1use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57 HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70 CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71 InstanceState,
72};
73use crate::vm::{
74 AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98 CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99 MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100 RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101 TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102 TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113 Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114 FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115 StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136const BLOCKED: u32 = 0xffff_ffff;
139
140#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143 Starting = 0,
144 Started = 1,
145 Returned = 2,
146 StartCancelled = 3,
147 ReturnCancelled = 4,
148}
149
150impl Status {
151 pub fn pack(self, waitable: Option<u32>) -> u32 {
157 assert!(matches!(self, Status::Returned) == waitable.is_none());
158 let waitable = waitable.unwrap_or(0);
159 assert!(waitable < (1 << 28));
160 (waitable << 4) | (self as u32)
161 }
162}
163
164#[derive(Clone, Copy, Debug)]
167enum Event {
168 None,
169 Subtask {
170 status: Status,
171 },
172 StreamRead {
173 code: ReturnCode,
174 pending: Option<(TypeStreamTableIndex, u32)>,
175 },
176 StreamWrite {
177 code: ReturnCode,
178 pending: Option<(TypeStreamTableIndex, u32)>,
179 },
180 FutureRead {
181 code: ReturnCode,
182 pending: Option<(TypeFutureTableIndex, u32)>,
183 },
184 FutureWrite {
185 code: ReturnCode,
186 pending: Option<(TypeFutureTableIndex, u32)>,
187 },
188 Cancelled,
189}
190
191impl Event {
192 fn parts(self) -> (u32, u32) {
197 const EVENT_NONE: u32 = 0;
198 const EVENT_SUBTASK: u32 = 1;
199 const EVENT_STREAM_READ: u32 = 2;
200 const EVENT_STREAM_WRITE: u32 = 3;
201 const EVENT_FUTURE_READ: u32 = 4;
202 const EVENT_FUTURE_WRITE: u32 = 5;
203 const EVENT_CANCELLED: u32 = 6;
204 match self {
205 Event::None => (EVENT_NONE, 0),
206 Event::Cancelled => (EVENT_CANCELLED, 0),
207 Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208 Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209 Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210 Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211 Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212 }
213 }
214}
215
216mod callback_code {
218 pub const EXIT: u32 = 0;
219 pub const YIELD: u32 = 1;
220 pub const WAIT: u32 = 2;
221}
222
223const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233 store: StoreContextMut<'a, T>,
234 get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239 D: HasData + ?Sized,
240 T: 'static,
241{
242 pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244 Self { store, get_data }
245 }
246
247 pub fn data_mut(&mut self) -> &mut T {
249 self.store.data_mut()
250 }
251
252 pub fn get(&mut self) -> D::Data<'_> {
254 (self.get_data)(self.data_mut())
255 }
256
257 pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261 where
262 T: 'static,
263 {
264 let accessor = Accessor {
265 get_data: self.get_data,
266 token: StoreToken::new(self.store.as_context_mut()),
267 };
268 self.store
269 .as_context_mut()
270 .spawn_with_accessor(accessor, task)
271 }
272
273 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276 self.get_data
277 }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282 D: HasData + ?Sized,
283 T: 'static,
284{
285 type Data = T;
286
287 fn as_context(&self) -> StoreContext<'_, T> {
288 self.store.as_context()
289 }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294 D: HasData + ?Sized,
295 T: 'static,
296{
297 fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298 self.store.as_context_mut()
299 }
300}
301
302pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363 D: HasData + ?Sized,
364{
365 token: StoreToken<T>,
366 get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369pub trait AsAccessor {
386 type Data: 'static;
388
389 type AccessorData: HasData + ?Sized;
392
393 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398 type Data = T::Data;
399 type AccessorData = T::AccessorData;
400
401 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402 T::as_accessor(self)
403 }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407 type Data = T;
408 type AccessorData = D;
409
410 fn as_accessor(&self) -> &Accessor<T, D> {
411 self
412 }
413}
414
415const _: () = {
438 const fn assert<T: Send + Sync>() {}
439 assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443 pub(crate) fn new(token: StoreToken<T>) -> Self {
452 Self {
453 token,
454 get_data: |x| x,
455 }
456 }
457}
458
459impl<T, D> Accessor<T, D>
460where
461 D: HasData + ?Sized,
462{
463 pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481 tls::get(|vmstore| {
482 fun(Access {
483 store: self.token.as_context_mut(vmstore),
484 get_data: self.get_data,
485 })
486 })
487 }
488
489 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492 self.get_data
493 }
494
495 pub fn with_getter<D2: HasData>(
512 &self,
513 get_data: fn(&mut T) -> D2::Data<'_>,
514 ) -> Accessor<T, D2> {
515 Accessor {
516 token: self.token,
517 get_data,
518 }
519 }
520
521 pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537 where
538 T: 'static,
539 {
540 let accessor = self.clone_for_spawn();
541 self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542 }
543
544 fn clone_for_spawn(&self) -> Self {
545 Self {
546 token: self.token,
547 get_data: self.get_data,
548 }
549 }
550
551 pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587 self.with(|mut access| {
588 let store = access.as_context_mut().0;
589 let state = store.concurrent_state_mut_without_forcing_current_thread();
590 if state.interesting_tasks == 0 {
591 Poll::Ready(())
592 } else {
593 state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594 Poll::Pending
595 }
596 })
597 }
598
599 pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623 self.with(|mut access| {
624 let store = access.as_context_mut().0;
625
626 {
627 let state = store.concurrent_state_mut_without_forcing_current_thread();
628 if state.switch_item.is_some()
629 || !state.high_priority.is_empty()
630 || !state.low_priority.is_empty()
631 {
632 state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633 return Poll::Pending;
634 }
635 }
636
637 let (_, _, _, raw_options) = func.abi_info(store);
638 let instance = func.instance().runtime_instance(raw_options.instance);
639 let state = store.instance_state(instance).concurrent_state();
640 if state.backpressure == 0 {
641 Poll::Ready(())
642 } else {
643 store
644 .concurrent_state_mut_without_forcing_current_thread()
645 .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646 Poll::Pending
647 }
648 })
649 }
650}
651
652pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674 AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676 D: HasData + ?Sized,
677{
678 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684 T: 'static,
685 F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686 F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687 Fut: Future<Output = Result<()>> + Send + 'fut,
688 D: HasData,
689{
690 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691 (self)(accessor)
692 }
693}
694
695enum CallerInfo {
698 Async {
700 params: Vec<ValRaw>,
701 has_result: bool,
702 },
703 Sync {
705 params: Vec<ValRaw>,
706 result_count: u32,
707 },
708}
709
710enum WaitMode {
712 Fiber(StoreFiber<'static>),
714 Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721 match self {
722 Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723 Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724 }
725 }
726}
727
728#[derive(Debug)]
730enum SuspendReason {
731 Waiting {
734 set: TableId<WaitableSet>,
735 thread: QualifiedThreadId,
736 },
737 YieldingToSubtask { thread: QualifiedThreadId },
740 NeedWork,
743 Yielding { thread: QualifiedThreadId },
746 ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751enum GuestCallKind {
753 DeliverEvent {
756 instance: Instance,
758 set: Option<TableId<WaitableSet>>,
763 },
764 StartImplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
767 StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
768}
769
770impl fmt::Debug for GuestCallKind {
771 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
772 match self {
773 Self::DeliverEvent { instance, set } => f
774 .debug_struct("DeliverEvent")
775 .field("instance", instance)
776 .field("set", set)
777 .finish(),
778 Self::StartImplicit(_) => f.debug_tuple("StartImplicit").finish(),
779 Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
780 }
781 }
782}
783
784#[derive(Copy, Clone, Debug)]
786pub enum SuspensionTarget {
787 Resume(u32),
788 Promote(u32),
789 None,
790}
791
792#[derive(Copy, Clone, Debug)]
794pub enum ResumeThread {
795 Promote,
796 Resume,
797 ResumeLater,
798}
799
800#[derive(Debug)]
802struct GuestCall {
803 thread: QualifiedThreadId,
804 kind: GuestCallKind,
805}
806
807impl GuestCall {
808 fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
819 let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
820 let async_typed = task.async_typed;
821 let async_lifted = task.async_lifted;
822 let has_callback = task.callback.is_some();
823 let instance = task.instance;
824 let state = store.instance_state(instance).concurrent_state();
825
826 let ready = match &self.kind {
827 GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
828 GuestCallKind::StartImplicit(_) => {
829 if async_typed {
830 let needs_exclusive = !async_lifted || has_callback;
833 !((needs_exclusive && state.do_not_enter) || state.backpressure > 0)
834 } else {
835 true
836 }
837 }
838 GuestCallKind::StartExplicit(_) => true,
839 };
840 log::trace!(
841 "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {})",
842 state.do_not_enter,
843 state.backpressure
844 );
845 Ok(ready)
846 }
847}
848
849enum WorkerItem {
851 GuestCall(GuestCall),
852 Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
853}
854
855enum WorkItem {
858 PushFuture(AlwaysMut<HostTaskFuture>),
860 ResumeFiber {
862 instance: RuntimeInstance,
863 thread: QualifiedThreadId,
864 fiber: StoreFiber<'static>,
865 },
866 ResumeThread {
868 instance: RuntimeInstance,
869 thread: QualifiedThreadId,
870 },
871 GuestCall {
873 instance: RuntimeInstance,
874 call: GuestCall,
875 },
876 WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
878}
879
880struct ResumingFiber<'a> {
886 store: &'a mut StoreOpaque,
887 fiber: Option<StoreFiber<'static>>,
888}
889
890impl Drop for ResumingFiber<'_> {
891 fn drop(&mut self) {
892 if let Some(mut fiber) = self.fiber.take() {
893 fiber.dispose(self.store);
894 }
895 }
896}
897
898impl fmt::Debug for WorkItem {
899 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
900 match self {
901 Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
902 Self::ResumeFiber {
903 instance, thread, ..
904 } => f
905 .debug_struct("ResumeFiber")
906 .field("instance", instance)
907 .field("thread", thread)
908 .finish(),
909 Self::ResumeThread { instance, thread } => f
910 .debug_struct("ResumeThread")
911 .field("instance", instance)
912 .field("thread", thread)
913 .finish(),
914 Self::GuestCall { instance, call } => f
915 .debug_struct("GuestCall")
916 .field("instance", instance)
917 .field("call", call)
918 .finish(),
919 Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
920 }
921 }
922}
923
924#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
926pub(crate) enum WaitResult {
927 Cancelled,
928 Completed,
929}
930
931pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
939 store: &mut dyn VMStore,
940 host_task: EnteredHostTask,
941 future: impl Future<Output = Result<R>> + Send + 'static,
942) -> Result<R> {
943 let mut future = Box::pin(future);
950 let poll = tls::set(store, || {
951 future
952 .as_mut()
953 .poll(&mut Context::from_waker(&Waker::noop()))
954 });
955
956 let caller = match host_task {
957 Some(caller) => caller,
958 None => bail_bug!("host task wasn't created but should have been"),
959 };
960
961 let task = match poll {
962 Poll::Ready(result) => return result,
964
965 Poll::Pending => {
970 let Some(task) = store.materialize_host_task_id()? else {
971 bail_bug!("current thread is not a host thread")
972 };
973
974 let future = Box::pin(async move {
977 let result = run_with_host_task_set(task, future).await??;
978 tls::get(move |store| {
979 let state = store.concurrent_state_mut()?;
980 let host_state = &mut state.get_mut(task)?.state;
981 assert!(matches!(host_state, HostTaskState::CalleeStarted));
982 *host_state = HostTaskState::CalleeFinished(Box::new(result));
983
984 Waitable::Host(task).set_event(
985 state,
986 Some(Event::Subtask {
987 status: Status::Returned,
988 }),
989 )?;
990
991 Ok(())
992 })
993 }) as HostTaskFuture;
994
995 let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
996 store.switch_or_trap_if_may_not_suspend(caller_instance)?;
997
998 let state = store.concurrent_state_mut()?;
999 state.push_future(future);
1000
1001 let set = state.get_mut(caller.thread)?.sync_call_set;
1002 Waitable::Host(task).join(state, Some(set))?;
1003
1004 store.suspend(SuspendReason::Waiting {
1005 set,
1006 thread: caller,
1007 })?;
1008
1009 Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1013 task
1014 }
1015 };
1016
1017 let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1019 match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1020 HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1021 Ok(result) => *result,
1022 Err(_) => bail_bug!("host task finished with wrong type of result"),
1023 }),
1024 _ => bail_bug!("unexpected host task state after completion"),
1025 }
1026}
1027
1028fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1030 match call.kind {
1031 GuestCallKind::DeliverEvent { instance, set } => {
1032 if let Some(set) = set {
1035 store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1036 }
1037 let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1038 Some(pair) => pair,
1039 None => match set {
1040 Some(set) => {
1045 log::trace!(
1046 "event for {:?} on {set:?} no longer present; waiting again",
1047 call.thread
1048 );
1049 return instance.wait_with_callback(
1050 store.store_opaque_mut(),
1051 call.thread,
1052 set,
1053 );
1054 }
1055 None => (Event::None, None),
1060 },
1061 };
1062 let state = store.concurrent_state_mut()?;
1063 if let Some(set) = set
1068 && waitable.is_none()
1069 && !state.get_mut(set)?.ready.is_empty()
1070 {
1071 state.wake_waiter(set)?;
1072 }
1073 let task = state.get_mut(call.thread.task)?;
1074 let runtime_instance = task.instance;
1075 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1076
1077 log::trace!(
1078 "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1079 call.thread,
1080 );
1081
1082 instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1083 invoke_callback(store, call.thread.task, event, handle)
1084 })?;
1085 }
1086 GuestCallKind::StartImplicit(fun) => {
1087 fun(store)?;
1088 }
1089 GuestCallKind::StartExplicit(fun) => {
1090 fun(store)?;
1091 }
1092 }
1093
1094 Ok(())
1095}
1096
1097fn invoke_callback(
1100 store: &mut dyn VMStore,
1101 task: TableId<GuestTask>,
1102 event: Event,
1103 handle: u32,
1104) -> Result<u32> {
1105 let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1106 bail_bug!("guest task callback field not present")
1107 };
1108
1109 let code = callback(store, event, handle)?;
1110
1111 store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1112
1113 Ok(code)
1114}
1115
1116impl<T> Store<T> {
1117 pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1119 where
1120 T: Send + 'static,
1121 {
1122 ensure!(
1123 self.as_context().0.concurrency_support(),
1124 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1125 );
1126 self.as_context_mut().run_concurrent(fun).await
1127 }
1128
1129 #[doc(hidden)]
1130 pub fn assert_concurrent_state_empty(&mut self) {
1131 self.as_context_mut().assert_concurrent_state_empty();
1132 }
1133
1134 #[doc(hidden)]
1135 pub fn concurrent_state_table_size(&mut self) -> usize {
1136 self.as_context_mut().concurrent_state_table_size()
1137 }
1138
1139 pub fn spawn(
1141 &mut self,
1142 task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1143 ) -> Result<JoinHandle>
1144 where
1145 T: 'static,
1146 {
1147 self.as_context_mut().spawn(task)
1148 }
1149}
1150
1151impl<T> StoreContextMut<'_, T> {
1152 #[doc(hidden)]
1163 pub fn assert_concurrent_state_empty(self) {
1164 let store = self.0;
1165 store
1166 .store_data_mut()
1167 .components
1168 .assert_instance_states_empty();
1169 let state = store.concurrent_state_mut().unwrap();
1170 assert!(
1171 state.table.get_mut().is_empty(),
1172 "non-empty table: {:?}",
1173 state.table.get_mut()
1174 );
1175 assert!(state.switch_item.is_none());
1176 assert!(state.next_switch_item.is_none());
1177 assert!(state.high_priority.is_empty());
1178 assert!(state.low_priority.is_empty());
1179 assert!(state.saved_next_switch_items.is_empty());
1180 assert!(state.unforced_current_thread.is_none());
1181 assert!(state.deferred_host_call_context.is_none());
1182 assert!(state.futures_mut().unwrap().is_empty());
1183 assert!(state.global_error_context_ref_counts.is_empty());
1184 }
1185
1186 #[doc(hidden)]
1191 pub fn concurrent_state_table_size(&mut self) -> usize {
1192 self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1193 }
1194
1195 pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1205 where
1206 T: 'static,
1207 {
1208 let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1209 self.spawn_with_accessor(accessor, task)
1210 }
1211
1212 fn spawn_with_accessor<D>(
1215 self,
1216 accessor: Accessor<T, D>,
1217 task: impl for<'fut> AccessorTask<'fut, T, D>,
1218 ) -> Result<JoinHandle>
1219 where
1220 T: 'static,
1221 D: HasData + ?Sized,
1222 {
1223 let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1227 self.0
1228 .concurrent_state_mut()?
1229 .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1230 Ok(handle)
1231 }
1232
1233 pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1317 where
1318 T: Send + 'static,
1319 {
1320 ensure!(
1321 self.0.concurrency_support(),
1322 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1323 );
1324 self.do_run_concurrent(fun, false).await
1325 }
1326
1327 pub(super) async fn run_concurrent_trap_on_idle<R>(
1328 self,
1329 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1330 ) -> Result<R> {
1331 self.do_run_concurrent(fun, true).await
1332 }
1333
1334 async fn do_run_concurrent<R>(
1335 mut self,
1336 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1337 trap_on_idle: bool,
1338 ) -> Result<R> {
1339 debug_assert!(self.0.concurrency_support());
1340 self.0.check_event_loop_not_running()?;
1341 let token = StoreToken::new(self.as_context_mut());
1342
1343 struct Dropper<'a, T: 'static, V> {
1344 store: StoreContextMut<'a, T>,
1345 value: ManuallyDrop<V>,
1346 }
1347
1348 impl<'a, T, V> Drop for Dropper<'a, T, V> {
1349 fn drop(&mut self) {
1350 self.store
1351 .0
1352 .concurrent_state_mut_already_forced_current_thread()
1353 .event_loop_running = false;
1354
1355 tls::set(self.store.0, || {
1356 unsafe { ManuallyDrop::drop(&mut self.value) }
1361 });
1362 }
1363 }
1364
1365 let accessor = &Accessor::new(token);
1366 self.0
1367 .concurrent_state_mut_already_forced_current_thread()
1368 .event_loop_running = true;
1369 let dropper = &mut Dropper {
1370 store: self,
1371 value: ManuallyDrop::new(fun(accessor)),
1372 };
1373 let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1375
1376 let result = dropper
1377 .store
1378 .as_context_mut()
1379 .poll_until(future, trap_on_idle)
1380 .await;
1381
1382 if result.is_err() {
1383 dropper.store.0.set_trapped();
1384 }
1385
1386 result
1387 }
1388
1389 async fn poll_until<R>(
1395 mut self,
1396 mut future: Pin<&mut impl Future<Output = R>>,
1397 trap_on_idle: bool,
1398 ) -> Result<R> {
1399 struct Reset<'a, T: 'static> {
1400 store: StoreContextMut<'a, T>,
1401 futures: Option<FuturesUnordered<HostTaskFuture>>,
1402 }
1403
1404 impl<'a, T> Drop for Reset<'a, T> {
1405 fn drop(&mut self) {
1406 if let Some(futures) = self.futures.take() {
1407 *self
1408 .store
1409 .0
1410 .concurrent_state_mut_already_forced_current_thread()
1411 .futures
1412 .get_mut() = Some(futures);
1413 }
1414 }
1415 }
1416
1417 const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1421 let mut turns_without_yield = 0;
1422
1423 loop {
1424 let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1428 let mut reset = Reset {
1429 store: self.as_context_mut(),
1430 futures,
1431 };
1432 let mut next = match reset.futures.as_mut() {
1433 Some(f) => pin!(f.next()),
1434 None => bail_bug!("concurrent state missing futures field"),
1435 };
1436
1437 enum PollResult<R> {
1438 Complete(R),
1439 ProcessWork {
1440 ready: Option<WorkItem>,
1441 low_priority: bool,
1442 },
1443 }
1444
1445 let result = future::poll_fn(|cx| {
1446 if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1449 return Poll::Ready(Ok(PollResult::Complete(value)));
1450 }
1451
1452 if reset.store.0.component_trapped() {
1461 return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1462 }
1463
1464 let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1468 Poll::Ready(Some(output)) => {
1469 match output {
1470 Err(e) => return Poll::Ready(Err(e)),
1471 Ok(()) => {}
1472 }
1473 Poll::Ready(true)
1474 }
1475 Poll::Ready(None) => Poll::Ready(false),
1476 Poll::Pending => Poll::Pending,
1477 };
1478
1479 let state = reset.store.0.concurrent_state_mut()?;
1494 let mut ready = state.switch_item.take();
1495 let mut low_priority = false;
1496 if ready.is_none() {
1497 ready = state.high_priority.pop_back();
1498 if ready.is_none() {
1499 ready = state.low_priority.pop_back();
1500 low_priority = true;
1501 }
1502 }
1503 if ready.is_some() {
1504 return Poll::Ready(Ok(PollResult::ProcessWork {
1505 ready,
1506 low_priority,
1507 }));
1508 }
1509
1510 if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1511 waker.wake();
1512 }
1513
1514 return match next {
1518 Poll::Ready(true) => {
1519 Poll::Ready(Ok(PollResult::ProcessWork {
1525 ready: None,
1526 low_priority: false,
1527 }))
1528 }
1529 Poll::Ready(false) => {
1530 if let Poll::Ready(value) =
1534 tls::set(reset.store.0, || future.as_mut().poll(cx))
1535 {
1536 Poll::Ready(Ok(PollResult::Complete(value)))
1537 } else {
1538 if trap_on_idle {
1544 Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1551 Trap::CannotBlockSyncTask.into()
1552 } else {
1553 Trap::AsyncDeadlock.into()
1555 }))
1556 } else {
1557 Poll::Pending
1561 }
1562 }
1563 }
1564 Poll::Pending => Poll::Pending,
1569 };
1570 })
1571 .await;
1572
1573 drop(reset);
1577
1578 match result? {
1579 PollResult::Complete(value) => break Ok(value),
1582 PollResult::ProcessWork {
1585 mut ready,
1586 low_priority,
1587 } => {
1588 if low_priority {
1616 if let Some(item) = ready.take() {
1617 self.0.concurrent_state_mut()?.low_priority.push_back(item);
1618 }
1619 self.0.yield_now().await;
1620 turns_without_yield = 0;
1621 ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1622 }
1623
1624 if let Some(item) = ready {
1625 self.as_context_mut().handle_work_item(item).await?;
1626 }
1627
1628 turns_without_yield += 1;
1629 if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1630 turns_without_yield = 0;
1631 self.0.yield_now().await;
1632 }
1633 }
1634 }
1635 }
1636 }
1637
1638 fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1640 let item = match item {
1644 WorkItem::ResumeFiber {
1645 instance,
1646 thread,
1647 fiber,
1648 } => {
1649 let future = self.0.resume_fiber(fiber);
1650 log::trace!(
1651 "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1652 );
1653 return futures::future::Either::Left(future);
1654 }
1655 item => item,
1656 };
1657 futures::future::Either::Right(async move {
1658 log::trace!("handle work item {item:?}");
1659 match item {
1660 WorkItem::PushFuture(future) => {
1661 self.0
1662 .concurrent_state_mut()?
1663 .futures_mut()?
1664 .push(future.into_inner());
1665 }
1666 WorkItem::ResumeFiber { .. } => unreachable!(),
1667 WorkItem::ResumeThread { thread, .. } => {
1668 let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1669 if !matches!(slot, GuestThreadState::Ready { .. }) {
1670 bail_bug!("cannot resume non-pending thread {thread:?}");
1671 }
1672 let GuestThreadState::Ready { fiber } =
1673 mem::replace(slot, GuestThreadState::Running)
1674 else {
1675 unreachable!()
1676 };
1677 self.0.resume_fiber(fiber).await?;
1678 }
1679 WorkItem::GuestCall { call, .. } => {
1680 if call.is_ready(self.0)? {
1681 self.0
1682 .concurrent_state_mut()?
1683 .get_mut(call.thread.thread)?
1684 .wake_on_cancel = WakeOnCancel::None;
1685 self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1686 } else {
1687 let state = self.0.concurrent_state_mut()?;
1688 let task = state.get_mut(call.thread.task)?;
1689 if !task.starting_sent {
1690 task.starting_sent = true;
1691 if let GuestCallKind::StartImplicit(_) = &call.kind {
1692 Waitable::Guest(call.thread.task).set_event(
1693 state,
1694 Some(Event::Subtask {
1695 status: Status::Starting,
1696 }),
1697 )?;
1698 }
1699 }
1700
1701 let instance = state.get_mut(call.thread.task)?.instance;
1702 if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1711 let state = self.0.concurrent_state_mut()?;
1712 if !state.get_mut(*set)?.ready.is_empty() {
1713 state.wake_waiter(*set)?;
1714 }
1715 }
1716 self.0
1717 .instance_state(instance)
1718 .concurrent_state()
1719 .pending
1720 .insert(call.thread, call.kind);
1721
1722 self.0.concurrent_state_mut()?.take_next_switch_item()?;
1726 }
1727 }
1728 WorkItem::WorkerFunction(fun) => {
1729 self.run_on_worker(WorkerItem::Function(fun)).await?;
1730 }
1731 }
1732
1733 Ok(())
1734 })
1735 }
1736
1737 async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1739 let state = self.0.concurrent_state_mut()?;
1740 if state.worker_item.is_some() {
1741 bail_bug!("worker item already set when taking worker fiber");
1742 }
1743 let worker = if let Some(fiber) = state.worker.take() {
1744 fiber
1745 } else {
1746 unsafe {
1765 fiber::make_fiber_unchecked(self.0, move |store| {
1766 loop {
1767 let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1768 bail_bug!("worker_item not present when resuming fiber")
1769 };
1770 match item {
1771 WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1772 WorkerItem::Function(fun) => fun.into_inner()(store)?,
1773 }
1774
1775 store.suspend(SuspendReason::NeedWork)?;
1776 }
1777 })?
1778 }
1779 };
1780
1781 self.0
1784 .concurrent_state_mut_already_forced_current_thread()
1785 .worker_item = Some(item);
1786
1787 self.0.resume_fiber(worker).await
1788 }
1789
1790 pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1795 where
1796 T: 'static,
1797 F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1798 + Send
1799 + Sync
1800 + 'static,
1801 R: Send + Sync + 'static,
1802 {
1803 let token = StoreToken::new(self);
1804 async move {
1805 let mut accessor = Accessor::new(token);
1806 closure(&mut accessor).await
1807 }
1808 }
1809
1810 pub(crate) async fn start_instance(
1811 &mut self,
1812 instance: ModuleInstance,
1813 callee: Option<RuntimeInstance>,
1814 ) -> Result<ModuleInstance> {
1815 self.0.check_event_loop_not_running()?;
1817 let (tx, rx) = oneshot::channel();
1818 let token = StoreToken::new(self.as_context_mut());
1819 self.0.queue_task(move |store| {
1820 _ = tx.send(
1821 super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1822 .map(|()| instance),
1823 );
1824 Ok(())
1825 })?;
1826 self.as_context_mut()
1827 .run_concurrent_trap_on_idle(async |_| {
1828 rx.await
1829 .map_err(|_| format_err!("oneshot channel canceled"))
1830 })
1831 .await??
1832 }
1833}
1834
1835pub type EnteredHostTask = Option<QualifiedThreadId>;
1842
1843impl StoreOpaque {
1844 pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1845 if self
1846 .concurrent_state_mut_already_forced_current_thread()
1847 .event_loop_running
1848 {
1849 bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1850 }
1851 Ok(())
1852 }
1853
1854 #[inline]
1858 pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1859 if !self.concurrency_support() {
1861 return Ok(CurrentThread::None);
1862 }
1863
1864 if !self
1867 .vm_store_context_mut()
1868 .current_thread_mut()
1869 .is_deferred()
1870 {
1871 return Ok(self
1872 .concurrent_state_mut_already_forced_current_thread()
1873 .unforced_current_thread);
1874 }
1875
1876 self.force_deferred_current_thread()
1877 }
1878
1879 #[cold]
1882 fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1883 let state = self.concurrent_state_mut_without_forcing_current_thread();
1892 let id = match state.unforced_current_thread.guest_task() {
1893 Some(task) => state.get_mut(task)?.instance.instance,
1894 None => bail_bug!("deferred component-model thread with non-guest base"),
1895 };
1896
1897 let mut frames = SmallVec::<[_; 8]>::new();
1900 let mut cur = *self.vm_store_context_mut().current_thread_mut();
1901 while let Some(ptr) = cur.as_deferred() {
1902 let deferred = unsafe { ptr.as_non_null().as_ref() };
1907 frames.push((
1908 deferred.callee_async != 0,
1909 deferred.callee_instance,
1910 deferred.saved_context,
1911 deferred.lazy_lenders_start,
1912 ));
1913 cur = deferred.parent;
1914 }
1915
1916 *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1920
1921 let current_context = *self.vm_store_context_mut().component_context_mut();
1924
1925 let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
1932 frames.reverse();
1933 let lazy_lenders_base = match frames.first() {
1934 Some((_, _, _, start)) => *start,
1935 None => bail_bug!("forcing a deferred thread with no deferred frames"),
1936 };
1937 let lazy_lender_ends = frames
1938 .iter()
1939 .skip(1)
1940 .map(|(_, _, _, start)| *start)
1941 .chain([lazy_lenders_end])
1942 .collect::<SmallVec<[_; 8]>>();
1943 for ((callee_async, callee_instance, saved_context, start), lenders_end) in
1944 frames.into_iter().zip(lazy_lender_ends)
1945 {
1946 *self.vm_store_context_mut().component_context_mut() = saved_context;
1950 let callee = RuntimeInstance {
1951 instance: id,
1952 index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
1953 };
1954 self.enter_guest_sync_call(callee_async, callee)?;
1955
1956 self.promote_current_lazy_lends(id, start, lenders_end)?;
1959 }
1960
1961 debug_assert_eq!(
1964 *self.vm_store_context_mut().lazy_lenders_len_mut(),
1965 lazy_lenders_end
1966 );
1967 *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
1968
1969 *self.vm_store_context_mut().component_context_mut() = current_context;
1971
1972 Ok(self
1973 .concurrent_state_mut_without_forcing_current_thread()
1974 .unforced_current_thread)
1975 }
1976
1977 fn promote_current_lazy_lends(
1983 &mut self,
1984 id: ComponentInstanceId,
1985 start: usize,
1986 end: usize,
1987 ) -> Result<()> {
1988 if start == end {
1989 return Ok(());
1990 }
1991
1992 let scope = match self
1993 .concurrent_state_mut_without_forcing_current_thread()
1994 .unforced_current_thread
1995 .guest_task()
1996 {
1997 Some(task) => VMScope::id(task.rep()),
1998 None => bail_bug!("replayed deferred frame is not a guest thread"),
1999 };
2000 let instance = Instance::from_wasmtime(self, id);
2001 self.promote_lazy_lends(instance, scope, start, end)
2002 }
2003
2004 fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2005 match self.current_thread()?.guest() {
2006 Some(id) => Ok(*id),
2007 None => bail_bug!("current thread is not a guest thread"),
2008 }
2009 }
2010
2011 pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2015 match self.current_thread()? {
2016 CurrentThread::Host(id) => Ok(Some(id)),
2017 CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2018 _ => bail_bug!("current thread is not a host thread"),
2019 }
2020 }
2021
2022 fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2025 Ok(self
2026 .concurrent_state_mut()?
2027 .materialize_current_host_task_id()?)
2028 }
2029
2030 fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2031 log::trace!("enter sync-typed call {callee:?}");
2032 let state = self.instance_state(callee).concurrent_state();
2033 let old_do_not_suspend = state.do_not_suspend;
2034 state.do_not_suspend = true;
2035
2036 let thread = self.current_guest_thread()?;
2037 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2038 if thread.old_do_not_suspend.is_some() {
2039 bail_bug!("current thread already has `old_do_not_suspend` value");
2040 }
2041
2042 thread.old_do_not_suspend = Some(old_do_not_suspend);
2043
2044 Ok(())
2045 }
2046
2047 fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2048 log::trace!("exit sync-typed call {callee:?}");
2049 let thread = self.current_guest_thread()?;
2050 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2051 let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2052 bail_bug!("current thread missing `old_do_not_suspend` value");
2053 };
2054 let state = self.instance_state(callee).concurrent_state();
2055 state.do_not_suspend = old_do_not_suspend;
2056 Ok(())
2057 }
2058
2059 pub(crate) fn enter_guest_sync_call(
2071 &mut self,
2072 callee_async_typed: bool,
2073 callee: RuntimeInstance,
2074 ) -> Result<()> {
2075 log::trace!("enter sync-lifted call {callee:?}");
2076 if !self.concurrency_support() {
2077 return self.enter_call_not_concurrent();
2078 }
2079
2080 let state = self.concurrent_state_mut()?;
2084 let item = state.next_switch_item.take();
2085 state.saved_next_switch_items.push(item);
2086
2087 let thread = self.current_thread()?;
2088 let caller = if let Some(thread) = thread.guest() {
2089 Caller::Guest { thread: *thread }
2090 } else {
2091 Caller::Host {
2092 tx: None,
2093 caller: self.materialize_host_task_id()?,
2094 }
2095 };
2096
2097 let state = self.concurrent_state_mut()?;
2098 let guest_thread = GuestTask::new(
2099 state,
2100 Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2101 LiftResult {
2102 lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2103 ty: TypeTupleIndex::reserved_value(),
2104 memory: None,
2105 string_encoding: StringEncoding::Utf8,
2106 },
2107 caller,
2108 None,
2109 callee,
2110 callee_async_typed,
2111 false,
2112 )?;
2113
2114 Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2115 guest_thread.thread,
2116 self,
2117 callee.index,
2118 )?;
2119 self.set_thread(guest_thread)?;
2120
2121 if !callee_async_typed {
2122 self.enter_sync_call(callee)?;
2123 }
2124
2125 Ok(())
2126 }
2127
2128 pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2136 if !self.concurrency_support() {
2137 return Ok(self.exit_call_not_concurrent());
2138 }
2139
2140 let thread = match self.current_thread()?.guest() {
2141 Some(t) => *t,
2142 None => bail_bug!("expected task when exiting"),
2143 };
2144 let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2145 let instance = task.instance;
2146
2147 let caller = match &task.caller {
2148 &Caller::Guest { thread } => thread.into(),
2149 &Caller::Host { caller, .. } => caller
2150 .map(CurrentThread::Host)
2151 .unwrap_or(CurrentThread::None),
2152 };
2153 task.lift_result = None;
2154 task.exited = true;
2155 let async_typed = task.async_typed;
2156
2157 if !async_typed {
2158 self.exit_sync_call(instance)?;
2159 }
2160
2161 self.set_thread(caller)?;
2162
2163 log::trace!("exit sync-lifted call {instance:?}");
2164
2165 if async_typed {
2166 self.switch_or_trap_if_may_not_suspend(instance)?;
2171 }
2172
2173 self.cleanup_thread(thread, instance)?;
2174
2175 self.concurrent_state_mut()?
2176 .decrement_task_ref_count(thread.task)?;
2177
2178 let state = self.concurrent_state_mut()?;
2179 let Some(item) = state.saved_next_switch_items.pop() else {
2180 bail_bug!("unable to pop from `saved_next_switch_items`");
2181 };
2182 if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2183 state.push_high_priority(item);
2186 bail_bug!("`next_switch_item` unexpectedly already set");
2187 }
2188
2189 Ok(())
2190 }
2191
2192 pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2199 if !self.concurrency_support() {
2200 self.enter_call_not_concurrent()?;
2201 return Ok(None);
2202 }
2203 let caller = self.current_guest_thread()?;
2204 log::trace!("new deferred host task with caller {caller:?}");
2205
2206 self.set_thread(CurrentThread::DeferredHost(caller))?;
2207 let state = self.concurrent_state_mut()?;
2208 debug_assert!(state.deferred_host_call_context.is_none());
2209 state.deferred_host_call_context = Some(CallContext::default());
2210 state.debug_assert_deferred_host_invariant();
2211 Ok(Some(caller))
2212 }
2213
2214 pub(crate) fn host_task_delete(
2221 &mut self,
2222 original_task: EnteredHostTask,
2223 materialized_task: Option<TableId<HostTask>>,
2224 ) -> Result<()> {
2225 match original_task {
2226 Some(caller) => {
2227 self.set_thread(caller)?;
2228 if materialized_task.is_none() {
2229 let state = self.concurrent_state_mut()?;
2230 let context = state
2231 .deferred_host_call_context
2232 .take()
2233 .expect("deferred host call context should be present");
2234 debug_assert!(context.is_empty());
2235 state.debug_assert_deferred_host_invariant();
2236 }
2237 log::trace!(
2238 "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2239 );
2240 if let Some(task) = materialized_task {
2241 self.concurrent_state_mut()?.delete_host_task(task)?;
2242 }
2243 }
2244 None => {
2245 debug_assert!(materialized_task.is_none());
2246 self.exit_call_not_concurrent();
2247 }
2248 }
2249 Ok(())
2250 }
2251
2252 fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2255 self.component_instance_mut(instance.instance)
2256 .instance_state(instance.index)
2257 }
2258
2259 fn instance_state_and_handle_counter(
2262 &mut self,
2263 instance: RuntimeInstance,
2264 ) -> (&mut InstanceState, &mut HandleCounter) {
2265 let (component, counter) = self
2266 .store_data_mut()
2267 .component_instance_and_handle_counter_mut(instance.instance);
2268 (component.instance_state(instance.index), counter)
2269 }
2270
2271 fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2273 let (component, counter) = self
2274 .store_data_mut()
2275 .component_instance_and_handle_counter_mut(instance.instance);
2276 component.handle_table_insertion(instance.index, counter)
2277 }
2278
2279 pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2285 let thread = thread.into();
2286 let state = self.concurrent_state_mut()?;
2287 state.debug_assert_deferred_host_invariant();
2288 let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2289
2290 state.handle_thread_switch(old_thread, thread)?;
2291
2292 if let Some(old_thread) = old_thread.guest() {
2300 let old_context = *self.vm_store_context_mut().component_context_mut();
2301 self.concurrent_state_mut()?
2302 .get_mut(old_thread.thread)?
2303 .context = old_context;
2304 }
2305 if cfg!(debug_assertions) {
2306 *self.vm_store_context_mut().component_context_mut() =
2307 [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2308 }
2309 if let Some(thread) = thread.guest() {
2310 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2311 let context = thread.context;
2312 if cfg!(debug_assertions) {
2313 thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2314 }
2315 *self.vm_store_context_mut().component_context_mut() = context;
2316 }
2317
2318 *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2320 VMLazyThread::none()
2321 } else {
2322 VMLazyThread::forced()
2323 };
2324
2325 Ok(old_thread)
2326 }
2327
2328 fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2330 if self.switch_if_may_not_suspend(instance)? {
2331 Ok(())
2332 } else {
2333 Err(Trap::CannotBlockSyncTask.into())
2334 }
2335 }
2336
2337 fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2341 self.concurrent_state_mut()?;
2345
2346 Ok(!self.concurrency_support()
2347 || !self
2348 .instance_state(instance)
2349 .concurrent_state()
2350 .do_not_suspend
2351 || self
2352 .concurrent_state_mut()?
2353 .promote_instance_local_thread_work_item(instance)?)
2354 }
2355
2356 fn enter_instance(&mut self, instance: RuntimeInstance) {
2360 log::trace!("enter {instance:?}");
2361 self.instance_state(instance)
2362 .concurrent_state()
2363 .do_not_enter = true;
2364 }
2365
2366 fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2370 log::trace!("exit {instance:?}");
2371 self.instance_state(instance)
2372 .concurrent_state()
2373 .do_not_enter = false;
2374 self.partition_pending(instance)
2375 }
2376
2377 fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2385 for (thread, kind) in
2386 mem::take(&mut self.instance_state(instance).concurrent_state().pending).into_iter()
2387 {
2388 let call = GuestCall { thread, kind };
2389 if call.is_ready(self)? {
2390 self.concurrent_state_mut()?
2391 .push_high_priority(WorkItem::GuestCall { instance, call });
2392 } else {
2393 self.instance_state(instance)
2394 .concurrent_state()
2395 .pending
2396 .insert(call.thread, call.kind);
2397 }
2398 }
2399
2400 if let Some(waker) = self
2401 .concurrent_state_mut()?
2402 .ready_for_concurrent_call_waker
2403 .take()
2404 {
2405 waker.wake();
2406 }
2407
2408 Ok(())
2409 }
2410
2411 pub(crate) fn backpressure_modify(
2413 &mut self,
2414 caller_instance: RuntimeInstance,
2415 modify: impl FnOnce(u16) -> Option<u16>,
2416 ) -> Result<()> {
2417 let state = self.instance_state(caller_instance).concurrent_state();
2418 let old = state.backpressure;
2419 let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2420 state.backpressure = new;
2421
2422 if old > 0 && new == 0 {
2423 self.partition_pending(caller_instance)?;
2426 }
2427
2428 Ok(())
2429 }
2430
2431 fn resume_fiber(
2434 &mut self,
2435 fiber: StoreFiber<'static>,
2436 ) -> impl Future<Output = Result<()>> + '_ {
2437 let mut resuming = ResumingFiber {
2438 store: self,
2439 fiber: Some(fiber),
2440 };
2441 async move {
2442 let old_thread = resuming.store.current_thread()?;
2443 log::trace!("resume_fiber: save current thread {old_thread:?}");
2444
2445 let fiber = resuming.fiber.take().unwrap();
2446 resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2447
2448 resuming.store.set_thread(old_thread)?;
2449
2450 let state = resuming.store.concurrent_state_mut()?;
2451
2452 if let Some(ot) = old_thread.guest() {
2453 state.set_thread_running(ot.thread)?;
2454 }
2455 log::trace!("resume_fiber: restore current thread {old_thread:?}");
2456
2457 if resuming.fiber.is_some() {
2458 log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2459 let reason = match state.suspend_reason.take() {
2461 Some(r) => r,
2462 None => bail_bug!("suspend reason missing when resuming fiber"),
2463 };
2464 match reason {
2465 SuspendReason::NeedWork => {
2466 if state.worker.is_none() {
2467 state.worker = resuming.fiber.take();
2468 }
2469 }
2470 SuspendReason::Yielding { thread } => {
2471 let slot = &mut state.get_mut(thread.thread)?.state;
2472 slot.check_no_fiber()?;
2473 *slot = GuestThreadState::Ready {
2474 fiber: resuming.fiber.take().unwrap(),
2475 };
2476 let instance = state.get_mut(thread.task)?.instance;
2477 state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2478 }
2479 SuspendReason::ExplicitlySuspending { thread } => {
2480 let slot = &mut state.get_mut(thread.thread)?.state;
2481 slot.check_no_fiber()?;
2482 *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2483 }
2484 SuspendReason::Waiting { set, thread } => {
2485 match state.get_mut(set)?.waiting.entry(thread) {
2486 Entry::Vacant(entry) => {
2487 entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2488 }
2489 Entry::Occupied(_) => {
2490 bail_bug!("entry unexpectedly already exists for {thread:?}");
2491 }
2492 }
2493 }
2494 SuspendReason::YieldingToSubtask { thread } => {
2495 let instance = state.get_mut(thread.task)?.instance;
2504 if state.next_switch_item.is_some() {
2505 bail_bug!(
2508 "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2509 a thread wanted to wait on a subtask"
2510 );
2511 }
2512 let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2513 instance,
2514 thread,
2515 fiber: resuming.fiber.take().unwrap(),
2516 });
2517 log::trace!("set next switch item to {item:?}");
2518 }
2519 };
2520 } else {
2521 log::trace!("resume_fiber: fiber has exited");
2522 }
2523
2524 Ok(())
2525 }
2526 }
2527
2528 fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2534 log::trace!("suspend fiber: {reason:?}");
2535
2536 if !self.fiber_async_state_mut().can_block() {
2539 return Err(format_err!("future dropped"));
2540 }
2541
2542 let state = self.concurrent_state_mut()?;
2543
2544 let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2551 SuspendReason::Yielding { .. }
2552 | SuspendReason::Waiting { .. }
2553 | SuspendReason::ExplicitlySuspending { .. } => {
2554 if state.switch_item.is_none() {
2557 state.take_next_switch_item()?;
2558 }
2559
2560 (true, false)
2561 }
2562 SuspendReason::YieldingToSubtask { .. } => (true, true),
2563 SuspendReason::NeedWork => (false, false),
2564 };
2565
2566 let old_next_switch_item = if save_and_restore_next_switch_item {
2567 Some(state.save_next_switch_item()?)
2568 } else {
2569 None
2570 };
2571
2572 let old_guest_thread = if save_and_restore_thread {
2573 self.current_thread()?
2574 } else {
2575 CurrentThread::None
2576 };
2577
2578 let waiting_set = match &reason {
2579 SuspendReason::Waiting { set, .. } => Some(*set),
2580 _ => None,
2581 };
2582
2583 let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2584 if suspend_reason.is_some() {
2585 bail_bug!("suspend reason unexpectedly already set for store");
2586 }
2587 *suspend_reason = Some(reason);
2588
2589 if let Some(set) = waiting_set {
2592 self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2593 }
2594
2595 self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2596
2597 if let Some(set) = waiting_set {
2598 self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2599 }
2600
2601 if save_and_restore_thread {
2602 self.set_thread(old_guest_thread)?;
2603 }
2604
2605 if let Some(item) = old_next_switch_item {
2606 let state = self.concurrent_state_mut()?;
2607 state.restore_next_switch_item(item)?;
2608 }
2609
2610 Ok(())
2611 }
2612
2613 fn wait_for_event(
2614 &mut self,
2615 caller_instance: RuntimeInstance,
2616 waitable: Waitable,
2617 ) -> Result<()> {
2618 let caller = self.current_guest_thread()?;
2619 let state = self.concurrent_state_mut()?;
2620
2621 waitable.trap_if_in_waitable_set(state)?;
2622
2623 let set = state.get_mut(caller.thread)?.sync_call_set;
2624 waitable.join(state, Some(set))?;
2625
2626 self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2627
2628 self.suspend(SuspendReason::Waiting {
2629 set,
2630 thread: caller,
2631 })?;
2632 let state = self.concurrent_state_mut()?;
2633
2634 waitable.join(state, None)
2635 }
2636
2637 fn cleanup_thread(
2659 &mut self,
2660 guest_thread: QualifiedThreadId,
2661 runtime_instance: RuntimeInstance,
2662 ) -> Result<()> {
2663 let state = self.concurrent_state_mut()?;
2664 state.take_next_switch_item()?;
2667 let thread_data = state.get_mut(guest_thread.thread)?;
2668 let sync_call_set = thread_data.sync_call_set;
2669 if let Some(guest_id) = thread_data.instance_rep {
2670 self.instance_state(runtime_instance)
2671 .thread_handle_table()
2672 .guest_thread_remove(guest_id)?;
2673 }
2674 let state = self.concurrent_state_mut()?;
2675
2676 for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2678 if let Some(Event::Subtask {
2679 status: Status::Returned | Status::ReturnCancelled,
2680 }) = waitable.common(state)?.event
2681 {
2682 waitable.delete_from(state)?;
2683 }
2684 }
2685
2686 state.delete(guest_thread.thread)?;
2687 state.delete(sync_call_set)?;
2688 let task = state.get_mut(guest_thread.task)?;
2689 task.threads.remove(&guest_thread.thread);
2690
2691 if task.threads.is_empty() && !task.returned_or_cancelled() {
2692 bail!(Trap::NoAsyncResult);
2693 }
2694
2695 if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2696 task.decremented_interesting_task_count = true;
2697
2698 debug_assert!(state.interesting_tasks > 0);
2699 state.interesting_tasks -= 1;
2700 if state.interesting_tasks == 0
2701 && let Some(waker) = state.interesting_tasks_empty_waker.take()
2702 {
2703 waker.wake();
2704 }
2705 }
2706
2707 state.decrement_task_ref_count(guest_thread.task)?;
2708
2709 Ok(())
2710 }
2711
2712 fn cancel_guest_subtask_without_lowered_parameters(
2721 &mut self,
2722 caller_instance: RuntimeInstance,
2723 guest_task: TableId<GuestTask>,
2724 ) -> Result<()> {
2725 let concurrent_state = self.concurrent_state_mut()?;
2726 let task = concurrent_state.get_mut(guest_task)?;
2727 assert!(!task.already_lowered_parameters());
2728 task.lower_params = None;
2732 task.lift_result = None;
2733 task.exited = true;
2734 let instance = task.instance;
2735
2736 assert_eq!(1, task.threads.len());
2743 let thread = *task.threads.iter().next().unwrap();
2744 let next_switch_item = concurrent_state.save_next_switch_item()?;
2745 self.cleanup_thread(
2746 QualifiedThreadId {
2747 task: guest_task,
2748 thread,
2749 },
2750 caller_instance,
2751 )?;
2752 self.concurrent_state_mut()?
2753 .restore_next_switch_item(next_switch_item)?;
2754
2755 let pending = &mut self.instance_state(instance).concurrent_state().pending;
2757 let pending_count = pending.len();
2758 pending.retain(|thread, _| thread.task != guest_task);
2759
2760 if pending.len() == pending_count {
2761 let matches = |item: &WorkItem| match item {
2764 WorkItem::ResumeThread { thread, .. }
2765 | WorkItem::GuestCall {
2766 call: GuestCall { thread, .. },
2767 ..
2768 } => thread.task == guest_task,
2769 _ => false,
2770 };
2771
2772 let remove = |queue: &mut VecDeque<WorkItem>| {
2773 let count = queue.len();
2774 queue.retain(|item| !matches(item));
2775 count != queue.len()
2776 };
2777
2778 let state = self.concurrent_state_mut()?;
2779 let mut found = remove(&mut state.high_priority);
2780 found |= remove(&mut state.low_priority);
2781
2782 for item in [&mut state.switch_item, &mut state.next_switch_item] {
2783 if matches!(item, Some(item) if matches(item)) {
2784 found = true;
2785 *item = None;
2786 }
2787 }
2788
2789 if !found {
2790 bail!(Trap::SubtaskCancelAfterTerminal);
2791 }
2792 }
2793 Ok(())
2794 }
2795
2796 pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2799 if !self.concurrency_support() {
2800 return Ok(self
2801 .current_scope_id_not_concurrent()?
2802 .map(|id| CurrentScope::Id(VMScope::id(id))));
2803 }
2804
2805 Ok(match self.current_thread()? {
2806 CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2807 CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2808 CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2809 CurrentThread::None => return Ok(None),
2810 })
2811 }
2812
2813 pub(crate) fn queue_task(
2814 &mut self,
2815 task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2816 ) -> Result<()> {
2817 self.concurrent_state_mut()?
2818 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2819 Ok(())
2820 }
2821
2822 fn any_may_not_suspend(&mut self) -> Result<bool> {
2831 Ok(self
2839 .concurrent_state_mut()?
2840 .table
2841 .get_mut()
2842 .iter_mut()
2843 .filter_map(|(_, entry)| {
2844 if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2845 Some(task.instance)
2846 } else {
2847 None
2848 }
2849 })
2850 .collect::<Vec<_>>()
2851 .into_iter()
2852 .any(|instance| {
2853 self.instance_state(instance)
2854 .concurrent_state()
2855 .do_not_suspend
2856 }))
2857 }
2858}
2859
2860impl Instance {
2861 fn get_event(
2864 self,
2865 store: &mut StoreOpaque,
2866 guest_task: TableId<GuestTask>,
2867 set: Option<TableId<WaitableSet>>,
2868 cancellable: bool,
2869 ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
2870 let state = store.concurrent_state_mut()?;
2871
2872 let task = state.get_mut(guest_task)?;
2873 let event = &mut task.event;
2874 if let Some(ev) = event
2875 && (cancellable || !matches!(ev, Event::Cancelled))
2876 {
2877 log::trace!("deliver event {ev:?} to {guest_task:?}");
2878
2879 if matches!(ev, Event::Cancelled) {
2880 task.cancel_request_delivered = true;
2881 }
2882
2883 let ev = *ev;
2884 *event = None;
2885 return Ok(Some((ev, None)));
2886 }
2887
2888 let set = match set {
2889 Some(set) => set,
2890 None => return Ok(None),
2891 };
2892 let waitable = match state.get_mut(set)?.ready.pop_first() {
2893 Some(v) => v,
2894 None => return Ok(None),
2895 };
2896
2897 let common = waitable.common(state)?;
2898 let handle = match common.handle {
2899 Some(h) => h,
2900 None => bail_bug!("handle not set when delivering event"),
2901 };
2902 let event = match common.event.take() {
2903 Some(e) => e,
2904 None => bail_bug!("event not set when delivering event"),
2905 };
2906
2907 log::trace!(
2908 "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
2909 );
2910
2911 waitable.on_delivery(store, self, event)?;
2912
2913 Ok(Some((event, Some((waitable, handle)))))
2914 }
2915
2916 fn handle_callback_code(
2923 self,
2924 store: &mut dyn VMStore,
2925 guest_thread: QualifiedThreadId,
2926 runtime_instance: RuntimeInstance,
2927 call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
2928 ) -> Result<()> {
2929 let old_thread = store.set_thread(guest_thread)?;
2930 log::trace!(
2931 "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
2932 );
2933
2934 store.enter_instance(runtime_instance);
2935
2936 let mut code = call(store)?;
2937
2938 if matches!(
2944 unpack_callback_code(code).0,
2945 callback_code::YIELD | callback_code::WAIT
2946 ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
2947 {
2948 let (unpacked, set) = unpack_callback_code(code);
2949 log::trace!(
2950 "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
2951 cancellation; delivering it via callback"
2952 );
2953 code = invoke_callback(store, guest_thread.task, event, 0)?;
2954 }
2955
2956 store.exit_instance(runtime_instance)?;
2957
2958 store.set_thread(old_thread)?;
2959 log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
2960
2961 let store = store.store_opaque_mut();
2962 let (code, set) = unpack_callback_code(code);
2963
2964 log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
2965
2966 let state = store.concurrent_state_mut()?;
2967
2968 state.take_next_switch_item()?;
2969
2970 let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
2971 let set = store
2972 .instance_state(runtime_instance)
2973 .handle_table()
2974 .waitable_set_rep(handle)?;
2975
2976 Ok(TableId::<WaitableSet>::new(set))
2977 };
2978
2979 match code {
2980 callback_code::EXIT => {
2981 log::trace!("implicit thread {guest_thread:?} completed");
2982 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
2983 task.exited = true;
2984 task.callback = None;
2985
2986 store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
2991
2992 store.cleanup_thread(guest_thread, runtime_instance)?;
2993 }
2994 callback_code::YIELD => {
2995 let old = state
2998 .get_mut(guest_thread.thread)?
2999 .wake_on_cancel
3000 .replace(WakeOnCancel::Yielding);
3001 if !old.is_none() {
3002 bail_bug!("thread unexpectedly had wake_on_cancel set");
3003 }
3004
3005 let call = GuestCall {
3012 thread: guest_thread,
3013 kind: GuestCallKind::DeliverEvent {
3014 instance: self,
3015 set: None,
3016 },
3017 };
3018 state.push_low_priority(WorkItem::GuestCall {
3021 instance: runtime_instance,
3022 call,
3023 });
3024 }
3025 callback_code::WAIT => {
3026 let set = get_set(store, set)?;
3027 self.wait_with_callback(store, guest_thread, set)?;
3028 }
3029 _ => bail!(Trap::UnsupportedCallbackCode),
3030 }
3031
3032 Ok(())
3033 }
3034
3035 fn wait_with_callback(
3041 self,
3042 store: &mut StoreOpaque,
3043 guest_thread: QualifiedThreadId,
3044 set: TableId<WaitableSet>,
3045 ) -> Result<()> {
3046 let state = store.concurrent_state_mut()?;
3047 state.get_mut(set)?.num_waiting += 1;
3050
3051 if state.get_mut(guest_thread.task)?.event.is_some()
3052 || !state.get_mut(set)?.ready.is_empty()
3053 {
3054 let instance = state.get_mut(guest_thread.task)?.instance;
3056 state.push_high_priority(WorkItem::GuestCall {
3057 instance,
3058 call: GuestCall {
3059 thread: guest_thread,
3060 kind: GuestCallKind::DeliverEvent {
3061 instance: self,
3062 set: Some(set),
3063 },
3064 },
3065 });
3066 return Ok(());
3067 }
3068
3069 let instance = store
3075 .concurrent_state_mut()?
3076 .get_mut(guest_thread.task)?
3077 .instance;
3078 store.switch_or_trap_if_may_not_suspend(instance)?;
3079
3080 let state = store.concurrent_state_mut()?;
3086 let old = state
3087 .get_mut(guest_thread.thread)?
3088 .wake_on_cancel
3089 .replace(WakeOnCancel::Waiting(set));
3090 if !old.is_none() {
3091 bail_bug!("thread unexpectedly had wake_on_cancel set");
3092 }
3093 let old = state
3094 .get_mut(set)?
3095 .waiting
3096 .insert(guest_thread, WaitMode::Callback(self));
3097 if !old.is_none() {
3098 bail_bug!("set's waiting set already had this thread registered");
3099 }
3100 Ok(())
3101 }
3102
3103 unsafe fn stage_call<T: 'static>(
3110 self,
3111 mut store: StoreContextMut<T>,
3112 guest_thread: QualifiedThreadId,
3113 callee: SendSyncPtr<VMFuncRef>,
3114 param_count: usize,
3115 result_count: usize,
3116 async_: bool,
3117 callback: Option<SendSyncPtr<VMFuncRef>>,
3118 post_return: Option<SendSyncPtr<VMFuncRef>>,
3119 host_caller: bool,
3120 ) -> Result<()> {
3121 unsafe fn make_call<T: 'static>(
3136 store: StoreContextMut<T>,
3137 guest_thread: QualifiedThreadId,
3138 callee: SendSyncPtr<VMFuncRef>,
3139 param_count: usize,
3140 result_count: usize,
3141 ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3142 + Send
3143 + Sync
3144 + 'static
3145 + use<T> {
3146 let token = StoreToken::new(store);
3147 move |store: &mut dyn VMStore| {
3148 let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3149
3150 store
3151 .concurrent_state_mut()?
3152 .set_thread_running(guest_thread.thread)?;
3153 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3154 let lower = match task.lower_params.take() {
3155 Some(l) => l,
3156 None => bail_bug!("lower_params missing"),
3157 };
3158
3159 lower(store, &mut storage[..param_count])?;
3160
3161 let mut store = token.as_context_mut(store);
3162
3163 unsafe {
3166 crate::Func::call_unchecked_raw(
3167 &mut store,
3168 callee.as_non_null(),
3169 NonNull::new(
3170 &mut storage[..param_count.max(result_count)]
3171 as *mut [MaybeUninit<ValRaw>] as _,
3172 )
3173 .unwrap(),
3174 UncaughtException::Trap,
3175 )?;
3176 }
3177
3178 Ok(storage)
3179 }
3180 }
3181
3182 let call = unsafe {
3186 make_call(
3187 store.as_context_mut(),
3188 guest_thread,
3189 callee,
3190 param_count,
3191 result_count,
3192 )
3193 };
3194
3195 let callee_instance = store
3196 .0
3197 .concurrent_state_mut()?
3198 .get_mut(guest_thread.task)?
3199 .instance;
3200
3201 let fun = if callback.is_some() {
3202 assert!(async_);
3203
3204 Box::new(move |store: &mut dyn VMStore| {
3205 self.add_guest_thread_to_instance_table(
3206 guest_thread.thread,
3207 store,
3208 callee_instance.index,
3209 )?;
3210 self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3211 let storage = call(store)?;
3212
3213 Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3216 })
3217 }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3218 } else {
3219 let token = StoreToken::new(store.as_context_mut());
3220 Box::new(move |store: &mut dyn VMStore| {
3221 self.add_guest_thread_to_instance_table(
3222 guest_thread.thread,
3223 store,
3224 callee_instance.index,
3225 )?;
3226 let old_thread = store.set_thread(guest_thread)?;
3227 log::trace!(
3228 "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3229 );
3230 let flags = self.id().get(store).instance_flags(callee_instance.index);
3231
3232 let callee_async_typed = store
3233 .concurrent_state_mut()?
3234 .get_mut(guest_thread.task)?
3235 .async_typed;
3236
3237 if !async_ && callee_async_typed {
3241 store.enter_instance(callee_instance);
3242 }
3243
3244 if !callee_async_typed {
3245 store.enter_sync_call(callee_instance)?;
3246 }
3247
3248 let storage = call(store)?;
3255
3256 if !callee_async_typed {
3257 store.exit_sync_call(callee_instance)?;
3258 }
3259
3260 if !async_ {
3261 if callee_async_typed {
3267 store.exit_instance(callee_instance)?;
3268 }
3269
3270 let lift = {
3271 let state = store.concurrent_state_mut()?;
3272 if !state.get_mut(guest_thread.task)?.result.is_none() {
3273 bail_bug!("task has already produced a result");
3274 }
3275
3276 match state.get_mut(guest_thread.task)?.lift_result.take() {
3277 Some(lift) => lift,
3278 None => bail_bug!("lift_result field is missing"),
3279 }
3280 };
3281
3282 let result = (lift.lift)(store, unsafe {
3285 mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3286 &storage[..result_count],
3287 )
3288 })?;
3289
3290 let post_return_arg = match result_count {
3291 0 => ValRaw::i32(0),
3292 1 => unsafe { storage[0].assume_init() },
3295 _ => unreachable!(),
3296 };
3297
3298 unsafe {
3299 call_post_return(
3300 token.as_context_mut(store),
3301 post_return.map(|v| v.as_non_null()),
3302 post_return_arg,
3303 flags,
3304 )?;
3305 }
3306
3307 self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3308 }
3309
3310 store.set_thread(old_thread)?;
3311
3312 store
3313 .concurrent_state_mut()?
3314 .get_mut(guest_thread.task)?
3315 .exited = true;
3316
3317 log::trace!(
3318 "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3319 );
3320
3321 if callee_async_typed {
3322 store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3327 }
3328
3329 store.cleanup_thread(guest_thread, callee_instance)?;
3331 Ok(())
3332 })
3333 };
3334
3335 store.0.concurrent_state_mut()?.push_work_item(
3336 WorkItem::GuestCall {
3337 instance: callee_instance,
3338 call: GuestCall {
3339 thread: guest_thread,
3340 kind: GuestCallKind::StartImplicit(fun),
3341 },
3342 },
3343 if host_caller {
3344 Priority::High
3345 } else {
3346 Priority::Switch
3347 },
3348 )?;
3349
3350 Ok(())
3351 }
3352
3353 unsafe fn prepare_call<T: 'static>(
3366 self,
3367 mut store: StoreContextMut<T>,
3368 start: NonNull<VMFuncRef>,
3369 return_: NonNull<VMFuncRef>,
3370 caller_instance: RuntimeComponentInstanceIndex,
3371 callee_instance: RuntimeComponentInstanceIndex,
3372 task_return_type: TypeTupleIndex,
3373 callee_async_typed: bool,
3374 memory: *mut VMMemoryDefinition,
3375 string_encoding: StringEncoding,
3376 caller_info: CallerInfo,
3377 ) -> Result<()> {
3378 enum ResultInfo {
3379 Heap { results: u32 },
3380 Stack { result_count: u32 },
3381 }
3382
3383 let result_info = match &caller_info {
3384 CallerInfo::Async {
3385 has_result: true,
3386 params,
3387 } => ResultInfo::Heap {
3388 results: match params.last() {
3390 Some(r) => r.get_u32(),
3391 None => bail_bug!("retptr missing"),
3392 },
3393 },
3394 CallerInfo::Async {
3395 has_result: false, ..
3396 } => ResultInfo::Stack { result_count: 0 },
3397 CallerInfo::Sync {
3398 result_count,
3399 params,
3400 } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3401 results: match params.last() {
3403 Some(r) => r.get_u32(),
3404 None => bail_bug!("arg ptr missing"),
3405 },
3406 },
3407 CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3408 result_count: *result_count,
3409 },
3410 };
3411
3412 let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3413
3414 let start = SendSyncPtr::new(start);
3418 let return_ = SendSyncPtr::new(return_);
3419 let token = StoreToken::new(store.as_context_mut());
3420 let old_thread = store.0.current_guest_thread()?;
3421
3422 let state = store.0.concurrent_state_mut()?;
3423
3424 debug_assert_eq!(
3425 state.get_mut(old_thread.task)?.instance,
3426 self.runtime_instance(caller_instance)
3427 );
3428
3429 let guest_thread = GuestTask::new(
3430 state,
3431 Box::new(move |store, dst| {
3432 let mut store = token.as_context_mut(store);
3433 assert!(dst.len() <= MAX_FLAT_PARAMS);
3434 let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3436 let count = match caller_info {
3437 CallerInfo::Async { params, has_result } => {
3441 let params = ¶ms[..params.len() - usize::from(has_result)];
3442 for (param, src) in params.iter().zip(&mut src) {
3443 src.write(*param);
3444 }
3445 params.len()
3446 }
3447
3448 CallerInfo::Sync { params, .. } => {
3450 for (param, src) in params.iter().zip(&mut src) {
3451 src.write(*param);
3452 }
3453 params.len()
3454 }
3455 };
3456 unsafe {
3463 crate::Func::call_unchecked_raw(
3464 &mut store,
3465 start.as_non_null(),
3466 NonNull::new(
3467 &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3468 )
3469 .unwrap(),
3470 UncaughtException::Trap,
3471 )?;
3472 }
3473 dst.copy_from_slice(&src[..dst.len()]);
3474 let task = store.0.current_guest_thread()?.task;
3475 let state = store.0.concurrent_state_mut()?;
3476 Waitable::Guest(task).set_event(
3477 state,
3478 Some(Event::Subtask {
3479 status: Status::Started,
3480 }),
3481 )?;
3482 Ok(())
3483 }),
3484 LiftResult {
3485 lift: Box::new(move |store, src| {
3486 let mut store = token.as_context_mut(store);
3489 let mut my_src = src.to_owned(); if let ResultInfo::Heap { results } = &result_info {
3491 my_src.push(ValRaw::u32(*results));
3492 }
3493
3494 unsafe {
3501 crate::Func::call_unchecked_raw(
3502 &mut store,
3503 return_.as_non_null(),
3504 my_src.as_mut_slice().into(),
3505 UncaughtException::Trap,
3506 )?;
3507 }
3508
3509 let thread = store.0.current_guest_thread()?;
3510 let state = store.0.concurrent_state_mut()?;
3511 if sync_caller {
3512 state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3513 if let ResultInfo::Stack { result_count } = &result_info {
3514 match result_count {
3515 0 => None,
3516 1 => Some(my_src[0]),
3517 _ => unreachable!(),
3518 }
3519 } else {
3520 None
3521 },
3522 );
3523 }
3524 Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3525 }),
3526 ty: task_return_type,
3527 memory: NonNull::new(memory).map(SendSyncPtr::new),
3528 string_encoding,
3529 },
3530 Caller::Guest { thread: old_thread },
3531 None,
3532 self.runtime_instance(callee_instance),
3533 callee_async_typed,
3534 false,
3537 )?;
3538
3539 store.0.set_thread(guest_thread)?;
3542 log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3543
3544 Ok(())
3545 }
3546
3547 unsafe fn call_callback<T>(
3552 self,
3553 mut store: StoreContextMut<T>,
3554 function: SendSyncPtr<VMFuncRef>,
3555 event: Event,
3556 handle: u32,
3557 ) -> Result<u32> {
3558 let (ordinal, result) = event.parts();
3559 let params = &mut [
3560 ValRaw::u32(ordinal),
3561 ValRaw::u32(handle),
3562 ValRaw::u32(result),
3563 ];
3564 unsafe {
3569 crate::Func::call_unchecked_raw(
3570 &mut store,
3571 function.as_non_null(),
3572 params.as_mut_slice().into(),
3573 UncaughtException::Trap,
3574 )?;
3575 }
3576 Ok(params[0].get_u32())
3577 }
3578
3579 unsafe fn start_call<T: 'static>(
3597 self,
3598 mut store: StoreContextMut<T>,
3599 callback: *mut VMFuncRef,
3600 post_return: *mut VMFuncRef,
3601 callee: NonNull<VMFuncRef>,
3602 param_count: u32,
3603 result_count: u32,
3604 flags: u32,
3605 storage: &mut [MaybeUninit<ValRaw>],
3606 ) -> Result<()> {
3607 let token = StoreToken::new(store.as_context_mut());
3608 let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3609 let guest_thread = store.0.current_guest_thread()?;
3610 let state = store.0.concurrent_state_mut()?;
3611
3612 if !state.event_loop_running {
3617 bail!(
3618 "async function started during an original synchronous \
3619 entrypoint; please use an async-style entrypoint instead"
3620 );
3621 }
3622
3623 let callee = SendSyncPtr::new(callee);
3624 let param_count = usize::try_from(param_count)?;
3625 assert!(param_count <= MAX_FLAT_PARAMS);
3626 let result_count = usize::try_from(result_count)?;
3627 assert!(result_count <= MAX_FLAT_RESULTS);
3628
3629 let task = state.get_mut(guest_thread.task)?;
3630 let callee_async_typed = task.async_typed;
3631 let callee_instance = task.instance;
3632
3633 task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3634
3635 if let Some(callback) = NonNull::new(callback) {
3636 let callback = SendSyncPtr::new(callback);
3640 task.callback = Some(Box::new(move |store, event, handle| {
3641 let store = token.as_context_mut(store);
3642 unsafe { self.call_callback::<T>(store, callback, event, handle) }
3643 }));
3644 }
3645
3646 let Caller::Guest { thread: caller } = &task.caller else {
3647 bail_bug!("start_call unexpectedly invoked for host->guest call");
3650 };
3651 let caller = *caller;
3652 let caller_instance = state.get_mut(caller.task)?.instance;
3653
3654 unsafe {
3656 self.stage_call(
3657 store.as_context_mut(),
3658 guest_thread,
3659 callee,
3660 param_count,
3661 result_count,
3662 (flags & START_FLAG_ASYNC_CALLEE) != 0,
3663 NonNull::new(callback).map(SendSyncPtr::new),
3664 NonNull::new(post_return).map(SendSyncPtr::new),
3665 false,
3666 )?;
3667 }
3668
3669 let old_do_not_suspend = if callee_async_typed {
3670 let state = store.0.instance_state(callee_instance).concurrent_state();
3677 let old_do_not_suspend = state.do_not_suspend;
3678 state.do_not_suspend = false;
3679 Some(old_do_not_suspend)
3680 } else {
3681 None
3682 };
3683
3684 let state = store.0.concurrent_state_mut()?;
3685
3686 let guest_waitable = Waitable::Guest(guest_thread.task);
3689 let old_set = guest_waitable.common(state)?.set;
3690 let set = state.get_mut(caller.thread)?.sync_call_set;
3691 guest_waitable.join(state, Some(set))?;
3692
3693 store.0.set_thread(CurrentThread::None)?;
3694
3695 let mut yielded = false;
3711 let (status, waitable) = loop {
3712 store.0.suspend(if yielded {
3713 SuspendReason::Waiting {
3714 set,
3715 thread: caller,
3716 }
3717 } else {
3718 yielded = true;
3719 SuspendReason::YieldingToSubtask { thread: caller }
3720 })?;
3721
3722 if let Some(old_do_not_suspend) = old_do_not_suspend {
3723 store
3724 .0
3725 .instance_state(callee_instance)
3726 .concurrent_state()
3727 .do_not_suspend = old_do_not_suspend;
3728 }
3729
3730 let state = store.0.concurrent_state_mut()?;
3731
3732 log::trace!("taking event for {:?}", guest_thread.task);
3733 let event = guest_waitable.take_event(state)?;
3734 let Some(Event::Subtask { status }) = event else {
3735 bail_bug!("subtasks should only get subtask events, got {event:?}")
3736 };
3737
3738 log::trace!("status {status:?} for {:?}", guest_thread.task);
3739
3740 if status == Status::Returned {
3741 break (status, None);
3743 } else if async_caller {
3744 let handle = store
3748 .0
3749 .handle_table_insertion(caller_instance)
3750 .subtask_guest(guest_thread.task.rep())?;
3751 store
3752 .0
3753 .concurrent_state_mut()?
3754 .get_mut(guest_thread.task)?
3755 .common
3756 .handle = Some(handle);
3757 break (status, Some(handle));
3758 } else {
3759 store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3763 }
3764 };
3765
3766 guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3767
3768 store.0.set_thread(caller)?;
3770
3771 store
3772 .0
3773 .concurrent_state_mut()?
3774 .set_thread_running(caller.thread)?;
3775
3776 log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3777
3778 if async_caller {
3779 let Some(slot) = storage.first_mut() else {
3780 bail_bug!("no storage for async call status");
3781 };
3782 *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3783 } else {
3784 let state = store.0.concurrent_state_mut()?;
3787 let task = state.get_mut(guest_thread.task)?;
3788 if let Some(Some(result)) = task.sync_result.take()? {
3789 storage[0] = MaybeUninit::new(result);
3790 }
3791 }
3792
3793 if status == Status::Returned {
3794 store
3795 .0
3796 .concurrent_state_mut()?
3797 .decrement_task_ref_count(guest_thread.task)?;
3798 }
3799
3800 Ok(())
3801 }
3802
3803 pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3819 self,
3820 mut store: StoreContextMut<'_, T>,
3821 host_task: EnteredHostTask,
3822 result_may_require_realloc: bool,
3823 future: impl Future<Output = Result<R>> + Send + 'static,
3824 lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3825 + Send
3826 + 'static,
3827 ) -> Result<u32> {
3828 let token = StoreToken::new(store.as_context_mut());
3829
3830 let (join_handle, future) = JoinHandle::run(future);
3833 let mut future = Box::pin(future);
3834
3835 let poll = tls::set(store.0, || {
3840 future
3841 .as_mut()
3842 .poll(&mut Context::from_waker(&Waker::noop()))
3843 });
3844
3845 match poll {
3846 Poll::Ready(result) => {
3848 let result = result.transpose()?;
3849 let task = store.0.current_materialized_host_task()?;
3852 lower(store.as_context_mut(), result, true, task)?;
3853 return Ok(Status::Returned.pack(None));
3854 }
3855
3856 Poll::Pending => {}
3858 }
3859
3860 let Some(task) = store.0.materialize_host_task_id()? else {
3864 bail_bug!("current thread is not a host thread")
3865 };
3866 {
3867 let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
3868 assert!(matches!(state, HostTaskState::CalleeStarted));
3869 *state = HostTaskState::CalleeRunning(join_handle);
3870 }
3871
3872 let future = Box::pin(async move {
3880 let result = match run_with_host_task_set(task, future).await? {
3881 Some(result) => Some(result?),
3882 None => None,
3883 };
3884 let on_complete = move |store: &mut dyn VMStore| {
3885 let mut store = token.as_context_mut(store);
3889 let old = store.0.set_thread(task)?;
3890
3891 let status = if result.is_some() {
3892 Status::Returned
3893 } else {
3894 Status::ReturnCancelled
3895 };
3896
3897 lower(store.as_context_mut(), result, false, Some(task))?;
3898 let state = store.0.concurrent_state_mut()?;
3899 match &mut state.get_mut(task)?.state {
3900 pending @ HostTaskState::CalleeCancelling => {
3903 *pending = HostTaskState::CalleeDone { cancelled: true };
3904 }
3905
3906 other => *other = HostTaskState::CalleeDone { cancelled: false },
3908 }
3909 Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
3910
3911 store.0.set_thread(old)?;
3912 Ok(())
3913 };
3914
3915 tls::get(move |store| {
3916 if result_may_require_realloc {
3917 store
3922 .concurrent_state_mut()?
3923 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
3924 on_complete,
3925 ))));
3926 Ok(())
3927 } else {
3928 on_complete(store)
3931 }
3932 })
3933 });
3934
3935 let caller = match host_task {
3938 Some(caller) => caller,
3939 None => bail_bug!("host task wasn't created but should have been"),
3940 };
3941 let state = store.0.concurrent_state_mut()?;
3942 state.push_future(future);
3943 let instance = state.get_mut(caller.task)?.instance;
3944 let handle = store
3945 .0
3946 .handle_table_insertion(instance)
3947 .subtask_host(task.rep())?;
3948 store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
3949 log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
3950
3951 store.0.set_thread(caller)?;
3955 Ok(Status::Started.pack(Some(handle)))
3956 }
3957
3958 pub(crate) fn task_return(
3961 self,
3962 store: &mut dyn VMStore,
3963 ty: TypeTupleIndex,
3964 options: OptionsIndex,
3965 storage: &[ValRaw],
3966 ) -> Result<()> {
3967 let guest_thread = store.current_guest_thread()?;
3968 let state = store.concurrent_state_mut()?;
3969 if !state.get_mut(guest_thread.task)?.async_lifted {
3970 bail!(Trap::TaskReturnOrCancelSyncLifted);
3971 }
3972 let lift = state
3973 .get_mut(guest_thread.task)?
3974 .lift_result
3975 .take()
3976 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
3977 if !state.get_mut(guest_thread.task)?.result.is_none() {
3978 bail_bug!("task result unexpectedly already set");
3979 }
3980
3981 let CanonicalOptions {
3982 string_encoding,
3983 data_model,
3984 ..
3985 } = &self.id().get(store).component().env_component().options[options];
3986
3987 let invalid = ty != lift.ty
3988 || string_encoding != &lift.string_encoding
3989 || match data_model {
3990 CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
3991 Some(memory) => {
3992 let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
3993 let actual = self.id().get(store).runtime_memory(memory);
3994 expected != actual.as_ptr()
3995 }
3996 None => false,
3999 },
4000 CanonicalOptionsDataModel::Gc { .. } => true,
4002 };
4003
4004 if invalid {
4005 bail!(Trap::TaskReturnInvalid);
4006 }
4007
4008 log::trace!("task.return for {guest_thread:?}");
4009
4010 let result = (lift.lift)(store, storage)?;
4011 self.task_complete(store, guest_thread.task, result, Status::Returned)
4012 }
4013
4014 pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4016 let guest_thread = store.current_guest_thread()?;
4017 let state = store.concurrent_state_mut()?;
4018 let task = state.get_mut(guest_thread.task)?;
4019 if !task.async_lifted {
4020 bail!(Trap::TaskReturnOrCancelSyncLifted);
4021 }
4022 if !task.cancel_request_delivered {
4023 bail!(Trap::TaskCancelNotCancelled);
4024 }
4025 _ = task
4026 .lift_result
4027 .take()
4028 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4029
4030 if !task.result.is_none() {
4031 bail_bug!("task result should not bet set yet");
4032 }
4033
4034 log::trace!("task.cancel for {guest_thread:?}");
4035
4036 self.task_complete(
4037 store,
4038 guest_thread.task,
4039 Box::new(DummyResult),
4040 Status::ReturnCancelled,
4041 )
4042 }
4043
4044 fn task_complete(
4050 self,
4051 store: &mut StoreOpaque,
4052 guest_task: TableId<GuestTask>,
4053 result: Box<dyn Any + Send + Sync>,
4054 status: Status,
4055 ) -> Result<()> {
4056 store
4057 .component_resource_tables(Some(self))?
4058 .validate_scope_exit()?;
4059
4060 let state = store.concurrent_state_mut()?;
4061 let task = state.get_mut(guest_task)?;
4062
4063 task.event = None;
4067
4068 if let Caller::Host { tx, .. } = &mut task.caller {
4069 if let Some(tx) = tx.take() {
4070 _ = tx.send(result);
4071 }
4072 } else {
4073 task.result = Some(result);
4074 Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4075 }
4076
4077 Ok(())
4078 }
4079
4080 pub(crate) fn waitable_set_new(
4082 self,
4083 store: &mut StoreOpaque,
4084 caller_instance: RuntimeComponentInstanceIndex,
4085 ) -> Result<u32> {
4086 let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4087 let handle = store
4088 .handle_table_insertion(self.runtime_instance(caller_instance))
4089 .waitable_set(set.rep())?;
4090 log::trace!("new waitable set {set:?} (handle {handle})");
4091 Ok(handle)
4092 }
4093
4094 pub(crate) fn waitable_set_drop(
4096 self,
4097 store: &mut StoreOpaque,
4098 caller_instance: RuntimeComponentInstanceIndex,
4099 set: u32,
4100 ) -> Result<()> {
4101 let rep = store
4102 .instance_state(self.runtime_instance(caller_instance))
4103 .handle_table()
4104 .waitable_set_remove(set)?;
4105
4106 log::trace!("drop waitable set {rep} (handle {set})");
4107
4108 let set = store
4112 .concurrent_state_mut()?
4113 .get_mut(TableId::<WaitableSet>::new(rep))?;
4114 if set.num_waiting > 0 {
4115 bail!(Trap::WaitableSetDropHasWaiters);
4116 }
4117
4118 store
4119 .concurrent_state_mut()?
4120 .delete(TableId::<WaitableSet>::new(rep))?;
4121
4122 Ok(())
4123 }
4124
4125 pub(crate) fn waitable_join(
4127 self,
4128 store: &mut StoreOpaque,
4129 caller_instance: RuntimeComponentInstanceIndex,
4130 waitable_handle: u32,
4131 set_handle: u32,
4132 ) -> Result<()> {
4133 let mut instance = self.id().get_mut(store);
4134 let waitable =
4135 Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4136
4137 let set = if set_handle == 0 {
4138 None
4139 } else {
4140 let set = instance.instance_states().0[caller_instance]
4141 .handle_table()
4142 .waitable_set_rep(set_handle)?;
4143
4144 Some(TableId::<WaitableSet>::new(set))
4145 };
4146
4147 waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4148
4149 log::trace!(
4150 "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4151 );
4152
4153 waitable.join(store.concurrent_state_mut()?, set)
4154 }
4155
4156 pub(crate) fn subtask_drop(
4158 self,
4159 store: &mut StoreOpaque,
4160 caller_instance: RuntimeComponentInstanceIndex,
4161 task_id: u32,
4162 ) -> Result<()> {
4163 self.waitable_join(store, caller_instance, task_id, 0)?;
4164
4165 let (rep, is_host) = store
4166 .instance_state(self.runtime_instance(caller_instance))
4167 .handle_table()
4168 .subtask_remove(task_id)?;
4169
4170 let concurrent_state = store.concurrent_state_mut()?;
4171 let waitable = if is_host {
4172 let id = TableId::<HostTask>::new(rep);
4173 let task = concurrent_state.get_mut(id)?;
4174 match &task.state {
4175 HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4176 bail!(Trap::SubtaskDropNotResolved)
4177 }
4178 HostTaskState::CalleeDone { .. } => {}
4179 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4180 bail_bug!("invalid state for callee in `subtask.drop`")
4181 }
4182 }
4183
4184 Waitable::Host(id)
4185 } else {
4186 let id = TableId::<GuestTask>::new(rep);
4187 let task = concurrent_state.get_mut(id)?;
4188 if task.lift_result.is_some() {
4189 bail!(Trap::SubtaskDropNotResolved);
4190 }
4191 Waitable::Guest(id)
4192 };
4193
4194 waitable.common(concurrent_state)?.handle = None;
4195
4196 if waitable.take_event(concurrent_state)?.is_some() {
4199 bail!(Trap::SubtaskDropNotResolved);
4200 }
4201
4202 waitable.delete_from(concurrent_state)?;
4203
4204 log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4205 Ok(())
4206 }
4207
4208 pub(crate) fn waitable_set_wait(
4210 self,
4211 store: &mut StoreOpaque,
4212 options: OptionsIndex,
4213 set: u32,
4214 payload: u32,
4215 ) -> Result<u32> {
4216 let &CanonicalOptions {
4217 instance: caller_instance,
4218 ..
4219 } = &self.id().get(store).component().env_component().options[options];
4220 let caller = self.runtime_instance(caller_instance);
4221 let rep = store
4222 .instance_state(self.runtime_instance(caller_instance))
4223 .handle_table()
4224 .waitable_set_rep(set)?;
4225
4226 self.waitable_check(
4227 store,
4228 caller,
4229 WaitableCheck::Wait,
4230 WaitableCheckParams {
4231 set: TableId::new(rep),
4232 options,
4233 payload,
4234 },
4235 )
4236 }
4237
4238 pub(crate) fn waitable_set_poll(
4240 self,
4241 store: &mut StoreOpaque,
4242 options: OptionsIndex,
4243 set: u32,
4244 payload: u32,
4245 ) -> Result<u32> {
4246 let &CanonicalOptions {
4247 instance: caller_instance,
4248 ..
4249 } = &self.id().get(store).component().env_component().options[options];
4250 let caller = self.runtime_instance(caller_instance);
4251 let rep = store
4252 .instance_state(caller)
4253 .handle_table()
4254 .waitable_set_rep(set)?;
4255
4256 self.waitable_check(
4257 store,
4258 caller,
4259 WaitableCheck::Poll,
4260 WaitableCheckParams {
4261 set: TableId::new(rep),
4262 options,
4263 payload,
4264 },
4265 )
4266 }
4267
4268 pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4270 let thread_id = store.current_guest_thread()?.thread;
4271 match store
4272 .concurrent_state_mut()?
4273 .get_mut(thread_id)?
4274 .instance_rep
4275 {
4276 Some(r) => Ok(r),
4277 None => bail_bug!("thread should have instance_rep by now"),
4278 }
4279 }
4280
4281 pub(crate) fn thread_new_indirect<T: 'static>(
4283 self,
4284 mut store: StoreContextMut<T>,
4285 runtime_instance: RuntimeComponentInstanceIndex,
4286 start_func_ty: ModuleInternedTypeIndex,
4287 start_func_table_idx: RuntimeTableIndex,
4288 start_func_idx: u32,
4289 context: i32,
4290 ) -> Result<u32> {
4291 log::trace!("creating new thread");
4292
4293 let (instance, registry) = self.id().get_mut_and_registry(store.0);
4294 let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4295 else {
4296 bail_bug!("thread.new-indirect start function type should be registered");
4297 };
4298 let callee = instance
4299 .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4300 .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4301 if callee.type_index(store.0) != start_func_ty {
4302 bail!(Trap::ThreadNewIndirectInvalidType);
4303 }
4304
4305 let token = StoreToken::new(store.as_context_mut());
4306 let start_func = Box::new(
4307 move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4308 let old_thread = store.set_thread(guest_thread)?;
4309 log::trace!(
4310 "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4311 );
4312
4313 let mut store = token.as_context_mut(store);
4314 let mut params = [ValRaw::i32(context)];
4315 unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4318
4319 store.0.set_thread(old_thread)?;
4320
4321 let runtime_instance = self.runtime_instance(runtime_instance);
4322
4323 store
4326 .0
4327 .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4328
4329 store.0.cleanup_thread(guest_thread, runtime_instance)?;
4330
4331 log::trace!("explicit thread {guest_thread:?} completed");
4332 let state = store.0.concurrent_state_mut()?;
4333 if let Some(t) = old_thread.guest() {
4334 state.set_thread_running(t.thread)?;
4335 }
4336 log::trace!("thread start: restored {old_thread:?} as current thread");
4337
4338 Ok(())
4339 },
4340 );
4341
4342 let current_thread = store.0.current_guest_thread()?;
4343 let state = store.0.concurrent_state_mut()?;
4344 let parent_task = current_thread.task;
4345
4346 let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4347 let thread_id = state.push(new_thread)?;
4348 state.get_mut(parent_task)?.threads.insert(thread_id);
4349
4350 log::trace!("new thread with id {thread_id:?} created");
4351
4352 self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4353 }
4354
4355 pub(crate) fn resume_thread(
4356 self,
4357 store: &mut StoreOpaque,
4358 runtime_instance: RuntimeComponentInstanceIndex,
4359 thread_idx: u32,
4360 how: ResumeThread,
4361 ) -> Result<bool> {
4362 let thread_id =
4363 GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4364 let state = store.concurrent_state_mut()?;
4365 let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4366
4367 if store.current_guest_thread()? == guest_thread {
4368 bail!(Trap::CannotResumeThread);
4369 }
4370
4371 let priority = match how {
4372 ResumeThread::Resume => Priority::Switch,
4373 ResumeThread::ResumeLater => Priority::Low,
4374
4375 ResumeThread::Promote => {
4378 let instance = self.runtime_instance(runtime_instance);
4379 let do_not_enter = store
4380 .instance_state(instance)
4381 .concurrent_state()
4382 .do_not_enter;
4383 return store.concurrent_state_mut()?.promote_work_item_matching(
4384 |item: &WorkItem| match item {
4385 WorkItem::ResumeThread { thread, .. }
4386 | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4387
4388 WorkItem::GuestCall {
4389 call: GuestCall { thread, kind },
4390 ..
4391 } => {
4392 *thread == guest_thread
4393 && match kind {
4394 GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4398
4399 GuestCallKind::StartExplicit(_) => true,
4402
4403 GuestCallKind::StartImplicit(_) => false,
4407 }
4408 }
4409
4410 WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4411 },
4412 );
4413 }
4414 };
4415
4416 let state = store.concurrent_state_mut()?;
4417 let thread = state.get_mut(guest_thread.thread)?;
4418
4419 match &thread.state {
4422 GuestThreadState::NotStartedExplicit(_) => {}
4423 GuestThreadState::Suspended(_) => {
4424 log::trace!("resuming thread {thread_id:?} that was suspended");
4425 state.schedule_suspended_thread(
4426 guest_thread,
4427 self.runtime_instance(runtime_instance),
4428 priority,
4429 )?;
4430 return Ok(true);
4431 }
4432 _ => bail!(Trap::CannotResumeThread),
4433 }
4434 match mem::replace(&mut thread.state, GuestThreadState::Running) {
4435 GuestThreadState::NotStartedExplicit(start_func) => {
4436 log::trace!("starting thread {guest_thread:?}");
4437 let guest_call = WorkItem::GuestCall {
4438 instance: self.runtime_instance(runtime_instance),
4439 call: GuestCall {
4440 thread: guest_thread,
4441 kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4442 start_func(store, guest_thread)
4443 })),
4444 },
4445 };
4446 state.push_work_item(guest_call, priority)?;
4447 }
4448 other => {
4449 thread.state = other;
4450 bail_bug!("thread state checked to be resumable above");
4451 }
4452 }
4453 Ok(true)
4454 }
4455
4456 fn add_guest_thread_to_instance_table(
4457 self,
4458 thread_id: TableId<GuestThread>,
4459 store: &mut StoreOpaque,
4460 runtime_instance: RuntimeComponentInstanceIndex,
4461 ) -> Result<u32> {
4462 let (state, counter) =
4463 store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4464 let guest_id = state
4465 .thread_handle_table()
4466 .guest_thread_insert(counter, thread_id.rep())?;
4467 store
4468 .concurrent_state_mut()?
4469 .get_mut(thread_id)?
4470 .instance_rep = Some(guest_id);
4471 Ok(guest_id)
4472 }
4473
4474 pub(crate) fn suspension_intrinsic(
4478 self,
4479 store: &mut StoreOpaque,
4480 caller: RuntimeComponentInstanceIndex,
4481 yielding: bool,
4482 to_thread: SuspensionTarget,
4483 ) -> Result<WaitResult> {
4484 let check_suspend = match to_thread {
4485 SuspensionTarget::Promote(thread) => {
4486 !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4487 }
4488 SuspensionTarget::Resume(thread) => {
4489 if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4490 bail_bug!(
4491 "`resume_thread` should only ever return false \
4492 when `ResumeThread::Promote` is passed to it"
4493 );
4494 }
4495 false
4496 }
4497 SuspensionTarget::None => true,
4498 };
4499
4500 if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4501 return if yielding {
4502 Ok(WaitResult::Completed)
4503 } else {
4504 Err(Trap::CannotBlockSyncTask.into())
4505 };
4506 }
4507
4508 let guest_thread = store.current_guest_thread()?;
4509
4510 let reason = if yielding {
4511 SuspendReason::Yielding {
4512 thread: guest_thread,
4513 }
4514 } else {
4515 SuspendReason::ExplicitlySuspending {
4516 thread: guest_thread,
4517 }
4518 };
4519
4520 store.suspend(reason)?;
4521
4522 Ok(WaitResult::Completed)
4523 }
4524
4525 fn waitable_check(
4527 self,
4528 store: &mut StoreOpaque,
4529 caller: RuntimeInstance,
4530 check: WaitableCheck,
4531 params: WaitableCheckParams,
4532 ) -> Result<u32> {
4533 let guest_thread = store.current_guest_thread()?;
4534
4535 log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4536
4537 match &check {
4540 WaitableCheck::Wait => {
4541 let set = params.set;
4542
4543 loop {
4548 let state = store.concurrent_state_mut()?;
4549 let task = state.get_mut(guest_thread.task)?;
4550 if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4551 || !state.get_mut(set)?.ready.is_empty()
4552 {
4553 break;
4554 }
4555
4556 store.switch_or_trap_if_may_not_suspend(caller)?;
4557
4558 store.suspend(SuspendReason::Waiting {
4559 set,
4560 thread: guest_thread,
4561 })?;
4562 }
4563 }
4564 WaitableCheck::Poll => {}
4565 }
4566
4567 log::trace!(
4568 "waitable check for {guest_thread:?}; set {:?}, part two",
4569 params.set
4570 );
4571
4572 let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4574
4575 let (ordinal, handle, result) = match &check {
4576 WaitableCheck::Wait => {
4577 let (event, waitable) = match event {
4578 Some(p) => p,
4579 None => bail_bug!("event expected to be present"),
4580 };
4581 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4582 let (ordinal, result) = event.parts();
4583 (ordinal, handle, result)
4584 }
4585 WaitableCheck::Poll => {
4586 if let Some((event, waitable)) = event {
4587 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4588 let (ordinal, result) = event.parts();
4589 (ordinal, handle, result)
4590 } else {
4591 log::trace!(
4592 "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4593 guest_thread.task,
4594 params.set
4595 );
4596 let (ordinal, result) = Event::None.parts();
4597 (ordinal, 0, result)
4598 }
4599 }
4600 };
4601 let ptr = crate::component::func::validate_inbounds_dynamic(
4602 &CanonicalAbiInfo::POINTER_PAIR,
4603 self.options_memory(store, params.options),
4604 &ValRaw::u32(params.payload),
4605 )?;
4606 let memory = self
4607 .options_memory_range_mut(store, params.options, ptr, 8)
4608 .unwrap();
4609 memory[0..4].copy_from_slice(&handle.to_le_bytes());
4610 memory[4..8].copy_from_slice(&result.to_le_bytes());
4611 Ok(ordinal)
4612 }
4613
4614 pub(crate) fn subtask_cancel(
4616 self,
4617 store: &mut StoreOpaque,
4618 caller_instance: RuntimeComponentInstanceIndex,
4619 async_: bool,
4620 task_id: u32,
4621 ) -> Result<u32> {
4622 let (rep, is_host) = store
4623 .instance_state(self.runtime_instance(caller_instance))
4624 .handle_table()
4625 .subtask_rep(task_id)?;
4626 let waitable = if is_host {
4627 Waitable::Host(TableId::<HostTask>::new(rep))
4628 } else {
4629 Waitable::Guest(TableId::<GuestTask>::new(rep))
4630 };
4631 let concurrent_state = store.concurrent_state_mut()?;
4632
4633 log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4634
4635 waitable.trap_if_in_waitable_set(concurrent_state)?;
4636
4637 let needs_block;
4638 if let Waitable::Host(host_task) = waitable {
4639 let state = &mut concurrent_state.get_mut(host_task)?.state;
4640 match state {
4641 HostTaskState::CalleeRunning(handle) => {
4648 handle.abort();
4649 *state = HostTaskState::CalleeCancelling;
4650 needs_block = true;
4651 }
4652
4653 HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4656 bail!(Trap::SubtaskCancelAfterTerminal);
4657 }
4658 HostTaskState::CalleeDone { cancelled: false } => {
4659 *state = HostTaskState::CalleeDone { cancelled: true };
4662 needs_block = false;
4663 }
4664
4665 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4668 bail_bug!("invalid states for host callee")
4669 }
4670 }
4671 } else {
4672 let guest_task = TableId::<GuestTask>::new(rep);
4673 let task = concurrent_state.get_mut(guest_task)?;
4674
4675 if task.cancel_requested {
4678 bail!(Trap::SubtaskCancelAfterTerminal);
4679 }
4680
4681 if !task.already_lowered_parameters() {
4682 store.cancel_guest_subtask_without_lowered_parameters(
4683 self.runtime_instance(caller_instance),
4684 guest_task,
4685 )?;
4686 return Ok(Status::StartCancelled as u32);
4687 } else if !task.returned_or_cancelled() {
4688 task.cancel_requested = true;
4695 task.event = Some(Event::Cancelled);
4696 let runtime_instance = task.instance;
4697 for thread in task.threads.clone() {
4698 let thread = QualifiedThreadId {
4699 task: guest_task,
4700 thread,
4701 };
4702 let concurrent_state = store.concurrent_state_mut()?;
4703 let thread_mut = concurrent_state.get_mut(thread.thread)?;
4704
4705 let yield_ = |store: &mut StoreOpaque| {
4706 let state = store.instance_state(runtime_instance).concurrent_state();
4711 let old_do_not_suspend = state.do_not_suspend;
4712 state.do_not_suspend = false;
4713
4714 let caller = store.current_guest_thread()?;
4715
4716 let state = store.concurrent_state_mut()?;
4721 let set = state.get_mut(caller.thread)?.sync_call_set;
4722 waitable.join(state, Some(set))?;
4723
4724 store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4725
4726 let state = store.concurrent_state_mut()?;
4727 waitable.join(state, None)?;
4728
4729 store
4730 .instance_state(runtime_instance)
4731 .concurrent_state()
4732 .do_not_suspend = old_do_not_suspend;
4733
4734 Ok::<(), crate::Error>(())
4735 };
4736
4737 match thread_mut.wake_on_cancel.take() {
4738 WakeOnCancel::Waiting(set) => {
4739 let instance = concurrent_state.take_callback_waiter(set, thread)?;
4741 let item = WorkItem::GuestCall {
4742 instance: runtime_instance,
4743 call: GuestCall {
4744 thread,
4745 kind: GuestCallKind::DeliverEvent {
4746 instance,
4747 set: Some(set),
4748 },
4749 },
4750 };
4751 concurrent_state.set_switch_item(item)?;
4752
4753 yield_(store)?;
4754
4755 break;
4756 }
4757 WakeOnCancel::Yielding => {
4758 if concurrent_state.promote_thread_work_item(thread)? {
4759 yield_(store)?;
4760 break;
4761 } else if store
4762 .instance_state(runtime_instance)
4763 .concurrent_state()
4764 .pending
4765 .contains_key(&thread)
4766 {
4767 store
4777 .concurrent_state_mut()?
4778 .get_mut(thread.thread)?
4779 .wake_on_cancel = WakeOnCancel::Yielding;
4780 } else {
4781 bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4782 }
4783 }
4784 WakeOnCancel::None => {}
4785 }
4786 }
4787
4788 needs_block = !store
4791 .concurrent_state_mut()?
4792 .get_mut(guest_task)?
4793 .returned_or_cancelled()
4794 } else {
4795 needs_block = false;
4796 }
4797 };
4798
4799 if needs_block {
4803 if async_ {
4804 return Ok(BLOCKED);
4805 }
4806
4807 let old_next_switch_item = if is_host {
4817 Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4818 } else {
4819 None
4820 };
4821
4822 store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4825
4826 if let Some(item) = old_next_switch_item {
4827 let state = store.concurrent_state_mut()?;
4828 state.restore_next_switch_item(item)?;
4829 }
4830
4831 }
4833
4834 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4835 if let Some(Event::Subtask {
4836 status: status @ (Status::Returned | Status::ReturnCancelled),
4837 }) = event
4838 {
4839 Ok(status as u32)
4840 } else {
4841 bail!(Trap::SubtaskCancelAfterTerminal);
4842 }
4843 }
4844}
4845
4846pub trait VMComponentAsyncStore {
4854 unsafe fn prepare_call(
4860 &mut self,
4861 instance: Instance,
4862 memory: *mut VMMemoryDefinition,
4863 start: NonNull<VMFuncRef>,
4864 return_: NonNull<VMFuncRef>,
4865 caller_instance: RuntimeComponentInstanceIndex,
4866 callee_instance: RuntimeComponentInstanceIndex,
4867 task_return_type: TypeTupleIndex,
4868 callee_async: bool,
4869 string_encoding: StringEncoding,
4870 result_count: u32,
4871 storage: *mut ValRaw,
4872 storage_len: usize,
4873 ) -> Result<()>;
4874
4875 unsafe fn start_call(
4878 &mut self,
4879 instance: Instance,
4880 callback: *mut VMFuncRef,
4881 post_return: *mut VMFuncRef,
4882 callee: NonNull<VMFuncRef>,
4883 param_count: u32,
4884 result_count: u32,
4885 flags: u32,
4886 storage: *mut MaybeUninit<ValRaw>,
4887 storage_len: usize,
4888 ) -> Result<()>;
4889
4890 fn future_write(
4892 &mut self,
4893 instance: Instance,
4894 caller: RuntimeComponentInstanceIndex,
4895 ty: TypeFutureTableIndex,
4896 options: OptionsIndex,
4897 future: u32,
4898 address: u32,
4899 ) -> Result<u32>;
4900
4901 fn future_read(
4903 &mut self,
4904 instance: Instance,
4905 caller: RuntimeComponentInstanceIndex,
4906 ty: TypeFutureTableIndex,
4907 options: OptionsIndex,
4908 future: u32,
4909 address: u32,
4910 ) -> Result<u32>;
4911
4912 fn future_drop_writable(
4914 &mut self,
4915 instance: Instance,
4916 ty: TypeFutureTableIndex,
4917 writer: u32,
4918 ) -> Result<()>;
4919
4920 fn stream_write(
4922 &mut self,
4923 instance: Instance,
4924 caller: RuntimeComponentInstanceIndex,
4925 ty: TypeStreamTableIndex,
4926 options: OptionsIndex,
4927 stream: u32,
4928 address: u32,
4929 count: u32,
4930 ) -> Result<u32>;
4931
4932 fn stream_read(
4934 &mut self,
4935 instance: Instance,
4936 caller: RuntimeComponentInstanceIndex,
4937 ty: TypeStreamTableIndex,
4938 options: OptionsIndex,
4939 stream: u32,
4940 address: u32,
4941 count: u32,
4942 ) -> Result<u32>;
4943
4944 fn flat_stream_write(
4947 &mut self,
4948 instance: Instance,
4949 caller: RuntimeComponentInstanceIndex,
4950 ty: TypeStreamTableIndex,
4951 options: OptionsIndex,
4952 payload_size: u32,
4953 payload_align: u32,
4954 stream: u32,
4955 address: u32,
4956 count: u32,
4957 ) -> Result<u32>;
4958
4959 fn flat_stream_read(
4962 &mut self,
4963 instance: Instance,
4964 caller: RuntimeComponentInstanceIndex,
4965 ty: TypeStreamTableIndex,
4966 options: OptionsIndex,
4967 payload_size: u32,
4968 payload_align: u32,
4969 stream: u32,
4970 address: u32,
4971 count: u32,
4972 ) -> Result<u32>;
4973
4974 fn stream_drop_writable(
4976 &mut self,
4977 instance: Instance,
4978 ty: TypeStreamTableIndex,
4979 writer: u32,
4980 ) -> Result<()>;
4981
4982 fn error_context_debug_message(
4984 &mut self,
4985 instance: Instance,
4986 ty: TypeComponentLocalErrorContextTableIndex,
4987 options: OptionsIndex,
4988 err_ctx_handle: u32,
4989 debug_msg_address: u32,
4990 ) -> Result<()>;
4991
4992 fn thread_new_indirect(
4994 &mut self,
4995 instance: Instance,
4996 caller: RuntimeComponentInstanceIndex,
4997 func_ty_idx: ModuleInternedTypeIndex,
4998 start_func_table_idx: RuntimeTableIndex,
4999 start_func_idx: u32,
5000 context: i32,
5001 ) -> Result<u32>;
5002}
5003
5004impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5006 unsafe fn prepare_call(
5007 &mut self,
5008 instance: Instance,
5009 memory: *mut VMMemoryDefinition,
5010 start: NonNull<VMFuncRef>,
5011 return_: NonNull<VMFuncRef>,
5012 caller_instance: RuntimeComponentInstanceIndex,
5013 callee_instance: RuntimeComponentInstanceIndex,
5014 task_return_type: TypeTupleIndex,
5015 callee_async: bool,
5016 string_encoding: StringEncoding,
5017 result_count_or_max_if_async: u32,
5018 storage: *mut ValRaw,
5019 storage_len: usize,
5020 ) -> Result<()> {
5021 let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5025
5026 unsafe {
5027 instance.prepare_call(
5028 StoreContextMut(self),
5029 start,
5030 return_,
5031 caller_instance,
5032 callee_instance,
5033 task_return_type,
5034 callee_async,
5035 memory,
5036 string_encoding,
5037 match result_count_or_max_if_async {
5038 PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5039 params,
5040 has_result: false,
5041 },
5042 PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5043 params,
5044 has_result: true,
5045 },
5046 result_count => CallerInfo::Sync {
5047 params,
5048 result_count,
5049 },
5050 },
5051 )
5052 }
5053 }
5054
5055 unsafe fn start_call(
5056 &mut self,
5057 instance: Instance,
5058 callback: *mut VMFuncRef,
5059 post_return: *mut VMFuncRef,
5060 callee: NonNull<VMFuncRef>,
5061 param_count: u32,
5062 result_count: u32,
5063 flags: u32,
5064 storage: *mut MaybeUninit<ValRaw>,
5065 storage_len: usize,
5066 ) -> Result<()> {
5067 unsafe {
5068 instance.start_call(
5069 StoreContextMut(self),
5070 callback,
5071 post_return,
5072 callee,
5073 param_count,
5074 result_count,
5075 flags,
5076 core::slice::from_raw_parts_mut(storage, storage_len),
5080 )
5081 }
5082 }
5083
5084 fn future_write(
5085 &mut self,
5086 instance: Instance,
5087 caller: RuntimeComponentInstanceIndex,
5088 ty: TypeFutureTableIndex,
5089 options: OptionsIndex,
5090 future: u32,
5091 address: u32,
5092 ) -> Result<u32> {
5093 instance
5094 .guest_write(
5095 StoreContextMut(self),
5096 caller,
5097 TransmitIndex::Future(ty),
5098 options,
5099 None,
5100 future,
5101 address,
5102 1,
5103 )
5104 .map(|result| result.encode())
5105 }
5106
5107 fn future_read(
5108 &mut self,
5109 instance: Instance,
5110 caller: RuntimeComponentInstanceIndex,
5111 ty: TypeFutureTableIndex,
5112 options: OptionsIndex,
5113 future: u32,
5114 address: u32,
5115 ) -> Result<u32> {
5116 instance
5117 .guest_read(
5118 StoreContextMut(self),
5119 caller,
5120 TransmitIndex::Future(ty),
5121 options,
5122 None,
5123 future,
5124 address,
5125 1,
5126 )
5127 .map(|result| result.encode())
5128 }
5129
5130 fn stream_write(
5131 &mut self,
5132 instance: Instance,
5133 caller: RuntimeComponentInstanceIndex,
5134 ty: TypeStreamTableIndex,
5135 options: OptionsIndex,
5136 stream: u32,
5137 address: u32,
5138 count: u32,
5139 ) -> Result<u32> {
5140 instance
5141 .guest_write(
5142 StoreContextMut(self),
5143 caller,
5144 TransmitIndex::Stream(ty),
5145 options,
5146 None,
5147 stream,
5148 address,
5149 count,
5150 )
5151 .map(|result| result.encode())
5152 }
5153
5154 fn stream_read(
5155 &mut self,
5156 instance: Instance,
5157 caller: RuntimeComponentInstanceIndex,
5158 ty: TypeStreamTableIndex,
5159 options: OptionsIndex,
5160 stream: u32,
5161 address: u32,
5162 count: u32,
5163 ) -> Result<u32> {
5164 instance
5165 .guest_read(
5166 StoreContextMut(self),
5167 caller,
5168 TransmitIndex::Stream(ty),
5169 options,
5170 None,
5171 stream,
5172 address,
5173 count,
5174 )
5175 .map(|result| result.encode())
5176 }
5177
5178 fn future_drop_writable(
5179 &mut self,
5180 instance: Instance,
5181 ty: TypeFutureTableIndex,
5182 writer: u32,
5183 ) -> Result<()> {
5184 instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5185 }
5186
5187 fn flat_stream_write(
5188 &mut self,
5189 instance: Instance,
5190 caller: RuntimeComponentInstanceIndex,
5191 ty: TypeStreamTableIndex,
5192 options: OptionsIndex,
5193 payload_size: u32,
5194 payload_align: u32,
5195 stream: u32,
5196 address: u32,
5197 count: u32,
5198 ) -> Result<u32> {
5199 instance
5200 .guest_write(
5201 StoreContextMut(self),
5202 caller,
5203 TransmitIndex::Stream(ty),
5204 options,
5205 Some(FlatAbi {
5206 size: payload_size,
5207 align: payload_align,
5208 }),
5209 stream,
5210 address,
5211 count,
5212 )
5213 .map(|result| result.encode())
5214 }
5215
5216 fn flat_stream_read(
5217 &mut self,
5218 instance: Instance,
5219 caller: RuntimeComponentInstanceIndex,
5220 ty: TypeStreamTableIndex,
5221 options: OptionsIndex,
5222 payload_size: u32,
5223 payload_align: u32,
5224 stream: u32,
5225 address: u32,
5226 count: u32,
5227 ) -> Result<u32> {
5228 instance
5229 .guest_read(
5230 StoreContextMut(self),
5231 caller,
5232 TransmitIndex::Stream(ty),
5233 options,
5234 Some(FlatAbi {
5235 size: payload_size,
5236 align: payload_align,
5237 }),
5238 stream,
5239 address,
5240 count,
5241 )
5242 .map(|result| result.encode())
5243 }
5244
5245 fn stream_drop_writable(
5246 &mut self,
5247 instance: Instance,
5248 ty: TypeStreamTableIndex,
5249 writer: u32,
5250 ) -> Result<()> {
5251 instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5252 }
5253
5254 fn error_context_debug_message(
5255 &mut self,
5256 instance: Instance,
5257 ty: TypeComponentLocalErrorContextTableIndex,
5258 options: OptionsIndex,
5259 err_ctx_handle: u32,
5260 debug_msg_address: u32,
5261 ) -> Result<()> {
5262 instance.error_context_debug_message(
5263 StoreContextMut(self),
5264 ty,
5265 options,
5266 err_ctx_handle,
5267 debug_msg_address,
5268 )
5269 }
5270
5271 fn thread_new_indirect(
5272 &mut self,
5273 instance: Instance,
5274 caller: RuntimeComponentInstanceIndex,
5275 func_ty_idx: ModuleInternedTypeIndex,
5276 start_func_table_idx: RuntimeTableIndex,
5277 start_func_idx: u32,
5278 context: i32,
5279 ) -> Result<u32> {
5280 instance.thread_new_indirect(
5281 StoreContextMut(self),
5282 caller,
5283 func_ty_idx,
5284 start_func_table_idx,
5285 start_func_idx,
5286 context,
5287 )
5288 }
5289}
5290
5291type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5292
5293async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5296where
5297 F: Future,
5298{
5299 let mut future = pin!(future);
5300 future::poll_fn(|cx| {
5301 let old_thread = match tls::get(|store| store.set_thread(task)) {
5302 Ok(thread) => thread,
5303 Err(error) => return Poll::Ready(Err(error)),
5304 };
5305 let result = future.as_mut().poll(cx);
5306 match tls::get(|store| store.set_thread(old_thread)) {
5307 Ok(_) => result.map(Ok),
5308 Err(error) => Poll::Ready(Err(error)),
5309 }
5310 })
5311 .await
5312}
5313
5314pub(crate) struct HostTask {
5318 common: WaitableCommon,
5319
5320 call_context: CallContext,
5323
5324 state: HostTaskState,
5325
5326 group: TaskGroupId,
5327}
5328
5329enum HostTaskState {
5330 CalleeStarted,
5335
5336 CalleeRunning(JoinHandle),
5341
5342 CalleeCancelling,
5346
5347 CalleeFinished(LiftedResult),
5351
5352 CalleeDone { cancelled: bool },
5355}
5356
5357impl HostTask {
5358 fn new(
5359 concurrent_state: &mut ConcurrentState,
5360 state: HostTaskState,
5361 caller: QualifiedThreadId,
5362 ) -> Result<Self> {
5363 let group = concurrent_state.get_mut(caller.task)?.group;
5364 concurrent_state.increment_group_ref_count(group)?;
5365
5366 Ok(Self {
5367 common: WaitableCommon::default(),
5368 call_context: CallContext::default(),
5369 state,
5370 group,
5371 })
5372 }
5373}
5374
5375impl TableDebug for HostTask {
5376 fn type_name() -> &'static str {
5377 "HostTask"
5378 }
5379}
5380
5381type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5382
5383enum Caller {
5385 Host {
5387 tx: Option<oneshot::Sender<LiftedResult>>,
5389 caller: Option<TableId<HostTask>>,
5393 },
5394 Guest {
5396 thread: QualifiedThreadId,
5398 },
5399}
5400
5401struct LiftResult {
5404 lift: RawLift,
5405 ty: TypeTupleIndex,
5406 memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5407 string_encoding: StringEncoding,
5408}
5409
5410#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5415pub(crate) struct QualifiedThreadId {
5416 task: TableId<GuestTask>,
5417 thread: TableId<GuestThread>,
5418}
5419
5420impl QualifiedThreadId {
5421 fn qualify(
5422 state: &mut ConcurrentState,
5423 thread: TableId<GuestThread>,
5424 ) -> Result<QualifiedThreadId> {
5425 Ok(QualifiedThreadId {
5426 task: state.get_mut(thread)?.parent_task,
5427 thread,
5428 })
5429 }
5430}
5431
5432impl fmt::Debug for QualifiedThreadId {
5433 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5434 f.debug_tuple("QualifiedThreadId")
5435 .field(&self.task.rep())
5436 .field(&self.thread.rep())
5437 .finish()
5438 }
5439}
5440
5441enum GuestThreadState {
5442 NotStartedImplicit,
5443 NotStartedExplicit(
5444 Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5445 ),
5446 Running,
5447 Suspended(StoreFiber<'static>),
5448 Ready {
5449 fiber: StoreFiber<'static>,
5450 },
5451 Completed,
5452}
5453
5454impl GuestThreadState {
5455 fn check_no_fiber(&self) -> Result<()> {
5456 match self {
5457 Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5458 Self::NotStartedImplicit
5459 | Self::NotStartedExplicit(_)
5460 | Self::Running
5461 | Self::Completed => Ok(()),
5462 }
5463 }
5464}
5465
5466impl fmt::Debug for GuestThreadState {
5467 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5468 match self {
5469 Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5470 Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5471 Self::Running => f.debug_tuple("Running").finish(),
5472 Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5473 Self::Ready { .. } => f.debug_struct("Ready").finish(),
5474 Self::Completed => f.debug_tuple("Completed").finish(),
5475 }
5476 }
5477}
5478
5479#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5480enum WakeOnCancel {
5481 None,
5482 Waiting(TableId<WaitableSet>),
5483 Yielding,
5484}
5485
5486impl WakeOnCancel {
5487 fn is_none(self) -> bool {
5488 matches!(self, WakeOnCancel::None)
5489 }
5490
5491 fn replace(&mut self, other: WakeOnCancel) -> Self {
5492 let old = *self;
5493 *self = other;
5494 old
5495 }
5496
5497 fn take(&mut self) -> Self {
5498 self.replace(WakeOnCancel::None)
5499 }
5500}
5501
5502pub struct GuestThread {
5503 context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5506 parent_task: TableId<GuestTask>,
5508 wake_on_cancel: WakeOnCancel,
5511 state: GuestThreadState,
5513 instance_rep: Option<u32>,
5516 sync_call_set: TableId<WaitableSet>,
5518 old_do_not_suspend: Option<bool>,
5521}
5522
5523impl GuestThread {
5524 fn from_instance(
5527 state: Pin<&mut ComponentInstance>,
5528 caller_instance: RuntimeComponentInstanceIndex,
5529 guest_thread: u32,
5530 ) -> Result<TableId<Self>> {
5531 let rep = state.instance_states().0[caller_instance]
5532 .thread_handle_table()
5533 .guest_thread_rep(guest_thread)?;
5534 Ok(TableId::new(rep))
5535 }
5536
5537 fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5538 let sync_call_set = state.push(WaitableSet {
5539 is_sync_call_set: true,
5540 ..WaitableSet::default()
5541 })?;
5542 Ok(Self {
5543 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5544 parent_task,
5545 wake_on_cancel: WakeOnCancel::None,
5546 state: GuestThreadState::NotStartedImplicit,
5547 instance_rep: None,
5548 sync_call_set,
5549 old_do_not_suspend: None,
5550 })
5551 }
5552
5553 fn new_explicit(
5554 state: &mut ConcurrentState,
5555 parent_task: TableId<GuestTask>,
5556 start_func: Box<
5557 dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5558 >,
5559 ) -> Result<Self> {
5560 state.increment_task_ref_count(parent_task)?;
5561 let sync_call_set = state.push(WaitableSet {
5562 is_sync_call_set: true,
5563 ..WaitableSet::default()
5564 })?;
5565 Ok(Self {
5566 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5567 parent_task,
5568 wake_on_cancel: WakeOnCancel::None,
5569 state: GuestThreadState::NotStartedExplicit(start_func),
5570 instance_rep: None,
5571 sync_call_set,
5572 old_do_not_suspend: None,
5573 })
5574 }
5575}
5576
5577impl TableDebug for GuestThread {
5578 fn type_name() -> &'static str {
5579 "GuestThread"
5580 }
5581}
5582
5583enum SyncResult {
5584 NotProduced,
5585 Produced(Option<ValRaw>),
5586 Taken,
5587}
5588
5589impl SyncResult {
5590 fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5591 Ok(match mem::replace(self, SyncResult::Taken) {
5592 SyncResult::NotProduced => None,
5593 SyncResult::Produced(val) => Some(val),
5594 SyncResult::Taken => {
5595 bail_bug!("attempted to take a synchronous result that was already taken")
5596 }
5597 })
5598 }
5599}
5600
5601pub(crate) struct GuestTask {
5603 common: WaitableCommon,
5605 lower_params: Option<RawLower>,
5607 lift_result: Option<LiftResult>,
5609 result: Option<LiftedResult>,
5612 callback: Option<CallbackFn>,
5615 caller: Caller,
5617 call_context: CallContext,
5622 sync_result: SyncResult,
5625 cancel_request_delivered: bool,
5629 cancel_requested: bool,
5634 starting_sent: bool,
5637 instance: RuntimeInstance,
5644 event: Option<Event>,
5646 exited: bool,
5648 threads: HashSet<TableId<GuestThread>>,
5650 async_typed: bool,
5653 async_lifted: bool,
5656 decremented_interesting_task_count: bool,
5659 group: TaskGroupId,
5661 ref_count: usize,
5666}
5667
5668impl GuestTask {
5669 fn already_lowered_parameters(&self) -> bool {
5670 self.lower_params.is_none()
5672 }
5673
5674 fn returned_or_cancelled(&self) -> bool {
5675 self.lift_result.is_none()
5677 }
5678
5679 fn new(
5680 state: &mut ConcurrentState,
5681 lower_params: RawLower,
5682 lift_result: LiftResult,
5683 caller: Caller,
5684 callback: Option<CallbackFn>,
5685 instance: RuntimeInstance,
5686 async_typed: bool,
5687 async_lifted: bool,
5688 ) -> Result<QualifiedThreadId> {
5689 let group = match caller {
5690 Caller::Guest { thread } => {
5691 let group = state.get_mut(thread.task)?.group;
5692 state.increment_group_ref_count(group)?;
5693 group
5694 }
5695 Caller::Host { .. } => state.make_task_group()?,
5696 };
5697
5698 let task = state.push(Self {
5699 common: WaitableCommon::default(),
5700 lower_params: Some(lower_params),
5701 lift_result: Some(lift_result),
5702 result: None,
5703 callback,
5704 caller,
5705 call_context: CallContext::default(),
5706 sync_result: SyncResult::NotProduced,
5707 cancel_request_delivered: false,
5708 cancel_requested: false,
5709 starting_sent: false,
5710 instance,
5711 event: None,
5712 exited: false,
5713 threads: HashSet::new(),
5714 async_typed,
5715 async_lifted,
5716 decremented_interesting_task_count: false,
5717 group,
5718 ref_count: 2,
5721 })?;
5722 let new_thread = GuestThread::new_implicit(state, task)?;
5723 let thread = state.push(new_thread)?;
5724 state.get_mut(task)?.threads.insert(thread);
5725 state.interesting_tasks += 1;
5726 let thread = QualifiedThreadId { task, thread };
5727 log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5728 Ok(thread)
5729 }
5730}
5731
5732impl TableDebug for GuestTask {
5733 fn type_name() -> &'static str {
5734 "GuestTask"
5735 }
5736}
5737
5738#[derive(Default)]
5740struct WaitableCommon {
5741 event: Option<Event>,
5743 set: Option<TableId<WaitableSet>>,
5745 handle: Option<u32>,
5747}
5748
5749#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5751enum Waitable {
5752 Host(TableId<HostTask>),
5754 Guest(TableId<GuestTask>),
5756 Transmit(TableId<TransmitHandle>),
5758}
5759
5760impl Waitable {
5761 fn from_instance(
5764 state: Pin<&mut ComponentInstance>,
5765 caller_instance: RuntimeComponentInstanceIndex,
5766 waitable: u32,
5767 ) -> Result<Self> {
5768 use crate::runtime::vm::component::Waitable;
5769
5770 let (waitable, kind) = state.instance_states().0[caller_instance]
5771 .handle_table()
5772 .waitable_rep(waitable)?;
5773
5774 Ok(match kind {
5775 Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5776 Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5777 Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5778 })
5779 }
5780
5781 fn rep(&self) -> u32 {
5783 match self {
5784 Self::Host(id) => id.rep(),
5785 Self::Guest(id) => id.rep(),
5786 Self::Transmit(id) => id.rep(),
5787 }
5788 }
5789
5790 fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5794 log::trace!("waitable {self:?} join set {set:?}");
5795
5796 let old = mem::replace(&mut self.common(state)?.set, set);
5797
5798 if let Some(old) = old {
5799 match *self {
5800 Waitable::Host(id) => state.remove_child(id, old),
5801 Waitable::Guest(id) => state.remove_child(id, old),
5802 Waitable::Transmit(id) => state.remove_child(id, old),
5803 }?;
5804
5805 state.get_mut(old)?.ready.remove(self);
5806 }
5807
5808 if let Some(set) = set {
5809 match *self {
5810 Waitable::Host(id) => state.add_child(id, set),
5811 Waitable::Guest(id) => state.add_child(id, set),
5812 Waitable::Transmit(id) => state.add_child(id, set),
5813 }?;
5814
5815 if self.common(state)?.event.is_some() {
5816 self.mark_ready(state)?;
5817 }
5818 }
5819
5820 Ok(())
5821 }
5822
5823 fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5825 Ok(match self {
5826 Self::Host(id) => &mut state.get_mut(*id)?.common,
5827 Self::Guest(id) => &mut state.get_mut(*id)?.common,
5828 Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5829 })
5830 }
5831
5832 fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5838 if self.common(state)?.set.is_some() {
5839 bail!(Trap::WaitableSyncAndAsync);
5840 }
5841 Ok(())
5842 }
5843
5844 fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5850 if let Some(set) = self.common(state)?.set
5851 && state.get_mut(set)?.is_sync_call_set
5852 {
5853 bail!(Trap::WaitableSyncAndAsync);
5854 }
5855 Ok(())
5856 }
5857
5858 fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
5862 log::trace!("set event for {self:?}: {event:?}");
5863 self.common(state)?.event = event;
5864 self.mark_ready(state)
5865 }
5866
5867 fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
5869 let common = self.common(state)?;
5870 let event = common.event.take();
5871 if let Some(set) = self.common(state)?.set {
5872 state.get_mut(set)?.ready.remove(self);
5873 }
5874
5875 Ok(event)
5876 }
5877
5878 fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
5882 if let Some(set) = self.common(state)?.set {
5883 state.get_mut(set)?.ready.insert(*self);
5884 state.wake_waiter(set)?;
5885 }
5886 Ok(())
5887 }
5888
5889 fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
5892 match self {
5893 Self::Host(task) => state.delete_host_task(*task),
5894 Self::Guest(task) => state.decrement_task_ref_count(*task),
5895 Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
5896 }
5897 }
5898}
5899
5900impl fmt::Debug for Waitable {
5901 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
5902 match self {
5903 Self::Host(id) => write!(f, "{id:?}"),
5904 Self::Guest(id) => write!(f, "{id:?}"),
5905 Self::Transmit(id) => write!(f, "{id:?}"),
5906 }
5907 }
5908}
5909
5910#[derive(Default)]
5912struct WaitableSet {
5913 ready: BTreeSet<Waitable>,
5915 waiting: BTreeMap<QualifiedThreadId, WaitMode>,
5917 num_waiting: usize,
5920 is_sync_call_set: bool,
5923}
5924
5925impl WaitableSet {
5926 fn stop_waiting(&mut self) -> Result<()> {
5928 self.num_waiting = match self.num_waiting.checked_sub(1) {
5929 Some(n) => n,
5930 None => bail_bug!("waiter not accounted for in waitable set"),
5931 };
5932 Ok(())
5933 }
5934}
5935
5936impl TableDebug for WaitableSet {
5937 fn type_name() -> &'static str {
5938 "WaitableSet"
5939 }
5940}
5941
5942type RawLower =
5944 Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
5945
5946type RawLift = Box<
5948 dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
5949>;
5950
5951type LiftedResult = Box<dyn Any + Send + Sync>;
5955
5956struct DummyResult;
5959
5960#[derive(Default)]
5962pub struct ConcurrentInstanceState {
5963 backpressure: u16,
5965 do_not_enter: bool,
5967 do_not_suspend: bool,
5970 pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
5973}
5974
5975impl ConcurrentInstanceState {
5976 pub fn pending_is_empty(&self) -> bool {
5977 self.pending.is_empty()
5978 }
5979}
5980
5981#[derive(Debug, Copy, Clone)]
5982pub(crate) enum CurrentThread {
5983 Guest(QualifiedThreadId),
5986 Host(TableId<HostTask>),
5988 DeferredHost(QualifiedThreadId),
5991 None,
5994}
5995
5996impl CurrentThread {
5997 fn guest(&self) -> Option<&QualifiedThreadId> {
5998 match self {
5999 Self::Guest(id) => Some(id),
6000 _ => None,
6001 }
6002 }
6003
6004 fn guest_task(&self) -> Option<TableId<GuestTask>> {
6005 match self {
6006 Self::Guest(id) => Some(id.task),
6007 _ => None,
6008 }
6009 }
6010
6011 fn is_none(&self) -> bool {
6012 matches!(self, Self::None)
6013 }
6014}
6015
6016impl From<QualifiedThreadId> for CurrentThread {
6017 fn from(id: QualifiedThreadId) -> Self {
6018 Self::Guest(id)
6019 }
6020}
6021
6022impl From<TableId<HostTask>> for CurrentThread {
6023 fn from(id: TableId<HostTask>) -> Self {
6024 Self::Host(id)
6025 }
6026}
6027
6028enum Priority {
6029 Switch,
6030 High,
6031 Low,
6032}
6033
6034pub struct ConcurrentState {
6036 unforced_current_thread: CurrentThread,
6042
6043 deferred_host_call_context: Option<CallContext>,
6049
6050 futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6055 table: AlwaysMut<ResourceTable>,
6057 switch_item: Option<WorkItem>,
6065 next_switch_item: Option<WorkItem>,
6071 high_priority: VecDeque<WorkItem>,
6073 low_priority: VecDeque<WorkItem>,
6075 suspend_reason: Option<SuspendReason>,
6079 worker: Option<StoreFiber<'static>>,
6083 worker_item: Option<WorkerItem>,
6085
6086 global_error_context_ref_counts:
6099 BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6100
6101 interesting_tasks: usize,
6114
6115 interesting_tasks_empty_waker: Option<Waker>,
6119
6120 ready_for_concurrent_call_waker: Option<Waker>,
6125
6126 event_loop_running: bool,
6128
6129 saved_next_switch_items: Vec<Option<WorkItem>>,
6132
6133 #[cfg(feature = "task-group-hook")]
6135 task_group_hook: Option<Box<dyn TaskGroupHook>>,
6136}
6137
6138impl Default for ConcurrentState {
6139 fn default() -> Self {
6140 Self {
6141 unforced_current_thread: CurrentThread::None,
6142 deferred_host_call_context: None,
6143 table: AlwaysMut::new(ResourceTable::new()),
6144 futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6145 switch_item: None,
6146 next_switch_item: None,
6147 high_priority: VecDeque::new(),
6148 low_priority: VecDeque::new(),
6149 suspend_reason: None,
6150 worker: None,
6151 worker_item: None,
6152 global_error_context_ref_counts: BTreeMap::new(),
6153 interesting_tasks: 0,
6154 interesting_tasks_empty_waker: None,
6155 ready_for_concurrent_call_waker: None,
6156 event_loop_running: false,
6157 saved_next_switch_items: Vec::new(),
6158 #[cfg(feature = "task-group-hook")]
6159 task_group_hook: None,
6160 }
6161 }
6162}
6163
6164impl ConcurrentState {
6165 pub(crate) fn take_fibers_and_futures(
6182 &mut self,
6183 fibers: &mut Vec<StoreFiber<'static>>,
6184 futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6185 ) {
6186 let ConcurrentState {
6187 table,
6188 worker,
6189 switch_item,
6190 next_switch_item,
6191 high_priority,
6192 low_priority,
6193 saved_next_switch_items,
6194 futures: pending_futures,
6195 worker_item: _,
6196 unforced_current_thread: _,
6197 deferred_host_call_context: _,
6198 suspend_reason: _,
6199 global_error_context_ref_counts: _,
6200 interesting_tasks: _,
6201 interesting_tasks_empty_waker: _,
6202 ready_for_concurrent_call_waker: _,
6203 event_loop_running: _,
6204 #[cfg(feature = "task-group-hook")]
6205 task_group_hook: _,
6206 } = self;
6207
6208 let mut items = Vec::new();
6209 for (_, entry) in table.get_mut().iter_mut() {
6210 let entry = entry.into_entry_mut();
6211 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6212 for mode in mem::take(&mut set.waiting).into_values() {
6213 match mode {
6214 WaitMode::Fiber(fiber) => {
6215 fibers.push(fiber);
6216 }
6217 WaitMode::Callback(_) => {}
6218 }
6219 }
6220 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6221 match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6222 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6223 fibers.push(fiber);
6224 }
6225 GuestThreadState::NotStartedImplicit
6226 | GuestThreadState::NotStartedExplicit(_)
6227 | GuestThreadState::Running
6228 | GuestThreadState::Completed => {}
6229 }
6230 } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6231 if let Some(item) = item.take() {
6232 items.push(item);
6233 }
6234 }
6235 }
6236
6237 if let Some(fiber) = worker.take() {
6238 fibers.push(fiber);
6239 }
6240
6241 let mut handle_item = |item| match item {
6242 WorkItem::ResumeFiber { fiber, .. } => {
6243 fibers.push(fiber);
6244 }
6245 WorkItem::PushFuture(future) => {
6246 pending_futures
6247 .get_mut()
6248 .as_mut()
6249 .unwrap()
6250 .push(future.into_inner());
6251 }
6252 WorkItem::ResumeThread { .. }
6253 | WorkItem::GuestCall { .. }
6254 | WorkItem::WorkerFunction(_) => {}
6255 };
6256
6257 for item in items {
6258 handle_item(item);
6259 }
6260 if let Some(item) = switch_item.take() {
6261 handle_item(item);
6262 }
6263 if let Some(item) = next_switch_item.take() {
6264 handle_item(item);
6265 }
6266 for item in mem::take(high_priority) {
6267 handle_item(item);
6268 }
6269 for item in mem::take(low_priority) {
6270 handle_item(item);
6271 }
6272 for item in mem::take(saved_next_switch_items)
6273 .into_iter()
6274 .filter_map(|v| v)
6275 {
6276 handle_item(item);
6277 }
6278
6279 if let Some(them) = pending_futures.get_mut().take() {
6280 futures.push(them);
6281 }
6282 }
6283
6284 #[cfg(feature = "gc")]
6285 pub(crate) fn trace_fiber_roots(
6286 &mut self,
6287 modules: &ModuleRegistry,
6288 unwind: &dyn Unwind,
6289 gc_roots_list: &mut GcRootsList,
6290 ) {
6291 let ConcurrentState {
6292 table,
6293 worker,
6294 switch_item,
6295 next_switch_item,
6296 high_priority,
6297 low_priority,
6298 saved_next_switch_items,
6299
6300 futures: _,
6304
6305 worker_item: _,
6307 unforced_current_thread: _,
6308 deferred_host_call_context: _,
6309 suspend_reason: _,
6310 global_error_context_ref_counts: _,
6311 interesting_tasks: _,
6312 interesting_tasks_empty_waker: _,
6313 ready_for_concurrent_call_waker: _,
6314 event_loop_running: _,
6315 #[cfg(feature = "task-group-hook")]
6316 task_group_hook: _,
6317 } = self;
6318
6319 for (_, entry) in table.get_mut().iter_mut() {
6320 let entry = entry.into_entry_mut();
6321 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6322 for mode in set.waiting.values_mut() {
6323 match mode {
6324 WaitMode::Fiber(fiber) => {
6325 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6326 }
6327 WaitMode::Callback(_) => {}
6328 }
6329 }
6330 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6331 match &mut thread.state {
6332 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6333 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6334 }
6335 GuestThreadState::NotStartedImplicit
6336 | GuestThreadState::NotStartedExplicit(_)
6337 | GuestThreadState::Running
6338 | GuestThreadState::Completed => {}
6339 }
6340 } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6341 entry.downcast_mut::<Option<WorkItem>>()
6342 {
6343 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6344 }
6345 }
6346
6347 if let Some(fiber) = worker {
6348 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6349 }
6350
6351 let mut handle_item = |item: &mut WorkItem| match item {
6352 WorkItem::ResumeFiber { fiber, .. } => {
6353 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6354 }
6355 WorkItem::PushFuture(_future) => {
6356 }
6359 WorkItem::ResumeThread { .. }
6360 | WorkItem::GuestCall { .. }
6361 | WorkItem::WorkerFunction(_) => {}
6362 };
6363
6364 if let Some(item) = switch_item {
6365 handle_item(item);
6366 }
6367 if let Some(item) = next_switch_item {
6368 handle_item(item);
6369 }
6370 for item in high_priority {
6371 handle_item(item);
6372 }
6373 for item in low_priority {
6374 handle_item(item);
6375 }
6376 for item in saved_next_switch_items
6377 .iter_mut()
6378 .filter_map(|v| v.as_mut())
6379 {
6380 handle_item(item);
6381 }
6382 }
6383
6384 fn schedule_suspended_thread(
6385 &mut self,
6386 thread: QualifiedThreadId,
6387 instance: RuntimeInstance,
6388 priority: Priority,
6389 ) -> Result<()> {
6390 if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6392 bail_bug!("switch item already set");
6393 }
6394 let slot = &mut self.get_mut(thread.thread)?.state;
6395 if !matches!(slot, GuestThreadState::Suspended(_)) {
6396 bail!(Trap::CannotResumeThread);
6397 }
6398 let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6399 else {
6400 unreachable!()
6401 };
6402 let item = WorkItem::ResumeFiber {
6403 instance,
6404 thread,
6405 fiber,
6406 };
6407 match priority {
6408 Priority::Switch => self.switch_item = Some(item),
6409 Priority::High => self.push_high_priority(item),
6410 Priority::Low => self.push_low_priority(item),
6411 }
6412 Ok(())
6413 }
6414
6415 fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6416 let slot = &mut self.get_mut(thread)?.state;
6417 slot.check_no_fiber()?;
6418 *slot = GuestThreadState::Running;
6419 Ok(())
6420 }
6421
6422 fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6423 let saved = self.push(None::<WorkItem>)?;
6426 let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6427 *slot = self.next_switch_item.take();
6428 Ok(saved)
6429 }
6430
6431 fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6432 if self.next_switch_item.is_some() {
6433 bail_bug!("next switch item already set when restoring");
6434 }
6435 self.next_switch_item = self.delete(saved)?;
6436 Ok(())
6437 }
6438
6439 fn take_callback_waiter(
6440 &mut self,
6441 set: TableId<WaitableSet>,
6442 thread: QualifiedThreadId,
6443 ) -> Result<Instance> {
6444 let waiting = &mut self.get_mut(set)?.waiting;
6445 let instance = match waiting.get(&thread) {
6446 Some(WaitMode::Callback(instance)) => *instance,
6447 other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6448 };
6449 waiting.remove(&thread);
6450 Ok(instance)
6451 }
6452
6453 fn push<V: Send + Sync + 'static>(
6454 &mut self,
6455 value: V,
6456 ) -> Result<TableId<V>, ResourceTableError> {
6457 self.table.get_mut().push(value).map(TableId::from)
6458 }
6459
6460 fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6461 self.table.get_mut().get_mut(&Resource::from(id))
6462 }
6463
6464 pub fn add_child<T: 'static, U: 'static>(
6465 &mut self,
6466 child: TableId<T>,
6467 parent: TableId<U>,
6468 ) -> Result<(), ResourceTableError> {
6469 self.table
6470 .get_mut()
6471 .add_child(Resource::from(child), Resource::from(parent))
6472 }
6473
6474 pub fn remove_child<T: 'static, U: 'static>(
6475 &mut self,
6476 child: TableId<T>,
6477 parent: TableId<U>,
6478 ) -> Result<(), ResourceTableError> {
6479 self.table
6480 .get_mut()
6481 .remove_child(Resource::from(child), Resource::from(parent))
6482 }
6483
6484 fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6485 self.table.get_mut().delete(Resource::from(id))
6486 }
6487
6488 fn push_future(&mut self, future: HostTaskFuture) {
6489 self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6496 }
6497
6498 fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6499 if self.switch_item.is_some() {
6500 self.push_high_priority(item);
6503 bail_bug!("switch item already set");
6504 }
6505
6506 self.switch_item = Some(item);
6507 log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6508
6509 Ok(())
6510 }
6511
6512 fn take_next_switch_item(&mut self) -> Result<()> {
6513 if let Some(item) = self.next_switch_item.take() {
6514 self.set_switch_item(item)?;
6515 }
6516 Ok(())
6517 }
6518
6519 fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6522 let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6525 return Ok(());
6526 };
6527 let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6528 if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6529 bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6530 }
6531 let instance = self.get_mut(thread.task)?.instance;
6532 self.get_mut(thread.thread)?.wake_on_cancel.take();
6533 let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6534
6535 let item = match mode {
6536 WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6537 instance,
6538 thread,
6539 fiber,
6540 },
6541 WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6542 instance,
6543 call: GuestCall {
6544 thread,
6545 kind: GuestCallKind::DeliverEvent {
6546 instance: callback_instance,
6547 set: Some(set),
6548 },
6549 },
6550 },
6551 };
6552 self.push_high_priority(item);
6553 Ok(())
6554 }
6555
6556 fn push_high_priority(&mut self, item: WorkItem) {
6557 self.high_priority.push_front(item);
6558 log::trace!(
6559 "push high priority: {:?}",
6560 self.high_priority.front().unwrap()
6561 );
6562 }
6563
6564 fn push_low_priority(&mut self, item: WorkItem) {
6565 self.low_priority.push_front(item);
6566 log::trace!(
6567 "push low priority: {:?}",
6568 self.low_priority.front().unwrap()
6569 );
6570 }
6571
6572 fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6573 match priority {
6574 Priority::Switch => self.set_switch_item(item)?,
6575 Priority::High => self.push_high_priority(item),
6576 Priority::Low => self.push_low_priority(item),
6577 }
6578
6579 Ok(())
6580 }
6581
6582 fn promote_instance_local_thread_work_item(
6583 &mut self,
6584 current_instance: RuntimeInstance,
6585 ) -> Result<bool> {
6586 log::trace!("promote thread work items for {current_instance:?}");
6587
6588 self.promote_work_item_matching(|item: &WorkItem| {
6589 let result = match item {
6590 WorkItem::ResumeThread { instance, .. }
6591 | WorkItem::ResumeFiber { instance, .. }
6592 | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6593 _ => false,
6594 };
6595
6596 log::trace!("candidate {item:?}: {result}");
6597 result
6598 })
6599 }
6600
6601 fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6602 self.promote_work_item_matching(|item: &WorkItem| match item {
6603 WorkItem::ResumeThread {
6604 thread: item_thread,
6605 ..
6606 }
6607 | WorkItem::GuestCall {
6608 call:
6609 GuestCall {
6610 thread: item_thread,
6611 ..
6612 },
6613 ..
6614 } => *item_thread == thread,
6615 _ => false,
6616 })
6617 }
6618
6619 fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6620 where
6621 F: FnMut(&WorkItem) -> bool,
6622 {
6623 if self.switch_item.is_none() {
6630 if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6631 self.switch_item = self.high_priority.remove(index);
6632 }
6633 }
6634 if self.switch_item.is_none() {
6635 if let Some(index) = self.low_priority.iter().rposition(predicate) {
6636 self.switch_item = self.low_priority.remove(index);
6637 }
6638 }
6639 Ok(self.switch_item.is_some())
6640 }
6641
6642 pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6645 match task.kind() {
6646 ScopeKind::HostId(task) => {
6647 let task: TableId<HostTask> = TableId::new(task);
6648 Ok(&mut self.get_mut(task)?.call_context)
6649 }
6650 ScopeKind::Id(task) => {
6651 let task: TableId<GuestTask> = TableId::new(task);
6652 Ok(&mut self.get_mut(task)?.call_context)
6653 }
6654 }
6655 }
6656
6657 pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6658 self.deferred_host_call_context.as_mut()
6659 }
6660
6661 fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6662 match self.futures.get_mut().as_mut() {
6663 Some(f) => Ok(f),
6664 None => bail_bug!("futures field of concurrent state is currently taken"),
6665 }
6666 }
6667
6668 pub(crate) fn table(&mut self) -> &mut ResourceTable {
6669 self.table.get_mut()
6670 }
6671
6672 fn debug_assert_deferred_host_invariant(&self) {
6673 debug_assert_eq!(
6674 self.deferred_host_call_context.is_some(),
6675 matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6676 "a deferred host thread and call context must exist together",
6677 );
6678 }
6679
6680 fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6681 self.debug_assert_deferred_host_invariant();
6682 let caller = match self.unforced_current_thread {
6683 CurrentThread::DeferredHost(caller) => caller,
6684 thread => return Ok(thread),
6685 };
6686
6687 let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6689 let task = self.push(task)?;
6690 let call_context = self
6691 .deferred_host_call_context
6692 .take()
6693 .expect("deferred host call context should be present");
6694 self.get_mut(task)
6695 .expect("newly inserted host task should be present")
6696 .call_context = call_context;
6697 self.unforced_current_thread = CurrentThread::Host(task);
6698 self.debug_assert_deferred_host_invariant();
6699 log::trace!("new host task materialized {task:?}");
6700 Ok(CurrentThread::Host(task))
6701 }
6702
6703 fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6704 match self.materialize_host_task()? {
6705 CurrentThread::Host(id) => Ok(Some(id)),
6706 CurrentThread::None => Ok(None),
6707 CurrentThread::Guest(_) => {
6708 bail_bug!("tried to materialize a host task id from a guest thread")
6709 }
6710 CurrentThread::DeferredHost(_) => {
6711 bail_bug!(
6712 "current thread is a deferred host thread which should have been materialized"
6713 )
6714 }
6715 }
6716 }
6717
6718 pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6719 match self.materialize_host_task()? {
6720 CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6721 _ => bail_bug!("current scope is not a deferred host scope"),
6722 }
6723 }
6724
6725 fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6726 let count = &mut self.get_mut(task)?.ref_count;
6727 *count += 1;
6728 log::trace!("increment {task:?} to {count}");
6729 Ok(())
6730 }
6731
6732 fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6733 let count = &mut self.get_mut(task)?.ref_count;
6734 if *count == 0 {
6735 bail_bug!("can't decrement task ref count that's already zero");
6736 }
6737 *count -= 1;
6738 log::trace!("decrement {task:?} to {count}");
6739 if *count == 0 {
6740 let task = self.delete(task)?;
6741
6742 self.decrement_group_ref_count(task.group)?;
6743
6744 debug_assert!(task.decremented_interesting_task_count);
6751 }
6752 Ok(())
6753 }
6754
6755 fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6756 log::trace!("delete host task {task:?}");
6757 let task = self.delete(task)?;
6758
6759 self.decrement_group_ref_count(task.group)
6760 }
6761}
6762
6763fn for_any_lower<
6766 F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6767>(
6768 fun: F,
6769) -> F {
6770 fun
6771}
6772
6773fn for_any_lift<
6775 F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6776>(
6777 fun: F,
6778) -> F {
6779 fun
6780}
6781
6782fn check_ambient_store(id: StoreId) {
6783 let message = "\
6784 `Future`s which depend on asynchronous component tasks, streams, or \
6785 futures to complete may only be polled from the event loop of the \
6786 store to which they belong. Please use \
6787 `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6788 ";
6789 tls::try_get(|store| {
6790 let matched = match store {
6791 tls::TryGet::Some(store) => store.id() == id,
6792 tls::TryGet::Taken | tls::TryGet::None => false,
6793 };
6794
6795 if !matched {
6796 panic!("{message}")
6797 }
6798 });
6799}
6800
6801fn unpack_callback_code(code: u32) -> (u32, u32) {
6802 (code & 0xF, code >> 4)
6803}
6804
6805struct WaitableCheckParams {
6809 set: TableId<WaitableSet>,
6810 options: OptionsIndex,
6811 payload: u32,
6812}
6813
6814enum WaitableCheck {
6817 Wait,
6818 Poll,
6819}
6820
6821pub(crate) struct PreparedCall<R> {
6823 handle: Func,
6825 thread: QualifiedThreadId,
6827 param_count: usize,
6829 rx: oneshot::Receiver<LiftedResult>,
6832 _phantom: PhantomData<R>,
6833}
6834
6835impl<R> PreparedCall<R> {
6836 pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6843 store
6844 .concurrent_state_mut()?
6845 .increment_task_ref_count(self.thread.task)?;
6846
6847 Ok(TaskId {
6848 task: self.thread.task,
6849 })
6850 }
6851}
6852
6853pub(crate) struct TaskId {
6855 task: TableId<GuestTask>,
6856}
6857
6858impl TaskId {
6859 pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
6871 let task = store.concurrent_state_mut()?.get_mut(self.task)?;
6872 if !task.already_lowered_parameters() {
6873 task.lower_params = Some(Box::new(|store, _| {
6878 store.set_trapped();
6879 bail!("the host future for this call was dropped before it started")
6880 }));
6881 } else {
6882 _ = store
6887 .concurrent_state_mut()?
6888 .decrement_task_ref_count(self.task);
6889 }
6890
6891 Ok(())
6892 }
6893}
6894
6895pub(crate) fn prepare_call<T, R>(
6901 mut store: StoreContextMut<T>,
6902 handle: Func,
6903 param_count: usize,
6904 lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
6905 + Send
6906 + Sync
6907 + 'static,
6908 lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
6909 + Send
6910 + Sync
6911 + 'static,
6912) -> Result<PreparedCall<R>> {
6913 if !store.0.may_enter() {
6917 bail!(Trap::CannotEnterComponent);
6918 }
6919
6920 let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
6921
6922 let instance = handle.instance().id().get(store.0);
6923 let options = &instance.component().env_component().options[options];
6924 let ty = &instance.component().types()[ty];
6925 let async_typed = ty.async_;
6926 let async_lifted = raw_options.async_;
6927 let task_return_type = ty.results;
6928 let component_instance = raw_options.instance;
6929 let callback = options.callback.map(|i| instance.runtime_callback(i));
6930 let memory = options
6931 .memory()
6932 .map(|i| instance.runtime_memory(i))
6933 .map(SendSyncPtr::new);
6934 let string_encoding = options.string_encoding;
6935 let token = StoreToken::new(store.as_context_mut());
6936 let caller = store.0.materialize_host_task_id()?;
6937 let state = store.0.concurrent_state_mut()?;
6938
6939 let (tx, rx) = oneshot::channel();
6940
6941 let instance = handle.instance().runtime_instance(component_instance);
6942 let thread = GuestTask::new(
6943 state,
6944 Box::new(for_any_lower(move |store, params| {
6945 lower_params(token.as_context_mut(store), params)
6946 })),
6947 LiftResult {
6948 lift: Box::new(for_any_lift(move |store, result| {
6949 lift_result(store, result)
6950 })),
6951 ty: task_return_type,
6952 memory,
6953 string_encoding,
6954 },
6955 Caller::Host {
6956 tx: Some(tx),
6957 caller,
6958 },
6959 callback.map(|callback| {
6960 let callback = SendSyncPtr::new(callback);
6961 let instance = handle.instance();
6962 Box::new(move |store: &mut dyn VMStore, event, handle| {
6963 let store = token.as_context_mut(store);
6964 unsafe { instance.call_callback(store, callback, event, handle) }
6967 }) as CallbackFn
6968 }),
6969 instance,
6970 async_typed,
6971 async_lifted,
6972 )?;
6973
6974 Ok(PreparedCall {
6975 handle,
6976 thread,
6977 param_count,
6978 rx,
6979 _phantom: PhantomData,
6980 })
6981}
6982
6983pub(crate) struct StagedCall<R> {
6984 store: StoreId,
6985 rx: oneshot::Receiver<LiftedResult>,
6986 _marker: PhantomData<fn() -> R>,
6987 group: TaskGroupId,
6995}
6996
6997impl<R> StagedCall<R> {
6998 pub(crate) fn new<T: 'static>(
7009 mut store: StoreContextMut<T>,
7010 prepared: PreparedCall<R>,
7011 ) -> Result<StagedCall<R>> {
7012 let PreparedCall {
7013 handle,
7014 thread,
7015 param_count,
7016 rx,
7017 ..
7018 } = prepared;
7019
7020 stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7021
7022 let state = store.0.concurrent_state_mut()?;
7023 let group = state.get_mut(thread.task)?.group;
7024 state.decrement_task_ref_count(thread.task)?;
7025
7026 Ok(StagedCall {
7027 store: store.0.id(),
7028 rx,
7029 _marker: PhantomData,
7030 group,
7031 })
7032 }
7033}
7034
7035impl<R> Future for StagedCall<R>
7036where
7037 R: 'static,
7038{
7039 type Output = Result<R>;
7040
7041 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7042 check_ambient_store(self.store);
7043 Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7044 Ok(r) => match r.downcast() {
7045 Ok(r) => Ok(*r),
7046 Err(_) => bail_bug!("wrong type of value produced"),
7047 },
7048 Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7049 })
7050 }
7051}
7052
7053fn stage_call0<T: 'static>(
7056 store: StoreContextMut<T>,
7057 handle: Func,
7058 guest_thread: QualifiedThreadId,
7059 param_count: usize,
7060) -> Result<()> {
7061 let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7062 let is_concurrent = raw_options.async_;
7063 let callback = raw_options.callback;
7064 let instance = handle.instance();
7065 let callee = handle.lifted_core_func(store.0);
7066 let post_return = raw_options
7067 .post_return
7068 .map(|i| instance.id().get(store.0).runtime_post_return(i));
7069 let callback = callback.map(|i| {
7070 let instance = instance.id().get(store.0);
7071 SendSyncPtr::new(instance.runtime_callback(i))
7072 });
7073
7074 log::trace!("queueing call {guest_thread:?}");
7075
7076 unsafe {
7080 instance.stage_call(
7081 store,
7082 guest_thread,
7083 SendSyncPtr::new(callee),
7084 param_count,
7085 1,
7086 is_concurrent,
7087 callback,
7088 post_return.map(SendSyncPtr::new),
7089 true,
7090 )
7091 }
7092}
7093
7094#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7095mod tests {
7096 use super::*;
7097 use crate::component::{Component, Linker};
7098 use crate::store::AsStoreOpaque;
7099 use crate::{Config, Engine};
7100
7101 fn host_subtask(
7102 state: HostTaskState,
7103 event: Option<Event>,
7104 ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7105 let mut config = Config::new();
7106 config.wasm_component_model_async(true);
7107 let engine = Engine::new(&config)?;
7108 let component = Component::new(&engine, "(component)")?;
7109 let mut store = Store::new(&engine, ());
7110 let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7111 let store_opaque = store.as_store_opaque();
7112 let concurrent_state = store_opaque.concurrent_state_mut()?;
7113 let group = concurrent_state.make_task_group()?;
7116 let task = concurrent_state.push(HostTask {
7117 common: WaitableCommon::default(),
7118 call_context: CallContext::default(),
7119 state,
7120 group,
7121 })?;
7122 let handle = store_opaque
7123 .handle_table_insertion(
7124 instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7125 )
7126 .subtask_host(task.rep())?;
7127 let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7128 common.handle = Some(handle);
7129 common.event = event;
7130 Ok((store, instance, task, handle))
7131 }
7132
7133 #[test]
7134 fn host_subtask_drop_during_cancellation() -> Result<()> {
7135 for abort_completed in [false, true] {
7136 let (handle, future) = JoinHandle::run(future::pending::<()>());
7137 let mut future = pin!(future);
7138 let (mut store, instance, task, handle) =
7139 host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7140 let store = store.as_store_opaque();
7141 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7142 assert_eq!(
7143 instance.subtask_cancel(store, caller, true, handle)?,
7144 BLOCKED
7145 );
7146 if abort_completed {
7147 assert!(matches!(
7150 future
7151 .as_mut()
7152 .poll(&mut Context::from_waker(Waker::noop())),
7153 Poll::Ready(None),
7154 ));
7155 }
7156 for async_ in [false, true] {
7157 let err = instance
7158 .subtask_cancel(store, caller, async_, handle)
7159 .unwrap_err();
7160 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7161 }
7162 let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7163 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7164 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7165 }
7166 Ok(())
7167 }
7168
7169 #[test]
7170 fn host_subtask_cancel_after_completion() -> Result<()> {
7171 for async_ in [false, true] {
7172 let (mut store, instance, task, handle) = host_subtask(
7173 HostTaskState::CalleeDone { cancelled: false },
7174 Some(Event::Subtask {
7175 status: Status::Returned,
7176 }),
7177 )?;
7178 let store = store.as_store_opaque();
7179 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7180 assert_eq!(
7181 instance.subtask_cancel(store, caller, async_, handle)?,
7182 Status::Returned as u32,
7183 );
7184 let err = instance
7185 .subtask_cancel(store, caller, async_, handle)
7186 .unwrap_err();
7187 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7188 instance.subtask_drop(store, caller, handle)?;
7189 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7190 }
7191 Ok(())
7192 }
7193
7194 #[test]
7195 fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7196 for (cancelled, status) in [
7197 (false, Status::Returned),
7198 (true, Status::Returned),
7199 (true, Status::ReturnCancelled),
7200 ] {
7201 for delivered in [false, true] {
7202 let event = if delivered {
7203 None
7204 } else {
7205 Some(Event::Subtask { status })
7206 };
7207 let (mut store, instance, task, handle) =
7208 host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7209 let store = store.as_store_opaque();
7210 let result = instance.subtask_drop(
7211 store,
7212 RuntimeComponentInstanceIndex::from_u32(0),
7213 handle,
7214 );
7215 if delivered {
7216 result?;
7217 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7218 } else {
7219 let err = result.unwrap_err();
7220 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7221 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7222 }
7223 }
7224 }
7225 Ok(())
7226 }
7227}