1use super::table::{TableDebug, TableId};
2use super::{Event, GlobalErrorContextRefCount, Waitable, WaitableCommon};
3use crate::component::concurrent::{ConcurrentState, CurrentThread, WorkItem, tls};
4use crate::component::func::{self, LiftContext, LowerContext};
5use crate::component::matching::InstanceType;
6use crate::component::resources::{HostResourceIndex, HostResourceTables};
7use crate::component::types;
8use crate::component::values::ErrorContextAny;
9use crate::component::{
10 AsAccessor, ComponentInstanceId, ComponentType, FutureAny, Instance, Lift, Lower,
11 RuntimeInstance, StreamAny, Val, WasmList,
12};
13use crate::prelude::*;
14use crate::store::{StoreOpaque, StoreToken};
15use crate::try_mutex::{TryMutex, TryMutexGuard};
16use crate::vm::VMTransmitLocalState;
17use crate::vm::component::TransmitLocalStateKind;
18use crate::vm::component::{ComponentInstance, HandleCounter, HandleTable, HandleTableInsertion};
19use crate::vm::{AlwaysMut, VMStore};
20use crate::{AsContext, AsContextMut, StoreContextMut, ValRaw};
21use crate::{
22 Error, Result, Trap, bail, bail_bug, ensure,
23 error::{Context as _, format_err},
24};
25use alloc::sync::Arc;
26use buffers::{Extender, SliceBuffer, UntypedWriteBuffer};
27use core::any::{Any, TypeId};
28use core::fmt;
29use core::future;
30use core::iter;
31use core::marker::PhantomData;
32use core::mem::{self, ManuallyDrop, MaybeUninit};
33use core::ops::{Deref, DerefMut};
34use core::pin::Pin;
35use core::task::{Context, Poll, Waker, ready};
36use futures::channel::oneshot;
37use futures::{FutureExt as _, stream};
38use wasmtime_environ::component::{
39 CanonicalAbiInfo, ComponentTypes, InterfaceType, OptionsIndex, RuntimeComponentInstanceIndex,
40 TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
41 TypeFutureTableIndex, TypeStreamTableIndex,
42};
43
44pub use buffers::{ReadBuffer, VecBuffer, WriteBuffer};
45
46mod buffers;
47
48#[derive(Copy, Clone, Debug)]
51pub enum TransmitKind {
52 Stream,
53 Future,
54}
55
56#[derive(Copy, Clone, Debug, PartialEq)]
58pub enum ReturnCode {
59 Blocked,
60 Completed(ItemCount),
61 Dropped(ItemCount),
62 Cancelled(ItemCount),
63}
64
65impl ReturnCode {
66 pub fn encode(&self) -> u32 {
71 const BLOCKED: u32 = 0xffff_ffff;
72 const COMPLETED: u32 = 0x0;
73 const DROPPED: u32 = 0x1;
74 const CANCELLED: u32 = 0x2;
75 match self {
76 ReturnCode::Blocked => BLOCKED,
77 ReturnCode::Completed(n) => (n.as_u32() << 4) | COMPLETED,
78 ReturnCode::Dropped(n) => (n.as_u32() << 4) | DROPPED,
79 ReturnCode::Cancelled(n) => (n.as_u32() << 4) | CANCELLED,
80 }
81 }
82
83 fn completed(kind: TransmitKind, count: ItemCount) -> Self {
86 Self::Completed(if let TransmitKind::Future = kind {
87 ItemCount::ZERO
88 } else {
89 count
90 })
91 }
92}
93
94#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
101#[repr(transparent)]
102pub struct ItemCount {
103 raw: u32,
104}
105
106impl ItemCount {
107 const MAX: u32 = 1 << 28;
108 const ZERO: ItemCount = ItemCount { raw: 0 };
109
110 fn new(count: u32) -> Result<Self, Trap> {
113 if count < Self::MAX {
114 Ok(Self { raw: count })
115 } else {
116 Err(Trap::StreamOpTooBig)
117 }
118 }
119
120 fn new_usize(count: usize) -> Result<Self, Trap> {
122 let count = u32::try_from(count).map_err(|_| Trap::StreamOpTooBig)?;
123 Self::new(count)
124 }
125
126 fn as_u32(&self) -> u32 {
127 self.raw
128 }
129
130 fn as_usize(&self) -> usize {
131 usize::try_from(self.raw).unwrap()
132 }
133
134 fn inc(&mut self, amt: usize) -> Result<(), Trap> {
137 let amt = u32::try_from(amt).map_err(|_| Trap::StreamOpTooBig)?;
138 let new_raw = self.raw.checked_add(amt).ok_or(Trap::StreamOpTooBig)?;
139 if new_raw < Self::MAX {
140 self.raw = new_raw;
141 Ok(())
142 } else {
143 Err(Trap::StreamOpTooBig)
144 }
145 }
146
147 fn add(&self, other: ItemCount) -> Result<ItemCount> {
153 match self.raw.checked_add(other.raw) {
154 Some(raw) => Ok(ItemCount::new(raw)?),
155 None => bail_bug!("overflow in `ItemCount::add`"),
156 }
157 }
158
159 fn sub(&self, other: ItemCount) -> Result<ItemCount> {
164 match self.raw.checked_sub(other.raw) {
165 Some(raw) => Ok(ItemCount { raw }),
166 None => bail_bug!("underflow in `ItemCount::sub`"),
167 }
168 }
169}
170
171impl fmt::Display for ItemCount {
172 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
173 self.raw.fmt(f)
174 }
175}
176
177impl fmt::Debug for ItemCount {
178 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
179 self.raw.fmt(f)
180 }
181}
182
183impl PartialEq<u32> for ItemCount {
184 fn eq(&self, other: &u32) -> bool {
185 self.raw == *other
186 }
187}
188
189impl PartialOrd<u32> for ItemCount {
190 fn partial_cmp(&self, other: &u32) -> Option<core::cmp::Ordering> {
191 self.raw.partial_cmp(other)
192 }
193}
194
195#[derive(Copy, Clone, Debug)]
200pub enum TransmitIndex {
201 Stream(TypeStreamTableIndex),
202 Future(TypeFutureTableIndex),
203}
204
205impl TransmitIndex {
206 pub fn kind(&self) -> TransmitKind {
207 match self {
208 TransmitIndex::Stream(_) => TransmitKind::Stream,
209 TransmitIndex::Future(_) => TransmitKind::Future,
210 }
211 }
212
213 fn payload<'a>(&self, types: &'a ComponentTypes) -> Option<&'a InterfaceType> {
216 match self {
217 TransmitIndex::Stream(i) => {
218 let ty = types[*i].ty;
219 types[ty].payload.as_ref()
220 }
221 TransmitIndex::Future(i) => {
222 let ty = types[*i].ty;
223 types[ty].payload.as_ref()
224 }
225 }
226 }
227}
228
229fn get_mut_by_index_from(
232 handle_table: &mut HandleTable,
233 ty: TransmitIndex,
234 index: u32,
235) -> Result<(u32, &mut VMTransmitLocalState)> {
236 match ty {
237 TransmitIndex::Stream(ty) => handle_table.stream_rep(ty, index),
238 TransmitIndex::Future(ty) => handle_table.future_rep(ty, index),
239 }
240}
241
242fn lower<T: func::Lower + Send + 'static, B: WriteBuffer<T>, U: 'static>(
250 mut store: StoreContextMut<U>,
251 instance: Instance,
252 options: OptionsIndex,
253 ty: TransmitIndex,
254 address: usize,
255 count: usize,
256 buffer: &mut B,
257) -> Result<()> {
258 let count = buffer.remaining().len().min(count);
259
260 let lower = &mut if T::MAY_REQUIRE_REALLOC {
261 LowerContext::new(store.as_context_mut(), options, instance)
262 } else {
263 LowerContext::new_without_realloc(store.as_context_mut(), options, instance)
264 };
265
266 if let Some(ty) = ty.payload(lower.types).copied() {
268 if address % usize::try_from(T::ALIGN32)? != 0 {
269 bail!("read pointer not aligned");
270 }
271 lower
272 .as_slice()
273 .get(address..)
274 .and_then(|b| b.get(..T::SIZE32 * count))
275 .ok_or_else(|| crate::format_err!("read pointer out of bounds of memory"))?;
276
277 T::linear_store_list_to_memory(lower, ty, address, &buffer.remaining()[..count])?;
278 }
279
280 buffer.skip(count);
281
282 Ok(())
283}
284
285fn lift<T: func::Lift + Send + 'static, B: ReadBuffer<T>>(
286 lift: &mut LiftContext<'_>,
287 ty: Option<InterfaceType>,
288 buffer: &mut B,
289 address: usize,
290 count: usize,
291) -> Result<()> {
292 let count = count.min(buffer.remaining_capacity());
293 if T::IS_RUST_UNIT_TYPE {
294 buffer.extend(
298 iter::repeat_with(|| unsafe { MaybeUninit::uninit().assume_init() }).take(count),
299 )
300 } else {
301 let ty = match ty {
302 Some(ty) => ty,
303 None => bail_bug!("type required for non-unit lift"),
304 };
305 if address % usize::try_from(T::ALIGN32)? != 0 {
306 bail!("write pointer not aligned");
307 }
308 lift.memory()
309 .get(address..)
310 .and_then(|b| b.get(..T::SIZE32 * count))
311 .ok_or_else(|| crate::format_err!("write pointer out of bounds of memory"))?;
312
313 let list = &WasmList::new(address, count, lift, ty)?;
314 T::linear_lift_into_from_memory(lift, list, &mut Extender(buffer))?
315 }
316 Ok(())
317}
318
319#[derive(Debug, PartialEq, Eq, PartialOrd)]
321pub(super) struct ErrorContextState {
322 pub(crate) debug_msg: String,
324}
325
326#[derive(Debug, Clone, Copy, PartialEq, Eq)]
329pub(super) struct FlatAbi {
330 pub(super) size: u32,
331 pub(super) align: u32,
332}
333
334struct HostBuffer<'a> {
335 dst: &'a mut Vec<u8>,
336 marked_written: &'a mut usize,
337}
338
339impl HostBuffer<'_> {
340 fn reborrow(&mut self) -> HostBuffer<'_> {
341 HostBuffer {
342 dst: &mut *self.dst,
343 marked_written: &mut *self.marked_written,
344 }
345 }
346}
347
348pub struct Destination<'a, T, B> {
350 id: TableId<TransmitState>,
351 buffer: &'a mut B,
352 host_buffer: Option<HostBuffer<'a>>,
353 _phantom: PhantomData<fn() -> T>,
354}
355
356impl<'a, T, B> Destination<'a, T, B> {
357 pub fn reborrow(&mut self) -> Destination<'_, T, B> {
359 Destination {
360 id: self.id,
361 buffer: &mut *self.buffer,
362 host_buffer: self.host_buffer.as_mut().map(|b| b.reborrow()),
363 _phantom: PhantomData,
364 }
365 }
366
367 pub fn take_buffer(&mut self) -> B
373 where
374 B: Default,
375 {
376 mem::take(self.buffer)
377 }
378
379 pub fn set_buffer(&mut self, buffer: B) {
389 *self.buffer = buffer;
390 }
391
392 pub fn remaining(&self, mut store: impl AsContextMut) -> Result<Option<usize>> {
409 self.remaining_(store.as_context_mut().0)
410 }
411
412 fn remaining_(&self, store: &mut StoreOpaque) -> Result<Option<usize>> {
413 let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;
414
415 if let &ReadState::GuestReady { count, .. } = &transmit.read {
416 let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
417 bail_bug!("expected WriteState::HostReady")
418 };
419
420 Ok(Some(count.as_usize() - guest_offset.as_usize()))
421 } else {
422 Ok(None)
423 }
424 }
425}
426
427impl<'a, B> Destination<'a, u8, B> {
428 pub fn as_direct<D>(
439 mut self,
440 store: StoreContextMut<'a, D>,
441 capacity: usize,
442 ) -> DirectDestination<'a, D> {
443 if let Some(buffer) = &mut self.host_buffer {
444 *buffer.marked_written = 0;
445 buffer.dst.resize(capacity, 0);
446 }
447
448 DirectDestination {
449 id: self.id,
450 host_buffer: self.host_buffer,
451 store,
452 }
453 }
454}
455
456pub struct DirectDestination<'a, D: 'static> {
459 id: TableId<TransmitState>,
460 host_buffer: Option<HostBuffer<'a>>,
461 store: StoreContextMut<'a, D>,
462}
463
464#[cfg(feature = "std")]
465impl<D: 'static> std::io::Write for DirectDestination<'_, D> {
466 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
467 let rem = self.remaining().map_err(std::io::Error::other)?;
468 let n = rem.len().min(buf.len());
469 rem[..n].copy_from_slice(&buf[..n]);
470 self.mark_written(n);
471 Ok(n)
472 }
473
474 fn flush(&mut self) -> std::io::Result<()> {
475 Ok(())
476 }
477}
478
479impl<D: 'static> DirectDestination<'_, D> {
480 pub fn remaining(&mut self) -> Result<&mut [u8]> {
482 if let Some(buffer) = self.host_buffer.as_mut() {
483 return Ok(&mut buffer.dst[*buffer.marked_written..]);
484 }
485 let transmit = self
486 .store
487 .as_context_mut()
488 .0
489 .concurrent_state_mut()?
490 .get_mut(self.id)?;
491
492 let &ReadState::GuestReady {
493 address,
494 count,
495 options,
496 instance,
497 ..
498 } = &transmit.read
499 else {
500 bail_bug!("expected ReadState::GuestReady")
501 };
502
503 let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
504 bail_bug!("expected WriteState::HostReady")
505 };
506
507 let memory = instance.options_memory_range_mut(
508 self.store.0,
509 options,
510 address + guest_offset.as_usize(),
511 count.as_usize() - guest_offset.as_usize(),
512 );
513 match memory {
514 Some(memory) => Ok(memory),
515 None => bail_bug!("guest buffer unexpectedly out of bounds"),
516 }
517 }
518
519 pub fn mark_written(&mut self, count: usize) {
526 self.mark_written_(count).unwrap()
530 }
531
532 fn mark_written_(&mut self, count: usize) -> Result<()> {
533 if let Some(buffer) = self.host_buffer.as_mut() {
534 *buffer.marked_written = buffer.marked_written.checked_add(count).unwrap();
537 } else {
538 let transmit = self
539 .store
540 .as_context_mut()
541 .0
542 .concurrent_state_mut()?
543 .get_mut(self.id)?;
544
545 let ReadState::GuestReady {
546 count: read_count, ..
547 } = &transmit.read
548 else {
549 bail_bug!("expected ReadState::GuestReady")
550 };
551
552 let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
553 bail_bug!("expected WriteState::HostReady");
554 };
555
556 if guest_offset.as_usize() + count > read_count.as_usize() {
557 panic!(
560 "write count ({count}) must be less than or equal to read count ({read_count})"
561 )
562 } else {
563 guest_offset.inc(count)?;
564 }
565 }
566 Ok(())
567 }
568}
569
570#[derive(Copy, Clone, Debug)]
572pub enum StreamResult {
573 Completed,
576 Cancelled,
581 Dropped,
584}
585
586pub trait StreamProducer<D>: Send + 'static {
588 type Item;
590
591 type Buffer: WriteBuffer<Self::Item> + Default;
593
594 fn poll_produce<'a>(
733 self: Pin<&mut Self>,
734 cx: &mut Context<'_>,
735 store: StoreContextMut<'a, D>,
736 destination: Destination<'a, Self::Item, Self::Buffer>,
737 finish: bool,
738 ) -> Poll<Result<StreamResult>>;
739
740 fn try_into(
752 me: Pin<Box<Self>>,
753 buffer: Self::Buffer,
754 _ty: TypeId,
755 ) -> Result<Box<dyn Any>, (Pin<Box<Self>>, Self::Buffer)> {
756 Err((me, buffer))
757 }
758}
759
760impl<T, D> StreamProducer<D> for iter::Empty<T>
761where
762 T: Send + Sync + 'static,
763{
764 type Item = T;
765 type Buffer = Option<Self::Item>;
766
767 fn poll_produce<'a>(
768 self: Pin<&mut Self>,
769 _: &mut Context<'_>,
770 _: StoreContextMut<'a, D>,
771 _: Destination<'a, Self::Item, Self::Buffer>,
772 _: bool,
773 ) -> Poll<Result<StreamResult>> {
774 Poll::Ready(Ok(StreamResult::Dropped))
775 }
776}
777
778impl<T, D> StreamProducer<D> for stream::Empty<T>
779where
780 T: Send + Sync + 'static,
781{
782 type Item = T;
783 type Buffer = Option<Self::Item>;
784
785 fn poll_produce<'a>(
786 self: Pin<&mut Self>,
787 _: &mut Context<'_>,
788 _: StoreContextMut<'a, D>,
789 _: Destination<'a, Self::Item, Self::Buffer>,
790 _: bool,
791 ) -> Poll<Result<StreamResult>> {
792 Poll::Ready(Ok(StreamResult::Dropped))
793 }
794}
795
796impl<T, D> StreamProducer<D> for Vec<T>
797where
798 T: Unpin + Send + Sync + 'static,
799{
800 type Item = T;
801 type Buffer = VecBuffer<T>;
802
803 fn poll_produce<'a>(
804 self: Pin<&mut Self>,
805 _: &mut Context<'_>,
806 _: StoreContextMut<'a, D>,
807 mut dst: Destination<'a, Self::Item, Self::Buffer>,
808 _: bool,
809 ) -> Poll<Result<StreamResult>> {
810 dst.set_buffer(mem::take(self.get_mut()).into());
811 Poll::Ready(Ok(StreamResult::Dropped))
812 }
813}
814
815impl<T, D> StreamProducer<D> for Box<[T]>
816where
817 T: Unpin + Send + Sync + 'static,
818{
819 type Item = T;
820 type Buffer = VecBuffer<T>;
821
822 fn poll_produce<'a>(
823 self: Pin<&mut Self>,
824 _: &mut Context<'_>,
825 _: StoreContextMut<'a, D>,
826 mut dst: Destination<'a, Self::Item, Self::Buffer>,
827 _: bool,
828 ) -> Poll<Result<StreamResult>> {
829 dst.set_buffer(mem::take(self.get_mut()).into_vec().into());
830 Poll::Ready(Ok(StreamResult::Dropped))
831 }
832}
833
834#[cfg(feature = "component-model-bytes")]
835impl<D> StreamProducer<D> for bytes::Bytes {
836 type Item = u8;
837 type Buffer = Self;
838
839 fn poll_produce<'a>(
840 self: Pin<&mut Self>,
841 _: &mut Context<'_>,
842 _store: StoreContextMut<'a, D>,
843 mut dst: Destination<'a, Self::Item, Self::Buffer>,
844 _: bool,
845 ) -> Poll<Result<StreamResult>> {
846 dst.set_buffer(mem::take(self.get_mut()));
847 Poll::Ready(Ok(StreamResult::Dropped))
848 }
849}
850
851#[cfg(feature = "component-model-bytes")]
852impl<D> StreamProducer<D> for bytes::BytesMut {
853 type Item = u8;
854 type Buffer = Self;
855
856 fn poll_produce<'a>(
857 self: Pin<&mut Self>,
858 _: &mut Context<'_>,
859 _store: StoreContextMut<'a, D>,
860 mut dst: Destination<'a, Self::Item, Self::Buffer>,
861 _: bool,
862 ) -> Poll<Result<StreamResult>> {
863 dst.set_buffer(mem::take(self.get_mut()));
864 Poll::Ready(Ok(StreamResult::Dropped))
865 }
866}
867
868pub struct Source<'a, T> {
870 id: TableId<TransmitState>,
871 host_buffer: Option<&'a mut dyn WriteBuffer<T>>,
872}
873
874impl<'a, T> Source<'a, T> {
875 pub fn reborrow(&mut self) -> Source<'_, T> {
877 Source {
878 id: self.id,
879 host_buffer: self.host_buffer.as_deref_mut(),
880 }
881 }
882
883 pub fn read<B, S: AsContextMut>(&mut self, mut store: S, buffer: &mut B) -> Result<()>
885 where
886 T: func::Lift + 'static,
887 B: ReadBuffer<T>,
888 {
889 if let Some(input) = &mut self.host_buffer {
890 let count = input.remaining().len().min(buffer.remaining_capacity());
891 buffer.move_from(*input, count);
892 } else {
893 let store = store.as_context_mut();
894 let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;
895
896 let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
897 bail_bug!("expected ReadState::HostReady");
898 };
899
900 let &WriteState::GuestReady {
901 ty,
902 address,
903 count,
904 options,
905 instance,
906 ..
907 } = &transmit.write
908 else {
909 bail_bug!("expected WriteState::GuestReady");
910 };
911
912 let cx = &mut LiftContext::new(store.0.store_opaque_mut(), options, instance)?;
913 let ty = ty.payload(cx.types);
914 let old_remaining = buffer.remaining_capacity();
915 lift::<T, B>(
916 cx,
917 ty.copied(),
918 buffer,
919 address + (T::SIZE32 * guest_offset.as_usize()),
920 count.as_usize() - guest_offset.as_usize(),
921 )?;
922
923 let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;
924
925 let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
926 bail_bug!("expected ReadState::HostReady");
927 };
928
929 guest_offset.inc(old_remaining - buffer.remaining_capacity())?;
930 }
931
932 Ok(())
933 }
934
935 pub fn remaining(&self, mut store: impl AsContextMut) -> Result<usize>
938 where
939 T: 'static,
940 {
941 self.remaining_(store.as_context_mut().0)
942 }
943
944 fn remaining_(&self, store: &mut StoreOpaque) -> Result<usize>
945 where
946 T: 'static,
947 {
948 let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;
949
950 if let &WriteState::GuestReady { count, .. } = &transmit.write {
951 let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
952 bail_bug!("expected ReadState::HostReady")
953 };
954
955 Ok(count.as_usize() - guest_offset.as_usize())
956 } else if let Some(host_buffer) = &self.host_buffer {
957 Ok(host_buffer.remaining().len())
958 } else {
959 bail_bug!("expected either WriteState::GuestReady or host buffer")
960 }
961 }
962}
963
964impl<'a> Source<'a, u8> {
965 pub fn as_direct<D>(self, store: StoreContextMut<'a, D>) -> DirectSource<'a, D> {
967 DirectSource {
968 id: self.id,
969 host_buffer: self.host_buffer,
970 store,
971 }
972 }
973}
974
975pub struct DirectSource<'a, D: 'static> {
978 id: TableId<TransmitState>,
979 host_buffer: Option<&'a mut dyn WriteBuffer<u8>>,
980 store: StoreContextMut<'a, D>,
981}
982
983#[cfg(feature = "std")]
984impl<D: 'static> std::io::Read for DirectSource<'_, D> {
985 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
986 let rem = self.remaining().map_err(std::io::Error::other)?;
987 let n = rem.len().min(buf.len());
988 buf[..n].copy_from_slice(&rem[..n]);
989 self.mark_read(n);
990 Ok(n)
991 }
992}
993
994impl<D: 'static> DirectSource<'_, D> {
995 pub fn remaining(&mut self) -> Result<&[u8]> {
997 if let Some(buffer) = self.host_buffer.as_deref_mut() {
998 return Ok(buffer.remaining());
999 }
1000 let transmit = self
1001 .store
1002 .as_context_mut()
1003 .0
1004 .concurrent_state_mut()?
1005 .get_mut(self.id)?;
1006
1007 let &WriteState::GuestReady {
1008 address,
1009 count,
1010 options,
1011 instance,
1012 ..
1013 } = &transmit.write
1014 else {
1015 bail_bug!("expected WriteState::GuestReady")
1016 };
1017
1018 let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
1019 bail_bug!("expected ReadState::HostReady")
1020 };
1021
1022 let memory = instance
1023 .options_memory(self.store.0, options)
1024 .get(address + guest_offset.as_usize()..)
1025 .and_then(|b| b.get(..(count.as_usize() - guest_offset.as_usize())));
1026 match memory {
1027 Some(memory) => Ok(memory),
1028 None => bail_bug!("guest buffer unexpectedly out of bounds"),
1029 }
1030 }
1031
1032 pub fn mark_read(&mut self, count: usize) {
1039 self.mark_read_(count).unwrap()
1043 }
1044
1045 fn mark_read_(&mut self, count: usize) -> Result<()> {
1046 if let Some(buffer) = self.host_buffer.as_deref_mut() {
1047 buffer.skip(count);
1048 return Ok(());
1049 }
1050
1051 let transmit = self
1052 .store
1053 .as_context_mut()
1054 .0
1055 .concurrent_state_mut()?
1056 .get_mut(self.id)?;
1057
1058 let WriteState::GuestReady {
1059 count: write_count, ..
1060 } = &transmit.write
1061 else {
1062 bail_bug!("expected WriteState::GuestReady");
1063 };
1064
1065 let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
1066 bail_bug!("expected ReadState::HostReady");
1067 };
1068
1069 if guest_offset.as_usize() + count > write_count.as_usize() {
1070 panic!("read count ({count}) must be less than or equal to write count ({write_count})")
1072 } else {
1073 guest_offset.inc(count)?;
1074 }
1075 Ok(())
1076 }
1077}
1078
1079pub trait StreamConsumer<D>: Send + 'static {
1081 type Item;
1083
1084 fn poll_consume(
1171 self: Pin<&mut Self>,
1172 cx: &mut Context<'_>,
1173 store: StoreContextMut<D>,
1174 source: Source<'_, Self::Item>,
1175 finish: bool,
1176 ) -> Poll<Result<StreamResult>>;
1177}
1178
1179pub trait FutureProducer<D>: Send + 'static {
1181 type Item;
1183
1184 fn poll_produce(
1194 self: Pin<&mut Self>,
1195 cx: &mut Context<'_>,
1196 store: StoreContextMut<D>,
1197 finish: bool,
1198 ) -> Poll<Result<Option<Self::Item>>>;
1199}
1200
1201impl<T, E, D, Fut> FutureProducer<D> for Fut
1202where
1203 E: Into<Error>,
1204 Fut: Future<Output = Result<T, E>> + ?Sized + Send + 'static,
1205{
1206 type Item = T;
1207
1208 fn poll_produce<'a>(
1209 self: Pin<&mut Self>,
1210 cx: &mut Context<'_>,
1211 _: StoreContextMut<'a, D>,
1212 finish: bool,
1213 ) -> Poll<Result<Option<T>>> {
1214 match self.poll(cx) {
1215 Poll::Ready(Ok(v)) => Poll::Ready(Ok(Some(v))),
1216 Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
1217 Poll::Pending if finish => Poll::Ready(Ok(None)),
1218 Poll::Pending => Poll::Pending,
1219 }
1220 }
1221}
1222
1223pub trait FutureConsumer<D>: Send + 'static {
1225 type Item;
1227
1228 fn poll_consume(
1240 self: Pin<&mut Self>,
1241 cx: &mut Context<'_>,
1242 store: StoreContextMut<D>,
1243 source: Source<'_, Self::Item>,
1244 finish: bool,
1245 ) -> Poll<Result<()>>;
1246}
1247
1248pub struct FutureReader<T> {
1255 idx: HostResourceIndex,
1256 producer_type_match_checked: bool,
1259 _phantom: PhantomData<T>,
1260}
1261
1262impl<T> FutureReader<T> {
1263 pub fn new<S: AsContextMut>(
1272 mut store: S,
1273 producer: impl FutureProducer<S::Data, Item = T>,
1274 ) -> Result<Self>
1275 where
1276 T: func::Lower + func::Lift + Send + Sync + 'static,
1277 {
1278 ensure!(
1279 store.as_context().0.concurrency_support(),
1280 "concurrency support is not enabled"
1281 );
1282
1283 struct Producer<P>(P);
1284
1285 impl<D, T: func::Lower + 'static, P: FutureProducer<D, Item = T>> StreamProducer<D>
1286 for Producer<P>
1287 {
1288 type Item = P::Item;
1289 type Buffer = Option<P::Item>;
1290
1291 fn poll_produce<'a>(
1292 self: Pin<&mut Self>,
1293 cx: &mut Context<'_>,
1294 store: StoreContextMut<D>,
1295 mut destination: Destination<'a, Self::Item, Self::Buffer>,
1296 finish: bool,
1297 ) -> Poll<Result<StreamResult>> {
1298 let producer = unsafe { self.map_unchecked_mut(|v| &mut v.0) };
1301
1302 Poll::Ready(Ok(
1303 if let Some(value) = ready!(producer.poll_produce(cx, store, finish))? {
1304 destination.set_buffer(Some(value));
1305
1306 StreamResult::Completed
1313 } else {
1314 StreamResult::Cancelled
1315 },
1316 ))
1317 }
1318 }
1319
1320 Ok(Self {
1321 idx: store
1322 .as_context_mut()
1323 .new_transmit(TransmitKind::Future, Producer(producer))?,
1324 producer_type_match_checked: true,
1325 _phantom: PhantomData,
1326 })
1327 }
1328
1329 pub(super) fn new_(idx: HostResourceIndex) -> Self {
1330 Self {
1331 idx,
1332 producer_type_match_checked: false,
1335 _phantom: PhantomData,
1336 }
1337 }
1338
1339 pub(super) fn idx(&self) -> HostResourceIndex {
1340 self.idx
1341 }
1342
1343 pub fn pipe<S: AsContextMut>(
1351 self,
1352 mut store: S,
1353 consumer: impl FutureConsumer<S::Data, Item = T> + Unpin,
1354 ) -> Result<()>
1355 where
1356 T: func::Lift + 'static,
1357 {
1358 struct Consumer<C>(C);
1359
1360 impl<D: 'static, T: func::Lift + 'static, C: FutureConsumer<D, Item = T>> StreamConsumer<D>
1361 for Consumer<C>
1362 {
1363 type Item = T;
1364
1365 fn poll_consume(
1366 self: Pin<&mut Self>,
1367 cx: &mut Context<'_>,
1368 mut store: StoreContextMut<D>,
1369 mut source: Source<Self::Item>,
1370 finish: bool,
1371 ) -> Poll<Result<StreamResult>> {
1372 let consumer = unsafe { self.map_unchecked_mut(|v| &mut v.0) };
1375
1376 ready!(consumer.poll_consume(
1377 cx,
1378 store.as_context_mut(),
1379 source.reborrow(),
1380 finish
1381 ))?;
1382
1383 Poll::Ready(Ok(if source.remaining(store)? == 0 {
1384 StreamResult::Completed
1390 } else {
1391 StreamResult::Cancelled
1392 }))
1393 }
1394 }
1395
1396 store
1397 .as_context_mut()
1398 .set_consumer(self.idx, TransmitKind::Future, Consumer(consumer))
1399 }
1400
1401 fn lift_from_index(cx: &mut LiftContext<'_>, ty: InterfaceType, index: u32) -> Result<Self> {
1403 Ok(Self::new_(lift_index_to_future(cx, ty, index)?))
1404 }
1405
1406 pub fn close(&mut self, mut store: impl AsContextMut) -> Result<()> {
1419 future_close(store.as_context_mut().0, self.idx)
1420 }
1421
1422 pub fn close_with(&mut self, accessor: impl AsAccessor) -> Result<()> {
1424 accessor.as_accessor().with(|access| self.close(access))
1425 }
1426
1427 pub fn guard<A>(self, accessor: A) -> GuardedFutureReader<T, A>
1433 where
1434 A: AsAccessor,
1435 {
1436 GuardedFutureReader::new(accessor, self)
1437 }
1438
1439 pub fn try_into_future_any(self, store: impl AsContextMut) -> Result<FutureAny>
1446 where
1447 T: ComponentType + 'static,
1448 {
1449 FutureAny::try_from_future_reader(store, self)
1450 }
1451
1452 pub fn try_from_future_any(future: FutureAny) -> Result<Self>
1459 where
1460 T: ComponentType + 'static,
1461 {
1462 future.try_into_future_reader()
1463 }
1464}
1465
1466impl<T> fmt::Debug for FutureReader<T> {
1467 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1468 f.debug_struct("FutureReader")
1469 .field("idx", &self.idx)
1470 .finish()
1471 }
1472}
1473
1474pub(super) fn future_close(store: &mut StoreOpaque, idx: HostResourceIndex) -> Result<()> {
1475 let id = store.host_reader_take(idx)?;
1476 store.host_drop_reader(id, TransmitKind::Future)
1477}
1478
1479pub(super) fn lift_index_to_future(
1481 cx: &mut LiftContext<'_>,
1482 ty: InterfaceType,
1483 index: u32,
1484) -> Result<HostResourceIndex> {
1485 match ty {
1486 InterfaceType::Future(src) => {
1487 let id = {
1488 let (state, instance) = cx.concurrent_state_and_instance_mut();
1489 lift_index_to_transmit(instance, state, TransmitIndex::Future(src), index)?
1490 };
1491 cx.host_resource_lower_own(id.rep(), None, None)
1492 }
1493 _ => func::bad_type_info(),
1494 }
1495}
1496
1497fn typecheck_lowered_transmit<U>(
1500 idx: HostResourceIndex,
1501 cx: &mut LowerContext<'_, U>,
1502 payload: Option<&InterfaceType>,
1503) -> Result<()> {
1504 let origin = cx.store.0.transmit_origin(idx)?;
1505 let dst = cx.instance_type();
1506 let equivalent = match origin {
1507 TransmitOrigin::Host { typecheck } => {
1508 return typecheck(payload, &dst).context("future or stream payload types differ");
1509 }
1510 TransmitOrigin::GuestFuture(id, ty) => {
1511 let src = InstanceType::new(cx.store.0.component_instance(id));
1512 types::FutureType::from(src.types[ty].ty, &src).equivalent_payload_guest(&dst, payload)
1513 }
1514 TransmitOrigin::GuestStream(id, ty) => {
1515 let src = InstanceType::new(cx.store.0.component_instance(id));
1516 types::StreamType::from(src.types[ty].ty, &src).equivalent_payload_guest(&dst, payload)
1517 }
1518 };
1519 if !equivalent {
1520 bail!("future or stream payload types differ");
1521 }
1522 Ok(())
1523}
1524
1525pub(super) fn lower_future_to_index<U>(
1529 idx: HostResourceIndex,
1530 cx: &mut LowerContext<'_, U>,
1531 ty: InterfaceType,
1532 item_already_typechecked: bool,
1533) -> Result<u32> {
1534 match ty {
1535 InterfaceType::Future(dst) => {
1536 if !item_already_typechecked {
1546 let types = cx.types;
1547 typecheck_lowered_transmit(idx, cx, types[types[dst].ty].payload.as_ref())?;
1548 }
1549 let id = cx.store.0.host_reader_take(idx)?;
1550 cx.instance_handle()
1551 .lower_transmit_to_index(cx.store.0, TransmitIndex::Future(dst), id)
1552 }
1553 _ => func::bad_type_info(),
1554 }
1555}
1556
1557unsafe impl<T: ComponentType> ComponentType for FutureReader<T> {
1560 const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1561
1562 type Lower = <u32 as func::ComponentType>::Lower;
1563
1564 fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1565 match ty {
1566 InterfaceType::Future(ty) => {
1567 let ty = types.types[*ty].ty;
1568 types::typecheck_payload::<T>(types.types[ty].payload.as_ref(), types)
1569 }
1570 other => bail!("expected `future`, found `{}`", func::desc(other)),
1571 }
1572 }
1573}
1574
1575unsafe impl<T: ComponentType> func::Lower for FutureReader<T> {
1577 fn linear_lower_to_flat<U>(
1578 &self,
1579 cx: &mut LowerContext<'_, U>,
1580 ty: InterfaceType,
1581 dst: &mut MaybeUninit<Self::Lower>,
1582 ) -> Result<()> {
1583 lower_future_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1584 .linear_lower_to_flat(cx, InterfaceType::U32, dst)
1585 }
1586
1587 fn linear_lower_to_memory<U>(
1588 &self,
1589 cx: &mut LowerContext<'_, U>,
1590 ty: InterfaceType,
1591 offset: usize,
1592 ) -> Result<()> {
1593 lower_future_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1594 .linear_lower_to_memory(cx, InterfaceType::U32, offset)
1595 }
1596}
1597
1598unsafe impl<T: ComponentType> func::Lift for FutureReader<T> {
1600 fn linear_lift_from_flat(
1601 cx: &mut LiftContext<'_>,
1602 ty: InterfaceType,
1603 src: &Self::Lower,
1604 ) -> Result<Self> {
1605 let index = u32::linear_lift_from_flat(cx, InterfaceType::U32, src)?;
1606 Self::lift_from_index(cx, ty, index)
1607 }
1608
1609 fn linear_lift_from_memory(
1610 cx: &mut LiftContext<'_>,
1611 ty: InterfaceType,
1612 bytes: &[u8],
1613 ) -> Result<Self> {
1614 let index = u32::linear_lift_from_memory(cx, InterfaceType::U32, bytes)?;
1615 Self::lift_from_index(cx, ty, index)
1616 }
1617}
1618
1619pub struct GuardedFutureReader<T, A>
1627where
1628 A: AsAccessor,
1629{
1630 reader: Option<FutureReader<T>>,
1634 accessor: A,
1635}
1636
1637impl<T, A> GuardedFutureReader<T, A>
1638where
1639 A: AsAccessor,
1640{
1641 pub fn new(accessor: A, reader: FutureReader<T>) -> Self {
1649 assert!(
1650 accessor
1651 .as_accessor()
1652 .with(|a| a.as_context().0.concurrency_support())
1653 );
1654 Self {
1655 reader: Some(reader),
1656 accessor,
1657 }
1658 }
1659
1660 pub fn into_future(self) -> FutureReader<T> {
1663 self.into()
1664 }
1665}
1666
1667impl<T, A> From<GuardedFutureReader<T, A>> for FutureReader<T>
1668where
1669 A: AsAccessor,
1670{
1671 fn from(mut guard: GuardedFutureReader<T, A>) -> Self {
1672 guard.reader.take().unwrap()
1673 }
1674}
1675
1676impl<T, A> Drop for GuardedFutureReader<T, A>
1677where
1678 A: AsAccessor,
1679{
1680 fn drop(&mut self) {
1681 if let Some(reader) = &mut self.reader {
1682 let result = reader.close_with(&self.accessor);
1688 debug_assert!(result.is_ok());
1689 }
1690 }
1691}
1692
1693pub struct StreamReader<T> {
1700 idx: HostResourceIndex,
1701 producer_type_match_checked: bool,
1704 _phantom: PhantomData<T>,
1705}
1706
1707impl<T> StreamReader<T> {
1708 pub fn new<S: AsContextMut>(
1717 mut store: S,
1718 producer: impl StreamProducer<S::Data, Item = T>,
1719 ) -> Result<Self>
1720 where
1721 T: func::Lower + func::Lift + Send + Sync + 'static,
1722 {
1723 ensure!(
1724 store.as_context().0.concurrency_support(),
1725 "concurrency support is not enabled",
1726 );
1727 Ok(Self {
1728 idx: store
1729 .as_context_mut()
1730 .new_transmit(TransmitKind::Stream, producer)?,
1731 producer_type_match_checked: true,
1732 _phantom: PhantomData,
1733 })
1734 }
1735
1736 pub(super) fn new_(idx: HostResourceIndex) -> Self {
1737 Self {
1738 idx,
1739 producer_type_match_checked: false,
1742 _phantom: PhantomData,
1743 }
1744 }
1745
1746 pub(super) fn idx(&self) -> HostResourceIndex {
1747 self.idx
1748 }
1749
1750 pub fn try_into<V: 'static>(mut self, mut store: impl AsContextMut) -> Result<V, Self> {
1772 let store = store.as_context_mut();
1773 let id = store.0.host_reader_rep(self.idx).unwrap();
1774 let state = store.0.concurrent_state_mut_already_forced_current_thread();
1775 let id = state.get_mut(id).unwrap().state;
1776 if let WriteState::HostReady { try_into, .. } = &state.get_mut(id).unwrap().write {
1777 match try_into(TypeId::of::<V>()) {
1778 Some(result) => {
1779 self.close(store).unwrap();
1780 Ok(*result.downcast::<V>().unwrap())
1781 }
1782 None => Err(self),
1783 }
1784 } else {
1785 Err(self)
1786 }
1787 }
1788
1789 pub fn pipe<S: AsContextMut>(
1797 self,
1798 mut store: S,
1799 consumer: impl StreamConsumer<S::Data, Item = T>,
1800 ) -> Result<()>
1801 where
1802 T: 'static,
1803 {
1804 store
1805 .as_context_mut()
1806 .set_consumer(self.idx, TransmitKind::Stream, consumer)
1807 }
1808
1809 fn lift_from_index(cx: &mut LiftContext<'_>, ty: InterfaceType, index: u32) -> Result<Self> {
1811 Ok(Self::new_(lift_index_to_stream(cx, ty, index)?))
1812 }
1813
1814 pub fn close(&mut self, mut store: impl AsContextMut) -> Result<()> {
1825 stream_close(store.as_context_mut().0, self.idx)
1826 }
1827
1828 pub fn close_with(&mut self, accessor: impl AsAccessor) -> Result<()> {
1830 accessor.as_accessor().with(|access| self.close(access))
1831 }
1832
1833 pub fn guard<A>(self, accessor: A) -> GuardedStreamReader<T, A>
1839 where
1840 A: AsAccessor,
1841 {
1842 GuardedStreamReader::new(accessor, self)
1843 }
1844
1845 pub fn try_into_stream_any(self, store: impl AsContextMut) -> Result<StreamAny>
1852 where
1853 T: ComponentType + 'static,
1854 {
1855 StreamAny::try_from_stream_reader(store, self)
1856 }
1857
1858 pub fn try_from_stream_any(stream: StreamAny) -> Result<Self>
1865 where
1866 T: ComponentType + 'static,
1867 {
1868 stream.try_into_stream_reader()
1869 }
1870}
1871
1872impl<T> fmt::Debug for StreamReader<T> {
1873 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1874 f.debug_struct("StreamReader")
1875 .field("idx", &self.idx)
1876 .finish()
1877 }
1878}
1879
1880pub(super) fn stream_close(store: &mut StoreOpaque, idx: HostResourceIndex) -> Result<()> {
1881 let id = store.host_reader_take(idx)?;
1882 store.host_drop_reader(id, TransmitKind::Stream)
1883}
1884
1885pub(super) fn lift_index_to_stream(
1887 cx: &mut LiftContext<'_>,
1888 ty: InterfaceType,
1889 index: u32,
1890) -> Result<HostResourceIndex> {
1891 match ty {
1892 InterfaceType::Stream(src) => {
1893 let id = {
1894 let (state, instance) = cx.concurrent_state_and_instance_mut();
1895 lift_index_to_transmit(instance, state, TransmitIndex::Stream(src), index)?
1896 };
1897 cx.host_resource_lower_own(id.rep(), None, None)
1898 }
1899 _ => func::bad_type_info(),
1900 }
1901}
1902
1903pub(super) fn lower_stream_to_index<U>(
1907 idx: HostResourceIndex,
1908 cx: &mut LowerContext<'_, U>,
1909 ty: InterfaceType,
1910 item_already_typechecked: bool,
1911) -> Result<u32> {
1912 match ty {
1913 InterfaceType::Stream(dst) => {
1914 if !item_already_typechecked {
1924 let types = cx.types;
1925 typecheck_lowered_transmit(idx, cx, types[types[dst].ty].payload.as_ref())?;
1926 }
1927 let id = cx.store.0.host_reader_take(idx)?;
1928 cx.instance_handle()
1929 .lower_transmit_to_index(cx.store.0, TransmitIndex::Stream(dst), id)
1930 }
1931 _ => func::bad_type_info(),
1932 }
1933}
1934
1935unsafe impl<T: ComponentType> ComponentType for StreamReader<T> {
1938 const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1939
1940 type Lower = <u32 as func::ComponentType>::Lower;
1941
1942 fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1943 match ty {
1944 InterfaceType::Stream(ty) => {
1945 let ty = types.types[*ty].ty;
1946 types::typecheck_payload::<T>(types.types[ty].payload.as_ref(), types)
1947 }
1948 other => bail!("expected `stream`, found `{}`", func::desc(other)),
1949 }
1950 }
1951}
1952
1953unsafe impl<T: ComponentType> func::Lower for StreamReader<T> {
1955 fn linear_lower_to_flat<U>(
1956 &self,
1957 cx: &mut LowerContext<'_, U>,
1958 ty: InterfaceType,
1959 dst: &mut MaybeUninit<Self::Lower>,
1960 ) -> Result<()> {
1961 lower_stream_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1962 .linear_lower_to_flat(cx, InterfaceType::U32, dst)
1963 }
1964
1965 fn linear_lower_to_memory<U>(
1966 &self,
1967 cx: &mut LowerContext<'_, U>,
1968 ty: InterfaceType,
1969 offset: usize,
1970 ) -> Result<()> {
1971 lower_stream_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1972 .linear_lower_to_memory(cx, InterfaceType::U32, offset)
1973 }
1974}
1975
1976unsafe impl<T: ComponentType> func::Lift for StreamReader<T> {
1978 fn linear_lift_from_flat(
1979 cx: &mut LiftContext<'_>,
1980 ty: InterfaceType,
1981 src: &Self::Lower,
1982 ) -> Result<Self> {
1983 let index = u32::linear_lift_from_flat(cx, InterfaceType::U32, src)?;
1984 Self::lift_from_index(cx, ty, index)
1985 }
1986
1987 fn linear_lift_from_memory(
1988 cx: &mut LiftContext<'_>,
1989 ty: InterfaceType,
1990 bytes: &[u8],
1991 ) -> Result<Self> {
1992 let index = u32::linear_lift_from_memory(cx, InterfaceType::U32, bytes)?;
1993 Self::lift_from_index(cx, ty, index)
1994 }
1995}
1996
1997pub struct GuardedStreamReader<T, A>
2005where
2006 A: AsAccessor,
2007{
2008 reader: Option<StreamReader<T>>,
2012 accessor: A,
2013}
2014
2015impl<T, A> GuardedStreamReader<T, A>
2016where
2017 A: AsAccessor,
2018{
2019 pub fn new(accessor: A, reader: StreamReader<T>) -> Self {
2028 assert!(
2029 accessor
2030 .as_accessor()
2031 .with(|a| a.as_context().0.concurrency_support())
2032 );
2033 Self {
2034 reader: Some(reader),
2035 accessor,
2036 }
2037 }
2038
2039 pub fn into_stream(self) -> StreamReader<T> {
2042 self.into()
2043 }
2044}
2045
2046impl<T, A> From<GuardedStreamReader<T, A>> for StreamReader<T>
2047where
2048 A: AsAccessor,
2049{
2050 fn from(mut guard: GuardedStreamReader<T, A>) -> Self {
2051 guard.reader.take().unwrap()
2052 }
2053}
2054
2055impl<T, A> Drop for GuardedStreamReader<T, A>
2056where
2057 A: AsAccessor,
2058{
2059 fn drop(&mut self) {
2060 if let Some(reader) = &mut self.reader {
2061 let result = reader.close_with(&self.accessor);
2063 debug_assert!(result.is_ok());
2064 }
2065 }
2066}
2067
2068pub struct ErrorContext {
2070 rep: u32,
2071}
2072
2073impl ErrorContext {
2074 pub(crate) fn new(rep: u32) -> Self {
2075 Self { rep }
2076 }
2077
2078 pub fn into_val(self) -> Val {
2080 Val::ErrorContext(ErrorContextAny(self.rep))
2081 }
2082
2083 pub fn from_val(_: impl AsContextMut, value: &Val) -> Result<Self> {
2085 let Val::ErrorContext(ErrorContextAny(rep)) = value else {
2086 bail!("expected `error-context`; got `{}`", value.desc());
2087 };
2088 Ok(Self::new(*rep))
2089 }
2090
2091 fn lift_from_index(cx: &mut LiftContext<'_>, ty: InterfaceType, index: u32) -> Result<Self> {
2092 match ty {
2093 InterfaceType::ErrorContext(src) => {
2094 let rep = cx
2095 .instance_mut()
2096 .table_for_error_context(src)
2097 .error_context_rep(index)?;
2098
2099 Ok(Self { rep })
2100 }
2101 _ => func::bad_type_info(),
2102 }
2103 }
2104}
2105
2106pub(crate) fn lower_error_context_to_index<U>(
2107 rep: u32,
2108 cx: &mut LowerContext<'_, U>,
2109 ty: InterfaceType,
2110) -> Result<u32> {
2111 match ty {
2112 InterfaceType::ErrorContext(dst) => {
2113 let (instance, counter) = cx
2114 .instance_handle()
2115 .id()
2116 .get_mut_and_handle_counter(cx.store.0);
2117 instance
2118 .error_context_table_insertion(dst, counter)
2119 .error_context(rep)
2120 }
2121 _ => func::bad_type_info(),
2122 }
2123}
2124unsafe impl func::ComponentType for ErrorContext {
2127 const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
2128
2129 type Lower = <u32 as func::ComponentType>::Lower;
2130
2131 fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
2132 match ty {
2133 InterfaceType::ErrorContext(_) => Ok(()),
2134 other => bail!("expected `error`, found `{}`", func::desc(other)),
2135 }
2136 }
2137}
2138
2139unsafe impl func::Lower for ErrorContext {
2141 fn linear_lower_to_flat<T>(
2142 &self,
2143 cx: &mut LowerContext<'_, T>,
2144 ty: InterfaceType,
2145 dst: &mut MaybeUninit<Self::Lower>,
2146 ) -> Result<()> {
2147 lower_error_context_to_index(self.rep, cx, ty)?.linear_lower_to_flat(
2148 cx,
2149 InterfaceType::U32,
2150 dst,
2151 )
2152 }
2153
2154 fn linear_lower_to_memory<T>(
2155 &self,
2156 cx: &mut LowerContext<'_, T>,
2157 ty: InterfaceType,
2158 offset: usize,
2159 ) -> Result<()> {
2160 lower_error_context_to_index(self.rep, cx, ty)?.linear_lower_to_memory(
2161 cx,
2162 InterfaceType::U32,
2163 offset,
2164 )
2165 }
2166}
2167
2168unsafe impl func::Lift for ErrorContext {
2170 fn linear_lift_from_flat(
2171 cx: &mut LiftContext<'_>,
2172 ty: InterfaceType,
2173 src: &Self::Lower,
2174 ) -> Result<Self> {
2175 let index = u32::linear_lift_from_flat(cx, InterfaceType::U32, src)?;
2176 Self::lift_from_index(cx, ty, index)
2177 }
2178
2179 fn linear_lift_from_memory(
2180 cx: &mut LiftContext<'_>,
2181 ty: InterfaceType,
2182 bytes: &[u8],
2183 ) -> Result<Self> {
2184 let index = u32::linear_lift_from_memory(cx, InterfaceType::U32, bytes)?;
2185 Self::lift_from_index(cx, ty, index)
2186 }
2187}
2188
2189pub(super) struct TransmitHandle {
2191 pub(super) common: WaitableCommon,
2192 state: TableId<TransmitState>,
2194}
2195
2196impl TransmitHandle {
2197 fn new(state: TableId<TransmitState>) -> Self {
2198 Self {
2199 common: WaitableCommon::default(),
2200 state,
2201 }
2202 }
2203}
2204
2205impl TableDebug for TransmitHandle {
2206 fn type_name() -> &'static str {
2207 "TransmitHandle"
2208 }
2209}
2210
2211struct TransmitState {
2213 write_handle: TableId<TransmitHandle>,
2215 read_handle: TableId<TransmitHandle>,
2217 write: WriteState,
2219 read: ReadState,
2221 done: bool,
2223 pub(super) origin: TransmitOrigin,
2227}
2228
2229#[derive(Copy, Clone)]
2230pub(super) enum TransmitOrigin {
2231 Host {
2232 typecheck: fn(Option<&InterfaceType>, &InstanceType<'_>) -> Result<()>,
2234 },
2235 GuestFuture(ComponentInstanceId, TypeFutureTableIndex),
2236 GuestStream(ComponentInstanceId, TypeStreamTableIndex),
2237}
2238
2239impl TransmitState {
2240 fn new(origin: TransmitOrigin) -> Self {
2241 Self {
2242 write_handle: TableId::new(u32::MAX),
2243 read_handle: TableId::new(u32::MAX),
2244 read: ReadState::Open,
2245 write: WriteState::Open,
2246 done: false,
2247 origin,
2248 }
2249 }
2250}
2251
2252impl TableDebug for TransmitState {
2253 fn type_name() -> &'static str {
2254 "TransmitState"
2255 }
2256}
2257
2258impl TransmitOrigin {
2259 fn guest(id: ComponentInstanceId, index: TransmitIndex) -> Self {
2260 match index {
2261 TransmitIndex::Future(ty) => TransmitOrigin::GuestFuture(id, ty),
2262 TransmitIndex::Stream(ty) => TransmitOrigin::GuestStream(id, ty),
2263 }
2264 }
2265}
2266
2267type PollStream = Box<
2268 dyn Fn() -> Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>> + Send + Sync,
2269>;
2270
2271type TryInto = Box<dyn Fn(TypeId) -> Option<Box<dyn Any>> + Send + Sync>;
2272
2273enum WriteState {
2275 Open,
2277 GuestReady {
2279 instance: Instance,
2280 caller: RuntimeComponentInstanceIndex,
2281 ty: TransmitIndex,
2282 flat_abi: Option<FlatAbi>,
2283 options: OptionsIndex,
2284 address: usize,
2285 count: ItemCount,
2286 handle: u32,
2287 },
2288 HostReady {
2290 produce: PollStream,
2291 try_into: TryInto,
2292 guest_offset: ItemCount,
2293 cancel: bool,
2294 cancel_waker: Option<Waker>,
2295 },
2296 Dropped,
2298}
2299
2300impl fmt::Debug for WriteState {
2301 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2302 match self {
2303 Self::Open => f.debug_tuple("Open").finish(),
2304 Self::GuestReady { .. } => f.debug_tuple("GuestReady").finish(),
2305 Self::HostReady { .. } => f.debug_tuple("HostReady").finish(),
2306 Self::Dropped => f.debug_tuple("Dropped").finish(),
2307 }
2308 }
2309}
2310
2311enum ReadState {
2313 Open,
2315 GuestReady {
2317 ty: TransmitIndex,
2318 caller_instance: RuntimeComponentInstanceIndex,
2319 flat_abi: Option<FlatAbi>,
2320 instance: Instance,
2321 options: OptionsIndex,
2322 address: usize,
2323 count: ItemCount,
2324 handle: u32,
2325 },
2326 HostReady {
2328 consume: PollStream,
2329 guest_offset: ItemCount,
2330 cancel: bool,
2331 cancel_waker: Option<Waker>,
2332 },
2333 HostToHost {
2335 accept: Box<
2336 dyn for<'a> Fn(
2337 &'a mut UntypedWriteBuffer<'a>,
2338 )
2339 -> Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'a>>
2340 + Send
2341 + Sync,
2342 >,
2343 buffer: Vec<u8>,
2344 limit: usize,
2345 },
2346 Dropped,
2348}
2349
2350impl fmt::Debug for ReadState {
2351 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2352 match self {
2353 Self::Open => f.debug_tuple("Open").finish(),
2354 Self::GuestReady { .. } => f.debug_tuple("GuestReady").finish(),
2355 Self::HostReady { .. } => f.debug_tuple("HostReady").finish(),
2356 Self::HostToHost { .. } => f.debug_tuple("HostToHost").finish(),
2357 Self::Dropped => f.debug_tuple("Dropped").finish(),
2358 }
2359 }
2360}
2361
2362fn return_code(
2363 kind: TransmitKind,
2364 state: StreamResult,
2365 count: ItemCount,
2366 cancel: bool,
2367) -> Result<ReturnCode> {
2368 Ok(match (state, kind) {
2378 (StreamResult::Dropped, _) => ReturnCode::Dropped(count),
2379 (StreamResult::Completed, TransmitKind::Stream) if cancel => ReturnCode::Cancelled(count),
2380 (StreamResult::Completed, _) => ReturnCode::completed(kind, count),
2381 (StreamResult::Cancelled, _) => ReturnCode::Cancelled(count),
2382 })
2383}
2384
2385fn settle_host_read(
2386 transmit: &mut TransmitState,
2387 kind: TransmitKind,
2388 state: StreamResult,
2389) -> Result<ReturnCode> {
2390 let ReadState::HostReady {
2391 consume,
2392 guest_offset,
2393 cancel,
2394 ..
2395 } = mem::replace(&mut transmit.read, ReadState::Open)
2396 else {
2397 bail_bug!("expected ReadState::HostReady")
2398 };
2399 let code = return_code(kind, state, guest_offset, cancel)?;
2400 transmit.read = match state {
2401 StreamResult::Dropped => ReadState::Dropped,
2402 StreamResult::Completed | StreamResult::Cancelled => ReadState::HostReady {
2403 consume,
2404 guest_offset: ItemCount::ZERO,
2405 cancel: false,
2406 cancel_waker: None,
2407 },
2408 };
2409 Ok(code)
2410}
2411
2412fn settle_host_write(
2413 transmit: &mut TransmitState,
2414 kind: TransmitKind,
2415 state: StreamResult,
2416) -> Result<ReturnCode> {
2417 let WriteState::HostReady {
2418 produce,
2419 try_into,
2420 guest_offset,
2421 cancel,
2422 ..
2423 } = mem::replace(&mut transmit.write, WriteState::Open)
2424 else {
2425 bail_bug!("expected WriteState::HostReady")
2426 };
2427 let code = return_code(kind, state, guest_offset, cancel)?;
2428 transmit.write = match state {
2429 StreamResult::Dropped => WriteState::Dropped,
2430 StreamResult::Completed | StreamResult::Cancelled => WriteState::HostReady {
2431 produce,
2432 try_into,
2433 guest_offset: ItemCount::ZERO,
2434 cancel: false,
2435 cancel_waker: None,
2436 },
2437 };
2438 Ok(code)
2439}
2440
2441impl StoreOpaque {
2442 fn pipe_from_guest(
2443 &mut self,
2444 kind: TransmitKind,
2445 id: TableId<TransmitState>,
2446 future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,
2447 ) {
2448 let future = async move {
2449 let stream_state = future.await?;
2450 tls::get(|store| {
2451 let state = store.concurrent_state_mut()?;
2452 let transmit = state.get_mut(id)?;
2453 let code = settle_host_read(transmit, kind, stream_state)?;
2454 let WriteState::GuestReady { ty, handle, .. } =
2455 mem::replace(&mut transmit.write, WriteState::Open)
2456 else {
2457 bail_bug!("expected WriteState::GuestReady")
2458 };
2459 state.send_write_result(ty, id, handle, code)?;
2460 Ok(())
2461 })
2462 };
2463
2464 self.concurrent_state_mut_already_forced_current_thread()
2465 .push_future(future.boxed());
2466 }
2467
2468 fn pipe_to_guest(
2469 &mut self,
2470 kind: TransmitKind,
2471 id: TableId<TransmitState>,
2472 future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,
2473 ) {
2474 let future = async move {
2475 let stream_state = future.await?;
2476 tls::get(|store| {
2477 let state = store.concurrent_state_mut()?;
2478 let transmit = state.get_mut(id)?;
2479 let code = settle_host_write(transmit, kind, stream_state)?;
2480 let ReadState::GuestReady { ty, handle, .. } =
2481 mem::replace(&mut transmit.read, ReadState::Open)
2482 else {
2483 bail_bug!("expected ReadState::GuestReady")
2484 };
2485 state.send_read_result(ty, id, handle, code)?;
2486 Ok(())
2487 })
2488 };
2489
2490 self.concurrent_state_mut_already_forced_current_thread()
2491 .push_future(future.boxed());
2492 }
2493
2494 fn host_drop_reader(&mut self, id: TableId<TransmitHandle>, kind: TransmitKind) -> Result<()> {
2496 let state = self.concurrent_state_mut()?;
2497 Waitable::Transmit(id).join(state, None)?;
2498 let transmit_id = state.get_mut(id)?.state;
2499 let transmit = state
2500 .get_mut(transmit_id)
2501 .with_context(|| format!("error closing reader {transmit_id:?}"))?;
2502 log::trace!(
2503 "host_drop_reader state {transmit_id:?}; read state {:?} write state {:?}",
2504 transmit.read,
2505 transmit.write
2506 );
2507
2508 transmit.read = ReadState::Dropped;
2509
2510 let new_state = if let WriteState::Dropped = &transmit.write {
2513 WriteState::Dropped
2514 } else {
2515 WriteState::Open
2516 };
2517
2518 let write_handle = transmit.write_handle;
2519 let done = transmit.done;
2520
2521 match mem::replace(&mut transmit.write, new_state) {
2522 WriteState::GuestReady { ty, handle, .. } => {
2525 state.update_event(
2526 write_handle.rep(),
2527 match ty {
2528 TransmitIndex::Future(ty) => Event::FutureWrite {
2529 code: ReturnCode::Dropped(ItemCount::ZERO),
2530 pending: Some((ty, handle)),
2531 },
2532 TransmitIndex::Stream(ty) => Event::StreamWrite {
2533 code: ReturnCode::Dropped(ItemCount::ZERO),
2534 pending: Some((ty, handle)),
2535 },
2536 },
2537 )?;
2538 }
2539
2540 WriteState::Open if done => {}
2543 WriteState::Open => {
2544 state.update_event(
2545 write_handle.rep(),
2546 match kind {
2547 TransmitKind::Future => Event::FutureWrite {
2548 code: ReturnCode::Dropped(ItemCount::ZERO),
2549 pending: None,
2550 },
2551 TransmitKind::Stream => Event::StreamWrite {
2552 code: ReturnCode::Dropped(ItemCount::ZERO),
2553 pending: None,
2554 },
2555 },
2556 )?;
2557 }
2558
2559 WriteState::Dropped | WriteState::HostReady { .. } => {
2564 log::trace!("host_drop_reader delete {transmit_id:?}");
2565 state.delete_transmit(transmit_id)?;
2566 }
2567 }
2568 Ok(())
2569 }
2570
2571 fn host_drop_writer(
2573 &mut self,
2574 id: TableId<TransmitHandle>,
2575 on_drop_open: Option<fn() -> Result<()>>,
2576 ) -> Result<()> {
2577 let state = self.concurrent_state_mut()?;
2578 Waitable::Transmit(id).join(state, None)?;
2579 let transmit_id = state.get_mut(id)?.state;
2580 let transmit = state
2581 .get_mut(transmit_id)
2582 .with_context(|| format!("error closing writer {transmit_id:?}"))?;
2583 log::trace!(
2584 "host_drop_writer state {transmit_id:?}; read state {:?} writer state {:?}",
2585 transmit.read,
2586 transmit.write
2587 );
2588
2589 match &mut transmit.write {
2591 WriteState::GuestReady { .. } => {
2592 bail_bug!("can't call `host_drop_writer` on a guest-owned writer");
2593 }
2594 WriteState::HostReady { .. } => {}
2595 v @ WriteState::Open => {
2596 if let (Some(on_drop_open), false) = (on_drop_open, transmit.done) {
2597 on_drop_open()?;
2598 } else {
2599 *v = WriteState::Dropped;
2600 }
2601 }
2602 WriteState::Dropped => bail_bug!("write state is already dropped"),
2603 }
2604
2605 let transmit = self.concurrent_state_mut()?.get_mut(transmit_id)?;
2606
2607 let new_state = if let ReadState::Dropped = &transmit.read {
2613 ReadState::Dropped
2614 } else {
2615 ReadState::Open
2616 };
2617
2618 let read_handle = transmit.read_handle;
2619 let done = transmit.done;
2620
2621 match mem::replace(&mut transmit.read, new_state) {
2623 ReadState::GuestReady { ty, handle, .. } => {
2627 self.concurrent_state_mut()?.update_event(
2629 read_handle.rep(),
2630 match ty {
2631 TransmitIndex::Future(ty) => Event::FutureRead {
2632 code: ReturnCode::Dropped(ItemCount::ZERO),
2633 pending: Some((ty, handle)),
2634 },
2635 TransmitIndex::Stream(ty) => Event::StreamRead {
2636 code: ReturnCode::Dropped(ItemCount::ZERO),
2637 pending: Some((ty, handle)),
2638 },
2639 },
2640 )?;
2641 }
2642
2643 ReadState::Open if done => {}
2645 ReadState::Open => {
2646 self.concurrent_state_mut()?.update_event(
2647 read_handle.rep(),
2648 match on_drop_open {
2649 Some(_) => Event::FutureRead {
2650 code: ReturnCode::Dropped(ItemCount::ZERO),
2651 pending: None,
2652 },
2653 None => Event::StreamRead {
2654 code: ReturnCode::Dropped(ItemCount::ZERO),
2655 pending: None,
2656 },
2657 },
2658 )?;
2659 }
2660
2661 ReadState::Dropped | ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {
2668 log::trace!("host_drop_writer delete {transmit_id:?}");
2669 self.concurrent_state_mut()?.delete_transmit(transmit_id)?;
2670 }
2671 }
2672 Ok(())
2673 }
2674
2675 pub(super) fn transmit_origin(&mut self, idx: HostResourceIndex) -> Result<TransmitOrigin> {
2676 let id = self.host_reader_rep(idx)?;
2677 let state = self.concurrent_state_mut()?;
2678 let state_id = state.get_mut(id)?.state;
2679 Ok(state.get_mut(state_id)?.origin)
2680 }
2681
2682 fn host_reader_insert(&mut self, id: TableId<TransmitHandle>) -> Result<HostResourceIndex> {
2686 HostResourceTables::new_host(self)?.host_resource_lower_own(id.rep(), None, None)
2687 }
2688
2689 fn host_reader_take(&mut self, idx: HostResourceIndex) -> Result<TableId<TransmitHandle>> {
2692 let rep = HostResourceTables::new_host(self)?.host_resource_lift_own(idx)?;
2693 Ok(TableId::new(rep))
2694 }
2695
2696 fn host_reader_rep(&mut self, idx: HostResourceIndex) -> Result<TableId<TransmitHandle>> {
2699 let rep = HostResourceTables::new_host(self)?.host_resource_rep(idx)?;
2700 Ok(TableId::new(rep))
2701 }
2702}
2703
2704struct ProducerState<P, B> {
2705 producer: P,
2706 buffer: B,
2707 dropped: bool,
2708}
2709
2710impl<T> StoreContextMut<'_, T> {
2711 fn new_transmit<P: StreamProducer<T>>(
2712 mut self,
2713 kind: TransmitKind,
2714 producer: P,
2715 ) -> Result<HostResourceIndex>
2716 where
2717 P::Item: func::Lower,
2718 {
2719 let token = StoreToken::new(self.as_context_mut());
2720 let state = self.0.concurrent_state_mut()?;
2721 let (_, read) = state.new_transmit(TransmitOrigin::Host {
2722 typecheck: types::typecheck_payload::<P::Item>,
2723 })?;
2724 let producer = Arc::new(LockedState::new(ProducerState {
2725 producer: Box::pin(producer),
2726 buffer: P::Buffer::default(),
2727 dropped: false,
2728 }));
2729 let id = state.get_mut(read)?.state;
2730 let produce = Box::new({
2731 let producer = producer.clone();
2732 move || {
2733 let producer = producer.clone();
2734 async move {
2735 let mut state = producer.take()?;
2736 let ProducerState { producer: mine, buffer, dropped } = &mut *state;
2737
2738 let (result, cancelled) = if buffer.remaining().is_empty() {
2739 future::poll_fn(|cx| {
2740 tls::get(|store| {
2741 let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2742
2743 let &WriteState::HostReady { cancel, .. } = &transmit.write else {
2744 bail_bug!("expected WriteState::HostReady")
2745 };
2746
2747 let mut host_written = 0;
2748 let mut host_buffer =
2749 if let ReadState::HostToHost { buffer, .. } = &mut transmit.read {
2750 Some(mem::take(buffer))
2751 } else {
2752 None
2753 };
2754
2755 let poll = mine.as_mut().poll_produce(
2756 cx,
2757 token.as_context_mut(store),
2758 Destination {
2759 id,
2760 buffer,
2761 host_buffer: host_buffer.as_mut().map(|b| {
2762 HostBuffer {
2763 dst: b,
2764 marked_written: &mut host_written,
2765 }
2766 }),
2767 _phantom: PhantomData,
2768 },
2769 cancel,
2770 );
2771
2772 let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2773
2774 let host_offset = if let (
2775 Some(host_buffer),
2776 ReadState::HostToHost { buffer, limit, .. },
2777 ) = (host_buffer, &mut transmit.read)
2778 {
2779 *limit = host_written;
2780 *buffer = host_buffer;
2781 *limit
2782 } else {
2783 0
2784 };
2785
2786 {
2787 let WriteState::HostReady {
2788 guest_offset,
2789 cancel_waker,
2790 ..
2791 } = &mut transmit.write
2792 else {
2793 bail_bug!("expected WriteState::HostReady")
2794 };
2795
2796 *cancel_waker = if poll.is_pending() {
2797 if !buffer.remaining().is_empty()
2798 || *guest_offset > 0
2799 || host_offset > 0
2800 {
2801 bail!(
2802 "StreamProducer::poll_produce returned Poll::Pending \
2803 after producing at least one item"
2804 )
2805 }
2806 Some(cx.waker().clone())
2807 } else {
2808 None
2809 };
2810 }
2811
2812 Ok(poll.map(|v| v.map(|result| (result, cancel))))
2813 })?
2814 })
2815 .await?
2816 } else {
2817 (StreamResult::Completed, false)
2818 };
2819
2820 let (guest_offset, host_offset, count) = tls::get(|store| {
2821 let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2822 let (count, host_offset) = match &transmit.read {
2823 &ReadState::GuestReady { count, .. } => (count.as_u32(), 0),
2824 &ReadState::HostToHost { limit, .. } => (1, limit),
2825 _ => bail_bug!("invalid read state"),
2826 };
2827 let guest_offset = match &transmit.write {
2828 &WriteState::HostReady { guest_offset, .. } => guest_offset,
2829 _ => bail_bug!("invalid write state"),
2830 };
2831 Ok((guest_offset, host_offset, count))
2832 })?;
2833
2834 let produced =
2835 !buffer.remaining().is_empty() || guest_offset > 0 || host_offset > 0;
2836
2837 match result {
2838 StreamResult::Completed => {
2839 if count > 0 && !produced {
2843 bail!(
2844 "StreamProducer::poll_produce returned StreamResult::Completed \
2845 without producing any items"
2846 );
2847 }
2848 }
2849 StreamResult::Cancelled => {
2850 if !cancelled {
2851 bail!(
2852 "StreamProducer::poll_produce returned StreamResult::Cancelled \
2853 without being given a `finish` parameter value of true"
2854 );
2855 }
2856 if produced {
2857 bail!(
2858 "StreamProducer::poll_produce returned StreamResult::Cancelled \
2859 after producing at least one item"
2860 );
2861 }
2862 }
2863 StreamResult::Dropped => {
2864 *dropped = true;
2865 }
2866 }
2867
2868 let write_buffer = !buffer.remaining().is_empty() || host_offset > 0;
2869
2870 let dropped = *dropped;
2871
2872 drop(state);
2873
2874 if write_buffer {
2875 write(token, id, producer.clone(), kind).await?;
2876 }
2877
2878 Ok(if dropped {
2879 if producer.with(|p| p.buffer.remaining().is_empty())? {
2880 StreamResult::Dropped
2881 } else {
2882 StreamResult::Completed
2883 }
2884 } else {
2885 result
2886 })
2887 }
2888 .boxed()
2889 }
2890 });
2891 let try_into = Box::new(move |ty| {
2892 let ProducerState {
2893 producer: mine,
2894 buffer,
2895 dropped,
2896 } = producer.try_lock().ok()?.take()?;
2897 match P::try_into(mine, buffer, ty) {
2898 Ok(value) => Some(value),
2899 Err((mine, buffer)) => {
2900 *producer.try_lock().ok()? = Some(ProducerState {
2901 producer: mine,
2902 buffer,
2903 dropped,
2904 });
2905 None
2906 }
2907 }
2908 });
2909 state.get_mut(id)?.write = WriteState::HostReady {
2910 produce,
2911 try_into,
2912 guest_offset: ItemCount::ZERO,
2913 cancel: false,
2914 cancel_waker: None,
2915 };
2916 self.0.host_reader_insert(read)
2917 }
2918
2919 fn set_consumer<C: StreamConsumer<T>>(
2920 mut self,
2921 idx: HostResourceIndex,
2922 kind: TransmitKind,
2923 consumer: C,
2924 ) -> Result<()> {
2925 let token = StoreToken::new(self.as_context_mut());
2926 let id = self.0.host_reader_take(idx)?;
2927 let state = self.0.concurrent_state_mut()?;
2928 let id = state.get_mut(id)?.state;
2929 let transmit = state.get_mut(id)?;
2930 let consumer = Arc::new(LockedState::new(Box::pin(consumer)));
2931 let consume_with_buffer = {
2932 let consumer = consumer.clone();
2933 async move |mut host_buffer: Option<&mut dyn WriteBuffer<C::Item>>| {
2934 let mut mine = consumer.take()?;
2935
2936 let host_buffer_remaining_before =
2937 host_buffer.as_deref_mut().map(|v| v.remaining().len());
2938
2939 let (result, cancelled) = future::poll_fn(|cx| {
2940 tls::get(|store| {
2941 let cancel = match &store.concurrent_state_mut()?.get_mut(id)?.read {
2942 &ReadState::HostReady { cancel, .. } => cancel,
2943 ReadState::Open => false,
2944 _ => bail_bug!("unexpected read state"),
2945 };
2946
2947 let poll = mine.as_mut().poll_consume(
2948 cx,
2949 token.as_context_mut(store),
2950 Source {
2951 id,
2952 host_buffer: host_buffer.as_deref_mut(),
2953 },
2954 cancel,
2955 );
2956
2957 if let ReadState::HostReady { cancel_waker, .. } =
2958 &mut store.concurrent_state_mut()?.get_mut(id)?.read
2959 {
2960 *cancel_waker = if poll.is_pending() {
2961 Some(cx.waker().clone())
2962 } else {
2963 None
2964 };
2965 }
2966
2967 Ok(poll.map(|v| v.map(|result| (result, cancel))))
2968 })?
2969 })
2970 .await?;
2971
2972 let (guest_offset, count) = tls::get(|store| {
2973 let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2974 Ok((
2975 match &transmit.read {
2976 &ReadState::HostReady { guest_offset, .. } => guest_offset,
2977 ReadState::Open => ItemCount::ZERO,
2978 _ => bail_bug!("invalid read state"),
2979 },
2980 match &transmit.write {
2981 WriteState::GuestReady { count, .. } => count.as_usize(),
2982 WriteState::HostReady { .. } => match host_buffer_remaining_before {
2983 Some(n) => n,
2984 None => bail_bug!("host_buffer_remaining_before should be set"),
2985 },
2986 _ => bail_bug!("invalid write state"),
2987 },
2988 ))
2989 })?;
2990
2991 match result {
2992 StreamResult::Completed => {
2993 let consumed = match (host_buffer_remaining_before, host_buffer) {
2997 (Some(before), Some(after)) => before != after.remaining().len(),
2998 _ => guest_offset > 0,
2999 };
3000 if count > 0 && !consumed {
3001 bail!(
3002 "StreamConsumer::poll_consume returned StreamResult::Completed \
3003 without consuming any items"
3004 );
3005 }
3006
3007 if let TransmitKind::Future = kind {
3008 tls::get(|store| {
3009 store.concurrent_state_mut()?.get_mut(id)?.done = true;
3010 crate::error::Ok(())
3011 })?;
3012 }
3013 }
3014 StreamResult::Cancelled => {
3015 if !cancelled {
3016 bail!(
3017 "StreamConsumer::poll_consume returned StreamResult::Cancelled \
3018 without being given a `finish` parameter value of true"
3019 );
3020 }
3021 }
3022 StreamResult::Dropped => {}
3023 }
3024
3025 Ok(result)
3026 }
3027 };
3028 let consume = {
3029 let consume = consume_with_buffer.clone();
3030 Box::new(move || {
3031 let consume = consume.clone();
3032 async move { consume(None).await }.boxed()
3033 })
3034 };
3035
3036 match &transmit.write {
3037 WriteState::Open => {
3038 transmit.read = ReadState::HostReady {
3039 consume,
3040 guest_offset: ItemCount::ZERO,
3041 cancel: false,
3042 cancel_waker: None,
3043 };
3044 }
3045 &WriteState::GuestReady { .. } => {
3046 let write_handle = transmit.write_handle;
3051 let write_has_event = Waitable::Transmit(write_handle)
3052 .common(state)?
3053 .event
3054 .is_some();
3055 let future = if write_has_event {
3056 None
3057 } else {
3058 Some(consume())
3059 };
3060 state.get_mut(id)?.read = ReadState::HostReady {
3061 consume,
3062 guest_offset: ItemCount::ZERO,
3063 cancel: false,
3064 cancel_waker: None,
3065 };
3066 if let Some(future) = future {
3067 self.0.pipe_from_guest(kind, id, future);
3068 }
3069 }
3070 WriteState::HostReady { .. } => {
3071 let WriteState::HostReady { produce, .. } = mem::replace(
3072 &mut transmit.write,
3073 WriteState::HostReady {
3074 produce: Box::new(|| {
3075 Box::pin(async { bail_bug!("unexpected invocation of `produce`") })
3076 }),
3077 try_into: Box::new(|_| None),
3078 guest_offset: ItemCount::ZERO,
3079 cancel: false,
3080 cancel_waker: None,
3081 },
3082 ) else {
3083 bail_bug!("expected WriteState::HostReady")
3084 };
3085
3086 transmit.read = ReadState::HostToHost {
3087 accept: Box::new(move |input| {
3088 let consume = consume_with_buffer.clone();
3089 async move { consume(Some(input.get_mut::<C::Item>()?)).await }.boxed()
3090 }),
3091 buffer: Vec::new(),
3092 limit: 0,
3093 };
3094
3095 let future = async move {
3096 loop {
3097 if tls::get(|store| {
3098 crate::error::Ok(matches!(
3099 store.concurrent_state_mut()?.get_mut(id)?.read,
3100 ReadState::Dropped
3101 ))
3102 })? {
3103 break Ok(());
3104 }
3105
3106 match produce().await? {
3107 StreamResult::Completed | StreamResult::Cancelled => {}
3108 StreamResult::Dropped => break Ok(()),
3109 }
3110
3111 if let TransmitKind::Future = kind {
3112 break Ok(());
3113 }
3114 }
3115 }
3116 .map(move |result| {
3117 tls::get(|store| store.concurrent_state_mut()?.delete_transmit(id))?;
3118 result
3119 });
3120
3121 state.push_future(Box::pin(future));
3122 }
3123 WriteState::Dropped => {
3124 let reader = transmit.read_handle;
3125 self.0.host_drop_reader(reader, kind)?;
3126 }
3127 }
3128 Ok(())
3129 }
3130}
3131
3132async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: WriteBuffer<T>>(
3133 token: StoreToken<D>,
3134 id: TableId<TransmitState>,
3135 producer: Arc<LockedState<ProducerState<P, B>>>,
3136 kind: TransmitKind,
3137) -> Result<()> {
3138 let (read, guest_offset) = tls::get(|store| {
3139 let transmit = store.concurrent_state_mut()?.get_mut(id)?;
3140
3141 let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write {
3142 Some(guest_offset)
3143 } else {
3144 None
3145 };
3146
3147 crate::error::Ok((
3148 mem::replace(&mut transmit.read, ReadState::Open),
3149 guest_offset,
3150 ))
3151 })?;
3152
3153 match read {
3154 ReadState::GuestReady {
3155 ty,
3156 flat_abi,
3157 options,
3158 address,
3159 count,
3160 handle,
3161 instance,
3162 caller_instance,
3163 } => {
3164 let guest_offset = match guest_offset {
3165 Some(i) => i,
3166 None => bail_bug!("guest_offset should be present if ready"),
3167 };
3168
3169 if let TransmitKind::Future = kind {
3170 tls::get(|store| {
3171 store.concurrent_state_mut()?.get_mut(id)?.done = true;
3172 crate::error::Ok(())
3173 })?;
3174 }
3175
3176 let old_remaining = producer.with(|p| p.buffer.remaining().len())?;
3177 let accept = {
3178 let producer = producer.clone();
3179 move |mut store: StoreContextMut<D>| {
3180 let mut state = producer.take()?;
3181 lower::<T, B, D>(
3182 store.as_context_mut(),
3183 instance,
3184 options,
3185 ty,
3186 address + (T::SIZE32 * guest_offset.as_usize()),
3187 count.as_usize() - guest_offset.as_usize(),
3188 &mut state.buffer,
3189 )?;
3190 crate::error::Ok(())
3191 }
3192 };
3193
3194 if guest_offset < count {
3195 if T::MAY_REQUIRE_REALLOC {
3196 let (tx, rx) = oneshot::channel();
3201 tls::get(move |store| {
3202 store
3203 .concurrent_state_mut()?
3204 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(
3205 Box::new(move |store| {
3206 _ = tx.send(accept(token.as_context_mut(store))?);
3207 Ok(())
3208 }),
3209 )));
3210 crate::error::Ok(())
3211 })?;
3212 match rx.await {
3213 Ok(r) => r,
3214 Err(oneshot::Canceled) => {
3222 bail!("stream write cancelled before it could complete")
3223 }
3224 }
3225 } else {
3226 tls::get(|store| accept(token.as_context_mut(store)))?
3231 }
3232 }
3233
3234 tls::get(|store| {
3235 let count = old_remaining - producer.with(|p| p.buffer.remaining().len())?;
3236
3237 let transmit = store.concurrent_state_mut()?.get_mut(id)?;
3238
3239 let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
3240 bail_bug!("expected WriteState::HostReady")
3241 };
3242
3243 guest_offset.inc(count)?;
3244
3245 transmit.read = ReadState::GuestReady {
3246 ty,
3247 flat_abi,
3248 options,
3249 address,
3250 count: ItemCount::new_usize(count)?,
3251 handle,
3252 instance,
3253 caller_instance,
3254 };
3255
3256 crate::error::Ok(())
3257 })?;
3258
3259 Ok(())
3260 }
3261
3262 ReadState::HostToHost {
3263 accept,
3264 mut buffer,
3265 limit,
3266 } => {
3267 let mut state = StreamResult::Completed;
3268 let mut position = 0;
3269
3270 while !matches!(state, StreamResult::Dropped) && position < limit {
3271 let mut slice_buffer = SliceBuffer::new(buffer, position, limit);
3272 state = accept(&mut UntypedWriteBuffer::new(&mut slice_buffer)).await?;
3273 (buffer, position, _) = slice_buffer.into_parts();
3274 }
3275
3276 {
3277 let mut producer = producer.take()?;
3278 let ProducerState { buffer, .. } = &mut *producer;
3279
3280 while !(matches!(state, StreamResult::Dropped) || buffer.remaining().is_empty()) {
3281 state = accept(&mut UntypedWriteBuffer::new(buffer)).await?;
3282 }
3283 }
3284
3285 tls::get(|store| {
3286 store.concurrent_state_mut()?.get_mut(id)?.read = match state {
3287 StreamResult::Dropped => ReadState::Dropped,
3288 StreamResult::Completed | StreamResult::Cancelled => ReadState::HostToHost {
3289 accept,
3290 buffer,
3291 limit: 0,
3292 },
3293 };
3294
3295 crate::error::Ok(())
3296 })?;
3297 Ok(())
3298 }
3299
3300 _ => bail_bug!("unexpected read state"),
3301 }
3302}
3303
3304fn poll_inline(
3311 store: &mut dyn VMStore,
3312 future: &mut Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,
3313) -> Result<Poll<Result<StreamResult>>> {
3314 let old_thread = store.set_thread(CurrentThread::None)?;
3315 let poll = tls::set(store, || {
3316 future
3317 .as_mut()
3318 .poll(&mut Context::from_waker(&Waker::noop()))
3319 });
3320 store.set_thread(old_thread)?;
3321 Ok(poll)
3322}
3323
3324impl Instance {
3325 fn consume(
3328 self,
3329 store: &mut dyn VMStore,
3330 kind: TransmitKind,
3331 transmit_id: TableId<TransmitState>,
3332 consume: PollStream,
3333 guest_offset: ItemCount,
3334 cancel: bool,
3335 ) -> Result<ReturnCode> {
3336 let mut future = consume();
3337 store.concurrent_state_mut()?.get_mut(transmit_id)?.read = ReadState::HostReady {
3338 consume,
3339 guest_offset,
3340 cancel,
3341 cancel_waker: None,
3342 };
3343 let poll = poll_inline(store, &mut future)?;
3344
3345 Ok(match poll {
3346 Poll::Ready(state) => {
3347 let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
3348 let code = settle_host_read(transmit, kind, state?)?;
3349 transmit.write = WriteState::Open;
3350 code
3351 }
3352 Poll::Pending => {
3353 store.pipe_from_guest(kind, transmit_id, future);
3354 ReturnCode::Blocked
3355 }
3356 })
3357 }
3358
3359 fn produce(
3362 self,
3363 store: &mut dyn VMStore,
3364 kind: TransmitKind,
3365 transmit_id: TableId<TransmitState>,
3366 produce: PollStream,
3367 try_into: TryInto,
3368 guest_offset: ItemCount,
3369 cancel: bool,
3370 ) -> Result<ReturnCode> {
3371 let mut future = produce();
3372 store.concurrent_state_mut()?.get_mut(transmit_id)?.write = WriteState::HostReady {
3373 produce,
3374 try_into,
3375 guest_offset,
3376 cancel,
3377 cancel_waker: None,
3378 };
3379 let poll = poll_inline(store, &mut future)?;
3380
3381 Ok(match poll {
3382 Poll::Ready(state) => {
3383 let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
3384 let code = settle_host_write(transmit, kind, state?)?;
3385 transmit.read = ReadState::Open;
3386 code
3387 }
3388 Poll::Pending => {
3389 store.pipe_to_guest(kind, transmit_id, future);
3390 ReturnCode::Blocked
3391 }
3392 })
3393 }
3394
3395 pub(super) fn guest_drop_writable(
3397 self,
3398 store: &mut StoreOpaque,
3399 ty: TransmitIndex,
3400 writer: u32,
3401 ) -> Result<()> {
3402 let table = self.id().get_mut(store).table_for_transmit(ty);
3403 let (transmit_rep, is_done) = match ty {
3404 TransmitIndex::Future(ty) => table.future_remove_writable(ty, writer)?,
3405 TransmitIndex::Stream(ty) => (table.stream_remove_writable(ty, writer)?, false),
3406 };
3407
3408 let id = TableId::<TransmitHandle>::new(transmit_rep);
3409 log::trace!("guest_drop_writable: drop writer {id:?}");
3410 match ty {
3411 TransmitIndex::Stream(_) => store.host_drop_writer(id, None),
3412 TransmitIndex::Future(_) => store.host_drop_writer(
3413 id,
3414 if is_done {
3415 None
3416 } else {
3417 Some(|| {
3418 Err(format_err!(
3419 "cannot drop future write end without first writing a value"
3420 ))
3421 })
3422 },
3423 ),
3424 }
3425 }
3426
3427 fn copy<T: 'static>(
3430 store: StoreContextMut<T>,
3431 flat_abi: Option<FlatAbi>,
3432 write_runtime_instance: RuntimeInstance,
3433 write_ty: TransmitIndex,
3434 write_options: OptionsIndex,
3435 write_address: usize,
3436 read_runtime_instance: RuntimeInstance,
3437 read_ty: TransmitIndex,
3438 read_options: OptionsIndex,
3439 read_address: usize,
3440 count: ItemCount,
3441 rep: u32,
3442 ) -> Result<()> {
3443 let write_instance = Instance::from_runtime_instance(store.0, write_runtime_instance);
3444 let read_instance = Instance::from_runtime_instance(store.0, read_runtime_instance);
3445 let (write_component, store) = write_instance.component_and_store_mut(store.0);
3446 let (read_component, mut store) = read_instance.component_and_store_mut(store);
3447 let write_types = write_component.types();
3448 let read_types = read_component.types();
3449 let count = count.as_usize();
3450
3451 let write_payload_ty = write_ty.payload(write_types);
3458 let write_abi = match write_payload_ty {
3459 Some(ty) => write_types.canonical_abi(ty),
3460 None => &CanonicalAbiInfo::ZERO,
3461 };
3462 let write_length_in_bytes = match flat_abi {
3463 Some(abi) => usize::try_from(abi.size)? * count,
3464 None => usize::try_from(write_abi.size32)? * count,
3465 };
3466 if write_payload_ty.is_some() {
3467 if write_address % usize::try_from(write_abi.align32)? != 0 {
3468 bail_bug!("write pointer not aligned");
3469 }
3470 if write_instance
3471 .options_memory(store, write_options)
3472 .get(write_address..)
3473 .and_then(|b| b.get(..write_length_in_bytes))
3474 .is_none()
3475 {
3476 bail_bug!("write pointer out of bounds");
3477 }
3478 }
3479
3480 let read_payload_ty = read_ty.payload(read_types);
3481 let read_abi = match read_payload_ty {
3482 Some(ty) => read_types.canonical_abi(ty),
3483 None => &CanonicalAbiInfo::ZERO,
3484 };
3485 let read_length_in_bytes = match flat_abi {
3486 Some(abi) => usize::try_from(abi.size)? * count,
3487 None => usize::try_from(read_abi.size32)? * count,
3488 };
3489 if read_payload_ty.is_some() {
3490 if read_address % usize::try_from(read_abi.align32)? != 0 {
3491 bail_bug!("read pointer not aligned");
3492 }
3493 if read_instance
3494 .options_memory(store, read_options)
3495 .get(read_address..)
3496 .and_then(|b| b.get(..read_length_in_bytes))
3497 .is_none()
3498 {
3499 bail_bug!("read pointer out of bounds");
3500 }
3501 }
3502
3503 if write_runtime_instance == read_runtime_instance
3504 && !allow_intra_component_read_write(write_payload_ty)
3505 {
3506 bail!(
3507 "cannot read from and write to intra-component future/stream with non-numeric payload"
3508 )
3509 }
3510
3511 match (write_ty, read_ty) {
3512 (TransmitIndex::Future(_), TransmitIndex::Future(_)) => {
3513 if count != 1 {
3514 bail_bug!("futures can only send 1 item");
3515 }
3516
3517 let val = write_payload_ty
3518 .map(|ty| {
3519 let lift = &mut LiftContext::new(store, write_options, write_instance)?;
3520 let bytes = &lift.memory()[write_address..][..write_length_in_bytes];
3521 Val::load(lift, *ty, bytes)
3522 })
3523 .transpose()?;
3524
3525 if let Some(val) = val {
3526 let lower =
3527 &mut LowerContext::new(store.as_context_mut(), read_options, read_instance);
3528 let ptr = func::validate_inbounds_dynamic(
3529 read_abi,
3530 lower.as_slice(),
3531 &ValRaw::u32(read_address.try_into()?),
3532 )?;
3533 let ty = match read_payload_ty {
3534 Some(ty) => ty,
3535 None => bail_bug!("expected read payload type to be present"),
3536 };
3537 val.store(lower, *ty, ptr)?;
3538 }
3539 }
3540 (TransmitIndex::Stream(_), TransmitIndex::Stream(_)) => {
3541 if write_length_in_bytes == 0 {
3542 return Ok(());
3543 }
3544 let write_payload_ty = match write_payload_ty {
3545 Some(ty) => ty,
3546 None => bail_bug!("expected write payload type to be present"),
3547 };
3548 let read_payload_ty = match read_payload_ty {
3549 Some(ty) => ty,
3550 None => bail_bug!("expected read payload type to be present"),
3551 };
3552 let lift =
3553 &mut LiftContext::new(store.store_opaque_mut(), write_options, write_instance)?;
3554 if let Some(abi) = flat_abi {
3555 lift.consume_fuel_array(count, usize::try_from(abi.size)?)?;
3558
3559 let store_opaque = store.store_opaque_mut();
3561
3562 assert_eq!(read_length_in_bytes, write_length_in_bytes);
3563
3564 if read_instance
3565 .options_memory(store_opaque, read_options)
3566 .as_ptr()
3567 == write_instance
3568 .options_memory(store_opaque, write_options)
3569 .as_ptr()
3570 {
3571 let memory = read_instance.options_memory_mut(store_opaque, read_options);
3572 memory.copy_within(
3573 write_address..write_address + write_length_in_bytes,
3574 read_address,
3575 );
3576 } else {
3577 let src = write_instance.options_memory(store_opaque, write_options)
3578 [write_address..][..write_length_in_bytes]
3579 .as_ptr();
3580 let dst = read_instance
3581 .options_memory_range_mut(
3582 store_opaque,
3583 read_options,
3584 read_address,
3585 read_length_in_bytes,
3586 )
3587 .unwrap()
3588 .as_mut_ptr();
3589
3590 unsafe {
3600 src.copy_to_nonoverlapping(dst, write_length_in_bytes);
3601 }
3602 }
3603 } else {
3604 let bytes = &lift.memory()[write_address..][..write_length_in_bytes];
3605 lift.consume_fuel_array(count, size_of::<Val>())?;
3606
3607 let values = (0..count)
3608 .map(|index| {
3609 let size = usize::try_from(write_abi.size32)?;
3610 Val::load(lift, *write_payload_ty, &bytes[(index * size)..][..size])
3611 })
3612 .collect::<Result<Vec<_>>>()?;
3613
3614 let id = TableId::<TransmitHandle>::new(rep);
3615 log::trace!("copy values {values:?} for {id:?}");
3616
3617 let lower =
3618 &mut LowerContext::new(store.as_context_mut(), read_options, read_instance);
3619 let mut ptr = read_address;
3620 for value in values {
3621 value.store(lower, *read_payload_ty, ptr)?;
3622 ptr += usize::try_from(read_abi.size32)?;
3623 }
3624 }
3625 }
3626 _ => bail_bug!("mismatched transmit types in copy"),
3627 }
3628
3629 Ok(())
3630 }
3631
3632 fn check_bounds(
3633 self,
3634 store: &StoreOpaque,
3635 options: OptionsIndex,
3636 ty: TransmitIndex,
3637 address: usize,
3638 count: usize,
3639 ) -> Result<()> {
3640 let types = self.id().get(store).component().types();
3641 let payload = match ty {
3642 TransmitIndex::Future(ty) => types[types[ty].ty].payload,
3643 TransmitIndex::Stream(ty) => types[types[ty].ty].payload,
3644 };
3645 let Some(payload) = payload else {
3648 return Ok(());
3649 };
3650 let abi = types.canonical_abi(&payload);
3651 let size = usize::try_from(abi.size32)?;
3653 let align = usize::try_from(abi.align32)?;
3654 if address % align != 0 {
3655 bail!("buffer pointer not aligned");
3656 }
3657 self.options_memory(store, options)
3658 .get(address..)
3659 .and_then(|b| b.get(..size.checked_mul(count)?))
3660 .map(drop)
3661 .ok_or_else(|| crate::format_err!("buffer pointer out of bounds of memory"))
3662 }
3663
3664 pub(super) fn guest_write<T: 'static>(
3666 self,
3667 mut store: StoreContextMut<T>,
3668 caller: RuntimeComponentInstanceIndex,
3669 ty: TransmitIndex,
3670 options: OptionsIndex,
3671 flat_abi: Option<FlatAbi>,
3672 handle: u32,
3673 address: u32,
3674 count: u32,
3675 ) -> Result<ReturnCode> {
3676 let count = ItemCount::new(count)?;
3677
3678 let address = usize::try_from(address)?;
3679 self.check_bounds(store.0, options, ty, address, count.as_usize())?;
3680 let async_ = self.options(store.0, options).async_;
3681 let (rep, state) = self.id().get_mut(store.0).get_mut_by_index(ty, handle)?;
3682 let TransmitLocalStateKind::Write { done } = state.kind() else {
3683 bail!(Trap::ConcurrentFutureStreamOp);
3684 };
3685
3686 *state = VMTransmitLocalState::busy();
3687 let transmit_handle = TableId::<TransmitHandle>::new(rep);
3688 let concurrent_state = store.0.concurrent_state_mut()?;
3689 let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;
3690 let transmit = concurrent_state.get_mut(transmit_id)?;
3691 log::trace!(
3692 "guest_write {count} to {transmit_handle:?} (handle {handle}; state {transmit_id:?}); {:?}",
3693 transmit.read
3694 );
3695
3696 if done || transmit.done {
3697 bail!(match ty {
3698 TransmitIndex::Future(_) => Trap::WriteToDroppedFuture,
3699 TransmitIndex::Stream(_) => Trap::WriteToDroppedStream,
3700 });
3701 }
3702
3703 if !async_ {
3704 Waitable::Transmit(transmit_handle).trap_if_in_waitable_set(concurrent_state)?;
3705 }
3706 let transmit = concurrent_state.get_mut(transmit_id)?;
3707
3708 let new_state = if let ReadState::Dropped = &transmit.read {
3709 ReadState::Dropped
3710 } else {
3711 ReadState::Open
3712 };
3713
3714 let set_guest_ready = |me: &mut ConcurrentState| {
3715 let transmit = me.get_mut(transmit_id)?;
3716 if !matches!(&transmit.write, WriteState::Open) {
3717 bail_bug!("expected `WriteState::Open`; got `{:?}`", transmit.write);
3718 }
3719 transmit.write = WriteState::GuestReady {
3720 instance: self,
3721 caller,
3722 ty,
3723 flat_abi,
3724 options,
3725 address,
3726 count,
3727 handle,
3728 };
3729 Ok::<_, crate::Error>(())
3730 };
3731
3732 let mut result = match mem::replace(&mut transmit.read, new_state) {
3733 ReadState::GuestReady {
3734 ty: read_ty,
3735 flat_abi: read_flat_abi,
3736 options: read_options,
3737 address: read_address,
3738 count: read_count,
3739 handle: read_handle,
3740 instance: read_instance,
3741 caller_instance: read_caller_instance,
3742 } => {
3743 if flat_abi != read_flat_abi {
3744 bail_bug!("expected flat ABI calculations to be the same");
3745 }
3746
3747 if let TransmitIndex::Future(_) = ty {
3748 transmit.done = true;
3749 }
3750
3751 let write_count = count;
3773 let write_complete = count == 0 || read_count > 0;
3774 let read_complete = count > 0;
3775 let read_buffer_remaining = count < read_count;
3776
3777 let read_handle_rep = transmit.read_handle.rep();
3778
3779 let count = count.min(read_count);
3780
3781 Instance::copy(
3782 store.as_context_mut(),
3783 flat_abi,
3784 self.runtime_instance(caller),
3785 ty,
3786 options,
3787 address,
3788 read_instance.runtime_instance(read_caller_instance),
3789 read_ty,
3790 read_options,
3791 read_address,
3792 count,
3793 rep,
3794 )?;
3795
3796 let instance = read_instance.id().get(store.0);
3797 let types = instance.component().types();
3798 let item_size = match read_ty.payload(types) {
3800 Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,
3801 None => 0,
3802 };
3803 let concurrent_state = store.0.concurrent_state_mut()?;
3804 if read_complete {
3805 let total = if let Some(Event::StreamRead {
3806 code: ReturnCode::Completed(old_total),
3807 ..
3808 }) = concurrent_state.take_event(read_handle_rep)?
3809 {
3810 count.add(old_total)?
3811 } else {
3812 count
3813 };
3814
3815 let code = ReturnCode::completed(ty.kind(), total);
3816
3817 concurrent_state.send_read_result(read_ty, transmit_id, read_handle, code)?;
3818 }
3819
3820 if read_buffer_remaining || (write_count == 0 && read_count == 0) {
3827 let transmit = concurrent_state.get_mut(transmit_id)?;
3828 transmit.read = ReadState::GuestReady {
3829 ty: read_ty,
3830 flat_abi: read_flat_abi,
3831 options: read_options,
3832 address: read_address + (count.as_usize() * item_size),
3833 count: read_count.sub(count)?,
3834 handle: read_handle,
3835 instance: read_instance,
3836 caller_instance: read_caller_instance,
3837 };
3838 }
3839
3840 if write_complete {
3841 ReturnCode::completed(ty.kind(), count)
3842 } else {
3843 set_guest_ready(concurrent_state)?;
3844 ReturnCode::Blocked
3845 }
3846 }
3847
3848 ReadState::HostReady {
3849 consume,
3850 guest_offset,
3851 cancel,
3852 cancel_waker,
3853 } => {
3854 if cancel_waker.is_some() {
3855 bail_bug!("expected cancel_waker to be none");
3856 }
3857 if cancel {
3858 bail_bug!("expected cancel to be false");
3859 }
3860 if guest_offset != 0 {
3861 bail_bug!("expected guest_offset to be 0");
3862 }
3863
3864 set_guest_ready(concurrent_state)?;
3865 self.consume(
3866 store.0,
3867 ty.kind(),
3868 transmit_id,
3869 consume,
3870 ItemCount::ZERO,
3871 false,
3872 )?
3873 }
3874
3875 ReadState::HostToHost { .. } => bail_bug!("unexpected HostToHost"),
3876
3877 ReadState::Open => {
3878 set_guest_ready(concurrent_state)?;
3879 ReturnCode::Blocked
3880 }
3881
3882 ReadState::Dropped => {
3883 if let TransmitIndex::Future(_) = ty {
3884 transmit.done = true;
3885 }
3886
3887 match Waitable::Transmit(transmit_handle).take_event(concurrent_state)? {
3888 Some(
3889 Event::StreamWrite {
3890 code: ReturnCode::Dropped(ItemCount::ZERO),
3891 ..
3892 }
3893 | Event::FutureWrite {
3894 code: ReturnCode::Dropped(ItemCount::ZERO),
3895 ..
3896 },
3897 ) => {}
3898 event => bail_bug!("expected pending dropped event for writer; got {event:?}"),
3899 }
3900
3901 ReturnCode::Dropped(ItemCount::ZERO)
3902 }
3903 };
3904
3905 if result == ReturnCode::Blocked && !async_ {
3906 result = self.wait_for_write(store.0, caller, transmit_handle)?;
3907 }
3908
3909 if result != ReturnCode::Blocked {
3910 *self.id().get_mut(store.0).get_mut_by_index(ty, handle)?.1 =
3911 VMTransmitLocalState::write(matches!(result, ReturnCode::Dropped(_)));
3912 }
3913
3914 log::trace!(
3915 "guest_write result for {transmit_handle:?} (handle {handle}; state {transmit_id:?}): {result:?}",
3916 );
3917
3918 Ok(result)
3919 }
3920
3921 pub(super) fn guest_read<T: 'static>(
3923 self,
3924 mut store: StoreContextMut<T>,
3925 caller_instance: RuntimeComponentInstanceIndex,
3926 ty: TransmitIndex,
3927 options: OptionsIndex,
3928 flat_abi: Option<FlatAbi>,
3929 handle: u32,
3930 address: u32,
3931 count: u32,
3932 ) -> Result<ReturnCode> {
3933 let count = ItemCount::new(count)?;
3934
3935 let address = usize::try_from(address)?;
3936 self.check_bounds(store.0, options, ty, address, count.as_usize())?;
3937 let async_ = self.options(store.0, options).async_;
3938 let (rep, state) = self.id().get_mut(store.0).get_mut_by_index(ty, handle)?;
3939 let TransmitLocalStateKind::Read { done } = state.kind() else {
3940 bail!(Trap::ConcurrentFutureStreamOp);
3941 };
3942
3943 *state = VMTransmitLocalState::busy();
3944 let transmit_handle = TableId::<TransmitHandle>::new(rep);
3945 let concurrent_state = store.0.concurrent_state_mut()?;
3946 let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;
3947 let transmit = concurrent_state.get_mut(transmit_id)?;
3948 log::trace!(
3949 "guest_read {count} from {transmit_handle:?} (handle {handle}; state {transmit_id:?}); {:?}",
3950 transmit.write
3951 );
3952
3953 if done || transmit.done {
3954 match ty {
3955 TransmitIndex::Future(_) => {
3956 bail!("cannot read from future after previous read succeeded")
3957 }
3958 TransmitIndex::Stream(_) => bail!(Trap::ReadFromDroppedStream),
3959 }
3960 }
3961
3962 if !async_ {
3963 Waitable::Transmit(transmit_handle).trap_if_in_waitable_set(concurrent_state)?;
3964 }
3965 let transmit = concurrent_state.get_mut(transmit_id)?;
3966
3967 let new_state = if let WriteState::Dropped = &transmit.write {
3968 WriteState::Dropped
3969 } else {
3970 WriteState::Open
3971 };
3972
3973 let set_guest_ready = |me: &mut ConcurrentState| {
3974 let transmit = me.get_mut(transmit_id)?;
3975 if !matches!(&transmit.read, ReadState::Open) {
3976 bail_bug!("expected `ReadState::Open`; got `{:?}`", transmit.read);
3977 }
3978 transmit.read = ReadState::GuestReady {
3979 ty,
3980 flat_abi,
3981 options,
3982 address,
3983 count,
3984 handle,
3985 instance: self,
3986 caller_instance,
3987 };
3988 Ok::<_, crate::Error>(())
3989 };
3990
3991 let mut result = match mem::replace(&mut transmit.write, new_state) {
3992 WriteState::GuestReady {
3993 instance: write_instance,
3994 ty: write_ty,
3995 flat_abi: write_flat_abi,
3996 options: write_options,
3997 address: write_address,
3998 count: write_count,
3999 handle: write_handle,
4000 caller: write_caller,
4001 } => {
4002 if flat_abi != write_flat_abi {
4003 bail_bug!("expected flat ABI calculations to be the same");
4004 }
4005
4006 if let TransmitIndex::Future(_) = ty {
4007 transmit.done = true;
4008 }
4009
4010 let write_handle_rep = transmit.write_handle.rep();
4011
4012 let write_complete = write_count == 0 || count > 0;
4017 let read_complete = write_count > 0;
4018 let write_buffer_remaining = count < write_count;
4019
4020 let count = count.min(write_count);
4021
4022 Instance::copy(
4023 store.as_context_mut(),
4024 flat_abi,
4025 write_instance.runtime_instance(write_caller),
4026 write_ty,
4027 write_options,
4028 write_address,
4029 self.runtime_instance(caller_instance),
4030 ty,
4031 options,
4032 address,
4033 count,
4034 rep,
4035 )?;
4036
4037 let instance = write_instance.id().get(store.0);
4038 let types = instance.component().types();
4039 let item_size = match write_ty.payload(types) {
4041 Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,
4042 None => 0,
4043 };
4044 let concurrent_state = store.0.concurrent_state_mut()?;
4045
4046 if write_complete {
4047 let total = if let Some(Event::StreamWrite {
4048 code: ReturnCode::Completed(old_total),
4049 ..
4050 }) = concurrent_state.take_event(write_handle_rep)?
4051 {
4052 count.add(old_total)?
4053 } else {
4054 count
4055 };
4056
4057 let code = ReturnCode::completed(ty.kind(), total);
4058
4059 concurrent_state.send_write_result(
4060 write_ty,
4061 transmit_id,
4062 write_handle,
4063 code,
4064 )?;
4065 }
4066
4067 if write_buffer_remaining {
4068 let transmit = concurrent_state.get_mut(transmit_id)?;
4069 transmit.write = WriteState::GuestReady {
4070 instance: write_instance,
4071 caller: write_caller,
4072 ty: write_ty,
4073 flat_abi: write_flat_abi,
4074 options: write_options,
4075 address: write_address + (count.as_usize() * item_size),
4076 count: write_count.sub(count)?,
4077 handle: write_handle,
4078 };
4079 }
4080
4081 if read_complete {
4082 ReturnCode::completed(ty.kind(), count)
4083 } else {
4084 set_guest_ready(concurrent_state)?;
4085 ReturnCode::Blocked
4086 }
4087 }
4088
4089 WriteState::HostReady {
4090 produce,
4091 try_into,
4092 guest_offset,
4093 cancel,
4094 cancel_waker,
4095 } => {
4096 if cancel_waker.is_some() {
4097 bail_bug!("expected cancel_waker to be none");
4098 }
4099 if cancel {
4100 bail_bug!("expected cancel to be false");
4101 }
4102 if guest_offset != 0 {
4103 bail_bug!("expected guest_offset to be 0");
4104 }
4105
4106 set_guest_ready(concurrent_state)?;
4107
4108 let code = self.produce(
4109 store.0,
4110 ty.kind(),
4111 transmit_id,
4112 produce,
4113 try_into,
4114 ItemCount::ZERO,
4115 false,
4116 )?;
4117
4118 if let (TransmitIndex::Future(_), ReturnCode::Completed(_)) = (ty, code) {
4119 store.0.concurrent_state_mut()?.get_mut(transmit_id)?.done = true;
4120 }
4121
4122 code
4123 }
4124
4125 WriteState::Open => {
4126 set_guest_ready(concurrent_state)?;
4127 ReturnCode::Blocked
4128 }
4129
4130 WriteState::Dropped => {
4131 if let TransmitIndex::Future(_) = ty {
4132 bail_bug!(
4133 "should not be possible to read from a future whose write end was dropped"
4134 );
4135 }
4136
4137 match Waitable::Transmit(transmit_handle).take_event(concurrent_state)? {
4138 Some(Event::StreamRead {
4139 code: ReturnCode::Dropped(ItemCount::ZERO),
4140 ..
4141 }) => {}
4142 event => bail_bug!("expected pending dropped event for reader; got {event:?}"),
4143 }
4144
4145 ReturnCode::Dropped(ItemCount::ZERO)
4146 }
4147 };
4148
4149 if result == ReturnCode::Blocked && !async_ {
4150 result = self.wait_for_read(store.0, caller_instance, transmit_handle)?;
4151 }
4152
4153 if result != ReturnCode::Blocked {
4154 *self.id().get_mut(store.0).get_mut_by_index(ty, handle)?.1 =
4155 VMTransmitLocalState::read(matches!(
4156 (result, ty),
4157 (ReturnCode::Dropped(_), TransmitIndex::Stream(_))
4158 ));
4159 }
4160
4161 log::trace!(
4162 "guest_read result for {transmit_handle:?} (handle {handle}; state {transmit_id:?}): {result:?}",
4163 );
4164
4165 Ok(result)
4166 }
4167
4168 fn wait_for_write(
4169 self,
4170 store: &mut StoreOpaque,
4171 caller: RuntimeComponentInstanceIndex,
4172 handle: TableId<TransmitHandle>,
4173 ) -> Result<ReturnCode> {
4174 let waitable = Waitable::Transmit(handle);
4175 store.wait_for_event(self.runtime_instance(caller), waitable)?;
4176 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4177 if let Some(event @ (Event::StreamWrite { code, .. } | Event::FutureWrite { code, .. })) =
4178 event
4179 {
4180 waitable.on_delivery(store, self, event)?;
4181 Ok(code)
4182 } else {
4183 bail_bug!("expected either a stream or future write event")
4184 }
4185 }
4186
4187 fn cancel_write(
4189 self,
4190 store: &mut StoreOpaque,
4191 caller: RuntimeComponentInstanceIndex,
4192 transmit_id: TableId<TransmitState>,
4193 async_: bool,
4194 ) -> Result<ReturnCode> {
4195 let state = store.concurrent_state_mut()?;
4196 let transmit = state.get_mut(transmit_id)?;
4197 log::trace!(
4198 "host_cancel_write state {transmit_id:?}; write state {:?} read state {:?}",
4199 transmit.read,
4200 transmit.write
4201 );
4202 let waitable = Waitable::Transmit(transmit.write_handle);
4203
4204 waitable.trap_if_has_sync_waiter(state)?;
4205 if !async_ {
4206 waitable.trap_if_in_waitable_set(state)?;
4207 }
4208
4209 let code = if let Some(event) = waitable.take_event(state)? {
4210 let (Event::FutureWrite { code, .. } | Event::StreamWrite { code, .. }) = event else {
4211 bail_bug!("expected either a stream or future write event")
4212 };
4213 waitable.on_delivery(store, self, event)?;
4214 match (code, event) {
4215 (ReturnCode::Completed(count), Event::StreamWrite { .. }) => {
4216 ReturnCode::Cancelled(count)
4217 }
4218 (ReturnCode::Dropped(_) | ReturnCode::Completed(_), _) => code,
4219 _ => bail_bug!("unexpected code/event combo"),
4220 }
4221 } else if let ReadState::HostReady {
4222 cancel,
4223 cancel_waker,
4224 ..
4225 } = &mut state.get_mut(transmit_id)?.read
4226 {
4227 *cancel = true;
4228 if let Some(waker) = cancel_waker.take() {
4229 waker.wake();
4230 }
4231
4232 if async_ {
4233 ReturnCode::Blocked
4234 } else {
4235 let handle = store
4236 .concurrent_state_mut()?
4237 .get_mut(transmit_id)?
4238 .write_handle;
4239 self.wait_for_write(store, caller, handle)?
4240 }
4241 } else {
4242 ReturnCode::Cancelled(ItemCount::ZERO)
4243 };
4244
4245 if !matches!(code, ReturnCode::Blocked) {
4246 let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
4247
4248 match &transmit.write {
4249 WriteState::GuestReady { .. } => {
4250 transmit.write = WriteState::Open;
4251 }
4252 WriteState::HostReady { .. } => bail_bug!("support host write cancellation"),
4253 WriteState::Open | WriteState::Dropped => {}
4254 }
4255 }
4256
4257 log::trace!("cancelled write {transmit_id:?}: {code:?}");
4258
4259 Ok(code)
4260 }
4261
4262 fn wait_for_read(
4263 self,
4264 store: &mut StoreOpaque,
4265 caller: RuntimeComponentInstanceIndex,
4266 handle: TableId<TransmitHandle>,
4267 ) -> Result<ReturnCode> {
4268 let waitable = Waitable::Transmit(handle);
4269 store.wait_for_event(self.runtime_instance(caller), waitable)?;
4270 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4271 if let Some(event @ (Event::StreamRead { code, .. } | Event::FutureRead { code, .. })) =
4272 event
4273 {
4274 waitable.on_delivery(store, self, event)?;
4275 Ok(code)
4276 } else {
4277 bail_bug!("expected either a stream or future read event")
4278 }
4279 }
4280
4281 fn cancel_read(
4283 self,
4284 store: &mut StoreOpaque,
4285 caller: RuntimeComponentInstanceIndex,
4286 transmit_id: TableId<TransmitState>,
4287 async_: bool,
4288 ) -> Result<ReturnCode> {
4289 let state = store.concurrent_state_mut()?;
4290 let transmit = state.get_mut(transmit_id)?;
4291 log::trace!(
4292 "host_cancel_read state {transmit_id:?}; read state {:?} write state {:?}",
4293 transmit.read,
4294 transmit.write
4295 );
4296
4297 let waitable = Waitable::Transmit(transmit.read_handle);
4298
4299 waitable.trap_if_has_sync_waiter(state)?;
4300 if !async_ {
4301 waitable.trap_if_in_waitable_set(state)?;
4302 }
4303
4304 let code = if let Some(event) = waitable.take_event(state)? {
4305 let (Event::FutureRead { code, .. } | Event::StreamRead { code, .. }) = event else {
4306 bail_bug!("expected either a stream or future read event")
4307 };
4308 waitable.on_delivery(store, self, event)?;
4309 match (code, event) {
4310 (ReturnCode::Completed(count), Event::StreamRead { .. }) => {
4311 ReturnCode::Cancelled(count)
4312 }
4313 (ReturnCode::Dropped(_) | ReturnCode::Completed(_), _) => code,
4314 _ => bail_bug!("unexpected code/event combo"),
4315 }
4316 } else if let WriteState::HostReady {
4317 cancel,
4318 cancel_waker,
4319 ..
4320 } = &mut state.get_mut(transmit_id)?.write
4321 {
4322 *cancel = true;
4323 if let Some(waker) = cancel_waker.take() {
4324 waker.wake();
4325 }
4326
4327 if async_ {
4328 ReturnCode::Blocked
4329 } else {
4330 let handle = store
4331 .concurrent_state_mut()?
4332 .get_mut(transmit_id)?
4333 .read_handle;
4334 self.wait_for_read(store, caller, handle)?
4335 }
4336 } else {
4337 ReturnCode::Cancelled(ItemCount::ZERO)
4338 };
4339
4340 if !matches!(code, ReturnCode::Blocked) {
4341 let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
4342
4343 match &transmit.read {
4344 ReadState::GuestReady { .. } => {
4345 transmit.read = ReadState::Open;
4346 }
4347 ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {
4348 bail_bug!("support host read cancellation")
4349 }
4350 ReadState::Open | ReadState::Dropped => {}
4351 }
4352 }
4353
4354 log::trace!("cancelled read {transmit_id:?}: {code:?}");
4355
4356 Ok(code)
4357 }
4358
4359 fn guest_cancel_write(
4361 self,
4362 store: &mut StoreOpaque,
4363 caller: RuntimeComponentInstanceIndex,
4364 ty: TransmitIndex,
4365 async_: bool,
4366 writer: u32,
4367 ) -> Result<ReturnCode> {
4368 let (rep, state) =
4369 get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, writer)?;
4370 let id = TableId::<TransmitHandle>::new(rep);
4371 log::trace!("guest cancel write {id:?} (handle {writer})");
4372
4373 match state.kind() {
4375 TransmitLocalStateKind::Write { .. } => {
4376 bail!("stream or future write cancelled when no write is pending")
4377 }
4378 TransmitLocalStateKind::Read { .. } => {
4379 bail!("passed read end to `{{stream|future}}.cancel-write`")
4380 }
4381 TransmitLocalStateKind::Cancelling => {
4382 bail!("stream or future write cancelled when a cancellation is already pending")
4383 }
4384 TransmitLocalStateKind::Busy => {}
4385 }
4386
4387 let state = store.concurrent_state_mut()?;
4390 let transmit_id = state.get_mut(id)?.state;
4391 if state.get_mut(transmit_id)?.write_handle != id {
4392 bail!("passed read end to `{{stream|future}}.cancel-write`")
4393 }
4394
4395 let code = self.cancel_write(store, caller, transmit_id, async_)?;
4396 let state =
4397 get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, writer)?.1;
4398 if let TransmitLocalStateKind::Busy = state.kind() {
4399 *state = if matches!(code, ReturnCode::Blocked) {
4400 VMTransmitLocalState::cancelling()
4401 } else {
4402 VMTransmitLocalState::write(false)
4403 };
4404 }
4405 Ok(code)
4406 }
4407
4408 fn guest_cancel_read(
4410 self,
4411 store: &mut StoreOpaque,
4412 caller: RuntimeComponentInstanceIndex,
4413 ty: TransmitIndex,
4414 async_: bool,
4415 reader: u32,
4416 ) -> Result<ReturnCode> {
4417 let (rep, state) =
4418 get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, reader)?;
4419 let id = TableId::<TransmitHandle>::new(rep);
4420 log::trace!("guest cancel read {id:?} (handle {reader})");
4421
4422 match state.kind() {
4424 TransmitLocalStateKind::Read { .. } => {
4425 bail!("stream or future read cancelled when no read is pending")
4426 }
4427 TransmitLocalStateKind::Write { .. } => {
4428 bail!("passed write end to `{{stream|future}}.cancel-read`")
4429 }
4430 TransmitLocalStateKind::Cancelling => {
4431 bail!("stream or future read cancelled when a cancellation is already pending")
4432 }
4433 TransmitLocalStateKind::Busy => {}
4434 }
4435
4436 let state = store.concurrent_state_mut()?;
4439 let transmit_id = state.get_mut(id)?.state;
4440 if state.get_mut(transmit_id)?.read_handle != id {
4441 bail!("passed write end to `{{stream|future}}.cancel-read`")
4442 }
4443
4444 let code = self.cancel_read(store, caller, transmit_id, async_)?;
4445 let state =
4446 get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, reader)?.1;
4447 if let TransmitLocalStateKind::Busy = state.kind() {
4448 *state = if matches!(code, ReturnCode::Blocked) {
4449 VMTransmitLocalState::cancelling()
4450 } else {
4451 VMTransmitLocalState::read(false)
4452 };
4453 }
4454 Ok(code)
4455 }
4456
4457 fn guest_drop_readable(
4459 self,
4460 store: &mut StoreOpaque,
4461 ty: TransmitIndex,
4462 reader: u32,
4463 ) -> Result<()> {
4464 let table = self.id().get_mut(store).table_for_transmit(ty);
4465 let (rep, _is_done) = match ty {
4466 TransmitIndex::Stream(ty) => table.stream_remove_readable(ty, reader)?,
4467 TransmitIndex::Future(ty) => table.future_remove_readable(ty, reader)?,
4468 };
4469 let kind = match ty {
4470 TransmitIndex::Stream(_) => TransmitKind::Stream,
4471 TransmitIndex::Future(_) => TransmitKind::Future,
4472 };
4473 let id = TableId::<TransmitHandle>::new(rep);
4474 log::trace!("guest_drop_readable: drop reader {id:?}");
4475 store.host_drop_reader(id, kind)
4476 }
4477
4478 pub(crate) fn error_context_new(
4480 self,
4481 store: &mut StoreOpaque,
4482 ty: TypeComponentLocalErrorContextTableIndex,
4483 options: OptionsIndex,
4484 debug_msg_address: u32,
4485 debug_msg_len: u32,
4486 ) -> Result<u32> {
4487 let lift_ctx = &mut LiftContext::new(store, options, self)?;
4488 let debug_msg = String::linear_lift_from_flat(
4489 lift_ctx,
4490 InterfaceType::String,
4491 &[ValRaw::u32(debug_msg_address), ValRaw::u32(debug_msg_len)],
4492 )?;
4493
4494 let err_ctx = ErrorContextState { debug_msg };
4496 let state = store.concurrent_state_mut()?;
4497 let table_id = state.push(err_ctx)?;
4498 let global_ref_count_idx =
4499 TypeComponentGlobalErrorContextTableIndex::from_u32(table_id.rep());
4500
4501 let _ = state
4503 .global_error_context_ref_counts
4504 .insert(global_ref_count_idx, GlobalErrorContextRefCount(1));
4505
4506 let (instance, counter) = self.id().get_mut_and_handle_counter(store);
4513 let local_idx = instance
4514 .error_context_table_insertion(ty, counter)
4515 .error_context(table_id.rep())?;
4516
4517 Ok(local_idx)
4518 }
4519
4520 pub(super) fn error_context_debug_message<T>(
4522 self,
4523 store: StoreContextMut<T>,
4524 ty: TypeComponentLocalErrorContextTableIndex,
4525 options: OptionsIndex,
4526 err_ctx_handle: u32,
4527 debug_msg_address: u32,
4528 ) -> Result<()> {
4529 let handle_table_id_rep = self
4531 .id()
4532 .get_mut(store.0)
4533 .table_for_error_context(ty)
4534 .error_context_rep(err_ctx_handle)?;
4535
4536 let state = store.0.concurrent_state_mut()?;
4537 let ErrorContextState { debug_msg } =
4539 state.get_mut(TableId::<ErrorContextState>::new(handle_table_id_rep))?;
4540 let debug_msg = debug_msg.clone();
4541
4542 let lower_cx = &mut LowerContext::new(store, options, self);
4543 let debug_msg_address = usize::try_from(debug_msg_address)?;
4544 if debug_msg_address % usize::try_from(<str as ComponentType>::ALIGN32)? != 0 {
4545 bail!("invalid debug message pointer: not aligned");
4546 }
4547 let offset = lower_cx
4553 .as_slice()
4554 .get(debug_msg_address..)
4555 .and_then(|b| b.get(..8))
4556 .map(|_| debug_msg_address)
4557 .ok_or_else(|| crate::format_err!("invalid debug message pointer: out of bounds"))?;
4558 debug_msg
4559 .as_str()
4560 .linear_lower_to_memory(lower_cx, InterfaceType::String, offset)?;
4561
4562 Ok(())
4563 }
4564
4565 pub(crate) fn future_cancel_read(
4567 self,
4568 store: &mut StoreOpaque,
4569 caller: RuntimeComponentInstanceIndex,
4570 ty: TypeFutureTableIndex,
4571 async_: bool,
4572 reader: u32,
4573 ) -> Result<u32> {
4574 self.guest_cancel_read(store, caller, TransmitIndex::Future(ty), async_, reader)
4575 .map(|v| v.encode())
4576 }
4577
4578 pub(crate) fn future_cancel_write(
4580 self,
4581 store: &mut StoreOpaque,
4582 caller: RuntimeComponentInstanceIndex,
4583 ty: TypeFutureTableIndex,
4584 async_: bool,
4585 writer: u32,
4586 ) -> Result<u32> {
4587 self.guest_cancel_write(store, caller, TransmitIndex::Future(ty), async_, writer)
4588 .map(|v| v.encode())
4589 }
4590
4591 pub(crate) fn stream_cancel_read(
4593 self,
4594 store: &mut StoreOpaque,
4595 caller: RuntimeComponentInstanceIndex,
4596 ty: TypeStreamTableIndex,
4597 async_: bool,
4598 reader: u32,
4599 ) -> Result<u32> {
4600 self.guest_cancel_read(store, caller, TransmitIndex::Stream(ty), async_, reader)
4601 .map(|v| v.encode())
4602 }
4603
4604 pub(crate) fn stream_cancel_write(
4606 self,
4607 store: &mut StoreOpaque,
4608 caller: RuntimeComponentInstanceIndex,
4609 ty: TypeStreamTableIndex,
4610 async_: bool,
4611 writer: u32,
4612 ) -> Result<u32> {
4613 self.guest_cancel_write(store, caller, TransmitIndex::Stream(ty), async_, writer)
4614 .map(|v| v.encode())
4615 }
4616
4617 pub(crate) fn future_drop_readable(
4619 self,
4620 store: &mut StoreOpaque,
4621 ty: TypeFutureTableIndex,
4622 reader: u32,
4623 ) -> Result<()> {
4624 self.guest_drop_readable(store, TransmitIndex::Future(ty), reader)
4625 }
4626
4627 pub(crate) fn stream_drop_readable(
4629 self,
4630 store: &mut StoreOpaque,
4631 ty: TypeStreamTableIndex,
4632 reader: u32,
4633 ) -> Result<()> {
4634 self.guest_drop_readable(store, TransmitIndex::Stream(ty), reader)
4635 }
4636
4637 fn guest_new(self, store: &mut StoreOpaque, ty: TransmitIndex) -> Result<ResourcePair> {
4641 let (write, read) = store
4642 .concurrent_state_mut()?
4643 .new_transmit(TransmitOrigin::guest(self.id().instance(), ty))?;
4644
4645 let (instance, counter) = self.id().get_mut_and_handle_counter(store);
4646 let mut insertion = instance.transmit_table_insertion(ty, counter);
4647 let (read_handle, write_handle) = match ty {
4648 TransmitIndex::Future(ty) => (
4649 insertion.future_read(ty, read.rep())?,
4650 insertion.future_write(ty, write.rep())?,
4651 ),
4652 TransmitIndex::Stream(ty) => (
4653 insertion.stream_read(ty, read.rep())?,
4654 insertion.stream_write(ty, write.rep())?,
4655 ),
4656 };
4657
4658 let state = store.concurrent_state_mut()?;
4659 state.get_mut(read)?.common.handle = Some(read_handle);
4660 state.get_mut(write)?.common.handle = Some(write_handle);
4661
4662 Ok(ResourcePair {
4663 write: write_handle,
4664 read: read_handle,
4665 })
4666 }
4667
4668 pub(crate) fn error_context_drop(
4670 self,
4671 store: &mut StoreOpaque,
4672 ty: TypeComponentLocalErrorContextTableIndex,
4673 error_context: u32,
4674 ) -> Result<()> {
4675 let instance = self.id().get_mut(store);
4676
4677 let local_handle_table = instance.table_for_error_context(ty);
4678
4679 let rep = local_handle_table.error_context_drop(error_context)?;
4680
4681 let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(rep);
4682
4683 let state = store.concurrent_state_mut()?;
4684 let Some(GlobalErrorContextRefCount(global_ref_count)) = state
4685 .global_error_context_ref_counts
4686 .get_mut(&global_ref_count_idx)
4687 else {
4688 bail_bug!("retrieve concurrent state for error context during drop")
4689 };
4690
4691 if *global_ref_count < 1 {
4693 bail_bug!("ref count unexpectedly zero");
4694 }
4695 *global_ref_count -= 1;
4696 if *global_ref_count == 0 {
4697 state
4698 .global_error_context_ref_counts
4699 .remove(&global_ref_count_idx);
4700
4701 state
4702 .delete(TableId::<ErrorContextState>::new(rep))
4703 .context("deleting component-global error context data")?;
4704 }
4705
4706 Ok(())
4707 }
4708
4709 fn guest_transfer(
4717 self,
4718 store: &mut StoreOpaque,
4719 src_idx: u32,
4720 src: TransmitIndex,
4721 dst: TransmitIndex,
4722 ) -> Result<u32> {
4723 let id = self.lift_index_to_transmit(store, src, src_idx)?;
4724 self.lower_transmit_to_index(store, dst, id)
4725 }
4726
4727 fn lift_index_to_transmit(
4728 self,
4729 store: &mut StoreOpaque,
4730 ty: TransmitIndex,
4731 src_idx: u32,
4732 ) -> Result<TableId<TransmitHandle>> {
4733 let (state, _, _, _, instance) = store.lift_context_parts(self);
4734 lift_index_to_transmit(instance, state.concurrent_state_mut(), ty, src_idx)
4735 }
4736
4737 fn lower_transmit_to_index(
4740 self,
4741 store: &mut StoreOpaque,
4742 ty: TransmitIndex,
4743 id: TableId<TransmitHandle>,
4744 ) -> Result<u32> {
4745 let (state, _, counter, _, instance) = store.lift_context_parts(self);
4746 lower_transmit_to_index(instance, counter, state.concurrent_state_mut(), ty, id)
4747 }
4748
4749 pub(crate) fn future_new(
4751 self,
4752 store: &mut StoreOpaque,
4753 ty: TypeFutureTableIndex,
4754 ) -> Result<ResourcePair> {
4755 self.guest_new(store, TransmitIndex::Future(ty))
4756 }
4757
4758 pub(crate) fn stream_new(
4760 self,
4761 store: &mut StoreOpaque,
4762 ty: TypeStreamTableIndex,
4763 ) -> Result<ResourcePair> {
4764 self.guest_new(store, TransmitIndex::Stream(ty))
4765 }
4766
4767 pub(crate) fn future_transfer(
4770 self,
4771 store: &mut StoreOpaque,
4772 src_idx: u32,
4773 src: TypeFutureTableIndex,
4774 dst: TypeFutureTableIndex,
4775 ) -> Result<u32> {
4776 self.guest_transfer(
4777 store,
4778 src_idx,
4779 TransmitIndex::Future(src),
4780 TransmitIndex::Future(dst),
4781 )
4782 }
4783
4784 pub(crate) fn stream_transfer(
4787 self,
4788 store: &mut StoreOpaque,
4789 src_idx: u32,
4790 src: TypeStreamTableIndex,
4791 dst: TypeStreamTableIndex,
4792 ) -> Result<u32> {
4793 self.guest_transfer(
4794 store,
4795 src_idx,
4796 TransmitIndex::Stream(src),
4797 TransmitIndex::Stream(dst),
4798 )
4799 }
4800
4801 pub(crate) fn error_context_transfer(
4803 self,
4804 store: &mut StoreOpaque,
4805 src_idx: u32,
4806 src: TypeComponentLocalErrorContextTableIndex,
4807 dst: TypeComponentLocalErrorContextTableIndex,
4808 ) -> Result<u32> {
4809 let (mut instance, counter) = self.id().get_mut_and_handle_counter(store);
4810 let rep = instance
4811 .as_mut()
4812 .table_for_error_context(src)
4813 .error_context_rep(src_idx)?;
4814 let dst_idx = instance
4815 .error_context_table_insertion(dst, counter)
4816 .error_context(rep)?;
4817
4818 let global_ref_count = store
4822 .concurrent_state_mut()?
4823 .global_error_context_ref_counts
4824 .get_mut(&TypeComponentGlobalErrorContextTableIndex::from_u32(rep))
4825 .context("global ref count present for existing (sub)component error context")?;
4826
4827 global_ref_count.0 = global_ref_count
4828 .0
4829 .checked_add(1)
4830 .ok_or_else(|| format_err!(Trap::ReferenceCountOverflow))?;
4831
4832 Ok(dst_idx)
4833 }
4834}
4835
4836fn lift_index_to_transmit(
4842 instance: Pin<&mut ComponentInstance>,
4843 concurrent_state: &mut ConcurrentState,
4844 ty: TransmitIndex,
4845 src_idx: u32,
4846) -> Result<TableId<TransmitHandle>> {
4847 let handle_table = instance.table_for_transmit(ty);
4848 let (rep, is_done) = match ty {
4849 TransmitIndex::Future(idx) => handle_table.future_remove_readable(idx, src_idx)?,
4850 TransmitIndex::Stream(idx) => handle_table.stream_remove_readable(idx, src_idx)?,
4851 };
4852 let desc = match ty {
4853 TransmitIndex::Future(_) => "future",
4854 TransmitIndex::Stream(_) => "stream",
4855 };
4856 if is_done {
4857 bail!("cannot lift {desc} after being notified that the writable end dropped");
4858 }
4859 let id = TableId::<TransmitHandle>::new(rep);
4860 let future = concurrent_state.get_mut(id)?;
4861 if future.common.set.is_some() {
4862 bail!("cannot lift {desc} while it's in a waitable set");
4863 }
4864 future.common.handle = None;
4865
4866 let state = future.state;
4867 if concurrent_state.get_mut(state)?.done {
4868 match ty {
4869 TransmitIndex::Future(_) => bail!("cannot lift {desc} after previous read succeeded"),
4870 TransmitIndex::Stream(_) => bail!(Trap::LiftDroppedStream),
4871 };
4872 }
4873
4874 Ok(id)
4875}
4876
4877fn lower_transmit_to_index(
4887 instance: Pin<&mut ComponentInstance>,
4888 counter: &mut HandleCounter,
4889 concurrent_state: &mut ConcurrentState,
4890 ty: TransmitIndex,
4891 id: TableId<TransmitHandle>,
4892) -> Result<u32> {
4893 let state = concurrent_state.get_mut(id)?.state;
4894 debug_assert_eq!(concurrent_state.get_mut(state)?.read_handle, id);
4895 let mut insertion = instance.transmit_table_insertion(ty, counter);
4896 let handle = match ty {
4897 TransmitIndex::Future(idx) => insertion.future_read(idx, id.rep()),
4898 TransmitIndex::Stream(idx) => insertion.stream_read(idx, id.rep()),
4899 }?;
4900 concurrent_state.get_mut(id)?.common.handle = Some(handle);
4901 Ok(handle)
4902}
4903
4904impl ComponentInstance {
4905 fn transmit_instance(&self, ty: TransmitIndex) -> RuntimeComponentInstanceIndex {
4906 let types = self.component().types();
4907 match ty {
4908 TransmitIndex::Stream(ty) => types[ty].instance,
4909 TransmitIndex::Future(ty) => types[ty].instance,
4910 }
4911 }
4912
4913 fn table_for_transmit(self: Pin<&mut Self>, ty: TransmitIndex) -> &mut HandleTable {
4914 let runtime_instance = self.transmit_instance(ty);
4915 self.instance_state(runtime_instance).handle_table()
4916 }
4917
4918 fn transmit_table_insertion<'a>(
4919 self: Pin<&'a mut Self>,
4920 ty: TransmitIndex,
4921 counter: &'a mut HandleCounter,
4922 ) -> HandleTableInsertion<'a> {
4923 let runtime_instance = self.transmit_instance(ty);
4924 self.handle_table_insertion(runtime_instance, counter)
4925 }
4926
4927 fn table_for_error_context(
4928 self: Pin<&mut Self>,
4929 ty: TypeComponentLocalErrorContextTableIndex,
4930 ) -> &mut HandleTable {
4931 let runtime_instance = self.component().types()[ty].instance;
4932 self.instance_state(runtime_instance).handle_table()
4933 }
4934
4935 fn error_context_table_insertion<'a>(
4936 self: Pin<&'a mut Self>,
4937 ty: TypeComponentLocalErrorContextTableIndex,
4938 counter: &'a mut HandleCounter,
4939 ) -> HandleTableInsertion<'a> {
4940 let runtime_instance = self.component().types()[ty].instance;
4941 self.handle_table_insertion(runtime_instance, counter)
4942 }
4943
4944 fn get_mut_by_index(
4945 self: Pin<&mut Self>,
4946 ty: TransmitIndex,
4947 index: u32,
4948 ) -> Result<(u32, &mut VMTransmitLocalState)> {
4949 get_mut_by_index_from(self.table_for_transmit(ty), ty, index)
4950 }
4951}
4952
4953impl ConcurrentState {
4954 fn send_write_result(
4955 &mut self,
4956 ty: TransmitIndex,
4957 id: TableId<TransmitState>,
4958 handle: u32,
4959 code: ReturnCode,
4960 ) -> Result<()> {
4961 let write_handle = self.get_mut(id)?.write_handle.rep();
4962 self.set_event(
4963 write_handle,
4964 match ty {
4965 TransmitIndex::Future(ty) => Event::FutureWrite {
4966 code,
4967 pending: Some((ty, handle)),
4968 },
4969 TransmitIndex::Stream(ty) => Event::StreamWrite {
4970 code,
4971 pending: Some((ty, handle)),
4972 },
4973 },
4974 )
4975 }
4976
4977 fn send_read_result(
4978 &mut self,
4979 ty: TransmitIndex,
4980 id: TableId<TransmitState>,
4981 handle: u32,
4982 code: ReturnCode,
4983 ) -> Result<()> {
4984 let read_handle = self.get_mut(id)?.read_handle.rep();
4985 self.set_event(
4986 read_handle,
4987 match ty {
4988 TransmitIndex::Future(ty) => Event::FutureRead {
4989 code,
4990 pending: Some((ty, handle)),
4991 },
4992 TransmitIndex::Stream(ty) => Event::StreamRead {
4993 code,
4994 pending: Some((ty, handle)),
4995 },
4996 },
4997 )
4998 }
4999
5000 fn take_event(&mut self, waitable: u32) -> Result<Option<Event>> {
5001 Waitable::Transmit(TableId::<TransmitHandle>::new(waitable)).take_event(self)
5002 }
5003
5004 fn set_event(&mut self, waitable: u32, event: Event) -> Result<()> {
5005 Waitable::Transmit(TableId::<TransmitHandle>::new(waitable)).set_event(self, Some(event))
5006 }
5007
5008 fn update_event(&mut self, waitable: u32, event: Event) -> Result<()> {
5019 let waitable = Waitable::Transmit(TableId::<TransmitHandle>::new(waitable));
5020
5021 fn update_code(old: ReturnCode, new: ReturnCode) -> Result<ReturnCode> {
5022 let (ReturnCode::Completed(count)
5023 | ReturnCode::Dropped(count)
5024 | ReturnCode::Cancelled(count)) = old
5025 else {
5026 bail_bug!("unexpected old return code")
5027 };
5028
5029 Ok(match new {
5030 ReturnCode::Dropped(ItemCount::ZERO) => ReturnCode::Dropped(count),
5031 ReturnCode::Cancelled(ItemCount::ZERO) => ReturnCode::Cancelled(count),
5032 _ => bail_bug!("unexpected new return code"),
5033 })
5034 }
5035
5036 let event = match (waitable.take_event(self)?, event) {
5037 (None, _) => event,
5038 (Some(old @ Event::FutureWrite { .. }), Event::FutureWrite { .. }) => old,
5039 (Some(old @ Event::FutureRead { .. }), Event::FutureRead { .. }) => old,
5040 (
5041 Some(Event::StreamWrite {
5042 code: old_code,
5043 pending: old_pending,
5044 }),
5045 Event::StreamWrite { code, pending },
5046 ) => Event::StreamWrite {
5047 code: update_code(old_code, code)?,
5048 pending: old_pending.or(pending),
5049 },
5050 (
5051 Some(Event::StreamRead {
5052 code: old_code,
5053 pending: old_pending,
5054 }),
5055 Event::StreamRead { code, pending },
5056 ) => Event::StreamRead {
5057 code: update_code(old_code, code)?,
5058 pending: old_pending.or(pending),
5059 },
5060 _ => bail_bug!("unexpected event combination"),
5061 };
5062
5063 waitable.set_event(self, Some(event))
5064 }
5065
5066 fn new_transmit(
5069 &mut self,
5070 origin: TransmitOrigin,
5071 ) -> Result<(TableId<TransmitHandle>, TableId<TransmitHandle>)> {
5072 self.table().ensure_capacity(3)?;
5075 let state_id = self.push(TransmitState::new(origin))?;
5076
5077 let write = self.push(TransmitHandle::new(state_id))?;
5078 let read = self.push(TransmitHandle::new(state_id))?;
5079
5080 let state = self.get_mut(state_id)?;
5081 state.write_handle = write;
5082 state.read_handle = read;
5083
5084 log::trace!("new transmit: state {state_id:?}; write {write:?}; read {read:?}",);
5085
5086 Ok((write, read))
5087 }
5088
5089 fn delete_transmit(&mut self, state_id: TableId<TransmitState>) -> Result<()> {
5091 let state = self.delete(state_id)?;
5092 self.delete(state.write_handle)?;
5093 self.delete(state.read_handle)?;
5094
5095 log::trace!(
5096 "delete transmit: state {state_id:?}; write {:?}; read {:?}",
5097 state.write_handle,
5098 state.read_handle,
5099 );
5100
5101 Ok(())
5102 }
5103}
5104
5105pub(crate) struct ResourcePair {
5106 pub(crate) write: u32,
5107 pub(crate) read: u32,
5108}
5109
5110impl Waitable {
5111 pub(super) fn on_delivery(
5114 &self,
5115 store: &mut StoreOpaque,
5116 instance: Instance,
5117 event: Event,
5118 ) -> Result<()> {
5119 if let Event::FutureRead {
5120 code: ReturnCode::Dropped(_),
5121 ..
5122 }
5123 | Event::FutureWrite {
5124 code: ReturnCode::Dropped(_),
5125 ..
5126 }
5127 | Event::StreamRead {
5128 code: ReturnCode::Dropped(_),
5129 ..
5130 }
5131 | Event::StreamWrite {
5132 code: ReturnCode::Dropped(_),
5133 ..
5134 } = event
5135 {
5136 let Waitable::Transmit(transmit_handle) = self else {
5137 bail_bug!("unexpected `{event:?}` for `{self:?}`");
5138 };
5139 let state = store.concurrent_state_mut()?;
5140 let transmit_id = state.get_mut(*transmit_handle)?.state;
5141 state.get_mut(transmit_id)?.done = true;
5142 }
5143
5144 let instance = instance.id().get_mut(store);
5145 let (rep, state, code) = match event {
5146 Event::FutureRead {
5147 pending: Some((ty, handle)),
5148 code,
5149 }
5150 | Event::FutureWrite {
5151 pending: Some((ty, handle)),
5152 code,
5153 } => {
5154 let runtime_instance = instance.component().types()[ty].instance;
5155 let (rep, state) = instance.instance_states().0[runtime_instance]
5156 .handle_table()
5157 .future_rep(ty, handle)?;
5158 (rep, state, code)
5159 }
5160 Event::StreamRead {
5161 pending: Some((ty, handle)),
5162 code,
5163 }
5164 | Event::StreamWrite {
5165 pending: Some((ty, handle)),
5166 code,
5167 } => {
5168 let runtime_instance = instance.component().types()[ty].instance;
5169 let (rep, state) = instance.instance_states().0[runtime_instance]
5170 .handle_table()
5171 .stream_rep(ty, handle)?;
5172 (rep, state, code)
5173 }
5174 _ => return Ok(()),
5175 };
5176 if rep != self.rep() {
5177 bail_bug!("unexpected rep mismatch");
5178 }
5179 if !matches!(
5180 state.kind(),
5181 TransmitLocalStateKind::Busy | TransmitLocalStateKind::Cancelling
5182 ) {
5183 bail_bug!("expected state to be busy or cancelling");
5184 }
5185 let done = matches!(code, ReturnCode::Dropped(_));
5186 *state = match event {
5187 Event::FutureRead { .. } | Event::StreamRead { .. } => VMTransmitLocalState::read(done),
5188 Event::FutureWrite { .. } | Event::StreamWrite { .. } => {
5189 VMTransmitLocalState::write(done)
5190 }
5191 _ => bail_bug!("unexpected event for stream"),
5192 };
5193
5194 let transmit_handle = TableId::<TransmitHandle>::new(rep);
5195 let state = store.concurrent_state_mut()?;
5196 let transmit_id = state.get_mut(transmit_handle)?.state;
5197 let transmit = state.get_mut(transmit_id)?;
5198
5199 match event {
5200 Event::StreamRead { .. } => {
5201 transmit.read = ReadState::Open;
5202 }
5203 Event::StreamWrite { .. } => transmit.write = WriteState::Open,
5204 _ => {}
5205 }
5206 Ok(())
5207 }
5208}
5209
5210fn allow_intra_component_read_write(ty: Option<&InterfaceType>) -> bool {
5214 matches!(
5215 ty,
5216 None | Some(
5217 InterfaceType::S8
5218 | InterfaceType::U8
5219 | InterfaceType::S16
5220 | InterfaceType::U16
5221 | InterfaceType::S32
5222 | InterfaceType::U32
5223 | InterfaceType::S64
5224 | InterfaceType::U64
5225 | InterfaceType::Float32
5226 | InterfaceType::Float64
5227 )
5228 )
5229}
5230
5231struct LockedState<T> {
5235 inner: TryMutex<Option<T>>,
5236}
5237
5238impl<T> LockedState<T> {
5239 fn new(value: T) -> Self {
5241 Self {
5242 inner: TryMutex::new(Some(value)),
5243 }
5244 }
5245
5246 fn try_lock(&self) -> Result<TryMutexGuard<'_, Option<T>>> {
5255 match self.inner.try_lock() {
5256 Some(lock) => Ok(lock),
5257 None => bail_bug!("should not have contention on state lock"),
5258 }
5259 }
5260
5261 fn take(&self) -> Result<LockedStateGuard<'_, T>> {
5268 let result = self.try_lock()?.take();
5269 match result {
5270 Some(result) => Ok(LockedStateGuard {
5271 value: ManuallyDrop::new(result),
5272 state: self,
5273 }),
5274 None => bail_bug!("lock value unexpectedly missing"),
5275 }
5276 }
5277
5278 fn with<R>(&self, f: impl FnOnce(&mut T) -> R) -> Result<R> {
5287 let mut inner = self.try_lock()?;
5288 match &mut *inner {
5289 Some(state) => Ok(f(state)),
5290 None => bail_bug!("lock value unexpectedly missing"),
5291 }
5292 }
5293}
5294
5295struct LockedStateGuard<'a, T> {
5298 value: ManuallyDrop<T>,
5299 state: &'a LockedState<T>,
5300}
5301
5302impl<T> Deref for LockedStateGuard<'_, T> {
5303 type Target = T;
5304
5305 fn deref(&self) -> &T {
5306 &self.value
5307 }
5308}
5309
5310impl<T> DerefMut for LockedStateGuard<'_, T> {
5311 fn deref_mut(&mut self) -> &mut T {
5312 &mut self.value
5313 }
5314}
5315
5316impl<T> Drop for LockedStateGuard<'_, T> {
5317 fn drop(&mut self) {
5318 let value = unsafe { ManuallyDrop::take(&mut self.value) };
5323
5324 if let Ok(mut lock) = self.state.try_lock() {
5328 *lock = Some(value);
5329 }
5330 }
5331}
5332
5333#[cfg(test)]
5334mod tests {
5335 use super::*;
5336 use crate::{Engine, Store};
5337 use core::future::pending;
5338 use core::pin::pin;
5339 use std::sync::LazyLock;
5340
5341 static ENGINE: LazyLock<Engine> = LazyLock::new(Engine::default);
5342
5343 fn poll_future_producer<T>(rx: Pin<&mut T>, finish: bool) -> Poll<Result<Option<T::Item>>>
5344 where
5345 T: FutureProducer<()>,
5346 {
5347 rx.poll_produce(
5348 &mut Context::from_waker(Waker::noop()),
5349 Store::new(&ENGINE, ()).as_context_mut(),
5350 finish,
5351 )
5352 }
5353
5354 #[test]
5355 fn future_producer() {
5356 let mut fut = pin!(async { crate::error::Ok(()) });
5357 assert!(matches!(
5358 poll_future_producer(fut.as_mut(), false),
5359 Poll::Ready(Ok(Some(()))),
5360 ));
5361
5362 let mut fut = pin!(async { crate::error::Ok(()) });
5363 assert!(matches!(
5364 poll_future_producer(fut.as_mut(), true),
5365 Poll::Ready(Ok(Some(()))),
5366 ));
5367
5368 let mut fut = pin!(pending::<Result<()>>());
5369 assert!(matches!(
5370 poll_future_producer(fut.as_mut(), false),
5371 Poll::Pending,
5372 ));
5373 assert!(matches!(
5374 poll_future_producer(fut.as_mut(), true),
5375 Poll::Ready(Ok(None)),
5376 ));
5377
5378 let (tx, rx) = oneshot::channel();
5379 let mut rx = pin!(rx);
5380 assert!(matches!(
5381 poll_future_producer(rx.as_mut(), false),
5382 Poll::Pending,
5383 ));
5384 assert!(matches!(
5385 poll_future_producer(rx.as_mut(), true),
5386 Poll::Ready(Ok(None)),
5387 ));
5388 tx.send(()).unwrap();
5389 assert!(matches!(
5390 poll_future_producer(rx.as_mut(), true),
5391 Poll::Ready(Ok(Some(()))),
5392 ));
5393
5394 let (tx, rx) = oneshot::channel();
5395 let mut rx = pin!(rx);
5396 tx.send(()).unwrap();
5397 assert!(matches!(
5398 poll_future_producer(rx.as_mut(), false),
5399 Poll::Ready(Ok(Some(()))),
5400 ));
5401
5402 let (tx, rx) = oneshot::channel::<()>();
5403 let mut rx = pin!(rx);
5404 drop(tx);
5405 assert!(matches!(
5406 poll_future_producer(rx.as_mut(), false),
5407 Poll::Ready(Err(..)),
5408 ));
5409
5410 let (tx, rx) = oneshot::channel::<()>();
5411 let mut rx = pin!(rx);
5412 drop(tx);
5413 assert!(matches!(
5414 poll_future_producer(rx.as_mut(), true),
5415 Poll::Ready(Err(..)),
5416 ));
5417 }
5418
5419 #[test]
5420 fn direct_destination_successive_writes() {
5421 use std::io::Write;
5422
5423 let mut store = Store::new(&ENGINE, ());
5424 let mut buffer = ();
5425 let mut dst = Vec::new();
5426 let mut marked_written = 0;
5427 let destination = Destination::<u8, ()> {
5428 id: TableId::new(0),
5429 buffer: &mut buffer,
5430 host_buffer: Some(HostBuffer {
5431 dst: &mut dst,
5432 marked_written: &mut marked_written,
5433 }),
5434 _phantom: PhantomData,
5435 };
5436
5437 let mut direct = destination.as_direct(store.as_context_mut(), 6);
5440 direct.write_all(&[1, 2, 3]).unwrap();
5441 direct.write_all(&[4, 5, 6]).unwrap();
5442 drop(direct);
5443
5444 assert_eq!(marked_written, 6);
5445 assert_eq!(dst, [1, 2, 3, 4, 5, 6]);
5446 }
5447}