Skip to main content

wasmtime/runtime/component/concurrent/
futures_and_streams.rs

1use super::table::{TableDebug, TableId};
2use super::{Event, GlobalErrorContextRefCount, Waitable, WaitableCommon};
3use crate::component::concurrent::{ConcurrentState, CurrentThread, WorkItem, tls};
4use crate::component::func::{self, LiftContext, LowerContext};
5use crate::component::matching::InstanceType;
6use crate::component::resources::{HostResourceIndex, HostResourceTables};
7use crate::component::types;
8use crate::component::values::ErrorContextAny;
9use crate::component::{
10    AsAccessor, ComponentInstanceId, ComponentType, FutureAny, Instance, Lift, Lower,
11    RuntimeInstance, StreamAny, Val, WasmList,
12};
13use crate::prelude::*;
14use crate::store::{StoreOpaque, StoreToken};
15use crate::try_mutex::{TryMutex, TryMutexGuard};
16use crate::vm::VMTransmitLocalState;
17use crate::vm::component::TransmitLocalStateKind;
18use crate::vm::component::{ComponentInstance, HandleCounter, HandleTable, HandleTableInsertion};
19use crate::vm::{AlwaysMut, VMStore};
20use crate::{AsContext, AsContextMut, StoreContextMut, ValRaw};
21use crate::{
22    Error, Result, Trap, bail, bail_bug, ensure,
23    error::{Context as _, format_err},
24};
25use alloc::sync::Arc;
26use buffers::{Extender, SliceBuffer, UntypedWriteBuffer};
27use core::any::{Any, TypeId};
28use core::fmt;
29use core::future;
30use core::iter;
31use core::marker::PhantomData;
32use core::mem::{self, ManuallyDrop, MaybeUninit};
33use core::ops::{Deref, DerefMut};
34use core::pin::Pin;
35use core::task::{Context, Poll, Waker, ready};
36use futures::channel::oneshot;
37use futures::{FutureExt as _, stream};
38use wasmtime_environ::component::{
39    CanonicalAbiInfo, ComponentTypes, InterfaceType, OptionsIndex, RuntimeComponentInstanceIndex,
40    TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
41    TypeFutureTableIndex, TypeStreamTableIndex,
42};
43
44pub use buffers::{ReadBuffer, VecBuffer, WriteBuffer};
45
46mod buffers;
47
48/// Enum for distinguishing between a stream or future in functions that handle
49/// both.
50#[derive(Copy, Clone, Debug)]
51pub enum TransmitKind {
52    Stream,
53    Future,
54}
55
56/// Represents `{stream,future}.{read,write}` results.
57#[derive(Copy, Clone, Debug, PartialEq)]
58pub enum ReturnCode {
59    Blocked,
60    Completed(ItemCount),
61    Dropped(ItemCount),
62    Cancelled(ItemCount),
63}
64
65impl ReturnCode {
66    /// Pack `self` into a single 32-bit integer that may be returned to the
67    /// guest.
68    ///
69    /// This corresponds to `pack_copy_result` in the Component Model spec.
70    pub fn encode(&self) -> u32 {
71        const BLOCKED: u32 = 0xffff_ffff;
72        const COMPLETED: u32 = 0x0;
73        const DROPPED: u32 = 0x1;
74        const CANCELLED: u32 = 0x2;
75        match self {
76            ReturnCode::Blocked => BLOCKED,
77            ReturnCode::Completed(n) => (n.as_u32() << 4) | COMPLETED,
78            ReturnCode::Dropped(n) => (n.as_u32() << 4) | DROPPED,
79            ReturnCode::Cancelled(n) => (n.as_u32() << 4) | CANCELLED,
80        }
81    }
82
83    /// Returns `Self::Completed` with the specified count (or zero if
84    /// `matches!(kind, TransmitKind::Future)`)
85    fn completed(kind: TransmitKind, count: ItemCount) -> Self {
86        Self::Completed(if let TransmitKind::Future = kind {
87            ItemCount::ZERO
88        } else {
89            count
90        })
91    }
92}
93
94/// Representation of how many items are being operated on in a stream read or
95/// write.
96///
97/// The component model requires that stream operations are limited to `1<<28`
98/// items in one go. This type is a newtype wrapper around `u32` with the
99/// invariant that the internal value is limited by this amount.
100#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
101#[repr(transparent)]
102pub struct ItemCount {
103    raw: u32,
104}
105
106impl ItemCount {
107    const MAX: u32 = 1 << 28;
108    const ZERO: ItemCount = ItemCount { raw: 0 };
109
110    /// Creates a new `ItemCount` with the specified count, or a trap if it's
111    /// too large.
112    fn new(count: u32) -> Result<Self, Trap> {
113        if count < Self::MAX {
114            Ok(Self { raw: count })
115        } else {
116            Err(Trap::StreamOpTooBig)
117        }
118    }
119
120    /// Same as `Self::new` but takes a `usize`.
121    fn new_usize(count: usize) -> Result<Self, Trap> {
122        let count = u32::try_from(count).map_err(|_| Trap::StreamOpTooBig)?;
123        Self::new(count)
124    }
125
126    fn as_u32(&self) -> u32 {
127        self.raw
128    }
129
130    fn as_usize(&self) -> usize {
131        usize::try_from(self.raw).unwrap()
132    }
133
134    /// Increments `self` by `amt`, returning a trap if the amount would exceed
135    /// the maximum item count.
136    fn inc(&mut self, amt: usize) -> Result<(), Trap> {
137        let amt = u32::try_from(amt).map_err(|_| Trap::StreamOpTooBig)?;
138        let new_raw = self.raw.checked_add(amt).ok_or(Trap::StreamOpTooBig)?;
139        if new_raw < Self::MAX {
140            self.raw = new_raw;
141            Ok(())
142        } else {
143            Err(Trap::StreamOpTooBig)
144        }
145    }
146
147    /// Helper to add two `ItemCount`s together, fallibly.
148    ///
149    /// It's considered a bug if this overflows, so this is only suitable in
150    /// situations where overflow and/or exceeding the total item count is known
151    /// that it may be possible.
152    fn add(&self, other: ItemCount) -> Result<ItemCount> {
153        match self.raw.checked_add(other.raw) {
154            Some(raw) => Ok(ItemCount::new(raw)?),
155            None => bail_bug!("overflow in `ItemCount::add`"),
156        }
157    }
158
159    /// Same as `add`, but for subtraction.
160    ///
161    /// Like with `add` this is only suitable for situations where the result is
162    /// known to not underflow.
163    fn sub(&self, other: ItemCount) -> Result<ItemCount> {
164        match self.raw.checked_sub(other.raw) {
165            Some(raw) => Ok(ItemCount { raw }),
166            None => bail_bug!("underflow in `ItemCount::sub`"),
167        }
168    }
169}
170
171impl fmt::Display for ItemCount {
172    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
173        self.raw.fmt(f)
174    }
175}
176
177impl fmt::Debug for ItemCount {
178    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
179        self.raw.fmt(f)
180    }
181}
182
183impl PartialEq<u32> for ItemCount {
184    fn eq(&self, other: &u32) -> bool {
185        self.raw == *other
186    }
187}
188
189impl PartialOrd<u32> for ItemCount {
190    fn partial_cmp(&self, other: &u32) -> Option<core::cmp::Ordering> {
191        self.raw.partial_cmp(other)
192    }
193}
194
195/// Represents a stream or future type index.
196///
197/// This is useful as a parameter type for functions which operate on either a
198/// future or a stream.
199#[derive(Copy, Clone, Debug)]
200pub enum TransmitIndex {
201    Stream(TypeStreamTableIndex),
202    Future(TypeFutureTableIndex),
203}
204
205impl TransmitIndex {
206    pub fn kind(&self) -> TransmitKind {
207        match self {
208            TransmitIndex::Stream(_) => TransmitKind::Stream,
209            TransmitIndex::Future(_) => TransmitKind::Future,
210        }
211    }
212
213    /// Retrieve the payload type of the specified stream or future, or `None`
214    /// if it has no payload type.
215    fn payload<'a>(&self, types: &'a ComponentTypes) -> Option<&'a InterfaceType> {
216        match self {
217            TransmitIndex::Stream(i) => {
218                let ty = types[*i].ty;
219                types[ty].payload.as_ref()
220            }
221            TransmitIndex::Future(i) => {
222                let ty = types[*i].ty;
223                types[ty].payload.as_ref()
224            }
225        }
226    }
227}
228
229/// Retrieve the host rep and state for the specified guest-visible waitable
230/// handle.
231fn get_mut_by_index_from(
232    handle_table: &mut HandleTable,
233    ty: TransmitIndex,
234    index: u32,
235) -> Result<(u32, &mut VMTransmitLocalState)> {
236    match ty {
237        TransmitIndex::Stream(ty) => handle_table.stream_rep(ty, index),
238        TransmitIndex::Future(ty) => handle_table.future_rep(ty, index),
239    }
240}
241
242/// Lowers items from the `buffer` of a host `{Stream,Future}Producer` `P` into
243/// a guest's linear memory, where `T` is `P::Item`.
244///
245/// `Lower` requires `T::typecheck` to pass for `ty`'s payload, but no check is
246/// done here. `ty` is the type of the read end's entry in the guest's handle
247/// table, so by the invariant on `lower_transmit_to_index`,
248/// `P::Item::typecheck` has already passed for `ty`'s payload.
249fn lower<T: func::Lower + Send + 'static, B: WriteBuffer<T>, U: 'static>(
250    mut store: StoreContextMut<U>,
251    instance: Instance,
252    options: OptionsIndex,
253    ty: TransmitIndex,
254    address: usize,
255    count: usize,
256    buffer: &mut B,
257) -> Result<()> {
258    let count = buffer.remaining().len().min(count);
259
260    let lower = &mut if T::MAY_REQUIRE_REALLOC {
261        LowerContext::new(store.as_context_mut(), options, instance)
262    } else {
263        LowerContext::new_without_realloc(store.as_context_mut(), options, instance)
264    };
265
266    // Note that validation only happens with operations that have a payload.
267    if let Some(ty) = ty.payload(lower.types).copied() {
268        if address % usize::try_from(T::ALIGN32)? != 0 {
269            bail!("read pointer not aligned");
270        }
271        lower
272            .as_slice()
273            .get(address..)
274            .and_then(|b| b.get(..T::SIZE32 * count))
275            .ok_or_else(|| crate::format_err!("read pointer out of bounds of memory"))?;
276
277        T::linear_store_list_to_memory(lower, ty, address, &buffer.remaining()[..count])?;
278    }
279
280    buffer.skip(count);
281
282    Ok(())
283}
284
285fn lift<T: func::Lift + Send + 'static, B: ReadBuffer<T>>(
286    lift: &mut LiftContext<'_>,
287    ty: Option<InterfaceType>,
288    buffer: &mut B,
289    address: usize,
290    count: usize,
291) -> Result<()> {
292    let count = count.min(buffer.remaining_capacity());
293    if T::IS_RUST_UNIT_TYPE {
294        // SAFETY: `T::IS_RUST_UNIT_TYPE` is only true for `()`, a
295        // zero-sized type, so `MaybeUninit::uninit().assume_init()`
296        // is a valid way to populate the zero-sized buffer.
297        buffer.extend(
298            iter::repeat_with(|| unsafe { MaybeUninit::uninit().assume_init() }).take(count),
299        )
300    } else {
301        let ty = match ty {
302            Some(ty) => ty,
303            None => bail_bug!("type required for non-unit lift"),
304        };
305        if address % usize::try_from(T::ALIGN32)? != 0 {
306            bail!("write pointer not aligned");
307        }
308        lift.memory()
309            .get(address..)
310            .and_then(|b| b.get(..T::SIZE32 * count))
311            .ok_or_else(|| crate::format_err!("write pointer out of bounds of memory"))?;
312
313        let list = &WasmList::new(address, count, lift, ty)?;
314        T::linear_lift_into_from_memory(lift, list, &mut Extender(buffer))?
315    }
316    Ok(())
317}
318
319/// Represents the state associated with an error context
320#[derive(Debug, PartialEq, Eq, PartialOrd)]
321pub(super) struct ErrorContextState {
322    /// Debug message associated with the error context
323    pub(crate) debug_msg: String,
324}
325
326/// Represents the size and alignment for a "flat" Component Model type,
327/// i.e. one containing no pointers or handles.
328#[derive(Debug, Clone, Copy, PartialEq, Eq)]
329pub(super) struct FlatAbi {
330    pub(super) size: u32,
331    pub(super) align: u32,
332}
333
334struct HostBuffer<'a> {
335    dst: &'a mut Vec<u8>,
336    marked_written: &'a mut usize,
337}
338
339impl HostBuffer<'_> {
340    fn reborrow(&mut self) -> HostBuffer<'_> {
341        HostBuffer {
342            dst: &mut *self.dst,
343            marked_written: &mut *self.marked_written,
344        }
345    }
346}
347
348/// Represents the buffer for a host- or guest-initiated stream read.
349pub struct Destination<'a, T, B> {
350    id: TableId<TransmitState>,
351    buffer: &'a mut B,
352    host_buffer: Option<HostBuffer<'a>>,
353    _phantom: PhantomData<fn() -> T>,
354}
355
356impl<'a, T, B> Destination<'a, T, B> {
357    /// Reborrow `self` so it can be used again later.
358    pub fn reborrow(&mut self) -> Destination<'_, T, B> {
359        Destination {
360            id: self.id,
361            buffer: &mut *self.buffer,
362            host_buffer: self.host_buffer.as_mut().map(|b| b.reborrow()),
363            _phantom: PhantomData,
364        }
365    }
366
367    /// Take the buffer out of `self`, leaving a default-initialized one in its
368    /// place.
369    ///
370    /// This can be useful for reusing the previously-stored buffer's capacity
371    /// instead of allocating a fresh one.
372    pub fn take_buffer(&mut self) -> B
373    where
374        B: Default,
375    {
376        mem::take(self.buffer)
377    }
378
379    /// Store the specified buffer in `self`.
380    ///
381    /// Any items contained in the buffer will be delivered to the reader after
382    /// the `StreamProducer::poll_produce` call to which this `Destination` was
383    /// passed returns (unless overwritten by another call to `set_buffer`).
384    ///
385    /// If items are stored via this buffer _and_ written via a
386    /// `DirectDestination` view of `self`, then the items in the buffer will be
387    /// delivered after the ones written using `DirectDestination`.
388    pub fn set_buffer(&mut self, buffer: B) {
389        *self.buffer = buffer;
390    }
391
392    /// Return the remaining number of items the current read has capacity to
393    /// accept, if known.
394    ///
395    /// This will return `Some(_)` if the reader is a guest; it will return
396    /// `None` if the reader is the host.
397    ///
398    /// Note that this can return `Some(0)`. This means that the guest is
399    /// attempting to perform a zero-length read which typically means that it's
400    /// trying to wait for this stream to be ready-to-read but is not actually
401    /// ready to receive the items yet. The host in this case is allowed to
402    /// either block waiting for readiness or immediately complete the
403    /// operation. The guest is expected to handle both cases. Some more
404    /// discussion about this case can be found in the discussion of ["Stream
405    /// Readiness" in the component-model repo][docs].
406    ///
407    /// [docs]: https://github.com/WebAssembly/component-model/blob/main/design/mvp/Concurrency.md#stream-readiness
408    pub fn remaining(&self, mut store: impl AsContextMut) -> Result<Option<usize>> {
409        self.remaining_(store.as_context_mut().0)
410    }
411
412    fn remaining_(&self, store: &mut StoreOpaque) -> Result<Option<usize>> {
413        let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;
414
415        if let &ReadState::GuestReady { count, .. } = &transmit.read {
416            let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
417                bail_bug!("expected WriteState::HostReady")
418            };
419
420            Ok(Some(count.as_usize() - guest_offset.as_usize()))
421        } else {
422            Ok(None)
423        }
424    }
425}
426
427impl<'a, B> Destination<'a, u8, B> {
428    /// Return a `DirectDestination` view of `self`.
429    ///
430    /// If the reader is a guest, this will provide direct access to the guest's
431    /// read buffer.  If the reader is a host, this will provide access to a
432    /// buffer which will be delivered to the host before any items stored using
433    /// `Destination::set_buffer`.
434    ///
435    /// `capacity` will only be used if the reader is a host, in which case it
436    /// will update the length of the buffer, possibly zero-initializing the new
437    /// elements if the new length is larger than the old length.
438    pub fn as_direct<D>(
439        mut self,
440        store: StoreContextMut<'a, D>,
441        capacity: usize,
442    ) -> DirectDestination<'a, D> {
443        if let Some(buffer) = &mut self.host_buffer {
444            *buffer.marked_written = 0;
445            buffer.dst.resize(capacity, 0);
446        }
447
448        DirectDestination {
449            id: self.id,
450            host_buffer: self.host_buffer,
451            store,
452        }
453    }
454}
455
456/// Represents a read from a `stream<u8>`, providing direct access to the
457/// writer's buffer.
458pub struct DirectDestination<'a, D: 'static> {
459    id: TableId<TransmitState>,
460    host_buffer: Option<HostBuffer<'a>>,
461    store: StoreContextMut<'a, D>,
462}
463
464#[cfg(feature = "std")]
465impl<D: 'static> std::io::Write for DirectDestination<'_, D> {
466    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
467        let rem = self.remaining().map_err(std::io::Error::other)?;
468        let n = rem.len().min(buf.len());
469        rem[..n].copy_from_slice(&buf[..n]);
470        self.mark_written(n);
471        Ok(n)
472    }
473
474    fn flush(&mut self) -> std::io::Result<()> {
475        Ok(())
476    }
477}
478
479impl<D: 'static> DirectDestination<'_, D> {
480    /// Provide direct access to the writer's buffer.
481    pub fn remaining(&mut self) -> Result<&mut [u8]> {
482        if let Some(buffer) = self.host_buffer.as_mut() {
483            return Ok(&mut buffer.dst[*buffer.marked_written..]);
484        }
485        let transmit = self
486            .store
487            .as_context_mut()
488            .0
489            .concurrent_state_mut()?
490            .get_mut(self.id)?;
491
492        let &ReadState::GuestReady {
493            address,
494            count,
495            options,
496            instance,
497            ..
498        } = &transmit.read
499        else {
500            bail_bug!("expected ReadState::GuestReady")
501        };
502
503        let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
504            bail_bug!("expected WriteState::HostReady")
505        };
506
507        let memory = instance.options_memory_range_mut(
508            self.store.0,
509            options,
510            address + guest_offset.as_usize(),
511            count.as_usize() - guest_offset.as_usize(),
512        );
513        match memory {
514            Some(memory) => Ok(memory),
515            None => bail_bug!("guest buffer unexpectedly out of bounds"),
516        }
517    }
518
519    /// Mark the specified number of bytes as written to the writer's buffer.
520    ///
521    /// # Panics
522    ///
523    /// This will panic if the count is larger than the size of the
524    /// buffer returned by `Self::remaining`.
525    pub fn mark_written(&mut self, count: usize) {
526        // Note that this unwrap should only trigger for bugs in Wasmtime, and
527        // this is modeled here to centralize the `.unwrap()` for this method in
528        // one location.
529        self.mark_written_(count).unwrap()
530    }
531
532    fn mark_written_(&mut self, count: usize) -> Result<()> {
533        if let Some(buffer) = self.host_buffer.as_mut() {
534            // Note that this `.unwrap` is a documented panic condition of
535            // `mark_written`.
536            *buffer.marked_written = buffer.marked_written.checked_add(count).unwrap();
537        } else {
538            let transmit = self
539                .store
540                .as_context_mut()
541                .0
542                .concurrent_state_mut()?
543                .get_mut(self.id)?;
544
545            let ReadState::GuestReady {
546                count: read_count, ..
547            } = &transmit.read
548            else {
549                bail_bug!("expected ReadState::GuestReady")
550            };
551
552            let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
553                bail_bug!("expected WriteState::HostReady");
554            };
555
556            if guest_offset.as_usize() + count > read_count.as_usize() {
557                // Note that this `panic` is a documented panic condition of
558                // `mark_written`.
559                panic!(
560                    "write count ({count}) must be less than or equal to read count ({read_count})"
561                )
562            } else {
563                guest_offset.inc(count)?;
564            }
565        }
566        Ok(())
567    }
568}
569
570/// Represents the state of a `Stream{Producer,Consumer}`.
571#[derive(Copy, Clone, Debug)]
572pub enum StreamResult {
573    /// The operation completed normally, and the producer or consumer may be
574    /// able to produce or consume more items, respectively.
575    Completed,
576    /// The operation was interrupted (i.e. it wrapped up early after receiving
577    /// a `finish` parameter value of true in a call to `poll_produce` or
578    /// `poll_consume`), and the producer or consumer may be able to produce or
579    /// consume more items, respectively.
580    Cancelled,
581    /// The operation completed normally, but the producer or consumer will
582    /// _not_ able to produce or consume more items, respectively.
583    Dropped,
584}
585
586/// Represents the host-owned write end of a stream.
587pub trait StreamProducer<D>: Send + 'static {
588    /// The payload type of this stream.
589    type Item;
590
591    /// The `WriteBuffer` type to use when delivering items.
592    type Buffer: WriteBuffer<Self::Item> + Default;
593
594    /// Handle a host- or guest-initiated read by delivering zero or more items
595    /// to the specified destination.
596    ///
597    /// This will be called whenever the reader starts a read.
598    ///
599    /// # Arguments
600    ///
601    /// * `self` - a `Pin`'d version of self to perform Rust-level
602    ///   future-related operations on.
603    /// * `cx` - a Rust-related [`Context`] which is passed to other
604    ///   future-related operations or used to acquire a waker.
605    /// * `store` - the Wasmtime store that this operation is happening within.
606    ///   Used, for example, to consult the state `D` associated with the store.
607    /// * `destination` - the location that items are to be written to.
608    /// * `finish` - a flag indicating whether the host should strive to
609    ///   immediately complete/cancel any pending operation. See below for more
610    ///   details.
611    ///
612    /// # Behavior
613    ///
614    /// If the implementation is able to produce one or more items immediately,
615    /// it should write them to `destination` and return either
616    /// `Poll::Ready(Ok(StreamResult::Completed))` if it expects to produce more
617    /// items, or `Poll::Ready(Ok(StreamResult::Dropped))` if it cannot produce
618    /// any more items.
619    ///
620    /// If the implementation is unable to produce any items immediately, but
621    /// expects to do so later, and `finish` is _false_, it should store the
622    /// waker from `cx` for later and return `Poll::Pending` without writing
623    /// anything to `destination`.  Later, it should alert the waker when either
624    /// the items arrive, the stream has ended, or an error occurs.
625    ///
626    /// If more items are written to `destination` than the reader has immediate
627    /// capacity to accept, they will be retained in memory by the caller and
628    /// used to satisfy future reads, in which case `poll_produce` will only be
629    /// called again once all those items have been delivered.
630    ///
631    /// # Zero-length reads
632    ///
633    /// This function may be called with a zero-length capacity buffer
634    /// (i.e. `Destination::remaining` returns `Some(0)`). This indicates that
635    /// the guest wants to wait to see if an item is ready without actually
636    /// reading the item. For example think of a UNIX `poll` function run on a
637    /// TCP stream, seeing if it's readable without actually reading it.
638    ///
639    /// In this situation the host is allowed to either return immediately or
640    /// wait for readiness. Note that waiting for readiness is not always
641    /// possible. For example it's impossible to test if a Rust-native `Future`
642    /// is ready without actually reading the item. Stream-specific
643    /// optimizations, such as testing if a TCP stream is readable, may be
644    /// possible however.
645    ///
646    /// For a zero-length read, the host is allowed to:
647    ///
648    /// - Return `Poll::Ready(Ok(StreamResult::Completed))` without writing
649    ///   anything if it expects to be able to produce items immediately (i.e.
650    ///   without first returning `Poll::Pending`) the next time `poll_produce`
651    ///   is called with non-zero capacity. This is the best-case scenario of
652    ///   fulfilling the guest's desire -- items aren't read/buffered but the
653    ///   host is saying it's ready when the guest is.
654    ///
655    /// - Return `Poll::Ready(Ok(StreamResult::Completed))` without actually
656    ///   testing for readiness. The guest doesn't know this yet, but the guest
657    ///   will realize that zero-length reads won't work on this stream when a
658    ///   subsequent nonzero read attempt is made which returns `Poll::Pending`
659    ///   here.
660    ///
661    /// - Return `Poll::Pending` if the host has performed necessary async work
662    ///   to wait for this stream to be readable without actually reading
663    ///   anything. This is also a best-case scenario where the host is letting
664    ///   the guest know that nothing is ready yet. Later the zero-length read
665    ///   will complete and then the guest will attempt a nonzero-length read to
666    ///   actually read some bytes.
667    ///
668    /// - Return `Poll::Ready(Ok(StreamResult::Completed))` after calling
669    ///   `Destination::set_buffer` with one more more items. Note, however,
670    ///   that this creates the hazard that the items will never be received by
671    ///   the guest if it decides not to do another non-zero-length read before
672    ///   closing the stream.  Moreover, if `Self::Item` is e.g. a
673    ///   `Resource<_>`, they may end up leaking in that scenario. It is not
674    ///   recommended to do this and it's better to return
675    ///   `StreamResult::Completed` without buffering anything instead.
676    ///
677    /// For more discussion on zero-length reads see the [documentation in the
678    /// component-model repo itself][docs].
679    ///
680    /// [docs]: https://github.com/WebAssembly/component-model/blob/main/design/mvp/Concurrency.md#stream-readiness
681    ///
682    /// # Return
683    ///
684    /// This function can return a number of possible cases from this function:
685    ///
686    /// * `Poll::Pending` - this operation cannot complete at this time. The
687    ///   Rust-level `Future::poll` contract applies here where a waker should
688    ///   be stored from the `cx` argument and be arranged to receive a
689    ///   notification when this implementation can make progress. For example
690    ///   if you call `Future::poll` on a sub-future, that's enough. If items
691    ///   were written to `destination` then a trap in the guest will be raised.
692    ///
693    ///   Note that implementations should strive to avoid this return value
694    ///   when `finish` is `true`. In such a situation the guest is attempting
695    ///   to, for example, cancel a previous operation. By returning
696    ///   `Poll::Pending` the guest will be blocked during the cancellation
697    ///   request. If `finish` is `true` then `StreamResult::Cancelled` is
698    ///   favored to indicate that no items were read. If a short read happened,
699    ///   however, it's ok to return `StreamResult::Completed` indicating some
700    ///   items were read.
701    ///
702    /// * `Poll::Ok(StreamResult::Completed)` - items, if applicable, were
703    ///   written to the `destination`. Only zero-length reads (see above) may
704    ///   complete without writing anything; if nothing was written for a
705    ///   nonzero-length read, or for any read by the host, then a trap in the
706    ///   guest will be raised.
707    ///
708    /// * `Poll::Ok(StreamResult::Cancelled)` - used when `finish` is `true` and
709    ///   the implementation was able to successfully cancel any async work that
710    ///   a previous read kicked off, if any. The host should not buffer values
711    ///   received after returning `Cancelled` because the guest will not be
712    ///   aware of these values and the guest could close the stream after
713    ///   cancelling a read. Hosts should only return `Cancelled` when there are
714    ///   no more async operations in flight for a previous read.
715    ///
716    ///   If items were written to `destination` then a trap in the guest will
717    ///   be raised. If `finish` is `false` then this return value will raise a
718    ///   trap in the guest.
719    ///
720    /// * `Poll::Ok(StreamResult::Dropped)` - end-of-stream marker, indicating
721    ///   that this producer should not be polled again. Note that items may
722    ///   still be written to `destination`.
723    ///
724    /// # Errors
725    ///
726    /// The implementation may alternatively choose to return `Err(_)` to
727    /// indicate an unrecoverable error. This will cause the guest (if any) to
728    /// trap and render the component instance (if any) unusable. The
729    /// implementation should report errors that _are_ recoverable by other
730    /// means (e.g. by writing to a `future`) and return
731    /// `Poll::Ready(Ok(StreamResult::Dropped))`.
732    fn poll_produce<'a>(
733        self: Pin<&mut Self>,
734        cx: &mut Context<'_>,
735        store: StoreContextMut<'a, D>,
736        destination: Destination<'a, Self::Item, Self::Buffer>,
737        finish: bool,
738    ) -> Poll<Result<StreamResult>>;
739
740    /// Attempt to convert the specified object into a `Box<dyn Any>` which may
741    /// be downcast to the specified type.
742    ///
743    /// `buffer` contains any items which this producer previously passed to
744    /// [`Destination::set_buffer`] but which have not yet been delivered to
745    /// the reader. If the implementation returns `Ok(_)`, it must ensure those
746    /// items are retained by the returned value, since they are otherwise
747    /// lost.
748    ///
749    /// The implementation must ensure that, if it returns `Ok(_)`, a downcast
750    /// to the specified type is guaranteed to succeed.
751    fn try_into(
752        me: Pin<Box<Self>>,
753        buffer: Self::Buffer,
754        _ty: TypeId,
755    ) -> Result<Box<dyn Any>, (Pin<Box<Self>>, Self::Buffer)> {
756        Err((me, buffer))
757    }
758}
759
760impl<T, D> StreamProducer<D> for iter::Empty<T>
761where
762    T: Send + Sync + 'static,
763{
764    type Item = T;
765    type Buffer = Option<Self::Item>;
766
767    fn poll_produce<'a>(
768        self: Pin<&mut Self>,
769        _: &mut Context<'_>,
770        _: StoreContextMut<'a, D>,
771        _: Destination<'a, Self::Item, Self::Buffer>,
772        _: bool,
773    ) -> Poll<Result<StreamResult>> {
774        Poll::Ready(Ok(StreamResult::Dropped))
775    }
776}
777
778impl<T, D> StreamProducer<D> for stream::Empty<T>
779where
780    T: Send + Sync + 'static,
781{
782    type Item = T;
783    type Buffer = Option<Self::Item>;
784
785    fn poll_produce<'a>(
786        self: Pin<&mut Self>,
787        _: &mut Context<'_>,
788        _: StoreContextMut<'a, D>,
789        _: Destination<'a, Self::Item, Self::Buffer>,
790        _: bool,
791    ) -> Poll<Result<StreamResult>> {
792        Poll::Ready(Ok(StreamResult::Dropped))
793    }
794}
795
796impl<T, D> StreamProducer<D> for Vec<T>
797where
798    T: Unpin + Send + Sync + 'static,
799{
800    type Item = T;
801    type Buffer = VecBuffer<T>;
802
803    fn poll_produce<'a>(
804        self: Pin<&mut Self>,
805        _: &mut Context<'_>,
806        _: StoreContextMut<'a, D>,
807        mut dst: Destination<'a, Self::Item, Self::Buffer>,
808        _: bool,
809    ) -> Poll<Result<StreamResult>> {
810        dst.set_buffer(mem::take(self.get_mut()).into());
811        Poll::Ready(Ok(StreamResult::Dropped))
812    }
813}
814
815impl<T, D> StreamProducer<D> for Box<[T]>
816where
817    T: Unpin + Send + Sync + 'static,
818{
819    type Item = T;
820    type Buffer = VecBuffer<T>;
821
822    fn poll_produce<'a>(
823        self: Pin<&mut Self>,
824        _: &mut Context<'_>,
825        _: StoreContextMut<'a, D>,
826        mut dst: Destination<'a, Self::Item, Self::Buffer>,
827        _: bool,
828    ) -> Poll<Result<StreamResult>> {
829        dst.set_buffer(mem::take(self.get_mut()).into_vec().into());
830        Poll::Ready(Ok(StreamResult::Dropped))
831    }
832}
833
834#[cfg(feature = "component-model-bytes")]
835impl<D> StreamProducer<D> for bytes::Bytes {
836    type Item = u8;
837    type Buffer = Self;
838
839    fn poll_produce<'a>(
840        self: Pin<&mut Self>,
841        _: &mut Context<'_>,
842        _store: StoreContextMut<'a, D>,
843        mut dst: Destination<'a, Self::Item, Self::Buffer>,
844        _: bool,
845    ) -> Poll<Result<StreamResult>> {
846        dst.set_buffer(mem::take(self.get_mut()));
847        Poll::Ready(Ok(StreamResult::Dropped))
848    }
849}
850
851#[cfg(feature = "component-model-bytes")]
852impl<D> StreamProducer<D> for bytes::BytesMut {
853    type Item = u8;
854    type Buffer = Self;
855
856    fn poll_produce<'a>(
857        self: Pin<&mut Self>,
858        _: &mut Context<'_>,
859        _store: StoreContextMut<'a, D>,
860        mut dst: Destination<'a, Self::Item, Self::Buffer>,
861        _: bool,
862    ) -> Poll<Result<StreamResult>> {
863        dst.set_buffer(mem::take(self.get_mut()));
864        Poll::Ready(Ok(StreamResult::Dropped))
865    }
866}
867
868/// Represents the buffer for a host- or guest-initiated stream write.
869pub struct Source<'a, T> {
870    id: TableId<TransmitState>,
871    host_buffer: Option<&'a mut dyn WriteBuffer<T>>,
872}
873
874impl<'a, T> Source<'a, T> {
875    /// Reborrow `self` so it can be used again later.
876    pub fn reborrow(&mut self) -> Source<'_, T> {
877        Source {
878            id: self.id,
879            host_buffer: self.host_buffer.as_deref_mut(),
880        }
881    }
882
883    /// Accept zero or more items from the writer.
884    pub fn read<B, S: AsContextMut>(&mut self, mut store: S, buffer: &mut B) -> Result<()>
885    where
886        T: func::Lift + 'static,
887        B: ReadBuffer<T>,
888    {
889        if let Some(input) = &mut self.host_buffer {
890            let count = input.remaining().len().min(buffer.remaining_capacity());
891            buffer.move_from(*input, count);
892        } else {
893            let store = store.as_context_mut();
894            let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;
895
896            let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
897                bail_bug!("expected ReadState::HostReady");
898            };
899
900            let &WriteState::GuestReady {
901                ty,
902                address,
903                count,
904                options,
905                instance,
906                ..
907            } = &transmit.write
908            else {
909                bail_bug!("expected WriteState::GuestReady");
910            };
911
912            let cx = &mut LiftContext::new(store.0.store_opaque_mut(), options, instance)?;
913            let ty = ty.payload(cx.types);
914            let old_remaining = buffer.remaining_capacity();
915            lift::<T, B>(
916                cx,
917                ty.copied(),
918                buffer,
919                address + (T::SIZE32 * guest_offset.as_usize()),
920                count.as_usize() - guest_offset.as_usize(),
921            )?;
922
923            let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;
924
925            let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
926                bail_bug!("expected ReadState::HostReady");
927            };
928
929            guest_offset.inc(old_remaining - buffer.remaining_capacity())?;
930        }
931
932        Ok(())
933    }
934
935    /// Return the number of items remaining to be read from the current write
936    /// operation.
937    pub fn remaining(&self, mut store: impl AsContextMut) -> Result<usize>
938    where
939        T: 'static,
940    {
941        self.remaining_(store.as_context_mut().0)
942    }
943
944    fn remaining_(&self, store: &mut StoreOpaque) -> Result<usize>
945    where
946        T: 'static,
947    {
948        let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;
949
950        if let &WriteState::GuestReady { count, .. } = &transmit.write {
951            let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
952                bail_bug!("expected ReadState::HostReady")
953            };
954
955            Ok(count.as_usize() - guest_offset.as_usize())
956        } else if let Some(host_buffer) = &self.host_buffer {
957            Ok(host_buffer.remaining().len())
958        } else {
959            bail_bug!("expected either WriteState::GuestReady or host buffer")
960        }
961    }
962}
963
964impl<'a> Source<'a, u8> {
965    /// Return a `DirectSource` view of `self`.
966    pub fn as_direct<D>(self, store: StoreContextMut<'a, D>) -> DirectSource<'a, D> {
967        DirectSource {
968            id: self.id,
969            host_buffer: self.host_buffer,
970            store,
971        }
972    }
973}
974
975/// Represents a write to a `stream<u8>`, providing direct access to the
976/// writer's buffer.
977pub struct DirectSource<'a, D: 'static> {
978    id: TableId<TransmitState>,
979    host_buffer: Option<&'a mut dyn WriteBuffer<u8>>,
980    store: StoreContextMut<'a, D>,
981}
982
983#[cfg(feature = "std")]
984impl<D: 'static> std::io::Read for DirectSource<'_, D> {
985    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
986        let rem = self.remaining().map_err(std::io::Error::other)?;
987        let n = rem.len().min(buf.len());
988        buf[..n].copy_from_slice(&rem[..n]);
989        self.mark_read(n);
990        Ok(n)
991    }
992}
993
994impl<D: 'static> DirectSource<'_, D> {
995    /// Provide direct access to the writer's buffer.
996    pub fn remaining(&mut self) -> Result<&[u8]> {
997        if let Some(buffer) = self.host_buffer.as_deref_mut() {
998            return Ok(buffer.remaining());
999        }
1000        let transmit = self
1001            .store
1002            .as_context_mut()
1003            .0
1004            .concurrent_state_mut()?
1005            .get_mut(self.id)?;
1006
1007        let &WriteState::GuestReady {
1008            address,
1009            count,
1010            options,
1011            instance,
1012            ..
1013        } = &transmit.write
1014        else {
1015            bail_bug!("expected WriteState::GuestReady")
1016        };
1017
1018        let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
1019            bail_bug!("expected ReadState::HostReady")
1020        };
1021
1022        let memory = instance
1023            .options_memory(self.store.0, options)
1024            .get(address + guest_offset.as_usize()..)
1025            .and_then(|b| b.get(..(count.as_usize() - guest_offset.as_usize())));
1026        match memory {
1027            Some(memory) => Ok(memory),
1028            None => bail_bug!("guest buffer unexpectedly out of bounds"),
1029        }
1030    }
1031
1032    /// Mark the specified number of bytes as read from the writer's buffer.
1033    ///
1034    /// # Panics
1035    ///
1036    /// This will panic if the count is larger than the size of the buffer
1037    /// returned by `Self::remaining`.
1038    pub fn mark_read(&mut self, count: usize) {
1039        // Note that this unwrap should only trigger for bugs in Wasmtime, and
1040        // this is modeled here to centralize the `.unwrap()` for this method in
1041        // one location.
1042        self.mark_read_(count).unwrap()
1043    }
1044
1045    fn mark_read_(&mut self, count: usize) -> Result<()> {
1046        if let Some(buffer) = self.host_buffer.as_deref_mut() {
1047            buffer.skip(count);
1048            return Ok(());
1049        }
1050
1051        let transmit = self
1052            .store
1053            .as_context_mut()
1054            .0
1055            .concurrent_state_mut()?
1056            .get_mut(self.id)?;
1057
1058        let WriteState::GuestReady {
1059            count: write_count, ..
1060        } = &transmit.write
1061        else {
1062            bail_bug!("expected WriteState::GuestReady");
1063        };
1064
1065        let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
1066            bail_bug!("expected ReadState::HostReady");
1067        };
1068
1069        if guest_offset.as_usize() + count > write_count.as_usize() {
1070            // Note that this is a documented panic condition of `mark_read`.
1071            panic!("read count ({count}) must be less than or equal to write count ({write_count})")
1072        } else {
1073            guest_offset.inc(count)?;
1074        }
1075        Ok(())
1076    }
1077}
1078
1079/// Represents the host-owned read end of a stream.
1080pub trait StreamConsumer<D>: Send + 'static {
1081    /// The payload type of this stream.
1082    type Item;
1083
1084    /// Handle a host- or guest-initiated write by accepting zero or more items
1085    /// from the specified source.
1086    ///
1087    /// This will be called whenever the writer starts a write.
1088    ///
1089    /// If the implementation is able to consume one or more items immediately,
1090    /// it should take them from `source` and return either
1091    /// `Poll::Ready(Ok(StreamResult::Completed))` if it expects to be able to consume
1092    /// more items, or `Poll::Ready(Ok(StreamResult::Dropped))` if it cannot
1093    /// accept any more items.  Alternatively, it may return `Poll::Pending` to
1094    /// indicate that the caller should delay sending a `COMPLETED` event to the
1095    /// writer until a later call to this function returns `Poll::Ready(_)`.
1096    /// For more about that, see the `Backpressure` section below.
1097    ///
1098    /// If the implementation cannot consume any items immediately and `finish`
1099    /// is _false_, it should store the waker from `cx` for later and return
1100    /// `Poll::Pending` without writing anything to `destination`.  Later, it
1101    /// should alert the waker when either (1) the items arrive, (2) the stream
1102    /// has ended, or (3) an error occurs.
1103    ///
1104    /// If the implementation cannot consume any items immediately and `finish`
1105    /// is _true_, it should, if possible, return
1106    /// `Poll::Ready(Ok(StreamResult::Cancelled))` immediately without taking
1107    /// anything from `source`.  However, that might not be possible if an
1108    /// earlier call to `poll_consume` kicked off an asynchronous operation
1109    /// which needs to be completed (and possibly interrupted) gracefully, in
1110    /// which case the implementation may return `Poll::Pending` and later alert
1111    /// the waker as described above.  In other words, when `finish` is true,
1112    /// the implementation should prioritize returning a result to the reader
1113    /// (even if no items can be consumed) rather than wait indefinitely for at
1114    /// capacity to free up.
1115    ///
1116    /// In all of the above cases, the implementation may alternatively choose
1117    /// to return `Err(_)` to indicate an unrecoverable error.  This will cause
1118    /// the guest (if any) to trap and render the component instance (if any)
1119    /// unusable.  The implementation should report errors that _are_
1120    /// recoverable by other means (e.g. by writing to a `future`) and return
1121    /// `Poll::Ready(Ok(StreamResult::Dropped))`.
1122    ///
1123    /// Note that the implementation should only return
1124    /// `Poll::Ready(Ok(StreamResult::Cancelled))` without having taken any
1125    /// items from `source` if called with `finish` set to true.  If it does so
1126    /// when `finish` is false, the caller will trap.  Additionally, it should
1127    /// only return `Poll::Ready(Ok(StreamResult::Completed))` after taking at
1128    /// least one item from `source` if there is an item available; otherwise,
1129    /// the caller will trap.  Items taken during earlier calls for the same
1130    /// write which returned `Poll::Pending` count towards this, so it's fine
1131    /// to return `Completed` without taking more items once those have been
1132    /// forwarded (see "Backpressure" below).  If `poll_consume` is called with
1133    /// no items in
1134    /// `source`, it should only return `Poll::Ready(_)` once it is able to
1135    /// accept at least one item during the next call to `poll_consume`.
1136    ///
1137    /// Note that any items which the implementation of this trait takes from
1138    /// `source` become the responsibility of that implementation.  For that
1139    /// reason, an implementation which forwards items to an upstream sink
1140    /// should reserve capacity in that sink before taking items out of
1141    /// `source`, if possible.  Alternatively, it might buffer items which can't
1142    /// be forwarded immediately and send them once capacity is freed up.
1143    ///
1144    /// ## Backpressure
1145    ///
1146    /// As mentioned above, an implementation might choose to return
1147    /// `Poll::Pending` after taking items from `source`, which tells the caller
1148    /// to delay sending a `COMPLETED` event to the writer.  This can be used as
1149    /// a form of backpressure when the items are forwarded to an upstream sink
1150    /// asynchronously.  Note, however, that it's not possible to "put back"
1151    /// items into `source` once they've been taken out, so if the upstream sink
1152    /// is unable to accept all the items, that cannot be communicated to the
1153    /// writer at this level of abstraction.  Just as with application-specific,
1154    /// recoverable errors, information about which items could be forwarded and
1155    /// which could not must be communicated out-of-band, e.g. by writing to an
1156    /// application-specific `future`.
1157    ///
1158    /// Similarly, if the writer cancels the write after items have been taken
1159    /// from `source` but before the items have all been forwarded to an
1160    /// upstream sink, `poll_consume` will be called with `finish` set to true,
1161    /// and the implementation may either:
1162    ///
1163    /// - Interrupt the forwarding process gracefully.  This may be preferable
1164    /// if there is an out-of-band channel for communicating to the writer how
1165    /// many items were forwarded before being interrupted.
1166    ///
1167    /// - Allow the forwarding to complete without interrupting it.  This is
1168    /// usually preferable if there's no out-of-band channel for reporting back
1169    /// to the writer how many items were forwarded.
1170    fn poll_consume(
1171        self: Pin<&mut Self>,
1172        cx: &mut Context<'_>,
1173        store: StoreContextMut<D>,
1174        source: Source<'_, Self::Item>,
1175        finish: bool,
1176    ) -> Poll<Result<StreamResult>>;
1177}
1178
1179/// Represents a host-owned write end of a future.
1180pub trait FutureProducer<D>: Send + 'static {
1181    /// The payload type of this future.
1182    type Item;
1183
1184    /// Handle a host- or guest-initiated read by producing a value.
1185    ///
1186    /// This is equivalent to `StreamProducer::poll_produce`, but with a
1187    /// simplified interface for futures.
1188    ///
1189    /// If `finish` is true, the implementation may return
1190    /// `Poll::Ready(Ok(None))` to indicate the operation was canceled before it
1191    /// could produce a value.  Otherwise, it must either return
1192    /// `Poll::Ready(Ok(Some(_)))`, `Poll::Ready(Err(_))`, or `Poll::Pending`.
1193    fn poll_produce(
1194        self: Pin<&mut Self>,
1195        cx: &mut Context<'_>,
1196        store: StoreContextMut<D>,
1197        finish: bool,
1198    ) -> Poll<Result<Option<Self::Item>>>;
1199}
1200
1201impl<T, E, D, Fut> FutureProducer<D> for Fut
1202where
1203    E: Into<Error>,
1204    Fut: Future<Output = Result<T, E>> + ?Sized + Send + 'static,
1205{
1206    type Item = T;
1207
1208    fn poll_produce<'a>(
1209        self: Pin<&mut Self>,
1210        cx: &mut Context<'_>,
1211        _: StoreContextMut<'a, D>,
1212        finish: bool,
1213    ) -> Poll<Result<Option<T>>> {
1214        match self.poll(cx) {
1215            Poll::Ready(Ok(v)) => Poll::Ready(Ok(Some(v))),
1216            Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
1217            Poll::Pending if finish => Poll::Ready(Ok(None)),
1218            Poll::Pending => Poll::Pending,
1219        }
1220    }
1221}
1222
1223/// Represents a host-owned read end of a future.
1224pub trait FutureConsumer<D>: Send + 'static {
1225    /// The payload type of this future.
1226    type Item;
1227
1228    /// Handle a host- or guest-initiated write by consuming a value.
1229    ///
1230    /// This is equivalent to `StreamProducer::poll_produce`, but with a
1231    /// simplified interface for futures.
1232    ///
1233    /// If `finish` is true, the implementation may return `Poll::Ready(Ok(()))`
1234    /// without taking the item from `source`, which indicates the operation was
1235    /// canceled before it could consume the value.  Otherwise, it must either
1236    /// take the item from `source` and return `Poll::Ready(Ok(()))`, or else
1237    /// return `Poll::Ready(Err(_))` or `Poll::Pending` (with or without taking
1238    /// the item).
1239    fn poll_consume(
1240        self: Pin<&mut Self>,
1241        cx: &mut Context<'_>,
1242        store: StoreContextMut<D>,
1243        source: Source<'_, Self::Item>,
1244        finish: bool,
1245    ) -> Poll<Result<()>>;
1246}
1247
1248/// Represents the readable end of a Component Model `future`.
1249///
1250/// Note that `FutureReader` instances must be disposed of using either `pipe`
1251/// or `close`; otherwise the in-store representation will leak and the writer
1252/// end will hang indefinitely.  Consider using [`GuardedFutureReader`] to
1253/// ensure that disposal happens automatically.
1254pub struct FutureReader<T> {
1255    idx: HostResourceIndex,
1256    // It has already been checked that this read end is connected to a
1257    // `FutureProducer` with `Item = T`.
1258    producer_type_match_checked: bool,
1259    _phantom: PhantomData<T>,
1260}
1261
1262impl<T> FutureReader<T> {
1263    /// Create a new future with the specified producer.
1264    ///
1265    /// # Errors
1266    ///
1267    /// Returns an error if the resource table for this store is full or if
1268    /// [`Config::concurrency_support`] is not enabled.
1269    ///
1270    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
1271    pub fn new<S: AsContextMut>(
1272        mut store: S,
1273        producer: impl FutureProducer<S::Data, Item = T>,
1274    ) -> Result<Self>
1275    where
1276        T: func::Lower + func::Lift + Send + Sync + 'static,
1277    {
1278        ensure!(
1279            store.as_context().0.concurrency_support(),
1280            "concurrency support is not enabled"
1281        );
1282
1283        struct Producer<P>(P);
1284
1285        impl<D, T: func::Lower + 'static, P: FutureProducer<D, Item = T>> StreamProducer<D>
1286            for Producer<P>
1287        {
1288            type Item = P::Item;
1289            type Buffer = Option<P::Item>;
1290
1291            fn poll_produce<'a>(
1292                self: Pin<&mut Self>,
1293                cx: &mut Context<'_>,
1294                store: StoreContextMut<D>,
1295                mut destination: Destination<'a, Self::Item, Self::Buffer>,
1296                finish: bool,
1297            ) -> Poll<Result<StreamResult>> {
1298                // SAFETY: This is a standard pin-projection, and we never move
1299                // out of `self`.
1300                let producer = unsafe { self.map_unchecked_mut(|v| &mut v.0) };
1301
1302                Poll::Ready(Ok(
1303                    if let Some(value) = ready!(producer.poll_produce(cx, store, finish))? {
1304                        destination.set_buffer(Some(value));
1305
1306                        // Here we return `StreamResult::Completed` even though
1307                        // we've produced the last item we'll ever produce.
1308                        // That's because the ABI expects
1309                        // `ReturnCode::Completed(1)` rather than
1310                        // `ReturnCode::Dropped(1)`.  In any case, we won't be
1311                        // called again since the future will have resolved.
1312                        StreamResult::Completed
1313                    } else {
1314                        StreamResult::Cancelled
1315                    },
1316                ))
1317            }
1318        }
1319
1320        Ok(Self {
1321            idx: store
1322                .as_context_mut()
1323                .new_transmit(TransmitKind::Future, Producer(producer))?,
1324            producer_type_match_checked: true,
1325            _phantom: PhantomData,
1326        })
1327    }
1328
1329    pub(super) fn new_(idx: HostResourceIndex) -> Self {
1330        Self {
1331            idx,
1332            // It isn't known if this future comes from a `FutureProducer` with
1333            // `Item = T`.
1334            producer_type_match_checked: false,
1335            _phantom: PhantomData,
1336        }
1337    }
1338
1339    pub(super) fn idx(&self) -> HostResourceIndex {
1340        self.idx
1341    }
1342
1343    /// Set the consumer that accepts the result of this future.
1344    ///
1345    /// # Errors
1346    ///
1347    /// Returns an error if this future has already been closed or otherwise
1348    /// transferred (e.g. lowered into a guest), or if it does not belong to
1349    /// `store`.
1350    pub fn pipe<S: AsContextMut>(
1351        self,
1352        mut store: S,
1353        consumer: impl FutureConsumer<S::Data, Item = T> + Unpin,
1354    ) -> Result<()>
1355    where
1356        T: func::Lift + 'static,
1357    {
1358        struct Consumer<C>(C);
1359
1360        impl<D: 'static, T: func::Lift + 'static, C: FutureConsumer<D, Item = T>> StreamConsumer<D>
1361            for Consumer<C>
1362        {
1363            type Item = T;
1364
1365            fn poll_consume(
1366                self: Pin<&mut Self>,
1367                cx: &mut Context<'_>,
1368                mut store: StoreContextMut<D>,
1369                mut source: Source<Self::Item>,
1370                finish: bool,
1371            ) -> Poll<Result<StreamResult>> {
1372                // SAFETY: This is a standard pin-projection, and we never move
1373                // out of `self`.
1374                let consumer = unsafe { self.map_unchecked_mut(|v| &mut v.0) };
1375
1376                ready!(consumer.poll_consume(
1377                    cx,
1378                    store.as_context_mut(),
1379                    source.reborrow(),
1380                    finish
1381                ))?;
1382
1383                Poll::Ready(Ok(if source.remaining(store)? == 0 {
1384                    // Here we return `StreamResult::Completed` even though
1385                    // we've consumed the last item we'll ever consume.  That's
1386                    // because the ABI expects `ReturnCode::Completed(1)` rather
1387                    // than `ReturnCode::Dropped(1)`.  In any case, we won't be
1388                    // called again since the future will have resolved.
1389                    StreamResult::Completed
1390                } else {
1391                    StreamResult::Cancelled
1392                }))
1393            }
1394        }
1395
1396        store
1397            .as_context_mut()
1398            .set_consumer(self.idx, TransmitKind::Future, Consumer(consumer))
1399    }
1400
1401    /// Transfer ownership of the read end of a future from a guest to the host.
1402    fn lift_from_index(cx: &mut LiftContext<'_>, ty: InterfaceType, index: u32) -> Result<Self> {
1403        Ok(Self::new_(lift_index_to_future(cx, ty, index)?))
1404    }
1405
1406    /// Close this `FutureReader`.
1407    ///
1408    /// This will close this half of the future which will signal to a pending
1409    /// write, if any, that the reader side is dropped. If the writer half has
1410    /// not yet written a value then when it attempts to write a value it will
1411    /// see that this end is closed.
1412    ///
1413    /// # Errors
1414    ///
1415    /// Returns an error if this future has already been closed or otherwise
1416    /// transferred (e.g. lowered into a guest), or if it does not belong to
1417    /// `store`.
1418    pub fn close(&mut self, mut store: impl AsContextMut) -> Result<()> {
1419        future_close(store.as_context_mut().0, self.idx)
1420    }
1421
1422    /// Convenience method around [`Self::close`].
1423    pub fn close_with(&mut self, accessor: impl AsAccessor) -> Result<()> {
1424        accessor.as_accessor().with(|access| self.close(access))
1425    }
1426
1427    /// Returns a [`GuardedFutureReader`] which will auto-close this future on
1428    /// drop and clean it up from the store.
1429    ///
1430    /// Note that the `accessor` provided must own this future and is
1431    /// additionally transferred to the `GuardedFutureReader` return value.
1432    pub fn guard<A>(self, accessor: A) -> GuardedFutureReader<T, A>
1433    where
1434        A: AsAccessor,
1435    {
1436        GuardedFutureReader::new(accessor, self)
1437    }
1438
1439    /// Attempts to convert this [`FutureReader<T>`] to a [`FutureAny`].
1440    ///
1441    /// # Errors
1442    ///
1443    /// This function will return an error if `self` does not belong to
1444    /// `store`.
1445    pub fn try_into_future_any(self, store: impl AsContextMut) -> Result<FutureAny>
1446    where
1447        T: ComponentType + 'static,
1448    {
1449        FutureAny::try_from_future_reader(store, self)
1450    }
1451
1452    /// Attempts to convert a [`FutureAny`] into a [`FutureReader<T>`].
1453    ///
1454    /// # Errors
1455    ///
1456    /// This function will fail if `T` doesn't match the type of the value that
1457    /// `future` is sending.
1458    pub fn try_from_future_any(future: FutureAny) -> Result<Self>
1459    where
1460        T: ComponentType + 'static,
1461    {
1462        future.try_into_future_reader()
1463    }
1464}
1465
1466impl<T> fmt::Debug for FutureReader<T> {
1467    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1468        f.debug_struct("FutureReader")
1469            .field("idx", &self.idx)
1470            .finish()
1471    }
1472}
1473
1474pub(super) fn future_close(store: &mut StoreOpaque, idx: HostResourceIndex) -> Result<()> {
1475    let id = store.host_reader_take(idx)?;
1476    store.host_drop_reader(id, TransmitKind::Future)
1477}
1478
1479/// Transfer ownership of the read end of a future from a guest to the host.
1480pub(super) fn lift_index_to_future(
1481    cx: &mut LiftContext<'_>,
1482    ty: InterfaceType,
1483    index: u32,
1484) -> Result<HostResourceIndex> {
1485    match ty {
1486        InterfaceType::Future(src) => {
1487            let id = {
1488                let (state, instance) = cx.concurrent_state_and_instance_mut();
1489                lift_index_to_transmit(instance, state, TransmitIndex::Future(src), index)?
1490            };
1491            cx.host_resource_lower_own(id.rep(), None, None)
1492        }
1493        _ => func::bad_type_info(),
1494    }
1495}
1496
1497/// Checks that the host-owned read end `idx` may be lowered into a guest as a
1498/// future or stream with the payload type `payload`.
1499fn typecheck_lowered_transmit<U>(
1500    idx: HostResourceIndex,
1501    cx: &mut LowerContext<'_, U>,
1502    payload: Option<&InterfaceType>,
1503) -> Result<()> {
1504    let origin = cx.store.0.transmit_origin(idx)?;
1505    let dst = cx.instance_type();
1506    let equivalent = match origin {
1507        TransmitOrigin::Host { typecheck } => {
1508            return typecheck(payload, &dst).context("future or stream payload types differ");
1509        }
1510        TransmitOrigin::GuestFuture(id, ty) => {
1511            let src = InstanceType::new(cx.store.0.component_instance(id));
1512            types::FutureType::from(src.types[ty].ty, &src).equivalent_payload_guest(&dst, payload)
1513        }
1514        TransmitOrigin::GuestStream(id, ty) => {
1515            let src = InstanceType::new(cx.store.0.component_instance(id));
1516            types::StreamType::from(src.types[ty].ty, &src).equivalent_payload_guest(&dst, payload)
1517        }
1518    };
1519    if !equivalent {
1520        bail!("future or stream payload types differ");
1521    }
1522    Ok(())
1523}
1524
1525/// Transfer ownership of the read end of a future from the host to a guest.
1526/// `item_already_typechecked` should only be set to `true` when lowering a
1527/// `FutureReader<T>` which contains a `FutureProducer<Item = T>`.
1528pub(super) fn lower_future_to_index<U>(
1529    idx: HostResourceIndex,
1530    cx: &mut LowerContext<'_, U>,
1531    ty: InterfaceType,
1532    item_already_typechecked: bool,
1533) -> Result<u32> {
1534    match ty {
1535        InterfaceType::Future(dst) => {
1536            // Maintains the invariant on `lower_transmit_to_index`: if this read
1537            // end comes from a host `FutureProducer` `P`, then
1538            // `P::Item::typecheck` must pass for `ty`'s payload.
1539            //
1540            // If `item_already_typechecked` is true, then we're lowering a
1541            // `FutureReader<T>`, which means `FutureReader<T>::typecheck` has
1542            // already passed for `ty` (as required by the `Lower` trait). This
1543            // includes running `T::typecheck` for `ty`'s payload, and
1544            // since `P::Item = T` the invariant is already satisfied.
1545            if !item_already_typechecked {
1546                let types = cx.types;
1547                typecheck_lowered_transmit(idx, cx, types[types[dst].ty].payload.as_ref())?;
1548            }
1549            let id = cx.store.0.host_reader_take(idx)?;
1550            cx.instance_handle()
1551                .lower_transmit_to_index(cx.store.0, TransmitIndex::Future(dst), id)
1552        }
1553        _ => func::bad_type_info(),
1554    }
1555}
1556
1557// SAFETY: This relies on the `ComponentType` implementation for `u32` being
1558// safe and correct since we lift and lower future handles as `u32`s.
1559unsafe impl<T: ComponentType> ComponentType for FutureReader<T> {
1560    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1561
1562    type Lower = <u32 as func::ComponentType>::Lower;
1563
1564    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1565        match ty {
1566            InterfaceType::Future(ty) => {
1567                let ty = types.types[*ty].ty;
1568                types::typecheck_payload::<T>(types.types[ty].payload.as_ref(), types)
1569            }
1570            other => bail!("expected `future`, found `{}`", func::desc(other)),
1571        }
1572    }
1573}
1574
1575// SAFETY: See the comment on the `ComponentType` `impl` for this type.
1576unsafe impl<T: ComponentType> func::Lower for FutureReader<T> {
1577    fn linear_lower_to_flat<U>(
1578        &self,
1579        cx: &mut LowerContext<'_, U>,
1580        ty: InterfaceType,
1581        dst: &mut MaybeUninit<Self::Lower>,
1582    ) -> Result<()> {
1583        lower_future_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1584            .linear_lower_to_flat(cx, InterfaceType::U32, dst)
1585    }
1586
1587    fn linear_lower_to_memory<U>(
1588        &self,
1589        cx: &mut LowerContext<'_, U>,
1590        ty: InterfaceType,
1591        offset: usize,
1592    ) -> Result<()> {
1593        lower_future_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1594            .linear_lower_to_memory(cx, InterfaceType::U32, offset)
1595    }
1596}
1597
1598// SAFETY: See the comment on the `ComponentType` `impl` for this type.
1599unsafe impl<T: ComponentType> func::Lift for FutureReader<T> {
1600    fn linear_lift_from_flat(
1601        cx: &mut LiftContext<'_>,
1602        ty: InterfaceType,
1603        src: &Self::Lower,
1604    ) -> Result<Self> {
1605        let index = u32::linear_lift_from_flat(cx, InterfaceType::U32, src)?;
1606        Self::lift_from_index(cx, ty, index)
1607    }
1608
1609    fn linear_lift_from_memory(
1610        cx: &mut LiftContext<'_>,
1611        ty: InterfaceType,
1612        bytes: &[u8],
1613    ) -> Result<Self> {
1614        let index = u32::linear_lift_from_memory(cx, InterfaceType::U32, bytes)?;
1615        Self::lift_from_index(cx, ty, index)
1616    }
1617}
1618
1619/// A [`FutureReader`] paired with an [`Accessor`].
1620///
1621/// This is an RAII wrapper around [`FutureReader`] that ensures it is closed
1622/// when dropped. This can be created through [`GuardedFutureReader::new`] or
1623/// [`FutureReader::guard`].
1624///
1625/// [`Accessor`]: crate::component::Accessor
1626pub struct GuardedFutureReader<T, A>
1627where
1628    A: AsAccessor,
1629{
1630    // This field is `None` to implement the conversion from this guard back to
1631    // `FutureReader`. When `None` is seen in the destructor it will cause the
1632    // destructor to do nothing.
1633    reader: Option<FutureReader<T>>,
1634    accessor: A,
1635}
1636
1637impl<T, A> GuardedFutureReader<T, A>
1638where
1639    A: AsAccessor,
1640{
1641    /// Create a new `GuardedFutureReader` with the specified `accessor` and `reader`.
1642    ///
1643    /// # Panics
1644    ///
1645    /// Panics if [`Config::concurrency_support`] is not enabled.
1646    ///
1647    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
1648    pub fn new(accessor: A, reader: FutureReader<T>) -> Self {
1649        assert!(
1650            accessor
1651                .as_accessor()
1652                .with(|a| a.as_context().0.concurrency_support())
1653        );
1654        Self {
1655            reader: Some(reader),
1656            accessor,
1657        }
1658    }
1659
1660    /// Extracts the underlying [`FutureReader`] from this guard, returning it
1661    /// back.
1662    pub fn into_future(self) -> FutureReader<T> {
1663        self.into()
1664    }
1665}
1666
1667impl<T, A> From<GuardedFutureReader<T, A>> for FutureReader<T>
1668where
1669    A: AsAccessor,
1670{
1671    fn from(mut guard: GuardedFutureReader<T, A>) -> Self {
1672        guard.reader.take().unwrap()
1673    }
1674}
1675
1676impl<T, A> Drop for GuardedFutureReader<T, A>
1677where
1678    A: AsAccessor,
1679{
1680    fn drop(&mut self) {
1681        if let Some(reader) = &mut self.reader {
1682            // Currently this can only fail if the future is closed twice or
1683            // the handle was cloned/closed elsewhere, all of which the embedder
1684            // is responsible for when using this guard. Hint during
1685            // development that's a bad idea, but otherwise ignore the error in
1686            // release builds.
1687            let result = reader.close_with(&self.accessor);
1688            debug_assert!(result.is_ok());
1689        }
1690    }
1691}
1692
1693/// Represents the readable end of a Component Model `stream`.
1694///
1695/// Note that `StreamReader` instances must be disposed of using `close`;
1696/// otherwise the in-store representation will leak and the writer end will hang
1697/// indefinitely.  Consider using [`GuardedStreamReader`] to ensure that
1698/// disposal happens automatically.
1699pub struct StreamReader<T> {
1700    idx: HostResourceIndex,
1701    // It has already been checked that this read end is connected to a
1702    // `StreamProducer` with `Item = T`.
1703    producer_type_match_checked: bool,
1704    _phantom: PhantomData<T>,
1705}
1706
1707impl<T> StreamReader<T> {
1708    /// Create a new stream with the specified producer.
1709    ///
1710    /// # Errors
1711    ///
1712    /// Returns an error if the resource table for this store is full or if
1713    /// [`Config::concurrency_support`] is not enabled.
1714    ///
1715    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
1716    pub fn new<S: AsContextMut>(
1717        mut store: S,
1718        producer: impl StreamProducer<S::Data, Item = T>,
1719    ) -> Result<Self>
1720    where
1721        T: func::Lower + func::Lift + Send + Sync + 'static,
1722    {
1723        ensure!(
1724            store.as_context().0.concurrency_support(),
1725            "concurrency support is not enabled",
1726        );
1727        Ok(Self {
1728            idx: store
1729                .as_context_mut()
1730                .new_transmit(TransmitKind::Stream, producer)?,
1731            producer_type_match_checked: true,
1732            _phantom: PhantomData,
1733        })
1734    }
1735
1736    pub(super) fn new_(idx: HostResourceIndex) -> Self {
1737        Self {
1738            idx,
1739            // It isn't known if this stream comes from a `StreamProducer` with
1740            // `Item = T`.
1741            producer_type_match_checked: false,
1742            _phantom: PhantomData,
1743        }
1744    }
1745
1746    pub(super) fn idx(&self) -> HostResourceIndex {
1747        self.idx
1748    }
1749
1750    /// Attempt to consume this object by converting it into the specified type.
1751    ///
1752    /// This can be useful for "short-circuiting" host-to-host streams,
1753    /// bypassing the guest entirely.  For example, if a guest task returns a
1754    /// host-created stream and then exits, this function may be used to
1755    /// retrieve the write end, after which the guest instance and store may be
1756    /// disposed of if no longer needed.
1757    ///
1758    /// This will return `Ok(_)` if and only if the following conditions are
1759    /// met:
1760    ///
1761    /// - The stream was created by the host (i.e. not by the guest).
1762    ///
1763    /// - The `StreamProducer::try_into` function returns `Ok(_)` when given the
1764    /// producer provided to `StreamReader::new` when the stream was created,
1765    /// along with `TypeId::of::<V>()`.
1766    ///
1767    /// # Panics
1768    ///
1769    /// Panics if this stream has already been closed, or if this stream doesn't
1770    /// belong to the specified `store`.
1771    pub fn try_into<V: 'static>(mut self, mut store: impl AsContextMut) -> Result<V, Self> {
1772        let store = store.as_context_mut();
1773        let id = store.0.host_reader_rep(self.idx).unwrap();
1774        let state = store.0.concurrent_state_mut_already_forced_current_thread();
1775        let id = state.get_mut(id).unwrap().state;
1776        if let WriteState::HostReady { try_into, .. } = &state.get_mut(id).unwrap().write {
1777            match try_into(TypeId::of::<V>()) {
1778                Some(result) => {
1779                    self.close(store).unwrap();
1780                    Ok(*result.downcast::<V>().unwrap())
1781                }
1782                None => Err(self),
1783            }
1784        } else {
1785            Err(self)
1786        }
1787    }
1788
1789    /// Set the consumer that accepts the items delivered to this stream.
1790    ///
1791    /// # Errors
1792    ///
1793    /// Returns an error if this stream has already been closed or otherwise
1794    /// transferred (e.g. lowered into a guest), or if it does not belong to
1795    /// `store`.
1796    pub fn pipe<S: AsContextMut>(
1797        self,
1798        mut store: S,
1799        consumer: impl StreamConsumer<S::Data, Item = T>,
1800    ) -> Result<()>
1801    where
1802        T: 'static,
1803    {
1804        store
1805            .as_context_mut()
1806            .set_consumer(self.idx, TransmitKind::Stream, consumer)
1807    }
1808
1809    /// Transfer ownership of the read end of a stream from a guest to the host.
1810    fn lift_from_index(cx: &mut LiftContext<'_>, ty: InterfaceType, index: u32) -> Result<Self> {
1811        Ok(Self::new_(lift_index_to_stream(cx, ty, index)?))
1812    }
1813
1814    /// Close this `StreamReader`.
1815    ///
1816    /// This will signal that this portion of the stream is closed causing all
1817    /// future writes to return immediately with "DROPPED".
1818    ///
1819    /// # Errors
1820    ///
1821    /// Returns an error if this stream has already been closed or otherwise
1822    /// transferred (e.g. lowered into a guest), or if it does not belong to
1823    /// `store`.
1824    pub fn close(&mut self, mut store: impl AsContextMut) -> Result<()> {
1825        stream_close(store.as_context_mut().0, self.idx)
1826    }
1827
1828    /// Convenience method around [`Self::close`].
1829    pub fn close_with(&mut self, accessor: impl AsAccessor) -> Result<()> {
1830        accessor.as_accessor().with(|access| self.close(access))
1831    }
1832
1833    /// Returns a [`GuardedStreamReader`] which will auto-close this stream on
1834    /// drop and clean it up from the store.
1835    ///
1836    /// Note that the `accessor` provided must own this future and is
1837    /// additionally transferred to the `GuardedStreamReader` return value.
1838    pub fn guard<A>(self, accessor: A) -> GuardedStreamReader<T, A>
1839    where
1840        A: AsAccessor,
1841    {
1842        GuardedStreamReader::new(accessor, self)
1843    }
1844
1845    /// Attempts to convert this [`StreamReader<T>`] to a [`StreamAny`].
1846    ///
1847    /// # Errors
1848    ///
1849    /// This function will return an error if `self` does not belong to
1850    /// `store`.
1851    pub fn try_into_stream_any(self, store: impl AsContextMut) -> Result<StreamAny>
1852    where
1853        T: ComponentType + 'static,
1854    {
1855        StreamAny::try_from_stream_reader(store, self)
1856    }
1857
1858    /// Attempts to convert a [`StreamAny`] into a [`StreamReader<T>`].
1859    ///
1860    /// # Errors
1861    ///
1862    /// This function will fail if `T` doesn't match the type of the value that
1863    /// `stream` is sending.
1864    pub fn try_from_stream_any(stream: StreamAny) -> Result<Self>
1865    where
1866        T: ComponentType + 'static,
1867    {
1868        stream.try_into_stream_reader()
1869    }
1870}
1871
1872impl<T> fmt::Debug for StreamReader<T> {
1873    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1874        f.debug_struct("StreamReader")
1875            .field("idx", &self.idx)
1876            .finish()
1877    }
1878}
1879
1880pub(super) fn stream_close(store: &mut StoreOpaque, idx: HostResourceIndex) -> Result<()> {
1881    let id = store.host_reader_take(idx)?;
1882    store.host_drop_reader(id, TransmitKind::Stream)
1883}
1884
1885/// Transfer ownership of the read end of a stream from a guest to the host.
1886pub(super) fn lift_index_to_stream(
1887    cx: &mut LiftContext<'_>,
1888    ty: InterfaceType,
1889    index: u32,
1890) -> Result<HostResourceIndex> {
1891    match ty {
1892        InterfaceType::Stream(src) => {
1893            let id = {
1894                let (state, instance) = cx.concurrent_state_and_instance_mut();
1895                lift_index_to_transmit(instance, state, TransmitIndex::Stream(src), index)?
1896            };
1897            cx.host_resource_lower_own(id.rep(), None, None)
1898        }
1899        _ => func::bad_type_info(),
1900    }
1901}
1902
1903/// Transfer ownership of the read end of a stream from the host to a guest.
1904/// `item_already_typechecked` should only be set to `true` when lowering a
1905/// `StreamReader<T>` which contains a `StreamProducer<Item = T>`.
1906pub(super) fn lower_stream_to_index<U>(
1907    idx: HostResourceIndex,
1908    cx: &mut LowerContext<'_, U>,
1909    ty: InterfaceType,
1910    item_already_typechecked: bool,
1911) -> Result<u32> {
1912    match ty {
1913        InterfaceType::Stream(dst) => {
1914            // Maintains the invariant on `lower_transmit_to_index`: if this read
1915            // end comes from a host `StreamProducer` `P`, then
1916            // `P::Item::typecheck` must pass for `ty`'s payload.
1917            //
1918            // If `item_already_typechecked` is true, then we're lowering a
1919            // `StreamReader<T>`, which means `StreamReader<T>::typecheck` has
1920            // already passed for `ty` (as required by the `Lower` trait). This
1921            // includes running `T::typecheck` for `ty`'s payload, and
1922            // since `P::Item = T` the invariant is already satisfied.
1923            if !item_already_typechecked {
1924                let types = cx.types;
1925                typecheck_lowered_transmit(idx, cx, types[types[dst].ty].payload.as_ref())?;
1926            }
1927            let id = cx.store.0.host_reader_take(idx)?;
1928            cx.instance_handle()
1929                .lower_transmit_to_index(cx.store.0, TransmitIndex::Stream(dst), id)
1930        }
1931        _ => func::bad_type_info(),
1932    }
1933}
1934
1935// SAFETY: This relies on the `ComponentType` implementation for `u32` being
1936// safe and correct since we lift and lower stream handles as `u32`s.
1937unsafe impl<T: ComponentType> ComponentType for StreamReader<T> {
1938    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1939
1940    type Lower = <u32 as func::ComponentType>::Lower;
1941
1942    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1943        match ty {
1944            InterfaceType::Stream(ty) => {
1945                let ty = types.types[*ty].ty;
1946                types::typecheck_payload::<T>(types.types[ty].payload.as_ref(), types)
1947            }
1948            other => bail!("expected `stream`, found `{}`", func::desc(other)),
1949        }
1950    }
1951}
1952
1953// SAFETY: See the comment on the `ComponentType` `impl` for this type.
1954unsafe impl<T: ComponentType> func::Lower for StreamReader<T> {
1955    fn linear_lower_to_flat<U>(
1956        &self,
1957        cx: &mut LowerContext<'_, U>,
1958        ty: InterfaceType,
1959        dst: &mut MaybeUninit<Self::Lower>,
1960    ) -> Result<()> {
1961        lower_stream_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1962            .linear_lower_to_flat(cx, InterfaceType::U32, dst)
1963    }
1964
1965    fn linear_lower_to_memory<U>(
1966        &self,
1967        cx: &mut LowerContext<'_, U>,
1968        ty: InterfaceType,
1969        offset: usize,
1970    ) -> Result<()> {
1971        lower_stream_to_index(self.idx, cx, ty, self.producer_type_match_checked)?
1972            .linear_lower_to_memory(cx, InterfaceType::U32, offset)
1973    }
1974}
1975
1976// SAFETY: See the comment on the `ComponentType` `impl` for this type.
1977unsafe impl<T: ComponentType> func::Lift for StreamReader<T> {
1978    fn linear_lift_from_flat(
1979        cx: &mut LiftContext<'_>,
1980        ty: InterfaceType,
1981        src: &Self::Lower,
1982    ) -> Result<Self> {
1983        let index = u32::linear_lift_from_flat(cx, InterfaceType::U32, src)?;
1984        Self::lift_from_index(cx, ty, index)
1985    }
1986
1987    fn linear_lift_from_memory(
1988        cx: &mut LiftContext<'_>,
1989        ty: InterfaceType,
1990        bytes: &[u8],
1991    ) -> Result<Self> {
1992        let index = u32::linear_lift_from_memory(cx, InterfaceType::U32, bytes)?;
1993        Self::lift_from_index(cx, ty, index)
1994    }
1995}
1996
1997/// A [`StreamReader`] paired with an [`Accessor`].
1998///
1999/// This is an RAII wrapper around [`StreamReader`] that ensures it is closed
2000/// when dropped. This can be created through [`GuardedStreamReader::new`] or
2001/// [`StreamReader::guard`].
2002///
2003/// [`Accessor`]: crate::component::Accessor
2004pub struct GuardedStreamReader<T, A>
2005where
2006    A: AsAccessor,
2007{
2008    // This field is `None` to implement the conversion from this guard back to
2009    // `StreamReader`. When `None` is seen in the destructor it will cause the
2010    // destructor to do nothing.
2011    reader: Option<StreamReader<T>>,
2012    accessor: A,
2013}
2014
2015impl<T, A> GuardedStreamReader<T, A>
2016where
2017    A: AsAccessor,
2018{
2019    /// Create a new `GuardedStreamReader` with the specified `accessor` and
2020    /// `reader`.
2021    ///
2022    /// # Panics
2023    ///
2024    /// Panics if [`Config::concurrency_support`] is not enabled.
2025    ///
2026    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
2027    pub fn new(accessor: A, reader: StreamReader<T>) -> Self {
2028        assert!(
2029            accessor
2030                .as_accessor()
2031                .with(|a| a.as_context().0.concurrency_support())
2032        );
2033        Self {
2034            reader: Some(reader),
2035            accessor,
2036        }
2037    }
2038
2039    /// Extracts the underlying [`StreamReader`] from this guard, returning it
2040    /// back.
2041    pub fn into_stream(self) -> StreamReader<T> {
2042        self.into()
2043    }
2044}
2045
2046impl<T, A> From<GuardedStreamReader<T, A>> for StreamReader<T>
2047where
2048    A: AsAccessor,
2049{
2050    fn from(mut guard: GuardedStreamReader<T, A>) -> Self {
2051        guard.reader.take().unwrap()
2052    }
2053}
2054
2055impl<T, A> Drop for GuardedStreamReader<T, A>
2056where
2057    A: AsAccessor,
2058{
2059    fn drop(&mut self) {
2060        if let Some(reader) = &mut self.reader {
2061            // See comments in `Drop for GuardedFutureReader` for this assert.
2062            let result = reader.close_with(&self.accessor);
2063            debug_assert!(result.is_ok());
2064        }
2065    }
2066}
2067
2068/// Represents a Component Model `error-context`.
2069pub struct ErrorContext {
2070    rep: u32,
2071}
2072
2073impl ErrorContext {
2074    pub(crate) fn new(rep: u32) -> Self {
2075        Self { rep }
2076    }
2077
2078    /// Convert this `ErrorContext` into a [`Val`].
2079    pub fn into_val(self) -> Val {
2080        Val::ErrorContext(ErrorContextAny(self.rep))
2081    }
2082
2083    /// Attempt to convert the specified [`Val`] to a `ErrorContext`.
2084    pub fn from_val(_: impl AsContextMut, value: &Val) -> Result<Self> {
2085        let Val::ErrorContext(ErrorContextAny(rep)) = value else {
2086            bail!("expected `error-context`; got `{}`", value.desc());
2087        };
2088        Ok(Self::new(*rep))
2089    }
2090
2091    fn lift_from_index(cx: &mut LiftContext<'_>, ty: InterfaceType, index: u32) -> Result<Self> {
2092        match ty {
2093            InterfaceType::ErrorContext(src) => {
2094                let rep = cx
2095                    .instance_mut()
2096                    .table_for_error_context(src)
2097                    .error_context_rep(index)?;
2098
2099                Ok(Self { rep })
2100            }
2101            _ => func::bad_type_info(),
2102        }
2103    }
2104}
2105
2106pub(crate) fn lower_error_context_to_index<U>(
2107    rep: u32,
2108    cx: &mut LowerContext<'_, U>,
2109    ty: InterfaceType,
2110) -> Result<u32> {
2111    match ty {
2112        InterfaceType::ErrorContext(dst) => {
2113            let (instance, counter) = cx
2114                .instance_handle()
2115                .id()
2116                .get_mut_and_handle_counter(cx.store.0);
2117            instance
2118                .error_context_table_insertion(dst, counter)
2119                .error_context(rep)
2120        }
2121        _ => func::bad_type_info(),
2122    }
2123}
2124// SAFETY: This relies on the `ComponentType` implementation for `u32` being
2125// safe and correct since we lift and lower future handles as `u32`s.
2126unsafe impl func::ComponentType for ErrorContext {
2127    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
2128
2129    type Lower = <u32 as func::ComponentType>::Lower;
2130
2131    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
2132        match ty {
2133            InterfaceType::ErrorContext(_) => Ok(()),
2134            other => bail!("expected `error`, found `{}`", func::desc(other)),
2135        }
2136    }
2137}
2138
2139// SAFETY: See the comment on the `ComponentType` `impl` for this type.
2140unsafe impl func::Lower for ErrorContext {
2141    fn linear_lower_to_flat<T>(
2142        &self,
2143        cx: &mut LowerContext<'_, T>,
2144        ty: InterfaceType,
2145        dst: &mut MaybeUninit<Self::Lower>,
2146    ) -> Result<()> {
2147        lower_error_context_to_index(self.rep, cx, ty)?.linear_lower_to_flat(
2148            cx,
2149            InterfaceType::U32,
2150            dst,
2151        )
2152    }
2153
2154    fn linear_lower_to_memory<T>(
2155        &self,
2156        cx: &mut LowerContext<'_, T>,
2157        ty: InterfaceType,
2158        offset: usize,
2159    ) -> Result<()> {
2160        lower_error_context_to_index(self.rep, cx, ty)?.linear_lower_to_memory(
2161            cx,
2162            InterfaceType::U32,
2163            offset,
2164        )
2165    }
2166}
2167
2168// SAFETY: See the comment on the `ComponentType` `impl` for this type.
2169unsafe impl func::Lift for ErrorContext {
2170    fn linear_lift_from_flat(
2171        cx: &mut LiftContext<'_>,
2172        ty: InterfaceType,
2173        src: &Self::Lower,
2174    ) -> Result<Self> {
2175        let index = u32::linear_lift_from_flat(cx, InterfaceType::U32, src)?;
2176        Self::lift_from_index(cx, ty, index)
2177    }
2178
2179    fn linear_lift_from_memory(
2180        cx: &mut LiftContext<'_>,
2181        ty: InterfaceType,
2182        bytes: &[u8],
2183    ) -> Result<Self> {
2184        let index = u32::linear_lift_from_memory(cx, InterfaceType::U32, bytes)?;
2185        Self::lift_from_index(cx, ty, index)
2186    }
2187}
2188
2189/// Represents the read or write end of a stream or future.
2190pub(super) struct TransmitHandle {
2191    pub(super) common: WaitableCommon,
2192    /// See `TransmitState`
2193    state: TableId<TransmitState>,
2194}
2195
2196impl TransmitHandle {
2197    fn new(state: TableId<TransmitState>) -> Self {
2198        Self {
2199            common: WaitableCommon::default(),
2200            state,
2201        }
2202    }
2203}
2204
2205impl TableDebug for TransmitHandle {
2206    fn type_name() -> &'static str {
2207        "TransmitHandle"
2208    }
2209}
2210
2211/// Represents the state of a stream or future.
2212struct TransmitState {
2213    /// The write end of the stream or future.
2214    write_handle: TableId<TransmitHandle>,
2215    /// The read end of the stream or future.
2216    read_handle: TableId<TransmitHandle>,
2217    /// See `WriteState`
2218    write: WriteState,
2219    /// See `ReadState`
2220    read: ReadState,
2221    /// Whether further values may be transmitted via this stream or future.
2222    done: bool,
2223    /// The original creator of this stream or future, whose payload type is
2224    /// checked against a guest's whenever the host lowers the read end into
2225    /// that guest, and which `{Future,Stream}Any` use as their initial type.
2226    pub(super) origin: TransmitOrigin,
2227}
2228
2229#[derive(Copy, Clone)]
2230pub(super) enum TransmitOrigin {
2231    Host {
2232        /// Type-checks the producer's items against the guest-supplied payload.
2233        typecheck: fn(Option<&InterfaceType>, &InstanceType<'_>) -> Result<()>,
2234    },
2235    GuestFuture(ComponentInstanceId, TypeFutureTableIndex),
2236    GuestStream(ComponentInstanceId, TypeStreamTableIndex),
2237}
2238
2239impl TransmitState {
2240    fn new(origin: TransmitOrigin) -> Self {
2241        Self {
2242            write_handle: TableId::new(u32::MAX),
2243            read_handle: TableId::new(u32::MAX),
2244            read: ReadState::Open,
2245            write: WriteState::Open,
2246            done: false,
2247            origin,
2248        }
2249    }
2250}
2251
2252impl TableDebug for TransmitState {
2253    fn type_name() -> &'static str {
2254        "TransmitState"
2255    }
2256}
2257
2258impl TransmitOrigin {
2259    fn guest(id: ComponentInstanceId, index: TransmitIndex) -> Self {
2260        match index {
2261            TransmitIndex::Future(ty) => TransmitOrigin::GuestFuture(id, ty),
2262            TransmitIndex::Stream(ty) => TransmitOrigin::GuestStream(id, ty),
2263        }
2264    }
2265}
2266
2267type PollStream = Box<
2268    dyn Fn() -> Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>> + Send + Sync,
2269>;
2270
2271type TryInto = Box<dyn Fn(TypeId) -> Option<Box<dyn Any>> + Send + Sync>;
2272
2273/// Represents the state of the write end of a stream or future.
2274enum WriteState {
2275    /// The write end is open, but no write is pending.
2276    Open,
2277    /// The write end is owned by a guest task and a write is pending.
2278    GuestReady {
2279        instance: Instance,
2280        caller: RuntimeComponentInstanceIndex,
2281        ty: TransmitIndex,
2282        flat_abi: Option<FlatAbi>,
2283        options: OptionsIndex,
2284        address: usize,
2285        count: ItemCount,
2286        handle: u32,
2287    },
2288    /// The write end is owned by the host, which is ready to produce items.
2289    HostReady {
2290        produce: PollStream,
2291        try_into: TryInto,
2292        guest_offset: ItemCount,
2293        cancel: bool,
2294        cancel_waker: Option<Waker>,
2295    },
2296    /// The write end has been dropped.
2297    Dropped,
2298}
2299
2300impl fmt::Debug for WriteState {
2301    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2302        match self {
2303            Self::Open => f.debug_tuple("Open").finish(),
2304            Self::GuestReady { .. } => f.debug_tuple("GuestReady").finish(),
2305            Self::HostReady { .. } => f.debug_tuple("HostReady").finish(),
2306            Self::Dropped => f.debug_tuple("Dropped").finish(),
2307        }
2308    }
2309}
2310
2311/// Represents the state of the read end of a stream or future.
2312enum ReadState {
2313    /// The read end is open, but no read is pending.
2314    Open,
2315    /// The read end is owned by a guest task and a read is pending.
2316    GuestReady {
2317        ty: TransmitIndex,
2318        caller_instance: RuntimeComponentInstanceIndex,
2319        flat_abi: Option<FlatAbi>,
2320        instance: Instance,
2321        options: OptionsIndex,
2322        address: usize,
2323        count: ItemCount,
2324        handle: u32,
2325    },
2326    /// The read end is owned by a host task, and it is ready to consume items.
2327    HostReady {
2328        consume: PollStream,
2329        guest_offset: ItemCount,
2330        cancel: bool,
2331        cancel_waker: Option<Waker>,
2332    },
2333    /// Both the read and write ends are owned by the host.
2334    HostToHost {
2335        accept: Box<
2336            dyn for<'a> Fn(
2337                    &'a mut UntypedWriteBuffer<'a>,
2338                )
2339                    -> Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'a>>
2340                + Send
2341                + Sync,
2342        >,
2343        buffer: Vec<u8>,
2344        limit: usize,
2345    },
2346    /// The read end has been dropped.
2347    Dropped,
2348}
2349
2350impl fmt::Debug for ReadState {
2351    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2352        match self {
2353            Self::Open => f.debug_tuple("Open").finish(),
2354            Self::GuestReady { .. } => f.debug_tuple("GuestReady").finish(),
2355            Self::HostReady { .. } => f.debug_tuple("HostReady").finish(),
2356            Self::HostToHost { .. } => f.debug_tuple("HostToHost").finish(),
2357            Self::Dropped => f.debug_tuple("Dropped").finish(),
2358        }
2359    }
2360}
2361
2362fn return_code(
2363    kind: TransmitKind,
2364    state: StreamResult,
2365    count: ItemCount,
2366    cancel: bool,
2367) -> Result<ReturnCode> {
2368    // The spec says that for streams the priority of return codes is
2369    //
2370    //   DROPPED > CANCELLED > COMPLETED
2371    //
2372    // where for futures it's
2373    //
2374    //   COMPLETED > DROPPED > CANCELLED
2375    //
2376    // which is what's implemented here.
2377    Ok(match (state, kind) {
2378        (StreamResult::Dropped, _) => ReturnCode::Dropped(count),
2379        (StreamResult::Completed, TransmitKind::Stream) if cancel => ReturnCode::Cancelled(count),
2380        (StreamResult::Completed, _) => ReturnCode::completed(kind, count),
2381        (StreamResult::Cancelled, _) => ReturnCode::Cancelled(count),
2382    })
2383}
2384
2385fn settle_host_read(
2386    transmit: &mut TransmitState,
2387    kind: TransmitKind,
2388    state: StreamResult,
2389) -> Result<ReturnCode> {
2390    let ReadState::HostReady {
2391        consume,
2392        guest_offset,
2393        cancel,
2394        ..
2395    } = mem::replace(&mut transmit.read, ReadState::Open)
2396    else {
2397        bail_bug!("expected ReadState::HostReady")
2398    };
2399    let code = return_code(kind, state, guest_offset, cancel)?;
2400    transmit.read = match state {
2401        StreamResult::Dropped => ReadState::Dropped,
2402        StreamResult::Completed | StreamResult::Cancelled => ReadState::HostReady {
2403            consume,
2404            guest_offset: ItemCount::ZERO,
2405            cancel: false,
2406            cancel_waker: None,
2407        },
2408    };
2409    Ok(code)
2410}
2411
2412fn settle_host_write(
2413    transmit: &mut TransmitState,
2414    kind: TransmitKind,
2415    state: StreamResult,
2416) -> Result<ReturnCode> {
2417    let WriteState::HostReady {
2418        produce,
2419        try_into,
2420        guest_offset,
2421        cancel,
2422        ..
2423    } = mem::replace(&mut transmit.write, WriteState::Open)
2424    else {
2425        bail_bug!("expected WriteState::HostReady")
2426    };
2427    let code = return_code(kind, state, guest_offset, cancel)?;
2428    transmit.write = match state {
2429        StreamResult::Dropped => WriteState::Dropped,
2430        StreamResult::Completed | StreamResult::Cancelled => WriteState::HostReady {
2431            produce,
2432            try_into,
2433            guest_offset: ItemCount::ZERO,
2434            cancel: false,
2435            cancel_waker: None,
2436        },
2437    };
2438    Ok(code)
2439}
2440
2441impl StoreOpaque {
2442    fn pipe_from_guest(
2443        &mut self,
2444        kind: TransmitKind,
2445        id: TableId<TransmitState>,
2446        future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,
2447    ) {
2448        let future = async move {
2449            let stream_state = future.await?;
2450            tls::get(|store| {
2451                let state = store.concurrent_state_mut()?;
2452                let transmit = state.get_mut(id)?;
2453                let code = settle_host_read(transmit, kind, stream_state)?;
2454                let WriteState::GuestReady { ty, handle, .. } =
2455                    mem::replace(&mut transmit.write, WriteState::Open)
2456                else {
2457                    bail_bug!("expected WriteState::GuestReady")
2458                };
2459                state.send_write_result(ty, id, handle, code)?;
2460                Ok(())
2461            })
2462        };
2463
2464        self.concurrent_state_mut_already_forced_current_thread()
2465            .push_future(future.boxed());
2466    }
2467
2468    fn pipe_to_guest(
2469        &mut self,
2470        kind: TransmitKind,
2471        id: TableId<TransmitState>,
2472        future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,
2473    ) {
2474        let future = async move {
2475            let stream_state = future.await?;
2476            tls::get(|store| {
2477                let state = store.concurrent_state_mut()?;
2478                let transmit = state.get_mut(id)?;
2479                let code = settle_host_write(transmit, kind, stream_state)?;
2480                let ReadState::GuestReady { ty, handle, .. } =
2481                    mem::replace(&mut transmit.read, ReadState::Open)
2482                else {
2483                    bail_bug!("expected ReadState::GuestReady")
2484                };
2485                state.send_read_result(ty, id, handle, code)?;
2486                Ok(())
2487            })
2488        };
2489
2490        self.concurrent_state_mut_already_forced_current_thread()
2491            .push_future(future.boxed());
2492    }
2493
2494    /// Drop the read end of a stream or future read from the host.
2495    fn host_drop_reader(&mut self, id: TableId<TransmitHandle>, kind: TransmitKind) -> Result<()> {
2496        let state = self.concurrent_state_mut()?;
2497        Waitable::Transmit(id).join(state, None)?;
2498        let transmit_id = state.get_mut(id)?.state;
2499        let transmit = state
2500            .get_mut(transmit_id)
2501            .with_context(|| format!("error closing reader {transmit_id:?}"))?;
2502        log::trace!(
2503            "host_drop_reader state {transmit_id:?}; read state {:?} write state {:?}",
2504            transmit.read,
2505            transmit.write
2506        );
2507
2508        transmit.read = ReadState::Dropped;
2509
2510        // If the write end is already dropped, it should stay dropped,
2511        // otherwise, it should be opened.
2512        let new_state = if let WriteState::Dropped = &transmit.write {
2513            WriteState::Dropped
2514        } else {
2515            WriteState::Open
2516        };
2517
2518        let write_handle = transmit.write_handle;
2519        let done = transmit.done;
2520
2521        match mem::replace(&mut transmit.write, new_state) {
2522            // If a guest is waiting to write, notify it that the read end has
2523            // been dropped.
2524            WriteState::GuestReady { ty, handle, .. } => {
2525                state.update_event(
2526                    write_handle.rep(),
2527                    match ty {
2528                        TransmitIndex::Future(ty) => Event::FutureWrite {
2529                            code: ReturnCode::Dropped(ItemCount::ZERO),
2530                            pending: Some((ty, handle)),
2531                        },
2532                        TransmitIndex::Stream(ty) => Event::StreamWrite {
2533                            code: ReturnCode::Dropped(ItemCount::ZERO),
2534                            pending: Some((ty, handle)),
2535                        },
2536                    },
2537                )?;
2538            }
2539
2540            // Otherwise notify the idle write end that the read end has been
2541            // dropped, unless it's already done.
2542            WriteState::Open if done => {}
2543            WriteState::Open => {
2544                state.update_event(
2545                    write_handle.rep(),
2546                    match kind {
2547                        TransmitKind::Future => Event::FutureWrite {
2548                            code: ReturnCode::Dropped(ItemCount::ZERO),
2549                            pending: None,
2550                        },
2551                        TransmitKind::Stream => Event::StreamWrite {
2552                            code: ReturnCode::Dropped(ItemCount::ZERO),
2553                            pending: None,
2554                        },
2555                    },
2556                )?;
2557            }
2558
2559            // If the writer has already been dropped, then this cleans out the
2560            // state that the reader is using. If the write is host-owned then
2561            // by cleaning this out we run the host's `Drop` implementation
2562            // which notifies it of this drop.
2563            WriteState::Dropped | WriteState::HostReady { .. } => {
2564                log::trace!("host_drop_reader delete {transmit_id:?}");
2565                state.delete_transmit(transmit_id)?;
2566            }
2567        }
2568        Ok(())
2569    }
2570
2571    /// Drop the write end of a stream or future read from the host.
2572    fn host_drop_writer(
2573        &mut self,
2574        id: TableId<TransmitHandle>,
2575        on_drop_open: Option<fn() -> Result<()>>,
2576    ) -> Result<()> {
2577        let state = self.concurrent_state_mut()?;
2578        Waitable::Transmit(id).join(state, None)?;
2579        let transmit_id = state.get_mut(id)?.state;
2580        let transmit = state
2581            .get_mut(transmit_id)
2582            .with_context(|| format!("error closing writer {transmit_id:?}"))?;
2583        log::trace!(
2584            "host_drop_writer state {transmit_id:?}; read state {:?} writer state {:?}",
2585            transmit.read,
2586            transmit.write
2587        );
2588
2589        // Existing queued transmits must be updated with information for the impending writer closure
2590        match &mut transmit.write {
2591            WriteState::GuestReady { .. } => {
2592                bail_bug!("can't call `host_drop_writer` on a guest-owned writer");
2593            }
2594            WriteState::HostReady { .. } => {}
2595            v @ WriteState::Open => {
2596                if let (Some(on_drop_open), false) = (on_drop_open, transmit.done) {
2597                    on_drop_open()?;
2598                } else {
2599                    *v = WriteState::Dropped;
2600                }
2601            }
2602            WriteState::Dropped => bail_bug!("write state is already dropped"),
2603        }
2604
2605        let transmit = self.concurrent_state_mut()?.get_mut(transmit_id)?;
2606
2607        // If the existing read state is dropped, then there's nothing to read
2608        // and we can keep it that way.
2609        //
2610        // If the read state was any other state, then we must set the new state to open
2611        // to indicate that there *is* data to be read
2612        let new_state = if let ReadState::Dropped = &transmit.read {
2613            ReadState::Dropped
2614        } else {
2615            ReadState::Open
2616        };
2617
2618        let read_handle = transmit.read_handle;
2619        let done = transmit.done;
2620
2621        // Swap in the new read state
2622        match mem::replace(&mut transmit.read, new_state) {
2623            // If the guest was ready to read, then we cannot drop the reader (or writer);
2624            // we must deliver the event, and update the state associated with the handle to
2625            // represent that a read must be performed
2626            ReadState::GuestReady { ty, handle, .. } => {
2627                // Ensure the final read of the guest is queued, with appropriate closure indicator
2628                self.concurrent_state_mut()?.update_event(
2629                    read_handle.rep(),
2630                    match ty {
2631                        TransmitIndex::Future(ty) => Event::FutureRead {
2632                            code: ReturnCode::Dropped(ItemCount::ZERO),
2633                            pending: Some((ty, handle)),
2634                        },
2635                        TransmitIndex::Stream(ty) => Event::StreamRead {
2636                            code: ReturnCode::Dropped(ItemCount::ZERO),
2637                            pending: Some((ty, handle)),
2638                        },
2639                    },
2640                )?;
2641            }
2642
2643            // Notify the read-end of the drop, unless we're already done.
2644            ReadState::Open if done => {}
2645            ReadState::Open => {
2646                self.concurrent_state_mut()?.update_event(
2647                    read_handle.rep(),
2648                    match on_drop_open {
2649                        Some(_) => Event::FutureRead {
2650                            code: ReturnCode::Dropped(ItemCount::ZERO),
2651                            pending: None,
2652                        },
2653                        None => Event::StreamRead {
2654                            code: ReturnCode::Dropped(ItemCount::ZERO),
2655                            pending: None,
2656                        },
2657                    },
2658                )?;
2659            }
2660
2661            // If the read state was already dropped, then we can remove the
2662            // transmit state completely (both writer and reader have been
2663            // dropped). If the read state is host-owned then it's additionally
2664            // deleted here as a notification that the read end has gone away.
2665            // Running the host's `Drop` implementation is what notifies it of
2666            // this event.
2667            ReadState::Dropped | ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {
2668                log::trace!("host_drop_writer delete {transmit_id:?}");
2669                self.concurrent_state_mut()?.delete_transmit(transmit_id)?;
2670            }
2671        }
2672        Ok(())
2673    }
2674
2675    pub(super) fn transmit_origin(&mut self, idx: HostResourceIndex) -> Result<TransmitOrigin> {
2676        let id = self.host_reader_rep(idx)?;
2677        let state = self.concurrent_state_mut()?;
2678        let state_id = state.get_mut(id)?.state;
2679        Ok(state.get_mut(state_id)?.origin)
2680    }
2681
2682    /// Insert `id` within the store, returning a `HostResourceIndex` suitable
2683    /// to re-acquire it via the `host_reader_take` and `host_reader_rep`
2684    /// methods below.
2685    fn host_reader_insert(&mut self, id: TableId<TransmitHandle>) -> Result<HostResourceIndex> {
2686        HostResourceTables::new_host(self)?.host_resource_lower_own(id.rep(), None, None)
2687    }
2688
2689    /// Removes `idx` from the store, returning the internal `id` it was
2690    /// created with.
2691    fn host_reader_take(&mut self, idx: HostResourceIndex) -> Result<TableId<TransmitHandle>> {
2692        let rep = HostResourceTables::new_host(self)?.host_resource_lift_own(idx)?;
2693        Ok(TableId::new(rep))
2694    }
2695
2696    /// Looks up `idx` within the store, returning the internal `id` it was
2697    /// created with.
2698    fn host_reader_rep(&mut self, idx: HostResourceIndex) -> Result<TableId<TransmitHandle>> {
2699        let rep = HostResourceTables::new_host(self)?.host_resource_rep(idx)?;
2700        Ok(TableId::new(rep))
2701    }
2702}
2703
2704struct ProducerState<P, B> {
2705    producer: P,
2706    buffer: B,
2707    dropped: bool,
2708}
2709
2710impl<T> StoreContextMut<'_, T> {
2711    fn new_transmit<P: StreamProducer<T>>(
2712        mut self,
2713        kind: TransmitKind,
2714        producer: P,
2715    ) -> Result<HostResourceIndex>
2716    where
2717        P::Item: func::Lower,
2718    {
2719        let token = StoreToken::new(self.as_context_mut());
2720        let state = self.0.concurrent_state_mut()?;
2721        let (_, read) = state.new_transmit(TransmitOrigin::Host {
2722            typecheck: types::typecheck_payload::<P::Item>,
2723        })?;
2724        let producer = Arc::new(LockedState::new(ProducerState {
2725            producer: Box::pin(producer),
2726            buffer: P::Buffer::default(),
2727            dropped: false,
2728        }));
2729        let id = state.get_mut(read)?.state;
2730        let produce = Box::new({
2731            let producer = producer.clone();
2732            move || {
2733                let producer = producer.clone();
2734                async move {
2735                    let mut state = producer.take()?;
2736                    let ProducerState { producer: mine, buffer, dropped } = &mut *state;
2737
2738                    let (result, cancelled) = if buffer.remaining().is_empty() {
2739                        future::poll_fn(|cx| {
2740                            tls::get(|store| {
2741                                let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2742
2743                                let &WriteState::HostReady { cancel, .. } = &transmit.write else {
2744                                    bail_bug!("expected WriteState::HostReady")
2745                                };
2746
2747                                let mut host_written = 0;
2748                                let mut host_buffer =
2749                                    if let ReadState::HostToHost { buffer, .. } = &mut transmit.read {
2750                                        Some(mem::take(buffer))
2751                                    } else {
2752                                        None
2753                                    };
2754
2755                                let poll = mine.as_mut().poll_produce(
2756                                    cx,
2757                                    token.as_context_mut(store),
2758                                    Destination {
2759                                        id,
2760                                        buffer,
2761                                        host_buffer: host_buffer.as_mut().map(|b| {
2762                                            HostBuffer {
2763                                                dst: b,
2764                                                marked_written: &mut host_written,
2765                                            }
2766                                        }),
2767                                        _phantom: PhantomData,
2768                                    },
2769                                    cancel,
2770                                );
2771
2772                                let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2773
2774                                let host_offset = if let (
2775                                    Some(host_buffer),
2776                                    ReadState::HostToHost { buffer, limit, .. },
2777                                ) = (host_buffer, &mut transmit.read)
2778                                {
2779                                    *limit = host_written;
2780                                    *buffer = host_buffer;
2781                                    *limit
2782                                } else {
2783                                    0
2784                                };
2785
2786                                {
2787                                    let WriteState::HostReady {
2788                                        guest_offset,
2789                                        cancel_waker,
2790                                        ..
2791                                    } = &mut transmit.write
2792                                    else {
2793                                        bail_bug!("expected WriteState::HostReady")
2794                                    };
2795
2796                                    *cancel_waker = if poll.is_pending() {
2797                                        if !buffer.remaining().is_empty()
2798                                            || *guest_offset > 0
2799                                            || host_offset > 0
2800                                        {
2801                                            bail!(
2802                                                "StreamProducer::poll_produce returned Poll::Pending \
2803                                                 after producing at least one item"
2804                                            )
2805                                        }
2806                                        Some(cx.waker().clone())
2807                                    } else {
2808                                        None
2809                                    };
2810                                }
2811
2812                                Ok(poll.map(|v| v.map(|result| (result, cancel))))
2813                            })?
2814                        })
2815                            .await?
2816                    } else {
2817                        (StreamResult::Completed, false)
2818                    };
2819
2820                    let (guest_offset, host_offset, count) = tls::get(|store| {
2821                        let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2822                        let (count, host_offset) = match &transmit.read {
2823                            &ReadState::GuestReady { count, .. } => (count.as_u32(), 0),
2824                            &ReadState::HostToHost { limit, .. } => (1, limit),
2825                            _ => bail_bug!("invalid read state"),
2826                        };
2827                        let guest_offset = match &transmit.write {
2828                            &WriteState::HostReady { guest_offset, .. } => guest_offset,
2829                            _ => bail_bug!("invalid write state"),
2830                        };
2831                        Ok((guest_offset, host_offset, count))
2832                    })?;
2833
2834                    let produced =
2835                        !buffer.remaining().is_empty() || guest_offset > 0 || host_offset > 0;
2836
2837                    match result {
2838                        StreamResult::Completed => {
2839                            // Only zero-length reads may complete without
2840                            // producing anything. Note that host-to-host reads
2841                            // always have a nonzero `count`.
2842                            if count > 0 && !produced {
2843                                bail!(
2844                                    "StreamProducer::poll_produce returned StreamResult::Completed \
2845                                     without producing any items"
2846                                );
2847                            }
2848                        }
2849                        StreamResult::Cancelled => {
2850                            if !cancelled {
2851                                bail!(
2852                                    "StreamProducer::poll_produce returned StreamResult::Cancelled \
2853                                     without being given a `finish` parameter value of true"
2854                                );
2855                            }
2856                            if produced {
2857                                bail!(
2858                                    "StreamProducer::poll_produce returned StreamResult::Cancelled \
2859                                     after producing at least one item"
2860                                );
2861                            }
2862                        }
2863                        StreamResult::Dropped => {
2864                            *dropped = true;
2865                        }
2866                    }
2867
2868                    let write_buffer = !buffer.remaining().is_empty() || host_offset > 0;
2869
2870                    let dropped = *dropped;
2871
2872                    drop(state);
2873
2874                    if write_buffer {
2875                        write(token, id, producer.clone(), kind).await?;
2876                    }
2877
2878                    Ok(if dropped {
2879                        if producer.with(|p| p.buffer.remaining().is_empty())? {
2880                            StreamResult::Dropped
2881                        } else {
2882                            StreamResult::Completed
2883                        }
2884                    } else {
2885                        result
2886                    })
2887                }
2888                .boxed()
2889            }
2890        });
2891        let try_into = Box::new(move |ty| {
2892            let ProducerState {
2893                producer: mine,
2894                buffer,
2895                dropped,
2896            } = producer.try_lock().ok()?.take()?;
2897            match P::try_into(mine, buffer, ty) {
2898                Ok(value) => Some(value),
2899                Err((mine, buffer)) => {
2900                    *producer.try_lock().ok()? = Some(ProducerState {
2901                        producer: mine,
2902                        buffer,
2903                        dropped,
2904                    });
2905                    None
2906                }
2907            }
2908        });
2909        state.get_mut(id)?.write = WriteState::HostReady {
2910            produce,
2911            try_into,
2912            guest_offset: ItemCount::ZERO,
2913            cancel: false,
2914            cancel_waker: None,
2915        };
2916        self.0.host_reader_insert(read)
2917    }
2918
2919    fn set_consumer<C: StreamConsumer<T>>(
2920        mut self,
2921        idx: HostResourceIndex,
2922        kind: TransmitKind,
2923        consumer: C,
2924    ) -> Result<()> {
2925        let token = StoreToken::new(self.as_context_mut());
2926        let id = self.0.host_reader_take(idx)?;
2927        let state = self.0.concurrent_state_mut()?;
2928        let id = state.get_mut(id)?.state;
2929        let transmit = state.get_mut(id)?;
2930        let consumer = Arc::new(LockedState::new(Box::pin(consumer)));
2931        let consume_with_buffer = {
2932            let consumer = consumer.clone();
2933            async move |mut host_buffer: Option<&mut dyn WriteBuffer<C::Item>>| {
2934                let mut mine = consumer.take()?;
2935
2936                let host_buffer_remaining_before =
2937                    host_buffer.as_deref_mut().map(|v| v.remaining().len());
2938
2939                let (result, cancelled) = future::poll_fn(|cx| {
2940                    tls::get(|store| {
2941                        let cancel = match &store.concurrent_state_mut()?.get_mut(id)?.read {
2942                            &ReadState::HostReady { cancel, .. } => cancel,
2943                            ReadState::Open => false,
2944                            _ => bail_bug!("unexpected read state"),
2945                        };
2946
2947                        let poll = mine.as_mut().poll_consume(
2948                            cx,
2949                            token.as_context_mut(store),
2950                            Source {
2951                                id,
2952                                host_buffer: host_buffer.as_deref_mut(),
2953                            },
2954                            cancel,
2955                        );
2956
2957                        if let ReadState::HostReady { cancel_waker, .. } =
2958                            &mut store.concurrent_state_mut()?.get_mut(id)?.read
2959                        {
2960                            *cancel_waker = if poll.is_pending() {
2961                                Some(cx.waker().clone())
2962                            } else {
2963                                None
2964                            };
2965                        }
2966
2967                        Ok(poll.map(|v| v.map(|result| (result, cancel))))
2968                    })?
2969                })
2970                .await?;
2971
2972                let (guest_offset, count) = tls::get(|store| {
2973                    let transmit = store.concurrent_state_mut()?.get_mut(id)?;
2974                    Ok((
2975                        match &transmit.read {
2976                            &ReadState::HostReady { guest_offset, .. } => guest_offset,
2977                            ReadState::Open => ItemCount::ZERO,
2978                            _ => bail_bug!("invalid read state"),
2979                        },
2980                        match &transmit.write {
2981                            WriteState::GuestReady { count, .. } => count.as_usize(),
2982                            WriteState::HostReady { .. } => match host_buffer_remaining_before {
2983                                Some(n) => n,
2984                                None => bail_bug!("host_buffer_remaining_before should be set"),
2985                            },
2986                            _ => bail_bug!("invalid write state"),
2987                        },
2988                    ))
2989                })?;
2990
2991                match result {
2992                    StreamResult::Completed => {
2993                        // Progress for a host writer is tracked by its buffer
2994                        // shrinking, whereas for a guest writer it's tracked
2995                        // in `guest_offset`.
2996                        let consumed = match (host_buffer_remaining_before, host_buffer) {
2997                            (Some(before), Some(after)) => before != after.remaining().len(),
2998                            _ => guest_offset > 0,
2999                        };
3000                        if count > 0 && !consumed {
3001                            bail!(
3002                                "StreamConsumer::poll_consume returned StreamResult::Completed \
3003                                 without consuming any items"
3004                            );
3005                        }
3006
3007                        if let TransmitKind::Future = kind {
3008                            tls::get(|store| {
3009                                store.concurrent_state_mut()?.get_mut(id)?.done = true;
3010                                crate::error::Ok(())
3011                            })?;
3012                        }
3013                    }
3014                    StreamResult::Cancelled => {
3015                        if !cancelled {
3016                            bail!(
3017                                "StreamConsumer::poll_consume returned StreamResult::Cancelled \
3018                                 without being given a `finish` parameter value of true"
3019                            );
3020                        }
3021                    }
3022                    StreamResult::Dropped => {}
3023                }
3024
3025                Ok(result)
3026            }
3027        };
3028        let consume = {
3029            let consume = consume_with_buffer.clone();
3030            Box::new(move || {
3031                let consume = consume.clone();
3032                async move { consume(None).await }.boxed()
3033            })
3034        };
3035
3036        match &transmit.write {
3037            WriteState::Open => {
3038                transmit.read = ReadState::HostReady {
3039                    consume,
3040                    guest_offset: ItemCount::ZERO,
3041                    cancel: false,
3042                    cancel_waker: None,
3043                };
3044            }
3045            &WriteState::GuestReady { .. } => {
3046                // If the guest writer has a pending event, then don't start
3047                // another read event here even if there's some partial data
3048                // available. Wait for that to be delivered and wait for the
3049                // writer to do something else before a read is triggered.
3050                let write_handle = transmit.write_handle;
3051                let write_has_event = Waitable::Transmit(write_handle)
3052                    .common(state)?
3053                    .event
3054                    .is_some();
3055                let future = if write_has_event {
3056                    None
3057                } else {
3058                    Some(consume())
3059                };
3060                state.get_mut(id)?.read = ReadState::HostReady {
3061                    consume,
3062                    guest_offset: ItemCount::ZERO,
3063                    cancel: false,
3064                    cancel_waker: None,
3065                };
3066                if let Some(future) = future {
3067                    self.0.pipe_from_guest(kind, id, future);
3068                }
3069            }
3070            WriteState::HostReady { .. } => {
3071                let WriteState::HostReady { produce, .. } = mem::replace(
3072                    &mut transmit.write,
3073                    WriteState::HostReady {
3074                        produce: Box::new(|| {
3075                            Box::pin(async { bail_bug!("unexpected invocation of `produce`") })
3076                        }),
3077                        try_into: Box::new(|_| None),
3078                        guest_offset: ItemCount::ZERO,
3079                        cancel: false,
3080                        cancel_waker: None,
3081                    },
3082                ) else {
3083                    bail_bug!("expected WriteState::HostReady")
3084                };
3085
3086                transmit.read = ReadState::HostToHost {
3087                    accept: Box::new(move |input| {
3088                        let consume = consume_with_buffer.clone();
3089                        async move { consume(Some(input.get_mut::<C::Item>()?)).await }.boxed()
3090                    }),
3091                    buffer: Vec::new(),
3092                    limit: 0,
3093                };
3094
3095                let future = async move {
3096                    loop {
3097                        if tls::get(|store| {
3098                            crate::error::Ok(matches!(
3099                                store.concurrent_state_mut()?.get_mut(id)?.read,
3100                                ReadState::Dropped
3101                            ))
3102                        })? {
3103                            break Ok(());
3104                        }
3105
3106                        match produce().await? {
3107                            StreamResult::Completed | StreamResult::Cancelled => {}
3108                            StreamResult::Dropped => break Ok(()),
3109                        }
3110
3111                        if let TransmitKind::Future = kind {
3112                            break Ok(());
3113                        }
3114                    }
3115                }
3116                .map(move |result| {
3117                    tls::get(|store| store.concurrent_state_mut()?.delete_transmit(id))?;
3118                    result
3119                });
3120
3121                state.push_future(Box::pin(future));
3122            }
3123            WriteState::Dropped => {
3124                let reader = transmit.read_handle;
3125                self.0.host_drop_reader(reader, kind)?;
3126            }
3127        }
3128        Ok(())
3129    }
3130}
3131
3132async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: WriteBuffer<T>>(
3133    token: StoreToken<D>,
3134    id: TableId<TransmitState>,
3135    producer: Arc<LockedState<ProducerState<P, B>>>,
3136    kind: TransmitKind,
3137) -> Result<()> {
3138    let (read, guest_offset) = tls::get(|store| {
3139        let transmit = store.concurrent_state_mut()?.get_mut(id)?;
3140
3141        let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write {
3142            Some(guest_offset)
3143        } else {
3144            None
3145        };
3146
3147        crate::error::Ok((
3148            mem::replace(&mut transmit.read, ReadState::Open),
3149            guest_offset,
3150        ))
3151    })?;
3152
3153    match read {
3154        ReadState::GuestReady {
3155            ty,
3156            flat_abi,
3157            options,
3158            address,
3159            count,
3160            handle,
3161            instance,
3162            caller_instance,
3163        } => {
3164            let guest_offset = match guest_offset {
3165                Some(i) => i,
3166                None => bail_bug!("guest_offset should be present if ready"),
3167            };
3168
3169            if let TransmitKind::Future = kind {
3170                tls::get(|store| {
3171                    store.concurrent_state_mut()?.get_mut(id)?.done = true;
3172                    crate::error::Ok(())
3173                })?;
3174            }
3175
3176            let old_remaining = producer.with(|p| p.buffer.remaining().len())?;
3177            let accept = {
3178                let producer = producer.clone();
3179                move |mut store: StoreContextMut<D>| {
3180                    let mut state = producer.take()?;
3181                    lower::<T, B, D>(
3182                        store.as_context_mut(),
3183                        instance,
3184                        options,
3185                        ty,
3186                        address + (T::SIZE32 * guest_offset.as_usize()),
3187                        count.as_usize() - guest_offset.as_usize(),
3188                        &mut state.buffer,
3189                    )?;
3190                    crate::error::Ok(())
3191                }
3192            };
3193
3194            if guest_offset < count {
3195                if T::MAY_REQUIRE_REALLOC {
3196                    // For payloads which may require a realloc call, use a
3197                    // oneshot::channel and background task.  This is
3198                    // necessary because calling the guest while there are
3199                    // host embedder frames on the stack is unsound.
3200                    let (tx, rx) = oneshot::channel();
3201                    tls::get(move |store| {
3202                        store
3203                            .concurrent_state_mut()?
3204                            .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(
3205                                Box::new(move |store| {
3206                                    _ = tx.send(accept(token.as_context_mut(store))?);
3207                                    Ok(())
3208                                }),
3209                            )));
3210                        crate::error::Ok(())
3211                    })?;
3212                    match rx.await {
3213                        Ok(r) => r,
3214                        // The worker function was dropped without sending a
3215                        // result, which can happen when lowering trapped. This
3216                        // isn't a bug but this specific handling here might be
3217                        // the last hurray of this task in the event queue, so
3218                        // don't kill anything other than our caller. The
3219                        // original trap should make its way back to the best
3220                        // location naturally.
3221                        Err(oneshot::Canceled) => {
3222                            bail!("stream write cancelled before it could complete")
3223                        }
3224                    }
3225                } else {
3226                    // Optimize flat payloads (i.e. those which do not
3227                    // require calling the guest's realloc function) by
3228                    // lowering directly instead of using a oneshot::channel
3229                    // and background task.
3230                    tls::get(|store| accept(token.as_context_mut(store)))?
3231                }
3232            }
3233
3234            tls::get(|store| {
3235                let count = old_remaining - producer.with(|p| p.buffer.remaining().len())?;
3236
3237                let transmit = store.concurrent_state_mut()?.get_mut(id)?;
3238
3239                let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
3240                    bail_bug!("expected WriteState::HostReady")
3241                };
3242
3243                guest_offset.inc(count)?;
3244
3245                transmit.read = ReadState::GuestReady {
3246                    ty,
3247                    flat_abi,
3248                    options,
3249                    address,
3250                    count: ItemCount::new_usize(count)?,
3251                    handle,
3252                    instance,
3253                    caller_instance,
3254                };
3255
3256                crate::error::Ok(())
3257            })?;
3258
3259            Ok(())
3260        }
3261
3262        ReadState::HostToHost {
3263            accept,
3264            mut buffer,
3265            limit,
3266        } => {
3267            let mut state = StreamResult::Completed;
3268            let mut position = 0;
3269
3270            while !matches!(state, StreamResult::Dropped) && position < limit {
3271                let mut slice_buffer = SliceBuffer::new(buffer, position, limit);
3272                state = accept(&mut UntypedWriteBuffer::new(&mut slice_buffer)).await?;
3273                (buffer, position, _) = slice_buffer.into_parts();
3274            }
3275
3276            {
3277                let mut producer = producer.take()?;
3278                let ProducerState { buffer, .. } = &mut *producer;
3279
3280                while !(matches!(state, StreamResult::Dropped) || buffer.remaining().is_empty()) {
3281                    state = accept(&mut UntypedWriteBuffer::new(buffer)).await?;
3282                }
3283            }
3284
3285            tls::get(|store| {
3286                store.concurrent_state_mut()?.get_mut(id)?.read = match state {
3287                    StreamResult::Dropped => ReadState::Dropped,
3288                    StreamResult::Completed | StreamResult::Cancelled => ReadState::HostToHost {
3289                        accept,
3290                        buffer,
3291                        limit: 0,
3292                    },
3293                };
3294
3295                crate::error::Ok(())
3296            })?;
3297            Ok(())
3298        }
3299
3300        _ => bail_bug!("unexpected read state"),
3301    }
3302}
3303
3304/// Polls a `StreamProducer` or `StreamConsumer` future once, inline, from
3305/// within a guest's `{stream,future}.{read,write}` intrinsic.
3306///
3307/// The producer or consumer is embedder code which doesn't belong to the
3308/// reading or writing guest's task, so it's polled as though from the
3309/// top-level host, just as it is when it's later polled from the event loop.
3310fn poll_inline(
3311    store: &mut dyn VMStore,
3312    future: &mut Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,
3313) -> Result<Poll<Result<StreamResult>>> {
3314    let old_thread = store.set_thread(CurrentThread::None)?;
3315    let poll = tls::set(store, || {
3316        future
3317            .as_mut()
3318            .poll(&mut Context::from_waker(&Waker::noop()))
3319    });
3320    store.set_thread(old_thread)?;
3321    Ok(poll)
3322}
3323
3324impl Instance {
3325    /// Handle a guest-initiated write by delivering the item(s) to the
3326    /// `StreamConsumer` for the specified stream or future.
3327    fn consume(
3328        self,
3329        store: &mut dyn VMStore,
3330        kind: TransmitKind,
3331        transmit_id: TableId<TransmitState>,
3332        consume: PollStream,
3333        guest_offset: ItemCount,
3334        cancel: bool,
3335    ) -> Result<ReturnCode> {
3336        let mut future = consume();
3337        store.concurrent_state_mut()?.get_mut(transmit_id)?.read = ReadState::HostReady {
3338            consume,
3339            guest_offset,
3340            cancel,
3341            cancel_waker: None,
3342        };
3343        let poll = poll_inline(store, &mut future)?;
3344
3345        Ok(match poll {
3346            Poll::Ready(state) => {
3347                let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
3348                let code = settle_host_read(transmit, kind, state?)?;
3349                transmit.write = WriteState::Open;
3350                code
3351            }
3352            Poll::Pending => {
3353                store.pipe_from_guest(kind, transmit_id, future);
3354                ReturnCode::Blocked
3355            }
3356        })
3357    }
3358
3359    /// Handle a guest-initiated read by polling the `StreamProducer` for the
3360    /// specified stream or future for items.
3361    fn produce(
3362        self,
3363        store: &mut dyn VMStore,
3364        kind: TransmitKind,
3365        transmit_id: TableId<TransmitState>,
3366        produce: PollStream,
3367        try_into: TryInto,
3368        guest_offset: ItemCount,
3369        cancel: bool,
3370    ) -> Result<ReturnCode> {
3371        let mut future = produce();
3372        store.concurrent_state_mut()?.get_mut(transmit_id)?.write = WriteState::HostReady {
3373            produce,
3374            try_into,
3375            guest_offset,
3376            cancel,
3377            cancel_waker: None,
3378        };
3379        let poll = poll_inline(store, &mut future)?;
3380
3381        Ok(match poll {
3382            Poll::Ready(state) => {
3383                let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
3384                let code = settle_host_write(transmit, kind, state?)?;
3385                transmit.read = ReadState::Open;
3386                code
3387            }
3388            Poll::Pending => {
3389                store.pipe_to_guest(kind, transmit_id, future);
3390                ReturnCode::Blocked
3391            }
3392        })
3393    }
3394
3395    /// Drop the writable end of the specified stream or future from the guest.
3396    pub(super) fn guest_drop_writable(
3397        self,
3398        store: &mut StoreOpaque,
3399        ty: TransmitIndex,
3400        writer: u32,
3401    ) -> Result<()> {
3402        let table = self.id().get_mut(store).table_for_transmit(ty);
3403        let (transmit_rep, is_done) = match ty {
3404            TransmitIndex::Future(ty) => table.future_remove_writable(ty, writer)?,
3405            TransmitIndex::Stream(ty) => (table.stream_remove_writable(ty, writer)?, false),
3406        };
3407
3408        let id = TableId::<TransmitHandle>::new(transmit_rep);
3409        log::trace!("guest_drop_writable: drop writer {id:?}");
3410        match ty {
3411            TransmitIndex::Stream(_) => store.host_drop_writer(id, None),
3412            TransmitIndex::Future(_) => store.host_drop_writer(
3413                id,
3414                if is_done {
3415                    None
3416                } else {
3417                    Some(|| {
3418                        Err(format_err!(
3419                            "cannot drop future write end without first writing a value"
3420                        ))
3421                    })
3422                },
3423            ),
3424        }
3425    }
3426
3427    /// Copy `count` items from `read_address` to `write_address` for the
3428    /// specified stream or future.
3429    fn copy<T: 'static>(
3430        store: StoreContextMut<T>,
3431        flat_abi: Option<FlatAbi>,
3432        write_runtime_instance: RuntimeInstance,
3433        write_ty: TransmitIndex,
3434        write_options: OptionsIndex,
3435        write_address: usize,
3436        read_runtime_instance: RuntimeInstance,
3437        read_ty: TransmitIndex,
3438        read_options: OptionsIndex,
3439        read_address: usize,
3440        count: ItemCount,
3441        rep: u32,
3442    ) -> Result<()> {
3443        let write_instance = Instance::from_runtime_instance(store.0, write_runtime_instance);
3444        let read_instance = Instance::from_runtime_instance(store.0, read_runtime_instance);
3445        let (write_component, store) = write_instance.component_and_store_mut(store.0);
3446        let (read_component, mut store) = read_instance.component_and_store_mut(store);
3447        let write_types = write_component.types();
3448        let read_types = read_component.types();
3449        let count = count.as_usize();
3450
3451        // Double-check that both buffers are properly aligned and in-bounds.
3452        // This was already validated in `check_bounds` when the read/write was
3453        // issued, so failure here is a bug.
3454        //
3455        // FIXME(#4311): this and the rest of this function use the 32-bit
3456        // layout of payloads, which needs memory64 handling.
3457        let write_payload_ty = write_ty.payload(write_types);
3458        let write_abi = match write_payload_ty {
3459            Some(ty) => write_types.canonical_abi(ty),
3460            None => &CanonicalAbiInfo::ZERO,
3461        };
3462        let write_length_in_bytes = match flat_abi {
3463            Some(abi) => usize::try_from(abi.size)? * count,
3464            None => usize::try_from(write_abi.size32)? * count,
3465        };
3466        if write_payload_ty.is_some() {
3467            if write_address % usize::try_from(write_abi.align32)? != 0 {
3468                bail_bug!("write pointer not aligned");
3469            }
3470            if write_instance
3471                .options_memory(store, write_options)
3472                .get(write_address..)
3473                .and_then(|b| b.get(..write_length_in_bytes))
3474                .is_none()
3475            {
3476                bail_bug!("write pointer out of bounds");
3477            }
3478        }
3479
3480        let read_payload_ty = read_ty.payload(read_types);
3481        let read_abi = match read_payload_ty {
3482            Some(ty) => read_types.canonical_abi(ty),
3483            None => &CanonicalAbiInfo::ZERO,
3484        };
3485        let read_length_in_bytes = match flat_abi {
3486            Some(abi) => usize::try_from(abi.size)? * count,
3487            None => usize::try_from(read_abi.size32)? * count,
3488        };
3489        if read_payload_ty.is_some() {
3490            if read_address % usize::try_from(read_abi.align32)? != 0 {
3491                bail_bug!("read pointer not aligned");
3492            }
3493            if read_instance
3494                .options_memory(store, read_options)
3495                .get(read_address..)
3496                .and_then(|b| b.get(..read_length_in_bytes))
3497                .is_none()
3498            {
3499                bail_bug!("read pointer out of bounds");
3500            }
3501        }
3502
3503        if write_runtime_instance == read_runtime_instance
3504            && !allow_intra_component_read_write(write_payload_ty)
3505        {
3506            bail!(
3507                "cannot read from and write to intra-component future/stream with non-numeric payload"
3508            )
3509        }
3510
3511        match (write_ty, read_ty) {
3512            (TransmitIndex::Future(_), TransmitIndex::Future(_)) => {
3513                if count != 1 {
3514                    bail_bug!("futures can only send 1 item");
3515                }
3516
3517                let val = write_payload_ty
3518                    .map(|ty| {
3519                        let lift = &mut LiftContext::new(store, write_options, write_instance)?;
3520                        let bytes = &lift.memory()[write_address..][..write_length_in_bytes];
3521                        Val::load(lift, *ty, bytes)
3522                    })
3523                    .transpose()?;
3524
3525                if let Some(val) = val {
3526                    let lower =
3527                        &mut LowerContext::new(store.as_context_mut(), read_options, read_instance);
3528                    let ptr = func::validate_inbounds_dynamic(
3529                        read_abi,
3530                        lower.as_slice(),
3531                        &ValRaw::u32(read_address.try_into()?),
3532                    )?;
3533                    let ty = match read_payload_ty {
3534                        Some(ty) => ty,
3535                        None => bail_bug!("expected read payload type to be present"),
3536                    };
3537                    val.store(lower, *ty, ptr)?;
3538                }
3539            }
3540            (TransmitIndex::Stream(_), TransmitIndex::Stream(_)) => {
3541                if write_length_in_bytes == 0 {
3542                    return Ok(());
3543                }
3544                let write_payload_ty = match write_payload_ty {
3545                    Some(ty) => ty,
3546                    None => bail_bug!("expected write payload type to be present"),
3547                };
3548                let read_payload_ty = match read_payload_ty {
3549                    Some(ty) => ty,
3550                    None => bail_bug!("expected read payload type to be present"),
3551                };
3552                let lift =
3553                    &mut LiftContext::new(store.store_opaque_mut(), write_options, write_instance)?;
3554                if let Some(abi) = flat_abi {
3555                    // The host copies these bytes itself rather than wasm, so
3556                    // charge them against hostcall fuel.
3557                    lift.consume_fuel_array(count, usize::try_from(abi.size)?)?;
3558
3559                    // Fast path memcpy for "flat" (i.e. no pointers or handles) payloads:
3560                    let store_opaque = store.store_opaque_mut();
3561
3562                    assert_eq!(read_length_in_bytes, write_length_in_bytes);
3563
3564                    if read_instance
3565                        .options_memory(store_opaque, read_options)
3566                        .as_ptr()
3567                        == write_instance
3568                            .options_memory(store_opaque, write_options)
3569                            .as_ptr()
3570                    {
3571                        let memory = read_instance.options_memory_mut(store_opaque, read_options);
3572                        memory.copy_within(
3573                            write_address..write_address + write_length_in_bytes,
3574                            read_address,
3575                        );
3576                    } else {
3577                        let src = write_instance.options_memory(store_opaque, write_options)
3578                            [write_address..][..write_length_in_bytes]
3579                            .as_ptr();
3580                        let dst = read_instance
3581                            .options_memory_range_mut(
3582                                store_opaque,
3583                                read_options,
3584                                read_address,
3585                                read_length_in_bytes,
3586                            )
3587                            .unwrap()
3588                            .as_mut_ptr();
3589
3590                        // SAFETY: Both `src` and `dst` have been validated
3591                        // above to be valid pointers as they're derived from
3592                        // slices that have the desired length with the desired
3593                        // read/write permission. The `unsafe` bit here is that
3594                        // the memories are disjoint (different base pointers)
3595                        // and there's no easy way to borrow both
3596                        // simultaneously from the store. Different memories
3597                        // are guaranteed to be disjoint, however, so the
3598                        // `unsafe` here should be ok.
3599                        unsafe {
3600                            src.copy_to_nonoverlapping(dst, write_length_in_bytes);
3601                        }
3602                    }
3603                } else {
3604                    let bytes = &lift.memory()[write_address..][..write_length_in_bytes];
3605                    lift.consume_fuel_array(count, size_of::<Val>())?;
3606
3607                    let values = (0..count)
3608                        .map(|index| {
3609                            let size = usize::try_from(write_abi.size32)?;
3610                            Val::load(lift, *write_payload_ty, &bytes[(index * size)..][..size])
3611                        })
3612                        .collect::<Result<Vec<_>>>()?;
3613
3614                    let id = TableId::<TransmitHandle>::new(rep);
3615                    log::trace!("copy values {values:?} for {id:?}");
3616
3617                    let lower =
3618                        &mut LowerContext::new(store.as_context_mut(), read_options, read_instance);
3619                    let mut ptr = read_address;
3620                    for value in values {
3621                        value.store(lower, *read_payload_ty, ptr)?;
3622                        ptr += usize::try_from(read_abi.size32)?;
3623                    }
3624                }
3625            }
3626            _ => bail_bug!("mismatched transmit types in copy"),
3627        }
3628
3629        Ok(())
3630    }
3631
3632    fn check_bounds(
3633        self,
3634        store: &StoreOpaque,
3635        options: OptionsIndex,
3636        ty: TransmitIndex,
3637        address: usize,
3638        count: usize,
3639    ) -> Result<()> {
3640        let types = self.id().get(store).component().types();
3641        let payload = match ty {
3642            TransmitIndex::Future(ty) => types[types[ty].ty].payload,
3643            TransmitIndex::Stream(ty) => types[types[ty].ty].payload,
3644        };
3645        // Empty payloads ignore the pointer entirely, but otherwise the
3646        // alignment and bounds checks apply even to zero-length reads/writes.
3647        let Some(payload) = payload else {
3648            return Ok(());
3649        };
3650        let abi = types.canonical_abi(&payload);
3651        // FIXME(#4311): needs memory64 handling
3652        let size = usize::try_from(abi.size32)?;
3653        let align = usize::try_from(abi.align32)?;
3654        if address % align != 0 {
3655            bail!("buffer pointer not aligned");
3656        }
3657        self.options_memory(store, options)
3658            .get(address..)
3659            .and_then(|b| b.get(..size.checked_mul(count)?))
3660            .map(drop)
3661            .ok_or_else(|| crate::format_err!("buffer pointer out of bounds of memory"))
3662    }
3663
3664    /// Write to the specified stream or future from the guest.
3665    pub(super) fn guest_write<T: 'static>(
3666        self,
3667        mut store: StoreContextMut<T>,
3668        caller: RuntimeComponentInstanceIndex,
3669        ty: TransmitIndex,
3670        options: OptionsIndex,
3671        flat_abi: Option<FlatAbi>,
3672        handle: u32,
3673        address: u32,
3674        count: u32,
3675    ) -> Result<ReturnCode> {
3676        let count = ItemCount::new(count)?;
3677
3678        let address = usize::try_from(address)?;
3679        self.check_bounds(store.0, options, ty, address, count.as_usize())?;
3680        let async_ = self.options(store.0, options).async_;
3681        let (rep, state) = self.id().get_mut(store.0).get_mut_by_index(ty, handle)?;
3682        let TransmitLocalStateKind::Write { done } = state.kind() else {
3683            bail!(Trap::ConcurrentFutureStreamOp);
3684        };
3685
3686        *state = VMTransmitLocalState::busy();
3687        let transmit_handle = TableId::<TransmitHandle>::new(rep);
3688        let concurrent_state = store.0.concurrent_state_mut()?;
3689        let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;
3690        let transmit = concurrent_state.get_mut(transmit_id)?;
3691        log::trace!(
3692            "guest_write {count} to {transmit_handle:?} (handle {handle}; state {transmit_id:?}); {:?}",
3693            transmit.read
3694        );
3695
3696        if done || transmit.done {
3697            bail!(match ty {
3698                TransmitIndex::Future(_) => Trap::WriteToDroppedFuture,
3699                TransmitIndex::Stream(_) => Trap::WriteToDroppedStream,
3700            });
3701        }
3702
3703        if !async_ {
3704            Waitable::Transmit(transmit_handle).trap_if_in_waitable_set(concurrent_state)?;
3705        }
3706        let transmit = concurrent_state.get_mut(transmit_id)?;
3707
3708        let new_state = if let ReadState::Dropped = &transmit.read {
3709            ReadState::Dropped
3710        } else {
3711            ReadState::Open
3712        };
3713
3714        let set_guest_ready = |me: &mut ConcurrentState| {
3715            let transmit = me.get_mut(transmit_id)?;
3716            if !matches!(&transmit.write, WriteState::Open) {
3717                bail_bug!("expected `WriteState::Open`; got `{:?}`", transmit.write);
3718            }
3719            transmit.write = WriteState::GuestReady {
3720                instance: self,
3721                caller,
3722                ty,
3723                flat_abi,
3724                options,
3725                address,
3726                count,
3727                handle,
3728            };
3729            Ok::<_, crate::Error>(())
3730        };
3731
3732        let mut result = match mem::replace(&mut transmit.read, new_state) {
3733            ReadState::GuestReady {
3734                ty: read_ty,
3735                flat_abi: read_flat_abi,
3736                options: read_options,
3737                address: read_address,
3738                count: read_count,
3739                handle: read_handle,
3740                instance: read_instance,
3741                caller_instance: read_caller_instance,
3742            } => {
3743                if flat_abi != read_flat_abi {
3744                    bail_bug!("expected flat ABI calculations to be the same");
3745                }
3746
3747                if let TransmitIndex::Future(_) = ty {
3748                    transmit.done = true;
3749                }
3750
3751                // Note that zero-length reads and writes are handling specially
3752                // by the spec to allow each end to signal readiness to the
3753                // other.  Quoting the spec:
3754                //
3755                // ```
3756                // The meaning of a read or write when the length is 0 is that
3757                // the caller is querying the "readiness" of the other
3758                // side. When a 0-length read/write rendezvous with a
3759                // non-0-length read/write, only the 0-length read/write
3760                // completes; the non-0-length read/write is kept pending (and
3761                // ready for a subsequent rendezvous).
3762                //
3763                // In the corner case where a 0-length read and write
3764                // rendezvous, only the writer is notified of readiness. To
3765                // avoid livelock, the Canonical ABI requires that a writer must
3766                // (eventually) follow a completed 0-length write with a
3767                // non-0-length write that is allowed to block (allowing the
3768                // reader end to run and rendezvous with its own non-0-length
3769                // read).
3770                // ```
3771
3772                let write_count = count;
3773                let write_complete = count == 0 || read_count > 0;
3774                let read_complete = count > 0;
3775                let read_buffer_remaining = count < read_count;
3776
3777                let read_handle_rep = transmit.read_handle.rep();
3778
3779                let count = count.min(read_count);
3780
3781                Instance::copy(
3782                    store.as_context_mut(),
3783                    flat_abi,
3784                    self.runtime_instance(caller),
3785                    ty,
3786                    options,
3787                    address,
3788                    read_instance.runtime_instance(read_caller_instance),
3789                    read_ty,
3790                    read_options,
3791                    read_address,
3792                    count,
3793                    rep,
3794                )?;
3795
3796                let instance = read_instance.id().get(store.0);
3797                let types = instance.component().types();
3798                // FIXME(#4311): needs memory64 handling
3799                let item_size = match read_ty.payload(types) {
3800                    Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,
3801                    None => 0,
3802                };
3803                let concurrent_state = store.0.concurrent_state_mut()?;
3804                if read_complete {
3805                    let total = if let Some(Event::StreamRead {
3806                        code: ReturnCode::Completed(old_total),
3807                        ..
3808                    }) = concurrent_state.take_event(read_handle_rep)?
3809                    {
3810                        count.add(old_total)?
3811                    } else {
3812                        count
3813                    };
3814
3815                    let code = ReturnCode::completed(ty.kind(), total);
3816
3817                    concurrent_state.send_read_result(read_ty, transmit_id, read_handle, code)?;
3818                }
3819
3820                // If the reader still has buffer remaining, or if this was a
3821                // zero-length rendezvous, then restore the state of the reader
3822                // back to what it was when we found it. Note that for the
3823                // zero-length rendezvous case this specifically won't execute
3824                // the `read_complete` logic above, which is intentional, as the
3825                // reader remains blocked.
3826                if read_buffer_remaining || (write_count == 0 && read_count == 0) {
3827                    let transmit = concurrent_state.get_mut(transmit_id)?;
3828                    transmit.read = ReadState::GuestReady {
3829                        ty: read_ty,
3830                        flat_abi: read_flat_abi,
3831                        options: read_options,
3832                        address: read_address + (count.as_usize() * item_size),
3833                        count: read_count.sub(count)?,
3834                        handle: read_handle,
3835                        instance: read_instance,
3836                        caller_instance: read_caller_instance,
3837                    };
3838                }
3839
3840                if write_complete {
3841                    ReturnCode::completed(ty.kind(), count)
3842                } else {
3843                    set_guest_ready(concurrent_state)?;
3844                    ReturnCode::Blocked
3845                }
3846            }
3847
3848            ReadState::HostReady {
3849                consume,
3850                guest_offset,
3851                cancel,
3852                cancel_waker,
3853            } => {
3854                if cancel_waker.is_some() {
3855                    bail_bug!("expected cancel_waker to be none");
3856                }
3857                if cancel {
3858                    bail_bug!("expected cancel to be false");
3859                }
3860                if guest_offset != 0 {
3861                    bail_bug!("expected guest_offset to be 0");
3862                }
3863
3864                set_guest_ready(concurrent_state)?;
3865                self.consume(
3866                    store.0,
3867                    ty.kind(),
3868                    transmit_id,
3869                    consume,
3870                    ItemCount::ZERO,
3871                    false,
3872                )?
3873            }
3874
3875            ReadState::HostToHost { .. } => bail_bug!("unexpected HostToHost"),
3876
3877            ReadState::Open => {
3878                set_guest_ready(concurrent_state)?;
3879                ReturnCode::Blocked
3880            }
3881
3882            ReadState::Dropped => {
3883                if let TransmitIndex::Future(_) = ty {
3884                    transmit.done = true;
3885                }
3886
3887                match Waitable::Transmit(transmit_handle).take_event(concurrent_state)? {
3888                    Some(
3889                        Event::StreamWrite {
3890                            code: ReturnCode::Dropped(ItemCount::ZERO),
3891                            ..
3892                        }
3893                        | Event::FutureWrite {
3894                            code: ReturnCode::Dropped(ItemCount::ZERO),
3895                            ..
3896                        },
3897                    ) => {}
3898                    event => bail_bug!("expected pending dropped event for writer; got {event:?}"),
3899                }
3900
3901                ReturnCode::Dropped(ItemCount::ZERO)
3902            }
3903        };
3904
3905        if result == ReturnCode::Blocked && !async_ {
3906            result = self.wait_for_write(store.0, caller, transmit_handle)?;
3907        }
3908
3909        if result != ReturnCode::Blocked {
3910            *self.id().get_mut(store.0).get_mut_by_index(ty, handle)?.1 =
3911                VMTransmitLocalState::write(matches!(result, ReturnCode::Dropped(_)));
3912        }
3913
3914        log::trace!(
3915            "guest_write result for {transmit_handle:?} (handle {handle}; state {transmit_id:?}): {result:?}",
3916        );
3917
3918        Ok(result)
3919    }
3920
3921    /// Read from the specified stream or future from the guest.
3922    pub(super) fn guest_read<T: 'static>(
3923        self,
3924        mut store: StoreContextMut<T>,
3925        caller_instance: RuntimeComponentInstanceIndex,
3926        ty: TransmitIndex,
3927        options: OptionsIndex,
3928        flat_abi: Option<FlatAbi>,
3929        handle: u32,
3930        address: u32,
3931        count: u32,
3932    ) -> Result<ReturnCode> {
3933        let count = ItemCount::new(count)?;
3934
3935        let address = usize::try_from(address)?;
3936        self.check_bounds(store.0, options, ty, address, count.as_usize())?;
3937        let async_ = self.options(store.0, options).async_;
3938        let (rep, state) = self.id().get_mut(store.0).get_mut_by_index(ty, handle)?;
3939        let TransmitLocalStateKind::Read { done } = state.kind() else {
3940            bail!(Trap::ConcurrentFutureStreamOp);
3941        };
3942
3943        *state = VMTransmitLocalState::busy();
3944        let transmit_handle = TableId::<TransmitHandle>::new(rep);
3945        let concurrent_state = store.0.concurrent_state_mut()?;
3946        let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;
3947        let transmit = concurrent_state.get_mut(transmit_id)?;
3948        log::trace!(
3949            "guest_read {count} from {transmit_handle:?} (handle {handle}; state {transmit_id:?}); {:?}",
3950            transmit.write
3951        );
3952
3953        if done || transmit.done {
3954            match ty {
3955                TransmitIndex::Future(_) => {
3956                    bail!("cannot read from future after previous read succeeded")
3957                }
3958                TransmitIndex::Stream(_) => bail!(Trap::ReadFromDroppedStream),
3959            }
3960        }
3961
3962        if !async_ {
3963            Waitable::Transmit(transmit_handle).trap_if_in_waitable_set(concurrent_state)?;
3964        }
3965        let transmit = concurrent_state.get_mut(transmit_id)?;
3966
3967        let new_state = if let WriteState::Dropped = &transmit.write {
3968            WriteState::Dropped
3969        } else {
3970            WriteState::Open
3971        };
3972
3973        let set_guest_ready = |me: &mut ConcurrentState| {
3974            let transmit = me.get_mut(transmit_id)?;
3975            if !matches!(&transmit.read, ReadState::Open) {
3976                bail_bug!("expected `ReadState::Open`; got `{:?}`", transmit.read);
3977            }
3978            transmit.read = ReadState::GuestReady {
3979                ty,
3980                flat_abi,
3981                options,
3982                address,
3983                count,
3984                handle,
3985                instance: self,
3986                caller_instance,
3987            };
3988            Ok::<_, crate::Error>(())
3989        };
3990
3991        let mut result = match mem::replace(&mut transmit.write, new_state) {
3992            WriteState::GuestReady {
3993                instance: write_instance,
3994                ty: write_ty,
3995                flat_abi: write_flat_abi,
3996                options: write_options,
3997                address: write_address,
3998                count: write_count,
3999                handle: write_handle,
4000                caller: write_caller,
4001            } => {
4002                if flat_abi != write_flat_abi {
4003                    bail_bug!("expected flat ABI calculations to be the same");
4004                }
4005
4006                if let TransmitIndex::Future(_) = ty {
4007                    transmit.done = true;
4008                }
4009
4010                let write_handle_rep = transmit.write_handle.rep();
4011
4012                // See the comment in `guest_write` for the
4013                // `ReadState::GuestReady` case concerning zero-length reads and
4014                // writes.
4015
4016                let write_complete = write_count == 0 || count > 0;
4017                let read_complete = write_count > 0;
4018                let write_buffer_remaining = count < write_count;
4019
4020                let count = count.min(write_count);
4021
4022                Instance::copy(
4023                    store.as_context_mut(),
4024                    flat_abi,
4025                    write_instance.runtime_instance(write_caller),
4026                    write_ty,
4027                    write_options,
4028                    write_address,
4029                    self.runtime_instance(caller_instance),
4030                    ty,
4031                    options,
4032                    address,
4033                    count,
4034                    rep,
4035                )?;
4036
4037                let instance = write_instance.id().get(store.0);
4038                let types = instance.component().types();
4039                // FIXME(#4311): needs memory64 handling
4040                let item_size = match write_ty.payload(types) {
4041                    Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,
4042                    None => 0,
4043                };
4044                let concurrent_state = store.0.concurrent_state_mut()?;
4045
4046                if write_complete {
4047                    let total = if let Some(Event::StreamWrite {
4048                        code: ReturnCode::Completed(old_total),
4049                        ..
4050                    }) = concurrent_state.take_event(write_handle_rep)?
4051                    {
4052                        count.add(old_total)?
4053                    } else {
4054                        count
4055                    };
4056
4057                    let code = ReturnCode::completed(ty.kind(), total);
4058
4059                    concurrent_state.send_write_result(
4060                        write_ty,
4061                        transmit_id,
4062                        write_handle,
4063                        code,
4064                    )?;
4065                }
4066
4067                if write_buffer_remaining {
4068                    let transmit = concurrent_state.get_mut(transmit_id)?;
4069                    transmit.write = WriteState::GuestReady {
4070                        instance: write_instance,
4071                        caller: write_caller,
4072                        ty: write_ty,
4073                        flat_abi: write_flat_abi,
4074                        options: write_options,
4075                        address: write_address + (count.as_usize() * item_size),
4076                        count: write_count.sub(count)?,
4077                        handle: write_handle,
4078                    };
4079                }
4080
4081                if read_complete {
4082                    ReturnCode::completed(ty.kind(), count)
4083                } else {
4084                    set_guest_ready(concurrent_state)?;
4085                    ReturnCode::Blocked
4086                }
4087            }
4088
4089            WriteState::HostReady {
4090                produce,
4091                try_into,
4092                guest_offset,
4093                cancel,
4094                cancel_waker,
4095            } => {
4096                if cancel_waker.is_some() {
4097                    bail_bug!("expected cancel_waker to be none");
4098                }
4099                if cancel {
4100                    bail_bug!("expected cancel to be false");
4101                }
4102                if guest_offset != 0 {
4103                    bail_bug!("expected guest_offset to be 0");
4104                }
4105
4106                set_guest_ready(concurrent_state)?;
4107
4108                let code = self.produce(
4109                    store.0,
4110                    ty.kind(),
4111                    transmit_id,
4112                    produce,
4113                    try_into,
4114                    ItemCount::ZERO,
4115                    false,
4116                )?;
4117
4118                if let (TransmitIndex::Future(_), ReturnCode::Completed(_)) = (ty, code) {
4119                    store.0.concurrent_state_mut()?.get_mut(transmit_id)?.done = true;
4120                }
4121
4122                code
4123            }
4124
4125            WriteState::Open => {
4126                set_guest_ready(concurrent_state)?;
4127                ReturnCode::Blocked
4128            }
4129
4130            WriteState::Dropped => {
4131                if let TransmitIndex::Future(_) = ty {
4132                    bail_bug!(
4133                        "should not be possible to read from a future whose write end was dropped"
4134                    );
4135                }
4136
4137                match Waitable::Transmit(transmit_handle).take_event(concurrent_state)? {
4138                    Some(Event::StreamRead {
4139                        code: ReturnCode::Dropped(ItemCount::ZERO),
4140                        ..
4141                    }) => {}
4142                    event => bail_bug!("expected pending dropped event for reader; got {event:?}"),
4143                }
4144
4145                ReturnCode::Dropped(ItemCount::ZERO)
4146            }
4147        };
4148
4149        if result == ReturnCode::Blocked && !async_ {
4150            result = self.wait_for_read(store.0, caller_instance, transmit_handle)?;
4151        }
4152
4153        if result != ReturnCode::Blocked {
4154            *self.id().get_mut(store.0).get_mut_by_index(ty, handle)?.1 =
4155                VMTransmitLocalState::read(matches!(
4156                    (result, ty),
4157                    (ReturnCode::Dropped(_), TransmitIndex::Stream(_))
4158                ));
4159        }
4160
4161        log::trace!(
4162            "guest_read result for {transmit_handle:?} (handle {handle}; state {transmit_id:?}): {result:?}",
4163        );
4164
4165        Ok(result)
4166    }
4167
4168    fn wait_for_write(
4169        self,
4170        store: &mut StoreOpaque,
4171        caller: RuntimeComponentInstanceIndex,
4172        handle: TableId<TransmitHandle>,
4173    ) -> Result<ReturnCode> {
4174        let waitable = Waitable::Transmit(handle);
4175        store.wait_for_event(self.runtime_instance(caller), waitable)?;
4176        let event = waitable.take_event(store.concurrent_state_mut()?)?;
4177        if let Some(event @ (Event::StreamWrite { code, .. } | Event::FutureWrite { code, .. })) =
4178            event
4179        {
4180            waitable.on_delivery(store, self, event)?;
4181            Ok(code)
4182        } else {
4183            bail_bug!("expected either a stream or future write event")
4184        }
4185    }
4186
4187    /// Cancel a pending stream or future write.
4188    fn cancel_write(
4189        self,
4190        store: &mut StoreOpaque,
4191        caller: RuntimeComponentInstanceIndex,
4192        transmit_id: TableId<TransmitState>,
4193        async_: bool,
4194    ) -> Result<ReturnCode> {
4195        let state = store.concurrent_state_mut()?;
4196        let transmit = state.get_mut(transmit_id)?;
4197        log::trace!(
4198            "host_cancel_write state {transmit_id:?}; write state {:?} read state {:?}",
4199            transmit.read,
4200            transmit.write
4201        );
4202        let waitable = Waitable::Transmit(transmit.write_handle);
4203
4204        waitable.trap_if_has_sync_waiter(state)?;
4205        if !async_ {
4206            waitable.trap_if_in_waitable_set(state)?;
4207        }
4208
4209        let code = if let Some(event) = waitable.take_event(state)? {
4210            let (Event::FutureWrite { code, .. } | Event::StreamWrite { code, .. }) = event else {
4211                bail_bug!("expected either a stream or future write event")
4212            };
4213            waitable.on_delivery(store, self, event)?;
4214            match (code, event) {
4215                (ReturnCode::Completed(count), Event::StreamWrite { .. }) => {
4216                    ReturnCode::Cancelled(count)
4217                }
4218                (ReturnCode::Dropped(_) | ReturnCode::Completed(_), _) => code,
4219                _ => bail_bug!("unexpected code/event combo"),
4220            }
4221        } else if let ReadState::HostReady {
4222            cancel,
4223            cancel_waker,
4224            ..
4225        } = &mut state.get_mut(transmit_id)?.read
4226        {
4227            *cancel = true;
4228            if let Some(waker) = cancel_waker.take() {
4229                waker.wake();
4230            }
4231
4232            if async_ {
4233                ReturnCode::Blocked
4234            } else {
4235                let handle = store
4236                    .concurrent_state_mut()?
4237                    .get_mut(transmit_id)?
4238                    .write_handle;
4239                self.wait_for_write(store, caller, handle)?
4240            }
4241        } else {
4242            ReturnCode::Cancelled(ItemCount::ZERO)
4243        };
4244
4245        if !matches!(code, ReturnCode::Blocked) {
4246            let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
4247
4248            match &transmit.write {
4249                WriteState::GuestReady { .. } => {
4250                    transmit.write = WriteState::Open;
4251                }
4252                WriteState::HostReady { .. } => bail_bug!("support host write cancellation"),
4253                WriteState::Open | WriteState::Dropped => {}
4254            }
4255        }
4256
4257        log::trace!("cancelled write {transmit_id:?}: {code:?}");
4258
4259        Ok(code)
4260    }
4261
4262    fn wait_for_read(
4263        self,
4264        store: &mut StoreOpaque,
4265        caller: RuntimeComponentInstanceIndex,
4266        handle: TableId<TransmitHandle>,
4267    ) -> Result<ReturnCode> {
4268        let waitable = Waitable::Transmit(handle);
4269        store.wait_for_event(self.runtime_instance(caller), waitable)?;
4270        let event = waitable.take_event(store.concurrent_state_mut()?)?;
4271        if let Some(event @ (Event::StreamRead { code, .. } | Event::FutureRead { code, .. })) =
4272            event
4273        {
4274            waitable.on_delivery(store, self, event)?;
4275            Ok(code)
4276        } else {
4277            bail_bug!("expected either a stream or future read event")
4278        }
4279    }
4280
4281    /// Cancel a pending stream or future read.
4282    fn cancel_read(
4283        self,
4284        store: &mut StoreOpaque,
4285        caller: RuntimeComponentInstanceIndex,
4286        transmit_id: TableId<TransmitState>,
4287        async_: bool,
4288    ) -> Result<ReturnCode> {
4289        let state = store.concurrent_state_mut()?;
4290        let transmit = state.get_mut(transmit_id)?;
4291        log::trace!(
4292            "host_cancel_read state {transmit_id:?}; read state {:?} write state {:?}",
4293            transmit.read,
4294            transmit.write
4295        );
4296
4297        let waitable = Waitable::Transmit(transmit.read_handle);
4298
4299        waitable.trap_if_has_sync_waiter(state)?;
4300        if !async_ {
4301            waitable.trap_if_in_waitable_set(state)?;
4302        }
4303
4304        let code = if let Some(event) = waitable.take_event(state)? {
4305            let (Event::FutureRead { code, .. } | Event::StreamRead { code, .. }) = event else {
4306                bail_bug!("expected either a stream or future read event")
4307            };
4308            waitable.on_delivery(store, self, event)?;
4309            match (code, event) {
4310                (ReturnCode::Completed(count), Event::StreamRead { .. }) => {
4311                    ReturnCode::Cancelled(count)
4312                }
4313                (ReturnCode::Dropped(_) | ReturnCode::Completed(_), _) => code,
4314                _ => bail_bug!("unexpected code/event combo"),
4315            }
4316        } else if let WriteState::HostReady {
4317            cancel,
4318            cancel_waker,
4319            ..
4320        } = &mut state.get_mut(transmit_id)?.write
4321        {
4322            *cancel = true;
4323            if let Some(waker) = cancel_waker.take() {
4324                waker.wake();
4325            }
4326
4327            if async_ {
4328                ReturnCode::Blocked
4329            } else {
4330                let handle = store
4331                    .concurrent_state_mut()?
4332                    .get_mut(transmit_id)?
4333                    .read_handle;
4334                self.wait_for_read(store, caller, handle)?
4335            }
4336        } else {
4337            ReturnCode::Cancelled(ItemCount::ZERO)
4338        };
4339
4340        if !matches!(code, ReturnCode::Blocked) {
4341            let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
4342
4343            match &transmit.read {
4344                ReadState::GuestReady { .. } => {
4345                    transmit.read = ReadState::Open;
4346                }
4347                ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {
4348                    bail_bug!("support host read cancellation")
4349                }
4350                ReadState::Open | ReadState::Dropped => {}
4351            }
4352        }
4353
4354        log::trace!("cancelled read {transmit_id:?}: {code:?}");
4355
4356        Ok(code)
4357    }
4358
4359    /// Cancel a pending write for the specified stream or future from the guest.
4360    fn guest_cancel_write(
4361        self,
4362        store: &mut StoreOpaque,
4363        caller: RuntimeComponentInstanceIndex,
4364        ty: TransmitIndex,
4365        async_: bool,
4366        writer: u32,
4367    ) -> Result<ReturnCode> {
4368        let (rep, state) =
4369            get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, writer)?;
4370        let id = TableId::<TransmitHandle>::new(rep);
4371        log::trace!("guest cancel write {id:?} (handle {writer})");
4372
4373        // This handle must be "busy" to have anything cancelled.
4374        match state.kind() {
4375            TransmitLocalStateKind::Write { .. } => {
4376                bail!("stream or future write cancelled when no write is pending")
4377            }
4378            TransmitLocalStateKind::Read { .. } => {
4379                bail!("passed read end to `{{stream|future}}.cancel-write`")
4380            }
4381            TransmitLocalStateKind::Cancelling => {
4382                bail!("stream or future write cancelled when a cancellation is already pending")
4383            }
4384            TransmitLocalStateKind::Busy => {}
4385        }
4386
4387        // Double-check that this is indeed the correct handle to cancel a write
4388        // from since `Busy` above didn't check that.
4389        let state = store.concurrent_state_mut()?;
4390        let transmit_id = state.get_mut(id)?.state;
4391        if state.get_mut(transmit_id)?.write_handle != id {
4392            bail!("passed read end to `{{stream|future}}.cancel-write`")
4393        }
4394
4395        let code = self.cancel_write(store, caller, transmit_id, async_)?;
4396        let state =
4397            get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, writer)?.1;
4398        if let TransmitLocalStateKind::Busy = state.kind() {
4399            *state = if matches!(code, ReturnCode::Blocked) {
4400                VMTransmitLocalState::cancelling()
4401            } else {
4402                VMTransmitLocalState::write(false)
4403            };
4404        }
4405        Ok(code)
4406    }
4407
4408    /// Cancel a pending read for the specified stream or future from the guest.
4409    fn guest_cancel_read(
4410        self,
4411        store: &mut StoreOpaque,
4412        caller: RuntimeComponentInstanceIndex,
4413        ty: TransmitIndex,
4414        async_: bool,
4415        reader: u32,
4416    ) -> Result<ReturnCode> {
4417        let (rep, state) =
4418            get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, reader)?;
4419        let id = TableId::<TransmitHandle>::new(rep);
4420        log::trace!("guest cancel read {id:?} (handle {reader})");
4421
4422        // This handle must be "busy" to have anything cancelled.
4423        match state.kind() {
4424            TransmitLocalStateKind::Read { .. } => {
4425                bail!("stream or future read cancelled when no read is pending")
4426            }
4427            TransmitLocalStateKind::Write { .. } => {
4428                bail!("passed write end to `{{stream|future}}.cancel-read`")
4429            }
4430            TransmitLocalStateKind::Cancelling => {
4431                bail!("stream or future read cancelled when a cancellation is already pending")
4432            }
4433            TransmitLocalStateKind::Busy => {}
4434        }
4435
4436        // Double-check that this is indeed the correct handle to cancel a read
4437        // from since `Busy` above didn't check that.
4438        let state = store.concurrent_state_mut()?;
4439        let transmit_id = state.get_mut(id)?.state;
4440        if state.get_mut(transmit_id)?.read_handle != id {
4441            bail!("passed write end to `{{stream|future}}.cancel-read`")
4442        }
4443
4444        let code = self.cancel_read(store, caller, transmit_id, async_)?;
4445        let state =
4446            get_mut_by_index_from(self.id().get_mut(store).table_for_transmit(ty), ty, reader)?.1;
4447        if let TransmitLocalStateKind::Busy = state.kind() {
4448            *state = if matches!(code, ReturnCode::Blocked) {
4449                VMTransmitLocalState::cancelling()
4450            } else {
4451                VMTransmitLocalState::read(false)
4452            };
4453        }
4454        Ok(code)
4455    }
4456
4457    /// Drop the readable end of the specified stream or future from the guest.
4458    fn guest_drop_readable(
4459        self,
4460        store: &mut StoreOpaque,
4461        ty: TransmitIndex,
4462        reader: u32,
4463    ) -> Result<()> {
4464        let table = self.id().get_mut(store).table_for_transmit(ty);
4465        let (rep, _is_done) = match ty {
4466            TransmitIndex::Stream(ty) => table.stream_remove_readable(ty, reader)?,
4467            TransmitIndex::Future(ty) => table.future_remove_readable(ty, reader)?,
4468        };
4469        let kind = match ty {
4470            TransmitIndex::Stream(_) => TransmitKind::Stream,
4471            TransmitIndex::Future(_) => TransmitKind::Future,
4472        };
4473        let id = TableId::<TransmitHandle>::new(rep);
4474        log::trace!("guest_drop_readable: drop reader {id:?}");
4475        store.host_drop_reader(id, kind)
4476    }
4477
4478    /// Create a new error context for the given component.
4479    pub(crate) fn error_context_new(
4480        self,
4481        store: &mut StoreOpaque,
4482        ty: TypeComponentLocalErrorContextTableIndex,
4483        options: OptionsIndex,
4484        debug_msg_address: u32,
4485        debug_msg_len: u32,
4486    ) -> Result<u32> {
4487        let lift_ctx = &mut LiftContext::new(store, options, self)?;
4488        let debug_msg = String::linear_lift_from_flat(
4489            lift_ctx,
4490            InterfaceType::String,
4491            &[ValRaw::u32(debug_msg_address), ValRaw::u32(debug_msg_len)],
4492        )?;
4493
4494        // Create a new ErrorContext that is tracked along with other concurrent state
4495        let err_ctx = ErrorContextState { debug_msg };
4496        let state = store.concurrent_state_mut()?;
4497        let table_id = state.push(err_ctx)?;
4498        let global_ref_count_idx =
4499            TypeComponentGlobalErrorContextTableIndex::from_u32(table_id.rep());
4500
4501        // Add to the global error context ref counts
4502        let _ = state
4503            .global_error_context_ref_counts
4504            .insert(global_ref_count_idx, GlobalErrorContextRefCount(1));
4505
4506        // Error context are tracked both locally (to a single component instance) and globally
4507        // the counts for both must stay in sync.
4508        //
4509        // Here we reflect the newly created global concurrent error context state into the
4510        // component instance's locally tracked count, along with the appropriate key into the global
4511        // ref tracking data structures to enable later lookup
4512        let (instance, counter) = self.id().get_mut_and_handle_counter(store);
4513        let local_idx = instance
4514            .error_context_table_insertion(ty, counter)
4515            .error_context(table_id.rep())?;
4516
4517        Ok(local_idx)
4518    }
4519
4520    /// Retrieve the debug message from the specified error context.
4521    pub(super) fn error_context_debug_message<T>(
4522        self,
4523        store: StoreContextMut<T>,
4524        ty: TypeComponentLocalErrorContextTableIndex,
4525        options: OptionsIndex,
4526        err_ctx_handle: u32,
4527        debug_msg_address: u32,
4528    ) -> Result<()> {
4529        // Retrieve the error context and internal debug message
4530        let handle_table_id_rep = self
4531            .id()
4532            .get_mut(store.0)
4533            .table_for_error_context(ty)
4534            .error_context_rep(err_ctx_handle)?;
4535
4536        let state = store.0.concurrent_state_mut()?;
4537        // Get the state associated with the error context
4538        let ErrorContextState { debug_msg } =
4539            state.get_mut(TableId::<ErrorContextState>::new(handle_table_id_rep))?;
4540        let debug_msg = debug_msg.clone();
4541
4542        let lower_cx = &mut LowerContext::new(store, options, self);
4543        let debug_msg_address = usize::try_from(debug_msg_address)?;
4544        if debug_msg_address % usize::try_from(<str as ComponentType>::ALIGN32)? != 0 {
4545            bail!("invalid debug message pointer: not aligned");
4546        }
4547        // Lower the string into the component's memory.
4548        //
4549        // Note that the "8" here is the size of a WIT `string` in linear
4550        // memory, the ptr+length. This'll need to be updated when `memory64`
4551        // comes along. (FIXME(#4311))
4552        let offset = lower_cx
4553            .as_slice()
4554            .get(debug_msg_address..)
4555            .and_then(|b| b.get(..8))
4556            .map(|_| debug_msg_address)
4557            .ok_or_else(|| crate::format_err!("invalid debug message pointer: out of bounds"))?;
4558        debug_msg
4559            .as_str()
4560            .linear_lower_to_memory(lower_cx, InterfaceType::String, offset)?;
4561
4562        Ok(())
4563    }
4564
4565    /// Implements the `future.cancel-read` intrinsic.
4566    pub(crate) fn future_cancel_read(
4567        self,
4568        store: &mut StoreOpaque,
4569        caller: RuntimeComponentInstanceIndex,
4570        ty: TypeFutureTableIndex,
4571        async_: bool,
4572        reader: u32,
4573    ) -> Result<u32> {
4574        self.guest_cancel_read(store, caller, TransmitIndex::Future(ty), async_, reader)
4575            .map(|v| v.encode())
4576    }
4577
4578    /// Implements the `future.cancel-write` intrinsic.
4579    pub(crate) fn future_cancel_write(
4580        self,
4581        store: &mut StoreOpaque,
4582        caller: RuntimeComponentInstanceIndex,
4583        ty: TypeFutureTableIndex,
4584        async_: bool,
4585        writer: u32,
4586    ) -> Result<u32> {
4587        self.guest_cancel_write(store, caller, TransmitIndex::Future(ty), async_, writer)
4588            .map(|v| v.encode())
4589    }
4590
4591    /// Implements the `stream.cancel-read` intrinsic.
4592    pub(crate) fn stream_cancel_read(
4593        self,
4594        store: &mut StoreOpaque,
4595        caller: RuntimeComponentInstanceIndex,
4596        ty: TypeStreamTableIndex,
4597        async_: bool,
4598        reader: u32,
4599    ) -> Result<u32> {
4600        self.guest_cancel_read(store, caller, TransmitIndex::Stream(ty), async_, reader)
4601            .map(|v| v.encode())
4602    }
4603
4604    /// Implements the `stream.cancel-write` intrinsic.
4605    pub(crate) fn stream_cancel_write(
4606        self,
4607        store: &mut StoreOpaque,
4608        caller: RuntimeComponentInstanceIndex,
4609        ty: TypeStreamTableIndex,
4610        async_: bool,
4611        writer: u32,
4612    ) -> Result<u32> {
4613        self.guest_cancel_write(store, caller, TransmitIndex::Stream(ty), async_, writer)
4614            .map(|v| v.encode())
4615    }
4616
4617    /// Implements the `future.drop-readable` intrinsic.
4618    pub(crate) fn future_drop_readable(
4619        self,
4620        store: &mut StoreOpaque,
4621        ty: TypeFutureTableIndex,
4622        reader: u32,
4623    ) -> Result<()> {
4624        self.guest_drop_readable(store, TransmitIndex::Future(ty), reader)
4625    }
4626
4627    /// Implements the `stream.drop-readable` intrinsic.
4628    pub(crate) fn stream_drop_readable(
4629        self,
4630        store: &mut StoreOpaque,
4631        ty: TypeStreamTableIndex,
4632        reader: u32,
4633    ) -> Result<()> {
4634        self.guest_drop_readable(store, TransmitIndex::Stream(ty), reader)
4635    }
4636
4637    /// Allocate a new future or stream and grant ownership of both the read and
4638    /// write ends to the (sub-)component instance to which the specified
4639    /// `TransmitIndex` belongs.
4640    fn guest_new(self, store: &mut StoreOpaque, ty: TransmitIndex) -> Result<ResourcePair> {
4641        let (write, read) = store
4642            .concurrent_state_mut()?
4643            .new_transmit(TransmitOrigin::guest(self.id().instance(), ty))?;
4644
4645        let (instance, counter) = self.id().get_mut_and_handle_counter(store);
4646        let mut insertion = instance.transmit_table_insertion(ty, counter);
4647        let (read_handle, write_handle) = match ty {
4648            TransmitIndex::Future(ty) => (
4649                insertion.future_read(ty, read.rep())?,
4650                insertion.future_write(ty, write.rep())?,
4651            ),
4652            TransmitIndex::Stream(ty) => (
4653                insertion.stream_read(ty, read.rep())?,
4654                insertion.stream_write(ty, write.rep())?,
4655            ),
4656        };
4657
4658        let state = store.concurrent_state_mut()?;
4659        state.get_mut(read)?.common.handle = Some(read_handle);
4660        state.get_mut(write)?.common.handle = Some(write_handle);
4661
4662        Ok(ResourcePair {
4663            write: write_handle,
4664            read: read_handle,
4665        })
4666    }
4667
4668    /// Drop the specified error context.
4669    pub(crate) fn error_context_drop(
4670        self,
4671        store: &mut StoreOpaque,
4672        ty: TypeComponentLocalErrorContextTableIndex,
4673        error_context: u32,
4674    ) -> Result<()> {
4675        let instance = self.id().get_mut(store);
4676
4677        let local_handle_table = instance.table_for_error_context(ty);
4678
4679        let rep = local_handle_table.error_context_drop(error_context)?;
4680
4681        let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(rep);
4682
4683        let state = store.concurrent_state_mut()?;
4684        let Some(GlobalErrorContextRefCount(global_ref_count)) = state
4685            .global_error_context_ref_counts
4686            .get_mut(&global_ref_count_idx)
4687        else {
4688            bail_bug!("retrieve concurrent state for error context during drop")
4689        };
4690
4691        // Reduce the component-global ref count, removing tracking if necessary
4692        if *global_ref_count < 1 {
4693            bail_bug!("ref count unexpectedly zero");
4694        }
4695        *global_ref_count -= 1;
4696        if *global_ref_count == 0 {
4697            state
4698                .global_error_context_ref_counts
4699                .remove(&global_ref_count_idx);
4700
4701            state
4702                .delete(TableId::<ErrorContextState>::new(rep))
4703                .context("deleting component-global error context data")?;
4704        }
4705
4706        Ok(())
4707    }
4708
4709    /// Transfer ownership of the specified stream or future read end from one
4710    /// guest to another.
4711    ///
4712    /// Maintains the invariant on `lower_transmit_to_index`: component
4713    /// validation requires `src` and `dst` to have equivalent payload types, so
4714    /// if `P::Item::typecheck` passed for `src`'s payload, it also passes for
4715    /// `dst`'s.
4716    fn guest_transfer(
4717        self,
4718        store: &mut StoreOpaque,
4719        src_idx: u32,
4720        src: TransmitIndex,
4721        dst: TransmitIndex,
4722    ) -> Result<u32> {
4723        let id = self.lift_index_to_transmit(store, src, src_idx)?;
4724        self.lower_transmit_to_index(store, dst, id)
4725    }
4726
4727    fn lift_index_to_transmit(
4728        self,
4729        store: &mut StoreOpaque,
4730        ty: TransmitIndex,
4731        src_idx: u32,
4732    ) -> Result<TableId<TransmitHandle>> {
4733        let (state, _, _, _, instance) = store.lift_context_parts(self);
4734        lift_index_to_transmit(instance, state.concurrent_state_mut(), ty, src_idx)
4735    }
4736
4737    /// Callers must maintain the invariant on the free function
4738    /// `lower_transmit_to_index`.
4739    fn lower_transmit_to_index(
4740        self,
4741        store: &mut StoreOpaque,
4742        ty: TransmitIndex,
4743        id: TableId<TransmitHandle>,
4744    ) -> Result<u32> {
4745        let (state, _, counter, _, instance) = store.lift_context_parts(self);
4746        lower_transmit_to_index(instance, counter, state.concurrent_state_mut(), ty, id)
4747    }
4748
4749    /// Implements the `future.new` intrinsic.
4750    pub(crate) fn future_new(
4751        self,
4752        store: &mut StoreOpaque,
4753        ty: TypeFutureTableIndex,
4754    ) -> Result<ResourcePair> {
4755        self.guest_new(store, TransmitIndex::Future(ty))
4756    }
4757
4758    /// Implements the `stream.new` intrinsic.
4759    pub(crate) fn stream_new(
4760        self,
4761        store: &mut StoreOpaque,
4762        ty: TypeStreamTableIndex,
4763    ) -> Result<ResourcePair> {
4764        self.guest_new(store, TransmitIndex::Stream(ty))
4765    }
4766
4767    /// Transfer ownership of the specified future read end from one guest to
4768    /// another.
4769    pub(crate) fn future_transfer(
4770        self,
4771        store: &mut StoreOpaque,
4772        src_idx: u32,
4773        src: TypeFutureTableIndex,
4774        dst: TypeFutureTableIndex,
4775    ) -> Result<u32> {
4776        self.guest_transfer(
4777            store,
4778            src_idx,
4779            TransmitIndex::Future(src),
4780            TransmitIndex::Future(dst),
4781        )
4782    }
4783
4784    /// Transfer ownership of the specified stream read end from one guest to
4785    /// another.
4786    pub(crate) fn stream_transfer(
4787        self,
4788        store: &mut StoreOpaque,
4789        src_idx: u32,
4790        src: TypeStreamTableIndex,
4791        dst: TypeStreamTableIndex,
4792    ) -> Result<u32> {
4793        self.guest_transfer(
4794            store,
4795            src_idx,
4796            TransmitIndex::Stream(src),
4797            TransmitIndex::Stream(dst),
4798        )
4799    }
4800
4801    /// Copy the specified error context from one component to another.
4802    pub(crate) fn error_context_transfer(
4803        self,
4804        store: &mut StoreOpaque,
4805        src_idx: u32,
4806        src: TypeComponentLocalErrorContextTableIndex,
4807        dst: TypeComponentLocalErrorContextTableIndex,
4808    ) -> Result<u32> {
4809        let (mut instance, counter) = self.id().get_mut_and_handle_counter(store);
4810        let rep = instance
4811            .as_mut()
4812            .table_for_error_context(src)
4813            .error_context_rep(src_idx)?;
4814        let dst_idx = instance
4815            .error_context_table_insertion(dst, counter)
4816            .error_context(rep)?;
4817
4818        // Update the global (cross-subcomponent) count for error contexts
4819        // as the new component has essentially created a new reference that will
4820        // be dropped/handled independently
4821        let global_ref_count = store
4822            .concurrent_state_mut()?
4823            .global_error_context_ref_counts
4824            .get_mut(&TypeComponentGlobalErrorContextTableIndex::from_u32(rep))
4825            .context("global ref count present for existing (sub)component error context")?;
4826
4827        global_ref_count.0 = global_ref_count
4828            .0
4829            .checked_add(1)
4830            .ok_or_else(|| format_err!(Trap::ReferenceCountOverflow))?;
4831
4832        Ok(dst_idx)
4833    }
4834}
4835
4836/// Performs the opertion of lifting a future or stream from and instance into
4837/// the `TransmitHandle` for it.
4838///
4839/// The `src_idx` is the guest-specified index within `instance` and `ty` is the
4840/// expected type of future/stream.
4841fn lift_index_to_transmit(
4842    instance: Pin<&mut ComponentInstance>,
4843    concurrent_state: &mut ConcurrentState,
4844    ty: TransmitIndex,
4845    src_idx: u32,
4846) -> Result<TableId<TransmitHandle>> {
4847    let handle_table = instance.table_for_transmit(ty);
4848    let (rep, is_done) = match ty {
4849        TransmitIndex::Future(idx) => handle_table.future_remove_readable(idx, src_idx)?,
4850        TransmitIndex::Stream(idx) => handle_table.stream_remove_readable(idx, src_idx)?,
4851    };
4852    let desc = match ty {
4853        TransmitIndex::Future(_) => "future",
4854        TransmitIndex::Stream(_) => "stream",
4855    };
4856    if is_done {
4857        bail!("cannot lift {desc} after being notified that the writable end dropped");
4858    }
4859    let id = TableId::<TransmitHandle>::new(rep);
4860    let future = concurrent_state.get_mut(id)?;
4861    if future.common.set.is_some() {
4862        bail!("cannot lift {desc} while it's in a waitable set");
4863    }
4864    future.common.handle = None;
4865
4866    let state = future.state;
4867    if concurrent_state.get_mut(state)?.done {
4868        match ty {
4869            TransmitIndex::Future(_) => bail!("cannot lift {desc} after previous read succeeded"),
4870            TransmitIndex::Stream(_) => bail!(Trap::LiftDroppedStream),
4871        };
4872    }
4873
4874    Ok(id)
4875}
4876
4877/// Performs the opertion of lowering a future or stream `TransmitHandle` into
4878/// an instance.
4879///
4880/// Invariant: if the read end of a stream or future is in a guest's handle
4881/// table under type `ty`, and it comes from a host `{Stream,Future}Producer`
4882/// `P`, then `P::Item::typecheck` has passed for `ty`'s payload. Apart from
4883/// `guest_new`, which only creates guest-written transmits, this is the only
4884/// place read ends are inserted into a guest's table, so callers must maintain
4885/// this invariant.
4886fn lower_transmit_to_index(
4887    instance: Pin<&mut ComponentInstance>,
4888    counter: &mut HandleCounter,
4889    concurrent_state: &mut ConcurrentState,
4890    ty: TransmitIndex,
4891    id: TableId<TransmitHandle>,
4892) -> Result<u32> {
4893    let state = concurrent_state.get_mut(id)?.state;
4894    debug_assert_eq!(concurrent_state.get_mut(state)?.read_handle, id);
4895    let mut insertion = instance.transmit_table_insertion(ty, counter);
4896    let handle = match ty {
4897        TransmitIndex::Future(idx) => insertion.future_read(idx, id.rep()),
4898        TransmitIndex::Stream(idx) => insertion.stream_read(idx, id.rep()),
4899    }?;
4900    concurrent_state.get_mut(id)?.common.handle = Some(handle);
4901    Ok(handle)
4902}
4903
4904impl ComponentInstance {
4905    fn transmit_instance(&self, ty: TransmitIndex) -> RuntimeComponentInstanceIndex {
4906        let types = self.component().types();
4907        match ty {
4908            TransmitIndex::Stream(ty) => types[ty].instance,
4909            TransmitIndex::Future(ty) => types[ty].instance,
4910        }
4911    }
4912
4913    fn table_for_transmit(self: Pin<&mut Self>, ty: TransmitIndex) -> &mut HandleTable {
4914        let runtime_instance = self.transmit_instance(ty);
4915        self.instance_state(runtime_instance).handle_table()
4916    }
4917
4918    fn transmit_table_insertion<'a>(
4919        self: Pin<&'a mut Self>,
4920        ty: TransmitIndex,
4921        counter: &'a mut HandleCounter,
4922    ) -> HandleTableInsertion<'a> {
4923        let runtime_instance = self.transmit_instance(ty);
4924        self.handle_table_insertion(runtime_instance, counter)
4925    }
4926
4927    fn table_for_error_context(
4928        self: Pin<&mut Self>,
4929        ty: TypeComponentLocalErrorContextTableIndex,
4930    ) -> &mut HandleTable {
4931        let runtime_instance = self.component().types()[ty].instance;
4932        self.instance_state(runtime_instance).handle_table()
4933    }
4934
4935    fn error_context_table_insertion<'a>(
4936        self: Pin<&'a mut Self>,
4937        ty: TypeComponentLocalErrorContextTableIndex,
4938        counter: &'a mut HandleCounter,
4939    ) -> HandleTableInsertion<'a> {
4940        let runtime_instance = self.component().types()[ty].instance;
4941        self.handle_table_insertion(runtime_instance, counter)
4942    }
4943
4944    fn get_mut_by_index(
4945        self: Pin<&mut Self>,
4946        ty: TransmitIndex,
4947        index: u32,
4948    ) -> Result<(u32, &mut VMTransmitLocalState)> {
4949        get_mut_by_index_from(self.table_for_transmit(ty), ty, index)
4950    }
4951}
4952
4953impl ConcurrentState {
4954    fn send_write_result(
4955        &mut self,
4956        ty: TransmitIndex,
4957        id: TableId<TransmitState>,
4958        handle: u32,
4959        code: ReturnCode,
4960    ) -> Result<()> {
4961        let write_handle = self.get_mut(id)?.write_handle.rep();
4962        self.set_event(
4963            write_handle,
4964            match ty {
4965                TransmitIndex::Future(ty) => Event::FutureWrite {
4966                    code,
4967                    pending: Some((ty, handle)),
4968                },
4969                TransmitIndex::Stream(ty) => Event::StreamWrite {
4970                    code,
4971                    pending: Some((ty, handle)),
4972                },
4973            },
4974        )
4975    }
4976
4977    fn send_read_result(
4978        &mut self,
4979        ty: TransmitIndex,
4980        id: TableId<TransmitState>,
4981        handle: u32,
4982        code: ReturnCode,
4983    ) -> Result<()> {
4984        let read_handle = self.get_mut(id)?.read_handle.rep();
4985        self.set_event(
4986            read_handle,
4987            match ty {
4988                TransmitIndex::Future(ty) => Event::FutureRead {
4989                    code,
4990                    pending: Some((ty, handle)),
4991                },
4992                TransmitIndex::Stream(ty) => Event::StreamRead {
4993                    code,
4994                    pending: Some((ty, handle)),
4995                },
4996            },
4997        )
4998    }
4999
5000    fn take_event(&mut self, waitable: u32) -> Result<Option<Event>> {
5001        Waitable::Transmit(TableId::<TransmitHandle>::new(waitable)).take_event(self)
5002    }
5003
5004    fn set_event(&mut self, waitable: u32, event: Event) -> Result<()> {
5005        Waitable::Transmit(TableId::<TransmitHandle>::new(waitable)).set_event(self, Some(event))
5006    }
5007
5008    /// Set or update the event for the specified waitable.
5009    ///
5010    /// If there is already an event set for this waitable, we assert that it is
5011    /// of the same variant as the new one and reuse the `ReturnCode` count and
5012    /// the `pending` field if applicable.
5013    // TODO: This is a bit awkward due to how
5014    // `Event::{Stream,Future}{Write,Read}` and
5015    // `ReturnCode::{Completed,Dropped,Cancelled}` are currently represented.
5016    // Consider updating those representations in a way that allows this
5017    // function to be simplified.
5018    fn update_event(&mut self, waitable: u32, event: Event) -> Result<()> {
5019        let waitable = Waitable::Transmit(TableId::<TransmitHandle>::new(waitable));
5020
5021        fn update_code(old: ReturnCode, new: ReturnCode) -> Result<ReturnCode> {
5022            let (ReturnCode::Completed(count)
5023            | ReturnCode::Dropped(count)
5024            | ReturnCode::Cancelled(count)) = old
5025            else {
5026                bail_bug!("unexpected old return code")
5027            };
5028
5029            Ok(match new {
5030                ReturnCode::Dropped(ItemCount::ZERO) => ReturnCode::Dropped(count),
5031                ReturnCode::Cancelled(ItemCount::ZERO) => ReturnCode::Cancelled(count),
5032                _ => bail_bug!("unexpected new return code"),
5033            })
5034        }
5035
5036        let event = match (waitable.take_event(self)?, event) {
5037            (None, _) => event,
5038            (Some(old @ Event::FutureWrite { .. }), Event::FutureWrite { .. }) => old,
5039            (Some(old @ Event::FutureRead { .. }), Event::FutureRead { .. }) => old,
5040            (
5041                Some(Event::StreamWrite {
5042                    code: old_code,
5043                    pending: old_pending,
5044                }),
5045                Event::StreamWrite { code, pending },
5046            ) => Event::StreamWrite {
5047                code: update_code(old_code, code)?,
5048                pending: old_pending.or(pending),
5049            },
5050            (
5051                Some(Event::StreamRead {
5052                    code: old_code,
5053                    pending: old_pending,
5054                }),
5055                Event::StreamRead { code, pending },
5056            ) => Event::StreamRead {
5057                code: update_code(old_code, code)?,
5058                pending: old_pending.or(pending),
5059            },
5060            _ => bail_bug!("unexpected event combination"),
5061        };
5062
5063        waitable.set_event(self, Some(event))
5064    }
5065
5066    /// Allocate a new future or stream, including the `TransmitState` and the
5067    /// `TransmitHandle`s corresponding to the read and write ends.
5068    fn new_transmit(
5069        &mut self,
5070        origin: TransmitOrigin,
5071    ) -> Result<(TableId<TransmitHandle>, TableId<TransmitHandle>)> {
5072        // All three entries must fit before inserting any of them so a full
5073        // table does not leave an inaccessible state or handle behind.
5074        self.table().ensure_capacity(3)?;
5075        let state_id = self.push(TransmitState::new(origin))?;
5076
5077        let write = self.push(TransmitHandle::new(state_id))?;
5078        let read = self.push(TransmitHandle::new(state_id))?;
5079
5080        let state = self.get_mut(state_id)?;
5081        state.write_handle = write;
5082        state.read_handle = read;
5083
5084        log::trace!("new transmit: state {state_id:?}; write {write:?}; read {read:?}",);
5085
5086        Ok((write, read))
5087    }
5088
5089    /// Delete the specified future or stream, including the read and write ends.
5090    fn delete_transmit(&mut self, state_id: TableId<TransmitState>) -> Result<()> {
5091        let state = self.delete(state_id)?;
5092        self.delete(state.write_handle)?;
5093        self.delete(state.read_handle)?;
5094
5095        log::trace!(
5096            "delete transmit: state {state_id:?}; write {:?}; read {:?}",
5097            state.write_handle,
5098            state.read_handle,
5099        );
5100
5101        Ok(())
5102    }
5103}
5104
5105pub(crate) struct ResourcePair {
5106    pub(crate) write: u32,
5107    pub(crate) read: u32,
5108}
5109
5110impl Waitable {
5111    /// Handle the imminent delivery of the specified event, e.g. by updating
5112    /// the state of the stream or future.
5113    pub(super) fn on_delivery(
5114        &self,
5115        store: &mut StoreOpaque,
5116        instance: Instance,
5117        event: Event,
5118    ) -> Result<()> {
5119        if let Event::FutureRead {
5120            code: ReturnCode::Dropped(_),
5121            ..
5122        }
5123        | Event::FutureWrite {
5124            code: ReturnCode::Dropped(_),
5125            ..
5126        }
5127        | Event::StreamRead {
5128            code: ReturnCode::Dropped(_),
5129            ..
5130        }
5131        | Event::StreamWrite {
5132            code: ReturnCode::Dropped(_),
5133            ..
5134        } = event
5135        {
5136            let Waitable::Transmit(transmit_handle) = self else {
5137                bail_bug!("unexpected `{event:?}` for `{self:?}`");
5138            };
5139            let state = store.concurrent_state_mut()?;
5140            let transmit_id = state.get_mut(*transmit_handle)?.state;
5141            state.get_mut(transmit_id)?.done = true;
5142        }
5143
5144        let instance = instance.id().get_mut(store);
5145        let (rep, state, code) = match event {
5146            Event::FutureRead {
5147                pending: Some((ty, handle)),
5148                code,
5149            }
5150            | Event::FutureWrite {
5151                pending: Some((ty, handle)),
5152                code,
5153            } => {
5154                let runtime_instance = instance.component().types()[ty].instance;
5155                let (rep, state) = instance.instance_states().0[runtime_instance]
5156                    .handle_table()
5157                    .future_rep(ty, handle)?;
5158                (rep, state, code)
5159            }
5160            Event::StreamRead {
5161                pending: Some((ty, handle)),
5162                code,
5163            }
5164            | Event::StreamWrite {
5165                pending: Some((ty, handle)),
5166                code,
5167            } => {
5168                let runtime_instance = instance.component().types()[ty].instance;
5169                let (rep, state) = instance.instance_states().0[runtime_instance]
5170                    .handle_table()
5171                    .stream_rep(ty, handle)?;
5172                (rep, state, code)
5173            }
5174            _ => return Ok(()),
5175        };
5176        if rep != self.rep() {
5177            bail_bug!("unexpected rep mismatch");
5178        }
5179        if !matches!(
5180            state.kind(),
5181            TransmitLocalStateKind::Busy | TransmitLocalStateKind::Cancelling
5182        ) {
5183            bail_bug!("expected state to be busy or cancelling");
5184        }
5185        let done = matches!(code, ReturnCode::Dropped(_));
5186        *state = match event {
5187            Event::FutureRead { .. } | Event::StreamRead { .. } => VMTransmitLocalState::read(done),
5188            Event::FutureWrite { .. } | Event::StreamWrite { .. } => {
5189                VMTransmitLocalState::write(done)
5190            }
5191            _ => bail_bug!("unexpected event for stream"),
5192        };
5193
5194        let transmit_handle = TableId::<TransmitHandle>::new(rep);
5195        let state = store.concurrent_state_mut()?;
5196        let transmit_id = state.get_mut(transmit_handle)?.state;
5197        let transmit = state.get_mut(transmit_id)?;
5198
5199        match event {
5200            Event::StreamRead { .. } => {
5201                transmit.read = ReadState::Open;
5202            }
5203            Event::StreamWrite { .. } => transmit.write = WriteState::Open,
5204            _ => {}
5205        }
5206        Ok(())
5207    }
5208}
5209
5210/// Determine whether an intra-component read/write is allowed for the specified
5211/// `stream` or `future` payload type according to the component model
5212/// specification.
5213fn allow_intra_component_read_write(ty: Option<&InterfaceType>) -> bool {
5214    matches!(
5215        ty,
5216        None | Some(
5217            InterfaceType::S8
5218                | InterfaceType::U8
5219                | InterfaceType::S16
5220                | InterfaceType::U16
5221                | InterfaceType::S32
5222                | InterfaceType::U32
5223                | InterfaceType::S64
5224                | InterfaceType::U64
5225                | InterfaceType::Float32
5226                | InterfaceType::Float64
5227        )
5228    )
5229}
5230
5231/// Helper structure to manage moving a `T` in/out of an interior `Mutex` which
5232/// contains an
5233/// `Option<T>`
5234struct LockedState<T> {
5235    inner: TryMutex<Option<T>>,
5236}
5237
5238impl<T> LockedState<T> {
5239    /// Creates a new initial state with `value` stored.
5240    fn new(value: T) -> Self {
5241        Self {
5242            inner: TryMutex::new(Some(value)),
5243        }
5244    }
5245
5246    /// Attempts to lock the inner mutex and return its guard.
5247    ///
5248    /// # Errors
5249    ///
5250    /// Fails if this lock is either poisoned or if it's currently locked.
5251    /// As-used in this file there should never actually be contention on this
5252    /// lock nor recursive access so failing to acquire the lock is a fatal
5253    /// error that gets propagated upwards.
5254    fn try_lock(&self) -> Result<TryMutexGuard<'_, Option<T>>> {
5255        match self.inner.try_lock() {
5256            Some(lock) => Ok(lock),
5257            None => bail_bug!("should not have contention on state lock"),
5258        }
5259    }
5260
5261    /// Takes the inner `T` out of this state, returning it as a guard which
5262    /// will put it back when finished.
5263    ///
5264    /// # Errors
5265    ///
5266    /// Returns an error if the state `T` isn't present.
5267    fn take(&self) -> Result<LockedStateGuard<'_, T>> {
5268        let result = self.try_lock()?.take();
5269        match result {
5270            Some(result) => Ok(LockedStateGuard {
5271                value: ManuallyDrop::new(result),
5272                state: self,
5273            }),
5274            None => bail_bug!("lock value unexpectedly missing"),
5275        }
5276    }
5277
5278    /// Performs the operation `f` on the inner state `&mut T`.
5279    ///
5280    /// This will acquire the internal lock and invoke `f`, so `f` should not
5281    /// expect to be able to recursively acquire this lock.
5282    ///
5283    /// # Errors
5284    ///
5285    /// Returns an error if the state `T` isn't present.
5286    fn with<R>(&self, f: impl FnOnce(&mut T) -> R) -> Result<R> {
5287        let mut inner = self.try_lock()?;
5288        match &mut *inner {
5289            Some(state) => Ok(f(state)),
5290            None => bail_bug!("lock value unexpectedly missing"),
5291        }
5292    }
5293}
5294
5295/// Helper structure returned from [`LockedState::take`] which will put the
5296/// state specified by `value` back into the original lock once this is dropped.
5297struct LockedStateGuard<'a, T> {
5298    value: ManuallyDrop<T>,
5299    state: &'a LockedState<T>,
5300}
5301
5302impl<T> Deref for LockedStateGuard<'_, T> {
5303    type Target = T;
5304
5305    fn deref(&self) -> &T {
5306        &self.value
5307    }
5308}
5309
5310impl<T> DerefMut for LockedStateGuard<'_, T> {
5311    fn deref_mut(&mut self) -> &mut T {
5312        &mut self.value
5313    }
5314}
5315
5316impl<T> Drop for LockedStateGuard<'_, T> {
5317    fn drop(&mut self) {
5318        // SAFETY: `ManuallyDrop::take` requires that after invoked the
5319        // original value is not read. This is the `Drop` for this type which
5320        // means we have exclusive ownership and it is not read further in the
5321        // destructor, satisfying this requirement.
5322        let value = unsafe { ManuallyDrop::take(&mut self.value) };
5323
5324        // If this fails due to contention that's a bug, but we're not in a
5325        // position to panic due to this being a destructor nor return an error,
5326        // so defer the bug to showing up later.
5327        if let Ok(mut lock) = self.state.try_lock() {
5328            *lock = Some(value);
5329        }
5330    }
5331}
5332
5333#[cfg(test)]
5334mod tests {
5335    use super::*;
5336    use crate::{Engine, Store};
5337    use core::future::pending;
5338    use core::pin::pin;
5339    use std::sync::LazyLock;
5340
5341    static ENGINE: LazyLock<Engine> = LazyLock::new(Engine::default);
5342
5343    fn poll_future_producer<T>(rx: Pin<&mut T>, finish: bool) -> Poll<Result<Option<T::Item>>>
5344    where
5345        T: FutureProducer<()>,
5346    {
5347        rx.poll_produce(
5348            &mut Context::from_waker(Waker::noop()),
5349            Store::new(&ENGINE, ()).as_context_mut(),
5350            finish,
5351        )
5352    }
5353
5354    #[test]
5355    fn future_producer() {
5356        let mut fut = pin!(async { crate::error::Ok(()) });
5357        assert!(matches!(
5358            poll_future_producer(fut.as_mut(), false),
5359            Poll::Ready(Ok(Some(()))),
5360        ));
5361
5362        let mut fut = pin!(async { crate::error::Ok(()) });
5363        assert!(matches!(
5364            poll_future_producer(fut.as_mut(), true),
5365            Poll::Ready(Ok(Some(()))),
5366        ));
5367
5368        let mut fut = pin!(pending::<Result<()>>());
5369        assert!(matches!(
5370            poll_future_producer(fut.as_mut(), false),
5371            Poll::Pending,
5372        ));
5373        assert!(matches!(
5374            poll_future_producer(fut.as_mut(), true),
5375            Poll::Ready(Ok(None)),
5376        ));
5377
5378        let (tx, rx) = oneshot::channel();
5379        let mut rx = pin!(rx);
5380        assert!(matches!(
5381            poll_future_producer(rx.as_mut(), false),
5382            Poll::Pending,
5383        ));
5384        assert!(matches!(
5385            poll_future_producer(rx.as_mut(), true),
5386            Poll::Ready(Ok(None)),
5387        ));
5388        tx.send(()).unwrap();
5389        assert!(matches!(
5390            poll_future_producer(rx.as_mut(), true),
5391            Poll::Ready(Ok(Some(()))),
5392        ));
5393
5394        let (tx, rx) = oneshot::channel();
5395        let mut rx = pin!(rx);
5396        tx.send(()).unwrap();
5397        assert!(matches!(
5398            poll_future_producer(rx.as_mut(), false),
5399            Poll::Ready(Ok(Some(()))),
5400        ));
5401
5402        let (tx, rx) = oneshot::channel::<()>();
5403        let mut rx = pin!(rx);
5404        drop(tx);
5405        assert!(matches!(
5406            poll_future_producer(rx.as_mut(), false),
5407            Poll::Ready(Err(..)),
5408        ));
5409
5410        let (tx, rx) = oneshot::channel::<()>();
5411        let mut rx = pin!(rx);
5412        drop(tx);
5413        assert!(matches!(
5414            poll_future_producer(rx.as_mut(), true),
5415            Poll::Ready(Err(..)),
5416        ));
5417    }
5418
5419    #[test]
5420    fn direct_destination_successive_writes() {
5421        use std::io::Write;
5422
5423        let mut store = Store::new(&ENGINE, ());
5424        let mut buffer = ();
5425        let mut dst = Vec::new();
5426        let mut marked_written = 0;
5427        let destination = Destination::<u8, ()> {
5428            id: TableId::new(0),
5429            buffer: &mut buffer,
5430            host_buffer: Some(HostBuffer {
5431                dst: &mut dst,
5432                marked_written: &mut marked_written,
5433            }),
5434            _phantom: PhantomData,
5435        };
5436
5437        // Each write must land after the bytes already marked as written,
5438        // not overwrite them.
5439        let mut direct = destination.as_direct(store.as_context_mut(), 6);
5440        direct.write_all(&[1, 2, 3]).unwrap();
5441        direct.write_all(&[4, 5, 6]).unwrap();
5442        drop(direct);
5443
5444        assert_eq!(marked_written, 6);
5445        assert_eq!(dst, [1, 2, 3, 4, 5, 6]);
5446    }
5447}