wasmtime/runtime/vm/
gc.rs

1#[cfg(feature = "gc")]
2mod enabled;
3#[cfg(feature = "gc")]
4pub use enabled::*;
5
6#[cfg(not(feature = "gc"))]
7mod disabled;
8#[cfg(not(feature = "gc"))]
9pub use disabled::*;
10
11#[cfg(feature = "stack-switching")]
12mod cont_ref;
13mod data;
14mod func_ref;
15mod gc_ref;
16mod gc_runtime;
17mod host_data;
18mod i31;
19
20#[cfg(feature = "stack-switching")]
21pub use cont_ref::*;
22pub use data::*;
23pub use func_ref::*;
24pub use gc_ref::*;
25pub use gc_runtime::*;
26pub use host_data::*;
27pub use i31::*;
28
29use crate::bail_bug;
30use crate::hash_map::HashMap;
31use crate::module::ModuleRegistry;
32use crate::prelude::*;
33use crate::runtime::vm::{GcHeapAllocationIndex, VMMemoryDefinition};
34use crate::store::Asyncness;
35use crate::type_registry::RegisteredType;
36use core::any::Any;
37use core::mem::MaybeUninit;
38use core::{alloc::Layout, num::NonZeroU32};
39use wasmtime_environ::{GcArrayLayout, GcLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
40
41/// GC-related data that is one-to-one with a `wasmtime::Store`.
42///
43/// Contains everything we need to do collections, invoke barriers, etc...
44///
45/// In general, exposes a very similar interface to `GcHeap`, but fills in some
46/// of the context arguments for callers (such as the `ExternRefHostDataTable`)
47/// since they are all stored together inside `GcStore`.
48pub struct GcStore {
49    /// This GC heap's allocation index (primarily used for integrating with the
50    /// pooling allocator).
51    pub allocation_index: GcHeapAllocationIndex,
52
53    /// The actual GC heap.
54    pub gc_heap: Box<dyn GcHeap>,
55
56    /// The `externref` host data table for this GC heap.
57    pub host_data_table: ExternRefHostDataTable,
58
59    /// The function-references table for this GC heap.
60    pub func_ref_table: FuncRefTable,
61
62    /// The continuation references table for this GC heap.
63    #[cfg(feature = "stack-switching")]
64    pub cont_ref_table: ContRefTable,
65
66    /// The total allocated bytes recorded after the last GC collection.
67    /// `None` if no collection has been performed yet. Used by the
68    /// grow-or-collect heuristic.
69    pub last_post_gc_allocated_bytes: Option<usize>,
70
71    /// An allocation counter that triggers GC when it reaches zero.
72    ///
73    /// Decremented on every allocation and when it hits zero, a GC is
74    /// forced and the counter is reset.
75    #[cfg(gc_zeal)]
76    gc_zeal_alloc_counter: Option<NonZeroU32>,
77
78    /// The initial value to reset the counter to after it triggers.
79    #[cfg(gc_zeal)]
80    gc_zeal_alloc_counter_init: Option<NonZeroU32>,
81}
82
83/// Convenience type definition for the storage, within a `Store`, of
84/// host-allocated types that are one-off used for host allocation.
85pub type StoreGcHostAllocTypes = HashMap<VMSharedTypeIndex, (RegisteredType, Option<TraceInfo>)>;
86
87/// Contextual information used when performing a GC to trace GC references as
88/// necesary.
89pub struct GcStoreTraceState<'a> {
90    /// The backing data of `externref`s, deleted from when a GC reference is
91    /// reclaimed, for example.
92    pub host_data_table: &'a mut ExternRefHostDataTable,
93    /// All known modules in this store.
94    pub modules: &'a ModuleRegistry,
95    /// All known host-registered types in this store.
96    pub gc_host_alloc_types: &'a StoreGcHostAllocTypes,
97}
98
99impl GcStore {
100    /// Create a new `GcStore`.
101    pub fn new(
102        allocation_index: GcHeapAllocationIndex,
103        gc_heap: Box<dyn GcHeap>,
104        gc_zeal_alloc_counter: Option<NonZeroU32>,
105    ) -> Self {
106        let host_data_table = ExternRefHostDataTable::default();
107        let func_ref_table = FuncRefTable::default();
108        #[cfg(feature = "stack-switching")]
109        let cont_ref_table = ContRefTable::default();
110
111        let _ = &gc_zeal_alloc_counter;
112
113        Self {
114            allocation_index,
115            gc_heap,
116            host_data_table,
117            func_ref_table,
118            #[cfg(feature = "stack-switching")]
119            cont_ref_table,
120            last_post_gc_allocated_bytes: None,
121            #[cfg(gc_zeal)]
122            gc_zeal_alloc_counter,
123            #[cfg(gc_zeal)]
124            gc_zeal_alloc_counter_init: gc_zeal_alloc_counter,
125        }
126    }
127
128    /// Get the `VMMemoryDefinition` for this GC heap.
129    pub fn vmmemory_definition(&self) -> VMMemoryDefinition {
130        self.gc_heap.vmmemory()
131    }
132
133    /// Get the current capacity (in bytes) of this GC heap.
134    pub fn gc_heap_capacity(&self) -> usize {
135        self.gc_heap.heap_slice().len()
136    }
137
138    /// Asynchronously perform garbage collection within this heap.
139    pub async fn gc(
140        &mut self,
141        asyncness: Asyncness,
142        roots: GcRootsIter<'_>,
143        modules: &ModuleRegistry,
144        gc_host_alloc_types: &StoreGcHostAllocTypes,
145        yield_fn: impl AsyncFn(),
146    ) -> Result<()> {
147        let mut trace_state = GcStoreTraceState {
148            host_data_table: &mut self.host_data_table,
149            modules,
150            gc_host_alloc_types,
151        };
152        let collection = self.gc_heap.gc(roots, &mut trace_state);
153        collect_async(collection, asyncness, yield_fn).await?;
154        self.last_post_gc_allocated_bytes = Some({
155            let size = self.gc_heap.allocated_bytes();
156            log::trace!("After collection, GC heap's allocated bytes = {size:#x} bytes");
157            size
158        });
159        Ok(())
160    }
161
162    /// Get the kind of the given GC reference.
163    pub fn kind(&self, gc_ref: &VMGcRef) -> Result<VMGcKind> {
164        debug_assert!(!gc_ref.is_i31());
165        match self.header(gc_ref)?.kind() {
166            Some(kind) => Ok(kind),
167            None => bail_bug!("invalid `VMGcKind`"),
168        }
169    }
170
171    /// Get the header of the given GC reference.
172    pub fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
173        debug_assert!(!gc_ref.is_i31());
174        self.gc_heap.header(gc_ref)
175    }
176
177    /// Clone a GC reference, calling GC write barriers as necessary.
178    pub fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
179        if gc_ref.is_i31() {
180            gc_ref.copy_i31()
181        } else {
182            self.gc_heap.clone_gc_ref(gc_ref)
183        }
184    }
185
186    /// Write the `source` GC reference into the uninitialized `destination`
187    /// slot, performing write barriers as necessary.
188    pub fn init_gc_ref(
189        &mut self,
190        destination: &mut MaybeUninit<Option<VMGcRef>>,
191        source: Option<&VMGcRef>,
192    ) -> Result<()> {
193        // Initialize the destination to `None`, at which point the regular GC
194        // write barrier is safe to reuse.
195        let destination = destination.write(None);
196        self.write_gc_ref(destination, source)
197    }
198
199    /// Dynamically tests whether a `init_gc_ref` is needed to write `gc_ref`
200    /// into an uninitialized destination.
201    pub(crate) fn needs_init_barrier(gc_ref: Option<&VMGcRef>) -> bool {
202        assert!(cfg!(feature = "gc") || gc_ref.is_none());
203        gc_ref.is_some_and(|r| !r.is_i31())
204    }
205
206    /// Dynamically tests whether a `write_gc_ref` is needed to write `gc_ref`
207    /// into `dest`.
208    pub(crate) fn needs_write_barrier(
209        dest: &mut Option<VMGcRef>,
210        gc_ref: Option<&VMGcRef>,
211    ) -> bool {
212        assert!(cfg!(feature = "gc") || gc_ref.is_none());
213        assert!(cfg!(feature = "gc") || dest.is_none());
214        dest.as_ref().is_some_and(|r| !r.is_i31()) || gc_ref.is_some_and(|r| !r.is_i31())
215    }
216
217    /// Same as [`Self::write_gc_ref`] but doesn't require a `store` when
218    /// possible.
219    ///
220    /// # Panics
221    ///
222    /// Panics if `store` is `None` and one of `dest` or `gc_ref` requires a
223    /// write barrier.
224    pub(crate) fn write_gc_ref_optional_store(
225        store: Option<&mut Self>,
226        dest: &mut Option<VMGcRef>,
227        gc_ref: Option<&VMGcRef>,
228    ) -> Result<()> {
229        if Self::needs_write_barrier(dest, gc_ref) {
230            store.unwrap().write_gc_ref(dest, gc_ref)
231        } else {
232            *dest = gc_ref.map(|r| r.copy_i31());
233            Ok(())
234        }
235    }
236
237    /// Write the `source` GC reference into the `destination` slot, performing
238    /// write barriers as necessary.
239    pub fn write_gc_ref(
240        &mut self,
241        destination: &mut Option<VMGcRef>,
242        source: Option<&VMGcRef>,
243    ) -> Result<()> {
244        // If neither the source nor destination actually point to a GC object
245        // (that is, they are both either null or `i31ref`s) then we can skip
246        // the GC barrier.
247        if Self::needs_write_barrier(destination, source) {
248            self.gc_heap.write_gc_ref(destination, source)?;
249        } else {
250            *destination = source.map(|s| s.copy_i31());
251        }
252        Ok(())
253    }
254
255    /// Drop the given GC reference, performing drop barriers as necessary.
256    pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
257        if !gc_ref.is_i31() {
258            self.gc_heap.drop_gc_ref(gc_ref);
259        }
260    }
261
262    /// Hook to call whenever a GC reference is about to be exposed to Wasm.
263    ///
264    /// Returns the raw representation of this GC ref, ready to be passed to
265    /// Wasm.
266    #[must_use]
267    pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> Result<NonZeroU32> {
268        let raw = gc_ref.as_raw_non_zero_u32();
269        if !gc_ref.is_i31() {
270            log::trace!("exposing GC ref to Wasm: {gc_ref:p}");
271            self.gc_heap.expose_gc_ref_to_wasm(gc_ref)?;
272        }
273        Ok(raw)
274    }
275
276    /// Allocate a new `externref`.
277    ///
278    /// Returns:
279    ///
280    /// * `Ok(Ok(_))`: Successfully allocated the `externref`.
281    ///
282    /// * `Ok(Err((value, n)))`: Failed to allocate the `externref`, but doing a GC
283    ///   and then trying again may succeed. Returns the given `value` as the
284    ///   error payload, along with the size of the failed allocation.
285    ///
286    /// * `Err(_)`: Unrecoverable allocation failure.
287    pub fn alloc_externref(
288        &mut self,
289        value: Box<dyn Any + Send + Sync>,
290    ) -> Result<Result<VMExternRef, (Box<dyn Any + Send + Sync>, u64)>> {
291        let host_data_id = self.host_data_table.alloc(value);
292        match self.gc_heap.alloc_externref(host_data_id)? {
293            Ok(x) => Ok(Ok(x)),
294            Err(n) => Ok(Err((self.host_data_table.dealloc(host_data_id)?, n))),
295        }
296    }
297
298    /// Get a shared borrow of the given `externref`'s host data.
299    ///
300    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
301    /// associated with a different heap is memory safe but will lead to general
302    /// incorrectness such as panics and wrong results.
303    pub fn externref_host_data(&self, externref: &VMExternRef) -> Result<&(dyn Any + Send + Sync)> {
304        let host_data_id = self.gc_heap.externref_host_data(externref)?;
305        self.host_data_table.get(host_data_id)
306    }
307
308    /// Get a mutable borrow of the given `externref`'s host data.
309    ///
310    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
311    /// associated with a different heap is memory safe but will lead to general
312    /// incorrectness such as panics and wrong results.
313    pub fn externref_host_data_mut(
314        &mut self,
315        externref: &VMExternRef,
316    ) -> Result<&mut (dyn Any + Send + Sync)> {
317        let host_data_id = self.gc_heap.externref_host_data(externref)?;
318        self.host_data_table.get_mut(host_data_id)
319    }
320
321    /// Allocate a raw object with the given header and layout.
322    pub fn alloc_raw(
323        &mut self,
324        header: VMGcHeader,
325        layout: Layout,
326    ) -> Result<Result<VMGcRef, u64>> {
327        // When gc_zeal is enabled with an allocation counter, decrement it and
328        // force a GC cycle when it reaches zero by returning a fake OOM.
329        #[cfg(gc_zeal)]
330        if let Some(counter) = self.gc_zeal_alloc_counter.take() {
331            match NonZeroU32::new(counter.get() - 1) {
332                Some(c) => self.gc_zeal_alloc_counter = Some(c),
333                None => {
334                    log::trace!("gc_zeal: allocation counter reached zero, forcing GC");
335                    self.gc_zeal_alloc_counter = self.gc_zeal_alloc_counter_init;
336                    return Ok(Err(0));
337                }
338            }
339        }
340
341        self.gc_heap.alloc_raw(header, layout)
342    }
343
344    /// Allocate an uninitialized struct with the given type index and layout.
345    ///
346    /// This does NOT check that the index is currently allocated in the types
347    /// registry or that the layout matches the index's type. Failure to uphold
348    /// those invariants is memory safe, but will lead to general incorrectness
349    /// such as panics and wrong results.
350    pub fn alloc_uninit_struct(
351        &mut self,
352        ty: VMSharedTypeIndex,
353        layout: &GcStructLayout,
354    ) -> Result<Result<VMStructRef, u64>> {
355        self.gc_heap
356            .alloc_uninit_struct_or_exn(ty, layout)
357            .map(|r| r.map(|r| r.into_structref_unchecked()))
358    }
359
360    /// Deallocate an uninitialized struct.
361    pub fn dealloc_uninit_struct(&mut self, structref: VMStructRef) -> Result<()> {
362        self.gc_heap.dealloc_uninit_struct_or_exn(structref.into())
363    }
364
365    /// Get the data for the given object reference.
366    ///
367    /// Panics when the structref and its size is out of the GC heap bounds.
368    pub fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcObjectData> {
369        self.gc_heap.gc_object_data_mut(gc_ref)
370    }
371
372    /// Allocate an uninitialized array with the given type index.
373    ///
374    /// This does NOT check that the index is currently allocated in the types
375    /// registry or that the layout matches the index's type. Failure to uphold
376    /// those invariants is memory safe, but will lead to general incorrectness
377    /// such as panics and wrong results.
378    pub fn alloc_uninit_array(
379        &mut self,
380        ty: VMSharedTypeIndex,
381        len: u32,
382        layout: &GcArrayLayout,
383    ) -> Result<Result<VMArrayRef, u64>> {
384        self.gc_heap.alloc_uninit_array(ty, len, layout)
385    }
386
387    /// Deallocate an uninitialized array.
388    pub fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) -> Result<()> {
389        self.gc_heap.dealloc_uninit_array(arrayref)
390    }
391
392    /// Get the length of the given array.
393    pub fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
394        self.gc_heap.array_len(arrayref)
395    }
396
397    /// Allocate an uninitialized exception object with the given type
398    /// index.
399    ///
400    /// This does NOT check that the index is currently allocated in the types
401    /// registry or that the layout matches the index's type. Failure to uphold
402    /// those invariants is memory safe, but will lead to general incorrectness
403    /// such as panics and wrong results.
404    pub fn alloc_uninit_exn(
405        &mut self,
406        ty: VMSharedTypeIndex,
407        layout: &GcStructLayout,
408    ) -> Result<Result<VMExnRef, u64>> {
409        self.gc_heap
410            .alloc_uninit_struct_or_exn(ty, layout)
411            .map(|r| r.map(|r| r.into_exnref_unchecked()))
412    }
413
414    /// Deallocate an uninitialized exception object.
415    pub fn dealloc_uninit_exn(&mut self, exnref: VMExnRef) -> Result<()> {
416        self.gc_heap.dealloc_uninit_struct_or_exn(exnref.into())
417    }
418
419    #[cfg(feature = "gc")]
420    pub(crate) fn replace_gc_zeal_alloc_counter(
421        &mut self,
422        new_value: Option<NonZeroU32>,
423    ) -> Option<NonZeroU32> {
424        #[cfg(gc_zeal)]
425        return core::mem::replace(&mut self.gc_zeal_alloc_counter, new_value);
426
427        #[cfg(not(gc_zeal))]
428        {
429            let _ = new_value;
430            return None;
431        }
432    }
433}
434
435/// How to trace a GC object.
436#[derive(Debug)]
437pub enum TraceInfo {
438    /// How to trace an array.
439    Array {
440        /// Whether this array type's elements are GC references, and need
441        /// tracing.
442        #[cfg_attr(
443            not(feature = "gc-drc"),
444            allow(dead_code, reason = "easier not to cfg on/off")
445        )]
446        gc_ref_elems: bool,
447    },
448
449    /// How to trace a struct.
450    Struct {
451        /// The offsets of each GC reference field that needs tracing in
452        /// instances of this struct type.
453        gc_ref_offsets: Box<[u32]>,
454    },
455}
456
457impl TraceInfo {
458    pub(crate) fn new(gc_layout: &GcLayout) -> Self {
459        match gc_layout {
460            GcLayout::Array(l) => TraceInfo::Array {
461                gc_ref_elems: l.elems_are_gc_refs,
462            },
463            GcLayout::Struct(l) => TraceInfo::Struct {
464                gc_ref_offsets: l
465                    .fields
466                    .iter()
467                    .filter_map(|f| if f.is_gc_ref { Some(f.offset) } else { None })
468                    .collect(),
469            },
470        }
471    }
472}
wasmtime/runtime/vm/gc.rs

wasmtime/runtime/vm/
gc.rs
