wasmtime/runtime/vm/
gc.rs

1#[cfg(feature = "gc")]
2mod enabled;
3#[cfg(feature = "gc")]
4pub use enabled::*;
5
6#[cfg(not(feature = "gc"))]
7mod disabled;
8#[cfg(not(feature = "gc"))]
9pub use disabled::*;
10
11mod data;
12mod func_ref;
13mod gc_ref;
14mod gc_runtime;
15mod host_data;
16mod i31;
17
18pub use data::*;
19pub use func_ref::*;
20pub use gc_ref::*;
21pub use gc_runtime::*;
22pub use host_data::*;
23pub use i31::*;
24
25use crate::hash_map::HashMap;
26use crate::module::ModuleRegistry;
27use crate::prelude::*;
28use crate::runtime::vm::{GcHeapAllocationIndex, VMMemoryDefinition};
29use crate::store::Asyncness;
30use crate::type_registry::RegisteredType;
31use core::any::Any;
32use core::mem::MaybeUninit;
33use core::{alloc::Layout, num::NonZeroU32};
34use wasmtime_environ::{GcArrayLayout, GcLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
35
36/// GC-related data that is one-to-one with a `wasmtime::Store`.
37///
38/// Contains everything we need to do collections, invoke barriers, etc...
39///
40/// In general, exposes a very similar interface to `GcHeap`, but fills in some
41/// of the context arguments for callers (such as the `ExternRefHostDataTable`)
42/// since they are all stored together inside `GcStore`.
43pub struct GcStore {
44    /// This GC heap's allocation index (primarily used for integrating with the
45    /// pooling allocator).
46    pub allocation_index: GcHeapAllocationIndex,
47
48    /// The actual GC heap.
49    pub gc_heap: Box<dyn GcHeap>,
50
51    /// The `externref` host data table for this GC heap.
52    pub host_data_table: ExternRefHostDataTable,
53
54    /// The function-references table for this GC heap.
55    pub func_ref_table: FuncRefTable,
56
57    /// The total allocated bytes recorded after the last GC collection.
58    /// `None` if no collection has been performed yet. Used by the
59    /// grow-or-collect heuristic.
60    pub last_post_gc_allocated_bytes: Option<usize>,
61
62    /// An allocation counter that triggers GC when it reaches zero.
63    ///
64    /// Decremented on every allocation and when it hits zero, a GC is
65    /// forced and the counter is reset.
66    #[cfg(gc_zeal)]
67    gc_zeal_alloc_counter: Option<NonZeroU32>,
68
69    /// The initial value to reset the counter to after it triggers.
70    #[cfg(gc_zeal)]
71    gc_zeal_alloc_counter_init: Option<NonZeroU32>,
72}
73
74/// Convenience type definition for the storage, within a `Store`, of
75/// host-allocated types that are one-off used for host allocation.
76pub type StoreGcHostAllocTypes = HashMap<VMSharedTypeIndex, (RegisteredType, Option<TraceInfo>)>;
77
78/// Contextual information used when performing a GC to trace GC references as
79/// necesary.
80pub struct GcStoreTraceState<'a> {
81    /// The backing data of `externref`s, deleted from when a GC reference is
82    /// reclaimed, for example.
83    pub host_data_table: &'a mut ExternRefHostDataTable,
84    /// All known modules in this store.
85    pub modules: &'a ModuleRegistry,
86    /// All known host-registered types in this store.
87    pub gc_host_alloc_types: &'a StoreGcHostAllocTypes,
88}
89
90impl GcStore {
91    /// Create a new `GcStore`.
92    pub fn new(
93        allocation_index: GcHeapAllocationIndex,
94        gc_heap: Box<dyn GcHeap>,
95        gc_zeal_alloc_counter: Option<NonZeroU32>,
96    ) -> Self {
97        let host_data_table = ExternRefHostDataTable::default();
98        let func_ref_table = FuncRefTable::default();
99
100        let _ = &gc_zeal_alloc_counter;
101
102        Self {
103            allocation_index,
104            gc_heap,
105            host_data_table,
106            func_ref_table,
107            last_post_gc_allocated_bytes: None,
108            #[cfg(gc_zeal)]
109            gc_zeal_alloc_counter,
110            #[cfg(gc_zeal)]
111            gc_zeal_alloc_counter_init: gc_zeal_alloc_counter,
112        }
113    }
114
115    /// Get the `VMMemoryDefinition` for this GC heap.
116    pub fn vmmemory_definition(&self) -> VMMemoryDefinition {
117        self.gc_heap.vmmemory()
118    }
119
120    /// Get the current capacity (in bytes) of this GC heap.
121    pub fn gc_heap_capacity(&self) -> usize {
122        self.gc_heap.heap_slice().len()
123    }
124
125    /// Asynchronously perform garbage collection within this heap.
126    pub async fn gc(
127        &mut self,
128        asyncness: Asyncness,
129        roots: GcRootsIter<'_>,
130        modules: &ModuleRegistry,
131        gc_host_alloc_types: &StoreGcHostAllocTypes,
132        yield_fn: impl AsyncFn(),
133    ) -> Result<()> {
134        let mut trace_state = GcStoreTraceState {
135            host_data_table: &mut self.host_data_table,
136            modules,
137            gc_host_alloc_types,
138        };
139        let collection = self.gc_heap.gc(roots, &mut trace_state);
140        collect_async(collection, asyncness, yield_fn).await?;
141        self.last_post_gc_allocated_bytes = Some({
142            let size = self.gc_heap.allocated_bytes();
143            log::trace!("After collection, GC heap's allocated bytes = {size:#x} bytes");
144            size
145        });
146        Ok(())
147    }
148
149    /// Get the kind of the given GC reference.
150    pub fn kind(&self, gc_ref: &VMGcRef) -> Result<VMGcKind> {
151        debug_assert!(!gc_ref.is_i31());
152        Ok(self.header(gc_ref)?.kind())
153    }
154
155    /// Get the header of the given GC reference.
156    pub fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
157        debug_assert!(!gc_ref.is_i31());
158        self.gc_heap.header(gc_ref)
159    }
160
161    /// Clone a GC reference, calling GC write barriers as necessary.
162    pub fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
163        if gc_ref.is_i31() {
164            gc_ref.copy_i31()
165        } else {
166            self.gc_heap.clone_gc_ref(gc_ref)
167        }
168    }
169
170    /// Write the `source` GC reference into the uninitialized `destination`
171    /// slot, performing write barriers as necessary.
172    pub fn init_gc_ref(
173        &mut self,
174        destination: &mut MaybeUninit<Option<VMGcRef>>,
175        source: Option<&VMGcRef>,
176    ) -> Result<()> {
177        // Initialize the destination to `None`, at which point the regular GC
178        // write barrier is safe to reuse.
179        let destination = destination.write(None);
180        self.write_gc_ref(destination, source)
181    }
182
183    /// Dynamically tests whether a `init_gc_ref` is needed to write `gc_ref`
184    /// into an uninitialized destination.
185    pub(crate) fn needs_init_barrier(gc_ref: Option<&VMGcRef>) -> bool {
186        assert!(cfg!(feature = "gc") || gc_ref.is_none());
187        gc_ref.is_some_and(|r| !r.is_i31())
188    }
189
190    /// Dynamically tests whether a `write_gc_ref` is needed to write `gc_ref`
191    /// into `dest`.
192    pub(crate) fn needs_write_barrier(
193        dest: &mut Option<VMGcRef>,
194        gc_ref: Option<&VMGcRef>,
195    ) -> bool {
196        assert!(cfg!(feature = "gc") || gc_ref.is_none());
197        assert!(cfg!(feature = "gc") || dest.is_none());
198        dest.as_ref().is_some_and(|r| !r.is_i31()) || gc_ref.is_some_and(|r| !r.is_i31())
199    }
200
201    /// Same as [`Self::write_gc_ref`] but doesn't require a `store` when
202    /// possible.
203    ///
204    /// # Panics
205    ///
206    /// Panics if `store` is `None` and one of `dest` or `gc_ref` requires a
207    /// write barrier.
208    pub(crate) fn write_gc_ref_optional_store(
209        store: Option<&mut Self>,
210        dest: &mut Option<VMGcRef>,
211        gc_ref: Option<&VMGcRef>,
212    ) -> Result<()> {
213        if Self::needs_write_barrier(dest, gc_ref) {
214            store.unwrap().write_gc_ref(dest, gc_ref)
215        } else {
216            *dest = gc_ref.map(|r| r.copy_i31());
217            Ok(())
218        }
219    }
220
221    /// Write the `source` GC reference into the `destination` slot, performing
222    /// write barriers as necessary.
223    pub fn write_gc_ref(
224        &mut self,
225        destination: &mut Option<VMGcRef>,
226        source: Option<&VMGcRef>,
227    ) -> Result<()> {
228        // If neither the source nor destination actually point to a GC object
229        // (that is, they are both either null or `i31ref`s) then we can skip
230        // the GC barrier.
231        if Self::needs_write_barrier(destination, source) {
232            self.gc_heap.write_gc_ref(destination, source)?;
233        } else {
234            *destination = source.map(|s| s.copy_i31());
235        }
236        Ok(())
237    }
238
239    /// Drop the given GC reference, performing drop barriers as necessary.
240    pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
241        if !gc_ref.is_i31() {
242            self.gc_heap.drop_gc_ref(gc_ref);
243        }
244    }
245
246    /// Hook to call whenever a GC reference is about to be exposed to Wasm.
247    ///
248    /// Returns the raw representation of this GC ref, ready to be passed to
249    /// Wasm.
250    #[must_use]
251    pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> Result<NonZeroU32> {
252        let raw = gc_ref.as_raw_non_zero_u32();
253        if !gc_ref.is_i31() {
254            log::trace!("exposing GC ref to Wasm: {gc_ref:p}");
255            self.gc_heap.expose_gc_ref_to_wasm(gc_ref)?;
256        }
257        Ok(raw)
258    }
259
260    /// Allocate a new `externref`.
261    ///
262    /// Returns:
263    ///
264    /// * `Ok(Ok(_))`: Successfully allocated the `externref`.
265    ///
266    /// * `Ok(Err((value, n)))`: Failed to allocate the `externref`, but doing a GC
267    ///   and then trying again may succeed. Returns the given `value` as the
268    ///   error payload, along with the size of the failed allocation.
269    ///
270    /// * `Err(_)`: Unrecoverable allocation failure.
271    pub fn alloc_externref(
272        &mut self,
273        value: Box<dyn Any + Send + Sync>,
274    ) -> Result<Result<VMExternRef, (Box<dyn Any + Send + Sync>, u64)>> {
275        let host_data_id = self.host_data_table.alloc(value);
276        match self.gc_heap.alloc_externref(host_data_id)? {
277            Ok(x) => Ok(Ok(x)),
278            Err(n) => Ok(Err((self.host_data_table.dealloc(host_data_id)?, n))),
279        }
280    }
281
282    /// Get a shared borrow of the given `externref`'s host data.
283    ///
284    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
285    /// associated with a different heap is memory safe but will lead to general
286    /// incorrectness such as panics and wrong results.
287    pub fn externref_host_data(&self, externref: &VMExternRef) -> Result<&(dyn Any + Send + Sync)> {
288        let host_data_id = self.gc_heap.externref_host_data(externref)?;
289        self.host_data_table.get(host_data_id)
290    }
291
292    /// Get a mutable borrow of the given `externref`'s host data.
293    ///
294    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
295    /// associated with a different heap is memory safe but will lead to general
296    /// incorrectness such as panics and wrong results.
297    pub fn externref_host_data_mut(
298        &mut self,
299        externref: &VMExternRef,
300    ) -> Result<&mut (dyn Any + Send + Sync)> {
301        let host_data_id = self.gc_heap.externref_host_data(externref)?;
302        self.host_data_table.get_mut(host_data_id)
303    }
304
305    /// Allocate a raw object with the given header and layout.
306    pub fn alloc_raw(
307        &mut self,
308        header: VMGcHeader,
309        layout: Layout,
310    ) -> Result<Result<VMGcRef, u64>> {
311        // When gc_zeal is enabled with an allocation counter, decrement it and
312        // force a GC cycle when it reaches zero by returning a fake OOM.
313        #[cfg(gc_zeal)]
314        if let Some(counter) = self.gc_zeal_alloc_counter.take() {
315            match NonZeroU32::new(counter.get() - 1) {
316                Some(c) => self.gc_zeal_alloc_counter = Some(c),
317                None => {
318                    log::trace!("gc_zeal: allocation counter reached zero, forcing GC");
319                    self.gc_zeal_alloc_counter = self.gc_zeal_alloc_counter_init;
320                    return Ok(Err(0));
321                }
322            }
323        }
324
325        self.gc_heap.alloc_raw(header, layout)
326    }
327
328    /// Allocate an uninitialized struct with the given type index and layout.
329    ///
330    /// This does NOT check that the index is currently allocated in the types
331    /// registry or that the layout matches the index's type. Failure to uphold
332    /// those invariants is memory safe, but will lead to general incorrectness
333    /// such as panics and wrong results.
334    pub fn alloc_uninit_struct(
335        &mut self,
336        ty: VMSharedTypeIndex,
337        layout: &GcStructLayout,
338    ) -> Result<Result<VMStructRef, u64>> {
339        self.gc_heap
340            .alloc_uninit_struct_or_exn(ty, layout)
341            .map(|r| r.map(|r| r.into_structref_unchecked()))
342    }
343
344    /// Deallocate an uninitialized struct.
345    pub fn dealloc_uninit_struct(&mut self, structref: VMStructRef) -> Result<()> {
346        self.gc_heap.dealloc_uninit_struct_or_exn(structref.into())
347    }
348
349    /// Get the data for the given object reference.
350    ///
351    /// Panics when the structref and its size is out of the GC heap bounds.
352    pub fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcObjectData> {
353        self.gc_heap.gc_object_data_mut(gc_ref)
354    }
355
356    /// Allocate an uninitialized array with the given type index.
357    ///
358    /// This does NOT check that the index is currently allocated in the types
359    /// registry or that the layout matches the index's type. Failure to uphold
360    /// those invariants is memory safe, but will lead to general incorrectness
361    /// such as panics and wrong results.
362    pub fn alloc_uninit_array(
363        &mut self,
364        ty: VMSharedTypeIndex,
365        len: u32,
366        layout: &GcArrayLayout,
367    ) -> Result<Result<VMArrayRef, u64>> {
368        self.gc_heap.alloc_uninit_array(ty, len, layout)
369    }
370
371    /// Deallocate an uninitialized array.
372    pub fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) -> Result<()> {
373        self.gc_heap.dealloc_uninit_array(arrayref)
374    }
375
376    /// Get the length of the given array.
377    pub fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
378        self.gc_heap.array_len(arrayref)
379    }
380
381    /// Allocate an uninitialized exception object with the given type
382    /// index.
383    ///
384    /// This does NOT check that the index is currently allocated in the types
385    /// registry or that the layout matches the index's type. Failure to uphold
386    /// those invariants is memory safe, but will lead to general incorrectness
387    /// such as panics and wrong results.
388    pub fn alloc_uninit_exn(
389        &mut self,
390        ty: VMSharedTypeIndex,
391        layout: &GcStructLayout,
392    ) -> Result<Result<VMExnRef, u64>> {
393        self.gc_heap
394            .alloc_uninit_struct_or_exn(ty, layout)
395            .map(|r| r.map(|r| r.into_exnref_unchecked()))
396    }
397
398    /// Deallocate an uninitialized exception object.
399    pub fn dealloc_uninit_exn(&mut self, exnref: VMExnRef) -> Result<()> {
400        self.gc_heap.dealloc_uninit_struct_or_exn(exnref.into())
401    }
402
403    #[cfg(feature = "gc")]
404    pub(crate) fn replace_gc_zeal_alloc_counter(
405        &mut self,
406        new_value: Option<NonZeroU32>,
407    ) -> Option<NonZeroU32> {
408        #[cfg(gc_zeal)]
409        return core::mem::replace(&mut self.gc_zeal_alloc_counter, new_value);
410
411        #[cfg(not(gc_zeal))]
412        {
413            let _ = new_value;
414            return None;
415        }
416    }
417}
418
419/// How to trace a GC object.
420#[derive(Debug)]
421pub enum TraceInfo {
422    /// How to trace an array.
423    Array {
424        /// Whether this array type's elements are GC references, and need
425        /// tracing.
426        #[cfg_attr(
427            not(feature = "gc-drc"),
428            allow(dead_code, reason = "easier not to cfg on/off")
429        )]
430        gc_ref_elems: bool,
431    },
432
433    /// How to trace a struct.
434    Struct {
435        /// The offsets of each GC reference field that needs tracing in
436        /// instances of this struct type.
437        gc_ref_offsets: Box<[u32]>,
438    },
439}
440
441impl TraceInfo {
442    pub(crate) fn new(gc_layout: &GcLayout) -> Self {
443        match gc_layout {
444            GcLayout::Array(l) => TraceInfo::Array {
445                gc_ref_elems: l.elems_are_gc_refs,
446            },
447            GcLayout::Struct(l) => TraceInfo::Struct {
448                gc_ref_offsets: l
449                    .fields
450                    .iter()
451                    .filter_map(|f| if f.is_gc_ref { Some(f.offset) } else { None })
452                    .collect(),
453            },
454        }
455    }
456}
wasmtime/runtime/vm/gc.rs

wasmtime/runtime/vm/
gc.rs
