wasmtime/runtime/vm/
gc.rs

1#[cfg(feature = "gc")]
2mod enabled;
3#[cfg(feature = "gc")]
4pub use enabled::*;
5
6#[cfg(not(feature = "gc"))]
7mod disabled;
8#[cfg(not(feature = "gc"))]
9pub use disabled::*;
10
11mod data;
12mod func_ref;
13mod gc_ref;
14mod gc_runtime;
15mod host_data;
16mod i31;
17
18pub use data::*;
19pub use func_ref::*;
20pub use gc_ref::*;
21pub use gc_runtime::*;
22pub use host_data::*;
23pub use i31::*;
24
25use crate::prelude::*;
26use crate::runtime::vm::{GcHeapAllocationIndex, VMMemoryDefinition};
27use crate::store::Asyncness;
28use core::any::Any;
29use core::mem::MaybeUninit;
30use core::{alloc::Layout, num::NonZeroU32};
31use wasmtime_environ::{GcArrayLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
32
33/// GC-related data that is one-to-one with a `wasmtime::Store`.
34///
35/// Contains everything we need to do collections, invoke barriers, etc...
36///
37/// In general, exposes a very similar interface to `GcHeap`, but fills in some
38/// of the context arguments for callers (such as the `ExternRefHostDataTable`)
39/// since they are all stored together inside `GcStore`.
40pub struct GcStore {
41    /// This GC heap's allocation index (primarily used for integrating with the
42    /// pooling allocator).
43    pub allocation_index: GcHeapAllocationIndex,
44
45    /// The actual GC heap.
46    pub gc_heap: Box<dyn GcHeap>,
47
48    /// The `externref` host data table for this GC heap.
49    pub host_data_table: ExternRefHostDataTable,
50
51    /// The function-references table for this GC heap.
52    pub func_ref_table: FuncRefTable,
53
54    /// The total allocated bytes recorded after the last GC collection.
55    /// `None` if no collection has been performed yet. Used by the
56    /// grow-or-collect heuristic.
57    pub last_post_gc_allocated_bytes: Option<usize>,
58
59    /// An allocation counter that triggers GC when it reaches zero.
60    ///
61    /// Decremented on every allocation and when it hits zero, a GC is
62    /// forced and the counter is reset.
63    #[cfg(gc_zeal)]
64    gc_zeal_alloc_counter: Option<NonZeroU32>,
65
66    /// The initial value to reset the counter to after it triggers.
67    #[cfg(gc_zeal)]
68    gc_zeal_alloc_counter_init: Option<NonZeroU32>,
69}
70
71impl GcStore {
72    /// Create a new `GcStore`.
73    pub fn new(
74        allocation_index: GcHeapAllocationIndex,
75        gc_heap: Box<dyn GcHeap>,
76        gc_zeal_alloc_counter: Option<NonZeroU32>,
77    ) -> Self {
78        let host_data_table = ExternRefHostDataTable::default();
79        let func_ref_table = FuncRefTable::default();
80
81        let _ = &gc_zeal_alloc_counter;
82
83        Self {
84            allocation_index,
85            gc_heap,
86            host_data_table,
87            func_ref_table,
88            last_post_gc_allocated_bytes: None,
89            #[cfg(gc_zeal)]
90            gc_zeal_alloc_counter,
91            #[cfg(gc_zeal)]
92            gc_zeal_alloc_counter_init: gc_zeal_alloc_counter,
93        }
94    }
95
96    /// Get the `VMMemoryDefinition` for this GC heap.
97    pub fn vmmemory_definition(&self) -> VMMemoryDefinition {
98        self.gc_heap.vmmemory()
99    }
100
101    /// Get the current capacity (in bytes) of this GC heap.
102    pub fn gc_heap_capacity(&self) -> usize {
103        self.gc_heap.heap_slice().len()
104    }
105
106    /// Asynchronously perform garbage collection within this heap.
107    pub async fn gc(
108        &mut self,
109        asyncness: Asyncness,
110        roots: GcRootsIter<'_>,
111        yield_fn: impl AsyncFn(),
112    ) -> Result<()> {
113        let collection = self.gc_heap.gc(roots, &mut self.host_data_table);
114        collect_async(collection, asyncness, yield_fn).await?;
115        self.last_post_gc_allocated_bytes = Some({
116            let size = self.gc_heap.allocated_bytes();
117            log::trace!("After collection, GC heap's allocated bytes = {size:#x} bytes");
118            size
119        });
120        Ok(())
121    }
122
123    /// Get the kind of the given GC reference.
124    pub fn kind(&self, gc_ref: &VMGcRef) -> Result<VMGcKind> {
125        debug_assert!(!gc_ref.is_i31());
126        Ok(self.header(gc_ref)?.kind())
127    }
128
129    /// Get the header of the given GC reference.
130    pub fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
131        debug_assert!(!gc_ref.is_i31());
132        self.gc_heap.header(gc_ref)
133    }
134
135    /// Clone a GC reference, calling GC write barriers as necessary.
136    pub fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
137        if gc_ref.is_i31() {
138            gc_ref.copy_i31()
139        } else {
140            self.gc_heap.clone_gc_ref(gc_ref)
141        }
142    }
143
144    /// Write the `source` GC reference into the uninitialized `destination`
145    /// slot, performing write barriers as necessary.
146    pub fn init_gc_ref(
147        &mut self,
148        destination: &mut MaybeUninit<Option<VMGcRef>>,
149        source: Option<&VMGcRef>,
150    ) -> Result<()> {
151        // Initialize the destination to `None`, at which point the regular GC
152        // write barrier is safe to reuse.
153        let destination = destination.write(None);
154        self.write_gc_ref(destination, source)
155    }
156
157    /// Dynamically tests whether a `init_gc_ref` is needed to write `gc_ref`
158    /// into an uninitialized destination.
159    pub(crate) fn needs_init_barrier(gc_ref: Option<&VMGcRef>) -> bool {
160        assert!(cfg!(feature = "gc") || gc_ref.is_none());
161        gc_ref.is_some_and(|r| !r.is_i31())
162    }
163
164    /// Dynamically tests whether a `write_gc_ref` is needed to write `gc_ref`
165    /// into `dest`.
166    pub(crate) fn needs_write_barrier(
167        dest: &mut Option<VMGcRef>,
168        gc_ref: Option<&VMGcRef>,
169    ) -> bool {
170        assert!(cfg!(feature = "gc") || gc_ref.is_none());
171        assert!(cfg!(feature = "gc") || dest.is_none());
172        dest.as_ref().is_some_and(|r| !r.is_i31()) || gc_ref.is_some_and(|r| !r.is_i31())
173    }
174
175    /// Same as [`Self::write_gc_ref`] but doesn't require a `store` when
176    /// possible.
177    ///
178    /// # Panics
179    ///
180    /// Panics if `store` is `None` and one of `dest` or `gc_ref` requires a
181    /// write barrier.
182    pub(crate) fn write_gc_ref_optional_store(
183        store: Option<&mut Self>,
184        dest: &mut Option<VMGcRef>,
185        gc_ref: Option<&VMGcRef>,
186    ) -> Result<()> {
187        if Self::needs_write_barrier(dest, gc_ref) {
188            store.unwrap().write_gc_ref(dest, gc_ref)
189        } else {
190            *dest = gc_ref.map(|r| r.copy_i31());
191            Ok(())
192        }
193    }
194
195    /// Write the `source` GC reference into the `destination` slot, performing
196    /// write barriers as necessary.
197    pub fn write_gc_ref(
198        &mut self,
199        destination: &mut Option<VMGcRef>,
200        source: Option<&VMGcRef>,
201    ) -> Result<()> {
202        // If neither the source nor destination actually point to a GC object
203        // (that is, they are both either null or `i31ref`s) then we can skip
204        // the GC barrier.
205        if Self::needs_write_barrier(destination, source) {
206            self.gc_heap
207                .write_gc_ref(&mut self.host_data_table, destination, source)?;
208        } else {
209            *destination = source.map(|s| s.copy_i31());
210        }
211        Ok(())
212    }
213
214    /// Drop the given GC reference, performing drop barriers as necessary.
215    pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
216        if !gc_ref.is_i31() {
217            self.gc_heap.drop_gc_ref(&mut self.host_data_table, gc_ref);
218        }
219    }
220
221    /// Hook to call whenever a GC reference is about to be exposed to Wasm.
222    ///
223    /// Returns the raw representation of this GC ref, ready to be passed to
224    /// Wasm.
225    #[must_use]
226    pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> Result<NonZeroU32> {
227        let raw = gc_ref.as_raw_non_zero_u32();
228        if !gc_ref.is_i31() {
229            log::trace!("exposing GC ref to Wasm: {gc_ref:p}");
230            self.gc_heap.expose_gc_ref_to_wasm(gc_ref)?;
231        }
232        Ok(raw)
233    }
234
235    /// Allocate a new `externref`.
236    ///
237    /// Returns:
238    ///
239    /// * `Ok(Ok(_))`: Successfully allocated the `externref`.
240    ///
241    /// * `Ok(Err((value, n)))`: Failed to allocate the `externref`, but doing a GC
242    ///   and then trying again may succeed. Returns the given `value` as the
243    ///   error payload, along with the size of the failed allocation.
244    ///
245    /// * `Err(_)`: Unrecoverable allocation failure.
246    pub fn alloc_externref(
247        &mut self,
248        value: Box<dyn Any + Send + Sync>,
249    ) -> Result<Result<VMExternRef, (Box<dyn Any + Send + Sync>, u64)>> {
250        let host_data_id = self.host_data_table.alloc(value);
251        match self.gc_heap.alloc_externref(host_data_id)? {
252            Ok(x) => Ok(Ok(x)),
253            Err(n) => Ok(Err((self.host_data_table.dealloc(host_data_id)?, n))),
254        }
255    }
256
257    /// Get a shared borrow of the given `externref`'s host data.
258    ///
259    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
260    /// associated with a different heap is memory safe but will lead to general
261    /// incorrectness such as panics and wrong results.
262    pub fn externref_host_data(&self, externref: &VMExternRef) -> Result<&(dyn Any + Send + Sync)> {
263        let host_data_id = self.gc_heap.externref_host_data(externref)?;
264        self.host_data_table.get(host_data_id)
265    }
266
267    /// Get a mutable borrow of the given `externref`'s host data.
268    ///
269    /// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
270    /// associated with a different heap is memory safe but will lead to general
271    /// incorrectness such as panics and wrong results.
272    pub fn externref_host_data_mut(
273        &mut self,
274        externref: &VMExternRef,
275    ) -> Result<&mut (dyn Any + Send + Sync)> {
276        let host_data_id = self.gc_heap.externref_host_data(externref)?;
277        self.host_data_table.get_mut(host_data_id)
278    }
279
280    /// Allocate a raw object with the given header and layout.
281    pub fn alloc_raw(
282        &mut self,
283        header: VMGcHeader,
284        layout: Layout,
285    ) -> Result<Result<VMGcRef, u64>> {
286        // When gc_zeal is enabled with an allocation counter, decrement it and
287        // force a GC cycle when it reaches zero by returning a fake OOM.
288        #[cfg(gc_zeal)]
289        if let Some(counter) = self.gc_zeal_alloc_counter.take() {
290            match NonZeroU32::new(counter.get() - 1) {
291                Some(c) => self.gc_zeal_alloc_counter = Some(c),
292                None => {
293                    log::trace!("gc_zeal: allocation counter reached zero, forcing GC");
294                    self.gc_zeal_alloc_counter = self.gc_zeal_alloc_counter_init;
295                    return Ok(Err(0));
296                }
297            }
298        }
299
300        self.gc_heap.alloc_raw(header, layout)
301    }
302
303    /// Eagerly ensure tracing info is registered for the given type.
304    pub fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {
305        self.gc_heap.ensure_trace_info(ty)
306    }
307
308    /// Allocate an uninitialized struct with the given type index and layout.
309    ///
310    /// This does NOT check that the index is currently allocated in the types
311    /// registry or that the layout matches the index's type. Failure to uphold
312    /// those invariants is memory safe, but will lead to general incorrectness
313    /// such as panics and wrong results.
314    pub fn alloc_uninit_struct(
315        &mut self,
316        ty: VMSharedTypeIndex,
317        layout: &GcStructLayout,
318    ) -> Result<Result<VMStructRef, u64>> {
319        self.gc_heap
320            .alloc_uninit_struct_or_exn(ty, layout)
321            .map(|r| r.map(|r| r.into_structref_unchecked()))
322    }
323
324    /// Deallocate an uninitialized struct.
325    pub fn dealloc_uninit_struct(&mut self, structref: VMStructRef) -> Result<()> {
326        self.gc_heap.dealloc_uninit_struct_or_exn(structref.into())
327    }
328
329    /// Get the data for the given object reference.
330    ///
331    /// Panics when the structref and its size is out of the GC heap bounds.
332    pub fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcObjectData> {
333        self.gc_heap.gc_object_data_mut(gc_ref)
334    }
335
336    /// Allocate an uninitialized array with the given type index.
337    ///
338    /// This does NOT check that the index is currently allocated in the types
339    /// registry or that the layout matches the index's type. Failure to uphold
340    /// those invariants is memory safe, but will lead to general incorrectness
341    /// such as panics and wrong results.
342    pub fn alloc_uninit_array(
343        &mut self,
344        ty: VMSharedTypeIndex,
345        len: u32,
346        layout: &GcArrayLayout,
347    ) -> Result<Result<VMArrayRef, u64>> {
348        self.gc_heap.alloc_uninit_array(ty, len, layout)
349    }
350
351    /// Deallocate an uninitialized array.
352    pub fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) -> Result<()> {
353        self.gc_heap.dealloc_uninit_array(arrayref)
354    }
355
356    /// Get the length of the given array.
357    pub fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
358        self.gc_heap.array_len(arrayref)
359    }
360
361    /// Allocate an uninitialized exception object with the given type
362    /// index.
363    ///
364    /// This does NOT check that the index is currently allocated in the types
365    /// registry or that the layout matches the index's type. Failure to uphold
366    /// those invariants is memory safe, but will lead to general incorrectness
367    /// such as panics and wrong results.
368    pub fn alloc_uninit_exn(
369        &mut self,
370        ty: VMSharedTypeIndex,
371        layout: &GcStructLayout,
372    ) -> Result<Result<VMExnRef, u64>> {
373        self.gc_heap
374            .alloc_uninit_struct_or_exn(ty, layout)
375            .map(|r| r.map(|r| r.into_exnref_unchecked()))
376    }
377
378    /// Deallocate an uninitialized exception object.
379    pub fn dealloc_uninit_exn(&mut self, exnref: VMExnRef) -> Result<()> {
380        self.gc_heap.dealloc_uninit_struct_or_exn(exnref.into())
381    }
382
383    #[cfg(feature = "gc")]
384    pub(crate) fn replace_gc_zeal_alloc_counter(
385        &mut self,
386        new_value: Option<NonZeroU32>,
387    ) -> Option<NonZeroU32> {
388        #[cfg(gc_zeal)]
389        return core::mem::replace(&mut self.gc_zeal_alloc_counter, new_value);
390
391        #[cfg(not(gc_zeal))]
392        {
393            let _ = new_value;
394            return None;
395        }
396    }
397}
wasmtime/runtime/vm/gc.rs

wasmtime/runtime/vm/
gc.rs
