Skip to main content

wasmtime/runtime/store/
gc.rs

1//! GC-related methods for stores.
2
3use crate::RootSet;
4use crate::error::{Context, ensure};
5use crate::module::ModuleRegistry;
6use crate::store::{
7    Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,
8};
9use crate::type_registry::RegisteredType;
10use crate::vm::{
11    self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, SendSyncPtr,
12    StoreGcHostAllocTypes, TraceInfo, VMGcRef,
13};
14use crate::{
15    Engine, ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, Tag,
16    ThrownException, bail,
17};
18use core::fmt;
19use core::mem::ManuallyDrop;
20use core::num::NonZeroU32;
21use core::ops::{Deref, DerefMut};
22use core::ptr::NonNull;
23use wasmtime_environ::DefinedTagIndex;
24use wasmtime_environ::packed_option::ReservedValue;
25
26#[derive(Default)]
27pub(crate) struct StoreGcData {
28    gc_roots: RootSet,
29    gc_roots_list: GcRootsList,
30    // Types for which the embedder has created an allocator for.
31    gc_host_alloc_types: StoreGcHostAllocTypes,
32    /// Pending exception, if any. This is also a GC root, because it
33    /// needs to be rooted somewhere between the time that a pending
34    /// exception is set and the time that the handling code takes the
35    /// exception object. We use this rooting strategy rather than a
36    /// root in an `Err` branch of a `Result` on the host side because
37    /// it is less error-prone with respect to rooting behavior. See
38    /// `throw()`, `take_pending_exception()`,
39    /// `peek_pending_exception()`, `has_pending_exception()`, and
40    /// `catch()`.
41    ///
42    /// Also note that the underlying reference here is a `VMExnRef`, a
43    /// refinement of `VMGcRef`, but rooting APIs right now make it difficult to
44    /// work with that directly so this is stored as `VMGcRef` instead.
45    pending_exception: Option<VMGcRef>,
46}
47
48impl<T> Store<T> {
49    /// Perform garbage collection.
50    ///
51    /// Note that it is not required to actively call this function. GC will
52    /// automatically happen according to various internal heuristics. This is
53    /// provided if fine-grained control over the GC is desired.
54    ///
55    /// If you are calling this method after an attempted allocation failed, you
56    /// may pass in the [`GcHeapOutOfMemory`][crate::GcHeapOutOfMemory] error.
57    /// When you do so, this method will attempt to create enough space in the
58    /// GC heap for that allocation, so that it will succeed on the next
59    /// attempt.
60    ///
61    /// # Errors
62    ///
63    /// This method will fail if an [async limiter is
64    /// configured](Store::limiter_async) in which case [`Store::gc_async`] must
65    /// be used instead.
66    pub fn gc(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>) -> Result<()> {
67        StoreContextMut(&mut self.inner).gc(why)
68    }
69
70    /// Returns the current capacity of the GC heap in bytes, or 0 if the GC
71    /// heap has not been initialized yet.
72    pub fn gc_heap_capacity(&self) -> usize {
73        self.inner.gc_heap_capacity()
74    }
75
76    /// Manually grow the GC heap by at least `bytes` bytes.
77    ///
78    /// This method will attempt to increase the size of the GC heap used for GC
79    /// objects by at least `bytes` bytes. The current capacity of the GC heap
80    /// can be determined by looking at [`Store::gc_heap_capacity`].
81    ///
82    /// This method can be useful, for example, to pre-allocate space in the GC
83    /// heap for guests that are known to have GC-heavy workloads. This can help
84    /// amortize startup costs in some situations.
85    ///
86    /// Note that GC heap capacity does not mean that an `(array i8)` of size
87    /// equal to the heap's capacity will succeed. Wasmtime's GC implementations
88    /// are responsible for how the heap is used and divvy'd up. As a result
89    /// the growth here does not have a precise semantic meaning and instead
90    /// it's recommended to primarily use this for performance tuning.
91    ///
92    /// # Errors
93    ///
94    /// GC heap growth is a resource-consuming operation that can fail for a
95    /// number of reasons:
96    ///
97    /// * The OS might reject growth of the GC heap.
98    /// * The GC heap's configuration may not allow it to grow further.
99    /// * The store's resource limiter might reject the growth.
100    /// * This method was used when the [`Store::gc_heap_grow_async`] method
101    ///   must be used instead.
102    pub fn gc_heap_grow(&mut self, bytes: u64) -> Result<()> {
103        StoreContextMut(&mut self.inner).gc_heap_grow(bytes)
104    }
105
106    /// Set an exception as the currently pending exception, and
107    /// return an error that propagates the throw.
108    ///
109    /// This method takes an exception object and stores it in the
110    /// `Store` as the currently pending exception. This is a special
111    /// rooted slot that holds the exception as long as it is
112    /// propagating. This method then returns a `ThrownException`
113    /// error, which is a special type that indicates a pending
114    /// exception exists. When this type propagates as an error
115    /// returned from a Wasm-to-host call, the pending exception is
116    /// thrown within the Wasm context, and either caught or
117    /// propagated further to the host-to-Wasm call boundary. If an
118    /// exception is thrown out of Wasm (or across Wasm from a
119    /// hostcall) back to the host-to-Wasm call boundary, *that*
120    /// invocation returns a `ThrownException`, and the pending
121    /// exception slot is again set. In other words, the
122    /// `ThrownException` error type should propagate upward exactly
123    /// and only when a pending exception is set.
124    ///
125    /// To take the pending exception, use [`Self::take_pending_exception`].
126    ///
127    /// This method is parameterized over `R` for convenience, but
128    /// will always return an `Err`.
129    ///
130    /// If there is already a pending exception in the store then the previous
131    /// one will be overwritten.
132    ///
133    /// # Errors
134    ///
135    /// This method will return an error if `exception` is unrooted. Otherwise
136    /// this method will always return `ThrownException`.
137    pub fn throw<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
138        self.inner.throw_impl(exception)
139    }
140
141    /// Take the currently pending exception, if any, and return it,
142    /// removing it from the "pending exception" slot.
143    ///
144    /// If there is no pending exception, returns `None`.
145    ///
146    /// Note: the returned exception is a LIFO root (see
147    /// [`crate::Rooted`]), rooted in the current handle scope. Take
148    /// care to ensure that it is re-rooted or otherwise does not
149    /// escape this scope! It is usually best to allow an exception
150    /// object to be rooted in the store's "pending exception" slot
151    /// until the final consumer has taken it, rather than root it and
152    /// pass it up the callstack in some other way.
153    ///
154    /// This method is useful to implement ad-hoc exception plumbing
155    /// in various ways, but for the most idiomatic handling, see
156    /// [`StoreContextMut::throw`].
157    pub fn take_pending_exception(&mut self) -> Option<Rooted<ExnRef>> {
158        self.inner.take_pending_exception_rooted()
159    }
160}
161
162impl<'a, T> StoreContextMut<'a, T> {
163    /// Perform garbage collection.
164    ///
165    /// Same as [`Store::gc`].
166    pub fn gc(&mut self, why: Option<&GcHeapOutOfMemory<()>>) -> Result<()> {
167        let (mut limiter, store) = self.0.validate_sync_resource_limiter_and_store_opaque()?;
168        vm::assert_ready(store.gc(
169            limiter.as_mut(),
170            None,
171            why.map(|e| e.bytes_needed()),
172            Asyncness::No,
173        ))?;
174        Ok(())
175    }
176
177    /// Manually grow the GC heap by at least `bytes` bytes.
178    ///
179    /// For more information, see the documentation of [`Store::gc_heap_grow`].
180    pub fn gc_heap_grow(&mut self, bytes: u64) -> Result<()> {
181        let (mut limiter, store) = self.0.validate_sync_resource_limiter_and_store_opaque()?;
182        vm::assert_ready(store.grow_gc_heap(limiter.as_mut(), bytes, crate::store::Asyncness::No))
183    }
184
185    /// Set an exception as the currently pending exception, and
186    /// return an error that propagates the throw.
187    ///
188    /// See [`Store::throw`] for more details.
189    #[cfg(feature = "gc")]
190    pub fn throw<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
191        self.0.inner.throw_impl(exception)
192    }
193
194    /// Take the currently pending exception, if any, and return it,
195    /// removing it from the "pending exception" slot.
196    ///
197    /// See [`Store::take_pending_exception`] for more details.
198    #[cfg(feature = "gc")]
199    pub fn take_pending_exception(&mut self) -> Option<Rooted<ExnRef>> {
200        self.0.inner.take_pending_exception_rooted()
201    }
202}
203
204#[derive(Debug)]
205struct GcHeapGrowthFailed;
206
207impl fmt::Display for GcHeapGrowthFailed {
208    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
209        f.write_str("GC heap growth failed")
210    }
211}
212
213impl core::error::Error for GcHeapGrowthFailed {}
214
215impl StoreOpaque {
216    /// Perform any growth or GC needed to allocate `bytes_needed` bytes.
217    ///
218    /// Note that even when this function returns it is not guaranteed
219    /// that a GC allocation of size `bytes_needed` will succeed. Growing the GC
220    /// heap could fail, and then performing a collection could succeed but
221    /// might not free up enough space. Therefore, callers should not assume
222    /// that a retried allocation will always succeed.
223    ///
224    /// The `root` argument passed in is considered a root for this GC operation
225    /// and its new value is returned as well.
226    pub(crate) async fn gc(
227        &mut self,
228        limiter: Option<&mut StoreResourceLimiter<'_>>,
229        root: Option<VMGcRef>,
230        bytes_needed: Option<u64>,
231        asyncness: Asyncness,
232    ) -> Result<Option<VMGcRef>> {
233        let mut scope = crate::OpaqueRootScope::new(self);
234        scope.trim_gc_liveness_flags(true);
235        let store_id = scope.id();
236        let root = root.map(|r| scope.gc_roots_mut().push_lifo_root(store_id, r));
237
238        scope
239            .collect_and_maybe_grow_gc_heap(limiter, bytes_needed, asyncness)
240            .await?;
241
242        Ok(root.map(|r| {
243            let r = r
244                .get_gc_ref(&scope)
245                .expect("still in scope")
246                .unchecked_copy();
247            scope.clone_gc_ref(&r)
248        }))
249    }
250
251    // This lives on the Store because it must simultaneously borrow
252    // `gc_store` and `gc_roots`, and is invoked from other modules to
253    // which we do not want to expose the raw fields for piecewise
254    // borrows.
255    pub(crate) fn trim_gc_liveness_flags(&mut self, eager: bool) {
256        if let Some(gc_store) = self.gc_store.as_mut() {
257            self.gc_data.gc_roots.trim_liveness_flags(gc_store, eager);
258        }
259    }
260
261    /// Helper invoked as part of `gc`, whose purpose is to GC and
262    /// maybe grow for a pending allocation of a given size.
263    async fn collect_and_maybe_grow_gc_heap(
264        &mut self,
265        limiter: Option<&mut StoreResourceLimiter<'_>>,
266        bytes_needed: Option<u64>,
267        asyncness: Asyncness,
268    ) -> Result<()> {
269        log::trace!("collect_and_maybe_grow_gc_heap(bytes_needed = {bytes_needed:#x?})");
270        self.do_gc(asyncness).await?;
271        let (allocatable_bytes, last_gc_heap_usage) = self.gc_store.as_ref().map_or((0, 0), |gc| {
272            (
273                gc.gc_heap_allocatable_bytes(),
274                gc.last_post_gc_allocated_bytes.unwrap_or(0),
275            )
276        });
277        if let Some(n) = bytes_needed
278            && n > u64::try_from(allocatable_bytes)?
279                .saturating_sub(u64::try_from(last_gc_heap_usage).unwrap())
280        {
281            if let Err(e) = self.grow_gc_heap(limiter, n, asyncness).await {
282                if e.is::<GcHeapGrowthFailed>() {
283                    log::trace!("ignoring GC heap growth failure: {e}");
284                } else {
285                    return Err(e);
286                }
287            }
288        }
289        Ok(())
290    }
291
292    /// Attempt to grow the GC heap by `bytes_needed` bytes.
293    ///
294    /// Returns an error if growing the GC heap fails.
295    pub(crate) async fn grow_gc_heap(
296        &mut self,
297        mut limiter: Option<&mut StoreResourceLimiter<'_>>,
298        bytes_needed: u64,
299        asyncness: Asyncness,
300    ) -> Result<()> {
301        log::trace!("Attempting to grow the GC heap by at least {bytes_needed:#x} bytes");
302
303        if bytes_needed == 0 {
304            return Ok(());
305        }
306
307        // If the GC heap needs a collection before growth (e.g. the copying
308        // collector's active space is the second half), do a GC first.
309        if self
310            .gc_store
311            .as_ref()
312            .map_or(false, |gc| gc.gc_heap.get().needs_gc_before_next_growth())
313        {
314            self.do_gc(asyncness).await?;
315            debug_assert!(
316                !self
317                    .gc_store
318                    .as_ref()
319                    .map_or(false, |gc| gc.gc_heap.get().needs_gc_before_next_growth()),
320                "needs_gc_before_next_growth should return false after a GC"
321            );
322        }
323
324        // Make sure the GC heap is actually allocated to get grown.
325        self.ensure_gc_store(limiter.as_deref_mut()).await?;
326
327        let page_size = self.engine().tunables().gc_heap_memory_type().page_size();
328
329        // Take the GC heap's underlying memory out of the GC heap, attempt to
330        // grow it, then replace it.
331        let mut heap = TakenGcHeap::new(self);
332
333        let current_size_in_bytes = u64::try_from(heap.memory.byte_size())?;
334        let current_size_in_pages = current_size_in_bytes / page_size;
335
336        // Aim to double the heap size, amortizing the cost of growth.
337        let doubled_size_in_pages = current_size_in_pages.saturating_mul(2);
338        assert!(doubled_size_in_pages >= current_size_in_pages);
339        let delta_pages_for_doubling = doubled_size_in_pages - current_size_in_pages;
340
341        // When doubling our size, saturate at the maximum memory size in pages.
342        //
343        // TODO: we should consult the instance allocator for its configured
344        // maximum memory size, if any, rather than assuming the index
345        // type's maximum size.
346        let max_size_in_bytes = 1 << 32;
347        let max_size_in_pages = max_size_in_bytes / page_size;
348        let delta_to_max_size_in_pages = max_size_in_pages - current_size_in_pages;
349        let delta_pages_for_alloc = delta_pages_for_doubling.min(delta_to_max_size_in_pages);
350
351        // But always make sure we are attempting to grow at least as many pages
352        // as needed by the requested allocation. This must happen *after* the
353        // max-size saturation, so that if we are at the max already, we do not
354        // succeed in growing by zero delta pages, and then return successfully
355        // to our caller, who would be assuming that there is now capacity for
356        // their allocation.
357        let pages_needed = bytes_needed.div_ceil(page_size);
358        assert!(pages_needed > 0);
359        let delta_pages_for_alloc = delta_pages_for_alloc.max(pages_needed);
360        assert!(delta_pages_for_alloc > 0);
361
362        // Safety: we pair growing the GC heap with updating its associated
363        // `VMMemoryDefinition` in the `VMStoreContext` immediately
364        // afterwards.
365        unsafe {
366            heap.memory
367                .grow(delta_pages_for_alloc, limiter)
368                .await
369                .context(GcHeapGrowthFailed)?
370                .ok_or(GcHeapGrowthFailed)?;
371        }
372        *heap.store.vm_store_context.gc_heap.get_mut() = heap.memory.vmmemory();
373
374        let new_size_in_bytes = u64::try_from(heap.memory.byte_size())?;
375        assert!(new_size_in_bytes > current_size_in_bytes);
376        heap.delta_bytes_grown = new_size_in_bytes - current_size_in_bytes;
377        let delta_bytes_for_alloc = delta_pages_for_alloc.checked_mul(page_size).unwrap();
378        assert!(
379            heap.delta_bytes_grown >= delta_bytes_for_alloc,
380            "{} should be greater than or equal to {delta_bytes_for_alloc}",
381            heap.delta_bytes_grown,
382        );
383        log::trace!(
384            "  -> grew GC heap by {:#x} bytes: new size is {new_size_in_bytes:#x} bytes",
385            heap.delta_bytes_grown
386        );
387        return Ok(());
388
389        struct TakenGcHeap<'a> {
390            store: &'a mut StoreOpaque,
391            memory: ManuallyDrop<vm::Memory>,
392            delta_bytes_grown: u64,
393        }
394
395        impl<'a> TakenGcHeap<'a> {
396            fn new(store: &'a mut StoreOpaque) -> TakenGcHeap<'a> {
397                TakenGcHeap {
398                    memory: ManuallyDrop::new(
399                        store.unwrap_gc_store_mut().gc_heap.get_mut().take_memory(),
400                    ),
401                    store,
402                    delta_bytes_grown: 0,
403                }
404            }
405        }
406
407        impl Drop for TakenGcHeap<'_> {
408            fn drop(&mut self) {
409                // SAFETY: this `Drop` guard ensures that this has exclusive
410                // ownership of fields and is thus safe to take `self.memory`.
411                // Additionally for `replace_memory` the memory was previously
412                // taken when this was created so it should be safe to place
413                // back inside the GC heap.
414                unsafe {
415                    self.store
416                        .unwrap_gc_store_mut()
417                        .gc_heap
418                        .get_mut()
419                        .replace_memory(
420                            ManuallyDrop::take(&mut self.memory),
421                            self.delta_bytes_grown,
422                        );
423                }
424            }
425        }
426    }
427
428    fn replace_gc_zeal_alloc_counter(
429        &mut self,
430        new_value: Option<NonZeroU32>,
431    ) -> Option<NonZeroU32> {
432        if let Some(gc_store) = &mut self.gc_store {
433            gc_store.replace_gc_zeal_alloc_counter(new_value)
434        } else {
435            None
436        }
437    }
438
439    /// Attempt an allocation, if it fails due to GC OOM, apply the
440    /// grow-or-collect heuristic and retry.
441    ///
442    /// The heuristic is:
443    /// - If the last post-collection heap usage is less than half the current
444    ///   capacity, collect first, then retry. If that still fails, grow and
445    ///   retry one final time.
446    /// - Otherwise, grow first and retry.
447    pub(crate) async fn retry_after_gc_async<T, U>(
448        &mut self,
449        mut limiter: Option<&mut StoreResourceLimiter<'_>>,
450        value: T,
451        asyncness: Asyncness,
452        alloc_func: impl Fn(&mut Self, T) -> Result<U>,
453    ) -> Result<U>
454    where
455        T: Send + Sync + 'static,
456    {
457        self.ensure_gc_store(limiter.as_deref_mut()).await?;
458
459        match alloc_func(self, value) {
460            Ok(x) => Ok(x),
461            Err(e) => match e.downcast::<crate::GcHeapOutOfMemory<T>>() {
462                Ok(oom) => {
463                    log::trace!("Got GC heap OOM: {oom}");
464
465                    let (value, oom) = oom.take_inner();
466                    let bytes_needed = oom.bytes_needed();
467
468                    let mut store = WithoutGcZealAllocCounter::new(self);
469
470                    let allocatable_bytes = store
471                        .gc_store
472                        .as_ref()
473                        .map_or(0, |gc_store| gc_store.gc_heap_allocatable_bytes());
474                    let last_gc_heap_usage = store.gc_store.as_ref().map_or(0, |gc_store| {
475                        gc_store.last_post_gc_allocated_bytes.unwrap_or(0)
476                    });
477
478                    if should_collect_first(bytes_needed, allocatable_bytes, last_gc_heap_usage) {
479                        log::trace!(
480                            "Collecting first, then retrying; growing GC heap if collecting didn't \
481                             free up enough space, then retrying again"
482                        );
483                        store
484                            .gc(limiter.as_deref_mut(), None, None, asyncness)
485                            .await?;
486
487                        match alloc_func(&mut store, value) {
488                            Ok(x) => Ok(x),
489                            Err(e) => match e.downcast::<crate::GcHeapOutOfMemory<T>>() {
490                                Ok(oom2) => {
491                                    // Collection wasn't enough; grow and try
492                                    // one final time.
493                                    let (value, _) = oom2.take_inner();
494                                    // Ignore error; we'll get one from
495                                    // `alloc_func` below if growth failed and
496                                    // failure to grow was fatal.
497                                    let _ =
498                                        store.grow_gc_heap(limiter, bytes_needed, asyncness).await;
499
500                                    alloc_func(&mut store, value)
501                                }
502                                Err(e) => Err(e),
503                            },
504                        }
505                    } else {
506                        log::trace!(
507                            "Grow GC heap first, collecting if growth failed, then retrying"
508                        );
509
510                        if let Err(e) = store
511                            .grow_gc_heap(limiter.as_deref_mut(), bytes_needed.max(1), asyncness)
512                            .await
513                        {
514                            log::trace!("growing GC heap failed: {e}");
515                            store.gc(limiter, None, None, asyncness).await?;
516                        }
517
518                        alloc_func(&mut store, value)
519                    }
520                }
521                Err(e) => Err(e),
522            },
523        }
524    }
525
526    /// Set a pending exception.
527    ///
528    /// The `exnref` is cloned internally and held on this store to be fetched
529    /// later by an unwind. This method does *not* set up an unwind request on
530    /// the TLS call state; that must be done separately.
531    ///
532    /// GC barriers are not required by the caller of this function.
533    pub(crate) fn set_pending_exception(&mut self, exnref: &VMGcRef) -> crate::Error {
534        debug_assert!(exnref.is_exnref(&**self.unwrap_gc_store_mut().gc_heap.get()));
535        let gc_store = self.gc_store.as_mut().unwrap();
536        match gc_store.write_gc_ref(&mut self.gc_data.pending_exception, Some(exnref)) {
537            Ok(()) => ThrownException.into(),
538            Err(e) => e,
539        }
540    }
541
542    /// Takes the pending exception from this store, if any, and exposes it to
543    /// WebAssembly, returning the raw representation.
544    pub(crate) fn expose_pending_exception_to_wasm(&mut self) -> Option<NonZeroU32> {
545        let exnref = self.gc_data.pending_exception.take()?;
546        let gc_store = self.unwrap_gc_store_mut();
547        debug_assert!(exnref.is_exnref(&**gc_store.gc_heap.get()));
548        Some(gc_store.expose_gc_ref_to_wasm(exnref).unwrap())
549    }
550
551    /// Takes the pending exception of the store, yielding ownership of its
552    /// reference to the `Rooted` that's returned.
553    fn take_pending_exception_rooted(&mut self) -> Option<Rooted<ExnRef>> {
554        let vmexnref = self.gc_data.pending_exception.take()?;
555        debug_assert!(vmexnref.is_exnref(&**self.unwrap_gc_store().gc_heap.get()));
556        let mut nogc = AutoAssertNoGc::new(self);
557        Some(Rooted::new(&mut nogc, vmexnref))
558    }
559
560    /// Returns the (instance,tag) pair that the pending exception in this
561    /// store, if any, references.
562    pub(crate) fn pending_exception_tag_and_instance(
563        &mut self,
564    ) -> Option<(InstanceId, DefinedTagIndex)> {
565        let pending_exnref = self.gc_data.pending_exception.as_ref()?.unchecked_copy();
566        debug_assert!(pending_exnref.is_exnref(&**self.unwrap_gc_store_mut().gc_heap.get()));
567        let store = AutoAssertNoGc::new(self);
568
569        // Note that if the GC heap is corrupt this will return an error, and in
570        // such as situation we return `None` here pretending that there's no
571        // pending exception. This defers the GC heap corruption to get detected
572        // later. This method is primarily called right now to determine if
573        // there's a handler for an exception, and by returning `None` here this
574        // turns into just any old embedder error.
575        let (instance, index) = pending_exnref.into_exnref_unchecked().tag(&store).ok()?;
576
577        // Validate the heap-derived indices: handler lookup only compares them
578        // against live tags, so an unvalidated pair can match the wrong
579        // handler, which then decodes the payload at the wrong types.
580        Some(Tag::from_raw_indices(&store, instance, index)?.to_raw_indices())
581    }
582
583    /// Get an owned rooted reference to the pending exception,
584    /// without taking it off the store.
585    #[cfg(feature = "debug")]
586    pub(crate) fn pending_exception_owned_rooted(
587        &mut self,
588    ) -> Result<Option<crate::OwnedRooted<ExnRef>>, crate::OutOfMemory> {
589        let pending = match &self.gc_data.pending_exception {
590            Some(r) => r,
591            None => return Ok(None),
592        };
593        let cloned = self.gc_store.as_mut().unwrap().clone_gc_ref(pending);
594        let mut nogc = AutoAssertNoGc::new(self);
595        Ok(Some(crate::OwnedRooted::new(&mut nogc, cloned)?))
596    }
597
598    /// Stores `exception` within the store to later get thrown.
599    ///
600    /// Delegates to `self.set_pending_exception` after accessing the internal
601    /// exception pointer.
602    fn throw_impl<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
603        let exception = exception.try_gc_ref(self)?.unchecked_copy();
604        Err(self.set_pending_exception(&exception))
605    }
606
607    /// Helper method to require that a `GcStore` was previously allocated for
608    /// this store, failing if it has not yet been allocated.
609    ///
610    /// Note that this should only be used in a context where allocation of a
611    /// `GcStore` is sure to have already happened prior, otherwise this may
612    /// return a confusing error to embedders which is a bug in Wasmtime.
613    ///
614    /// Some situations where it's safe to call this method:
615    ///
616    /// * There's already a non-null and non-i31 `VMGcRef` in scope. By existing
617    ///   this shows proof that the `GcStore` was previously allocated.
618    /// * During instantiation and instance's `needs_gc_heap` flag will be
619    ///   handled and instantiation will automatically create a GC store.
620    #[inline]
621    pub(crate) fn require_gc_store(&self) -> Result<&GcStore> {
622        match &self.gc_store {
623            Some(gc_store) => Ok(gc_store),
624            None => bail!("GC heap not initialized yet"),
625        }
626    }
627
628    /// Same as [`Self::require_gc_store`], but mutable.
629    #[inline]
630    pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {
631        match &mut self.gc_store {
632            Some(gc_store) => Ok(gc_store),
633            None => bail!("GC heap not initialized yet"),
634        }
635    }
636
637    /// Returns the current capacity of the GC heap in bytes, or 0 if the GC
638    /// heap has not been initialized yet.
639    pub(crate) fn gc_heap_capacity(&self) -> usize {
640        match self.gc_store.as_ref() {
641            Some(gc_store) => gc_store.gc_heap_capacity(),
642            None => 0,
643        }
644    }
645
646    async fn do_gc(&mut self, asyncness: Asyncness) -> Result<()> {
647        // If the GC heap hasn't been initialized, there is nothing to collect.
648        if self.gc_store.is_none() {
649            return Ok(());
650        }
651
652        if log::log_enabled!(log::Level::Trace) {
653            let gc_store = self.gc_store.as_ref().unwrap();
654            let capacity = gc_store.gc_heap_capacity();
655            let live_set_size = gc_store.last_post_gc_allocated_bytes.unwrap_or(0);
656            let utilization = live_set_size as f64 / capacity as f64 * 100.0;
657            log::trace!(
658                "============ Begin GC ===========\n\
659                 \t          GC heap capacity = {capacity:#010x} bytes\n\
660                 \tlast post-GC live-set size = {live_set_size:#010x} bytes\n\
661                 \t       GC heap utilization = {utilization:.02}%",
662            );
663        }
664
665        // Take the GC roots out of `self` so we can borrow it mutably but still
666        // call mutable methods on `self`.
667        let mut roots = core::mem::take(&mut self.gc_data.gc_roots_list);
668
669        self.trace_roots(&mut roots, asyncness).await;
670        self.gc_store
671            .as_mut()
672            .unwrap()
673            .gc(
674                asyncness,
675                unsafe { roots.iter() },
676                &self.modules,
677                &self.gc_data.gc_host_alloc_types,
678                // TODO: Once `Config` has an optional `AsyncFn` field for
679                // yielding to the current async runtime
680                // (e.g. `tokio::task::yield_now`), use that if set; otherwise
681                // fall back to the runtime-agnostic code.
682                yield_now,
683            )
684            .await?;
685
686        // Restore the GC roots for the next GC.
687        roots.clear();
688        self.gc_data.gc_roots_list = roots;
689
690        if log::log_enabled!(log::Level::Trace) {
691            let gc_store = self.gc_store.as_ref().unwrap();
692            let capacity = gc_store.gc_heap_capacity();
693            let live_set_size = gc_store.last_post_gc_allocated_bytes.unwrap_or(0);
694            let utilization = live_set_size as f64 / capacity as f64 * 100.0;
695            log::trace!(
696                "============ End GC ===========\n\
697                 \t     GC heap capacity = {capacity:#010x} bytes\n\
698                 \tpost-GC live-set size = {live_set_size:#010x} bytes\n\
699                 \t  GC heap utilization = {utilization:.02}%",
700            );
701        }
702        Ok(())
703    }
704
705    async fn trace_roots(&mut self, gc_roots_list: &mut GcRootsList, asyncness: Asyncness) {
706        log::trace!("Begin trace GC roots");
707
708        // We shouldn't have any leftover, stale GC roots.
709        assert!(gc_roots_list.is_empty());
710
711        self.trace_wasm_stack_roots(gc_roots_list);
712        if asyncness != Asyncness::No {
713            self.yield_now().await;
714        }
715
716        #[cfg(feature = "stack-switching")]
717        {
718            self.trace_wasm_continuation_roots(gc_roots_list);
719            if asyncness != Asyncness::No {
720                self.yield_now().await;
721            }
722        }
723
724        self.trace_vmctx_roots(gc_roots_list);
725        if asyncness != Asyncness::No {
726            self.yield_now().await;
727        }
728
729        self.trace_instance_roots(gc_roots_list);
730        if asyncness != Asyncness::No {
731            self.yield_now().await;
732        }
733
734        self.trace_user_roots(gc_roots_list);
735        if asyncness != Asyncness::No {
736            self.yield_now().await;
737        }
738
739        self.trace_pending_exception_roots(gc_roots_list);
740
741        log::trace!("End trace GC roots")
742    }
743
744    pub(crate) fn trace_wasm_stack_frame(
745        modules: &ModuleRegistry,
746        gc_roots_list: &mut GcRootsList,
747        frame: Frame,
748    ) {
749        let pc = frame.pc();
750        debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames");
751
752        let fp = frame.fp() as *mut usize;
753        debug_assert!(
754            !fp.is_null(),
755            "we should always get a valid frame pointer for Wasm frames"
756        );
757
758        let (store_code, offset) = modules
759            .store_code_by_pc(pc)
760            .expect("should have store code for Wasm frame");
761        let offset = u32::try_from(offset).unwrap();
762
763        let stack_map =
764            wasmtime_environ::StackMap::lookup(offset, store_code.code_memory().stack_map_data());
765
766        if let Some(stack_map) = stack_map {
767            log::trace!(
768                "We have a stack map that maps {} bytes in this Wasm frame",
769                stack_map.frame_size()
770            );
771
772            let sp = unsafe { stack_map.sp(fp) };
773            for stack_slot in unsafe { stack_map.live_gc_refs(sp) } {
774                unsafe {
775                    Self::trace_wasm_stack_slot(gc_roots_list, stack_slot);
776                }
777            }
778        }
779
780        #[cfg(feature = "debug")]
781        if let Some(frame_table) = store_code.code_memory().frame_table() {
782            for stack_slot in crate::debug::gc_refs_in_frame(frame_table, offset, fp) {
783                unsafe {
784                    Self::trace_wasm_stack_slot(gc_roots_list, stack_slot);
785                }
786            }
787        }
788    }
789
790    unsafe fn trace_wasm_stack_slot(gc_roots_list: &mut GcRootsList, stack_slot: *mut u32) {
791        let raw: u32 = unsafe { core::ptr::read(stack_slot) };
792        log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}");
793
794        let gc_ref = vm::VMGcRef::from_raw_u32(raw);
795        if gc_ref.is_some() {
796            unsafe {
797                gc_roots_list
798                    .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap()));
799            }
800        }
801    }
802
803    fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) {
804        log::trace!("Begin trace GC roots :: Wasm stack");
805
806        Backtrace::trace(self, |frame| {
807            Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);
808            core::ops::ControlFlow::Continue(())
809        });
810
811        #[cfg(feature = "component-model-async")]
812        if self.concurrency_support() {
813            let unwind = self.unwinder();
814            let StoreOpaque {
815                modules,
816                store_data,
817                ..
818            } = self;
819            store_data
820                .components
821                .task_state_mut()
822                .concurrent_state_mut()
823                .trace_fiber_roots(modules, unwind, gc_roots_list);
824        }
825
826        log::trace!("End trace GC roots :: Wasm stack");
827    }
828
829    #[cfg(feature = "stack-switching")]
830    fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) {
831        use crate::vm::{VMPayloads, VMStackState, ValRaw};
832
833        unsafe fn trace_payload_roots(gc_roots_list: &mut GcRootsList, payloads: &VMPayloads) {
834            let gc_ref_data = payloads.gc_ref_data;
835            let payloads = &payloads.buffer;
836            assert!(payloads.length <= payloads.capacity);
837            let Some(gc_ref_data) = gc_ref_data else {
838                return;
839            };
840            let gc_ref_data = gc_ref_data.as_ptr();
841            let data = payloads
842                .data
843                .expect("payload GC metadata requires an allocated payload buffer")
844                .as_ptr();
845
846            for index in 0..usize::try_from(payloads.length).unwrap() {
847                let marker = unsafe { gc_ref_data.add(index).read() };
848                if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF {
849                    let slot = unsafe { data.cast::<ValRaw>().add(index).cast::<u32>() };
850                    unsafe {
851                        StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot);
852                    }
853                }
854            }
855        }
856
857        log::trace!("Begin trace GC roots :: continuations");
858
859        for continuation in &self.continuations {
860            let state = continuation.common_stack_information.state;
861
862            match state {
863                VMStackState::Suspended => {
864                    unsafe {
865                        trace_payload_roots(gc_roots_list, &continuation.values);
866                    }
867                    Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| {
868                        Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);
869                        core::ops::ControlFlow::Continue(())
870                    });
871                }
872                VMStackState::Running => {
873                    // Handled by `trace_wasm_stack_roots`.
874                }
875                VMStackState::Parent => {
876                    // We don't know whether our child is suspended or running, but in
877                    // either case things should be handled correctly when traversing
878                    // further along in the chain, nothing required at this point.
879                }
880                VMStackState::Fresh => unsafe {
881                    trace_payload_roots(gc_roots_list, &continuation.args);
882                },
883                VMStackState::Returned | VMStackState::Trapped => {
884                    // Terminal continuations have no live GC values.
885                }
886            }
887        }
888
889        log::trace!("End trace GC roots :: continuations");
890    }
891
892    fn trace_vmctx_roots(&mut self, gc_roots_list: &mut GcRootsList) {
893        log::trace!("Begin trace GC roots :: vmctx");
894        self.for_each_global(|store, global| global.trace_root(store, gc_roots_list));
895        self.for_each_table(|store, table| table.trace_roots(store, gc_roots_list));
896        log::trace!("End trace GC roots :: vmctx");
897    }
898
899    fn trace_instance_roots(&mut self, gc_roots_list: &mut GcRootsList) {
900        log::trace!("Begin trace GC roots :: instance");
901        for (_id, instance) in &mut self.instances {
902            // SAFETY: the instance's GC roots will remain valid for the
903            // duration of this GC cycle.
904            unsafe {
905                instance
906                    .handle
907                    .get_mut()
908                    .trace_element_segment_roots(gc_roots_list);
909            }
910        }
911        log::trace!("End trace GC roots :: instance");
912    }
913
914    fn trace_user_roots(&mut self, gc_roots_list: &mut GcRootsList) {
915        log::trace!("Begin trace GC roots :: user");
916        self.gc_data.gc_roots.trace_roots(gc_roots_list);
917        log::trace!("End trace GC roots :: user");
918    }
919
920    fn trace_pending_exception_roots(&mut self, gc_roots_list: &mut GcRootsList) {
921        log::trace!("Begin trace GC roots :: pending exception");
922        if let Some(pending_exception) = self.gc_data.pending_exception.as_mut() {
923            unsafe {
924                gc_roots_list.add_vmgcref_root(pending_exception.into(), "Pending exception");
925            }
926        }
927        log::trace!("End trace GC roots :: pending exception");
928    }
929
930    /// Insert a host-allocated GC type into this store.
931    ///
932    /// This makes it suitable for the embedder to allocate instances of this
933    /// type in this store, and we don't have to worry about the type being
934    /// reclaimed (since it is possible that none of the Wasm modules in this
935    /// store are holding it alive).
936    ///
937    /// # Panics
938    ///
939    /// Panics if `ty` was not registered with this store's engine. The types
940    /// here are keyed by their `VMSharedTypeIndex`, which only means anything
941    /// within the engine that assigned it.
942    pub(crate) fn insert_gc_host_alloc_type(&mut self, ty: RegisteredType) {
943        assert!(
944            Engine::same(self.engine(), ty.engine()),
945            "type used with wrong engine"
946        );
947        let trace_info = ty.layout().map(TraceInfo::new);
948        self.gc_data
949            .gc_host_alloc_types
950            .insert(ty.index(), (ty, trace_info));
951    }
952
953    /// Performs a lazy allocation of the `GcStore` within this store, returning
954    /// the previous allocation if it's already present.
955    ///
956    /// This method will, if necessary, allocate a new `GcStore` -- linear
957    /// memory and all. This is a blocking operation due to
958    /// `ResourceLimiterAsync` which means that this should only be executed
959    /// in a fiber context at this time.
960    #[inline]
961    pub(crate) async fn ensure_gc_store(
962        &mut self,
963        limiter: Option<&mut StoreResourceLimiter<'_>>,
964    ) -> Result<&mut GcStore> {
965        if self.gc_store.is_some() {
966            return Ok(self.gc_store.as_mut().unwrap());
967        }
968        self.allocate_gc_store(limiter).await
969    }
970
971    #[inline(never)]
972    async fn allocate_gc_store(
973        &mut self,
974        limiter: Option<&mut StoreResourceLimiter<'_>>,
975    ) -> Result<&mut GcStore> {
976        log::trace!("allocating GC heap for store {:?}", self.id());
977
978        assert!(self.gc_store.is_none());
979        assert_eq!(
980            self.vm_store_context.gc_heap.get_mut().base.as_non_null(),
981            NonNull::dangling(),
982        );
983        assert_eq!(self.vm_store_context.gc_heap.get_mut().current_length(), 0);
984
985        let engine = self.engine();
986        let mem_ty = engine.tunables().gc_heap_memory_type();
987        ensure!(
988            engine.features().gc_types(),
989            "cannot allocate a GC store when GC is disabled at configuration time"
990        );
991        let gc_runtime = engine
992            .gc_runtime()
993            .context("no GC runtime: GC disabled at compile time or configuration time")?;
994
995        // First, allocate the memory that will be our GC heap's storage.
996        let mut request = InstanceAllocationRequest {
997            id: InstanceId::reserved_value(),
998            runtime_info: engine.empty_module_runtime_info(),
999            imports: vm::Imports::default(),
1000            store: self,
1001            limiter,
1002        };
1003
1004        let (mem_alloc_index, mem) = engine
1005            .allocator()
1006            .allocate_memory(
1007                &mut request,
1008                &mem_ty,
1009                None,
1010                wasmtime_environ::MemoryKind::GcHeap,
1011            )
1012            .await?;
1013
1014        // Then, allocate the actual GC heap, passing in that memory
1015        // storage.
1016        let (index, mut heap) =
1017            match engine
1018                .allocator()
1019                .allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index)
1020            {
1021                Ok(pair) => pair,
1022                Err(e) => unsafe {
1023                    engine
1024                        .allocator()
1025                        .deallocate_memory(None, mem_alloc_index, mem);
1026                    return Err(e);
1027                },
1028            };
1029        heap.attach(mem);
1030
1031        let gc_store = GcStore::new(
1032            self.id(),
1033            index,
1034            heap,
1035            engine.tunables().gc_zeal_alloc_counter,
1036        );
1037        *self.vm_store_context.gc_heap.get_mut() = gc_store.vmmemory_definition();
1038        Ok(self.gc_store.insert(gc_store))
1039    }
1040
1041    /// Tests whether there is a pending exception.
1042    pub fn has_pending_exception(&self) -> bool {
1043        self.gc_data.pending_exception.is_some()
1044    }
1045
1046    #[inline]
1047    pub(crate) fn gc_roots(&self) -> &RootSet {
1048        &self.gc_data.gc_roots
1049    }
1050
1051    #[inline]
1052    pub(crate) fn gc_roots_mut(&mut self) -> &mut RootSet {
1053        &mut self.gc_data.gc_roots
1054    }
1055
1056    #[inline]
1057    pub(crate) fn enter_gc_lifo_scope(&self) -> usize {
1058        self.gc_data.gc_roots.enter_lifo_scope()
1059    }
1060
1061    #[inline]
1062    pub(crate) fn exit_gc_lifo_scope(&mut self, scope: usize) {
1063        self.gc_data
1064            .gc_roots
1065            .exit_lifo_scope(self.gc_store.as_mut(), scope);
1066    }
1067}
1068
1069/// RAII type to temporarily disable the GC zeal allocation counter.
1070struct WithoutGcZealAllocCounter<'a> {
1071    store: &'a mut StoreOpaque,
1072    counter: Option<NonZeroU32>,
1073}
1074
1075impl Deref for WithoutGcZealAllocCounter<'_> {
1076    type Target = StoreOpaque;
1077
1078    fn deref(&self) -> &Self::Target {
1079        &self.store
1080    }
1081}
1082
1083impl DerefMut for WithoutGcZealAllocCounter<'_> {
1084    fn deref_mut(&mut self) -> &mut Self::Target {
1085        &mut self.store
1086    }
1087}
1088
1089impl Drop for WithoutGcZealAllocCounter<'_> {
1090    fn drop(&mut self) {
1091        self.store.replace_gc_zeal_alloc_counter(self.counter);
1092    }
1093}
1094
1095impl<'a> WithoutGcZealAllocCounter<'a> {
1096    pub fn new(store: &'a mut StoreOpaque) -> Self {
1097        let counter = store.replace_gc_zeal_alloc_counter(None);
1098        WithoutGcZealAllocCounter { store, counter }
1099    }
1100}
1101
1102/// Given that we've hit a `GcHeapOutOfMemory` error, should we try freeing up
1103/// space by collecting first or by growing the GC heap first?
1104///
1105/// * `bytes_needed`: the number of bytes the mutator wants to allocate
1106///
1107/// * `allocatable_bytes`: The number of bytes that can be allocated into the GC
1108///   heap before a collection is required.
1109///
1110/// * `last_gc_heap_usage`: The precise GC heap usage after the last collection.
1111#[track_caller]
1112fn should_collect_first(
1113    bytes_needed: u64,
1114    allocatable_bytes: usize,
1115    last_gc_heap_usage: usize,
1116) -> bool {
1117    debug_assert!(last_gc_heap_usage <= allocatable_bytes);
1118
1119    // If we haven't allocated the GC heap yet, there's nothing to collect.
1120    //
1121    // Make sure to grow in this scenario even when the GC zeal infrastructure
1122    // passes `bytes_needed = 0`. This way our retry-after-gc logic doesn't
1123    // auto-fail on its second attempt, which would be bad because it doesn't
1124    // necessarily retry more than once.
1125    if allocatable_bytes == 0 {
1126        return false;
1127    }
1128
1129    // The GC zeal infrastructure will use `bytes_needed = 0` to trigger extra
1130    // collections.
1131    if bytes_needed == 0 {
1132        return true;
1133    }
1134
1135    let Ok(bytes_needed) = usize::try_from(bytes_needed) else {
1136        // No point wasting time on collection if we will never be able to
1137        // satisfy the allocation.
1138        return false;
1139    };
1140
1141    if bytes_needed > isize::MAX.cast_unsigned() {
1142        // Similarly, no allocation can be larger than `isize::MAX` in Rust (or
1143        // LLVM), so don't bother wasting time on collection if we will never be
1144        // able to satisfy the allocation.
1145        return false;
1146    }
1147
1148    let Some(predicted_usage) = last_gc_heap_usage.checked_add(bytes_needed) else {
1149        // If we can't represent our predicted usage as a `usize`, we won't be
1150        // able to grow the GC heap to that size, so try collecting first to
1151        // free up space.
1152        return true;
1153    };
1154
1155    // Common case: to balance collection frequency (and its time overhead) with
1156    // GC heap growth (and its space overhead), only prefer growing first if the
1157    // predicted GC heap utilization is greater than half of the bytes that can
1158    // be allocated into the heap before a collection is required.
1159    predicted_usage < allocatable_bytes / 2
1160}
1161
1162#[cfg(test)]
1163mod tests {
1164    use super::should_collect_first;
1165    use crate::{AsContextMut, Config, Engine, ExternRef, Result, Store};
1166
1167    #[test]
1168    fn test_should_collect_first() {
1169        // No GC heap yet special case.
1170        for bytes_needed in 0..256 {
1171            assert_eq!(should_collect_first(bytes_needed, 0, 0), false);
1172        }
1173
1174        // GC zeal special case.
1175        for cap in 1..256 {
1176            for usage in 0..=cap {
1177                assert_eq!(should_collect_first(0, cap, usage), true);
1178            }
1179        }
1180
1181        let max_alloc_usize = isize::MAX.cast_unsigned();
1182        let max_alloc_u64 = u64::try_from(max_alloc_usize).unwrap();
1183
1184        // Allocation size larger than `isize::MAX` --> will never succeed, do
1185        // not bother collecting.
1186        assert_eq!(
1187            should_collect_first(max_alloc_u64 + 1, max_alloc_usize, 0),
1188            false,
1189        );
1190
1191        // Predicted usage overflow --> growth will likely fail, collect first.
1192        assert_eq!(should_collect_first(1, usize::MAX, usize::MAX), true);
1193
1194        // Common case: predicted usage is low --> we likely have more than
1195        // enough space already, so collect first.
1196        assert_eq!(should_collect_first(16, 1024, 64), true);
1197
1198        // Common case: predicted usage is high --> plausible we may not have
1199        // enough space, and we want to amortize the cost of collections, so
1200        // grow first.
1201        assert_eq!(should_collect_first(16, 1024, 512), false);
1202    }
1203
1204    #[test]
1205    fn gc_heap_initial_size() -> Result<()> {
1206        let mut config = Config::new();
1207        config.gc_heap_initial_size(1 << 20);
1208        let engine = Engine::new(&config)?;
1209        let mut store = Store::new(&engine, ());
1210        ExternRef::new(&mut store, 1)?;
1211
1212        let gc_store = store.as_context_mut().0.unwrap_gc_store();
1213        assert_eq!(gc_store.gc_heap_capacity(), 1 << 20);
1214        Ok(())
1215    }
1216}