1use crate::RootSet;
4use crate::error::{Context, ensure};
5use crate::module::ModuleRegistry;
6use crate::store::{
7 Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,
8};
9use crate::type_registry::RegisteredType;
10use crate::vm::{
11 self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, SendSyncPtr,
12 StoreGcHostAllocTypes, TraceInfo, VMGcRef,
13};
14use crate::{
15 Engine, ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, Tag,
16 ThrownException, bail,
17};
18use core::fmt;
19use core::mem::ManuallyDrop;
20use core::num::NonZeroU32;
21use core::ops::{Deref, DerefMut};
22use core::ptr::NonNull;
23use wasmtime_environ::DefinedTagIndex;
24use wasmtime_environ::packed_option::ReservedValue;
25
26#[derive(Default)]
27pub(crate) struct StoreGcData {
28 gc_roots: RootSet,
29 gc_roots_list: GcRootsList,
30 gc_host_alloc_types: StoreGcHostAllocTypes,
32 pending_exception: Option<VMGcRef>,
46}
47
48impl<T> Store<T> {
49 pub fn gc(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>) -> Result<()> {
67 StoreContextMut(&mut self.inner).gc(why)
68 }
69
70 pub fn gc_heap_capacity(&self) -> usize {
73 self.inner.gc_heap_capacity()
74 }
75
76 pub fn gc_heap_grow(&mut self, bytes: u64) -> Result<()> {
103 StoreContextMut(&mut self.inner).gc_heap_grow(bytes)
104 }
105
106 pub fn throw<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
138 self.inner.throw_impl(exception)
139 }
140
141 pub fn take_pending_exception(&mut self) -> Option<Rooted<ExnRef>> {
158 self.inner.take_pending_exception_rooted()
159 }
160}
161
162impl<'a, T> StoreContextMut<'a, T> {
163 pub fn gc(&mut self, why: Option<&GcHeapOutOfMemory<()>>) -> Result<()> {
167 let (mut limiter, store) = self.0.validate_sync_resource_limiter_and_store_opaque()?;
168 vm::assert_ready(store.gc(
169 limiter.as_mut(),
170 None,
171 why.map(|e| e.bytes_needed()),
172 Asyncness::No,
173 ))?;
174 Ok(())
175 }
176
177 pub fn gc_heap_grow(&mut self, bytes: u64) -> Result<()> {
181 let (mut limiter, store) = self.0.validate_sync_resource_limiter_and_store_opaque()?;
182 vm::assert_ready(store.grow_gc_heap(limiter.as_mut(), bytes, crate::store::Asyncness::No))
183 }
184
185 #[cfg(feature = "gc")]
190 pub fn throw<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
191 self.0.inner.throw_impl(exception)
192 }
193
194 #[cfg(feature = "gc")]
199 pub fn take_pending_exception(&mut self) -> Option<Rooted<ExnRef>> {
200 self.0.inner.take_pending_exception_rooted()
201 }
202}
203
204#[derive(Debug)]
205struct GcHeapGrowthFailed;
206
207impl fmt::Display for GcHeapGrowthFailed {
208 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
209 f.write_str("GC heap growth failed")
210 }
211}
212
213impl core::error::Error for GcHeapGrowthFailed {}
214
215impl StoreOpaque {
216 pub(crate) async fn gc(
227 &mut self,
228 limiter: Option<&mut StoreResourceLimiter<'_>>,
229 root: Option<VMGcRef>,
230 bytes_needed: Option<u64>,
231 asyncness: Asyncness,
232 ) -> Result<Option<VMGcRef>> {
233 let mut scope = crate::OpaqueRootScope::new(self);
234 scope.trim_gc_liveness_flags(true);
235 let store_id = scope.id();
236 let root = root.map(|r| scope.gc_roots_mut().push_lifo_root(store_id, r));
237
238 scope
239 .collect_and_maybe_grow_gc_heap(limiter, bytes_needed, asyncness)
240 .await?;
241
242 Ok(root.map(|r| {
243 let r = r
244 .get_gc_ref(&scope)
245 .expect("still in scope")
246 .unchecked_copy();
247 scope.clone_gc_ref(&r)
248 }))
249 }
250
251 pub(crate) fn trim_gc_liveness_flags(&mut self, eager: bool) {
256 if let Some(gc_store) = self.gc_store.as_mut() {
257 self.gc_data.gc_roots.trim_liveness_flags(gc_store, eager);
258 }
259 }
260
261 async fn collect_and_maybe_grow_gc_heap(
264 &mut self,
265 limiter: Option<&mut StoreResourceLimiter<'_>>,
266 bytes_needed: Option<u64>,
267 asyncness: Asyncness,
268 ) -> Result<()> {
269 log::trace!("collect_and_maybe_grow_gc_heap(bytes_needed = {bytes_needed:#x?})");
270 self.do_gc(asyncness).await?;
271 let (allocatable_bytes, last_gc_heap_usage) = self.gc_store.as_ref().map_or((0, 0), |gc| {
272 (
273 gc.gc_heap_allocatable_bytes(),
274 gc.last_post_gc_allocated_bytes.unwrap_or(0),
275 )
276 });
277 if let Some(n) = bytes_needed
278 && n > u64::try_from(allocatable_bytes)?
279 .saturating_sub(u64::try_from(last_gc_heap_usage).unwrap())
280 {
281 if let Err(e) = self.grow_gc_heap(limiter, n, asyncness).await {
282 if e.is::<GcHeapGrowthFailed>() {
283 log::trace!("ignoring GC heap growth failure: {e}");
284 } else {
285 return Err(e);
286 }
287 }
288 }
289 Ok(())
290 }
291
292 pub(crate) async fn grow_gc_heap(
296 &mut self,
297 mut limiter: Option<&mut StoreResourceLimiter<'_>>,
298 bytes_needed: u64,
299 asyncness: Asyncness,
300 ) -> Result<()> {
301 log::trace!("Attempting to grow the GC heap by at least {bytes_needed:#x} bytes");
302
303 if bytes_needed == 0 {
304 return Ok(());
305 }
306
307 if self
310 .gc_store
311 .as_ref()
312 .map_or(false, |gc| gc.gc_heap.get().needs_gc_before_next_growth())
313 {
314 self.do_gc(asyncness).await?;
315 debug_assert!(
316 !self
317 .gc_store
318 .as_ref()
319 .map_or(false, |gc| gc.gc_heap.get().needs_gc_before_next_growth()),
320 "needs_gc_before_next_growth should return false after a GC"
321 );
322 }
323
324 self.ensure_gc_store(limiter.as_deref_mut()).await?;
326
327 let page_size = self.engine().tunables().gc_heap_memory_type().page_size();
328
329 let mut heap = TakenGcHeap::new(self);
332
333 let current_size_in_bytes = u64::try_from(heap.memory.byte_size())?;
334 let current_size_in_pages = current_size_in_bytes / page_size;
335
336 let doubled_size_in_pages = current_size_in_pages.saturating_mul(2);
338 assert!(doubled_size_in_pages >= current_size_in_pages);
339 let delta_pages_for_doubling = doubled_size_in_pages - current_size_in_pages;
340
341 let max_size_in_bytes = 1 << 32;
347 let max_size_in_pages = max_size_in_bytes / page_size;
348 let delta_to_max_size_in_pages = max_size_in_pages - current_size_in_pages;
349 let delta_pages_for_alloc = delta_pages_for_doubling.min(delta_to_max_size_in_pages);
350
351 let pages_needed = bytes_needed.div_ceil(page_size);
358 assert!(pages_needed > 0);
359 let delta_pages_for_alloc = delta_pages_for_alloc.max(pages_needed);
360 assert!(delta_pages_for_alloc > 0);
361
362 unsafe {
366 heap.memory
367 .grow(delta_pages_for_alloc, limiter)
368 .await
369 .context(GcHeapGrowthFailed)?
370 .ok_or(GcHeapGrowthFailed)?;
371 }
372 *heap.store.vm_store_context.gc_heap.get_mut() = heap.memory.vmmemory();
373
374 let new_size_in_bytes = u64::try_from(heap.memory.byte_size())?;
375 assert!(new_size_in_bytes > current_size_in_bytes);
376 heap.delta_bytes_grown = new_size_in_bytes - current_size_in_bytes;
377 let delta_bytes_for_alloc = delta_pages_for_alloc.checked_mul(page_size).unwrap();
378 assert!(
379 heap.delta_bytes_grown >= delta_bytes_for_alloc,
380 "{} should be greater than or equal to {delta_bytes_for_alloc}",
381 heap.delta_bytes_grown,
382 );
383 log::trace!(
384 " -> grew GC heap by {:#x} bytes: new size is {new_size_in_bytes:#x} bytes",
385 heap.delta_bytes_grown
386 );
387 return Ok(());
388
389 struct TakenGcHeap<'a> {
390 store: &'a mut StoreOpaque,
391 memory: ManuallyDrop<vm::Memory>,
392 delta_bytes_grown: u64,
393 }
394
395 impl<'a> TakenGcHeap<'a> {
396 fn new(store: &'a mut StoreOpaque) -> TakenGcHeap<'a> {
397 TakenGcHeap {
398 memory: ManuallyDrop::new(
399 store.unwrap_gc_store_mut().gc_heap.get_mut().take_memory(),
400 ),
401 store,
402 delta_bytes_grown: 0,
403 }
404 }
405 }
406
407 impl Drop for TakenGcHeap<'_> {
408 fn drop(&mut self) {
409 unsafe {
415 self.store
416 .unwrap_gc_store_mut()
417 .gc_heap
418 .get_mut()
419 .replace_memory(
420 ManuallyDrop::take(&mut self.memory),
421 self.delta_bytes_grown,
422 );
423 }
424 }
425 }
426 }
427
428 fn replace_gc_zeal_alloc_counter(
429 &mut self,
430 new_value: Option<NonZeroU32>,
431 ) -> Option<NonZeroU32> {
432 if let Some(gc_store) = &mut self.gc_store {
433 gc_store.replace_gc_zeal_alloc_counter(new_value)
434 } else {
435 None
436 }
437 }
438
439 pub(crate) async fn retry_after_gc_async<T, U>(
448 &mut self,
449 mut limiter: Option<&mut StoreResourceLimiter<'_>>,
450 value: T,
451 asyncness: Asyncness,
452 alloc_func: impl Fn(&mut Self, T) -> Result<U>,
453 ) -> Result<U>
454 where
455 T: Send + Sync + 'static,
456 {
457 self.ensure_gc_store(limiter.as_deref_mut()).await?;
458
459 match alloc_func(self, value) {
460 Ok(x) => Ok(x),
461 Err(e) => match e.downcast::<crate::GcHeapOutOfMemory<T>>() {
462 Ok(oom) => {
463 log::trace!("Got GC heap OOM: {oom}");
464
465 let (value, oom) = oom.take_inner();
466 let bytes_needed = oom.bytes_needed();
467
468 let mut store = WithoutGcZealAllocCounter::new(self);
469
470 let allocatable_bytes = store
471 .gc_store
472 .as_ref()
473 .map_or(0, |gc_store| gc_store.gc_heap_allocatable_bytes());
474 let last_gc_heap_usage = store.gc_store.as_ref().map_or(0, |gc_store| {
475 gc_store.last_post_gc_allocated_bytes.unwrap_or(0)
476 });
477
478 if should_collect_first(bytes_needed, allocatable_bytes, last_gc_heap_usage) {
479 log::trace!(
480 "Collecting first, then retrying; growing GC heap if collecting didn't \
481 free up enough space, then retrying again"
482 );
483 store
484 .gc(limiter.as_deref_mut(), None, None, asyncness)
485 .await?;
486
487 match alloc_func(&mut store, value) {
488 Ok(x) => Ok(x),
489 Err(e) => match e.downcast::<crate::GcHeapOutOfMemory<T>>() {
490 Ok(oom2) => {
491 let (value, _) = oom2.take_inner();
494 let _ =
498 store.grow_gc_heap(limiter, bytes_needed, asyncness).await;
499
500 alloc_func(&mut store, value)
501 }
502 Err(e) => Err(e),
503 },
504 }
505 } else {
506 log::trace!(
507 "Grow GC heap first, collecting if growth failed, then retrying"
508 );
509
510 if let Err(e) = store
511 .grow_gc_heap(limiter.as_deref_mut(), bytes_needed.max(1), asyncness)
512 .await
513 {
514 log::trace!("growing GC heap failed: {e}");
515 store.gc(limiter, None, None, asyncness).await?;
516 }
517
518 alloc_func(&mut store, value)
519 }
520 }
521 Err(e) => Err(e),
522 },
523 }
524 }
525
526 pub(crate) fn set_pending_exception(&mut self, exnref: &VMGcRef) -> crate::Error {
534 debug_assert!(exnref.is_exnref(&**self.unwrap_gc_store_mut().gc_heap.get()));
535 let gc_store = self.gc_store.as_mut().unwrap();
536 match gc_store.write_gc_ref(&mut self.gc_data.pending_exception, Some(exnref)) {
537 Ok(()) => ThrownException.into(),
538 Err(e) => e,
539 }
540 }
541
542 pub(crate) fn expose_pending_exception_to_wasm(&mut self) -> Option<NonZeroU32> {
545 let exnref = self.gc_data.pending_exception.take()?;
546 let gc_store = self.unwrap_gc_store_mut();
547 debug_assert!(exnref.is_exnref(&**gc_store.gc_heap.get()));
548 Some(gc_store.expose_gc_ref_to_wasm(exnref).unwrap())
549 }
550
551 fn take_pending_exception_rooted(&mut self) -> Option<Rooted<ExnRef>> {
554 let vmexnref = self.gc_data.pending_exception.take()?;
555 debug_assert!(vmexnref.is_exnref(&**self.unwrap_gc_store().gc_heap.get()));
556 let mut nogc = AutoAssertNoGc::new(self);
557 Some(Rooted::new(&mut nogc, vmexnref))
558 }
559
560 pub(crate) fn pending_exception_tag_and_instance(
563 &mut self,
564 ) -> Option<(InstanceId, DefinedTagIndex)> {
565 let pending_exnref = self.gc_data.pending_exception.as_ref()?.unchecked_copy();
566 debug_assert!(pending_exnref.is_exnref(&**self.unwrap_gc_store_mut().gc_heap.get()));
567 let store = AutoAssertNoGc::new(self);
568
569 let (instance, index) = pending_exnref.into_exnref_unchecked().tag(&store).ok()?;
576
577 Some(Tag::from_raw_indices(&store, instance, index)?.to_raw_indices())
581 }
582
583 #[cfg(feature = "debug")]
586 pub(crate) fn pending_exception_owned_rooted(
587 &mut self,
588 ) -> Result<Option<crate::OwnedRooted<ExnRef>>, crate::OutOfMemory> {
589 let pending = match &self.gc_data.pending_exception {
590 Some(r) => r,
591 None => return Ok(None),
592 };
593 let cloned = self.gc_store.as_mut().unwrap().clone_gc_ref(pending);
594 let mut nogc = AutoAssertNoGc::new(self);
595 Ok(Some(crate::OwnedRooted::new(&mut nogc, cloned)?))
596 }
597
598 fn throw_impl<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
603 let exception = exception.try_gc_ref(self)?.unchecked_copy();
604 Err(self.set_pending_exception(&exception))
605 }
606
607 #[inline]
621 pub(crate) fn require_gc_store(&self) -> Result<&GcStore> {
622 match &self.gc_store {
623 Some(gc_store) => Ok(gc_store),
624 None => bail!("GC heap not initialized yet"),
625 }
626 }
627
628 #[inline]
630 pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {
631 match &mut self.gc_store {
632 Some(gc_store) => Ok(gc_store),
633 None => bail!("GC heap not initialized yet"),
634 }
635 }
636
637 pub(crate) fn gc_heap_capacity(&self) -> usize {
640 match self.gc_store.as_ref() {
641 Some(gc_store) => gc_store.gc_heap_capacity(),
642 None => 0,
643 }
644 }
645
646 async fn do_gc(&mut self, asyncness: Asyncness) -> Result<()> {
647 if self.gc_store.is_none() {
649 return Ok(());
650 }
651
652 if log::log_enabled!(log::Level::Trace) {
653 let gc_store = self.gc_store.as_ref().unwrap();
654 let capacity = gc_store.gc_heap_capacity();
655 let live_set_size = gc_store.last_post_gc_allocated_bytes.unwrap_or(0);
656 let utilization = live_set_size as f64 / capacity as f64 * 100.0;
657 log::trace!(
658 "============ Begin GC ===========\n\
659 \t GC heap capacity = {capacity:#010x} bytes\n\
660 \tlast post-GC live-set size = {live_set_size:#010x} bytes\n\
661 \t GC heap utilization = {utilization:.02}%",
662 );
663 }
664
665 let mut roots = core::mem::take(&mut self.gc_data.gc_roots_list);
668
669 self.trace_roots(&mut roots, asyncness).await;
670 self.gc_store
671 .as_mut()
672 .unwrap()
673 .gc(
674 asyncness,
675 unsafe { roots.iter() },
676 &self.modules,
677 &self.gc_data.gc_host_alloc_types,
678 yield_now,
683 )
684 .await?;
685
686 roots.clear();
688 self.gc_data.gc_roots_list = roots;
689
690 if log::log_enabled!(log::Level::Trace) {
691 let gc_store = self.gc_store.as_ref().unwrap();
692 let capacity = gc_store.gc_heap_capacity();
693 let live_set_size = gc_store.last_post_gc_allocated_bytes.unwrap_or(0);
694 let utilization = live_set_size as f64 / capacity as f64 * 100.0;
695 log::trace!(
696 "============ End GC ===========\n\
697 \t GC heap capacity = {capacity:#010x} bytes\n\
698 \tpost-GC live-set size = {live_set_size:#010x} bytes\n\
699 \t GC heap utilization = {utilization:.02}%",
700 );
701 }
702 Ok(())
703 }
704
705 async fn trace_roots(&mut self, gc_roots_list: &mut GcRootsList, asyncness: Asyncness) {
706 log::trace!("Begin trace GC roots");
707
708 assert!(gc_roots_list.is_empty());
710
711 self.trace_wasm_stack_roots(gc_roots_list);
712 if asyncness != Asyncness::No {
713 self.yield_now().await;
714 }
715
716 #[cfg(feature = "stack-switching")]
717 {
718 self.trace_wasm_continuation_roots(gc_roots_list);
719 if asyncness != Asyncness::No {
720 self.yield_now().await;
721 }
722 }
723
724 self.trace_vmctx_roots(gc_roots_list);
725 if asyncness != Asyncness::No {
726 self.yield_now().await;
727 }
728
729 self.trace_instance_roots(gc_roots_list);
730 if asyncness != Asyncness::No {
731 self.yield_now().await;
732 }
733
734 self.trace_user_roots(gc_roots_list);
735 if asyncness != Asyncness::No {
736 self.yield_now().await;
737 }
738
739 self.trace_pending_exception_roots(gc_roots_list);
740
741 log::trace!("End trace GC roots")
742 }
743
744 pub(crate) fn trace_wasm_stack_frame(
745 modules: &ModuleRegistry,
746 gc_roots_list: &mut GcRootsList,
747 frame: Frame,
748 ) {
749 let pc = frame.pc();
750 debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames");
751
752 let fp = frame.fp() as *mut usize;
753 debug_assert!(
754 !fp.is_null(),
755 "we should always get a valid frame pointer for Wasm frames"
756 );
757
758 let (store_code, offset) = modules
759 .store_code_by_pc(pc)
760 .expect("should have store code for Wasm frame");
761 let offset = u32::try_from(offset).unwrap();
762
763 let stack_map =
764 wasmtime_environ::StackMap::lookup(offset, store_code.code_memory().stack_map_data());
765
766 if let Some(stack_map) = stack_map {
767 log::trace!(
768 "We have a stack map that maps {} bytes in this Wasm frame",
769 stack_map.frame_size()
770 );
771
772 let sp = unsafe { stack_map.sp(fp) };
773 for stack_slot in unsafe { stack_map.live_gc_refs(sp) } {
774 unsafe {
775 Self::trace_wasm_stack_slot(gc_roots_list, stack_slot);
776 }
777 }
778 }
779
780 #[cfg(feature = "debug")]
781 if let Some(frame_table) = store_code.code_memory().frame_table() {
782 for stack_slot in crate::debug::gc_refs_in_frame(frame_table, offset, fp) {
783 unsafe {
784 Self::trace_wasm_stack_slot(gc_roots_list, stack_slot);
785 }
786 }
787 }
788 }
789
790 unsafe fn trace_wasm_stack_slot(gc_roots_list: &mut GcRootsList, stack_slot: *mut u32) {
791 let raw: u32 = unsafe { core::ptr::read(stack_slot) };
792 log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}");
793
794 let gc_ref = vm::VMGcRef::from_raw_u32(raw);
795 if gc_ref.is_some() {
796 unsafe {
797 gc_roots_list
798 .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap()));
799 }
800 }
801 }
802
803 fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) {
804 log::trace!("Begin trace GC roots :: Wasm stack");
805
806 Backtrace::trace(self, |frame| {
807 Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);
808 core::ops::ControlFlow::Continue(())
809 });
810
811 #[cfg(feature = "component-model-async")]
812 if self.concurrency_support() {
813 let unwind = self.unwinder();
814 let StoreOpaque {
815 modules,
816 store_data,
817 ..
818 } = self;
819 store_data
820 .components
821 .task_state_mut()
822 .concurrent_state_mut()
823 .trace_fiber_roots(modules, unwind, gc_roots_list);
824 }
825
826 log::trace!("End trace GC roots :: Wasm stack");
827 }
828
829 #[cfg(feature = "stack-switching")]
830 fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) {
831 use crate::vm::{VMPayloads, VMStackState, ValRaw};
832
833 unsafe fn trace_payload_roots(gc_roots_list: &mut GcRootsList, payloads: &VMPayloads) {
834 let gc_ref_data = payloads.gc_ref_data;
835 let payloads = &payloads.buffer;
836 assert!(payloads.length <= payloads.capacity);
837 let Some(gc_ref_data) = gc_ref_data else {
838 return;
839 };
840 let gc_ref_data = gc_ref_data.as_ptr();
841 let data = payloads
842 .data
843 .expect("payload GC metadata requires an allocated payload buffer")
844 .as_ptr();
845
846 for index in 0..usize::try_from(payloads.length).unwrap() {
847 let marker = unsafe { gc_ref_data.add(index).read() };
848 if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF {
849 let slot = unsafe { data.cast::<ValRaw>().add(index).cast::<u32>() };
850 unsafe {
851 StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot);
852 }
853 }
854 }
855 }
856
857 log::trace!("Begin trace GC roots :: continuations");
858
859 for continuation in &self.continuations {
860 let state = continuation.common_stack_information.state;
861
862 match state {
863 VMStackState::Suspended => {
864 unsafe {
865 trace_payload_roots(gc_roots_list, &continuation.values);
866 }
867 Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| {
868 Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);
869 core::ops::ControlFlow::Continue(())
870 });
871 }
872 VMStackState::Running => {
873 }
875 VMStackState::Parent => {
876 }
880 VMStackState::Fresh => unsafe {
881 trace_payload_roots(gc_roots_list, &continuation.args);
882 },
883 VMStackState::Returned | VMStackState::Trapped => {
884 }
886 }
887 }
888
889 log::trace!("End trace GC roots :: continuations");
890 }
891
892 fn trace_vmctx_roots(&mut self, gc_roots_list: &mut GcRootsList) {
893 log::trace!("Begin trace GC roots :: vmctx");
894 self.for_each_global(|store, global| global.trace_root(store, gc_roots_list));
895 self.for_each_table(|store, table| table.trace_roots(store, gc_roots_list));
896 log::trace!("End trace GC roots :: vmctx");
897 }
898
899 fn trace_instance_roots(&mut self, gc_roots_list: &mut GcRootsList) {
900 log::trace!("Begin trace GC roots :: instance");
901 for (_id, instance) in &mut self.instances {
902 unsafe {
905 instance
906 .handle
907 .get_mut()
908 .trace_element_segment_roots(gc_roots_list);
909 }
910 }
911 log::trace!("End trace GC roots :: instance");
912 }
913
914 fn trace_user_roots(&mut self, gc_roots_list: &mut GcRootsList) {
915 log::trace!("Begin trace GC roots :: user");
916 self.gc_data.gc_roots.trace_roots(gc_roots_list);
917 log::trace!("End trace GC roots :: user");
918 }
919
920 fn trace_pending_exception_roots(&mut self, gc_roots_list: &mut GcRootsList) {
921 log::trace!("Begin trace GC roots :: pending exception");
922 if let Some(pending_exception) = self.gc_data.pending_exception.as_mut() {
923 unsafe {
924 gc_roots_list.add_vmgcref_root(pending_exception.into(), "Pending exception");
925 }
926 }
927 log::trace!("End trace GC roots :: pending exception");
928 }
929
930 pub(crate) fn insert_gc_host_alloc_type(&mut self, ty: RegisteredType) {
943 assert!(
944 Engine::same(self.engine(), ty.engine()),
945 "type used with wrong engine"
946 );
947 let trace_info = ty.layout().map(TraceInfo::new);
948 self.gc_data
949 .gc_host_alloc_types
950 .insert(ty.index(), (ty, trace_info));
951 }
952
953 #[inline]
961 pub(crate) async fn ensure_gc_store(
962 &mut self,
963 limiter: Option<&mut StoreResourceLimiter<'_>>,
964 ) -> Result<&mut GcStore> {
965 if self.gc_store.is_some() {
966 return Ok(self.gc_store.as_mut().unwrap());
967 }
968 self.allocate_gc_store(limiter).await
969 }
970
971 #[inline(never)]
972 async fn allocate_gc_store(
973 &mut self,
974 limiter: Option<&mut StoreResourceLimiter<'_>>,
975 ) -> Result<&mut GcStore> {
976 log::trace!("allocating GC heap for store {:?}", self.id());
977
978 assert!(self.gc_store.is_none());
979 assert_eq!(
980 self.vm_store_context.gc_heap.get_mut().base.as_non_null(),
981 NonNull::dangling(),
982 );
983 assert_eq!(self.vm_store_context.gc_heap.get_mut().current_length(), 0);
984
985 let engine = self.engine();
986 let mem_ty = engine.tunables().gc_heap_memory_type();
987 ensure!(
988 engine.features().gc_types(),
989 "cannot allocate a GC store when GC is disabled at configuration time"
990 );
991 let gc_runtime = engine
992 .gc_runtime()
993 .context("no GC runtime: GC disabled at compile time or configuration time")?;
994
995 let mut request = InstanceAllocationRequest {
997 id: InstanceId::reserved_value(),
998 runtime_info: engine.empty_module_runtime_info(),
999 imports: vm::Imports::default(),
1000 store: self,
1001 limiter,
1002 };
1003
1004 let (mem_alloc_index, mem) = engine
1005 .allocator()
1006 .allocate_memory(
1007 &mut request,
1008 &mem_ty,
1009 None,
1010 wasmtime_environ::MemoryKind::GcHeap,
1011 )
1012 .await?;
1013
1014 let (index, mut heap) =
1017 match engine
1018 .allocator()
1019 .allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index)
1020 {
1021 Ok(pair) => pair,
1022 Err(e) => unsafe {
1023 engine
1024 .allocator()
1025 .deallocate_memory(None, mem_alloc_index, mem);
1026 return Err(e);
1027 },
1028 };
1029 heap.attach(mem);
1030
1031 let gc_store = GcStore::new(
1032 self.id(),
1033 index,
1034 heap,
1035 engine.tunables().gc_zeal_alloc_counter,
1036 );
1037 *self.vm_store_context.gc_heap.get_mut() = gc_store.vmmemory_definition();
1038 Ok(self.gc_store.insert(gc_store))
1039 }
1040
1041 pub fn has_pending_exception(&self) -> bool {
1043 self.gc_data.pending_exception.is_some()
1044 }
1045
1046 #[inline]
1047 pub(crate) fn gc_roots(&self) -> &RootSet {
1048 &self.gc_data.gc_roots
1049 }
1050
1051 #[inline]
1052 pub(crate) fn gc_roots_mut(&mut self) -> &mut RootSet {
1053 &mut self.gc_data.gc_roots
1054 }
1055
1056 #[inline]
1057 pub(crate) fn enter_gc_lifo_scope(&self) -> usize {
1058 self.gc_data.gc_roots.enter_lifo_scope()
1059 }
1060
1061 #[inline]
1062 pub(crate) fn exit_gc_lifo_scope(&mut self, scope: usize) {
1063 self.gc_data
1064 .gc_roots
1065 .exit_lifo_scope(self.gc_store.as_mut(), scope);
1066 }
1067}
1068
1069struct WithoutGcZealAllocCounter<'a> {
1071 store: &'a mut StoreOpaque,
1072 counter: Option<NonZeroU32>,
1073}
1074
1075impl Deref for WithoutGcZealAllocCounter<'_> {
1076 type Target = StoreOpaque;
1077
1078 fn deref(&self) -> &Self::Target {
1079 &self.store
1080 }
1081}
1082
1083impl DerefMut for WithoutGcZealAllocCounter<'_> {
1084 fn deref_mut(&mut self) -> &mut Self::Target {
1085 &mut self.store
1086 }
1087}
1088
1089impl Drop for WithoutGcZealAllocCounter<'_> {
1090 fn drop(&mut self) {
1091 self.store.replace_gc_zeal_alloc_counter(self.counter);
1092 }
1093}
1094
1095impl<'a> WithoutGcZealAllocCounter<'a> {
1096 pub fn new(store: &'a mut StoreOpaque) -> Self {
1097 let counter = store.replace_gc_zeal_alloc_counter(None);
1098 WithoutGcZealAllocCounter { store, counter }
1099 }
1100}
1101
1102#[track_caller]
1112fn should_collect_first(
1113 bytes_needed: u64,
1114 allocatable_bytes: usize,
1115 last_gc_heap_usage: usize,
1116) -> bool {
1117 debug_assert!(last_gc_heap_usage <= allocatable_bytes);
1118
1119 if allocatable_bytes == 0 {
1126 return false;
1127 }
1128
1129 if bytes_needed == 0 {
1132 return true;
1133 }
1134
1135 let Ok(bytes_needed) = usize::try_from(bytes_needed) else {
1136 return false;
1139 };
1140
1141 if bytes_needed > isize::MAX.cast_unsigned() {
1142 return false;
1146 }
1147
1148 let Some(predicted_usage) = last_gc_heap_usage.checked_add(bytes_needed) else {
1149 return true;
1153 };
1154
1155 predicted_usage < allocatable_bytes / 2
1160}
1161
1162#[cfg(test)]
1163mod tests {
1164 use super::should_collect_first;
1165 use crate::{AsContextMut, Config, Engine, ExternRef, Result, Store};
1166
1167 #[test]
1168 fn test_should_collect_first() {
1169 for bytes_needed in 0..256 {
1171 assert_eq!(should_collect_first(bytes_needed, 0, 0), false);
1172 }
1173
1174 for cap in 1..256 {
1176 for usage in 0..=cap {
1177 assert_eq!(should_collect_first(0, cap, usage), true);
1178 }
1179 }
1180
1181 let max_alloc_usize = isize::MAX.cast_unsigned();
1182 let max_alloc_u64 = u64::try_from(max_alloc_usize).unwrap();
1183
1184 assert_eq!(
1187 should_collect_first(max_alloc_u64 + 1, max_alloc_usize, 0),
1188 false,
1189 );
1190
1191 assert_eq!(should_collect_first(1, usize::MAX, usize::MAX), true);
1193
1194 assert_eq!(should_collect_first(16, 1024, 64), true);
1197
1198 assert_eq!(should_collect_first(16, 1024, 512), false);
1202 }
1203
1204 #[test]
1205 fn gc_heap_initial_size() -> Result<()> {
1206 let mut config = Config::new();
1207 config.gc_heap_initial_size(1 << 20);
1208 let engine = Engine::new(&config)?;
1209 let mut store = Store::new(&engine, ());
1210 ExternRef::new(&mut store, 1)?;
1211
1212 let gc_store = store.as_context_mut().0.unwrap_gc_store();
1213 assert_eq!(gc_store.gc_heap_capacity(), 1 << 20);
1214 Ok(())
1215 }
1216}