wasmtime/runtime/vm/gc/enabled/
null.rs

1//! The null collector.
2//!
3//! The null collector bump allocates objects until it runs out of space, at
4//! which point it returns an out-of-memory error. It never collects garbage.
5//! It does not require any GC barriers.
6
7use super::*;
8use crate::{
9    Engine, Trap,
10    prelude::*,
11    vm::{
12        ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
13        GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,
14        VMMemoryDefinition,
15    },
16};
17use core::ptr::NonNull;
18use core::{alloc::Layout, any::Any, num::NonZeroU32};
19use wasmtime_environ::{
20    GcArrayLayout, GcStructLayout, GcTypeLayouts, VMGcKind, VMSharedTypeIndex,
21    null::NullTypeLayouts,
22};
23
24/// The null collector.
25#[derive(Default)]
26pub struct NullCollector {
27    layouts: NullTypeLayouts,
28}
29
30unsafe impl GcRuntime for NullCollector {
31    fn layouts(&self) -> &dyn GcTypeLayouts {
32        &self.layouts
33    }
34
35    fn new_gc_heap(&self, _: &Engine) -> Result<Box<dyn GcHeap>> {
36        let heap = NullHeap::new()?;
37        Ok(Box::new(heap) as _)
38    }
39}
40
41/// A GC heap for the null collector.
42#[repr(C)]
43struct NullHeap {
44    /// Bump-allocation finger indexing within `1..self.heap.len()`.
45    ///
46    /// NB: this is an `UnsafeCell` because it is written to by compiled Wasm
47    /// code.
48    next: SendSyncUnsafeCell<NonZeroU32>,
49
50    /// The number of active no-gc scopes at the current moment.
51    no_gc_count: usize,
52
53    /// The actual storage for the GC heap.
54    memory: Option<crate::vm::Memory>,
55}
56
57/// The common header for all arrays in the null collector.
58#[repr(C)]
59struct VMNullArrayHeader {
60    header: VMGcHeader,
61    length: u32,
62}
63
64unsafe impl GcHeapObject for VMNullArrayHeader {
65    #[inline]
66    fn is(header: &VMGcHeader) -> bool {
67        header.kind() == VMGcKind::ArrayRef
68    }
69}
70
71impl VMNullArrayHeader {
72    fn typed_ref<'a>(
73        gc_heap: &NullHeap,
74        array: &'a VMArrayRef,
75    ) -> &'a TypedGcRef<VMNullArrayHeader> {
76        let gc_ref = array.as_gc_ref();
77        debug_assert!(gc_ref.is_typed::<VMNullArrayHeader>(gc_heap));
78        gc_ref.as_typed_unchecked()
79    }
80}
81
82/// The representation of an `externref` in the null collector.
83#[repr(C)]
84struct VMNullExternRef {
85    header: VMGcHeader,
86    host_data: ExternRefHostDataId,
87}
88
89unsafe impl GcHeapObject for VMNullExternRef {
90    #[inline]
91    fn is(header: &VMGcHeader) -> bool {
92        header.kind() == VMGcKind::ExternRef
93    }
94}
95
96impl VMNullExternRef {
97    /// Convert a generic `externref` to a typed reference to our concrete
98    /// `externref` type.
99    fn typed_ref<'a>(
100        gc_heap: &NullHeap,
101        externref: &'a VMExternRef,
102    ) -> &'a TypedGcRef<VMNullExternRef> {
103        let gc_ref = externref.as_gc_ref();
104        debug_assert!(gc_ref.is_typed::<VMNullExternRef>(gc_heap));
105        gc_ref.as_typed_unchecked()
106    }
107}
108
109impl NullHeap {
110    /// Construct a new, default heap for the null collector.
111    fn new() -> Result<Self> {
112        Ok(Self {
113            no_gc_count: 0,
114            next: SendSyncUnsafeCell::new(NonZeroU32::new(u32::MAX).unwrap()),
115            memory: None,
116        })
117    }
118
119    /// Attempt to bump-allocate an object with the given layout and
120    /// header.
121    ///
122    /// Returns `Ok(Ok(r))` on success, `Ok(Err(bytes_needed))` when we don't
123    /// have enough heap space but growing the GC heap could make it
124    /// allocatable, and `Err(_)` when we don't have enough space and growing
125    /// the GC heap won't help.
126    fn alloc(&mut self, mut header: VMGcHeader, layout: Layout) -> Result<Result<VMGcRef, u64>> {
127        debug_assert!(layout.size() >= core::mem::size_of::<VMGcHeader>());
128        debug_assert!(layout.align() >= core::mem::align_of::<VMGcHeader>());
129
130        // Make sure that the requested allocation's size fits in the GC
131        // header's unused bits.
132        let size = match u32::try_from(layout.size()).ok().and_then(|size| {
133            if VMGcKind::value_fits_in_unused_bits(size) {
134                Some(size)
135            } else {
136                None
137            }
138        }) {
139            Some(size) => size,
140            None => return Err(crate::Trap::AllocationTooLarge.into()),
141        };
142
143        let next = *self.next.get_mut();
144
145        // Increment the bump pointer to the layout's requested alignment.
146        let aligned = match u32::try_from(layout.align())
147            .ok()
148            .and_then(|align| next.get().checked_next_multiple_of(align))
149        {
150            Some(aligned) => aligned,
151            None => return Err(crate::Trap::AllocationTooLarge.into()),
152        };
153
154        // Check whether the allocation fits in the heap space we have left.
155        let end_of_object = match aligned.checked_add(size) {
156            Some(end) => end,
157            None => return Err(crate::Trap::AllocationTooLarge.into()),
158        };
159        let len = self.memory.as_ref().unwrap().byte_size();
160        let len = u32::try_from(len).unwrap_or(u32::MAX);
161        if end_of_object > len {
162            return Ok(Err(u64::try_from(layout.size())?));
163        }
164
165        // Update the bump pointer, write the header, and return the GC ref.
166        *self.next.get_mut() = NonZeroU32::new(end_of_object).unwrap();
167
168        let aligned = NonZeroU32::new(aligned).unwrap();
169        let gc_ref = VMGcRef::from_heap_index(aligned).unwrap();
170
171        debug_assert_eq!(header.reserved_u26(), 0);
172        header.set_reserved_u26(size);
173        *self.header_mut(&gc_ref)? = header;
174
175        Ok(Ok(gc_ref))
176    }
177}
178
179unsafe impl GcHeap for NullHeap {
180    fn is_attached(&self) -> bool {
181        self.memory.is_some()
182    }
183
184    fn attach(&mut self, memory: crate::vm::Memory) {
185        assert!(!self.is_attached());
186        self.memory = Some(memory);
187        self.next = SendSyncUnsafeCell::new(NonZeroU32::new(1).unwrap());
188    }
189
190    fn detach(&mut self) -> crate::vm::Memory {
191        assert!(self.is_attached());
192
193        let NullHeap {
194            next,
195            no_gc_count,
196            memory,
197        } = self;
198
199        *next.get_mut() = NonZeroU32::new(1).unwrap();
200        *no_gc_count = 0;
201
202        self.next = SendSyncUnsafeCell::new(NonZeroU32::new(u32::MAX).unwrap());
203        memory.take().unwrap()
204    }
205
206    fn as_any(&self) -> &dyn Any {
207        self as _
208    }
209
210    fn as_any_mut(&mut self) -> &mut dyn Any {
211        self as _
212    }
213
214    fn enter_no_gc_scope(&mut self) {
215        self.no_gc_count += 1;
216    }
217
218    fn exit_no_gc_scope(&mut self) {
219        self.no_gc_count -= 1;
220    }
221
222    fn take_memory(&mut self) -> crate::vm::Memory {
223        debug_assert!(self.is_attached());
224        self.memory.take().unwrap()
225    }
226
227    unsafe fn replace_memory(&mut self, memory: crate::vm::Memory, _delta_bytes_grown: u64) {
228        debug_assert!(self.memory.is_none());
229        self.memory = Some(memory);
230    }
231
232    fn vmmemory(&self) -> VMMemoryDefinition {
233        debug_assert!(self.is_attached());
234        self.memory.as_ref().unwrap().vmmemory()
235    }
236
237    fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
238        gc_ref.unchecked_copy()
239    }
240
241    fn write_gc_ref(
242        &mut self,
243        _host_data_table: &mut ExternRefHostDataTable,
244        destination: &mut Option<VMGcRef>,
245        source: Option<&VMGcRef>,
246    ) -> Result<()> {
247        *destination = source.map(|s| s.unchecked_copy());
248        Ok(())
249    }
250
251    fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) -> Result<()> {
252        // Don't need to do anything special here.
253        Ok(())
254    }
255
256    fn alloc_externref(
257        &mut self,
258        host_data: ExternRefHostDataId,
259    ) -> Result<Result<VMExternRef, u64>> {
260        let gc_ref = match self.alloc(VMGcHeader::externref(), Layout::new::<VMNullExternRef>())? {
261            Ok(r) => r,
262            Err(bytes_needed) => return Ok(Err(bytes_needed)),
263        };
264        self.index_mut::<VMNullExternRef>(gc_ref.as_typed_unchecked())?
265            .host_data = host_data;
266        Ok(Ok(gc_ref.into_externref_unchecked()))
267    }
268
269    fn externref_host_data(&self, externref: &VMExternRef) -> Result<ExternRefHostDataId> {
270        let typed_ref = VMNullExternRef::typed_ref(self, externref);
271        Ok(self.index(typed_ref)?.host_data)
272    }
273
274    fn object_size(&self, gc_ref: &VMGcRef) -> Result<usize> {
275        let size = self.header(gc_ref)?.reserved_u26();
276        Ok(usize::try_from(size)?)
277    }
278
279    fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
280        self.index(gc_ref.as_typed_unchecked())
281    }
282
283    fn header_mut(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcHeader> {
284        self.index_mut(gc_ref.as_typed_unchecked())
285    }
286
287    fn alloc_raw(&mut self, header: VMGcHeader, layout: Layout) -> Result<Result<VMGcRef, u64>> {
288        self.alloc(header, layout)
289    }
290
291    fn alloc_uninit_struct_or_exn(
292        &mut self,
293        ty: VMSharedTypeIndex,
294        layout: &GcStructLayout,
295    ) -> Result<Result<VMGcRef, u64>> {
296        let kind = if layout.is_exception {
297            VMGcKind::ExnRef
298        } else {
299            VMGcKind::StructRef
300        };
301        self.alloc(VMGcHeader::from_kind_and_index(kind, ty), layout.layout())
302    }
303
304    fn dealloc_uninit_struct_or_exn(&mut self, _struct_ref: VMGcRef) -> Result<()> {
305        Ok(())
306    }
307
308    fn alloc_uninit_array(
309        &mut self,
310        ty: VMSharedTypeIndex,
311        length: u32,
312        layout: &GcArrayLayout,
313    ) -> Result<Result<VMArrayRef, u64>> {
314        let layout = layout.layout(length).ok_or(Trap::AllocationTooLarge)?;
315        let gc_ref = match self.alloc(
316            VMGcHeader::from_kind_and_index(VMGcKind::ArrayRef, ty),
317            layout,
318        )? {
319            Ok(r) => r,
320            Err(bytes_needed) => return Ok(Err(bytes_needed)),
321        };
322        self.index_mut::<VMNullArrayHeader>(gc_ref.as_typed_unchecked())?
323            .length = length;
324        Ok(Ok(gc_ref.into_arrayref_unchecked()))
325    }
326
327    fn dealloc_uninit_array(&mut self, _array_ref: VMArrayRef) -> Result<()> {
328        Ok(())
329    }
330
331    fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
332        let arrayref = VMNullArrayHeader::typed_ref(self, arrayref);
333        Ok(self.index(arrayref)?.length)
334    }
335
336    fn allocated_bytes(&self) -> usize {
337        // The null collector never frees, so everything from the start of
338        // the heap up to the bump pointer is allocated. Subtract 1 because
339        // the bump pointer starts at index 1 (index 0 is unused since
340        // `VMGcRef` uses `NonZeroU32`), not because any byte was allocated.
341        let next = unsafe { *self.next.get() };
342        usize::try_from(next.get()).unwrap() - 1
343    }
344
345    fn gc<'a>(
346        &'a mut self,
347        _roots: GcRootsIter<'a>,
348        _host_data_table: &'a mut ExternRefHostDataTable,
349    ) -> Box<dyn GarbageCollection<'a> + 'a> {
350        assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");
351        Box::new(NullCollection {})
352    }
353
354    unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {
355        let ptr_to_next: *mut NonZeroU32 = unsafe { self.next.get() };
356        NonNull::new(ptr_to_next).unwrap().cast()
357    }
358}
359
360struct NullCollection {}
361
362impl<'a> GarbageCollection<'a> for NullCollection {
363    fn collect_increment(&mut self) -> Result<GcProgress> {
364        Ok(GcProgress::Complete)
365    }
366}
367
368#[cfg(test)]
369mod tests {
370    use super::*;
371
372    #[test]
373    fn vm_gc_null_header_size_align() {
374        assert_eq!(
375            (wasmtime_environ::null::HEADER_SIZE as usize),
376            core::mem::size_of::<VMGcHeader>()
377        );
378        assert_eq!(
379            (wasmtime_environ::null::HEADER_ALIGN as usize),
380            core::mem::align_of::<VMGcHeader>()
381        );
382    }
383
384    #[test]
385    fn vm_null_array_header_length_offset() {
386        assert_eq!(
387            wasmtime_environ::null::ARRAY_LENGTH_OFFSET,
388            u32::try_from(core::mem::offset_of!(VMNullArrayHeader, length)).unwrap(),
389        );
390    }
391}
wasmtime/runtime/vm/gc/enabled/null.rs

wasmtime/runtime/vm/gc/enabled/
null.rs
