wasmtime/runtime/vm/
table.rs

1//! Memory management for tables.
2//!
3//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
4
5use crate::prelude::*;
6use crate::runtime::store::StoreResourceLimiter;
7use crate::runtime::vm::stack_switching::VMContObj;
8use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition};
9use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VmPtr};
10use core::alloc::Layout;
11use core::mem;
12use core::ops::Range;
13use core::ptr::NonNull;
14use core::slice;
15use core::{cmp, usize};
16use wasmtime_environ::{
17    FUNCREF_INIT_BIT, FUNCREF_MASK, IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType,
18};
19
20#[derive(Copy, Clone, PartialEq, Eq, Debug)]
21pub enum TableElementType {
22    Func,
23    GcRef,
24    Cont,
25}
26
27impl TableElementType {
28    /// Returns the size required to actually store an element of this particular type
29    pub fn element_size(&self) -> usize {
30        match self {
31            TableElementType::Func => core::mem::size_of::<FuncTableElem>(),
32            TableElementType::GcRef => core::mem::size_of::<Option<VMGcRef>>(),
33            TableElementType::Cont => core::mem::size_of::<ContTableElem>(),
34        }
35    }
36}
37
38/// At-rest representation of a function in a funcref table.
39///
40/// Note that whether or not these pointers are tagged is a property of `Engine`
41/// configuration. Also note that this specifically uses `VmPtr<T>` to handle
42/// provenance here when loading/storing values to a table.
43///
44/// The possible values here are:
45///
46/// * `None` for untagged tables - a null function element
47/// * `Some(_)` for untagged tables - a non-null function element
48/// * `None` for tagged tables - an uninitialized element
49/// * `Some(1)` for tagged tables - a null function element
50/// * `Some(addr | 1)` for tagged tables - a non-null function element
51#[derive(Copy, Clone)]
52#[repr(transparent)]
53struct MaybeTaggedFuncRef(Option<VmPtr<VMFuncRef>>);
54
55impl MaybeTaggedFuncRef {
56    /// Converts the given `ptr`, a valid funcref pointer, into a tagged pointer
57    /// by adding in the `FUNCREF_INIT_BIT`.
58    fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self {
59        let maybe_tagged = if lazy_init {
60            Some(match ptr {
61                Some(ptr) => ptr.map_addr(|a| a | FUNCREF_INIT_BIT),
62                None => NonNull::new(core::ptr::without_provenance_mut(FUNCREF_INIT_BIT)).unwrap(),
63            })
64        } else {
65            ptr
66        };
67        MaybeTaggedFuncRef(maybe_tagged.map(Into::into))
68    }
69
70    /// Converts a tagged pointer into a `TableElement`, returning `UninitFunc`
71    /// for null (not a tagged value) or `FuncRef` for otherwise tagged values.
72    fn into_funcref(self, lazy_init: bool) -> Option<Option<NonNull<VMFuncRef>>> {
73        let ptr = self.0;
74        if lazy_init && ptr.is_none() {
75            None
76        } else {
77            // Masking off the tag bit is harmless whether the table uses lazy
78            // init or not.
79            Some(ptr.and_then(|ptr| NonNull::new(ptr.as_ptr().map_addr(|a| a & FUNCREF_MASK))))
80        }
81    }
82}
83
84pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>;
85pub type ContTableElem = Option<VMContObj>;
86
87/// The maximum of the sizes of any of the table element types
88#[cfg(feature = "pooling-allocator")]
89pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = {
90    // ContTableElem intentionally excluded for "nominal" calculation.
91    let sizes = [
92        core::mem::size_of::<FuncTableElem>(),
93        core::mem::size_of::<Option<VMGcRef>>(),
94    ];
95
96    // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`,
97    // but as a `const` function, so we can use it to define a constant.
98    const fn slice_max(data: &[usize]) -> usize {
99        match data {
100            [] => 0,
101            [head, tail @ ..] => {
102                let tail_max = slice_max(tail);
103                if *head >= tail_max { *head } else { tail_max }
104            }
105        }
106    }
107
108    slice_max(&sizes)
109};
110
111pub enum StaticTable {
112    Func(StaticFuncTable),
113    GcRef(StaticGcRefTable),
114    Cont(StaticContTable),
115}
116
117impl From<StaticFuncTable> for StaticTable {
118    fn from(value: StaticFuncTable) -> Self {
119        Self::Func(value)
120    }
121}
122
123impl From<StaticGcRefTable> for StaticTable {
124    fn from(value: StaticGcRefTable) -> Self {
125        Self::GcRef(value)
126    }
127}
128
129impl From<StaticContTable> for StaticTable {
130    fn from(value: StaticContTable) -> Self {
131        Self::Cont(value)
132    }
133}
134
135pub struct StaticFuncTable {
136    /// Where data for this table is stored. The length of this list is the
137    /// maximum size of the table.
138    data: SendSyncPtr<[FuncTableElem]>,
139    /// The current size of the table.
140    size: usize,
141    /// Whether elements of this table are initialized lazily.
142    lazy_init: bool,
143}
144
145pub struct StaticGcRefTable {
146    /// Where data for this table is stored. The length of this list is the
147    /// maximum size of the table.
148    data: SendSyncPtr<[Option<VMGcRef>]>,
149    /// The current size of the table.
150    size: usize,
151}
152
153pub struct StaticContTable {
154    /// Where data for this table is stored. The length of this list is the
155    /// maximum size of the table.
156    data: SendSyncPtr<[ContTableElem]>,
157    /// The current size of the table.
158    size: usize,
159}
160
161pub enum DynamicTable {
162    Func(DynamicFuncTable),
163    GcRef(DynamicGcRefTable),
164    Cont(DynamicContTable),
165}
166
167impl From<DynamicFuncTable> for DynamicTable {
168    fn from(value: DynamicFuncTable) -> Self {
169        Self::Func(value)
170    }
171}
172
173impl From<DynamicGcRefTable> for DynamicTable {
174    fn from(value: DynamicGcRefTable) -> Self {
175        Self::GcRef(value)
176    }
177}
178
179impl From<DynamicContTable> for DynamicTable {
180    fn from(value: DynamicContTable) -> Self {
181        Self::Cont(value)
182    }
183}
184
185pub struct DynamicFuncTable {
186    /// Dynamically managed storage space for this table. The length of this
187    /// vector is the current size of the table.
188    elements: TryVec<FuncTableElem>,
189    /// Maximum size that `elements` can grow to.
190    maximum: Option<usize>,
191    /// Whether elements of this table are initialized lazily.
192    lazy_init: bool,
193}
194
195pub struct DynamicGcRefTable {
196    /// Dynamically managed storage space for this table. The length of this
197    /// vector is the current size of the table.
198    elements: TryVec<Option<VMGcRef>>,
199    /// Maximum size that `elements` can grow to.
200    maximum: Option<usize>,
201}
202
203pub struct DynamicContTable {
204    /// Dynamically managed storage space for this table. The length of this
205    /// vector is the current size of the table.
206    elements: TryVec<ContTableElem>,
207    /// Maximum size that `elements` can grow to.
208    maximum: Option<usize>,
209}
210
211/// Represents an instance's table.
212pub enum Table {
213    /// A "static" table where storage space is managed externally, currently
214    /// used with the pooling allocator.
215    Static(StaticTable),
216    /// A "dynamic" table where table storage space is dynamically allocated via
217    /// `malloc` (aka Rust's `Vec`).
218    Dynamic(DynamicTable),
219}
220
221impl From<StaticTable> for Table {
222    fn from(value: StaticTable) -> Self {
223        Self::Static(value)
224    }
225}
226
227impl From<StaticFuncTable> for Table {
228    fn from(value: StaticFuncTable) -> Self {
229        let t: StaticTable = value.into();
230        t.into()
231    }
232}
233
234impl From<StaticGcRefTable> for Table {
235    fn from(value: StaticGcRefTable) -> Self {
236        let t: StaticTable = value.into();
237        t.into()
238    }
239}
240
241impl From<StaticContTable> for Table {
242    fn from(value: StaticContTable) -> Self {
243        let t: StaticTable = value.into();
244        t.into()
245    }
246}
247
248impl From<DynamicTable> for Table {
249    fn from(value: DynamicTable) -> Self {
250        Self::Dynamic(value)
251    }
252}
253
254impl From<DynamicFuncTable> for Table {
255    fn from(value: DynamicFuncTable) -> Self {
256        let t: DynamicTable = value.into();
257        t.into()
258    }
259}
260
261impl From<DynamicGcRefTable> for Table {
262    fn from(value: DynamicGcRefTable) -> Self {
263        let t: DynamicTable = value.into();
264        t.into()
265    }
266}
267
268impl From<DynamicContTable> for Table {
269    fn from(value: DynamicContTable) -> Self {
270        let t: DynamicTable = value.into();
271        t.into()
272    }
273}
274
275pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType {
276    match ty.heap_type.top() {
277        WasmHeapTopType::Func => TableElementType::Func,
278        WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef,
279        WasmHeapTopType::Cont => TableElementType::Cont,
280        WasmHeapTopType::Exn => TableElementType::GcRef,
281    }
282}
283
284/// Allocate dynamic table elements of the given length.
285///
286/// Relies on the fact that our tables' elements are initialized to `None`,
287/// which is represented by zero, to allocate pre-zeroed memory from the global
288/// allocator and avoid manual zero-initialization.
289///
290/// # Safety
291///
292/// Should only ever be called with a `T` that is a table element type and where
293/// `Option<T>`'s `None` variant is represented with zero.
294unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<TryVec<Option<T>>> {
295    debug_assert!(
296        unsafe {
297            core::mem::MaybeUninit::<Option<T>>::zeroed()
298                .assume_init()
299                .is_none()
300        },
301        "null table elements are represented with zeroed memory"
302    );
303
304    if len == 0 {
305        return Ok(TryVec::new());
306    }
307
308    let align = mem::align_of::<Option<T>>();
309
310    let size = mem::size_of::<Option<T>>();
311    let size = size.next_multiple_of(align);
312    let size = size
313        .checked_mul(len)
314        .ok_or_else(|| format_err!("overflow calculating table allocation size"))?;
315
316    let layout = Layout::from_size_align(size, align)?;
317
318    let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) };
319    if ptr.is_null() {
320        return Err(OutOfMemory::new(size).into());
321    }
322
323    let elems = unsafe { TryVec::<Option<T>>::from_raw_parts(ptr.cast(), len, len) };
324    debug_assert!(elems.iter().all(|e| e.is_none()));
325
326    Ok(elems)
327}
328
329impl Table {
330    /// Create a new dynamic (movable) table instance for the specified table plan.
331    pub async fn new_dynamic(
332        ty: &wasmtime_environ::Table,
333        tunables: &Tunables,
334        limiter: Option<&mut StoreResourceLimiter<'_>>,
335    ) -> Result<Self> {
336        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
337        match wasm_to_table_type(ty.ref_type) {
338            TableElementType::Func => Ok(Self::from(DynamicFuncTable {
339                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
340                maximum,
341                lazy_init: tunables.table_lazy_init,
342            })),
343            TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable {
344                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
345                maximum,
346            })),
347            TableElementType::Cont => {
348                let mut elements = TryVec::new();
349                elements.resize_with(minimum, || None)?;
350                Ok(Self::from(DynamicContTable { elements, maximum }))
351            }
352        }
353    }
354
355    /// Create a new static (immovable) table instance for the specified table plan.
356    pub async unsafe fn new_static(
357        ty: &wasmtime_environ::Table,
358        tunables: &Tunables,
359        data: SendSyncPtr<[u8]>,
360        limiter: Option<&mut StoreResourceLimiter<'_>>,
361    ) -> Result<Self> {
362        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
363        let size = minimum;
364        let max = maximum.unwrap_or(usize::MAX);
365
366        match wasm_to_table_type(ty.ref_type) {
367            TableElementType::Func => {
368                let len = {
369                    let (before, data, after) = unsafe {
370                        let data = data.as_non_null().as_ref();
371                        data.align_to::<FuncTableElem>()
372                    };
373                    assert!(before.is_empty());
374                    assert!(after.is_empty());
375                    data.len()
376                };
377                ensure!(
378                    usize::try_from(ty.limits.min).unwrap() <= len,
379                    "initial table size of {} exceeds the pooling allocator's \
380                     configured maximum table size of {len} elements",
381                    ty.limits.min,
382                );
383                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
384                    data.as_non_null().cast::<FuncTableElem>(),
385                    cmp::min(len, max),
386                ));
387                Ok(Self::from(StaticFuncTable {
388                    data,
389                    size,
390                    lazy_init: tunables.table_lazy_init,
391                }))
392            }
393            TableElementType::GcRef => {
394                let len = {
395                    let (before, data, after) = unsafe {
396                        let data = data.as_non_null().as_ref();
397                        data.align_to::<Option<VMGcRef>>()
398                    };
399                    assert!(before.is_empty());
400                    assert!(after.is_empty());
401                    data.len()
402                };
403                ensure!(
404                    usize::try_from(ty.limits.min).unwrap() <= len,
405                    "initial table size of {} exceeds the pooling allocator's \
406                     configured maximum table size of {len} elements",
407                    ty.limits.min,
408                );
409                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
410                    data.as_non_null().cast::<Option<VMGcRef>>(),
411                    cmp::min(len, max),
412                ));
413                Ok(Self::from(StaticGcRefTable { data, size }))
414            }
415            TableElementType::Cont => {
416                let len = {
417                    let (before, data, after) = unsafe {
418                        let data = data.as_non_null().as_ref();
419                        data.align_to::<ContTableElem>()
420                    };
421                    assert!(before.is_empty());
422                    assert!(after.is_empty());
423                    data.len()
424                };
425                ensure!(
426                    usize::try_from(ty.limits.min).unwrap() <= len,
427                    "initial table size of {} exceeds the pooling allocator's \
428                     configured maximum table size of {len} elements",
429                    ty.limits.min,
430                );
431                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
432                    data.as_non_null().cast::<ContTableElem>(),
433                    cmp::min(len, max),
434                ));
435                Ok(Self::from(StaticContTable { data, size }))
436            }
437        }
438    }
439
440    // Calls the `store`'s limiter to optionally prevent the table from being created.
441    //
442    // Returns the minimum and maximum size of the table if the table can be created.
443    async fn limit_new(
444        ty: &wasmtime_environ::Table,
445        limiter: Option<&mut StoreResourceLimiter<'_>>,
446    ) -> Result<(usize, Option<usize>)> {
447        // No matter how the table limits are specified
448        // The table size is limited by the host's pointer size
449        let absolute_max = usize::MAX;
450
451        // If the minimum overflows the host's pointer size, then we can't satisfy this request.
452        // We defer the error to later so the `store` can be informed.
453        let minimum = usize::try_from(ty.limits.min).ok();
454
455        // The maximum size of the table is limited by:
456        // * the host's pointer size.
457        // * the table's maximum size if defined.
458        // * if the table is 64-bit.
459        let maximum = match (ty.limits.max, ty.idx_type) {
460            (Some(max), _) => usize::try_from(max).ok(),
461            (None, IndexType::I64) => usize::try_from(u64::MAX).ok(),
462            (None, IndexType::I32) => usize::try_from(u32::MAX).ok(),
463        };
464
465        // Inform the store's limiter what's about to happen.
466        if let Some(limiter) = limiter {
467            if !limiter
468                .table_growing(0, minimum.unwrap_or(absolute_max), maximum)
469                .await?
470            {
471                bail!(
472                    "table minimum size of {} elements exceeds table limits",
473                    ty.limits.min
474                );
475            }
476        }
477
478        // At this point we need to actually handle overflows, so bail out with
479        // an error if we made it this far.
480        let minimum = minimum.ok_or_else(|| {
481            format_err!(
482                "table minimum size of {} elements exceeds table limits",
483                ty.limits.min
484            )
485        })?;
486        Ok((minimum, maximum))
487    }
488
489    /// Returns the type of the elements in this table.
490    pub fn element_type(&self) -> TableElementType {
491        match self {
492            Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => {
493                TableElementType::Func
494            }
495            Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => {
496                TableElementType::GcRef
497            }
498            Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => {
499                TableElementType::Cont
500            }
501        }
502    }
503
504    /// Returns whether or not the underlying storage of the table is "static".
505    #[cfg(feature = "pooling-allocator")]
506    pub(crate) fn is_static(&self) -> bool {
507        matches!(self, Table::Static(_))
508    }
509
510    /// Returns the number of allocated elements.
511    pub fn size(&self) -> usize {
512        match self {
513            Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size,
514            Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size,
515            Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size,
516            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(),
517            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
518                elements.len()
519            }
520            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(),
521        }
522    }
523
524    /// Returns the maximum number of elements at runtime.
525    ///
526    /// Returns `None` if the table is unbounded.
527    ///
528    /// The runtime maximum may not be equal to the maximum from the table's Wasm type
529    /// when it is being constrained by an instance allocator.
530    pub fn maximum(&self) -> Option<usize> {
531        match self {
532            Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()),
533            Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()),
534            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()),
535            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum,
536            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum,
537            Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum,
538        }
539    }
540
541    /// Fill `table[dst..dst + len]` with `val`.
542    ///
543    /// Returns a trap error on out-of-bounds accesses.
544    ///
545    /// # Panics
546    ///
547    /// Panics if `val` does not have a type that matches this table.
548    pub fn fill_func(
549        &mut self,
550        dst: u64,
551        val: Option<NonNull<VMFuncRef>>,
552        len: u64,
553    ) -> Result<(), Trap> {
554        let range = self.validate_fill(dst, len)?;
555        let (funcrefs, lazy_init) = self.funcrefs_mut();
556        funcrefs[range].fill(MaybeTaggedFuncRef::from(val, lazy_init));
557        Ok(())
558    }
559
560    /// Same as [`Self::fill_func`], but for GC references.
561    ///
562    /// # Panics
563    ///
564    /// Also panics if `gc_store.is_none()` and it's needed.
565    pub fn fill_gc_ref(
566        &mut self,
567        mut gc_store: Option<&mut GcStore>,
568        dst: u64,
569        val: Option<&VMGcRef>,
570        len: u64,
571    ) -> Result<()> {
572        let range = self.validate_fill(dst, len)?;
573
574        // Clone the init GC reference into each table slot.
575        for slot in &mut self.gc_refs_mut()[range] {
576            GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), slot, val)?;
577        }
578
579        Ok(())
580    }
581    /// Same as [`Self::fill_func`], but for continuations.
582    pub fn fill_cont(&mut self, dst: u64, val: Option<VMContObj>, len: u64) -> Result<(), Trap> {
583        let range = self.validate_fill(dst, len)?;
584        self.contrefs_mut()[range].fill(val);
585        Ok(())
586    }
587
588    fn validate_fill(&mut self, dst: u64, len: u64) -> Result<Range<usize>, Trap> {
589        let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
590        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
591        let end = start
592            .checked_add(len)
593            .ok_or_else(|| Trap::TableOutOfBounds)?;
594
595        if end > self.size() {
596            return Err(Trap::TableOutOfBounds);
597        }
598        Ok(start..end)
599    }
600
601    /// Grow table by the specified amount of elements.
602    ///
603    /// Returns the previous size of the table if growth is successful.
604    ///
605    /// Returns `None` if table can't be grown by the specified amount of
606    /// elements.
607    ///
608    /// # Panics
609    ///
610    /// Panics if `init_value` does not have a type that matches this table.
611    ///
612    /// # Unsafety
613    ///
614    /// Resizing the table can reallocate its internal elements buffer. This
615    /// table's instance's `VMContext` has raw pointers to the elements buffer
616    /// that are used by Wasm, and they need to be fixed up before we call into
617    /// Wasm again. Failure to do so will result in use-after-free inside Wasm.
618    ///
619    /// Additionally after growth this table will have null filled in for all
620    /// new entries, if any. The embedder must fill in these slots with an
621    /// appropriately typed value for this table's usage to still be sound
622    /// because this table may contain non-null elements, for example.
623    pub async unsafe fn grow(
624        &mut self,
625        mut limiter: Option<&mut StoreResourceLimiter<'_>>,
626        delta: u64,
627    ) -> Result<Option<usize>, Error> {
628        let old_size = self.size();
629
630        // Don't try to resize the table if its size isn't changing, just return
631        // success.
632        if delta == 0 {
633            return Ok(Some(old_size));
634        }
635        let delta = usize::try_from(delta).map_err(|_| Trap::TableOutOfBounds)?;
636
637        let new_size = match old_size.checked_add(delta) {
638            Some(s) => s,
639            None => {
640                if let Some(limiter) = limiter {
641                    limiter
642                        .table_grow_failed(format_err!("overflow calculating new table size"))?;
643                }
644                return Ok(None);
645            }
646        };
647
648        if let Some(limiter) = &mut limiter {
649            if !limiter
650                .table_growing(old_size, new_size, self.maximum())
651                .await?
652            {
653                return Ok(None);
654            }
655        }
656
657        // The WebAssembly spec requires failing a `table.grow` request if
658        // it exceeds the declared limits of the table. We may have set lower
659        // limits in the instance allocator as well.
660        if let Some(max) = self.maximum() {
661            if new_size > max {
662                if let Some(limiter) = limiter {
663                    limiter.table_grow_failed(format_err!("Table maximum size exceeded"))?;
664                }
665                return Ok(None);
666            }
667        }
668
669        // First resize the storage and then fill with the init value
670        match self {
671            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
672                unsafe {
673                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
674                }
675                *size = new_size;
676            }
677            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
678                unsafe {
679                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
680                }
681                *size = new_size;
682            }
683            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => {
684                unsafe {
685                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
686                }
687                *size = new_size;
688            }
689
690            // These calls to `resize` could move the base address of
691            // `elements`. If this table's limits declare it to be fixed-size,
692            // then during AOT compilation we may have promised Cranelift that
693            // the table base address won't change, so it is allowed to optimize
694            // loading the base address. However, in that case the above checks
695            // that delta is non-zero and the new size doesn't exceed the
696            // maximum mean we can't get here.
697            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
698                elements.resize_with(new_size, || None)?;
699            }
700            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
701                elements.resize_with(new_size, || None)?;
702            }
703            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
704                elements.resize_with(new_size, || None)?;
705            }
706        }
707
708        Ok(Some(old_size))
709    }
710
711    /// Get reference to the specified element.
712    ///
713    /// Returns `Ok(None)` if the element is null or uninitialized.
714    /// Returns `Err` if the index is out of bounds.
715    ///
716    /// Panics if this is a table of GC references and `gc_store` is `None`.
717    pub fn get_func(&self, index: u64) -> Result<Option<NonNull<VMFuncRef>>, Trap> {
718        match self.get_func_maybe_init(index)? {
719            Some(elem) => Ok(elem),
720            None => panic!("function index should have been initialized"),
721        }
722    }
723
724    /// Same as [`Self::get_func`], except plumbs through the uninitialized
725    /// variant of functions too as `Ok(None)`. An initialized function element
726    /// is `Ok(Some(element))`
727    pub fn get_func_maybe_init(
728        &self,
729        index: u64,
730    ) -> Result<Option<Option<NonNull<VMFuncRef>>>, Trap> {
731        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
732        let (funcrefs, lazy_init) = self.funcrefs();
733        Ok(funcrefs
734            .get(index)
735            .ok_or(Trap::TableOutOfBounds)?
736            .into_funcref(lazy_init))
737    }
738
739    /// Same as [`Self::get_func`], but for GC references.
740    pub fn get_gc_ref(&self, index: u64) -> Result<Option<&VMGcRef>, Trap> {
741        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
742        let gcref = self.gc_refs().get(index).ok_or(Trap::TableOutOfBounds)?;
743        Ok(gcref.as_ref())
744    }
745
746    /// Same as [`Self::get_func`], but for continuations.
747    pub fn get_cont(&self, index: u64) -> Result<Option<VMContObj>, Trap> {
748        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
749        let cont = self.contrefs().get(index).ok_or(Trap::TableOutOfBounds)?;
750        Ok(*cont)
751    }
752
753    /// Set reference to the specified element.
754    ///
755    /// # Errors
756    ///
757    /// Returns an error if `index` is out of bounds or if this table type does
758    /// not match the element type.
759    ///
760    /// # Panics
761    ///
762    /// Panics if `elem` is not of the right type for this table.
763    pub fn set_func(&mut self, index: u64, elem: Option<NonNull<VMFuncRef>>) -> Result<(), Trap> {
764        let trap = Trap::TableOutOfBounds;
765        let index: usize = index.try_into().map_err(|_| trap)?;
766        let (funcrefs, lazy_init) = self.funcrefs_mut();
767        *funcrefs.get_mut(index).ok_or(trap)? = MaybeTaggedFuncRef::from(elem, lazy_init);
768        Ok(())
769    }
770
771    /// Same as [`Self::set_func`] except for GC references.
772    pub fn set_gc_ref(
773        &mut self,
774        store: Option<&mut GcStore>,
775        index: u64,
776        elem: Option<&VMGcRef>,
777    ) -> Result<()> {
778        let trap = Trap::TableOutOfBounds;
779        let index: usize = index.try_into().map_err(|_| trap)?;
780        GcStore::write_gc_ref_optional_store(
781            store,
782            self.gc_refs_mut().get_mut(index).ok_or(trap)?,
783            elem,
784        )?;
785        Ok(())
786    }
787
788    /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
789    pub fn vmtable(&mut self) -> VMTableDefinition {
790        match self {
791            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
792                VMTableDefinition {
793                    base: data.cast().into(),
794                    current_elements: *size,
795                }
796            }
797            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
798                VMTableDefinition {
799                    base: data.cast().into(),
800                    current_elements: *size,
801                }
802            }
803            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition {
804                base: data.cast().into(),
805                current_elements: *size,
806            },
807            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
808                VMTableDefinition {
809                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
810                    current_elements: elements.len(),
811                }
812            }
813            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
814                VMTableDefinition {
815                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
816                    current_elements: elements.len(),
817                }
818            }
819            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
820                VMTableDefinition {
821                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
822                    current_elements: elements.len(),
823                }
824            }
825        }
826    }
827
828    fn funcrefs(&self) -> (&[MaybeTaggedFuncRef], bool) {
829        assert_eq!(self.element_type(), TableElementType::Func);
830        match self {
831            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
832                elements,
833                lazy_init,
834                ..
835            })) => (
836                unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) },
837                *lazy_init,
838            ),
839            Self::Static(StaticTable::Func(StaticFuncTable {
840                data,
841                size,
842                lazy_init,
843            })) => (
844                unsafe { slice::from_raw_parts(data.as_ptr().cast(), *size) },
845                *lazy_init,
846            ),
847            _ => unreachable!(),
848        }
849    }
850
851    fn funcrefs_mut(&mut self) -> (&mut [MaybeTaggedFuncRef], bool) {
852        assert_eq!(self.element_type(), TableElementType::Func);
853        match self {
854            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
855                elements,
856                lazy_init,
857                ..
858            })) => (
859                unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) },
860                *lazy_init,
861            ),
862            Self::Static(StaticTable::Func(StaticFuncTable {
863                data,
864                size,
865                lazy_init,
866            })) => (
867                unsafe { slice::from_raw_parts_mut(data.as_ptr().cast(), *size) },
868                *lazy_init,
869            ),
870            _ => unreachable!(),
871        }
872    }
873
874    fn gc_refs(&self) -> &[Option<VMGcRef>] {
875        assert_eq!(self.element_type(), TableElementType::GcRef);
876        match self {
877            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
878            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
879                &data.as_non_null().as_ref()[..*size]
880            },
881            _ => unreachable!(),
882        }
883    }
884
885    fn contrefs(&self) -> &[Option<VMContObj>] {
886        assert_eq!(self.element_type(), TableElementType::Cont);
887        match self {
888            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
889                slice::from_raw_parts(elements.as_ptr().cast(), elements.len())
890            },
891            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
892                slice::from_raw_parts(data.as_ptr().cast(), *size)
893            },
894            _ => unreachable!(),
895        }
896    }
897
898    fn contrefs_mut(&mut self) -> &mut [Option<VMContObj>] {
899        assert_eq!(self.element_type(), TableElementType::Cont);
900        match self {
901            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
902                slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len())
903            },
904            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
905                slice::from_raw_parts_mut(data.as_ptr().cast(), *size)
906            },
907            _ => unreachable!(),
908        }
909    }
910
911    /// Get this table's GC references as a slice.
912    ///
913    /// Panics if this is not a table of GC references.
914    pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] {
915        assert_eq!(self.element_type(), TableElementType::GcRef);
916        match self {
917            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
918            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
919                &mut data.as_non_null().as_mut()[..*size]
920            },
921            _ => unreachable!(),
922        }
923    }
924
925    /// Manually resets all table elements to a null bit pattern.
926    ///
927    /// Used in the pooling allocator when `madvise` fails to reset pages back
928    /// to their original contents. In such a situation the previous contents of
929    /// the table are unknown so this resets them all to defined values.
930    pub fn manually_memset_zeros(&mut self) {
931        match self.element_type() {
932            TableElementType::Func => {
933                let (funcrefs, _lazy_init) = self.funcrefs_mut();
934                funcrefs.fill(MaybeTaggedFuncRef(None));
935            }
936            TableElementType::GcRef => {
937                // Note that this explicitly contains no barriers as all table
938                // GC elements should already have been dropped before returning
939                // the table back to the pool.
940                for r in self.gc_refs_mut() {
941                    *r = None;
942                }
943            }
944            TableElementType::Cont => {
945                self.contrefs_mut().fill(None);
946            }
947        }
948    }
949
950    pub fn debug_assert_all_zero(&self) {
951        match self.element_type() {
952            TableElementType::Func => {
953                let (funcrefs, _lazy_init) = self.funcrefs();
954                debug_assert!(funcrefs.iter().all(|f| f.0.is_none()));
955            }
956            TableElementType::GcRef => {
957                debug_assert!(self.gc_refs().iter().all(|r| r.is_none()));
958            }
959            TableElementType::Cont => {
960                debug_assert!(self.contrefs().iter().all(|c| c.is_none()));
961            }
962        }
963    }
964}
965
966// The default table representation is an empty funcref table that cannot grow.
967impl Default for Table {
968    fn default() -> Self {
969        Self::from(StaticFuncTable {
970            data: SendSyncPtr::new(NonNull::from(&mut [])),
971            size: 0,
972            lazy_init: false,
973        })
974    }
975}
wasmtime/runtime/vm/table.rs

wasmtime/runtime/vm/
table.rs
