wasmtime/runtime/vm/
table.rs

1//! Memory management for tables.
2//!
3//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
4
5use crate::prelude::*;
6use crate::runtime::store::StoreResourceLimiter;
7use crate::runtime::vm::stack_switching::VMContObj;
8use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition};
9use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VmPtr};
10use core::alloc::Layout;
11use core::cmp;
12use core::mem;
13use core::ops::Range;
14use core::ptr::NonNull;
15use core::slice;
16use wasmtime_environ::{
17    FUNCREF_INIT_BIT, FUNCREF_MASK, IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType,
18};
19
20#[derive(Copy, Clone, PartialEq, Eq, Debug)]
21pub enum TableElementType {
22    Func,
23    GcRef,
24    Cont,
25}
26
27impl TableElementType {
28    /// Returns the size required to actually store an element of this particular type
29    pub fn element_size(&self) -> usize {
30        match self {
31            TableElementType::Func => core::mem::size_of::<FuncTableElem>(),
32            TableElementType::GcRef => core::mem::size_of::<Option<VMGcRef>>(),
33            TableElementType::Cont => core::mem::size_of::<ContTableElem>(),
34        }
35    }
36}
37
38/// At-rest representation of a function in a funcref table.
39///
40/// Note that whether or not these pointers are tagged is a property of `Engine`
41/// configuration. Also note that this specifically uses `VmPtr<T>` to handle
42/// provenance here when loading/storing values to a table.
43///
44/// The possible values here are:
45///
46/// * `None` for untagged tables - a null function element
47/// * `Some(_)` for untagged tables - a non-null function element
48/// * `None` for tagged tables - an uninitialized element
49/// * `Some(1)` for tagged tables - a null function element
50/// * `Some(addr | 1)` for tagged tables - a non-null function element
51#[derive(Copy, Clone)]
52#[repr(transparent)]
53struct MaybeTaggedFuncRef(Option<VmPtr<VMFuncRef>>);
54
55impl MaybeTaggedFuncRef {
56    /// Converts the given `ptr`, a valid funcref pointer, into a tagged pointer
57    /// by adding in the `FUNCREF_INIT_BIT`.
58    fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self {
59        let maybe_tagged = if lazy_init {
60            Some(match ptr {
61                Some(ptr) => ptr.map_addr(|a| a | FUNCREF_INIT_BIT),
62                None => NonNull::new(core::ptr::without_provenance_mut(FUNCREF_INIT_BIT)).unwrap(),
63            })
64        } else {
65            ptr
66        };
67        MaybeTaggedFuncRef(maybe_tagged.map(Into::into))
68    }
69
70    /// Converts a tagged pointer into a `TableElement`, returning `UninitFunc`
71    /// for null (not a tagged value) or `FuncRef` for otherwise tagged values.
72    fn into_funcref(self, lazy_init: bool) -> Option<Option<NonNull<VMFuncRef>>> {
73        let ptr = self.0;
74        if lazy_init && ptr.is_none() {
75            None
76        } else {
77            // Masking off the tag bit is harmless whether the table uses lazy
78            // init or not.
79            Some(ptr.and_then(|ptr| NonNull::new(ptr.as_ptr().map_addr(|a| a & FUNCREF_MASK))))
80        }
81    }
82}
83
84pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>;
85pub type ContTableElem = Option<VMContObj>;
86
87/// The maximum of the sizes of any of the table element types
88#[cfg(feature = "pooling-allocator")]
89pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = {
90    // ContTableElem intentionally excluded for "nominal" calculation.
91    let sizes = [
92        core::mem::size_of::<FuncTableElem>(),
93        core::mem::size_of::<Option<VMGcRef>>(),
94    ];
95
96    // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`,
97    // but as a `const` function, so we can use it to define a constant.
98    const fn slice_max(data: &[usize]) -> usize {
99        match data {
100            [] => 0,
101            [head, tail @ ..] => {
102                let tail_max = slice_max(tail);
103                if *head >= tail_max { *head } else { tail_max }
104            }
105        }
106    }
107
108    slice_max(&sizes)
109};
110
111pub enum StaticTable {
112    Func(StaticFuncTable),
113    GcRef(StaticGcRefTable),
114    Cont(StaticContTable),
115}
116
117impl From<StaticFuncTable> for StaticTable {
118    fn from(value: StaticFuncTable) -> Self {
119        Self::Func(value)
120    }
121}
122
123impl From<StaticGcRefTable> for StaticTable {
124    fn from(value: StaticGcRefTable) -> Self {
125        Self::GcRef(value)
126    }
127}
128
129impl From<StaticContTable> for StaticTable {
130    fn from(value: StaticContTable) -> Self {
131        Self::Cont(value)
132    }
133}
134
135pub struct StaticFuncTable {
136    /// Where data for this table is stored. The length of this list is the
137    /// maximum size of the table.
138    data: SendSyncPtr<[FuncTableElem]>,
139    /// The current size of the table.
140    size: usize,
141    /// Whether elements of this table are initialized lazily.
142    lazy_init: bool,
143}
144
145pub struct StaticGcRefTable {
146    /// Where data for this table is stored. The length of this list is the
147    /// maximum size of the table.
148    data: SendSyncPtr<[Option<VMGcRef>]>,
149    /// The current size of the table.
150    size: usize,
151}
152
153pub struct StaticContTable {
154    /// Where data for this table is stored. The length of this list is the
155    /// maximum size of the table.
156    data: SendSyncPtr<[ContTableElem]>,
157    /// The current size of the table.
158    size: usize,
159}
160
161pub enum DynamicTable {
162    Func(DynamicFuncTable),
163    GcRef(DynamicGcRefTable),
164    Cont(DynamicContTable),
165}
166
167impl From<DynamicFuncTable> for DynamicTable {
168    fn from(value: DynamicFuncTable) -> Self {
169        Self::Func(value)
170    }
171}
172
173impl From<DynamicGcRefTable> for DynamicTable {
174    fn from(value: DynamicGcRefTable) -> Self {
175        Self::GcRef(value)
176    }
177}
178
179impl From<DynamicContTable> for DynamicTable {
180    fn from(value: DynamicContTable) -> Self {
181        Self::Cont(value)
182    }
183}
184
185pub struct DynamicFuncTable {
186    /// Dynamically managed storage space for this table. The length of this
187    /// vector is the current size of the table.
188    elements: TryVec<FuncTableElem>,
189    /// Maximum size that `elements` can grow to.
190    maximum: Option<usize>,
191    /// Whether elements of this table are initialized lazily.
192    lazy_init: bool,
193}
194
195pub struct DynamicGcRefTable {
196    /// Dynamically managed storage space for this table. The length of this
197    /// vector is the current size of the table.
198    elements: TryVec<Option<VMGcRef>>,
199    /// Maximum size that `elements` can grow to.
200    maximum: Option<usize>,
201}
202
203pub struct DynamicContTable {
204    /// Dynamically managed storage space for this table. The length of this
205    /// vector is the current size of the table.
206    elements: TryVec<ContTableElem>,
207    /// Maximum size that `elements` can grow to.
208    maximum: Option<usize>,
209}
210
211/// Represents an instance's table.
212pub enum Table {
213    /// A "static" table where storage space is managed externally, currently
214    /// used with the pooling allocator.
215    Static(StaticTable),
216    /// A "dynamic" table where table storage space is dynamically allocated via
217    /// `malloc` (aka Rust's `Vec`).
218    Dynamic(DynamicTable),
219}
220
221impl From<StaticTable> for Table {
222    fn from(value: StaticTable) -> Self {
223        Self::Static(value)
224    }
225}
226
227impl From<StaticFuncTable> for Table {
228    fn from(value: StaticFuncTable) -> Self {
229        let t: StaticTable = value.into();
230        t.into()
231    }
232}
233
234impl From<StaticGcRefTable> for Table {
235    fn from(value: StaticGcRefTable) -> Self {
236        let t: StaticTable = value.into();
237        t.into()
238    }
239}
240
241impl From<StaticContTable> for Table {
242    fn from(value: StaticContTable) -> Self {
243        let t: StaticTable = value.into();
244        t.into()
245    }
246}
247
248impl From<DynamicTable> for Table {
249    fn from(value: DynamicTable) -> Self {
250        Self::Dynamic(value)
251    }
252}
253
254impl From<DynamicFuncTable> for Table {
255    fn from(value: DynamicFuncTable) -> Self {
256        let t: DynamicTable = value.into();
257        t.into()
258    }
259}
260
261impl From<DynamicGcRefTable> for Table {
262    fn from(value: DynamicGcRefTable) -> Self {
263        let t: DynamicTable = value.into();
264        t.into()
265    }
266}
267
268impl From<DynamicContTable> for Table {
269    fn from(value: DynamicContTable) -> Self {
270        let t: DynamicTable = value.into();
271        t.into()
272    }
273}
274
275pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType {
276    match ty.heap_type.top() {
277        WasmHeapTopType::Func => TableElementType::Func,
278        WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef,
279        WasmHeapTopType::Cont => TableElementType::Cont,
280        WasmHeapTopType::Exn => TableElementType::GcRef,
281    }
282}
283
284/// Allocate dynamic table elements of the given length.
285///
286/// Relies on the fact that our tables' elements are initialized to `None`,
287/// which is represented by zero, to allocate pre-zeroed memory from the global
288/// allocator and avoid manual zero-initialization.
289///
290/// # Safety
291///
292/// Should only ever be called with a `T` that is a table element type and where
293/// `Option<T>`'s `None` variant is represented with zero.
294unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<TryVec<Option<T>>> {
295    debug_assert!(
296        unsafe {
297            core::mem::MaybeUninit::<Option<T>>::zeroed()
298                .assume_init()
299                .is_none()
300        },
301        "null table elements are represented with zeroed memory"
302    );
303
304    if len == 0 {
305        return Ok(TryVec::new());
306    }
307
308    let align = mem::align_of::<Option<T>>();
309
310    let size = mem::size_of::<Option<T>>();
311    let size = size.next_multiple_of(align);
312    let size = size
313        .checked_mul(len)
314        .ok_or_else(|| format_err!("overflow calculating table allocation size"))?;
315
316    let layout = Layout::from_size_align(size, align)?;
317
318    let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) };
319    if ptr.is_null() {
320        return Err(OutOfMemory::new(size).into());
321    }
322
323    let elems = unsafe { TryVec::<Option<T>>::from_raw_parts(ptr.cast(), len, len) };
324    debug_assert!(elems.iter().all(|e| e.is_none()));
325
326    Ok(elems)
327}
328
329impl Table {
330    /// Create a new dynamic (movable) table instance for the specified table plan.
331    pub async fn new_dynamic(
332        ty: &wasmtime_environ::Table,
333        tunables: &Tunables,
334        limiter: Option<&mut StoreResourceLimiter<'_>>,
335    ) -> Result<Self> {
336        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
337        match wasm_to_table_type(ty.ref_type) {
338            TableElementType::Func => Ok(Self::from(DynamicFuncTable {
339                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
340                maximum,
341                lazy_init: tunables.table_lazy_init,
342            })),
343            TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable {
344                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
345                maximum,
346            })),
347            TableElementType::Cont => {
348                let mut elements = TryVec::new();
349                elements.resize_with(minimum, || None)?;
350                Ok(Self::from(DynamicContTable { elements, maximum }))
351            }
352        }
353    }
354
355    /// Create a new static (immovable) table instance for the specified table plan.
356    pub async unsafe fn new_static(
357        ty: &wasmtime_environ::Table,
358        tunables: &Tunables,
359        data: SendSyncPtr<[u8]>,
360        limiter: Option<&mut StoreResourceLimiter<'_>>,
361    ) -> Result<Self> {
362        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
363        let size = minimum;
364        let max = maximum.unwrap_or(usize::MAX);
365
366        match wasm_to_table_type(ty.ref_type) {
367            TableElementType::Func => {
368                let len = {
369                    let (before, data, after) = unsafe {
370                        let data = data.as_non_null().as_ref();
371                        data.align_to::<FuncTableElem>()
372                    };
373                    assert!(before.is_empty());
374                    assert!(after.is_empty());
375                    data.len()
376                };
377                ensure!(
378                    usize::try_from(ty.limits.min).unwrap() <= len,
379                    "initial table size of {} exceeds the pooling allocator's \
380                     configured maximum table size of {len} elements",
381                    ty.limits.min,
382                );
383                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
384                    data.as_non_null().cast::<FuncTableElem>(),
385                    cmp::min(len, max),
386                ));
387                Ok(Self::from(StaticFuncTable {
388                    data,
389                    size,
390                    lazy_init: tunables.table_lazy_init,
391                }))
392            }
393            TableElementType::GcRef => {
394                let len = {
395                    let (before, data, after) = unsafe {
396                        let data = data.as_non_null().as_ref();
397                        data.align_to::<Option<VMGcRef>>()
398                    };
399                    assert!(before.is_empty());
400                    assert!(after.is_empty());
401                    data.len()
402                };
403                ensure!(
404                    usize::try_from(ty.limits.min).unwrap() <= len,
405                    "initial table size of {} exceeds the pooling allocator's \
406                     configured maximum table size of {len} elements",
407                    ty.limits.min,
408                );
409                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
410                    data.as_non_null().cast::<Option<VMGcRef>>(),
411                    cmp::min(len, max),
412                ));
413                Ok(Self::from(StaticGcRefTable { data, size }))
414            }
415            TableElementType::Cont => {
416                let len = {
417                    let (before, data, after) = unsafe {
418                        let data = data.as_non_null().as_ref();
419                        data.align_to::<ContTableElem>()
420                    };
421                    assert!(before.is_empty());
422                    assert!(after.is_empty());
423                    data.len()
424                };
425                ensure!(
426                    usize::try_from(ty.limits.min).unwrap() <= len,
427                    "initial table size of {} exceeds the pooling allocator's \
428                     configured maximum table size of {len} elements",
429                    ty.limits.min,
430                );
431                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
432                    data.as_non_null().cast::<ContTableElem>(),
433                    cmp::min(len, max),
434                ));
435                Ok(Self::from(StaticContTable { data, size }))
436            }
437        }
438    }
439
440    // Calls the `store`'s limiter to optionally prevent the table from being created.
441    //
442    // Returns the minimum and maximum size of the table if the table can be created.
443    async fn limit_new(
444        ty: &wasmtime_environ::Table,
445        limiter: Option<&mut StoreResourceLimiter<'_>>,
446    ) -> Result<(usize, Option<usize>)> {
447        // No matter how the table limits are specified
448        // The table size is limited by the host's pointer size
449        let absolute_max = usize::MAX;
450
451        // If the minimum overflows the host's pointer size, then we can't satisfy this request.
452        // We defer the error to later so the `store` can be informed.
453        let minimum = usize::try_from(ty.limits.min).ok();
454
455        // The maximum size of the table is limited by:
456        // * the host's pointer size.
457        // * the table's maximum size if defined.
458        // * if the table is 64-bit.
459        let maximum = match (ty.limits.max, ty.idx_type) {
460            (Some(max), _) => usize::try_from(max).ok(),
461            (None, IndexType::I64) => usize::try_from(u64::MAX).ok(),
462            (None, IndexType::I32) => usize::try_from(u32::MAX).ok(),
463        };
464
465        // Inform the store's limiter what's about to happen.
466        if let Some(limiter) = limiter {
467            if !limiter
468                .table_growing(0, minimum.unwrap_or(absolute_max), maximum)
469                .await?
470            {
471                bail!(
472                    "table minimum size of {} elements exceeds table limits",
473                    ty.limits.min
474                );
475            }
476        }
477
478        // At this point we need to actually handle overflows, so bail out with
479        // an error if we made it this far.
480        let minimum = minimum.ok_or_else(|| {
481            format_err!(
482                "table minimum size of {} elements exceeds table limits",
483                ty.limits.min
484            )
485        })?;
486        Ok((minimum, maximum))
487    }
488
489    /// Returns the type of the elements in this table.
490    pub fn element_type(&self) -> TableElementType {
491        match self {
492            Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => {
493                TableElementType::Func
494            }
495            Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => {
496                TableElementType::GcRef
497            }
498            Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => {
499                TableElementType::Cont
500            }
501        }
502    }
503
504    /// Returns whether or not the underlying storage of the table is "static".
505    #[cfg(feature = "pooling-allocator")]
506    pub(crate) fn is_static(&self) -> bool {
507        matches!(self, Table::Static(_))
508    }
509
510    /// Returns the number of allocated elements.
511    pub fn size(&self) -> usize {
512        match self {
513            Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size,
514            Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size,
515            Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size,
516            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(),
517            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
518                elements.len()
519            }
520            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(),
521        }
522    }
523
524    /// Returns the maximum number of elements at runtime.
525    ///
526    /// Returns `None` if the table is unbounded.
527    ///
528    /// The runtime maximum may not be equal to the maximum from the table's Wasm type
529    /// when it is being constrained by an instance allocator.
530    pub fn maximum(&self) -> Option<usize> {
531        match self {
532            Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()),
533            Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()),
534            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()),
535            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum,
536            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum,
537            Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum,
538        }
539    }
540
541    /// Fill `table[dst..dst + len]` with `val`.
542    ///
543    /// Returns a trap error on out-of-bounds accesses.
544    ///
545    /// # Panics
546    ///
547    /// Panics if `val` does not have a type that matches this table.
548    pub fn fill_func(
549        &mut self,
550        dst: u64,
551        val: Option<NonNull<VMFuncRef>>,
552        len: u64,
553    ) -> Result<(), Trap> {
554        let range = self.validate_fill(dst, len)?;
555        let (funcrefs, lazy_init) = self.funcrefs_mut();
556        funcrefs[range].fill(MaybeTaggedFuncRef::from(val, lazy_init));
557        Ok(())
558    }
559
560    /// Same as [`Self::fill_func`], but for GC references.
561    ///
562    /// # Panics
563    ///
564    /// Also panics if `gc_store.is_none()` and it's needed.
565    pub fn fill_gc_ref(
566        &mut self,
567        mut gc_store: Option<&mut GcStore>,
568        dst: u64,
569        val: Option<&VMGcRef>,
570        len: u64,
571    ) -> Result<()> {
572        let range = self.validate_fill(dst, len)?;
573
574        // Clone the init GC reference into each table slot.
575        for slot in &mut self.gc_refs_mut()[range] {
576            GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), slot, val)?;
577        }
578
579        Ok(())
580    }
581    /// Same as [`Self::fill_func`], but for continuations.
582    pub fn fill_cont(&mut self, dst: u64, val: Option<VMContObj>, len: u64) -> Result<(), Trap> {
583        let range = self.validate_fill(dst, len)?;
584        self.contrefs_mut()[range].fill(val);
585        Ok(())
586    }
587
588    fn validate_fill(&mut self, dst: u64, len: u64) -> Result<Range<usize>, Trap> {
589        let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
590        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
591        let end = start
592            .checked_add(len)
593            .ok_or_else(|| Trap::TableOutOfBounds)?;
594
595        if end > self.size() {
596            return Err(Trap::TableOutOfBounds);
597        }
598        Ok(start..end)
599    }
600
601    /// Grow table by the specified amount of elements.
602    ///
603    /// Returns the previous size of the table if growth is successful.
604    ///
605    /// Returns `None` if table can't be grown by the specified amount of
606    /// elements.
607    ///
608    /// # Panics
609    ///
610    /// Panics if `init_value` does not have a type that matches this table.
611    ///
612    /// # Unsafety
613    ///
614    /// Resizing the table can reallocate its internal elements buffer. This
615    /// table's instance's `VMContext` has raw pointers to the elements buffer
616    /// that are used by Wasm, and they need to be fixed up before we call into
617    /// Wasm again. Failure to do so will result in use-after-free inside Wasm.
618    ///
619    /// Additionally after growth this table will have null filled in for all
620    /// new entries, if any. The embedder must fill in these slots with an
621    /// appropriately typed value for this table's usage to still be sound
622    /// because this table may contain non-null elements, for example.
623    pub async unsafe fn grow(
624        &mut self,
625        mut limiter: Option<&mut StoreResourceLimiter<'_>>,
626        delta: u64,
627    ) -> Result<Option<usize>, Error> {
628        let old_size = self.size();
629
630        // Don't try to resize the table if its size isn't changing, just return
631        // success.
632        if delta == 0 {
633            return Ok(Some(old_size));
634        }
635
636        // Clamp to maximum usize, the system won't be able to allocate this
637        // anyway and we'll go to the normal failure path.
638        let delta = usize::try_from(delta).unwrap_or(usize::MAX);
639
640        let new_size = match old_size.checked_add(delta) {
641            Some(s) => s,
642            None => {
643                if let Some(limiter) = limiter {
644                    limiter
645                        .table_grow_failed(format_err!("overflow calculating new table size"))?;
646                }
647                return Ok(None);
648            }
649        };
650
651        if let Some(limiter) = &mut limiter {
652            if !limiter
653                .table_growing(old_size, new_size, self.maximum())
654                .await?
655            {
656                return Ok(None);
657            }
658        }
659
660        // The WebAssembly spec requires failing a `table.grow` request if
661        // it exceeds the declared limits of the table. We may have set lower
662        // limits in the instance allocator as well.
663        if let Some(max) = self.maximum() {
664            if new_size > max {
665                if let Some(limiter) = limiter {
666                    limiter.table_grow_failed(format_err!("Table maximum size exceeded"))?;
667                }
668                return Ok(None);
669            }
670        }
671
672        // First resize the storage and then fill with the init value
673        match self {
674            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
675                unsafe {
676                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
677                }
678                *size = new_size;
679            }
680            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
681                unsafe {
682                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
683                }
684                *size = new_size;
685            }
686            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => {
687                unsafe {
688                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
689                }
690                *size = new_size;
691            }
692
693            // These calls to `resize` could move the base address of
694            // `elements`. If this table's limits declare it to be fixed-size,
695            // then during AOT compilation we may have promised Cranelift that
696            // the table base address won't change, so it is allowed to optimize
697            // loading the base address. However, in that case the above checks
698            // that delta is non-zero and the new size doesn't exceed the
699            // maximum mean we can't get here.
700            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
701                elements.resize_with(new_size, || None)?;
702            }
703            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
704                elements.resize_with(new_size, || None)?;
705            }
706            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
707                elements.resize_with(new_size, || None)?;
708            }
709        }
710
711        Ok(Some(old_size))
712    }
713
714    /// Get reference to the specified element.
715    ///
716    /// Returns `Ok(None)` if the element is null or uninitialized.
717    /// Returns `Err` if the index is out of bounds.
718    ///
719    /// Panics if this is a table of GC references and `gc_store` is `None`.
720    pub fn get_func(&self, index: u64) -> Result<Option<NonNull<VMFuncRef>>, Trap> {
721        match self.get_func_maybe_init(index)? {
722            Some(elem) => Ok(elem),
723            None => panic!("function index should have been initialized"),
724        }
725    }
726
727    /// Same as [`Self::get_func`], except plumbs through the uninitialized
728    /// variant of functions too as `Ok(None)`. An initialized function element
729    /// is `Ok(Some(element))`
730    pub fn get_func_maybe_init(
731        &self,
732        index: u64,
733    ) -> Result<Option<Option<NonNull<VMFuncRef>>>, Trap> {
734        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
735        let (funcrefs, lazy_init) = self.funcrefs();
736        Ok(funcrefs
737            .get(index)
738            .ok_or(Trap::TableOutOfBounds)?
739            .into_funcref(lazy_init))
740    }
741
742    /// Same as [`Self::get_func`], but for GC references.
743    pub fn get_gc_ref(&self, index: u64) -> Result<Option<&VMGcRef>, Trap> {
744        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
745        let gcref = self.gc_refs().get(index).ok_or(Trap::TableOutOfBounds)?;
746        Ok(gcref.as_ref())
747    }
748
749    /// Same as [`Self::get_func`], but for continuations.
750    pub fn get_cont(&self, index: u64) -> Result<Option<VMContObj>, Trap> {
751        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
752        let cont = self.contrefs().get(index).ok_or(Trap::TableOutOfBounds)?;
753        Ok(*cont)
754    }
755
756    /// Set reference to the specified element.
757    ///
758    /// # Errors
759    ///
760    /// Returns an error if `index` is out of bounds or if this table type does
761    /// not match the element type.
762    ///
763    /// # Panics
764    ///
765    /// Panics if `elem` is not of the right type for this table.
766    pub fn set_func(&mut self, index: u64, elem: Option<NonNull<VMFuncRef>>) -> Result<(), Trap> {
767        let trap = Trap::TableOutOfBounds;
768        let index: usize = index.try_into().map_err(|_| trap)?;
769        let (funcrefs, lazy_init) = self.funcrefs_mut();
770        *funcrefs.get_mut(index).ok_or(trap)? = MaybeTaggedFuncRef::from(elem, lazy_init);
771        Ok(())
772    }
773
774    /// Same as [`Self::set_func`] except for GC references.
775    pub fn set_gc_ref(
776        &mut self,
777        store: Option<&mut GcStore>,
778        index: u64,
779        elem: Option<&VMGcRef>,
780    ) -> Result<()> {
781        let trap = Trap::TableOutOfBounds;
782        let index: usize = index.try_into().map_err(|_| trap)?;
783        GcStore::write_gc_ref_optional_store(
784            store,
785            self.gc_refs_mut().get_mut(index).ok_or(trap)?,
786            elem,
787        )?;
788        Ok(())
789    }
790
791    /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
792    pub fn vmtable(&mut self) -> VMTableDefinition {
793        match self {
794            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
795                VMTableDefinition {
796                    base: data.cast().into(),
797                    current_elements: *size,
798                }
799            }
800            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
801                VMTableDefinition {
802                    base: data.cast().into(),
803                    current_elements: *size,
804                }
805            }
806            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition {
807                base: data.cast().into(),
808                current_elements: *size,
809            },
810            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
811                VMTableDefinition {
812                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
813                    current_elements: elements.len(),
814                }
815            }
816            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
817                VMTableDefinition {
818                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
819                    current_elements: elements.len(),
820                }
821            }
822            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
823                VMTableDefinition {
824                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
825                    current_elements: elements.len(),
826                }
827            }
828        }
829    }
830
831    fn funcrefs(&self) -> (&[MaybeTaggedFuncRef], bool) {
832        assert_eq!(self.element_type(), TableElementType::Func);
833        match self {
834            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
835                elements,
836                lazy_init,
837                ..
838            })) => (
839                unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) },
840                *lazy_init,
841            ),
842            Self::Static(StaticTable::Func(StaticFuncTable {
843                data,
844                size,
845                lazy_init,
846            })) => (
847                unsafe { slice::from_raw_parts(data.as_ptr().cast(), *size) },
848                *lazy_init,
849            ),
850            _ => unreachable!(),
851        }
852    }
853
854    fn funcrefs_mut(&mut self) -> (&mut [MaybeTaggedFuncRef], bool) {
855        assert_eq!(self.element_type(), TableElementType::Func);
856        match self {
857            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
858                elements,
859                lazy_init,
860                ..
861            })) => (
862                unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) },
863                *lazy_init,
864            ),
865            Self::Static(StaticTable::Func(StaticFuncTable {
866                data,
867                size,
868                lazy_init,
869            })) => (
870                unsafe { slice::from_raw_parts_mut(data.as_ptr().cast(), *size) },
871                *lazy_init,
872            ),
873            _ => unreachable!(),
874        }
875    }
876
877    fn gc_refs(&self) -> &[Option<VMGcRef>] {
878        assert_eq!(self.element_type(), TableElementType::GcRef);
879        match self {
880            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
881            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
882                &data.as_non_null().as_ref()[..*size]
883            },
884            _ => unreachable!(),
885        }
886    }
887
888    fn contrefs(&self) -> &[Option<VMContObj>] {
889        assert_eq!(self.element_type(), TableElementType::Cont);
890        match self {
891            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
892                slice::from_raw_parts(elements.as_ptr().cast(), elements.len())
893            },
894            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
895                slice::from_raw_parts(data.as_ptr().cast(), *size)
896            },
897            _ => unreachable!(),
898        }
899    }
900
901    fn contrefs_mut(&mut self) -> &mut [Option<VMContObj>] {
902        assert_eq!(self.element_type(), TableElementType::Cont);
903        match self {
904            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
905                slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len())
906            },
907            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
908                slice::from_raw_parts_mut(data.as_ptr().cast(), *size)
909            },
910            _ => unreachable!(),
911        }
912    }
913
914    /// Get this table's GC references as a slice.
915    ///
916    /// Panics if this is not a table of GC references.
917    pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] {
918        assert_eq!(self.element_type(), TableElementType::GcRef);
919        match self {
920            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
921            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
922                &mut data.as_non_null().as_mut()[..*size]
923            },
924            _ => unreachable!(),
925        }
926    }
927
928    /// Manually resets all table elements to a null bit pattern.
929    ///
930    /// Used in the pooling allocator when `madvise` fails to reset pages back
931    /// to their original contents. In such a situation the previous contents of
932    /// the table are unknown so this resets them all to defined values.
933    pub fn manually_memset_zeros(&mut self) {
934        match self.element_type() {
935            TableElementType::Func => {
936                let (funcrefs, _lazy_init) = self.funcrefs_mut();
937                funcrefs.fill(MaybeTaggedFuncRef(None));
938            }
939            TableElementType::GcRef => {
940                // Note that this explicitly contains no barriers as all table
941                // GC elements should already have been dropped before returning
942                // the table back to the pool.
943                for r in self.gc_refs_mut() {
944                    *r = None;
945                }
946            }
947            TableElementType::Cont => {
948                self.contrefs_mut().fill(None);
949            }
950        }
951    }
952
953    pub fn debug_assert_all_zero(&self) {
954        match self.element_type() {
955            TableElementType::Func => {
956                let (funcrefs, _lazy_init) = self.funcrefs();
957                debug_assert!(funcrefs.iter().all(|f| f.0.is_none()));
958            }
959            TableElementType::GcRef => {
960                debug_assert!(self.gc_refs().iter().all(|r| r.is_none()));
961            }
962            TableElementType::Cont => {
963                debug_assert!(self.contrefs().iter().all(|c| c.is_none()));
964            }
965        }
966    }
967}
968
969// The default table representation is an empty funcref table that cannot grow.
970impl Default for Table {
971    fn default() -> Self {
972        Self::from(StaticFuncTable {
973            data: SendSyncPtr::new(NonNull::from(&mut [])),
974            size: 0,
975            lazy_init: false,
976        })
977    }
978}
wasmtime/runtime/vm/table.rs

wasmtime/runtime/vm/
table.rs
