wasmtime/runtime/vm/
table.rs

Help
1//! Memory management for tables.
2//!
3//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
4
5#![cfg_attr(feature = "gc", allow(irrefutable_let_patterns))]
6
7use crate::prelude::*;
8use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition};
9use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore};
10use core::alloc::Layout;
11use core::mem;
12use core::ops::Range;
13use core::ptr::{self, NonNull};
14use core::slice;
15use core::{cmp, usize};
16use sptr::Strict;
17use wasmtime_environ::{
18    IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType, FUNCREF_INIT_BIT, FUNCREF_MASK,
19};
20
21/// An element going into or coming out of a table.
22///
23/// Table elements are stored as pointers and are default-initialized with
24/// `ptr::null_mut`.
25pub enum TableElement {
26    /// A `funcref`.
27    FuncRef(Option<NonNull<VMFuncRef>>),
28
29    /// A GC reference.
30    GcRef(Option<VMGcRef>),
31
32    /// An uninitialized funcref value. This should never be exposed
33    /// beyond the `wasmtime` crate boundary; the upper-level code
34    /// (which has access to the info needed for lazy initialization)
35    /// will replace it when fetched.
36    UninitFunc,
37}
38
39#[derive(Copy, Clone, PartialEq, Eq, Debug)]
40pub enum TableElementType {
41    Func,
42    GcRef,
43}
44
45impl TableElementType {
46    fn matches(&self, val: &TableElement) -> bool {
47        match (val, self) {
48            (TableElement::FuncRef(_), TableElementType::Func) => true,
49            (TableElement::GcRef(_), TableElementType::GcRef) => true,
50            _ => false,
51        }
52    }
53}
54
55// The usage of `*mut VMFuncRef` is safe w.r.t. thread safety, this just relies
56// on thread-safety of `VMGcRef` itself.
57unsafe impl Send for TableElement where VMGcRef: Send {}
58unsafe impl Sync for TableElement where VMGcRef: Sync {}
59
60impl TableElement {
61    /// Consumes a table element into a pointer/reference, as it
62    /// exists outside the table itself. This strips off any tag bits
63    /// or other information that only lives inside the table.
64    ///
65    /// Can only be done to an initialized table element; lazy init
66    /// must occur first. (In other words, lazy values do not survive
67    /// beyond the table, as every table read path initializes them.)
68    ///
69    /// # Safety
70    ///
71    /// The same warnings as for `into_table_values()` apply.
72    pub(crate) unsafe fn into_func_ref_asserting_initialized(self) -> Option<NonNull<VMFuncRef>> {
73        match self {
74            Self::FuncRef(e) => e,
75            Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"),
76            Self::GcRef(_) => panic!("GC reference is not a function reference"),
77        }
78    }
79
80    /// Indicates whether this value is the "uninitialized element"
81    /// value.
82    pub(crate) fn is_uninit(&self) -> bool {
83        match self {
84            Self::UninitFunc => true,
85            _ => false,
86        }
87    }
88}
89
90impl From<Option<NonNull<VMFuncRef>>> for TableElement {
91    fn from(f: Option<NonNull<VMFuncRef>>) -> TableElement {
92        TableElement::FuncRef(f)
93    }
94}
95
96impl From<Option<VMGcRef>> for TableElement {
97    fn from(x: Option<VMGcRef>) -> TableElement {
98        TableElement::GcRef(x)
99    }
100}
101
102impl From<VMGcRef> for TableElement {
103    fn from(x: VMGcRef) -> TableElement {
104        TableElement::GcRef(Some(x))
105    }
106}
107
108#[derive(Copy, Clone)]
109#[repr(transparent)]
110struct TaggedFuncRef(*mut VMFuncRef);
111
112impl TaggedFuncRef {
113    const UNINIT: TaggedFuncRef = TaggedFuncRef(ptr::null_mut());
114
115    /// Converts the given `ptr`, a valid funcref pointer, into a tagged pointer
116    /// by adding in the `FUNCREF_INIT_BIT`.
117    fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self {
118        let ptr = ptr.map(|p| p.as_ptr()).unwrap_or(ptr::null_mut());
119        if lazy_init {
120            let masked = Strict::map_addr(ptr, |a| a | FUNCREF_INIT_BIT);
121            TaggedFuncRef(masked)
122        } else {
123            TaggedFuncRef(ptr)
124        }
125    }
126
127    /// Converts a tagged pointer into a `TableElement`, returning `UninitFunc`
128    /// for null (not a tagged value) or `FuncRef` for otherwise tagged values.
129    fn into_table_element(self, lazy_init: bool) -> TableElement {
130        let ptr = self.0;
131        if lazy_init && ptr.is_null() {
132            TableElement::UninitFunc
133        } else {
134            // Masking off the tag bit is harmless whether the table uses lazy
135            // init or not.
136            let unmasked = Strict::map_addr(ptr, |a| a & FUNCREF_MASK);
137            TableElement::FuncRef(NonNull::new(unmasked))
138        }
139    }
140}
141
142pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>;
143
144pub enum StaticTable {
145    Func(StaticFuncTable),
146    GcRef(StaticGcRefTable),
147}
148
149impl From<StaticFuncTable> for StaticTable {
150    fn from(value: StaticFuncTable) -> Self {
151        Self::Func(value)
152    }
153}
154
155impl From<StaticGcRefTable> for StaticTable {
156    fn from(value: StaticGcRefTable) -> Self {
157        Self::GcRef(value)
158    }
159}
160
161pub struct StaticFuncTable {
162    /// Where data for this table is stored. The length of this list is the
163    /// maximum size of the table.
164    data: SendSyncPtr<[FuncTableElem]>,
165    /// The current size of the table.
166    size: usize,
167    /// Whether elements of this table are initialized lazily.
168    lazy_init: bool,
169}
170
171pub struct StaticGcRefTable {
172    /// Where data for this table is stored. The length of this list is the
173    /// maximum size of the table.
174    data: SendSyncPtr<[Option<VMGcRef>]>,
175    /// The current size of the table.
176    size: usize,
177}
178
179pub enum DynamicTable {
180    Func(DynamicFuncTable),
181    GcRef(DynamicGcRefTable),
182}
183
184impl From<DynamicFuncTable> for DynamicTable {
185    fn from(value: DynamicFuncTable) -> Self {
186        Self::Func(value)
187    }
188}
189
190impl From<DynamicGcRefTable> for DynamicTable {
191    fn from(value: DynamicGcRefTable) -> Self {
192        Self::GcRef(value)
193    }
194}
195
196pub struct DynamicFuncTable {
197    /// Dynamically managed storage space for this table. The length of this
198    /// vector is the current size of the table.
199    elements: Vec<FuncTableElem>,
200    /// Maximum size that `elements` can grow to.
201    maximum: Option<usize>,
202    /// Whether elements of this table are initialized lazily.
203    lazy_init: bool,
204}
205
206pub struct DynamicGcRefTable {
207    /// Dynamically managed storage space for this table. The length of this
208    /// vector is the current size of the table.
209    elements: Vec<Option<VMGcRef>>,
210    /// Maximum size that `elements` can grow to.
211    maximum: Option<usize>,
212}
213
214/// Represents an instance's table.
215pub enum Table {
216    /// A "static" table where storage space is managed externally, currently
217    /// used with the pooling allocator.
218    Static(StaticTable),
219    /// A "dynamic" table where table storage space is dynamically allocated via
220    /// `malloc` (aka Rust's `Vec`).
221    Dynamic(DynamicTable),
222}
223
224impl From<StaticTable> for Table {
225    fn from(value: StaticTable) -> Self {
226        Self::Static(value)
227    }
228}
229
230impl From<StaticFuncTable> for Table {
231    fn from(value: StaticFuncTable) -> Self {
232        let t: StaticTable = value.into();
233        t.into()
234    }
235}
236
237impl From<StaticGcRefTable> for Table {
238    fn from(value: StaticGcRefTable) -> Self {
239        let t: StaticTable = value.into();
240        t.into()
241    }
242}
243
244impl From<DynamicTable> for Table {
245    fn from(value: DynamicTable) -> Self {
246        Self::Dynamic(value)
247    }
248}
249
250impl From<DynamicFuncTable> for Table {
251    fn from(value: DynamicFuncTable) -> Self {
252        let t: DynamicTable = value.into();
253        t.into()
254    }
255}
256
257impl From<DynamicGcRefTable> for Table {
258    fn from(value: DynamicGcRefTable) -> Self {
259        let t: DynamicTable = value.into();
260        t.into()
261    }
262}
263
264fn wasm_to_table_type(ty: WasmRefType) -> TableElementType {
265    match ty.heap_type.top() {
266        WasmHeapTopType::Func => TableElementType::Func,
267        WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef,
268        WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
269    }
270}
271
272/// Allocate dynamic table elements of the given length.
273///
274/// Relies on the fact that our tables' elements are initialized to `None`,
275/// which is represented by zero, to allocate pre-zeroed memory from the global
276/// allocator and avoid manual zero-initialization.
277///
278/// # Safety
279///
280/// Should only ever be called with a `T` that is a table element type and where
281/// `Option<T>`'s `None` variant is represented with zero.
282unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<Vec<Option<T>>> {
283    debug_assert!(
284        core::mem::MaybeUninit::<Option<T>>::zeroed()
285            .assume_init()
286            .is_none(),
287        "null table elements are represented with zeroed memory"
288    );
289
290    if len == 0 {
291        return Ok(vec![]);
292    }
293
294    let align = mem::align_of::<Option<T>>();
295
296    let size = mem::size_of::<Option<T>>();
297    let size = size.next_multiple_of(align);
298    let size = size.checked_mul(len).unwrap();
299
300    let layout = Layout::from_size_align(size, align)?;
301
302    let ptr = alloc::alloc::alloc_zeroed(layout);
303    ensure!(!ptr.is_null(), "failed to allocate memory for table");
304
305    let elems = Vec::<Option<T>>::from_raw_parts(ptr.cast(), len, len);
306    debug_assert!(elems.iter().all(|e| e.is_none()));
307
308    Ok(elems)
309}
310
311impl Table {
312    /// Create a new dynamic (movable) table instance for the specified table plan.
313    pub fn new_dynamic(
314        ty: &wasmtime_environ::Table,
315        tunables: &Tunables,
316        store: &mut dyn VMStore,
317    ) -> Result<Self> {
318        let (minimum, maximum) = Self::limit_new(ty, store)?;
319        match wasm_to_table_type(ty.ref_type) {
320            TableElementType::Func => Ok(Self::from(DynamicFuncTable {
321                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
322                maximum,
323                lazy_init: tunables.table_lazy_init,
324            })),
325            TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable {
326                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
327                maximum,
328            })),
329        }
330    }
331
332    /// Create a new static (immovable) table instance for the specified table plan.
333    pub unsafe fn new_static(
334        ty: &wasmtime_environ::Table,
335        tunables: &Tunables,
336        data: SendSyncPtr<[u8]>,
337        store: &mut dyn VMStore,
338    ) -> Result<Self> {
339        let (minimum, maximum) = Self::limit_new(ty, store)?;
340        let size = minimum;
341        let max = maximum.unwrap_or(usize::MAX);
342
343        match wasm_to_table_type(ty.ref_type) {
344            TableElementType::Func => {
345                let len = {
346                    let data = data.as_non_null().as_ref();
347                    let (before, data, after) = data.align_to::<FuncTableElem>();
348                    assert!(before.is_empty());
349                    assert!(after.is_empty());
350                    data.len()
351                };
352                ensure!(
353                    usize::try_from(ty.limits.min).unwrap() <= len,
354                    "initial table size of {} exceeds the pooling allocator's \
355                     configured maximum table size of {len} elements",
356                    ty.limits.min,
357                );
358                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
359                    data.as_non_null().cast::<FuncTableElem>(),
360                    cmp::min(len, max),
361                ));
362                Ok(Self::from(StaticFuncTable {
363                    data,
364                    size,
365                    lazy_init: tunables.table_lazy_init,
366                }))
367            }
368            TableElementType::GcRef => {
369                let len = {
370                    let data = data.as_non_null().as_ref();
371                    let (before, data, after) = data.align_to::<Option<VMGcRef>>();
372                    assert!(before.is_empty());
373                    assert!(after.is_empty());
374                    data.len()
375                };
376                ensure!(
377                    usize::try_from(ty.limits.min).unwrap() <= len,
378                    "initial table size of {} exceeds the pooling allocator's \
379                     configured maximum table size of {len} elements",
380                    ty.limits.min,
381                );
382                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
383                    data.as_non_null().cast::<Option<VMGcRef>>(),
384                    cmp::min(len, max),
385                ));
386                Ok(Self::from(StaticGcRefTable { data, size }))
387            }
388        }
389    }
390
391    // Calls the `store`'s limiter to optionally prevent the table from being created.
392    //
393    // Returns the minimum and maximum size of the table if the table can be created.
394    fn limit_new(
395        ty: &wasmtime_environ::Table,
396        store: &mut dyn VMStore,
397    ) -> Result<(usize, Option<usize>)> {
398        // No matter how the table limits are specified
399        // The table size is limited by the host's pointer size
400        let absolute_max = usize::MAX;
401
402        // If the minimum overflows the host's pointer size, then we can't satisfy this request.
403        // We defer the error to later so the `store` can be informed.
404        let minimum = usize::try_from(ty.limits.min).ok();
405
406        // The maximum size of the table is limited by:
407        // * the host's pointer size.
408        // * the table's maximum size if defined.
409        // * if the table is 64-bit.
410        let maximum = match (ty.limits.max, ty.idx_type) {
411            (Some(max), _) => usize::try_from(max).ok(),
412            (None, IndexType::I64) => usize::try_from(u64::MAX).ok(),
413            (None, IndexType::I32) => usize::try_from(u32::MAX).ok(),
414        };
415
416        // Inform the store's limiter what's about to happen.
417        if !store.table_growing(0, minimum.unwrap_or(absolute_max), maximum)? {
418            bail!(
419                "table minimum size of {} elements exceeds table limits",
420                ty.limits.min
421            );
422        }
423
424        // At this point we need to actually handle overflows, so bail out with
425        // an error if we made it this far.
426        let minimum = minimum.ok_or_else(|| {
427            format_err!(
428                "table minimum size of {} elements exceeds table limits",
429                ty.limits.min
430            )
431        })?;
432        Ok((minimum, maximum))
433    }
434
435    /// Returns the type of the elements in this table.
436    pub fn element_type(&self) -> TableElementType {
437        match self {
438            Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => {
439                TableElementType::Func
440            }
441            Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => {
442                TableElementType::GcRef
443            }
444        }
445    }
446
447    /// Returns whether or not the underlying storage of the table is "static".
448    #[cfg(feature = "pooling-allocator")]
449    pub(crate) fn is_static(&self) -> bool {
450        matches!(self, Table::Static(_))
451    }
452
453    /// Returns the number of allocated elements.
454    pub fn size(&self) -> usize {
455        match self {
456            Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size,
457            Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size,
458            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(),
459            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
460                elements.len()
461            }
462        }
463    }
464
465    /// Returns the maximum number of elements at runtime.
466    ///
467    /// Returns `None` if the table is unbounded.
468    ///
469    /// The runtime maximum may not be equal to the maximum from the table's Wasm type
470    /// when it is being constrained by an instance allocator.
471    pub fn maximum(&self) -> Option<usize> {
472        match self {
473            Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()),
474            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()),
475            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum,
476            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum,
477        }
478    }
479
480    /// Initializes the contents of this table to the specified function
481    ///
482    /// # Panics
483    ///
484    /// Panics if the table is not a function table.
485    pub fn init_func(
486        &mut self,
487        dst: u64,
488        items: impl ExactSizeIterator<Item = Option<NonNull<VMFuncRef>>>,
489    ) -> Result<(), Trap> {
490        let dst = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
491
492        let (funcrefs, lazy_init) = self.funcrefs_mut();
493        let elements = funcrefs
494            .get_mut(dst..)
495            .and_then(|s| s.get_mut(..items.len()))
496            .ok_or(Trap::TableOutOfBounds)?;
497
498        for (item, slot) in items.zip(elements) {
499            *slot = TaggedFuncRef::from(item, lazy_init);
500        }
501        Ok(())
502    }
503
504    /// Fill `table[dst..]` with values from `items`
505    ///
506    /// Returns a trap error on out-of-bounds accesses.
507    pub fn init_gc_refs(
508        &mut self,
509        dst: u64,
510        items: impl ExactSizeIterator<Item = Option<VMGcRef>>,
511    ) -> Result<(), Trap> {
512        let dst = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
513
514        let elements = self
515            .gc_refs_mut()
516            .get_mut(dst..)
517            .and_then(|s| s.get_mut(..items.len()))
518            .ok_or(Trap::TableOutOfBounds)?;
519
520        for (item, slot) in items.zip(elements) {
521            *slot = item;
522        }
523        Ok(())
524    }
525
526    /// Fill `table[dst..dst + len]` with `val`.
527    ///
528    /// Returns a trap error on out-of-bounds accesses.
529    ///
530    /// # Panics
531    ///
532    /// Panics if `val` does not have a type that matches this table, or if
533    /// `gc_store.is_none()` and this is a table of GC references.
534    pub fn fill(
535        &mut self,
536        gc_store: Option<&mut GcStore>,
537        dst: u64,
538        val: TableElement,
539        len: u64,
540    ) -> Result<(), Trap> {
541        let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
542        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
543        let end = start
544            .checked_add(len)
545            .ok_or_else(|| Trap::TableOutOfBounds)?;
546
547        if end > self.size() {
548            return Err(Trap::TableOutOfBounds);
549        }
550
551        match val {
552            TableElement::FuncRef(f) => {
553                let (funcrefs, lazy_init) = self.funcrefs_mut();
554                funcrefs[start..end].fill(TaggedFuncRef::from(f, lazy_init));
555            }
556            TableElement::GcRef(r) => {
557                let gc_store =
558                    gc_store.expect("must provide a GcStore for tables of GC references");
559
560                // Clone the init GC reference into each table slot.
561                for slot in &mut self.gc_refs_mut()[start..end] {
562                    gc_store.write_gc_ref(slot, r.as_ref());
563                }
564
565                // Drop the init GC reference, since we aren't holding onto this
566                // reference anymore, only the clones in the table.
567                if let Some(r) = r {
568                    gc_store.drop_gc_ref(r);
569                }
570            }
571            TableElement::UninitFunc => {
572                let (funcrefs, _lazy_init) = self.funcrefs_mut();
573                funcrefs[start..end].fill(TaggedFuncRef::UNINIT);
574            }
575        }
576
577        Ok(())
578    }
579
580    /// Grow table by the specified amount of elements.
581    ///
582    /// Returns the previous size of the table if growth is successful.
583    ///
584    /// Returns `None` if table can't be grown by the specified amount of
585    /// elements, or if the `init_value` is the wrong kind of table element.
586    ///
587    /// # Panics
588    ///
589    /// Panics if `init_value` does not have a type that matches this table.
590    ///
591    /// # Unsafety
592    ///
593    /// Resizing the table can reallocate its internal elements buffer. This
594    /// table's instance's `VMContext` has raw pointers to the elements buffer
595    /// that are used by Wasm, and they need to be fixed up before we call into
596    /// Wasm again. Failure to do so will result in use-after-free inside Wasm.
597    ///
598    /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates
599    /// this unsafety.
600    pub unsafe fn grow(
601        &mut self,
602        delta: u64,
603        init_value: TableElement,
604        store: &mut dyn VMStore,
605    ) -> Result<Option<usize>, Error> {
606        let old_size = self.size();
607
608        // Don't try to resize the table if its size isn't changing, just return
609        // success.
610        if delta == 0 {
611            return Ok(Some(old_size));
612        }
613        // Cannot return `Trap::TableOutOfBounds` here because `impl std::error::Error for Trap` is not available in no-std.
614        let delta =
615            usize::try_from(delta).map_err(|_| format_err!("delta exceeds host pointer size"))?;
616
617        let new_size = match old_size.checked_add(delta) {
618            Some(s) => s,
619            None => {
620                store.table_grow_failed(format_err!("overflow calculating new table size"))?;
621                return Ok(None);
622            }
623        };
624
625        if !store.table_growing(old_size, new_size, self.maximum())? {
626            return Ok(None);
627        }
628
629        // The WebAssembly spec requires failing a `table.grow` request if
630        // it exceeds the declared limits of the table. We may have set lower
631        // limits in the instance allocator as well.
632        if let Some(max) = self.maximum() {
633            if new_size > max {
634                store.table_grow_failed(format_err!("Table maximum size exceeded"))?;
635                return Ok(None);
636            }
637        }
638
639        debug_assert!(self.type_matches(&init_value));
640
641        // First resize the storage and then fill with the init value
642        match self {
643            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
644                unsafe {
645                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
646                }
647                *size = new_size;
648            }
649            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
650                unsafe {
651                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
652                }
653                *size = new_size;
654            }
655
656            // These calls to `resize` could move the base address of
657            // `elements`. If this table's limits declare it to be fixed-size,
658            // then during AOT compilation we may have promised Cranelift that
659            // the table base address won't change, so it is allowed to optimize
660            // loading the base address. However, in that case the above checks
661            // that delta is non-zero and the new size doesn't exceed the
662            // maximum mean we can't get here.
663            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
664                elements.resize(usize::try_from(new_size).unwrap(), None);
665            }
666            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
667                elements.resize_with(usize::try_from(new_size).unwrap(), || None);
668            }
669        }
670
671        self.fill(
672            store.store_opaque_mut().optional_gc_store_mut()?,
673            u64::try_from(old_size).unwrap(),
674            init_value,
675            u64::try_from(delta).unwrap(),
676        )
677        .expect("table should not be out of bounds");
678
679        Ok(Some(old_size))
680    }
681
682    /// Get reference to the specified element.
683    ///
684    /// Returns `None` if the index is out of bounds.
685    ///
686    /// Panics if this is a table of GC references and `gc_store` is `None`.
687    pub fn get(&self, gc_store: Option<&mut GcStore>, index: u64) -> Option<TableElement> {
688        let index = usize::try_from(index).ok()?;
689        match self.element_type() {
690            TableElementType::Func => {
691                let (funcrefs, lazy_init) = self.funcrefs();
692                funcrefs
693                    .get(index)
694                    .copied()
695                    .map(|e| e.into_table_element(lazy_init))
696            }
697            TableElementType::GcRef => self.gc_refs().get(index).map(|r| {
698                let r = r.as_ref().map(|r| gc_store.unwrap().clone_gc_ref(r));
699                TableElement::GcRef(r)
700            }),
701        }
702    }
703
704    /// Set reference to the specified element.
705    ///
706    /// # Errors
707    ///
708    /// Returns an error if `index` is out of bounds or if this table type does
709    /// not match the element type.
710    ///
711    /// # Panics
712    ///
713    /// Panics if `elem` is not of the right type for this table.
714    pub fn set(&mut self, index: u64, elem: TableElement) -> Result<(), ()> {
715        let index: usize = index.try_into().map_err(|_| ())?;
716        match elem {
717            TableElement::FuncRef(f) => {
718                let (funcrefs, lazy_init) = self.funcrefs_mut();
719                *funcrefs.get_mut(index).ok_or(())? = TaggedFuncRef::from(f, lazy_init);
720            }
721            TableElement::UninitFunc => {
722                let (funcrefs, _lazy_init) = self.funcrefs_mut();
723                *funcrefs.get_mut(index).ok_or(())? = TaggedFuncRef::UNINIT;
724            }
725            TableElement::GcRef(e) => {
726                *self.gc_refs_mut().get_mut(index).ok_or(())? = e;
727            }
728        }
729        Ok(())
730    }
731
732    /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
733    ///
734    /// # Errors
735    ///
736    /// Returns an error if the range is out of bounds of either the source or
737    /// destination tables.
738    pub unsafe fn copy(
739        gc_store: Option<&mut GcStore>,
740        dst_table: *mut Self,
741        src_table: *mut Self,
742        dst_index: u64,
743        src_index: u64,
744        len: u64,
745    ) -> Result<(), Trap> {
746        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy
747
748        let src_index = usize::try_from(src_index).map_err(|_| Trap::TableOutOfBounds)?;
749        let dst_index = usize::try_from(dst_index).map_err(|_| Trap::TableOutOfBounds)?;
750        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
751
752        if src_index
753            .checked_add(len)
754            .map_or(true, |n| n > (*src_table).size())
755            || dst_index
756                .checked_add(len)
757                .map_or(true, |m| m > (*dst_table).size())
758        {
759            return Err(Trap::TableOutOfBounds);
760        }
761
762        debug_assert!(
763            (*dst_table).element_type() == (*src_table).element_type(),
764            "table element type mismatch"
765        );
766
767        let src_range = src_index..src_index + len;
768        let dst_range = dst_index..dst_index + len;
769
770        // Check if the tables are the same as we cannot mutably borrow and also borrow the same `RefCell`
771        if ptr::eq(dst_table, src_table) {
772            (*dst_table).copy_elements_within(gc_store, dst_range, src_range);
773        } else {
774            Self::copy_elements(gc_store, &mut *dst_table, &*src_table, dst_range, src_range);
775        }
776
777        Ok(())
778    }
779
780    /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
781    pub fn vmtable(&mut self) -> VMTableDefinition {
782        match self {
783            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
784                VMTableDefinition {
785                    base: data.cast().into(),
786                    current_elements: *size,
787                }
788            }
789            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
790                VMTableDefinition {
791                    base: data.cast().into(),
792                    current_elements: *size,
793                }
794            }
795            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
796                VMTableDefinition {
797                    base: NonNull::<[FuncTableElem]>::from(&mut elements[..])
798                        .cast()
799                        .into(),
800                    current_elements: elements.len(),
801                }
802            }
803            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
804                VMTableDefinition {
805                    base: NonNull::<[Option<VMGcRef>]>::from(&mut elements[..])
806                        .cast()
807                        .into(),
808                    current_elements: elements.len(),
809                }
810            }
811        }
812    }
813
814    fn type_matches(&self, val: &TableElement) -> bool {
815        self.element_type().matches(val)
816    }
817
818    fn funcrefs(&self) -> (&[TaggedFuncRef], bool) {
819        assert_eq!(self.element_type(), TableElementType::Func);
820        match self {
821            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
822                elements,
823                lazy_init,
824                ..
825            })) => (
826                unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) },
827                *lazy_init,
828            ),
829            Self::Static(StaticTable::Func(StaticFuncTable {
830                data,
831                size,
832                lazy_init,
833            })) => (
834                unsafe {
835                    slice::from_raw_parts(data.as_ptr().cast(), usize::try_from(*size).unwrap())
836                },
837                *lazy_init,
838            ),
839            _ => unreachable!(),
840        }
841    }
842
843    fn funcrefs_mut(&mut self) -> (&mut [TaggedFuncRef], bool) {
844        assert_eq!(self.element_type(), TableElementType::Func);
845        match self {
846            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
847                elements,
848                lazy_init,
849                ..
850            })) => (
851                unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) },
852                *lazy_init,
853            ),
854            Self::Static(StaticTable::Func(StaticFuncTable {
855                data,
856                size,
857                lazy_init,
858            })) => (
859                unsafe {
860                    slice::from_raw_parts_mut(data.as_ptr().cast(), usize::try_from(*size).unwrap())
861                },
862                *lazy_init,
863            ),
864            _ => unreachable!(),
865        }
866    }
867
868    fn gc_refs(&self) -> &[Option<VMGcRef>] {
869        assert_eq!(self.element_type(), TableElementType::GcRef);
870        match self {
871            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
872            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
873                &data.as_non_null().as_ref()[..usize::try_from(*size).unwrap()]
874            },
875            _ => unreachable!(),
876        }
877    }
878
879    /// Get this table's GC references as a slice.
880    ///
881    /// Panics if this is not a table of GC references.
882    pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] {
883        assert_eq!(self.element_type(), TableElementType::GcRef);
884        match self {
885            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
886            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
887                &mut data.as_non_null().as_mut()[..usize::try_from(*size).unwrap()]
888            },
889            _ => unreachable!(),
890        }
891    }
892
893    fn copy_elements(
894        gc_store: Option<&mut GcStore>,
895        dst_table: &mut Self,
896        src_table: &Self,
897        dst_range: Range<usize>,
898        src_range: Range<usize>,
899    ) {
900        // This can only be used when copying between different tables
901        debug_assert!(!ptr::eq(dst_table, src_table));
902
903        let ty = dst_table.element_type();
904
905        match ty {
906            TableElementType::Func => {
907                // `funcref` are `Copy`, so just do a mempcy
908                let (dst_funcrefs, _lazy_init) = dst_table.funcrefs_mut();
909                let (src_funcrefs, _lazy_init) = src_table.funcrefs();
910                dst_funcrefs[dst_range].copy_from_slice(&src_funcrefs[src_range]);
911            }
912            TableElementType::GcRef => {
913                assert_eq!(
914                    dst_range.end - dst_range.start,
915                    src_range.end - src_range.start
916                );
917                assert!(dst_range.end <= dst_table.gc_refs().len());
918                assert!(src_range.end <= src_table.gc_refs().len());
919                let gc_store = gc_store.unwrap();
920                for (dst, src) in dst_range.zip(src_range) {
921                    gc_store.write_gc_ref(
922                        &mut dst_table.gc_refs_mut()[dst],
923                        src_table.gc_refs()[src].as_ref(),
924                    );
925                }
926            }
927        }
928    }
929
930    fn copy_elements_within(
931        &mut self,
932        gc_store: Option<&mut GcStore>,
933        dst_range: Range<usize>,
934        src_range: Range<usize>,
935    ) {
936        assert_eq!(
937            dst_range.end - dst_range.start,
938            src_range.end - src_range.start
939        );
940
941        // This is a no-op.
942        if src_range.start == dst_range.start {
943            return;
944        }
945
946        let ty = self.element_type();
947        match ty {
948            TableElementType::Func => {
949                // `funcref` are `Copy`, so just do a memmove
950                let (funcrefs, _lazy_init) = self.funcrefs_mut();
951                funcrefs.copy_within(src_range, dst_range.start);
952            }
953            TableElementType::GcRef => {
954                let gc_store = gc_store.unwrap();
955
956                // We need to clone each `externref` while handling overlapping
957                // ranges
958                let elements = self.gc_refs_mut();
959                if dst_range.start < src_range.start {
960                    for (d, s) in dst_range.zip(src_range) {
961                        let (ds, ss) = elements.split_at_mut(s);
962                        let dst = &mut ds[d];
963                        let src = ss[0].as_ref();
964                        gc_store.write_gc_ref(dst, src);
965                    }
966                } else {
967                    for (s, d) in src_range.rev().zip(dst_range.rev()) {
968                        let (ss, ds) = elements.split_at_mut(d);
969                        let dst = &mut ds[0];
970                        let src = ss[s].as_ref();
971                        gc_store.write_gc_ref(dst, src);
972                    }
973                }
974            }
975        }
976    }
977}
978
979// The default table representation is an empty funcref table that cannot grow.
980impl Default for Table {
981    fn default() -> Self {
982        Self::from(StaticFuncTable {
983            data: SendSyncPtr::new(NonNull::from(&mut [])),
984            size: 0,
985            lazy_init: false,
986        })
987    }
988}
wasmtime/runtime/vm/table.rs

wasmtime/runtime/vm/
table.rs
