wasmtime/runtime/vm/
table.rs

1//! Memory management for tables.
2//!
3//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
4
5use crate::prelude::*;
6use crate::runtime::store::StoreResourceLimiter;
7use crate::runtime::vm::stack_switching::VMContObj;
8use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition};
9use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VmPtr};
10use core::alloc::Layout;
11use core::mem;
12use core::ops::Range;
13use core::ptr::{self, NonNull};
14use core::slice;
15use core::{cmp, usize};
16use wasmtime_environ::{
17    FUNCREF_INIT_BIT, FUNCREF_MASK, IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType,
18};
19
20#[derive(Copy, Clone, PartialEq, Eq, Debug)]
21pub enum TableElementType {
22    Func,
23    GcRef,
24    Cont,
25}
26
27impl TableElementType {
28    /// Returns the size required to actually store an element of this particular type
29    pub fn element_size(&self) -> usize {
30        match self {
31            TableElementType::Func => core::mem::size_of::<FuncTableElem>(),
32            TableElementType::GcRef => core::mem::size_of::<Option<VMGcRef>>(),
33            TableElementType::Cont => core::mem::size_of::<ContTableElem>(),
34        }
35    }
36}
37
38/// At-rest representation of a function in a funcref table.
39///
40/// Note that whether or not these pointers are tagged is a property of `Engine`
41/// configuration. Also note that this specifically uses `VmPtr<T>` to handle
42/// provenance here when loading/storing values to a table.
43///
44/// The possible values here are:
45///
46/// * `None` for untagged tables - a null function element
47/// * `Some(_)` for untagged tables - a non-null function element
48/// * `None` for tagged tables - an uninitialized element
49/// * `Some(1)` for tagged tables - a null function element
50/// * `Some(addr | 1)` for tagged tables - a non-null function element
51#[derive(Copy, Clone)]
52#[repr(transparent)]
53struct MaybeTaggedFuncRef(Option<VmPtr<VMFuncRef>>);
54
55impl MaybeTaggedFuncRef {
56    /// Converts the given `ptr`, a valid funcref pointer, into a tagged pointer
57    /// by adding in the `FUNCREF_INIT_BIT`.
58    fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self {
59        let maybe_tagged = if lazy_init {
60            Some(match ptr {
61                Some(ptr) => ptr.map_addr(|a| a | FUNCREF_INIT_BIT),
62                None => NonNull::new(core::ptr::without_provenance_mut(FUNCREF_INIT_BIT)).unwrap(),
63            })
64        } else {
65            ptr
66        };
67        MaybeTaggedFuncRef(maybe_tagged.map(Into::into))
68    }
69
70    /// Converts a tagged pointer into a `TableElement`, returning `UninitFunc`
71    /// for null (not a tagged value) or `FuncRef` for otherwise tagged values.
72    fn into_funcref(self, lazy_init: bool) -> Option<Option<NonNull<VMFuncRef>>> {
73        let ptr = self.0;
74        if lazy_init && ptr.is_none() {
75            None
76        } else {
77            // Masking off the tag bit is harmless whether the table uses lazy
78            // init or not.
79            Some(ptr.and_then(|ptr| NonNull::new(ptr.as_ptr().map_addr(|a| a & FUNCREF_MASK))))
80        }
81    }
82}
83
84pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>;
85pub type ContTableElem = Option<VMContObj>;
86
87/// The maximum of the sizes of any of the table element types
88#[cfg(feature = "pooling-allocator")]
89pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = {
90    // ContTableElem intentionally excluded for "nominal" calculation.
91    let sizes = [
92        core::mem::size_of::<FuncTableElem>(),
93        core::mem::size_of::<Option<VMGcRef>>(),
94    ];
95
96    // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`,
97    // but as a `const` function, so we can use it to define a constant.
98    const fn slice_max(data: &[usize]) -> usize {
99        match data {
100            [] => 0,
101            [head, tail @ ..] => {
102                let tail_max = slice_max(tail);
103                if *head >= tail_max { *head } else { tail_max }
104            }
105        }
106    }
107
108    slice_max(&sizes)
109};
110
111pub enum StaticTable {
112    Func(StaticFuncTable),
113    GcRef(StaticGcRefTable),
114    Cont(StaticContTable),
115}
116
117impl From<StaticFuncTable> for StaticTable {
118    fn from(value: StaticFuncTable) -> Self {
119        Self::Func(value)
120    }
121}
122
123impl From<StaticGcRefTable> for StaticTable {
124    fn from(value: StaticGcRefTable) -> Self {
125        Self::GcRef(value)
126    }
127}
128
129impl From<StaticContTable> for StaticTable {
130    fn from(value: StaticContTable) -> Self {
131        Self::Cont(value)
132    }
133}
134
135pub struct StaticFuncTable {
136    /// Where data for this table is stored. The length of this list is the
137    /// maximum size of the table.
138    data: SendSyncPtr<[FuncTableElem]>,
139    /// The current size of the table.
140    size: usize,
141    /// Whether elements of this table are initialized lazily.
142    lazy_init: bool,
143}
144
145pub struct StaticGcRefTable {
146    /// Where data for this table is stored. The length of this list is the
147    /// maximum size of the table.
148    data: SendSyncPtr<[Option<VMGcRef>]>,
149    /// The current size of the table.
150    size: usize,
151}
152
153pub struct StaticContTable {
154    /// Where data for this table is stored. The length of this list is the
155    /// maximum size of the table.
156    data: SendSyncPtr<[ContTableElem]>,
157    /// The current size of the table.
158    size: usize,
159}
160
161pub enum DynamicTable {
162    Func(DynamicFuncTable),
163    GcRef(DynamicGcRefTable),
164    Cont(DynamicContTable),
165}
166
167impl From<DynamicFuncTable> for DynamicTable {
168    fn from(value: DynamicFuncTable) -> Self {
169        Self::Func(value)
170    }
171}
172
173impl From<DynamicGcRefTable> for DynamicTable {
174    fn from(value: DynamicGcRefTable) -> Self {
175        Self::GcRef(value)
176    }
177}
178
179impl From<DynamicContTable> for DynamicTable {
180    fn from(value: DynamicContTable) -> Self {
181        Self::Cont(value)
182    }
183}
184
185pub struct DynamicFuncTable {
186    /// Dynamically managed storage space for this table. The length of this
187    /// vector is the current size of the table.
188    elements: Vec<FuncTableElem>,
189    /// Maximum size that `elements` can grow to.
190    maximum: Option<usize>,
191    /// Whether elements of this table are initialized lazily.
192    lazy_init: bool,
193}
194
195pub struct DynamicGcRefTable {
196    /// Dynamically managed storage space for this table. The length of this
197    /// vector is the current size of the table.
198    elements: Vec<Option<VMGcRef>>,
199    /// Maximum size that `elements` can grow to.
200    maximum: Option<usize>,
201}
202
203pub struct DynamicContTable {
204    /// Dynamically managed storage space for this table. The length of this
205    /// vector is the current size of the table.
206    elements: Vec<ContTableElem>,
207    /// Maximum size that `elements` can grow to.
208    maximum: Option<usize>,
209}
210
211/// Represents an instance's table.
212pub enum Table {
213    /// A "static" table where storage space is managed externally, currently
214    /// used with the pooling allocator.
215    Static(StaticTable),
216    /// A "dynamic" table where table storage space is dynamically allocated via
217    /// `malloc` (aka Rust's `Vec`).
218    Dynamic(DynamicTable),
219}
220
221impl From<StaticTable> for Table {
222    fn from(value: StaticTable) -> Self {
223        Self::Static(value)
224    }
225}
226
227impl From<StaticFuncTable> for Table {
228    fn from(value: StaticFuncTable) -> Self {
229        let t: StaticTable = value.into();
230        t.into()
231    }
232}
233
234impl From<StaticGcRefTable> for Table {
235    fn from(value: StaticGcRefTable) -> Self {
236        let t: StaticTable = value.into();
237        t.into()
238    }
239}
240
241impl From<StaticContTable> for Table {
242    fn from(value: StaticContTable) -> Self {
243        let t: StaticTable = value.into();
244        t.into()
245    }
246}
247
248impl From<DynamicTable> for Table {
249    fn from(value: DynamicTable) -> Self {
250        Self::Dynamic(value)
251    }
252}
253
254impl From<DynamicFuncTable> for Table {
255    fn from(value: DynamicFuncTable) -> Self {
256        let t: DynamicTable = value.into();
257        t.into()
258    }
259}
260
261impl From<DynamicGcRefTable> for Table {
262    fn from(value: DynamicGcRefTable) -> Self {
263        let t: DynamicTable = value.into();
264        t.into()
265    }
266}
267
268impl From<DynamicContTable> for Table {
269    fn from(value: DynamicContTable) -> Self {
270        let t: DynamicTable = value.into();
271        t.into()
272    }
273}
274
275pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType {
276    match ty.heap_type.top() {
277        WasmHeapTopType::Func => TableElementType::Func,
278        WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef,
279        WasmHeapTopType::Cont => TableElementType::Cont,
280        WasmHeapTopType::Exn => TableElementType::GcRef,
281    }
282}
283
284/// Allocate dynamic table elements of the given length.
285///
286/// Relies on the fact that our tables' elements are initialized to `None`,
287/// which is represented by zero, to allocate pre-zeroed memory from the global
288/// allocator and avoid manual zero-initialization.
289///
290/// # Safety
291///
292/// Should only ever be called with a `T` that is a table element type and where
293/// `Option<T>`'s `None` variant is represented with zero.
294unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<Vec<Option<T>>> {
295    debug_assert!(
296        unsafe {
297            core::mem::MaybeUninit::<Option<T>>::zeroed()
298                .assume_init()
299                .is_none()
300        },
301        "null table elements are represented with zeroed memory"
302    );
303
304    if len == 0 {
305        return Ok(vec![]);
306    }
307
308    let align = mem::align_of::<Option<T>>();
309
310    let size = mem::size_of::<Option<T>>();
311    let size = size.next_multiple_of(align);
312    let size = size.checked_mul(len).unwrap();
313
314    let layout = Layout::from_size_align(size, align)?;
315
316    let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) };
317    ensure!(!ptr.is_null(), "failed to allocate memory for table");
318
319    let elems = unsafe { Vec::<Option<T>>::from_raw_parts(ptr.cast(), len, len) };
320    debug_assert!(elems.iter().all(|e| e.is_none()));
321
322    Ok(elems)
323}
324
325impl Table {
326    /// Create a new dynamic (movable) table instance for the specified table plan.
327    pub async fn new_dynamic(
328        ty: &wasmtime_environ::Table,
329        tunables: &Tunables,
330        limiter: Option<&mut StoreResourceLimiter<'_>>,
331    ) -> Result<Self> {
332        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
333        match wasm_to_table_type(ty.ref_type) {
334            TableElementType::Func => Ok(Self::from(DynamicFuncTable {
335                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
336                maximum,
337                lazy_init: tunables.table_lazy_init,
338            })),
339            TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable {
340                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
341                maximum,
342            })),
343            TableElementType::Cont => Ok(Self::from(DynamicContTable {
344                elements: vec![None; minimum],
345                maximum,
346            })),
347        }
348    }
349
350    /// Create a new static (immovable) table instance for the specified table plan.
351    pub async unsafe fn new_static(
352        ty: &wasmtime_environ::Table,
353        tunables: &Tunables,
354        data: SendSyncPtr<[u8]>,
355        limiter: Option<&mut StoreResourceLimiter<'_>>,
356    ) -> Result<Self> {
357        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
358        let size = minimum;
359        let max = maximum.unwrap_or(usize::MAX);
360
361        match wasm_to_table_type(ty.ref_type) {
362            TableElementType::Func => {
363                let len = {
364                    let (before, data, after) = unsafe {
365                        let data = data.as_non_null().as_ref();
366                        data.align_to::<FuncTableElem>()
367                    };
368                    assert!(before.is_empty());
369                    assert!(after.is_empty());
370                    data.len()
371                };
372                ensure!(
373                    usize::try_from(ty.limits.min).unwrap() <= len,
374                    "initial table size of {} exceeds the pooling allocator's \
375                     configured maximum table size of {len} elements",
376                    ty.limits.min,
377                );
378                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
379                    data.as_non_null().cast::<FuncTableElem>(),
380                    cmp::min(len, max),
381                ));
382                Ok(Self::from(StaticFuncTable {
383                    data,
384                    size,
385                    lazy_init: tunables.table_lazy_init,
386                }))
387            }
388            TableElementType::GcRef => {
389                let len = {
390                    let (before, data, after) = unsafe {
391                        let data = data.as_non_null().as_ref();
392                        data.align_to::<Option<VMGcRef>>()
393                    };
394                    assert!(before.is_empty());
395                    assert!(after.is_empty());
396                    data.len()
397                };
398                ensure!(
399                    usize::try_from(ty.limits.min).unwrap() <= len,
400                    "initial table size of {} exceeds the pooling allocator's \
401                     configured maximum table size of {len} elements",
402                    ty.limits.min,
403                );
404                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
405                    data.as_non_null().cast::<Option<VMGcRef>>(),
406                    cmp::min(len, max),
407                ));
408                Ok(Self::from(StaticGcRefTable { data, size }))
409            }
410            TableElementType::Cont => {
411                let len = {
412                    let (before, data, after) = unsafe {
413                        let data = data.as_non_null().as_ref();
414                        data.align_to::<ContTableElem>()
415                    };
416                    assert!(before.is_empty());
417                    assert!(after.is_empty());
418                    data.len()
419                };
420                ensure!(
421                    usize::try_from(ty.limits.min).unwrap() <= len,
422                    "initial table size of {} exceeds the pooling allocator's \
423                     configured maximum table size of {len} elements",
424                    ty.limits.min,
425                );
426                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
427                    data.as_non_null().cast::<ContTableElem>(),
428                    cmp::min(len, max),
429                ));
430                Ok(Self::from(StaticContTable { data, size }))
431            }
432        }
433    }
434
435    // Calls the `store`'s limiter to optionally prevent the table from being created.
436    //
437    // Returns the minimum and maximum size of the table if the table can be created.
438    async fn limit_new(
439        ty: &wasmtime_environ::Table,
440        limiter: Option<&mut StoreResourceLimiter<'_>>,
441    ) -> Result<(usize, Option<usize>)> {
442        // No matter how the table limits are specified
443        // The table size is limited by the host's pointer size
444        let absolute_max = usize::MAX;
445
446        // If the minimum overflows the host's pointer size, then we can't satisfy this request.
447        // We defer the error to later so the `store` can be informed.
448        let minimum = usize::try_from(ty.limits.min).ok();
449
450        // The maximum size of the table is limited by:
451        // * the host's pointer size.
452        // * the table's maximum size if defined.
453        // * if the table is 64-bit.
454        let maximum = match (ty.limits.max, ty.idx_type) {
455            (Some(max), _) => usize::try_from(max).ok(),
456            (None, IndexType::I64) => usize::try_from(u64::MAX).ok(),
457            (None, IndexType::I32) => usize::try_from(u32::MAX).ok(),
458        };
459
460        // Inform the store's limiter what's about to happen.
461        if let Some(limiter) = limiter {
462            if !limiter
463                .table_growing(0, minimum.unwrap_or(absolute_max), maximum)
464                .await?
465            {
466                bail!(
467                    "table minimum size of {} elements exceeds table limits",
468                    ty.limits.min
469                );
470            }
471        }
472
473        // At this point we need to actually handle overflows, so bail out with
474        // an error if we made it this far.
475        let minimum = minimum.ok_or_else(|| {
476            format_err!(
477                "table minimum size of {} elements exceeds table limits",
478                ty.limits.min
479            )
480        })?;
481        Ok((minimum, maximum))
482    }
483
484    /// Returns the type of the elements in this table.
485    pub fn element_type(&self) -> TableElementType {
486        match self {
487            Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => {
488                TableElementType::Func
489            }
490            Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => {
491                TableElementType::GcRef
492            }
493            Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => {
494                TableElementType::Cont
495            }
496        }
497    }
498
499    /// Returns whether or not the underlying storage of the table is "static".
500    #[cfg(feature = "pooling-allocator")]
501    pub(crate) fn is_static(&self) -> bool {
502        matches!(self, Table::Static(_))
503    }
504
505    /// Returns the number of allocated elements.
506    pub fn size(&self) -> usize {
507        match self {
508            Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size,
509            Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size,
510            Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size,
511            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(),
512            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
513                elements.len()
514            }
515            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(),
516        }
517    }
518
519    /// Returns the maximum number of elements at runtime.
520    ///
521    /// Returns `None` if the table is unbounded.
522    ///
523    /// The runtime maximum may not be equal to the maximum from the table's Wasm type
524    /// when it is being constrained by an instance allocator.
525    pub fn maximum(&self) -> Option<usize> {
526        match self {
527            Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()),
528            Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()),
529            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()),
530            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum,
531            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum,
532            Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum,
533        }
534    }
535
536    /// Fill `table[dst..dst + len]` with `val`.
537    ///
538    /// Returns a trap error on out-of-bounds accesses.
539    ///
540    /// # Panics
541    ///
542    /// Panics if `val` does not have a type that matches this table.
543    pub fn fill_func(
544        &mut self,
545        dst: u64,
546        val: Option<NonNull<VMFuncRef>>,
547        len: u64,
548    ) -> Result<(), Trap> {
549        let range = self.validate_fill(dst, len)?;
550        let (funcrefs, lazy_init) = self.funcrefs_mut();
551        funcrefs[range].fill(MaybeTaggedFuncRef::from(val, lazy_init));
552        Ok(())
553    }
554
555    /// Same as [`Self::fill_func`], but for GC references.
556    ///
557    /// # Panics
558    ///
559    /// Also panics if `gc_store.is_none()` and it's needed.
560    pub fn fill_gc_ref(
561        &mut self,
562        mut gc_store: Option<&mut GcStore>,
563        dst: u64,
564        val: Option<&VMGcRef>,
565        len: u64,
566    ) -> Result<(), Trap> {
567        let range = self.validate_fill(dst, len)?;
568
569        // Clone the init GC reference into each table slot.
570        for slot in &mut self.gc_refs_mut()[range] {
571            GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), slot, val);
572        }
573
574        Ok(())
575    }
576    /// Same as [`Self::fill_func`], but for continuations.
577    pub fn fill_cont(&mut self, dst: u64, val: Option<VMContObj>, len: u64) -> Result<(), Trap> {
578        let range = self.validate_fill(dst, len)?;
579        self.contrefs_mut()[range].fill(val);
580        Ok(())
581    }
582
583    fn validate_fill(&mut self, dst: u64, len: u64) -> Result<Range<usize>, Trap> {
584        let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
585        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
586        let end = start
587            .checked_add(len)
588            .ok_or_else(|| Trap::TableOutOfBounds)?;
589
590        if end > self.size() {
591            return Err(Trap::TableOutOfBounds);
592        }
593        Ok(start..end)
594    }
595
596    /// Grow table by the specified amount of elements.
597    ///
598    /// Returns the previous size of the table if growth is successful.
599    ///
600    /// Returns `None` if table can't be grown by the specified amount of
601    /// elements, or if the `init_value` is the wrong kind of table element.
602    ///
603    /// # Panics
604    ///
605    /// Panics if `init_value` does not have a type that matches this table.
606    ///
607    /// # Unsafety
608    ///
609    /// Resizing the table can reallocate its internal elements buffer. This
610    /// table's instance's `VMContext` has raw pointers to the elements buffer
611    /// that are used by Wasm, and they need to be fixed up before we call into
612    /// Wasm again. Failure to do so will result in use-after-free inside Wasm.
613    ///
614    /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates
615    /// this unsafety.
616    pub async unsafe fn grow_func(
617        &mut self,
618        limiter: Option<&mut StoreResourceLimiter<'_>>,
619        delta: u64,
620        init_value: Option<SendSyncPtr<VMFuncRef>>,
621    ) -> Result<Option<usize>, Error> {
622        self._grow(delta, limiter, |me, base, len| {
623            me.fill_func(base, init_value.map(|p| p.as_non_null()), len)
624        })
625        .await
626    }
627
628    /// Same as [`Self::grow_func`], but for GC references.
629    pub async unsafe fn grow_gc_ref(
630        &mut self,
631        limiter: Option<&mut StoreResourceLimiter<'_>>,
632        gc_store: Option<&mut GcStore>,
633        delta: u64,
634        init_value: Option<&VMGcRef>,
635    ) -> Result<Option<usize>, Error> {
636        self._grow(delta, limiter, |me, base, len| {
637            me.fill_gc_ref(gc_store, base, init_value, len)
638        })
639        .await
640    }
641
642    /// Same as [`Self::grow_func`], but for continuations.
643    pub async unsafe fn grow_cont(
644        &mut self,
645        limiter: Option<&mut StoreResourceLimiter<'_>>,
646        delta: u64,
647        init_value: Option<VMContObj>,
648    ) -> Result<Option<usize>, Error> {
649        self._grow(delta, limiter, |me, base, len| {
650            me.fill_cont(base, init_value, len)
651        })
652        .await
653    }
654
655    async fn _grow(
656        &mut self,
657        delta: u64,
658        mut limiter: Option<&mut StoreResourceLimiter<'_>>,
659        fill: impl FnOnce(&mut Self, u64, u64) -> Result<(), Trap>,
660    ) -> Result<Option<usize>, Error> {
661        let old_size = self.size();
662
663        // Don't try to resize the table if its size isn't changing, just return
664        // success.
665        if delta == 0 {
666            return Ok(Some(old_size));
667        }
668        let delta = usize::try_from(delta).map_err(|_| Trap::TableOutOfBounds)?;
669
670        let new_size = match old_size.checked_add(delta) {
671            Some(s) => s,
672            None => {
673                if let Some(limiter) = limiter {
674                    limiter
675                        .table_grow_failed(format_err!("overflow calculating new table size"))?;
676                }
677                return Ok(None);
678            }
679        };
680
681        if let Some(limiter) = &mut limiter {
682            if !limiter
683                .table_growing(old_size, new_size, self.maximum())
684                .await?
685            {
686                return Ok(None);
687            }
688        }
689
690        // The WebAssembly spec requires failing a `table.grow` request if
691        // it exceeds the declared limits of the table. We may have set lower
692        // limits in the instance allocator as well.
693        if let Some(max) = self.maximum() {
694            if new_size > max {
695                if let Some(limiter) = limiter {
696                    limiter.table_grow_failed(format_err!("Table maximum size exceeded"))?;
697                }
698                return Ok(None);
699            }
700        }
701
702        // First resize the storage and then fill with the init value
703        match self {
704            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
705                unsafe {
706                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
707                }
708                *size = new_size;
709            }
710            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
711                unsafe {
712                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
713                }
714                *size = new_size;
715            }
716            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => {
717                unsafe {
718                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
719                }
720                *size = new_size;
721            }
722
723            // These calls to `resize` could move the base address of
724            // `elements`. If this table's limits declare it to be fixed-size,
725            // then during AOT compilation we may have promised Cranelift that
726            // the table base address won't change, so it is allowed to optimize
727            // loading the base address. However, in that case the above checks
728            // that delta is non-zero and the new size doesn't exceed the
729            // maximum mean we can't get here.
730            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
731                elements.resize(new_size, None);
732            }
733            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
734                elements.resize_with(new_size, || None);
735            }
736            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
737                elements.resize(new_size, None);
738            }
739        }
740
741        fill(
742            self,
743            u64::try_from(old_size).unwrap(),
744            u64::try_from(delta).unwrap(),
745        )
746        .expect("table should not be out of bounds");
747
748        Ok(Some(old_size))
749    }
750
751    /// Get reference to the specified element.
752    ///
753    /// Returns `None` if the index is out of bounds.
754    ///
755    /// Panics if this is a table of GC references and `gc_store` is `None`.
756    pub fn get_func(&self, index: u64) -> Result<Option<NonNull<VMFuncRef>>, Trap> {
757        match self.get_func_maybe_init(index)? {
758            Some(elem) => Ok(elem),
759            None => panic!("function index should have been initialized"),
760        }
761    }
762
763    /// Same as [`Self::get_func`], except plumbs through the uninitialized
764    /// variant of functions too as `Ok(None)`. An initialized function element
765    /// is `Ok(Some(element))`
766    pub fn get_func_maybe_init(
767        &self,
768        index: u64,
769    ) -> Result<Option<Option<NonNull<VMFuncRef>>>, Trap> {
770        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
771        let (funcrefs, lazy_init) = self.funcrefs();
772        Ok(funcrefs
773            .get(index)
774            .ok_or(Trap::TableOutOfBounds)?
775            .into_funcref(lazy_init))
776    }
777
778    /// Same as [`Self::get_func`], but for GC references.
779    pub fn get_gc_ref(&self, index: u64) -> Result<Option<&VMGcRef>, Trap> {
780        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
781        let gcref = self.gc_refs().get(index).ok_or(Trap::TableOutOfBounds)?;
782        Ok(gcref.as_ref())
783    }
784
785    /// Same as [`Self::get_func`], but for continuations.
786    pub fn get_cont(&self, index: u64) -> Result<Option<VMContObj>, Trap> {
787        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
788        let cont = self.contrefs().get(index).ok_or(Trap::TableOutOfBounds)?;
789        Ok(*cont)
790    }
791
792    /// Set reference to the specified element.
793    ///
794    /// # Errors
795    ///
796    /// Returns an error if `index` is out of bounds or if this table type does
797    /// not match the element type.
798    ///
799    /// # Panics
800    ///
801    /// Panics if `elem` is not of the right type for this table.
802    pub fn set_func(&mut self, index: u64, elem: Option<NonNull<VMFuncRef>>) -> Result<(), Trap> {
803        let trap = Trap::TableOutOfBounds;
804        let index: usize = index.try_into().map_err(|_| trap)?;
805        let (funcrefs, lazy_init) = self.funcrefs_mut();
806        *funcrefs.get_mut(index).ok_or(trap)? = MaybeTaggedFuncRef::from(elem, lazy_init);
807        Ok(())
808    }
809
810    /// Same as [`Self::set_func`] except for GC references.
811    pub fn set_gc_ref(
812        &mut self,
813        store: Option<&mut GcStore>,
814        index: u64,
815        elem: Option<&VMGcRef>,
816    ) -> Result<(), Trap> {
817        let trap = Trap::TableOutOfBounds;
818        let index: usize = index.try_into().map_err(|_| trap)?;
819        GcStore::write_gc_ref_optional_store(
820            store,
821            self.gc_refs_mut().get_mut(index).ok_or(trap)?,
822            elem,
823        );
824        Ok(())
825    }
826
827    /// Copy `len` elements from `self[src_index..][..len]` into
828    /// `dst_table[dst_index..][..len]`.
829    ///
830    /// # Errors
831    ///
832    /// Returns an error if the range is out of bounds of either the source or
833    /// destination tables.
834    pub fn copy_to(
835        &self,
836        dst: &mut Table,
837        gc_store: Option<&mut GcStore>,
838        dst_index: u64,
839        src_index: u64,
840        len: u64,
841    ) -> Result<(), Trap> {
842        let (src_range, dst_range) = Table::validate_copy(self, dst, dst_index, src_index, len)?;
843        Self::copy_elements(gc_store, dst, self, dst_range, src_range);
844        Ok(())
845    }
846
847    /// Copy `len` elements from `self[src_index..][..len]` into
848    /// `self[dst_index..][..len]`.
849    ///
850    /// # Errors
851    ///
852    /// Returns an error if the range is out of bounds of either the source or
853    /// destination tables.
854    pub fn copy_within(
855        &mut self,
856        gc_store: Option<&mut GcStore>,
857        dst_index: u64,
858        src_index: u64,
859        len: u64,
860    ) -> Result<(), Trap> {
861        let (src_range, dst_range) = Table::validate_copy(self, self, dst_index, src_index, len)?;
862        self.copy_elements_within(gc_store, dst_range, src_range);
863        Ok(())
864    }
865
866    /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
867    ///
868    /// # Errors
869    ///
870    /// Returns an error if the range is out of bounds of either the source or
871    /// destination tables.
872    fn validate_copy(
873        src: &Table,
874        dst: &Table,
875        dst_index: u64,
876        src_index: u64,
877        len: u64,
878    ) -> Result<(Range<usize>, Range<usize>), Trap> {
879        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy
880
881        let src_index = usize::try_from(src_index).map_err(|_| Trap::TableOutOfBounds)?;
882        let dst_index = usize::try_from(dst_index).map_err(|_| Trap::TableOutOfBounds)?;
883        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
884
885        if src_index.checked_add(len).map_or(true, |n| n > src.size())
886            || dst_index.checked_add(len).map_or(true, |m| m > dst.size())
887        {
888            return Err(Trap::TableOutOfBounds);
889        }
890
891        debug_assert!(
892            dst.element_type() == src.element_type(),
893            "table element type mismatch"
894        );
895
896        let src_range = src_index..src_index + len;
897        let dst_range = dst_index..dst_index + len;
898
899        Ok((src_range, dst_range))
900    }
901
902    /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
903    pub fn vmtable(&mut self) -> VMTableDefinition {
904        match self {
905            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
906                VMTableDefinition {
907                    base: data.cast().into(),
908                    current_elements: *size,
909                }
910            }
911            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
912                VMTableDefinition {
913                    base: data.cast().into(),
914                    current_elements: *size,
915                }
916            }
917            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition {
918                base: data.cast().into(),
919                current_elements: *size,
920            },
921            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
922                VMTableDefinition {
923                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
924                    current_elements: elements.len(),
925                }
926            }
927            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
928                VMTableDefinition {
929                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
930                    current_elements: elements.len(),
931                }
932            }
933            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
934                VMTableDefinition {
935                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
936                    current_elements: elements.len(),
937                }
938            }
939        }
940    }
941
942    fn funcrefs(&self) -> (&[MaybeTaggedFuncRef], bool) {
943        assert_eq!(self.element_type(), TableElementType::Func);
944        match self {
945            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
946                elements,
947                lazy_init,
948                ..
949            })) => (
950                unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) },
951                *lazy_init,
952            ),
953            Self::Static(StaticTable::Func(StaticFuncTable {
954                data,
955                size,
956                lazy_init,
957            })) => (
958                unsafe { slice::from_raw_parts(data.as_ptr().cast(), *size) },
959                *lazy_init,
960            ),
961            _ => unreachable!(),
962        }
963    }
964
965    fn funcrefs_mut(&mut self) -> (&mut [MaybeTaggedFuncRef], bool) {
966        assert_eq!(self.element_type(), TableElementType::Func);
967        match self {
968            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
969                elements,
970                lazy_init,
971                ..
972            })) => (
973                unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) },
974                *lazy_init,
975            ),
976            Self::Static(StaticTable::Func(StaticFuncTable {
977                data,
978                size,
979                lazy_init,
980            })) => (
981                unsafe { slice::from_raw_parts_mut(data.as_ptr().cast(), *size) },
982                *lazy_init,
983            ),
984            _ => unreachable!(),
985        }
986    }
987
988    fn gc_refs(&self) -> &[Option<VMGcRef>] {
989        assert_eq!(self.element_type(), TableElementType::GcRef);
990        match self {
991            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
992            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
993                &data.as_non_null().as_ref()[..*size]
994            },
995            _ => unreachable!(),
996        }
997    }
998
999    fn contrefs(&self) -> &[Option<VMContObj>] {
1000        assert_eq!(self.element_type(), TableElementType::Cont);
1001        match self {
1002            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
1003                slice::from_raw_parts(elements.as_ptr().cast(), elements.len())
1004            },
1005            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
1006                slice::from_raw_parts(data.as_ptr().cast(), *size)
1007            },
1008            _ => unreachable!(),
1009        }
1010    }
1011
1012    fn contrefs_mut(&mut self) -> &mut [Option<VMContObj>] {
1013        assert_eq!(self.element_type(), TableElementType::Cont);
1014        match self {
1015            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
1016                slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len())
1017            },
1018            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
1019                slice::from_raw_parts_mut(data.as_ptr().cast(), *size)
1020            },
1021            _ => unreachable!(),
1022        }
1023    }
1024
1025    /// Get this table's GC references as a slice.
1026    ///
1027    /// Panics if this is not a table of GC references.
1028    pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] {
1029        assert_eq!(self.element_type(), TableElementType::GcRef);
1030        match self {
1031            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
1032            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
1033                &mut data.as_non_null().as_mut()[..*size]
1034            },
1035            _ => unreachable!(),
1036        }
1037    }
1038
1039    fn copy_elements(
1040        mut gc_store: Option<&mut GcStore>,
1041        dst_table: &mut Self,
1042        src_table: &Self,
1043        dst_range: Range<usize>,
1044        src_range: Range<usize>,
1045    ) {
1046        // This can only be used when copying between different tables
1047        debug_assert!(!ptr::eq(dst_table, src_table));
1048
1049        let ty = dst_table.element_type();
1050
1051        match ty {
1052            TableElementType::Func => {
1053                // `funcref` are `Copy`, so just do a mempcy
1054                let (dst_funcrefs, _lazy_init) = dst_table.funcrefs_mut();
1055                let (src_funcrefs, _lazy_init) = src_table.funcrefs();
1056                dst_funcrefs[dst_range].copy_from_slice(&src_funcrefs[src_range]);
1057            }
1058            TableElementType::GcRef => {
1059                assert_eq!(
1060                    dst_range.end - dst_range.start,
1061                    src_range.end - src_range.start
1062                );
1063                assert!(dst_range.end <= dst_table.gc_refs().len());
1064                assert!(src_range.end <= src_table.gc_refs().len());
1065                for (dst, src) in dst_range.zip(src_range) {
1066                    GcStore::write_gc_ref_optional_store(
1067                        gc_store.as_deref_mut(),
1068                        &mut dst_table.gc_refs_mut()[dst],
1069                        src_table.gc_refs()[src].as_ref(),
1070                    );
1071                }
1072            }
1073            TableElementType::Cont => {
1074                // `contref` are `Copy`, so just do a mempcy
1075                dst_table.contrefs_mut()[dst_range]
1076                    .copy_from_slice(&src_table.contrefs()[src_range]);
1077            }
1078        }
1079    }
1080
1081    fn copy_elements_within(
1082        &mut self,
1083        mut gc_store: Option<&mut GcStore>,
1084        dst_range: Range<usize>,
1085        src_range: Range<usize>,
1086    ) {
1087        assert_eq!(
1088            dst_range.end - dst_range.start,
1089            src_range.end - src_range.start
1090        );
1091
1092        // This is a no-op.
1093        if src_range.start == dst_range.start {
1094            return;
1095        }
1096
1097        let ty = self.element_type();
1098        match ty {
1099            TableElementType::Func => {
1100                // `funcref` are `Copy`, so just do a memmove
1101                let (funcrefs, _lazy_init) = self.funcrefs_mut();
1102                funcrefs.copy_within(src_range, dst_range.start);
1103            }
1104            TableElementType::GcRef => {
1105                // We need to clone each `externref` while handling overlapping
1106                // ranges
1107                let elements = self.gc_refs_mut();
1108                if dst_range.start < src_range.start {
1109                    for (d, s) in dst_range.zip(src_range) {
1110                        let (ds, ss) = elements.split_at_mut(s);
1111                        let dst = &mut ds[d];
1112                        let src = ss[0].as_ref();
1113                        GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), dst, src);
1114                    }
1115                } else {
1116                    for (s, d) in src_range.rev().zip(dst_range.rev()) {
1117                        let (ss, ds) = elements.split_at_mut(d);
1118                        let dst = &mut ds[0];
1119                        let src = ss[s].as_ref();
1120                        GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), dst, src);
1121                    }
1122                }
1123            }
1124            TableElementType::Cont => {
1125                // `contref` are `Copy`, so just do a memmove
1126                self.contrefs_mut().copy_within(src_range, dst_range.start);
1127            }
1128        }
1129    }
1130}
1131
1132// The default table representation is an empty funcref table that cannot grow.
1133impl Default for Table {
1134    fn default() -> Self {
1135        Self::from(StaticFuncTable {
1136            data: SendSyncPtr::new(NonNull::from(&mut [])),
1137            size: 0,
1138            lazy_init: false,
1139        })
1140    }
1141}
wasmtime/runtime/vm/table.rs

wasmtime/runtime/vm/
table.rs
