wasmtime/runtime/vm/
table.rs

1//! Memory management for tables.
2//!
3//! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
4
5use crate::prelude::*;
6use crate::runtime::store::StoreResourceLimiter;
7use crate::runtime::vm::stack_switching::VMContObj;
8use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition};
9use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VmPtr};
10use core::alloc::Layout;
11use core::mem;
12use core::ops::Range;
13use core::ptr::{self, NonNull};
14use core::slice;
15use core::{cmp, usize};
16use wasmtime_environ::{
17    FUNCREF_INIT_BIT, FUNCREF_MASK, IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType,
18};
19
20#[derive(Copy, Clone, PartialEq, Eq, Debug)]
21pub enum TableElementType {
22    Func,
23    GcRef,
24    Cont,
25}
26
27impl TableElementType {
28    /// Returns the size required to actually store an element of this particular type
29    pub fn element_size(&self) -> usize {
30        match self {
31            TableElementType::Func => core::mem::size_of::<FuncTableElem>(),
32            TableElementType::GcRef => core::mem::size_of::<Option<VMGcRef>>(),
33            TableElementType::Cont => core::mem::size_of::<ContTableElem>(),
34        }
35    }
36}
37
38/// At-rest representation of a function in a funcref table.
39///
40/// Note that whether or not these pointers are tagged is a property of `Engine`
41/// configuration. Also note that this specifically uses `VmPtr<T>` to handle
42/// provenance here when loading/storing values to a table.
43///
44/// The possible values here are:
45///
46/// * `None` for untagged tables - a null function element
47/// * `Some(_)` for untagged tables - a non-null function element
48/// * `None` for tagged tables - an uninitialized element
49/// * `Some(1)` for tagged tables - a null function element
50/// * `Some(addr | 1)` for tagged tables - a non-null function element
51#[derive(Copy, Clone)]
52#[repr(transparent)]
53struct MaybeTaggedFuncRef(Option<VmPtr<VMFuncRef>>);
54
55impl MaybeTaggedFuncRef {
56    /// Converts the given `ptr`, a valid funcref pointer, into a tagged pointer
57    /// by adding in the `FUNCREF_INIT_BIT`.
58    fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self {
59        let maybe_tagged = if lazy_init {
60            Some(match ptr {
61                Some(ptr) => ptr.map_addr(|a| a | FUNCREF_INIT_BIT),
62                None => NonNull::new(core::ptr::without_provenance_mut(FUNCREF_INIT_BIT)).unwrap(),
63            })
64        } else {
65            ptr
66        };
67        MaybeTaggedFuncRef(maybe_tagged.map(Into::into))
68    }
69
70    /// Converts a tagged pointer into a `TableElement`, returning `UninitFunc`
71    /// for null (not a tagged value) or `FuncRef` for otherwise tagged values.
72    fn into_funcref(self, lazy_init: bool) -> Option<Option<NonNull<VMFuncRef>>> {
73        let ptr = self.0;
74        if lazy_init && ptr.is_none() {
75            None
76        } else {
77            // Masking off the tag bit is harmless whether the table uses lazy
78            // init or not.
79            Some(ptr.and_then(|ptr| NonNull::new(ptr.as_ptr().map_addr(|a| a & FUNCREF_MASK))))
80        }
81    }
82}
83
84pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>;
85pub type ContTableElem = Option<VMContObj>;
86
87/// The maximum of the sizes of any of the table element types
88#[cfg(feature = "pooling-allocator")]
89pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = {
90    // ContTableElem intentionally excluded for "nominal" calculation.
91    let sizes = [
92        core::mem::size_of::<FuncTableElem>(),
93        core::mem::size_of::<Option<VMGcRef>>(),
94    ];
95
96    // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`,
97    // but as a `const` function, so we can use it to define a constant.
98    const fn slice_max(data: &[usize]) -> usize {
99        match data {
100            [] => 0,
101            [head, tail @ ..] => {
102                let tail_max = slice_max(tail);
103                if *head >= tail_max { *head } else { tail_max }
104            }
105        }
106    }
107
108    slice_max(&sizes)
109};
110
111pub enum StaticTable {
112    Func(StaticFuncTable),
113    GcRef(StaticGcRefTable),
114    Cont(StaticContTable),
115}
116
117impl From<StaticFuncTable> for StaticTable {
118    fn from(value: StaticFuncTable) -> Self {
119        Self::Func(value)
120    }
121}
122
123impl From<StaticGcRefTable> for StaticTable {
124    fn from(value: StaticGcRefTable) -> Self {
125        Self::GcRef(value)
126    }
127}
128
129impl From<StaticContTable> for StaticTable {
130    fn from(value: StaticContTable) -> Self {
131        Self::Cont(value)
132    }
133}
134
135pub struct StaticFuncTable {
136    /// Where data for this table is stored. The length of this list is the
137    /// maximum size of the table.
138    data: SendSyncPtr<[FuncTableElem]>,
139    /// The current size of the table.
140    size: usize,
141    /// Whether elements of this table are initialized lazily.
142    lazy_init: bool,
143}
144
145pub struct StaticGcRefTable {
146    /// Where data for this table is stored. The length of this list is the
147    /// maximum size of the table.
148    data: SendSyncPtr<[Option<VMGcRef>]>,
149    /// The current size of the table.
150    size: usize,
151}
152
153pub struct StaticContTable {
154    /// Where data for this table is stored. The length of this list is the
155    /// maximum size of the table.
156    data: SendSyncPtr<[ContTableElem]>,
157    /// The current size of the table.
158    size: usize,
159}
160
161pub enum DynamicTable {
162    Func(DynamicFuncTable),
163    GcRef(DynamicGcRefTable),
164    Cont(DynamicContTable),
165}
166
167impl From<DynamicFuncTable> for DynamicTable {
168    fn from(value: DynamicFuncTable) -> Self {
169        Self::Func(value)
170    }
171}
172
173impl From<DynamicGcRefTable> for DynamicTable {
174    fn from(value: DynamicGcRefTable) -> Self {
175        Self::GcRef(value)
176    }
177}
178
179impl From<DynamicContTable> for DynamicTable {
180    fn from(value: DynamicContTable) -> Self {
181        Self::Cont(value)
182    }
183}
184
185pub struct DynamicFuncTable {
186    /// Dynamically managed storage space for this table. The length of this
187    /// vector is the current size of the table.
188    elements: Vec<FuncTableElem>,
189    /// Maximum size that `elements` can grow to.
190    maximum: Option<usize>,
191    /// Whether elements of this table are initialized lazily.
192    lazy_init: bool,
193}
194
195pub struct DynamicGcRefTable {
196    /// Dynamically managed storage space for this table. The length of this
197    /// vector is the current size of the table.
198    elements: Vec<Option<VMGcRef>>,
199    /// Maximum size that `elements` can grow to.
200    maximum: Option<usize>,
201}
202
203pub struct DynamicContTable {
204    /// Dynamically managed storage space for this table. The length of this
205    /// vector is the current size of the table.
206    elements: Vec<ContTableElem>,
207    /// Maximum size that `elements` can grow to.
208    maximum: Option<usize>,
209}
210
211/// Represents an instance's table.
212pub enum Table {
213    /// A "static" table where storage space is managed externally, currently
214    /// used with the pooling allocator.
215    Static(StaticTable),
216    /// A "dynamic" table where table storage space is dynamically allocated via
217    /// `malloc` (aka Rust's `Vec`).
218    Dynamic(DynamicTable),
219}
220
221impl From<StaticTable> for Table {
222    fn from(value: StaticTable) -> Self {
223        Self::Static(value)
224    }
225}
226
227impl From<StaticFuncTable> for Table {
228    fn from(value: StaticFuncTable) -> Self {
229        let t: StaticTable = value.into();
230        t.into()
231    }
232}
233
234impl From<StaticGcRefTable> for Table {
235    fn from(value: StaticGcRefTable) -> Self {
236        let t: StaticTable = value.into();
237        t.into()
238    }
239}
240
241impl From<StaticContTable> for Table {
242    fn from(value: StaticContTable) -> Self {
243        let t: StaticTable = value.into();
244        t.into()
245    }
246}
247
248impl From<DynamicTable> for Table {
249    fn from(value: DynamicTable) -> Self {
250        Self::Dynamic(value)
251    }
252}
253
254impl From<DynamicFuncTable> for Table {
255    fn from(value: DynamicFuncTable) -> Self {
256        let t: DynamicTable = value.into();
257        t.into()
258    }
259}
260
261impl From<DynamicGcRefTable> for Table {
262    fn from(value: DynamicGcRefTable) -> Self {
263        let t: DynamicTable = value.into();
264        t.into()
265    }
266}
267
268impl From<DynamicContTable> for Table {
269    fn from(value: DynamicContTable) -> Self {
270        let t: DynamicTable = value.into();
271        t.into()
272    }
273}
274
275pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType {
276    match ty.heap_type.top() {
277        WasmHeapTopType::Func => TableElementType::Func,
278        WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef,
279        WasmHeapTopType::Cont => TableElementType::Cont,
280        WasmHeapTopType::Exn => TableElementType::GcRef,
281    }
282}
283
284/// Allocate dynamic table elements of the given length.
285///
286/// Relies on the fact that our tables' elements are initialized to `None`,
287/// which is represented by zero, to allocate pre-zeroed memory from the global
288/// allocator and avoid manual zero-initialization.
289///
290/// # Safety
291///
292/// Should only ever be called with a `T` that is a table element type and where
293/// `Option<T>`'s `None` variant is represented with zero.
294unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<Vec<Option<T>>> {
295    debug_assert!(
296        unsafe {
297            core::mem::MaybeUninit::<Option<T>>::zeroed()
298                .assume_init()
299                .is_none()
300        },
301        "null table elements are represented with zeroed memory"
302    );
303
304    if len == 0 {
305        return Ok(vec![]);
306    }
307
308    let align = mem::align_of::<Option<T>>();
309
310    let size = mem::size_of::<Option<T>>();
311    let size = size.next_multiple_of(align);
312    let size = size.checked_mul(len).unwrap();
313
314    let layout = Layout::from_size_align(size, align)?;
315
316    let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) };
317    ensure!(!ptr.is_null(), "failed to allocate memory for table");
318
319    let elems = unsafe { Vec::<Option<T>>::from_raw_parts(ptr.cast(), len, len) };
320    debug_assert!(elems.iter().all(|e| e.is_none()));
321
322    Ok(elems)
323}
324
325impl Table {
326    /// Create a new dynamic (movable) table instance for the specified table plan.
327    pub async fn new_dynamic(
328        ty: &wasmtime_environ::Table,
329        tunables: &Tunables,
330        limiter: Option<&mut StoreResourceLimiter<'_>>,
331    ) -> Result<Self> {
332        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
333        match wasm_to_table_type(ty.ref_type) {
334            TableElementType::Func => Ok(Self::from(DynamicFuncTable {
335                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
336                maximum,
337                lazy_init: tunables.table_lazy_init,
338            })),
339            TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable {
340                elements: unsafe { alloc_dynamic_table_elements(minimum)? },
341                maximum,
342            })),
343            TableElementType::Cont => Ok(Self::from(DynamicContTable {
344                elements: vec![None; minimum],
345                maximum,
346            })),
347        }
348    }
349
350    /// Create a new static (immovable) table instance for the specified table plan.
351    pub async unsafe fn new_static(
352        ty: &wasmtime_environ::Table,
353        tunables: &Tunables,
354        data: SendSyncPtr<[u8]>,
355        limiter: Option<&mut StoreResourceLimiter<'_>>,
356    ) -> Result<Self> {
357        let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
358        let size = minimum;
359        let max = maximum.unwrap_or(usize::MAX);
360
361        match wasm_to_table_type(ty.ref_type) {
362            TableElementType::Func => {
363                let len = {
364                    let (before, data, after) = unsafe {
365                        let data = data.as_non_null().as_ref();
366                        data.align_to::<FuncTableElem>()
367                    };
368                    assert!(before.is_empty());
369                    assert!(after.is_empty());
370                    data.len()
371                };
372                ensure!(
373                    usize::try_from(ty.limits.min).unwrap() <= len,
374                    "initial table size of {} exceeds the pooling allocator's \
375                     configured maximum table size of {len} elements",
376                    ty.limits.min,
377                );
378                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
379                    data.as_non_null().cast::<FuncTableElem>(),
380                    cmp::min(len, max),
381                ));
382                Ok(Self::from(StaticFuncTable {
383                    data,
384                    size,
385                    lazy_init: tunables.table_lazy_init,
386                }))
387            }
388            TableElementType::GcRef => {
389                let len = {
390                    let (before, data, after) = unsafe {
391                        let data = data.as_non_null().as_ref();
392                        data.align_to::<Option<VMGcRef>>()
393                    };
394                    assert!(before.is_empty());
395                    assert!(after.is_empty());
396                    data.len()
397                };
398                ensure!(
399                    usize::try_from(ty.limits.min).unwrap() <= len,
400                    "initial table size of {} exceeds the pooling allocator's \
401                     configured maximum table size of {len} elements",
402                    ty.limits.min,
403                );
404                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
405                    data.as_non_null().cast::<Option<VMGcRef>>(),
406                    cmp::min(len, max),
407                ));
408                Ok(Self::from(StaticGcRefTable { data, size }))
409            }
410            TableElementType::Cont => {
411                let len = {
412                    let (before, data, after) = unsafe {
413                        let data = data.as_non_null().as_ref();
414                        data.align_to::<ContTableElem>()
415                    };
416                    assert!(before.is_empty());
417                    assert!(after.is_empty());
418                    data.len()
419                };
420                ensure!(
421                    usize::try_from(ty.limits.min).unwrap() <= len,
422                    "initial table size of {} exceeds the pooling allocator's \
423                     configured maximum table size of {len} elements",
424                    ty.limits.min,
425                );
426                let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
427                    data.as_non_null().cast::<ContTableElem>(),
428                    cmp::min(len, max),
429                ));
430                Ok(Self::from(StaticContTable { data, size }))
431            }
432        }
433    }
434
435    // Calls the `store`'s limiter to optionally prevent the table from being created.
436    //
437    // Returns the minimum and maximum size of the table if the table can be created.
438    async fn limit_new(
439        ty: &wasmtime_environ::Table,
440        limiter: Option<&mut StoreResourceLimiter<'_>>,
441    ) -> Result<(usize, Option<usize>)> {
442        // No matter how the table limits are specified
443        // The table size is limited by the host's pointer size
444        let absolute_max = usize::MAX;
445
446        // If the minimum overflows the host's pointer size, then we can't satisfy this request.
447        // We defer the error to later so the `store` can be informed.
448        let minimum = usize::try_from(ty.limits.min).ok();
449
450        // The maximum size of the table is limited by:
451        // * the host's pointer size.
452        // * the table's maximum size if defined.
453        // * if the table is 64-bit.
454        let maximum = match (ty.limits.max, ty.idx_type) {
455            (Some(max), _) => usize::try_from(max).ok(),
456            (None, IndexType::I64) => usize::try_from(u64::MAX).ok(),
457            (None, IndexType::I32) => usize::try_from(u32::MAX).ok(),
458        };
459
460        // Inform the store's limiter what's about to happen.
461        if let Some(limiter) = limiter {
462            if !limiter
463                .table_growing(0, minimum.unwrap_or(absolute_max), maximum)
464                .await?
465            {
466                bail!(
467                    "table minimum size of {} elements exceeds table limits",
468                    ty.limits.min
469                );
470            }
471        }
472
473        // At this point we need to actually handle overflows, so bail out with
474        // an error if we made it this far.
475        let minimum = minimum.ok_or_else(|| {
476            format_err!(
477                "table minimum size of {} elements exceeds table limits",
478                ty.limits.min
479            )
480        })?;
481        Ok((minimum, maximum))
482    }
483
484    /// Returns the type of the elements in this table.
485    pub fn element_type(&self) -> TableElementType {
486        match self {
487            Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => {
488                TableElementType::Func
489            }
490            Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => {
491                TableElementType::GcRef
492            }
493            Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => {
494                TableElementType::Cont
495            }
496        }
497    }
498
499    /// Returns whether or not the underlying storage of the table is "static".
500    #[cfg(feature = "pooling-allocator")]
501    pub(crate) fn is_static(&self) -> bool {
502        matches!(self, Table::Static(_))
503    }
504
505    /// Returns the number of allocated elements.
506    pub fn size(&self) -> usize {
507        match self {
508            Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size,
509            Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size,
510            Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size,
511            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(),
512            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
513                elements.len()
514            }
515            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(),
516        }
517    }
518
519    /// Returns the maximum number of elements at runtime.
520    ///
521    /// Returns `None` if the table is unbounded.
522    ///
523    /// The runtime maximum may not be equal to the maximum from the table's Wasm type
524    /// when it is being constrained by an instance allocator.
525    pub fn maximum(&self) -> Option<usize> {
526        match self {
527            Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()),
528            Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()),
529            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()),
530            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum,
531            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum,
532            Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum,
533        }
534    }
535
536    /// Fill `table[dst..dst + len]` with `val`.
537    ///
538    /// Returns a trap error on out-of-bounds accesses.
539    ///
540    /// # Panics
541    ///
542    /// Panics if `val` does not have a type that matches this table.
543    pub fn fill_func(
544        &mut self,
545        dst: u64,
546        val: Option<NonNull<VMFuncRef>>,
547        len: u64,
548    ) -> Result<(), Trap> {
549        let range = self.validate_fill(dst, len)?;
550        let (funcrefs, lazy_init) = self.funcrefs_mut();
551        funcrefs[range].fill(MaybeTaggedFuncRef::from(val, lazy_init));
552        Ok(())
553    }
554
555    /// Same as [`Self::fill_func`], but for GC references.
556    ///
557    /// # Panics
558    ///
559    /// Also panics if `gc_store.is_none()` and it's needed.
560    pub fn fill_gc_ref(
561        &mut self,
562        mut gc_store: Option<&mut GcStore>,
563        dst: u64,
564        val: Option<&VMGcRef>,
565        len: u64,
566    ) -> Result<(), Trap> {
567        let range = self.validate_fill(dst, len)?;
568
569        // Clone the init GC reference into each table slot.
570        for slot in &mut self.gc_refs_mut()[range] {
571            GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), slot, val);
572        }
573
574        Ok(())
575    }
576    /// Same as [`Self::fill_func`], but for continuations.
577    pub fn fill_cont(&mut self, dst: u64, val: Option<VMContObj>, len: u64) -> Result<(), Trap> {
578        let range = self.validate_fill(dst, len)?;
579        self.contrefs_mut()[range].fill(val);
580        Ok(())
581    }
582
583    fn validate_fill(&mut self, dst: u64, len: u64) -> Result<Range<usize>, Trap> {
584        let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
585        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
586        let end = start
587            .checked_add(len)
588            .ok_or_else(|| Trap::TableOutOfBounds)?;
589
590        if end > self.size() {
591            return Err(Trap::TableOutOfBounds);
592        }
593        Ok(start..end)
594    }
595
596    /// Grow table by the specified amount of elements.
597    ///
598    /// Returns the previous size of the table if growth is successful.
599    ///
600    /// Returns `None` if table can't be grown by the specified amount of
601    /// elements, or if the `init_value` is the wrong kind of table element.
602    ///
603    /// # Panics
604    ///
605    /// Panics if `init_value` does not have a type that matches this table.
606    ///
607    /// # Unsafety
608    ///
609    /// Resizing the table can reallocate its internal elements buffer. This
610    /// table's instance's `VMContext` has raw pointers to the elements buffer
611    /// that are used by Wasm, and they need to be fixed up before we call into
612    /// Wasm again. Failure to do so will result in use-after-free inside Wasm.
613    ///
614    /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates
615    /// this unsafety.
616    pub async unsafe fn grow_func(
617        &mut self,
618        limiter: Option<&mut StoreResourceLimiter<'_>>,
619        delta: u64,
620        init_value: Option<SendSyncPtr<VMFuncRef>>,
621    ) -> Result<Option<usize>, Error> {
622        self._grow(delta, limiter, |me, base, len| {
623            me.fill_func(base, init_value.map(|p| p.as_non_null()), len)
624        })
625        .await
626    }
627
628    /// Same as [`Self::grow_func`], but for GC references.
629    pub async unsafe fn grow_gc_ref(
630        &mut self,
631        limiter: Option<&mut StoreResourceLimiter<'_>>,
632        gc_store: Option<&mut GcStore>,
633        delta: u64,
634        init_value: Option<&VMGcRef>,
635    ) -> Result<Option<usize>, Error> {
636        self._grow(delta, limiter, |me, base, len| {
637            me.fill_gc_ref(gc_store, base, init_value, len)
638        })
639        .await
640    }
641
642    /// Same as [`Self::grow_func`], but for continuations.
643    pub async unsafe fn grow_cont(
644        &mut self,
645        limiter: Option<&mut StoreResourceLimiter<'_>>,
646        delta: u64,
647        init_value: Option<VMContObj>,
648    ) -> Result<Option<usize>, Error> {
649        self._grow(delta, limiter, |me, base, len| {
650            me.fill_cont(base, init_value, len)
651        })
652        .await
653    }
654
655    async fn _grow(
656        &mut self,
657        delta: u64,
658        mut limiter: Option<&mut StoreResourceLimiter<'_>>,
659        fill: impl FnOnce(&mut Self, u64, u64) -> Result<(), Trap>,
660    ) -> Result<Option<usize>, Error> {
661        let old_size = self.size();
662
663        // Don't try to resize the table if its size isn't changing, just return
664        // success.
665        if delta == 0 {
666            return Ok(Some(old_size));
667        }
668        let delta = usize::try_from(delta).map_err(|_| Trap::TableOutOfBounds)?;
669
670        let new_size = match old_size.checked_add(delta) {
671            Some(s) => s,
672            None => {
673                if let Some(limiter) = limiter {
674                    limiter
675                        .table_grow_failed(format_err!("overflow calculating new table size"))?;
676                }
677                return Ok(None);
678            }
679        };
680
681        if let Some(limiter) = &mut limiter {
682            if !limiter
683                .table_growing(old_size, new_size, self.maximum())
684                .await?
685            {
686                return Ok(None);
687            }
688        }
689
690        // The WebAssembly spec requires failing a `table.grow` request if
691        // it exceeds the declared limits of the table. We may have set lower
692        // limits in the instance allocator as well.
693        if let Some(max) = self.maximum() {
694            if new_size > max {
695                if let Some(limiter) = limiter {
696                    limiter.table_grow_failed(format_err!("Table maximum size exceeded"))?;
697                }
698                return Ok(None);
699            }
700        }
701
702        // First resize the storage and then fill with the init value
703        match self {
704            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
705                unsafe {
706                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
707                }
708                *size = new_size;
709            }
710            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
711                unsafe {
712                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
713                }
714                *size = new_size;
715            }
716            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => {
717                unsafe {
718                    debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
719                }
720                *size = new_size;
721            }
722
723            // These calls to `resize` could move the base address of
724            // `elements`. If this table's limits declare it to be fixed-size,
725            // then during AOT compilation we may have promised Cranelift that
726            // the table base address won't change, so it is allowed to optimize
727            // loading the base address. However, in that case the above checks
728            // that delta is non-zero and the new size doesn't exceed the
729            // maximum mean we can't get here.
730            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
731                elements.resize(new_size, None);
732            }
733            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
734                elements.resize_with(new_size, || None);
735            }
736            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
737                elements.resize(new_size, None);
738            }
739        }
740
741        fill(
742            self,
743            u64::try_from(old_size).unwrap(),
744            u64::try_from(delta).unwrap(),
745        )
746        .expect("table should not be out of bounds");
747
748        Ok(Some(old_size))
749    }
750
751    /// Get reference to the specified element.
752    ///
753    /// Returns `Ok(None)` if the element is null or uninitialized.
754    /// Returns `Err` if the index is out of bounds.
755    ///
756    /// Panics if this is a table of GC references and `gc_store` is `None`.
757    pub fn get_func(&self, index: u64) -> Result<Option<NonNull<VMFuncRef>>, Trap> {
758        match self.get_func_maybe_init(index)? {
759            Some(elem) => Ok(elem),
760            None => panic!("function index should have been initialized"),
761        }
762    }
763
764    /// Same as [`Self::get_func`], except plumbs through the uninitialized
765    /// variant of functions too as `Ok(None)`. An initialized function element
766    /// is `Ok(Some(element))`
767    pub fn get_func_maybe_init(
768        &self,
769        index: u64,
770    ) -> Result<Option<Option<NonNull<VMFuncRef>>>, Trap> {
771        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
772        let (funcrefs, lazy_init) = self.funcrefs();
773        Ok(funcrefs
774            .get(index)
775            .ok_or(Trap::TableOutOfBounds)?
776            .into_funcref(lazy_init))
777    }
778
779    /// Same as [`Self::get_func`], but for GC references.
780    pub fn get_gc_ref(&self, index: u64) -> Result<Option<&VMGcRef>, Trap> {
781        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
782        let gcref = self.gc_refs().get(index).ok_or(Trap::TableOutOfBounds)?;
783        Ok(gcref.as_ref())
784    }
785
786    /// Same as [`Self::get_func`], but for continuations.
787    pub fn get_cont(&self, index: u64) -> Result<Option<VMContObj>, Trap> {
788        let index = usize::try_from(index).map_err(|_| Trap::TableOutOfBounds)?;
789        let cont = self.contrefs().get(index).ok_or(Trap::TableOutOfBounds)?;
790        Ok(*cont)
791    }
792
793    /// Set reference to the specified element.
794    ///
795    /// # Errors
796    ///
797    /// Returns an error if `index` is out of bounds or if this table type does
798    /// not match the element type.
799    ///
800    /// # Panics
801    ///
802    /// Panics if `elem` is not of the right type for this table.
803    pub fn set_func(&mut self, index: u64, elem: Option<NonNull<VMFuncRef>>) -> Result<(), Trap> {
804        let trap = Trap::TableOutOfBounds;
805        let index: usize = index.try_into().map_err(|_| trap)?;
806        let (funcrefs, lazy_init) = self.funcrefs_mut();
807        *funcrefs.get_mut(index).ok_or(trap)? = MaybeTaggedFuncRef::from(elem, lazy_init);
808        Ok(())
809    }
810
811    /// Same as [`Self::set_func`] except for GC references.
812    pub fn set_gc_ref(
813        &mut self,
814        store: Option<&mut GcStore>,
815        index: u64,
816        elem: Option<&VMGcRef>,
817    ) -> Result<(), Trap> {
818        let trap = Trap::TableOutOfBounds;
819        let index: usize = index.try_into().map_err(|_| trap)?;
820        GcStore::write_gc_ref_optional_store(
821            store,
822            self.gc_refs_mut().get_mut(index).ok_or(trap)?,
823            elem,
824        );
825        Ok(())
826    }
827
828    /// Copy `len` elements from `self[src_index..][..len]` into
829    /// `dst_table[dst_index..][..len]`.
830    ///
831    /// # Errors
832    ///
833    /// Returns an error if the range is out of bounds of either the source or
834    /// destination tables.
835    pub fn copy_to(
836        &self,
837        dst: &mut Table,
838        gc_store: Option<&mut GcStore>,
839        dst_index: u64,
840        src_index: u64,
841        len: u64,
842    ) -> Result<(), Trap> {
843        let (src_range, dst_range) = Table::validate_copy(self, dst, dst_index, src_index, len)?;
844        Self::copy_elements(gc_store, dst, self, dst_range, src_range);
845        Ok(())
846    }
847
848    /// Copy `len` elements from `self[src_index..][..len]` into
849    /// `self[dst_index..][..len]`.
850    ///
851    /// # Errors
852    ///
853    /// Returns an error if the range is out of bounds of either the source or
854    /// destination tables.
855    pub fn copy_within(
856        &mut self,
857        gc_store: Option<&mut GcStore>,
858        dst_index: u64,
859        src_index: u64,
860        len: u64,
861    ) -> Result<(), Trap> {
862        let (src_range, dst_range) = Table::validate_copy(self, self, dst_index, src_index, len)?;
863        self.copy_elements_within(gc_store, dst_range, src_range);
864        Ok(())
865    }
866
867    /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
868    ///
869    /// # Errors
870    ///
871    /// Returns an error if the range is out of bounds of either the source or
872    /// destination tables.
873    fn validate_copy(
874        src: &Table,
875        dst: &Table,
876        dst_index: u64,
877        src_index: u64,
878        len: u64,
879    ) -> Result<(Range<usize>, Range<usize>), Trap> {
880        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy
881
882        let src_index = usize::try_from(src_index).map_err(|_| Trap::TableOutOfBounds)?;
883        let dst_index = usize::try_from(dst_index).map_err(|_| Trap::TableOutOfBounds)?;
884        let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
885
886        if src_index.checked_add(len).map_or(true, |n| n > src.size())
887            || dst_index.checked_add(len).map_or(true, |m| m > dst.size())
888        {
889            return Err(Trap::TableOutOfBounds);
890        }
891
892        debug_assert!(
893            dst.element_type() == src.element_type(),
894            "table element type mismatch"
895        );
896
897        let src_range = src_index..src_index + len;
898        let dst_range = dst_index..dst_index + len;
899
900        Ok((src_range, dst_range))
901    }
902
903    /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
904    pub fn vmtable(&mut self) -> VMTableDefinition {
905        match self {
906            Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
907                VMTableDefinition {
908                    base: data.cast().into(),
909                    current_elements: *size,
910                }
911            }
912            Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
913                VMTableDefinition {
914                    base: data.cast().into(),
915                    current_elements: *size,
916                }
917            }
918            Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition {
919                base: data.cast().into(),
920                current_elements: *size,
921            },
922            Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
923                VMTableDefinition {
924                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
925                    current_elements: elements.len(),
926                }
927            }
928            Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
929                VMTableDefinition {
930                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
931                    current_elements: elements.len(),
932                }
933            }
934            Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
935                VMTableDefinition {
936                    base: NonNull::new(elements.as_mut_ptr()).unwrap().cast().into(),
937                    current_elements: elements.len(),
938                }
939            }
940        }
941    }
942
943    fn funcrefs(&self) -> (&[MaybeTaggedFuncRef], bool) {
944        assert_eq!(self.element_type(), TableElementType::Func);
945        match self {
946            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
947                elements,
948                lazy_init,
949                ..
950            })) => (
951                unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) },
952                *lazy_init,
953            ),
954            Self::Static(StaticTable::Func(StaticFuncTable {
955                data,
956                size,
957                lazy_init,
958            })) => (
959                unsafe { slice::from_raw_parts(data.as_ptr().cast(), *size) },
960                *lazy_init,
961            ),
962            _ => unreachable!(),
963        }
964    }
965
966    fn funcrefs_mut(&mut self) -> (&mut [MaybeTaggedFuncRef], bool) {
967        assert_eq!(self.element_type(), TableElementType::Func);
968        match self {
969            Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
970                elements,
971                lazy_init,
972                ..
973            })) => (
974                unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) },
975                *lazy_init,
976            ),
977            Self::Static(StaticTable::Func(StaticFuncTable {
978                data,
979                size,
980                lazy_init,
981            })) => (
982                unsafe { slice::from_raw_parts_mut(data.as_ptr().cast(), *size) },
983                *lazy_init,
984            ),
985            _ => unreachable!(),
986        }
987    }
988
989    fn gc_refs(&self) -> &[Option<VMGcRef>] {
990        assert_eq!(self.element_type(), TableElementType::GcRef);
991        match self {
992            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
993            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
994                &data.as_non_null().as_ref()[..*size]
995            },
996            _ => unreachable!(),
997        }
998    }
999
1000    fn contrefs(&self) -> &[Option<VMContObj>] {
1001        assert_eq!(self.element_type(), TableElementType::Cont);
1002        match self {
1003            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
1004                slice::from_raw_parts(elements.as_ptr().cast(), elements.len())
1005            },
1006            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
1007                slice::from_raw_parts(data.as_ptr().cast(), *size)
1008            },
1009            _ => unreachable!(),
1010        }
1011    }
1012
1013    fn contrefs_mut(&mut self) -> &mut [Option<VMContObj>] {
1014        assert_eq!(self.element_type(), TableElementType::Cont);
1015        match self {
1016            Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe {
1017                slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len())
1018            },
1019            Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe {
1020                slice::from_raw_parts_mut(data.as_ptr().cast(), *size)
1021            },
1022            _ => unreachable!(),
1023        }
1024    }
1025
1026    /// Get this table's GC references as a slice.
1027    ///
1028    /// Panics if this is not a table of GC references.
1029    pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] {
1030        assert_eq!(self.element_type(), TableElementType::GcRef);
1031        match self {
1032            Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
1033            Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
1034                &mut data.as_non_null().as_mut()[..*size]
1035            },
1036            _ => unreachable!(),
1037        }
1038    }
1039
1040    fn copy_elements(
1041        mut gc_store: Option<&mut GcStore>,
1042        dst_table: &mut Self,
1043        src_table: &Self,
1044        dst_range: Range<usize>,
1045        src_range: Range<usize>,
1046    ) {
1047        // This can only be used when copying between different tables
1048        debug_assert!(!ptr::eq(dst_table, src_table));
1049
1050        let ty = dst_table.element_type();
1051
1052        match ty {
1053            TableElementType::Func => {
1054                // `funcref` are `Copy`, so just do a mempcy
1055                let (dst_funcrefs, _lazy_init) = dst_table.funcrefs_mut();
1056                let (src_funcrefs, _lazy_init) = src_table.funcrefs();
1057                dst_funcrefs[dst_range].copy_from_slice(&src_funcrefs[src_range]);
1058            }
1059            TableElementType::GcRef => {
1060                assert_eq!(
1061                    dst_range.end - dst_range.start,
1062                    src_range.end - src_range.start
1063                );
1064                assert!(dst_range.end <= dst_table.gc_refs().len());
1065                assert!(src_range.end <= src_table.gc_refs().len());
1066                for (dst, src) in dst_range.zip(src_range) {
1067                    GcStore::write_gc_ref_optional_store(
1068                        gc_store.as_deref_mut(),
1069                        &mut dst_table.gc_refs_mut()[dst],
1070                        src_table.gc_refs()[src].as_ref(),
1071                    );
1072                }
1073            }
1074            TableElementType::Cont => {
1075                // `contref` are `Copy`, so just do a mempcy
1076                dst_table.contrefs_mut()[dst_range]
1077                    .copy_from_slice(&src_table.contrefs()[src_range]);
1078            }
1079        }
1080    }
1081
1082    fn copy_elements_within(
1083        &mut self,
1084        mut gc_store: Option<&mut GcStore>,
1085        dst_range: Range<usize>,
1086        src_range: Range<usize>,
1087    ) {
1088        assert_eq!(
1089            dst_range.end - dst_range.start,
1090            src_range.end - src_range.start
1091        );
1092
1093        // This is a no-op.
1094        if src_range.start == dst_range.start {
1095            return;
1096        }
1097
1098        let ty = self.element_type();
1099        match ty {
1100            TableElementType::Func => {
1101                // `funcref` are `Copy`, so just do a memmove
1102                let (funcrefs, _lazy_init) = self.funcrefs_mut();
1103                funcrefs.copy_within(src_range, dst_range.start);
1104            }
1105            TableElementType::GcRef => {
1106                // We need to clone each `externref` while handling overlapping
1107                // ranges
1108                let elements = self.gc_refs_mut();
1109                if dst_range.start < src_range.start {
1110                    for (d, s) in dst_range.zip(src_range) {
1111                        let (ds, ss) = elements.split_at_mut(s);
1112                        let dst = &mut ds[d];
1113                        let src = ss[0].as_ref();
1114                        GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), dst, src);
1115                    }
1116                } else {
1117                    for (s, d) in src_range.rev().zip(dst_range.rev()) {
1118                        let (ss, ds) = elements.split_at_mut(d);
1119                        let dst = &mut ds[0];
1120                        let src = ss[s].as_ref();
1121                        GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), dst, src);
1122                    }
1123                }
1124            }
1125            TableElementType::Cont => {
1126                // `contref` are `Copy`, so just do a memmove
1127                self.contrefs_mut().copy_within(src_range, dst_range.start);
1128            }
1129        }
1130    }
1131}
1132
1133// The default table representation is an empty funcref table that cannot grow.
1134impl Default for Table {
1135    fn default() -> Self {
1136        Self::from(StaticFuncTable {
1137            data: SendSyncPtr::new(NonNull::from(&mut [])),
1138            size: 0,
1139            lazy_init: false,
1140        })
1141    }
1142}
wasmtime/runtime/vm/table.rs

wasmtime/runtime/vm/
table.rs
