1#![cfg_attr(feature = "gc", allow(irrefutable_let_patterns))]
6
7use crate::prelude::*;
8use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition};
9use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore};
10use core::alloc::Layout;
11use core::mem;
12use core::ops::Range;
13use core::ptr::{self, NonNull};
14use core::slice;
15use core::{cmp, usize};
16use sptr::Strict;
17use wasmtime_environ::{
18 IndexType, Trap, Tunables, WasmHeapTopType, WasmRefType, FUNCREF_INIT_BIT, FUNCREF_MASK,
19};
20
21pub enum TableElement {
26 FuncRef(Option<NonNull<VMFuncRef>>),
28
29 GcRef(Option<VMGcRef>),
31
32 UninitFunc,
37}
38
39#[derive(Copy, Clone, PartialEq, Eq, Debug)]
40pub enum TableElementType {
41 Func,
42 GcRef,
43}
44
45impl TableElementType {
46 fn matches(&self, val: &TableElement) -> bool {
47 match (val, self) {
48 (TableElement::FuncRef(_), TableElementType::Func) => true,
49 (TableElement::GcRef(_), TableElementType::GcRef) => true,
50 _ => false,
51 }
52 }
53}
54
55unsafe impl Send for TableElement where VMGcRef: Send {}
58unsafe impl Sync for TableElement where VMGcRef: Sync {}
59
60impl TableElement {
61 pub(crate) unsafe fn into_func_ref_asserting_initialized(self) -> Option<NonNull<VMFuncRef>> {
73 match self {
74 Self::FuncRef(e) => e,
75 Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"),
76 Self::GcRef(_) => panic!("GC reference is not a function reference"),
77 }
78 }
79
80 pub(crate) fn is_uninit(&self) -> bool {
83 match self {
84 Self::UninitFunc => true,
85 _ => false,
86 }
87 }
88}
89
90impl From<Option<NonNull<VMFuncRef>>> for TableElement {
91 fn from(f: Option<NonNull<VMFuncRef>>) -> TableElement {
92 TableElement::FuncRef(f)
93 }
94}
95
96impl From<Option<VMGcRef>> for TableElement {
97 fn from(x: Option<VMGcRef>) -> TableElement {
98 TableElement::GcRef(x)
99 }
100}
101
102impl From<VMGcRef> for TableElement {
103 fn from(x: VMGcRef) -> TableElement {
104 TableElement::GcRef(Some(x))
105 }
106}
107
108#[derive(Copy, Clone)]
109#[repr(transparent)]
110struct TaggedFuncRef(*mut VMFuncRef);
111
112impl TaggedFuncRef {
113 const UNINIT: TaggedFuncRef = TaggedFuncRef(ptr::null_mut());
114
115 fn from(ptr: Option<NonNull<VMFuncRef>>, lazy_init: bool) -> Self {
118 let ptr = ptr.map(|p| p.as_ptr()).unwrap_or(ptr::null_mut());
119 if lazy_init {
120 let masked = Strict::map_addr(ptr, |a| a | FUNCREF_INIT_BIT);
121 TaggedFuncRef(masked)
122 } else {
123 TaggedFuncRef(ptr)
124 }
125 }
126
127 fn into_table_element(self, lazy_init: bool) -> TableElement {
130 let ptr = self.0;
131 if lazy_init && ptr.is_null() {
132 TableElement::UninitFunc
133 } else {
134 let unmasked = Strict::map_addr(ptr, |a| a & FUNCREF_MASK);
137 TableElement::FuncRef(NonNull::new(unmasked))
138 }
139 }
140}
141
142pub type FuncTableElem = Option<SendSyncPtr<VMFuncRef>>;
143
144pub enum StaticTable {
145 Func(StaticFuncTable),
146 GcRef(StaticGcRefTable),
147}
148
149impl From<StaticFuncTable> for StaticTable {
150 fn from(value: StaticFuncTable) -> Self {
151 Self::Func(value)
152 }
153}
154
155impl From<StaticGcRefTable> for StaticTable {
156 fn from(value: StaticGcRefTable) -> Self {
157 Self::GcRef(value)
158 }
159}
160
161pub struct StaticFuncTable {
162 data: SendSyncPtr<[FuncTableElem]>,
165 size: usize,
167 lazy_init: bool,
169}
170
171pub struct StaticGcRefTable {
172 data: SendSyncPtr<[Option<VMGcRef>]>,
175 size: usize,
177}
178
179pub enum DynamicTable {
180 Func(DynamicFuncTable),
181 GcRef(DynamicGcRefTable),
182}
183
184impl From<DynamicFuncTable> for DynamicTable {
185 fn from(value: DynamicFuncTable) -> Self {
186 Self::Func(value)
187 }
188}
189
190impl From<DynamicGcRefTable> for DynamicTable {
191 fn from(value: DynamicGcRefTable) -> Self {
192 Self::GcRef(value)
193 }
194}
195
196pub struct DynamicFuncTable {
197 elements: Vec<FuncTableElem>,
200 maximum: Option<usize>,
202 lazy_init: bool,
204}
205
206pub struct DynamicGcRefTable {
207 elements: Vec<Option<VMGcRef>>,
210 maximum: Option<usize>,
212}
213
214pub enum Table {
216 Static(StaticTable),
219 Dynamic(DynamicTable),
222}
223
224impl From<StaticTable> for Table {
225 fn from(value: StaticTable) -> Self {
226 Self::Static(value)
227 }
228}
229
230impl From<StaticFuncTable> for Table {
231 fn from(value: StaticFuncTable) -> Self {
232 let t: StaticTable = value.into();
233 t.into()
234 }
235}
236
237impl From<StaticGcRefTable> for Table {
238 fn from(value: StaticGcRefTable) -> Self {
239 let t: StaticTable = value.into();
240 t.into()
241 }
242}
243
244impl From<DynamicTable> for Table {
245 fn from(value: DynamicTable) -> Self {
246 Self::Dynamic(value)
247 }
248}
249
250impl From<DynamicFuncTable> for Table {
251 fn from(value: DynamicFuncTable) -> Self {
252 let t: DynamicTable = value.into();
253 t.into()
254 }
255}
256
257impl From<DynamicGcRefTable> for Table {
258 fn from(value: DynamicGcRefTable) -> Self {
259 let t: DynamicTable = value.into();
260 t.into()
261 }
262}
263
264fn wasm_to_table_type(ty: WasmRefType) -> TableElementType {
265 match ty.heap_type.top() {
266 WasmHeapTopType::Func => TableElementType::Func,
267 WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef,
268 WasmHeapTopType::Cont => todo!(), }
270}
271
272unsafe fn alloc_dynamic_table_elements<T>(len: usize) -> Result<Vec<Option<T>>> {
283 debug_assert!(
284 core::mem::MaybeUninit::<Option<T>>::zeroed()
285 .assume_init()
286 .is_none(),
287 "null table elements are represented with zeroed memory"
288 );
289
290 if len == 0 {
291 return Ok(vec![]);
292 }
293
294 let align = mem::align_of::<Option<T>>();
295
296 let size = mem::size_of::<Option<T>>();
297 let size = size.next_multiple_of(align);
298 let size = size.checked_mul(len).unwrap();
299
300 let layout = Layout::from_size_align(size, align)?;
301
302 let ptr = alloc::alloc::alloc_zeroed(layout);
303 ensure!(!ptr.is_null(), "failed to allocate memory for table");
304
305 let elems = Vec::<Option<T>>::from_raw_parts(ptr.cast(), len, len);
306 debug_assert!(elems.iter().all(|e| e.is_none()));
307
308 Ok(elems)
309}
310
311impl Table {
312 pub fn new_dynamic(
314 ty: &wasmtime_environ::Table,
315 tunables: &Tunables,
316 store: &mut dyn VMStore,
317 ) -> Result<Self> {
318 let (minimum, maximum) = Self::limit_new(ty, store)?;
319 match wasm_to_table_type(ty.ref_type) {
320 TableElementType::Func => Ok(Self::from(DynamicFuncTable {
321 elements: unsafe { alloc_dynamic_table_elements(minimum)? },
322 maximum,
323 lazy_init: tunables.table_lazy_init,
324 })),
325 TableElementType::GcRef => Ok(Self::from(DynamicGcRefTable {
326 elements: unsafe { alloc_dynamic_table_elements(minimum)? },
327 maximum,
328 })),
329 }
330 }
331
332 pub unsafe fn new_static(
334 ty: &wasmtime_environ::Table,
335 tunables: &Tunables,
336 data: SendSyncPtr<[u8]>,
337 store: &mut dyn VMStore,
338 ) -> Result<Self> {
339 let (minimum, maximum) = Self::limit_new(ty, store)?;
340 let size = minimum;
341 let max = maximum.unwrap_or(usize::MAX);
342
343 match wasm_to_table_type(ty.ref_type) {
344 TableElementType::Func => {
345 let len = {
346 let data = data.as_non_null().as_ref();
347 let (before, data, after) = data.align_to::<FuncTableElem>();
348 assert!(before.is_empty());
349 assert!(after.is_empty());
350 data.len()
351 };
352 ensure!(
353 usize::try_from(ty.limits.min).unwrap() <= len,
354 "initial table size of {} exceeds the pooling allocator's \
355 configured maximum table size of {len} elements",
356 ty.limits.min,
357 );
358 let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
359 data.as_non_null().cast::<FuncTableElem>(),
360 cmp::min(len, max),
361 ));
362 Ok(Self::from(StaticFuncTable {
363 data,
364 size,
365 lazy_init: tunables.table_lazy_init,
366 }))
367 }
368 TableElementType::GcRef => {
369 let len = {
370 let data = data.as_non_null().as_ref();
371 let (before, data, after) = data.align_to::<Option<VMGcRef>>();
372 assert!(before.is_empty());
373 assert!(after.is_empty());
374 data.len()
375 };
376 ensure!(
377 usize::try_from(ty.limits.min).unwrap() <= len,
378 "initial table size of {} exceeds the pooling allocator's \
379 configured maximum table size of {len} elements",
380 ty.limits.min,
381 );
382 let data = SendSyncPtr::new(NonNull::slice_from_raw_parts(
383 data.as_non_null().cast::<Option<VMGcRef>>(),
384 cmp::min(len, max),
385 ));
386 Ok(Self::from(StaticGcRefTable { data, size }))
387 }
388 }
389 }
390
391 fn limit_new(
395 ty: &wasmtime_environ::Table,
396 store: &mut dyn VMStore,
397 ) -> Result<(usize, Option<usize>)> {
398 let absolute_max = usize::MAX;
401
402 let minimum = usize::try_from(ty.limits.min).ok();
405
406 let maximum = match (ty.limits.max, ty.idx_type) {
411 (Some(max), _) => usize::try_from(max).ok(),
412 (None, IndexType::I64) => usize::try_from(u64::MAX).ok(),
413 (None, IndexType::I32) => usize::try_from(u32::MAX).ok(),
414 };
415
416 if !store.table_growing(0, minimum.unwrap_or(absolute_max), maximum)? {
418 bail!(
419 "table minimum size of {} elements exceeds table limits",
420 ty.limits.min
421 );
422 }
423
424 let minimum = minimum.ok_or_else(|| {
427 format_err!(
428 "table minimum size of {} elements exceeds table limits",
429 ty.limits.min
430 )
431 })?;
432 Ok((minimum, maximum))
433 }
434
435 pub fn element_type(&self) -> TableElementType {
437 match self {
438 Table::Static(StaticTable::Func(_)) | Table::Dynamic(DynamicTable::Func(_)) => {
439 TableElementType::Func
440 }
441 Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => {
442 TableElementType::GcRef
443 }
444 }
445 }
446
447 #[cfg(feature = "pooling-allocator")]
449 pub(crate) fn is_static(&self) -> bool {
450 matches!(self, Table::Static(_))
451 }
452
453 pub fn size(&self) -> usize {
455 match self {
456 Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size,
457 Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size,
458 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(),
459 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
460 elements.len()
461 }
462 }
463 }
464
465 pub fn maximum(&self) -> Option<usize> {
472 match self {
473 Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()),
474 Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()),
475 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum,
476 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum,
477 }
478 }
479
480 pub fn init_func(
486 &mut self,
487 dst: u64,
488 items: impl ExactSizeIterator<Item = Option<NonNull<VMFuncRef>>>,
489 ) -> Result<(), Trap> {
490 let dst = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
491
492 let (funcrefs, lazy_init) = self.funcrefs_mut();
493 let elements = funcrefs
494 .get_mut(dst..)
495 .and_then(|s| s.get_mut(..items.len()))
496 .ok_or(Trap::TableOutOfBounds)?;
497
498 for (item, slot) in items.zip(elements) {
499 *slot = TaggedFuncRef::from(item, lazy_init);
500 }
501 Ok(())
502 }
503
504 pub fn init_gc_refs(
508 &mut self,
509 dst: u64,
510 items: impl ExactSizeIterator<Item = Option<VMGcRef>>,
511 ) -> Result<(), Trap> {
512 let dst = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
513
514 let elements = self
515 .gc_refs_mut()
516 .get_mut(dst..)
517 .and_then(|s| s.get_mut(..items.len()))
518 .ok_or(Trap::TableOutOfBounds)?;
519
520 for (item, slot) in items.zip(elements) {
521 *slot = item;
522 }
523 Ok(())
524 }
525
526 pub fn fill(
535 &mut self,
536 gc_store: Option<&mut GcStore>,
537 dst: u64,
538 val: TableElement,
539 len: u64,
540 ) -> Result<(), Trap> {
541 let start = usize::try_from(dst).map_err(|_| Trap::TableOutOfBounds)?;
542 let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
543 let end = start
544 .checked_add(len)
545 .ok_or_else(|| Trap::TableOutOfBounds)?;
546
547 if end > self.size() {
548 return Err(Trap::TableOutOfBounds);
549 }
550
551 match val {
552 TableElement::FuncRef(f) => {
553 let (funcrefs, lazy_init) = self.funcrefs_mut();
554 funcrefs[start..end].fill(TaggedFuncRef::from(f, lazy_init));
555 }
556 TableElement::GcRef(r) => {
557 let gc_store =
558 gc_store.expect("must provide a GcStore for tables of GC references");
559
560 for slot in &mut self.gc_refs_mut()[start..end] {
562 gc_store.write_gc_ref(slot, r.as_ref());
563 }
564
565 if let Some(r) = r {
568 gc_store.drop_gc_ref(r);
569 }
570 }
571 TableElement::UninitFunc => {
572 let (funcrefs, _lazy_init) = self.funcrefs_mut();
573 funcrefs[start..end].fill(TaggedFuncRef::UNINIT);
574 }
575 }
576
577 Ok(())
578 }
579
580 pub unsafe fn grow(
601 &mut self,
602 delta: u64,
603 init_value: TableElement,
604 store: &mut dyn VMStore,
605 ) -> Result<Option<usize>, Error> {
606 let old_size = self.size();
607
608 if delta == 0 {
611 return Ok(Some(old_size));
612 }
613 let delta =
615 usize::try_from(delta).map_err(|_| format_err!("delta exceeds host pointer size"))?;
616
617 let new_size = match old_size.checked_add(delta) {
618 Some(s) => s,
619 None => {
620 store.table_grow_failed(format_err!("overflow calculating new table size"))?;
621 return Ok(None);
622 }
623 };
624
625 if !store.table_growing(old_size, new_size, self.maximum())? {
626 return Ok(None);
627 }
628
629 if let Some(max) = self.maximum() {
633 if new_size > max {
634 store.table_grow_failed(format_err!("Table maximum size exceeded"))?;
635 return Ok(None);
636 }
637 }
638
639 debug_assert!(self.type_matches(&init_value));
640
641 match self {
643 Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
644 unsafe {
645 debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
646 }
647 *size = new_size;
648 }
649 Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
650 unsafe {
651 debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
652 }
653 *size = new_size;
654 }
655
656 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
664 elements.resize(usize::try_from(new_size).unwrap(), None);
665 }
666 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
667 elements.resize_with(usize::try_from(new_size).unwrap(), || None);
668 }
669 }
670
671 self.fill(
672 store.store_opaque_mut().optional_gc_store_mut()?,
673 u64::try_from(old_size).unwrap(),
674 init_value,
675 u64::try_from(delta).unwrap(),
676 )
677 .expect("table should not be out of bounds");
678
679 Ok(Some(old_size))
680 }
681
682 pub fn get(&self, gc_store: Option<&mut GcStore>, index: u64) -> Option<TableElement> {
688 let index = usize::try_from(index).ok()?;
689 match self.element_type() {
690 TableElementType::Func => {
691 let (funcrefs, lazy_init) = self.funcrefs();
692 funcrefs
693 .get(index)
694 .copied()
695 .map(|e| e.into_table_element(lazy_init))
696 }
697 TableElementType::GcRef => self.gc_refs().get(index).map(|r| {
698 let r = r.as_ref().map(|r| gc_store.unwrap().clone_gc_ref(r));
699 TableElement::GcRef(r)
700 }),
701 }
702 }
703
704 pub fn set(&mut self, index: u64, elem: TableElement) -> Result<(), ()> {
715 let index: usize = index.try_into().map_err(|_| ())?;
716 match elem {
717 TableElement::FuncRef(f) => {
718 let (funcrefs, lazy_init) = self.funcrefs_mut();
719 *funcrefs.get_mut(index).ok_or(())? = TaggedFuncRef::from(f, lazy_init);
720 }
721 TableElement::UninitFunc => {
722 let (funcrefs, _lazy_init) = self.funcrefs_mut();
723 *funcrefs.get_mut(index).ok_or(())? = TaggedFuncRef::UNINIT;
724 }
725 TableElement::GcRef(e) => {
726 *self.gc_refs_mut().get_mut(index).ok_or(())? = e;
727 }
728 }
729 Ok(())
730 }
731
732 pub unsafe fn copy(
739 gc_store: Option<&mut GcStore>,
740 dst_table: *mut Self,
741 src_table: *mut Self,
742 dst_index: u64,
743 src_index: u64,
744 len: u64,
745 ) -> Result<(), Trap> {
746 let src_index = usize::try_from(src_index).map_err(|_| Trap::TableOutOfBounds)?;
749 let dst_index = usize::try_from(dst_index).map_err(|_| Trap::TableOutOfBounds)?;
750 let len = usize::try_from(len).map_err(|_| Trap::TableOutOfBounds)?;
751
752 if src_index
753 .checked_add(len)
754 .map_or(true, |n| n > (*src_table).size())
755 || dst_index
756 .checked_add(len)
757 .map_or(true, |m| m > (*dst_table).size())
758 {
759 return Err(Trap::TableOutOfBounds);
760 }
761
762 debug_assert!(
763 (*dst_table).element_type() == (*src_table).element_type(),
764 "table element type mismatch"
765 );
766
767 let src_range = src_index..src_index + len;
768 let dst_range = dst_index..dst_index + len;
769
770 if ptr::eq(dst_table, src_table) {
772 (*dst_table).copy_elements_within(gc_store, dst_range, src_range);
773 } else {
774 Self::copy_elements(gc_store, &mut *dst_table, &*src_table, dst_range, src_range);
775 }
776
777 Ok(())
778 }
779
780 pub fn vmtable(&mut self) -> VMTableDefinition {
782 match self {
783 Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
784 VMTableDefinition {
785 base: data.cast().into(),
786 current_elements: *size,
787 }
788 }
789 Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
790 VMTableDefinition {
791 base: data.cast().into(),
792 current_elements: *size,
793 }
794 }
795 Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
796 VMTableDefinition {
797 base: NonNull::<[FuncTableElem]>::from(&mut elements[..])
798 .cast()
799 .into(),
800 current_elements: elements.len(),
801 }
802 }
803 Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
804 VMTableDefinition {
805 base: NonNull::<[Option<VMGcRef>]>::from(&mut elements[..])
806 .cast()
807 .into(),
808 current_elements: elements.len(),
809 }
810 }
811 }
812 }
813
814 fn type_matches(&self, val: &TableElement) -> bool {
815 self.element_type().matches(val)
816 }
817
818 fn funcrefs(&self) -> (&[TaggedFuncRef], bool) {
819 assert_eq!(self.element_type(), TableElementType::Func);
820 match self {
821 Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
822 elements,
823 lazy_init,
824 ..
825 })) => (
826 unsafe { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) },
827 *lazy_init,
828 ),
829 Self::Static(StaticTable::Func(StaticFuncTable {
830 data,
831 size,
832 lazy_init,
833 })) => (
834 unsafe {
835 slice::from_raw_parts(data.as_ptr().cast(), usize::try_from(*size).unwrap())
836 },
837 *lazy_init,
838 ),
839 _ => unreachable!(),
840 }
841 }
842
843 fn funcrefs_mut(&mut self) -> (&mut [TaggedFuncRef], bool) {
844 assert_eq!(self.element_type(), TableElementType::Func);
845 match self {
846 Self::Dynamic(DynamicTable::Func(DynamicFuncTable {
847 elements,
848 lazy_init,
849 ..
850 })) => (
851 unsafe { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) },
852 *lazy_init,
853 ),
854 Self::Static(StaticTable::Func(StaticFuncTable {
855 data,
856 size,
857 lazy_init,
858 })) => (
859 unsafe {
860 slice::from_raw_parts_mut(data.as_ptr().cast(), usize::try_from(*size).unwrap())
861 },
862 *lazy_init,
863 ),
864 _ => unreachable!(),
865 }
866 }
867
868 fn gc_refs(&self) -> &[Option<VMGcRef>] {
869 assert_eq!(self.element_type(), TableElementType::GcRef);
870 match self {
871 Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
872 Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
873 &data.as_non_null().as_ref()[..usize::try_from(*size).unwrap()]
874 },
875 _ => unreachable!(),
876 }
877 }
878
879 pub fn gc_refs_mut(&mut self) -> &mut [Option<VMGcRef>] {
883 assert_eq!(self.element_type(), TableElementType::GcRef);
884 match self {
885 Self::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => elements,
886 Self::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => unsafe {
887 &mut data.as_non_null().as_mut()[..usize::try_from(*size).unwrap()]
888 },
889 _ => unreachable!(),
890 }
891 }
892
893 fn copy_elements(
894 gc_store: Option<&mut GcStore>,
895 dst_table: &mut Self,
896 src_table: &Self,
897 dst_range: Range<usize>,
898 src_range: Range<usize>,
899 ) {
900 debug_assert!(!ptr::eq(dst_table, src_table));
902
903 let ty = dst_table.element_type();
904
905 match ty {
906 TableElementType::Func => {
907 let (dst_funcrefs, _lazy_init) = dst_table.funcrefs_mut();
909 let (src_funcrefs, _lazy_init) = src_table.funcrefs();
910 dst_funcrefs[dst_range].copy_from_slice(&src_funcrefs[src_range]);
911 }
912 TableElementType::GcRef => {
913 assert_eq!(
914 dst_range.end - dst_range.start,
915 src_range.end - src_range.start
916 );
917 assert!(dst_range.end <= dst_table.gc_refs().len());
918 assert!(src_range.end <= src_table.gc_refs().len());
919 let gc_store = gc_store.unwrap();
920 for (dst, src) in dst_range.zip(src_range) {
921 gc_store.write_gc_ref(
922 &mut dst_table.gc_refs_mut()[dst],
923 src_table.gc_refs()[src].as_ref(),
924 );
925 }
926 }
927 }
928 }
929
930 fn copy_elements_within(
931 &mut self,
932 gc_store: Option<&mut GcStore>,
933 dst_range: Range<usize>,
934 src_range: Range<usize>,
935 ) {
936 assert_eq!(
937 dst_range.end - dst_range.start,
938 src_range.end - src_range.start
939 );
940
941 if src_range.start == dst_range.start {
943 return;
944 }
945
946 let ty = self.element_type();
947 match ty {
948 TableElementType::Func => {
949 let (funcrefs, _lazy_init) = self.funcrefs_mut();
951 funcrefs.copy_within(src_range, dst_range.start);
952 }
953 TableElementType::GcRef => {
954 let gc_store = gc_store.unwrap();
955
956 let elements = self.gc_refs_mut();
959 if dst_range.start < src_range.start {
960 for (d, s) in dst_range.zip(src_range) {
961 let (ds, ss) = elements.split_at_mut(s);
962 let dst = &mut ds[d];
963 let src = ss[0].as_ref();
964 gc_store.write_gc_ref(dst, src);
965 }
966 } else {
967 for (s, d) in src_range.rev().zip(dst_range.rev()) {
968 let (ss, ds) = elements.split_at_mut(d);
969 let dst = &mut ds[0];
970 let src = ss[s].as_ref();
971 gc_store.write_gc_ref(dst, src);
972 }
973 }
974 }
975 }
976 }
977}
978
979impl Default for Table {
981 fn default() -> Self {
982 Self::from(StaticFuncTable {
983 data: SendSyncPtr::new(NonNull::from(&mut [])),
984 size: 0,
985 lazy_init: false,
986 })
987 }
988}