1use super::invoke_wasm_and_catch_traps;
2use crate::prelude::*;
3use crate::runtime::vm::{VMFuncRef, VMOpaqueContext};
4use crate::store::{AutoAssertNoGc, StoreOpaque};
5use crate::{
6 AsContext, AsContextMut, Engine, Func, FuncType, HeapType, NoFunc, RefType, StoreContextMut,
7 ValRaw, ValType,
8};
9use core::ffi::c_void;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::ptr::{self, NonNull};
13use wasmtime_environ::VMSharedTypeIndex;
14
15pub struct TypedFunc<Params, Results> {
24 _a: marker::PhantomData<fn(Params) -> Results>,
25 ty: FuncType,
26 func: Func,
27}
28
29impl<Params, Results> Clone for TypedFunc<Params, Results> {
30 fn clone(&self) -> TypedFunc<Params, Results> {
31 Self {
32 _a: marker::PhantomData,
33 ty: self.ty.clone(),
34 func: self.func,
35 }
36 }
37}
38
39impl<Params, Results> TypedFunc<Params, Results>
40where
41 Params: WasmParams,
42 Results: WasmResults,
43{
44 pub unsafe fn new_unchecked(store: impl AsContext, func: Func) -> TypedFunc<Params, Results> {
57 let store = store.as_context().0;
58 Self::_new_unchecked(store, func)
59 }
60
61 pub(crate) unsafe fn _new_unchecked(
62 store: &StoreOpaque,
63 func: Func,
64 ) -> TypedFunc<Params, Results> {
65 let ty = func.load_ty(store);
66 TypedFunc {
67 _a: marker::PhantomData,
68 ty,
69 func,
70 }
71 }
72
73 pub fn func(&self) -> &Func {
76 &self.func
77 }
78
79 #[inline]
97 pub fn call(&self, mut store: impl AsContextMut, params: Params) -> Result<Results> {
98 let mut store = store.as_context_mut();
99 assert!(
100 !store.0.async_support(),
101 "must use `call_async` with async stores"
102 );
103
104 #[cfg(feature = "gc")]
105 if Self::need_gc_before_call_raw(store.0, ¶ms) {
106 store.gc(None);
107 }
108
109 let func = self.func.vm_func_ref(store.0);
110 unsafe { Self::call_raw(&mut store, &self.ty, func, params) }
111 }
112
113 #[cfg(feature = "async")]
131 pub async fn call_async(
132 &self,
133 mut store: impl AsContextMut<Data: Send>,
134 params: Params,
135 ) -> Result<Results> {
136 let mut store = store.as_context_mut();
137 assert!(
138 store.0.async_support(),
139 "must use `call` with non-async stores"
140 );
141
142 #[cfg(feature = "gc")]
143 if Self::need_gc_before_call_raw(store.0, ¶ms) {
144 store.gc_async(None).await?;
145 }
146
147 store
148 .on_fiber(|store| {
149 let func = self.func.vm_func_ref(store.0);
150 unsafe { Self::call_raw(store, &self.ty, func, params) }
151 })
152 .await?
153 }
154
155 #[inline]
156 #[cfg(feature = "gc")]
157 pub(crate) fn need_gc_before_call_raw(_store: &StoreOpaque, _params: &Params) -> bool {
158 {
159 let num_gc_refs = _params.vmgcref_pointing_to_object_count();
161 if let Some(num_gc_refs) = core::num::NonZeroUsize::new(num_gc_refs) {
162 return _store
163 .unwrap_gc_store()
164 .gc_heap
165 .need_gc_before_entering_wasm(num_gc_refs);
166 }
167 }
168
169 false
170 }
171
172 pub(crate) unsafe fn call_raw<T>(
181 store: &mut StoreContextMut<'_, T>,
182 ty: &FuncType,
183 func: ptr::NonNull<VMFuncRef>,
184 params: Params,
185 ) -> Result<Results> {
186 if cfg!(debug_assertions) {
189 Self::debug_typecheck(store.0, func.as_ref().type_index);
190 }
191
192 union Storage<T: Copy, U: Copy> {
197 params: MaybeUninit<T>,
198 results: U,
199 }
200
201 let mut storage = Storage::<Params::ValRawStorage, Results::ValRawStorage> {
202 params: MaybeUninit::uninit(),
203 };
204
205 {
206 let mut store = AutoAssertNoGc::new(store.0);
207 params.store(&mut store, ty, &mut storage.params)?;
208 }
209
210 let mut captures = (func, storage);
216
217 let result = invoke_wasm_and_catch_traps(store, |caller, vm| {
218 let (func_ref, storage) = &mut captures;
219 let storage_len = mem::size_of_val::<Storage<_, _>>(storage) / mem::size_of::<ValRaw>();
220 let storage: *mut Storage<_, _> = storage;
221 let storage = storage.cast::<ValRaw>();
222 let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len);
223 let storage = NonNull::new(storage).unwrap();
224 func_ref
225 .as_ref()
226 .array_call(vm, VMOpaqueContext::from_vmcontext(caller), storage)
227 });
228
229 let (_, storage) = captures;
230 result?;
231
232 let mut store = AutoAssertNoGc::new(store.0);
233 Ok(Results::load(&mut store, &storage.results))
234 }
235
236 fn debug_typecheck(store: &StoreOpaque, func: VMSharedTypeIndex) {
238 let ty = FuncType::from_shared_type_index(store.engine(), func);
239 Params::typecheck(store.engine(), ty.params(), TypeCheckPosition::Param)
240 .expect("params should match");
241 Results::typecheck(store.engine(), ty.results(), TypeCheckPosition::Result)
242 .expect("results should match");
243 }
244}
245
246#[doc(hidden)]
247#[derive(Copy, Clone)]
248pub enum TypeCheckPosition {
249 Param,
250 Result,
251}
252
253pub unsafe trait WasmTy: Send {
262 #[doc(hidden)]
266 #[inline]
267 fn typecheck(engine: &Engine, actual: ValType, position: TypeCheckPosition) -> Result<()> {
268 let expected = Self::valtype();
269 debug_assert!(expected.comes_from_same_engine(engine));
270 debug_assert!(actual.comes_from_same_engine(engine));
271 match position {
272 TypeCheckPosition::Result => actual.ensure_matches(engine, &expected),
275 TypeCheckPosition::Param => match (expected.as_ref(), actual.as_ref()) {
278 (Some(expected_ref), Some(actual_ref)) if actual_ref.heap_type().is_concrete() => {
308 expected_ref
309 .heap_type()
310 .top()
311 .ensure_matches(engine, &actual_ref.heap_type().top())
312 }
313 _ => expected.ensure_matches(engine, &actual),
314 },
315 }
316 }
317
318 #[doc(hidden)]
320 fn valtype() -> ValType;
321
322 #[doc(hidden)]
323 fn may_gc() -> bool {
324 match Self::valtype() {
325 ValType::Ref(_) => true,
326 ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => false,
327 }
328 }
329
330 #[doc(hidden)]
333 fn compatible_with_store(&self, store: &StoreOpaque) -> bool;
334
335 #[doc(hidden)]
342 fn dynamic_concrete_type_check(
343 &self,
344 store: &StoreOpaque,
345 nullable: bool,
346 actual: &HeapType,
347 ) -> Result<()>;
348
349 #[doc(hidden)]
357 #[inline]
358 fn is_vmgcref_and_points_to_object(&self) -> bool {
359 Self::valtype().is_vmgcref_type_and_points_to_object()
360 }
361
362 #[doc(hidden)]
391 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()>;
392
393 #[doc(hidden)]
400 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self;
401}
402
403macro_rules! integers {
404 ($($primitive:ident/$get_primitive:ident => $ty:ident)*) => ($(
405 unsafe impl WasmTy for $primitive {
406 #[inline]
407 fn valtype() -> ValType {
408 ValType::$ty
409 }
410 #[inline]
411 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
412 true
413 }
414 #[inline]
415 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
416 unreachable!()
417 }
418 #[inline]
419 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
420 ptr.write(ValRaw::$primitive(self));
421 Ok(())
422 }
423 #[inline]
424 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
425 ptr.$get_primitive()
426 }
427 }
428 )*)
429}
430
431integers! {
432 i32/get_i32 => I32
433 i64/get_i64 => I64
434 u32/get_u32 => I32
435 u64/get_u64 => I64
436}
437
438macro_rules! floats {
439 ($($float:ident/$int:ident/$get_float:ident => $ty:ident)*) => ($(
440 unsafe impl WasmTy for $float {
441 #[inline]
442 fn valtype() -> ValType {
443 ValType::$ty
444 }
445 #[inline]
446 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
447 true
448 }
449 #[inline]
450 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
451 unreachable!()
452 }
453 #[inline]
454 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
455 ptr.write(ValRaw::$float(self.to_bits()));
456 Ok(())
457 }
458 #[inline]
459 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
460 $float::from_bits(ptr.$get_float())
461 }
462 }
463 )*)
464}
465
466floats! {
467 f32/u32/get_f32 => F32
468 f64/u64/get_f64 => F64
469}
470
471unsafe impl WasmTy for NoFunc {
472 #[inline]
473 fn valtype() -> ValType {
474 ValType::Ref(RefType::new(false, HeapType::NoFunc))
475 }
476
477 #[inline]
478 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
479 match self._inner {}
480 }
481
482 #[inline]
483 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
484 match self._inner {}
485 }
486
487 #[inline]
488 fn is_vmgcref_and_points_to_object(&self) -> bool {
489 match self._inner {}
490 }
491
492 #[inline]
493 fn store(self, _store: &mut AutoAssertNoGc<'_>, _ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
494 match self._inner {}
495 }
496
497 #[inline]
498 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
499 unreachable!("NoFunc is uninhabited")
500 }
501}
502
503unsafe impl WasmTy for Option<NoFunc> {
504 #[inline]
505 fn valtype() -> ValType {
506 ValType::Ref(RefType::new(true, HeapType::NoFunc))
507 }
508
509 #[inline]
510 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
511 true
512 }
513
514 #[inline]
515 fn dynamic_concrete_type_check(
516 &self,
517 _: &StoreOpaque,
518 nullable: bool,
519 ty: &HeapType,
520 ) -> Result<()> {
521 if nullable {
522 Ok(())
524 } else {
525 bail!("argument type mismatch: expected non-nullable (ref {ty}), found null reference")
526 }
527 }
528
529 #[inline]
530 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
531 ptr.write(ValRaw::funcref(ptr::null_mut()));
532 Ok(())
533 }
534
535 #[inline]
536 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
537 None
538 }
539}
540
541unsafe impl WasmTy for Func {
542 #[inline]
543 fn valtype() -> ValType {
544 ValType::Ref(RefType::new(false, HeapType::Func))
545 }
546
547 #[inline]
548 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
549 self.store == store.id()
550 }
551
552 #[inline]
553 fn dynamic_concrete_type_check(
554 &self,
555 store: &StoreOpaque,
556 _nullable: bool,
557 expected: &HeapType,
558 ) -> Result<()> {
559 let expected = expected.unwrap_concrete_func();
560 self.ensure_matches_ty(store, expected)
561 .context("argument type mismatch for reference to concrete type")
562 }
563
564 #[inline]
565 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
566 let abi = self.vm_func_ref(store);
567 ptr.write(ValRaw::funcref(abi.cast::<c_void>().as_ptr()));
568 Ok(())
569 }
570
571 #[inline]
572 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
573 let p = NonNull::new(ptr.get_funcref()).unwrap().cast();
574 Func::from_vm_func_ref(store, p)
575 }
576}
577
578unsafe impl WasmTy for Option<Func> {
579 #[inline]
580 fn valtype() -> ValType {
581 ValType::FUNCREF
582 }
583
584 #[inline]
585 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
586 if let Some(f) = self {
587 f.compatible_with_store(store)
588 } else {
589 true
590 }
591 }
592
593 fn dynamic_concrete_type_check(
594 &self,
595 store: &StoreOpaque,
596 nullable: bool,
597 expected: &HeapType,
598 ) -> Result<()> {
599 if let Some(f) = self {
600 let expected = expected.unwrap_concrete_func();
601 f.ensure_matches_ty(store, expected)
602 .context("argument type mismatch for reference to concrete type")
603 } else if nullable {
604 Ok(())
605 } else {
606 bail!(
607 "argument type mismatch: expected non-nullable (ref {expected}), found null reference"
608 )
609 }
610 }
611
612 #[inline]
613 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
614 let raw = if let Some(f) = self {
615 f.vm_func_ref(store).as_ptr()
616 } else {
617 ptr::null_mut()
618 };
619 ptr.write(ValRaw::funcref(raw.cast::<c_void>()));
620 Ok(())
621 }
622
623 #[inline]
624 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
625 let ptr = NonNull::new(ptr.get_funcref())?.cast();
626 Some(Func::from_vm_func_ref(store, ptr))
627 }
628}
629
630pub unsafe trait WasmParams: Send {
636 #[doc(hidden)]
637 type ValRawStorage: Copy;
638
639 #[doc(hidden)]
640 fn typecheck(
641 engine: &Engine,
642 params: impl ExactSizeIterator<Item = crate::ValType>,
643 position: TypeCheckPosition,
644 ) -> Result<()>;
645
646 #[doc(hidden)]
647 fn vmgcref_pointing_to_object_count(&self) -> usize;
648
649 #[doc(hidden)]
650 fn store(
651 self,
652 store: &mut AutoAssertNoGc<'_>,
653 func_ty: &FuncType,
654 dst: &mut MaybeUninit<Self::ValRawStorage>,
655 ) -> Result<()>;
656}
657
658unsafe impl<T> WasmParams for T
661where
662 T: WasmTy,
663{
664 type ValRawStorage = <(T,) as WasmParams>::ValRawStorage;
665
666 fn typecheck(
667 engine: &Engine,
668 params: impl ExactSizeIterator<Item = crate::ValType>,
669 position: TypeCheckPosition,
670 ) -> Result<()> {
671 <(T,) as WasmParams>::typecheck(engine, params, position)
672 }
673
674 #[inline]
675 fn vmgcref_pointing_to_object_count(&self) -> usize {
676 T::is_vmgcref_and_points_to_object(self) as usize
677 }
678
679 #[inline]
680 fn store(
681 self,
682 store: &mut AutoAssertNoGc<'_>,
683 func_ty: &FuncType,
684 dst: &mut MaybeUninit<Self::ValRawStorage>,
685 ) -> Result<()> {
686 <(T,) as WasmParams>::store((self,), store, func_ty, dst)
687 }
688}
689
690macro_rules! impl_wasm_params {
691 ($n:tt $($t:ident)*) => {
692 #[allow(non_snake_case)]
693 unsafe impl<$($t: WasmTy,)*> WasmParams for ($($t,)*) {
694 type ValRawStorage = [ValRaw; $n];
695
696 fn typecheck(
697 _engine: &Engine,
698 mut params: impl ExactSizeIterator<Item = crate::ValType>,
699 _position: TypeCheckPosition,
700 ) -> Result<()> {
701 let mut _n = 0;
702
703 $(
704 match params.next() {
705 Some(t) => {
706 _n += 1;
707 $t::typecheck(_engine, t, _position)?
708 },
709 None => bail!("expected {} types, found {}", $n, params.len() + _n),
710 }
711 )*
712
713 match params.next() {
714 None => Ok(()),
715 Some(_) => {
716 _n += 1;
717 bail!("expected {} types, found {}", $n, params.len() + _n)
718 },
719 }
720 }
721
722 #[inline]
723 fn vmgcref_pointing_to_object_count(&self) -> usize {
724 let ($($t,)*) = self;
725 0 $(
726 + $t.is_vmgcref_and_points_to_object() as usize
727 )*
728 }
729
730
731 #[inline]
732 fn store(
733 self,
734 _store: &mut AutoAssertNoGc<'_>,
735 _func_ty: &FuncType,
736 _ptr: &mut MaybeUninit<Self::ValRawStorage>,
737 ) -> Result<()> {
738 let ($($t,)*) = self;
739
740 let mut _i = 0;
741 $(
742 if !$t.compatible_with_store(_store) {
743 bail!("attempt to pass cross-`Store` value to Wasm as function argument");
744 }
745
746 if $t::valtype().is_ref() {
747 let param_ty = _func_ty.param(_i).unwrap();
748 let ref_ty = param_ty.unwrap_ref();
749 let heap_ty = ref_ty.heap_type();
750 if heap_ty.is_concrete() {
751 $t.dynamic_concrete_type_check(_store, ref_ty.is_nullable(), heap_ty)?;
752 }
753 }
754
755 let dst = map_maybe_uninit!(_ptr[_i]);
756 $t.store(_store, dst)?;
757
758 _i += 1;
759 )*
760 Ok(())
761 }
762 }
763 };
764}
765
766for_each_function_signature!(impl_wasm_params);
767
768pub unsafe trait WasmResults: WasmParams {
771 #[doc(hidden)]
772 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self;
773}
774
775unsafe impl<T: WasmTy> WasmResults for T {
777 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
778 <(T,) as WasmResults>::load(store, abi).0
779 }
780}
781
782macro_rules! impl_wasm_results {
783 ($n:tt $($t:ident)*) => {
784 #[allow(non_snake_case, unused_variables)]
785 unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) {
786 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
787 let [$($t,)*] = abi;
788 ($($t::load(store, $t),)*)
789 }
790 }
791 };
792}
793
794for_each_function_signature!(impl_wasm_results);