1use super::invoke_wasm_and_catch_traps;
2use crate::prelude::*;
3use crate::runtime::vm::VMFuncRef;
4use crate::store::{AutoAssertNoGc, StoreOpaque};
5use crate::{
6 AsContext, AsContextMut, Engine, Func, FuncType, HeapType, NoFunc, RefType, StoreContextMut,
7 ValRaw, ValType,
8};
9use core::ffi::c_void;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::ptr::{self, NonNull};
13use wasmtime_environ::VMSharedTypeIndex;
14
15pub struct TypedFunc<Params, Results> {
24 _a: marker::PhantomData<fn(Params) -> Results>,
25 ty: FuncType,
26 func: Func,
27}
28
29impl<Params, Results> Clone for TypedFunc<Params, Results> {
30 fn clone(&self) -> TypedFunc<Params, Results> {
31 Self {
32 _a: marker::PhantomData,
33 ty: self.ty.clone(),
34 func: self.func,
35 }
36 }
37}
38
39impl<Params, Results> TypedFunc<Params, Results>
40where
41 Params: WasmParams,
42 Results: WasmResults,
43{
44 pub unsafe fn new_unchecked(store: impl AsContext, func: Func) -> TypedFunc<Params, Results> {
57 let store = store.as_context().0;
58 Self::_new_unchecked(store, func)
59 }
60
61 pub(crate) unsafe fn _new_unchecked(
62 store: &StoreOpaque,
63 func: Func,
64 ) -> TypedFunc<Params, Results> {
65 let ty = func.load_ty(store);
66 TypedFunc {
67 _a: marker::PhantomData,
68 ty,
69 func,
70 }
71 }
72
73 pub fn func(&self) -> &Func {
76 &self.func
77 }
78
79 #[inline]
97 pub fn call(&self, mut store: impl AsContextMut, params: Params) -> Result<Results> {
98 let mut store = store.as_context_mut();
99 assert!(
100 !store.0.async_support(),
101 "must use `call_async` with async stores"
102 );
103
104 #[cfg(feature = "gc")]
105 if Self::need_gc_before_call_raw(store.0, ¶ms) {
106 store.gc(None);
107 }
108
109 let func = self.func.vm_func_ref(store.0);
110 unsafe { Self::call_raw(&mut store, &self.ty, func, params) }
111 }
112
113 #[cfg(feature = "async")]
131 pub async fn call_async(
132 &self,
133 mut store: impl AsContextMut<Data: Send>,
134 params: Params,
135 ) -> Result<Results>
136 where
137 Params: Sync,
138 Results: Sync,
139 {
140 let mut store = store.as_context_mut();
141 assert!(
142 store.0.async_support(),
143 "must use `call` with non-async stores"
144 );
145
146 #[cfg(feature = "gc")]
147 if Self::need_gc_before_call_raw(store.0, ¶ms) {
148 store.gc_async(None).await?;
149 }
150
151 store
152 .on_fiber(|store| {
153 let func = self.func.vm_func_ref(store.0);
154 unsafe { Self::call_raw(store, &self.ty, func, params) }
155 })
156 .await?
157 }
158
159 #[inline]
160 #[cfg(feature = "gc")]
161 pub(crate) fn need_gc_before_call_raw(_store: &StoreOpaque, _params: &Params) -> bool {
162 {
163 let num_gc_refs = _params.vmgcref_pointing_to_object_count();
165 if let Some(num_gc_refs) = core::num::NonZeroUsize::new(num_gc_refs) {
166 return _store
167 .unwrap_gc_store()
168 .gc_heap
169 .need_gc_before_entering_wasm(num_gc_refs);
170 }
171 }
172
173 false
174 }
175
176 pub(crate) unsafe fn call_raw<T>(
185 store: &mut StoreContextMut<'_, T>,
186 ty: &FuncType,
187 func: ptr::NonNull<VMFuncRef>,
188 params: Params,
189 ) -> Result<Results> {
190 if cfg!(debug_assertions) {
193 Self::debug_typecheck(store.0, func.as_ref().type_index);
194 }
195
196 union Storage<T: Copy, U: Copy> {
201 params: MaybeUninit<T>,
202 results: U,
203 }
204
205 let mut storage = Storage::<Params::ValRawStorage, Results::ValRawStorage> {
206 params: MaybeUninit::uninit(),
207 };
208
209 {
210 let mut store = AutoAssertNoGc::new(store.0);
211 params.store(&mut store, ty, &mut storage.params)?;
212 }
213
214 let mut captures = (func, storage);
220
221 let result = invoke_wasm_and_catch_traps(store, |caller, vm| {
222 let (func_ref, storage) = &mut captures;
223 let storage_len = mem::size_of_val::<Storage<_, _>>(storage) / mem::size_of::<ValRaw>();
224 let storage: *mut Storage<_, _> = storage;
225 let storage = storage.cast::<ValRaw>();
226 let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len);
227 let storage = NonNull::new(storage).unwrap();
228 VMFuncRef::array_call(*func_ref, vm, caller, storage)
229 });
230
231 let (_, storage) = captures;
232 result?;
233
234 let mut store = AutoAssertNoGc::new(store.0);
235 Ok(Results::load(&mut store, &storage.results))
236 }
237
238 fn debug_typecheck(store: &StoreOpaque, func: VMSharedTypeIndex) {
240 let ty = FuncType::from_shared_type_index(store.engine(), func);
241 Params::typecheck(store.engine(), ty.params(), TypeCheckPosition::Param)
242 .expect("params should match");
243 Results::typecheck(store.engine(), ty.results(), TypeCheckPosition::Result)
244 .expect("results should match");
245 }
246}
247
248#[doc(hidden)]
249#[derive(Copy, Clone)]
250pub enum TypeCheckPosition {
251 Param,
252 Result,
253}
254
255pub unsafe trait WasmTy: Send {
264 #[doc(hidden)]
268 #[inline]
269 fn typecheck(engine: &Engine, actual: ValType, position: TypeCheckPosition) -> Result<()> {
270 let expected = Self::valtype();
271 debug_assert!(expected.comes_from_same_engine(engine));
272 debug_assert!(actual.comes_from_same_engine(engine));
273 match position {
274 TypeCheckPosition::Result => actual.ensure_matches(engine, &expected),
277 TypeCheckPosition::Param => match (expected.as_ref(), actual.as_ref()) {
280 (Some(expected_ref), Some(actual_ref)) if actual_ref.heap_type().is_concrete() => {
310 expected_ref
311 .heap_type()
312 .top()
313 .ensure_matches(engine, &actual_ref.heap_type().top())
314 }
315 _ => expected.ensure_matches(engine, &actual),
316 },
317 }
318 }
319
320 #[doc(hidden)]
322 fn valtype() -> ValType;
323
324 #[doc(hidden)]
325 fn may_gc() -> bool {
326 match Self::valtype() {
327 ValType::Ref(_) => true,
328 ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => false,
329 }
330 }
331
332 #[doc(hidden)]
335 fn compatible_with_store(&self, store: &StoreOpaque) -> bool;
336
337 #[doc(hidden)]
344 fn dynamic_concrete_type_check(
345 &self,
346 store: &StoreOpaque,
347 nullable: bool,
348 actual: &HeapType,
349 ) -> Result<()>;
350
351 #[doc(hidden)]
359 #[inline]
360 fn is_vmgcref_and_points_to_object(&self) -> bool {
361 Self::valtype().is_vmgcref_type_and_points_to_object()
362 }
363
364 #[doc(hidden)]
393 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()>;
394
395 #[doc(hidden)]
402 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self;
403}
404
405macro_rules! integers {
406 ($($primitive:ident/$get_primitive:ident => $ty:ident)*) => ($(
407 unsafe impl WasmTy for $primitive {
408 #[inline]
409 fn valtype() -> ValType {
410 ValType::$ty
411 }
412 #[inline]
413 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
414 true
415 }
416 #[inline]
417 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
418 unreachable!()
419 }
420 #[inline]
421 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
422 ptr.write(ValRaw::$primitive(self));
423 Ok(())
424 }
425 #[inline]
426 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
427 ptr.$get_primitive()
428 }
429 }
430 )*)
431}
432
433integers! {
434 i32/get_i32 => I32
435 i64/get_i64 => I64
436 u32/get_u32 => I32
437 u64/get_u64 => I64
438}
439
440macro_rules! floats {
441 ($($float:ident/$int:ident/$get_float:ident => $ty:ident)*) => ($(
442 unsafe impl WasmTy for $float {
443 #[inline]
444 fn valtype() -> ValType {
445 ValType::$ty
446 }
447 #[inline]
448 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
449 true
450 }
451 #[inline]
452 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
453 unreachable!()
454 }
455 #[inline]
456 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
457 ptr.write(ValRaw::$float(self.to_bits()));
458 Ok(())
459 }
460 #[inline]
461 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
462 $float::from_bits(ptr.$get_float())
463 }
464 }
465 )*)
466}
467
468floats! {
469 f32/u32/get_f32 => F32
470 f64/u64/get_f64 => F64
471}
472
473unsafe impl WasmTy for NoFunc {
474 #[inline]
475 fn valtype() -> ValType {
476 ValType::Ref(RefType::new(false, HeapType::NoFunc))
477 }
478
479 #[inline]
480 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
481 match self._inner {}
482 }
483
484 #[inline]
485 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
486 match self._inner {}
487 }
488
489 #[inline]
490 fn is_vmgcref_and_points_to_object(&self) -> bool {
491 match self._inner {}
492 }
493
494 #[inline]
495 fn store(self, _store: &mut AutoAssertNoGc<'_>, _ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
496 match self._inner {}
497 }
498
499 #[inline]
500 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
501 unreachable!("NoFunc is uninhabited")
502 }
503}
504
505unsafe impl WasmTy for Option<NoFunc> {
506 #[inline]
507 fn valtype() -> ValType {
508 ValType::Ref(RefType::new(true, HeapType::NoFunc))
509 }
510
511 #[inline]
512 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
513 true
514 }
515
516 #[inline]
517 fn dynamic_concrete_type_check(
518 &self,
519 _: &StoreOpaque,
520 nullable: bool,
521 ty: &HeapType,
522 ) -> Result<()> {
523 if nullable {
524 Ok(())
526 } else {
527 bail!("argument type mismatch: expected non-nullable (ref {ty}), found null reference")
528 }
529 }
530
531 #[inline]
532 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
533 ptr.write(ValRaw::funcref(ptr::null_mut()));
534 Ok(())
535 }
536
537 #[inline]
538 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
539 None
540 }
541}
542
543unsafe impl WasmTy for Func {
544 #[inline]
545 fn valtype() -> ValType {
546 ValType::Ref(RefType::new(false, HeapType::Func))
547 }
548
549 #[inline]
550 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
551 self.store == store.id()
552 }
553
554 #[inline]
555 fn dynamic_concrete_type_check(
556 &self,
557 store: &StoreOpaque,
558 _nullable: bool,
559 expected: &HeapType,
560 ) -> Result<()> {
561 let expected = expected.unwrap_concrete_func();
562 self.ensure_matches_ty(store, expected)
563 .context("argument type mismatch for reference to concrete type")
564 }
565
566 #[inline]
567 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
568 let abi = self.vm_func_ref(store);
569 ptr.write(ValRaw::funcref(abi.cast::<c_void>().as_ptr()));
570 Ok(())
571 }
572
573 #[inline]
574 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
575 let p = NonNull::new(ptr.get_funcref()).unwrap().cast();
576 Func::from_vm_func_ref(store, p)
577 }
578}
579
580unsafe impl WasmTy for Option<Func> {
581 #[inline]
582 fn valtype() -> ValType {
583 ValType::FUNCREF
584 }
585
586 #[inline]
587 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
588 if let Some(f) = self {
589 f.compatible_with_store(store)
590 } else {
591 true
592 }
593 }
594
595 fn dynamic_concrete_type_check(
596 &self,
597 store: &StoreOpaque,
598 nullable: bool,
599 expected: &HeapType,
600 ) -> Result<()> {
601 if let Some(f) = self {
602 let expected = expected.unwrap_concrete_func();
603 f.ensure_matches_ty(store, expected)
604 .context("argument type mismatch for reference to concrete type")
605 } else if nullable {
606 Ok(())
607 } else {
608 bail!(
609 "argument type mismatch: expected non-nullable (ref {expected}), found null reference"
610 )
611 }
612 }
613
614 #[inline]
615 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
616 let raw = if let Some(f) = self {
617 f.vm_func_ref(store).as_ptr()
618 } else {
619 ptr::null_mut()
620 };
621 ptr.write(ValRaw::funcref(raw.cast::<c_void>()));
622 Ok(())
623 }
624
625 #[inline]
626 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
627 let ptr = NonNull::new(ptr.get_funcref())?.cast();
628 Some(Func::from_vm_func_ref(store, ptr))
629 }
630}
631
632pub unsafe trait WasmParams: Send {
638 #[doc(hidden)]
639 type ValRawStorage: Copy;
640
641 #[doc(hidden)]
642 fn typecheck(
643 engine: &Engine,
644 params: impl ExactSizeIterator<Item = crate::ValType>,
645 position: TypeCheckPosition,
646 ) -> Result<()>;
647
648 #[doc(hidden)]
649 fn vmgcref_pointing_to_object_count(&self) -> usize;
650
651 #[doc(hidden)]
652 fn store(
653 self,
654 store: &mut AutoAssertNoGc<'_>,
655 func_ty: &FuncType,
656 dst: &mut MaybeUninit<Self::ValRawStorage>,
657 ) -> Result<()>;
658}
659
660unsafe impl<T> WasmParams for T
663where
664 T: WasmTy,
665{
666 type ValRawStorage = <(T,) as WasmParams>::ValRawStorage;
667
668 fn typecheck(
669 engine: &Engine,
670 params: impl ExactSizeIterator<Item = crate::ValType>,
671 position: TypeCheckPosition,
672 ) -> Result<()> {
673 <(T,) as WasmParams>::typecheck(engine, params, position)
674 }
675
676 #[inline]
677 fn vmgcref_pointing_to_object_count(&self) -> usize {
678 T::is_vmgcref_and_points_to_object(self) as usize
679 }
680
681 #[inline]
682 fn store(
683 self,
684 store: &mut AutoAssertNoGc<'_>,
685 func_ty: &FuncType,
686 dst: &mut MaybeUninit<Self::ValRawStorage>,
687 ) -> Result<()> {
688 <(T,) as WasmParams>::store((self,), store, func_ty, dst)
689 }
690}
691
692macro_rules! impl_wasm_params {
693 ($n:tt $($t:ident)*) => {
694 #[allow(non_snake_case)]
695 unsafe impl<$($t: WasmTy,)*> WasmParams for ($($t,)*) {
696 type ValRawStorage = [ValRaw; $n];
697
698 fn typecheck(
699 _engine: &Engine,
700 mut params: impl ExactSizeIterator<Item = crate::ValType>,
701 _position: TypeCheckPosition,
702 ) -> Result<()> {
703 let mut _n = 0;
704
705 $(
706 match params.next() {
707 Some(t) => {
708 _n += 1;
709 $t::typecheck(_engine, t, _position)?
710 },
711 None => bail!("expected {} types, found {}", $n, params.len() + _n),
712 }
713 )*
714
715 match params.next() {
716 None => Ok(()),
717 Some(_) => {
718 _n += 1;
719 bail!("expected {} types, found {}", $n, params.len() + _n)
720 },
721 }
722 }
723
724 #[inline]
725 fn vmgcref_pointing_to_object_count(&self) -> usize {
726 let ($($t,)*) = self;
727 0 $(
728 + $t.is_vmgcref_and_points_to_object() as usize
729 )*
730 }
731
732
733 #[inline]
734 fn store(
735 self,
736 _store: &mut AutoAssertNoGc<'_>,
737 _func_ty: &FuncType,
738 _ptr: &mut MaybeUninit<Self::ValRawStorage>,
739 ) -> Result<()> {
740 let ($($t,)*) = self;
741
742 let mut _i = 0;
743 $(
744 if !$t.compatible_with_store(_store) {
745 bail!("attempt to pass cross-`Store` value to Wasm as function argument");
746 }
747
748 if $t::valtype().is_ref() {
749 let param_ty = _func_ty.param(_i).unwrap();
750 let ref_ty = param_ty.unwrap_ref();
751 let heap_ty = ref_ty.heap_type();
752 if heap_ty.is_concrete() {
753 $t.dynamic_concrete_type_check(_store, ref_ty.is_nullable(), heap_ty)?;
754 }
755 }
756
757 let dst = map_maybe_uninit!(_ptr[_i]);
758 $t.store(_store, dst)?;
759
760 _i += 1;
761 )*
762 Ok(())
763 }
764 }
765 };
766}
767
768for_each_function_signature!(impl_wasm_params);
769
770pub unsafe trait WasmResults: WasmParams {
773 #[doc(hidden)]
774 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self;
775}
776
777unsafe impl<T: WasmTy> WasmResults for T {
779 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
780 <(T,) as WasmResults>::load(store, abi).0
781 }
782}
783
784macro_rules! impl_wasm_results {
785 ($n:tt $($t:ident)*) => {
786 #[allow(non_snake_case, unused_variables)]
787 unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) {
788 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
789 let [$($t,)*] = abi;
790 ($($t::load(store, $t),)*)
791 }
792 }
793 };
794}
795
796for_each_function_signature!(impl_wasm_results);