1use super::invoke_wasm_and_catch_traps;
2use crate::prelude::*;
3use crate::runtime::vm::{VMFuncRef, VMOpaqueContext};
4use crate::store::{AutoAssertNoGc, StoreOpaque};
5use crate::{
6 AsContext, AsContextMut, Engine, Func, FuncType, HeapType, NoFunc, RefType, StoreContextMut,
7 ValRaw, ValType,
8};
9use core::ffi::c_void;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::ptr::{self, NonNull};
13use wasmtime_environ::VMSharedTypeIndex;
14
15pub struct TypedFunc<Params, Results> {
24 _a: marker::PhantomData<fn(Params) -> Results>,
25 ty: FuncType,
26 func: Func,
27}
28
29impl<Params, Results> Clone for TypedFunc<Params, Results> {
30 fn clone(&self) -> TypedFunc<Params, Results> {
31 Self {
32 _a: marker::PhantomData,
33 ty: self.ty.clone(),
34 func: self.func,
35 }
36 }
37}
38
39impl<Params, Results> TypedFunc<Params, Results>
40where
41 Params: WasmParams,
42 Results: WasmResults,
43{
44 pub unsafe fn new_unchecked(store: impl AsContext, func: Func) -> TypedFunc<Params, Results> {
57 let store = store.as_context().0;
58 Self::_new_unchecked(store, func)
59 }
60
61 pub(crate) unsafe fn _new_unchecked(
62 store: &StoreOpaque,
63 func: Func,
64 ) -> TypedFunc<Params, Results> {
65 let ty = func.load_ty(store);
66 TypedFunc {
67 _a: marker::PhantomData,
68 ty,
69 func,
70 }
71 }
72
73 pub fn func(&self) -> &Func {
76 &self.func
77 }
78
79 pub fn call(&self, mut store: impl AsContextMut, params: Params) -> Result<Results> {
97 let mut store = store.as_context_mut();
98 assert!(
99 !store.0.async_support(),
100 "must use `call_async` with async stores"
101 );
102
103 #[cfg(feature = "gc")]
104 if Self::need_gc_before_call_raw(store.0, ¶ms) {
105 store.gc(None);
106 }
107
108 let func = self.func.vm_func_ref(store.0);
109 unsafe { Self::call_raw(&mut store, &self.ty, func, params) }
110 }
111
112 #[cfg(feature = "async")]
130 pub async fn call_async<T>(
131 &self,
132 mut store: impl AsContextMut<Data = T>,
133 params: Params,
134 ) -> Result<Results>
135 where
136 T: Send,
137 {
138 let mut store = store.as_context_mut();
139 assert!(
140 store.0.async_support(),
141 "must use `call` with non-async stores"
142 );
143
144 #[cfg(feature = "gc")]
145 if Self::need_gc_before_call_raw(store.0, ¶ms) {
146 store.gc_async(None).await?;
147 }
148
149 store
150 .on_fiber(|store| {
151 let func = self.func.vm_func_ref(store.0);
152 unsafe { Self::call_raw(store, &self.ty, func, params) }
153 })
154 .await?
155 }
156
157 #[inline]
158 #[cfg(feature = "gc")]
159 pub(crate) fn need_gc_before_call_raw(_store: &StoreOpaque, _params: &Params) -> bool {
160 {
161 let num_gc_refs = _params.vmgcref_pointing_to_object_count();
163 if let Some(num_gc_refs) = core::num::NonZeroUsize::new(num_gc_refs) {
164 return _store
165 .unwrap_gc_store()
166 .gc_heap
167 .need_gc_before_entering_wasm(num_gc_refs);
168 }
169 }
170
171 false
172 }
173
174 pub(crate) unsafe fn call_raw<T>(
183 store: &mut StoreContextMut<'_, T>,
184 ty: &FuncType,
185 func: ptr::NonNull<VMFuncRef>,
186 params: Params,
187 ) -> Result<Results> {
188 if cfg!(debug_assertions) {
191 Self::debug_typecheck(store.0, func.as_ref().type_index);
192 }
193
194 union Storage<T: Copy, U: Copy> {
199 params: MaybeUninit<T>,
200 results: U,
201 }
202
203 let mut storage = Storage::<Params::ValRawStorage, Results::ValRawStorage> {
204 params: MaybeUninit::uninit(),
205 };
206
207 {
208 let mut store = AutoAssertNoGc::new(store.0);
209 params.store(&mut store, ty, &mut storage.params)?;
210 }
211
212 let mut captures = (func, storage);
218
219 let result = invoke_wasm_and_catch_traps(store, |caller, vm| {
220 let (func_ref, storage) = &mut captures;
221 let storage_len = mem::size_of_val::<Storage<_, _>>(storage) / mem::size_of::<ValRaw>();
222 let storage: *mut Storage<_, _> = storage;
223 let storage = storage.cast::<ValRaw>();
224 let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len);
225 let storage = NonNull::new(storage).unwrap();
226 func_ref
227 .as_ref()
228 .array_call(vm, VMOpaqueContext::from_vmcontext(caller), storage)
229 });
230
231 let (_, storage) = captures;
232 result?;
233
234 let mut store = AutoAssertNoGc::new(store.0);
235 Ok(Results::load(&mut store, &storage.results))
236 }
237
238 fn debug_typecheck(store: &StoreOpaque, func: VMSharedTypeIndex) {
240 let ty = FuncType::from_shared_type_index(store.engine(), func);
241 Params::typecheck(store.engine(), ty.params(), TypeCheckPosition::Param)
242 .expect("params should match");
243 Results::typecheck(store.engine(), ty.results(), TypeCheckPosition::Result)
244 .expect("results should match");
245 }
246}
247
248#[doc(hidden)]
249#[derive(Copy, Clone)]
250pub enum TypeCheckPosition {
251 Param,
252 Result,
253}
254
255pub unsafe trait WasmTy: Send {
264 #[doc(hidden)]
268 #[inline]
269 fn typecheck(engine: &Engine, actual: ValType, position: TypeCheckPosition) -> Result<()> {
270 let expected = Self::valtype();
271 debug_assert!(expected.comes_from_same_engine(engine));
272 debug_assert!(actual.comes_from_same_engine(engine));
273 match position {
274 TypeCheckPosition::Result => actual.ensure_matches(engine, &expected),
277 TypeCheckPosition::Param => match (expected.as_ref(), actual.as_ref()) {
280 (Some(expected_ref), Some(actual_ref)) if actual_ref.heap_type().is_concrete() => {
310 expected_ref
311 .heap_type()
312 .top()
313 .ensure_matches(engine, &actual_ref.heap_type().top())
314 }
315 _ => expected.ensure_matches(engine, &actual),
316 },
317 }
318 }
319
320 #[doc(hidden)]
322 fn valtype() -> ValType;
323
324 #[doc(hidden)]
325 fn may_gc() -> bool {
326 match Self::valtype() {
327 ValType::Ref(_) => true,
328 ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => false,
329 }
330 }
331
332 #[doc(hidden)]
335 fn compatible_with_store(&self, store: &StoreOpaque) -> bool;
336
337 #[doc(hidden)]
344 fn dynamic_concrete_type_check(
345 &self,
346 store: &StoreOpaque,
347 nullable: bool,
348 actual: &HeapType,
349 ) -> Result<()>;
350
351 #[doc(hidden)]
359 #[inline]
360 fn is_vmgcref_and_points_to_object(&self) -> bool {
361 Self::valtype().is_vmgcref_type_and_points_to_object()
362 }
363
364 #[doc(hidden)]
393 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()>;
394
395 #[doc(hidden)]
402 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self;
403}
404
405macro_rules! integers {
406 ($($primitive:ident/$get_primitive:ident => $ty:ident)*) => ($(
407 unsafe impl WasmTy for $primitive {
408 #[inline]
409 fn valtype() -> ValType {
410 ValType::$ty
411 }
412 #[inline]
413 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
414 true
415 }
416 #[inline]
417 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
418 unreachable!()
419 }
420 #[inline]
421 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
422 ptr.write(ValRaw::$primitive(self));
423 Ok(())
424 }
425 #[inline]
426 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
427 ptr.$get_primitive()
428 }
429 }
430 )*)
431}
432
433integers! {
434 i32/get_i32 => I32
435 i64/get_i64 => I64
436 u32/get_u32 => I32
437 u64/get_u64 => I64
438}
439
440macro_rules! floats {
441 ($($float:ident/$int:ident/$get_float:ident => $ty:ident)*) => ($(
442 unsafe impl WasmTy for $float {
443 #[inline]
444 fn valtype() -> ValType {
445 ValType::$ty
446 }
447 #[inline]
448 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
449 true
450 }
451 #[inline]
452 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
453 unreachable!()
454 }
455 #[inline]
456 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
457 ptr.write(ValRaw::$float(self.to_bits()));
458 Ok(())
459 }
460 #[inline]
461 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
462 $float::from_bits(ptr.$get_float())
463 }
464 }
465 )*)
466}
467
468floats! {
469 f32/u32/get_f32 => F32
470 f64/u64/get_f64 => F64
471}
472
473unsafe impl WasmTy for NoFunc {
474 #[inline]
475 fn valtype() -> ValType {
476 ValType::Ref(RefType::new(false, HeapType::NoFunc))
477 }
478
479 #[inline]
480 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
481 match self._inner {}
482 }
483
484 #[inline]
485 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
486 match self._inner {}
487 }
488
489 #[inline]
490 fn is_vmgcref_and_points_to_object(&self) -> bool {
491 match self._inner {}
492 }
493
494 #[inline]
495 fn store(self, _store: &mut AutoAssertNoGc<'_>, _ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
496 match self._inner {}
497 }
498
499 #[inline]
500 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
501 unreachable!("NoFunc is uninhabited")
502 }
503}
504
505unsafe impl WasmTy for Option<NoFunc> {
506 #[inline]
507 fn valtype() -> ValType {
508 ValType::Ref(RefType::new(true, HeapType::NoFunc))
509 }
510
511 #[inline]
512 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
513 true
514 }
515
516 #[inline]
517 fn dynamic_concrete_type_check(
518 &self,
519 _: &StoreOpaque,
520 nullable: bool,
521 ty: &HeapType,
522 ) -> Result<()> {
523 if nullable {
524 Ok(())
526 } else {
527 bail!("argument type mismatch: expected non-nullable (ref {ty}), found null reference")
528 }
529 }
530
531 #[inline]
532 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
533 ptr.write(ValRaw::funcref(ptr::null_mut()));
534 Ok(())
535 }
536
537 #[inline]
538 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
539 None
540 }
541}
542
543unsafe impl WasmTy for Func {
544 #[inline]
545 fn valtype() -> ValType {
546 ValType::Ref(RefType::new(false, HeapType::Func))
547 }
548
549 #[inline]
550 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
551 store.store_data().contains(self.0)
552 }
553
554 #[inline]
555 fn dynamic_concrete_type_check(
556 &self,
557 store: &StoreOpaque,
558 _nullable: bool,
559 expected: &HeapType,
560 ) -> Result<()> {
561 let expected = expected.unwrap_concrete_func();
562 self.ensure_matches_ty(store, expected)
563 .context("argument type mismatch for reference to concrete type")
564 }
565
566 #[inline]
567 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
568 let abi = self.vm_func_ref(store);
569 ptr.write(ValRaw::funcref(abi.cast::<c_void>().as_ptr()));
570 Ok(())
571 }
572
573 #[inline]
574 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
575 let p = NonNull::new(ptr.get_funcref()).unwrap().cast();
576 Func::from_vm_func_ref(store, p)
577 }
578}
579
580unsafe impl WasmTy for Option<Func> {
581 #[inline]
582 fn valtype() -> ValType {
583 ValType::FUNCREF
584 }
585
586 #[inline]
587 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
588 if let Some(f) = self {
589 store.store_data().contains(f.0)
590 } else {
591 true
592 }
593 }
594
595 fn dynamic_concrete_type_check(
596 &self,
597 store: &StoreOpaque,
598 nullable: bool,
599 expected: &HeapType,
600 ) -> Result<()> {
601 if let Some(f) = self {
602 let expected = expected.unwrap_concrete_func();
603 f.ensure_matches_ty(store, expected)
604 .context("argument type mismatch for reference to concrete type")
605 } else if nullable {
606 Ok(())
607 } else {
608 bail!("argument type mismatch: expected non-nullable (ref {expected}), found null reference")
609 }
610 }
611
612 #[inline]
613 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
614 let raw = if let Some(f) = self {
615 f.vm_func_ref(store).as_ptr()
616 } else {
617 ptr::null_mut()
618 };
619 ptr.write(ValRaw::funcref(raw.cast::<c_void>()));
620 Ok(())
621 }
622
623 #[inline]
624 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
625 let ptr = NonNull::new(ptr.get_funcref())?.cast();
626 Some(Func::from_vm_func_ref(store, ptr))
627 }
628}
629
630pub unsafe trait WasmParams: Send {
636 #[doc(hidden)]
637 type ValRawStorage: Copy;
638
639 #[doc(hidden)]
640 fn typecheck(
641 engine: &Engine,
642 params: impl ExactSizeIterator<Item = crate::ValType>,
643 position: TypeCheckPosition,
644 ) -> Result<()>;
645
646 #[doc(hidden)]
647 fn vmgcref_pointing_to_object_count(&self) -> usize;
648
649 #[doc(hidden)]
650 fn store(
651 self,
652 store: &mut AutoAssertNoGc<'_>,
653 func_ty: &FuncType,
654 dst: &mut MaybeUninit<Self::ValRawStorage>,
655 ) -> Result<()>;
656}
657
658unsafe impl<T> WasmParams for T
661where
662 T: WasmTy,
663{
664 type ValRawStorage = <(T,) as WasmParams>::ValRawStorage;
665
666 fn typecheck(
667 engine: &Engine,
668 params: impl ExactSizeIterator<Item = crate::ValType>,
669 position: TypeCheckPosition,
670 ) -> Result<()> {
671 <(T,) as WasmParams>::typecheck(engine, params, position)
672 }
673
674 #[inline]
675 fn vmgcref_pointing_to_object_count(&self) -> usize {
676 T::is_vmgcref_and_points_to_object(self) as usize
677 }
678
679 #[inline]
680 fn store(
681 self,
682 store: &mut AutoAssertNoGc<'_>,
683 func_ty: &FuncType,
684 dst: &mut MaybeUninit<Self::ValRawStorage>,
685 ) -> Result<()> {
686 <(T,) as WasmParams>::store((self,), store, func_ty, dst)
687 }
688}
689
690macro_rules! impl_wasm_params {
691 ($n:tt $($t:ident)*) => {
692 #[allow(non_snake_case)]
693 unsafe impl<$($t: WasmTy,)*> WasmParams for ($($t,)*) {
694 type ValRawStorage = [ValRaw; $n];
695
696 fn typecheck(
697 _engine: &Engine,
698 mut params: impl ExactSizeIterator<Item = crate::ValType>,
699 _position: TypeCheckPosition,
700 ) -> Result<()> {
701 let mut _n = 0;
702
703 $(
704 match params.next() {
705 Some(t) => {
706 _n += 1;
707 $t::typecheck(_engine, t, _position)?
708 },
709 None => bail!("expected {} types, found {}", $n, params.len() + _n),
710 }
711 )*
712
713 match params.next() {
714 None => Ok(()),
715 Some(_) => {
716 _n += 1;
717 bail!("expected {} types, found {}", $n, params.len() + _n)
718 },
719 }
720 }
721
722 #[inline]
723 fn vmgcref_pointing_to_object_count(&self) -> usize {
724 let ($($t,)*) = self;
725 0 $(
726 + $t.is_vmgcref_and_points_to_object() as usize
727 )*
728 }
729
730
731 #[inline]
732 fn store(
733 self,
734 _store: &mut AutoAssertNoGc<'_>,
735 _func_ty: &FuncType,
736 _ptr: &mut MaybeUninit<Self::ValRawStorage>,
737 ) -> Result<()> {
738 let ($($t,)*) = self;
739
740 let mut _i = 0;
741 $(
742 if !$t.compatible_with_store(_store) {
743 bail!("attempt to pass cross-`Store` value to Wasm as function argument");
744 }
745
746 if $t::valtype().is_ref() {
747 let param_ty = _func_ty.param(_i).unwrap();
748 let ref_ty = param_ty.unwrap_ref();
749 let heap_ty = ref_ty.heap_type();
750 if heap_ty.is_concrete() {
751 $t.dynamic_concrete_type_check(_store, ref_ty.is_nullable(), heap_ty)?;
752 }
753 }
754
755 let dst = map_maybe_uninit!(_ptr[_i]);
756 $t.store(_store, dst)?;
757
758 _i += 1;
759 )*
760 Ok(())
761 }
762 }
763 };
764}
765
766for_each_function_signature!(impl_wasm_params);
767
768pub unsafe trait WasmResults: WasmParams {
771 #[doc(hidden)]
772 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self;
773}
774
775unsafe impl<T: WasmTy> WasmResults for T {
777 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
778 <(T,) as WasmResults>::load(store, abi).0
779 }
780}
781
782macro_rules! impl_wasm_results {
783 ($n:tt $($t:ident)*) => {
784 #[allow(non_snake_case, unused_variables)]
785 unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) {
786 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
787 let [$($t,)*] = abi;
788 ($($t::load(store, $t),)*)
789 }
790 }
791 };
792}
793
794for_each_function_signature!(impl_wasm_results);