1use super::invoke_wasm_and_catch_traps;
2use crate::prelude::*;
3use crate::runtime::vm::VMFuncRef;
4use crate::store::{AutoAssertNoGc, StoreOpaque};
5use crate::{
6 AsContext, AsContextMut, Engine, Func, FuncType, HeapType, NoFunc, RefType, StoreContextMut,
7 ValRaw, ValType,
8};
9use core::ffi::c_void;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::ptr::{self, NonNull};
13use wasmtime_environ::VMSharedTypeIndex;
14
15pub struct TypedFunc<Params, Results> {
24 _a: marker::PhantomData<fn(Params) -> Results>,
25 ty: FuncType,
26 func: Func,
27}
28
29impl<Params, Results> Clone for TypedFunc<Params, Results> {
30 fn clone(&self) -> TypedFunc<Params, Results> {
31 Self {
32 _a: marker::PhantomData,
33 ty: self.ty.clone(),
34 func: self.func,
35 }
36 }
37}
38
39impl<Params, Results> TypedFunc<Params, Results>
40where
41 Params: WasmParams,
42 Results: WasmResults,
43{
44 pub unsafe fn new_unchecked(store: impl AsContext, func: Func) -> TypedFunc<Params, Results> {
57 let store = store.as_context().0;
58 unsafe { Self::_new_unchecked(store, func) }
59 }
60
61 pub(crate) unsafe fn _new_unchecked(
62 store: &StoreOpaque,
63 func: Func,
64 ) -> TypedFunc<Params, Results> {
65 let ty = func.load_ty(store);
66 TypedFunc {
67 _a: marker::PhantomData,
68 ty,
69 func,
70 }
71 }
72
73 pub fn func(&self) -> &Func {
76 &self.func
77 }
78
79 #[inline]
97 pub fn call(&self, mut store: impl AsContextMut, params: Params) -> Result<Results> {
98 let mut store = store.as_context_mut();
99 assert!(
100 !store.0.async_support(),
101 "must use `call_async` with async stores"
102 );
103
104 let func = self.func.vm_func_ref(store.0);
105 unsafe { Self::call_raw(&mut store, &self.ty, func, params) }
106 }
107
108 #[cfg(feature = "async")]
126 pub async fn call_async(
127 &self,
128 mut store: impl AsContextMut<Data: Send>,
129 params: Params,
130 ) -> Result<Results>
131 where
132 Params: Sync,
133 Results: Sync,
134 {
135 let mut store = store.as_context_mut();
136 assert!(
137 store.0.async_support(),
138 "must use `call` with non-async stores"
139 );
140
141 store
142 .on_fiber(|store| {
143 let func = self.func.vm_func_ref(store.0);
144 unsafe { Self::call_raw(store, &self.ty, func, params) }
145 })
146 .await?
147 }
148
149 pub(crate) unsafe fn call_raw<T>(
156 store: &mut StoreContextMut<'_, T>,
157 ty: &FuncType,
158 func: ptr::NonNull<VMFuncRef>,
159 params: Params,
160 ) -> Result<Results> {
161 unsafe {
167 if cfg!(debug_assertions) {
168 Self::debug_typecheck(store.0, func.as_ref().type_index);
169 }
170 }
171
172 union Storage<T: Copy, U: Copy> {
177 params: MaybeUninit<T>,
178 results: U,
179 }
180
181 let mut storage = Storage::<Params::ValRawStorage, Results::ValRawStorage> {
182 params: MaybeUninit::uninit(),
183 };
184
185 {
186 let mut store = AutoAssertNoGc::new(store.0);
187 let dst: &mut MaybeUninit<_> = unsafe { &mut storage.params };
191 params.store(&mut store, ty, dst)?;
192 }
193
194 let mut captures = (func, storage);
200
201 let result = invoke_wasm_and_catch_traps(store, |caller, vm| {
202 let (func_ref, storage) = &mut captures;
203 let storage_len = mem::size_of_val::<Storage<_, _>>(storage) / mem::size_of::<ValRaw>();
204 let storage: *mut Storage<_, _> = storage;
205 let storage = storage.cast::<ValRaw>();
206 let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len);
207 let storage = NonNull::new(storage).unwrap();
208
209 unsafe { VMFuncRef::array_call(*func_ref, vm, caller, storage) }
213 });
214
215 let (_, storage) = captures;
216 result?;
217
218 let mut store = AutoAssertNoGc::new(store.0);
219 unsafe { Ok(Results::load(&mut store, &storage.results)) }
224 }
225
226 fn debug_typecheck(store: &StoreOpaque, func: VMSharedTypeIndex) {
228 let ty = FuncType::from_shared_type_index(store.engine(), func);
229 Params::typecheck(store.engine(), ty.params(), TypeCheckPosition::Param)
230 .expect("params should match");
231 Results::typecheck(store.engine(), ty.results(), TypeCheckPosition::Result)
232 .expect("results should match");
233 }
234}
235
236#[doc(hidden)]
237#[derive(Copy, Clone)]
238pub enum TypeCheckPosition {
239 Param,
240 Result,
241}
242
243pub unsafe trait WasmTy: Send {
252 #[doc(hidden)]
256 #[inline]
257 fn typecheck(engine: &Engine, actual: ValType, position: TypeCheckPosition) -> Result<()> {
258 let expected = Self::valtype();
259 debug_assert!(expected.comes_from_same_engine(engine));
260 debug_assert!(actual.comes_from_same_engine(engine));
261 match position {
262 TypeCheckPosition::Result => actual.ensure_matches(engine, &expected),
265 TypeCheckPosition::Param => match (expected.as_ref(), actual.as_ref()) {
268 (Some(expected_ref), Some(actual_ref)) if actual_ref.heap_type().is_concrete() => {
298 expected_ref
299 .heap_type()
300 .top()
301 .ensure_matches(engine, &actual_ref.heap_type().top())
302 }
303 _ => expected.ensure_matches(engine, &actual),
304 },
305 }
306 }
307
308 #[doc(hidden)]
310 fn valtype() -> ValType;
311
312 #[doc(hidden)]
313 fn may_gc() -> bool {
314 match Self::valtype() {
315 ValType::Ref(_) => true,
316 ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => false,
317 }
318 }
319
320 #[doc(hidden)]
323 fn compatible_with_store(&self, store: &StoreOpaque) -> bool;
324
325 #[doc(hidden)]
332 fn dynamic_concrete_type_check(
333 &self,
334 store: &StoreOpaque,
335 nullable: bool,
336 actual: &HeapType,
337 ) -> Result<()>;
338
339 #[doc(hidden)]
347 #[inline]
348 fn is_vmgcref_and_points_to_object(&self) -> bool {
349 Self::valtype().is_vmgcref_type_and_points_to_object()
350 }
351
352 #[doc(hidden)]
381 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()>;
382
383 #[doc(hidden)]
390 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self;
391}
392
393macro_rules! integers {
394 ($($primitive:ident/$get_primitive:ident => $ty:ident)*) => ($(
395 unsafe impl WasmTy for $primitive {
396 #[inline]
397 fn valtype() -> ValType {
398 ValType::$ty
399 }
400 #[inline]
401 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
402 true
403 }
404 #[inline]
405 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
406 unreachable!()
407 }
408 #[inline]
409 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
410 ptr.write(ValRaw::$primitive(self));
411 Ok(())
412 }
413 #[inline]
414 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
415 ptr.$get_primitive()
416 }
417 }
418 )*)
419}
420
421integers! {
422 i32/get_i32 => I32
423 i64/get_i64 => I64
424 u32/get_u32 => I32
425 u64/get_u64 => I64
426}
427
428macro_rules! floats {
429 ($($float:ident/$int:ident/$get_float:ident => $ty:ident)*) => ($(
430 unsafe impl WasmTy for $float {
431 #[inline]
432 fn valtype() -> ValType {
433 ValType::$ty
434 }
435 #[inline]
436 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
437 true
438 }
439 #[inline]
440 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
441 unreachable!()
442 }
443 #[inline]
444 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
445 ptr.write(ValRaw::$float(self.to_bits()));
446 Ok(())
447 }
448 #[inline]
449 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
450 $float::from_bits(ptr.$get_float())
451 }
452 }
453 )*)
454}
455
456floats! {
457 f32/u32/get_f32 => F32
458 f64/u64/get_f64 => F64
459}
460
461unsafe impl WasmTy for NoFunc {
462 #[inline]
463 fn valtype() -> ValType {
464 ValType::Ref(RefType::new(false, HeapType::NoFunc))
465 }
466
467 #[inline]
468 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
469 match self._inner {}
470 }
471
472 #[inline]
473 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
474 match self._inner {}
475 }
476
477 #[inline]
478 fn is_vmgcref_and_points_to_object(&self) -> bool {
479 match self._inner {}
480 }
481
482 #[inline]
483 fn store(self, _store: &mut AutoAssertNoGc<'_>, _ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
484 match self._inner {}
485 }
486
487 #[inline]
488 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
489 unreachable!("NoFunc is uninhabited")
490 }
491}
492
493unsafe impl WasmTy for Option<NoFunc> {
494 #[inline]
495 fn valtype() -> ValType {
496 ValType::Ref(RefType::new(true, HeapType::NoFunc))
497 }
498
499 #[inline]
500 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
501 true
502 }
503
504 #[inline]
505 fn dynamic_concrete_type_check(
506 &self,
507 _: &StoreOpaque,
508 nullable: bool,
509 ty: &HeapType,
510 ) -> Result<()> {
511 if nullable {
512 Ok(())
514 } else {
515 bail!("argument type mismatch: expected non-nullable (ref {ty}), found null reference")
516 }
517 }
518
519 #[inline]
520 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
521 ptr.write(ValRaw::funcref(ptr::null_mut()));
522 Ok(())
523 }
524
525 #[inline]
526 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
527 None
528 }
529}
530
531unsafe impl WasmTy for Func {
532 #[inline]
533 fn valtype() -> ValType {
534 ValType::Ref(RefType::new(false, HeapType::Func))
535 }
536
537 #[inline]
538 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
539 self.store == store.id()
540 }
541
542 #[inline]
543 fn dynamic_concrete_type_check(
544 &self,
545 store: &StoreOpaque,
546 _nullable: bool,
547 expected: &HeapType,
548 ) -> Result<()> {
549 let expected = expected.unwrap_concrete_func();
550 self.ensure_matches_ty(store, expected)
551 .context("argument type mismatch for reference to concrete type")
552 }
553
554 #[inline]
555 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
556 let abi = self.vm_func_ref(store);
557 ptr.write(ValRaw::funcref(abi.cast::<c_void>().as_ptr()));
558 Ok(())
559 }
560
561 #[inline]
562 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
563 let p = NonNull::new(ptr.get_funcref()).unwrap().cast();
564
565 unsafe { Func::from_vm_func_ref(store.id(), p) }
568 }
569}
570
571unsafe impl WasmTy for Option<Func> {
572 #[inline]
573 fn valtype() -> ValType {
574 ValType::FUNCREF
575 }
576
577 #[inline]
578 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
579 if let Some(f) = self {
580 f.compatible_with_store(store)
581 } else {
582 true
583 }
584 }
585
586 fn dynamic_concrete_type_check(
587 &self,
588 store: &StoreOpaque,
589 nullable: bool,
590 expected: &HeapType,
591 ) -> Result<()> {
592 if let Some(f) = self {
593 let expected = expected.unwrap_concrete_func();
594 f.ensure_matches_ty(store, expected)
595 .context("argument type mismatch for reference to concrete type")
596 } else if nullable {
597 Ok(())
598 } else {
599 bail!(
600 "argument type mismatch: expected non-nullable (ref {expected}), found null reference"
601 )
602 }
603 }
604
605 #[inline]
606 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
607 let raw = if let Some(f) = self {
608 f.vm_func_ref(store).as_ptr()
609 } else {
610 ptr::null_mut()
611 };
612 ptr.write(ValRaw::funcref(raw.cast::<c_void>()));
613 Ok(())
614 }
615
616 #[inline]
617 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
618 let ptr = NonNull::new(ptr.get_funcref())?.cast();
619
620 unsafe { Some(Func::from_vm_func_ref(store.id(), ptr)) }
623 }
624}
625
626pub unsafe trait WasmParams: Send {
632 #[doc(hidden)]
633 type ValRawStorage: Copy;
634
635 #[doc(hidden)]
636 fn typecheck(
637 engine: &Engine,
638 params: impl ExactSizeIterator<Item = crate::ValType>,
639 position: TypeCheckPosition,
640 ) -> Result<()>;
641
642 #[doc(hidden)]
643 fn vmgcref_pointing_to_object_count(&self) -> usize;
644
645 #[doc(hidden)]
646 fn store(
647 self,
648 store: &mut AutoAssertNoGc<'_>,
649 func_ty: &FuncType,
650 dst: &mut MaybeUninit<Self::ValRawStorage>,
651 ) -> Result<()>;
652}
653
654unsafe impl<T> WasmParams for T
657where
658 T: WasmTy,
659{
660 type ValRawStorage = <(T,) as WasmParams>::ValRawStorage;
661
662 fn typecheck(
663 engine: &Engine,
664 params: impl ExactSizeIterator<Item = crate::ValType>,
665 position: TypeCheckPosition,
666 ) -> Result<()> {
667 <(T,) as WasmParams>::typecheck(engine, params, position)
668 }
669
670 #[inline]
671 fn vmgcref_pointing_to_object_count(&self) -> usize {
672 T::is_vmgcref_and_points_to_object(self) as usize
673 }
674
675 #[inline]
676 fn store(
677 self,
678 store: &mut AutoAssertNoGc<'_>,
679 func_ty: &FuncType,
680 dst: &mut MaybeUninit<Self::ValRawStorage>,
681 ) -> Result<()> {
682 <(T,) as WasmParams>::store((self,), store, func_ty, dst)
683 }
684}
685
686macro_rules! impl_wasm_params {
687 ($n:tt $($t:ident)*) => {
688 #[allow(non_snake_case, reason = "macro-generated code")]
689 unsafe impl<$($t: WasmTy,)*> WasmParams for ($($t,)*) {
690 type ValRawStorage = [ValRaw; $n];
691
692 fn typecheck(
693 _engine: &Engine,
694 mut params: impl ExactSizeIterator<Item = crate::ValType>,
695 _position: TypeCheckPosition,
696 ) -> Result<()> {
697 let mut _n = 0;
698
699 $(
700 match params.next() {
701 Some(t) => {
702 _n += 1;
703 $t::typecheck(_engine, t, _position)?
704 },
705 None => bail!("expected {} types, found {}", $n, params.len() + _n),
706 }
707 )*
708
709 match params.next() {
710 None => Ok(()),
711 Some(_) => {
712 _n += 1;
713 bail!("expected {} types, found {}", $n, params.len() + _n)
714 },
715 }
716 }
717
718 #[inline]
719 fn vmgcref_pointing_to_object_count(&self) -> usize {
720 let ($($t,)*) = self;
721 0 $(
722 + $t.is_vmgcref_and_points_to_object() as usize
723 )*
724 }
725
726
727 #[inline]
728 fn store(
729 self,
730 _store: &mut AutoAssertNoGc<'_>,
731 _func_ty: &FuncType,
732 _ptr: &mut MaybeUninit<Self::ValRawStorage>,
733 ) -> Result<()> {
734 let ($($t,)*) = self;
735
736 let mut _i = 0;
737 $(
738 if !$t.compatible_with_store(_store) {
739 bail!("attempt to pass cross-`Store` value to Wasm as function argument");
740 }
741
742 if $t::valtype().is_ref() {
743 let param_ty = _func_ty.param(_i).unwrap();
744 let ref_ty = param_ty.unwrap_ref();
745 let heap_ty = ref_ty.heap_type();
746 if heap_ty.is_concrete() {
747 $t.dynamic_concrete_type_check(_store, ref_ty.is_nullable(), heap_ty)?;
748 }
749 }
750
751 let dst = map_maybe_uninit!(_ptr[_i]);
752 $t.store(_store, dst)?;
753
754 _i += 1;
755 )*
756 Ok(())
757 }
758 }
759 };
760}
761
762for_each_function_signature!(impl_wasm_params);
763
764pub unsafe trait WasmResults: WasmParams {
767 #[doc(hidden)]
768 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self;
769}
770
771unsafe impl<T: WasmTy> WasmResults for T {
773 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
774 unsafe { <(T,) as WasmResults>::load(store, abi).0 }
777 }
778}
779
780macro_rules! impl_wasm_results {
781 ($n:tt $($t:ident)*) => {
782 #[allow(non_snake_case, reason = "macro-generated code")]
783 unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) {
784 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
785 let [$($t,)*] = abi;
786
787 (
788 $(unsafe { $t::load(_store, $t) },)*
791 )
792 }
793 }
794 };
795}
796
797for_each_function_signature!(impl_wasm_results);