1use super::invoke_wasm_and_catch_traps;
2use crate::prelude::*;
3use crate::runtime::vm::{VMFuncRef, VMOpaqueContext};
4use crate::store::{AutoAssertNoGc, StoreOpaque};
5use crate::{
6 AsContext, AsContextMut, Engine, Func, FuncType, HeapType, NoFunc, RefType, StoreContextMut,
7 ValRaw, ValType,
8};
9use core::ffi::c_void;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::ptr::{self, NonNull};
13use wasmtime_environ::VMSharedTypeIndex;
14
15pub struct TypedFunc<Params, Results> {
24 _a: marker::PhantomData<fn(Params) -> Results>,
25 ty: FuncType,
26 func: Func,
27}
28
29impl<Params, Results> Clone for TypedFunc<Params, Results> {
30 fn clone(&self) -> TypedFunc<Params, Results> {
31 Self {
32 _a: marker::PhantomData,
33 ty: self.ty.clone(),
34 func: self.func,
35 }
36 }
37}
38
39impl<Params, Results> TypedFunc<Params, Results>
40where
41 Params: WasmParams,
42 Results: WasmResults,
43{
44 pub unsafe fn new_unchecked(store: impl AsContext, func: Func) -> TypedFunc<Params, Results> {
57 let store = store.as_context().0;
58 Self::_new_unchecked(store, func)
59 }
60
61 pub(crate) unsafe fn _new_unchecked(
62 store: &StoreOpaque,
63 func: Func,
64 ) -> TypedFunc<Params, Results> {
65 let ty = func.load_ty(store);
66 TypedFunc {
67 _a: marker::PhantomData,
68 ty,
69 func,
70 }
71 }
72
73 pub fn func(&self) -> &Func {
76 &self.func
77 }
78
79 #[inline]
97 pub fn call(&self, mut store: impl AsContextMut, params: Params) -> Result<Results> {
98 let mut store = store.as_context_mut();
99 assert!(
100 !store.0.async_support(),
101 "must use `call_async` with async stores"
102 );
103
104 #[cfg(feature = "gc")]
105 if Self::need_gc_before_call_raw(store.0, ¶ms) {
106 store.gc(None);
107 }
108
109 let func = self.func.vm_func_ref(store.0);
110 unsafe { Self::call_raw(&mut store, &self.ty, func, params) }
111 }
112
113 #[cfg(feature = "async")]
131 pub async fn call_async<T>(
132 &self,
133 mut store: impl AsContextMut<Data = T>,
134 params: Params,
135 ) -> Result<Results>
136 where
137 T: Send,
138 {
139 let mut store = store.as_context_mut();
140 assert!(
141 store.0.async_support(),
142 "must use `call` with non-async stores"
143 );
144
145 #[cfg(feature = "gc")]
146 if Self::need_gc_before_call_raw(store.0, ¶ms) {
147 store.gc_async(None).await?;
148 }
149
150 store
151 .on_fiber(|store| {
152 let func = self.func.vm_func_ref(store.0);
153 unsafe { Self::call_raw(store, &self.ty, func, params) }
154 })
155 .await?
156 }
157
158 #[inline]
159 #[cfg(feature = "gc")]
160 pub(crate) fn need_gc_before_call_raw(_store: &StoreOpaque, _params: &Params) -> bool {
161 {
162 let num_gc_refs = _params.vmgcref_pointing_to_object_count();
164 if let Some(num_gc_refs) = core::num::NonZeroUsize::new(num_gc_refs) {
165 return _store
166 .unwrap_gc_store()
167 .gc_heap
168 .need_gc_before_entering_wasm(num_gc_refs);
169 }
170 }
171
172 false
173 }
174
175 pub(crate) unsafe fn call_raw<T>(
184 store: &mut StoreContextMut<'_, T>,
185 ty: &FuncType,
186 func: ptr::NonNull<VMFuncRef>,
187 params: Params,
188 ) -> Result<Results> {
189 if cfg!(debug_assertions) {
192 Self::debug_typecheck(store.0, func.as_ref().type_index);
193 }
194
195 union Storage<T: Copy, U: Copy> {
200 params: MaybeUninit<T>,
201 results: U,
202 }
203
204 let mut storage = Storage::<Params::ValRawStorage, Results::ValRawStorage> {
205 params: MaybeUninit::uninit(),
206 };
207
208 {
209 let mut store = AutoAssertNoGc::new(store.0);
210 params.store(&mut store, ty, &mut storage.params)?;
211 }
212
213 let mut captures = (func, storage);
219
220 let result = invoke_wasm_and_catch_traps(store, |caller, vm| {
221 let (func_ref, storage) = &mut captures;
222 let storage_len = mem::size_of_val::<Storage<_, _>>(storage) / mem::size_of::<ValRaw>();
223 let storage: *mut Storage<_, _> = storage;
224 let storage = storage.cast::<ValRaw>();
225 let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len);
226 let storage = NonNull::new(storage).unwrap();
227 func_ref
228 .as_ref()
229 .array_call(vm, VMOpaqueContext::from_vmcontext(caller), storage)
230 });
231
232 let (_, storage) = captures;
233 result?;
234
235 let mut store = AutoAssertNoGc::new(store.0);
236 Ok(Results::load(&mut store, &storage.results))
237 }
238
239 fn debug_typecheck(store: &StoreOpaque, func: VMSharedTypeIndex) {
241 let ty = FuncType::from_shared_type_index(store.engine(), func);
242 Params::typecheck(store.engine(), ty.params(), TypeCheckPosition::Param)
243 .expect("params should match");
244 Results::typecheck(store.engine(), ty.results(), TypeCheckPosition::Result)
245 .expect("results should match");
246 }
247}
248
249#[doc(hidden)]
250#[derive(Copy, Clone)]
251pub enum TypeCheckPosition {
252 Param,
253 Result,
254}
255
256pub unsafe trait WasmTy: Send {
265 #[doc(hidden)]
269 #[inline]
270 fn typecheck(engine: &Engine, actual: ValType, position: TypeCheckPosition) -> Result<()> {
271 let expected = Self::valtype();
272 debug_assert!(expected.comes_from_same_engine(engine));
273 debug_assert!(actual.comes_from_same_engine(engine));
274 match position {
275 TypeCheckPosition::Result => actual.ensure_matches(engine, &expected),
278 TypeCheckPosition::Param => match (expected.as_ref(), actual.as_ref()) {
281 (Some(expected_ref), Some(actual_ref)) if actual_ref.heap_type().is_concrete() => {
311 expected_ref
312 .heap_type()
313 .top()
314 .ensure_matches(engine, &actual_ref.heap_type().top())
315 }
316 _ => expected.ensure_matches(engine, &actual),
317 },
318 }
319 }
320
321 #[doc(hidden)]
323 fn valtype() -> ValType;
324
325 #[doc(hidden)]
326 fn may_gc() -> bool {
327 match Self::valtype() {
328 ValType::Ref(_) => true,
329 ValType::I32 | ValType::I64 | ValType::F32 | ValType::F64 | ValType::V128 => false,
330 }
331 }
332
333 #[doc(hidden)]
336 fn compatible_with_store(&self, store: &StoreOpaque) -> bool;
337
338 #[doc(hidden)]
345 fn dynamic_concrete_type_check(
346 &self,
347 store: &StoreOpaque,
348 nullable: bool,
349 actual: &HeapType,
350 ) -> Result<()>;
351
352 #[doc(hidden)]
360 #[inline]
361 fn is_vmgcref_and_points_to_object(&self) -> bool {
362 Self::valtype().is_vmgcref_type_and_points_to_object()
363 }
364
365 #[doc(hidden)]
394 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()>;
395
396 #[doc(hidden)]
403 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self;
404}
405
406macro_rules! integers {
407 ($($primitive:ident/$get_primitive:ident => $ty:ident)*) => ($(
408 unsafe impl WasmTy for $primitive {
409 #[inline]
410 fn valtype() -> ValType {
411 ValType::$ty
412 }
413 #[inline]
414 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
415 true
416 }
417 #[inline]
418 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
419 unreachable!()
420 }
421 #[inline]
422 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
423 ptr.write(ValRaw::$primitive(self));
424 Ok(())
425 }
426 #[inline]
427 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
428 ptr.$get_primitive()
429 }
430 }
431 )*)
432}
433
434integers! {
435 i32/get_i32 => I32
436 i64/get_i64 => I64
437 u32/get_u32 => I32
438 u64/get_u64 => I64
439}
440
441macro_rules! floats {
442 ($($float:ident/$int:ident/$get_float:ident => $ty:ident)*) => ($(
443 unsafe impl WasmTy for $float {
444 #[inline]
445 fn valtype() -> ValType {
446 ValType::$ty
447 }
448 #[inline]
449 fn compatible_with_store(&self, _: &StoreOpaque) -> bool {
450 true
451 }
452 #[inline]
453 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
454 unreachable!()
455 }
456 #[inline]
457 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
458 ptr.write(ValRaw::$float(self.to_bits()));
459 Ok(())
460 }
461 #[inline]
462 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
463 $float::from_bits(ptr.$get_float())
464 }
465 }
466 )*)
467}
468
469floats! {
470 f32/u32/get_f32 => F32
471 f64/u64/get_f64 => F64
472}
473
474unsafe impl WasmTy for NoFunc {
475 #[inline]
476 fn valtype() -> ValType {
477 ValType::Ref(RefType::new(false, HeapType::NoFunc))
478 }
479
480 #[inline]
481 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
482 match self._inner {}
483 }
484
485 #[inline]
486 fn dynamic_concrete_type_check(&self, _: &StoreOpaque, _: bool, _: &HeapType) -> Result<()> {
487 match self._inner {}
488 }
489
490 #[inline]
491 fn is_vmgcref_and_points_to_object(&self) -> bool {
492 match self._inner {}
493 }
494
495 #[inline]
496 fn store(self, _store: &mut AutoAssertNoGc<'_>, _ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
497 match self._inner {}
498 }
499
500 #[inline]
501 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
502 unreachable!("NoFunc is uninhabited")
503 }
504}
505
506unsafe impl WasmTy for Option<NoFunc> {
507 #[inline]
508 fn valtype() -> ValType {
509 ValType::Ref(RefType::new(true, HeapType::NoFunc))
510 }
511
512 #[inline]
513 fn compatible_with_store(&self, _store: &StoreOpaque) -> bool {
514 true
515 }
516
517 #[inline]
518 fn dynamic_concrete_type_check(
519 &self,
520 _: &StoreOpaque,
521 nullable: bool,
522 ty: &HeapType,
523 ) -> Result<()> {
524 if nullable {
525 Ok(())
527 } else {
528 bail!("argument type mismatch: expected non-nullable (ref {ty}), found null reference")
529 }
530 }
531
532 #[inline]
533 fn store(self, _store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
534 ptr.write(ValRaw::funcref(ptr::null_mut()));
535 Ok(())
536 }
537
538 #[inline]
539 unsafe fn load(_store: &mut AutoAssertNoGc<'_>, _ptr: &ValRaw) -> Self {
540 None
541 }
542}
543
544unsafe impl WasmTy for Func {
545 #[inline]
546 fn valtype() -> ValType {
547 ValType::Ref(RefType::new(false, HeapType::Func))
548 }
549
550 #[inline]
551 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
552 store.store_data().contains(self.0)
553 }
554
555 #[inline]
556 fn dynamic_concrete_type_check(
557 &self,
558 store: &StoreOpaque,
559 _nullable: bool,
560 expected: &HeapType,
561 ) -> Result<()> {
562 let expected = expected.unwrap_concrete_func();
563 self.ensure_matches_ty(store, expected)
564 .context("argument type mismatch for reference to concrete type")
565 }
566
567 #[inline]
568 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
569 let abi = self.vm_func_ref(store);
570 ptr.write(ValRaw::funcref(abi.cast::<c_void>().as_ptr()));
571 Ok(())
572 }
573
574 #[inline]
575 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
576 let p = NonNull::new(ptr.get_funcref()).unwrap().cast();
577 Func::from_vm_func_ref(store, p)
578 }
579}
580
581unsafe impl WasmTy for Option<Func> {
582 #[inline]
583 fn valtype() -> ValType {
584 ValType::FUNCREF
585 }
586
587 #[inline]
588 fn compatible_with_store(&self, store: &StoreOpaque) -> bool {
589 if let Some(f) = self {
590 store.store_data().contains(f.0)
591 } else {
592 true
593 }
594 }
595
596 fn dynamic_concrete_type_check(
597 &self,
598 store: &StoreOpaque,
599 nullable: bool,
600 expected: &HeapType,
601 ) -> Result<()> {
602 if let Some(f) = self {
603 let expected = expected.unwrap_concrete_func();
604 f.ensure_matches_ty(store, expected)
605 .context("argument type mismatch for reference to concrete type")
606 } else if nullable {
607 Ok(())
608 } else {
609 bail!("argument type mismatch: expected non-nullable (ref {expected}), found null reference")
610 }
611 }
612
613 #[inline]
614 fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
615 let raw = if let Some(f) = self {
616 f.vm_func_ref(store).as_ptr()
617 } else {
618 ptr::null_mut()
619 };
620 ptr.write(ValRaw::funcref(raw.cast::<c_void>()));
621 Ok(())
622 }
623
624 #[inline]
625 unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
626 let ptr = NonNull::new(ptr.get_funcref())?.cast();
627 Some(Func::from_vm_func_ref(store, ptr))
628 }
629}
630
631pub unsafe trait WasmParams: Send {
637 #[doc(hidden)]
638 type ValRawStorage: Copy;
639
640 #[doc(hidden)]
641 fn typecheck(
642 engine: &Engine,
643 params: impl ExactSizeIterator<Item = crate::ValType>,
644 position: TypeCheckPosition,
645 ) -> Result<()>;
646
647 #[doc(hidden)]
648 fn vmgcref_pointing_to_object_count(&self) -> usize;
649
650 #[doc(hidden)]
651 fn store(
652 self,
653 store: &mut AutoAssertNoGc<'_>,
654 func_ty: &FuncType,
655 dst: &mut MaybeUninit<Self::ValRawStorage>,
656 ) -> Result<()>;
657}
658
659unsafe impl<T> WasmParams for T
662where
663 T: WasmTy,
664{
665 type ValRawStorage = <(T,) as WasmParams>::ValRawStorage;
666
667 fn typecheck(
668 engine: &Engine,
669 params: impl ExactSizeIterator<Item = crate::ValType>,
670 position: TypeCheckPosition,
671 ) -> Result<()> {
672 <(T,) as WasmParams>::typecheck(engine, params, position)
673 }
674
675 #[inline]
676 fn vmgcref_pointing_to_object_count(&self) -> usize {
677 T::is_vmgcref_and_points_to_object(self) as usize
678 }
679
680 #[inline]
681 fn store(
682 self,
683 store: &mut AutoAssertNoGc<'_>,
684 func_ty: &FuncType,
685 dst: &mut MaybeUninit<Self::ValRawStorage>,
686 ) -> Result<()> {
687 <(T,) as WasmParams>::store((self,), store, func_ty, dst)
688 }
689}
690
691macro_rules! impl_wasm_params {
692 ($n:tt $($t:ident)*) => {
693 #[allow(non_snake_case)]
694 unsafe impl<$($t: WasmTy,)*> WasmParams for ($($t,)*) {
695 type ValRawStorage = [ValRaw; $n];
696
697 fn typecheck(
698 _engine: &Engine,
699 mut params: impl ExactSizeIterator<Item = crate::ValType>,
700 _position: TypeCheckPosition,
701 ) -> Result<()> {
702 let mut _n = 0;
703
704 $(
705 match params.next() {
706 Some(t) => {
707 _n += 1;
708 $t::typecheck(_engine, t, _position)?
709 },
710 None => bail!("expected {} types, found {}", $n, params.len() + _n),
711 }
712 )*
713
714 match params.next() {
715 None => Ok(()),
716 Some(_) => {
717 _n += 1;
718 bail!("expected {} types, found {}", $n, params.len() + _n)
719 },
720 }
721 }
722
723 #[inline]
724 fn vmgcref_pointing_to_object_count(&self) -> usize {
725 let ($($t,)*) = self;
726 0 $(
727 + $t.is_vmgcref_and_points_to_object() as usize
728 )*
729 }
730
731
732 #[inline]
733 fn store(
734 self,
735 _store: &mut AutoAssertNoGc<'_>,
736 _func_ty: &FuncType,
737 _ptr: &mut MaybeUninit<Self::ValRawStorage>,
738 ) -> Result<()> {
739 let ($($t,)*) = self;
740
741 let mut _i = 0;
742 $(
743 if !$t.compatible_with_store(_store) {
744 bail!("attempt to pass cross-`Store` value to Wasm as function argument");
745 }
746
747 if $t::valtype().is_ref() {
748 let param_ty = _func_ty.param(_i).unwrap();
749 let ref_ty = param_ty.unwrap_ref();
750 let heap_ty = ref_ty.heap_type();
751 if heap_ty.is_concrete() {
752 $t.dynamic_concrete_type_check(_store, ref_ty.is_nullable(), heap_ty)?;
753 }
754 }
755
756 let dst = map_maybe_uninit!(_ptr[_i]);
757 $t.store(_store, dst)?;
758
759 _i += 1;
760 )*
761 Ok(())
762 }
763 }
764 };
765}
766
767for_each_function_signature!(impl_wasm_params);
768
769pub unsafe trait WasmResults: WasmParams {
772 #[doc(hidden)]
773 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self;
774}
775
776unsafe impl<T: WasmTy> WasmResults for T {
778 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
779 <(T,) as WasmResults>::load(store, abi).0
780 }
781}
782
783macro_rules! impl_wasm_results {
784 ($n:tt $($t:ident)*) => {
785 #[allow(non_snake_case, unused_variables)]
786 unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) {
787 unsafe fn load(store: &mut AutoAssertNoGc<'_>, abi: &Self::ValRawStorage) -> Self {
788 let [$($t,)*] = abi;
789 ($($t::load(store, $t),)*)
790 }
791 }
792 };
793}
794
795for_each_function_signature!(impl_wasm_results);