1#![deny(missing_docs)]
4#![warn(clippy::cast_sign_loss)]
7
8use wasmtime_versioned_export_macros as _;
12
13#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
15#[expect(non_camel_case_types, reason = "matching wasm conventions")]
16pub(crate) type i8x16 = core::arch::x86_64::__m128i;
17#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
18#[expect(non_camel_case_types, reason = "matching wasm conventions")]
19pub(crate) type f32x4 = core::arch::x86_64::__m128;
20#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
21#[expect(non_camel_case_types, reason = "matching wasm conventions")]
22pub(crate) type f64x2 = core::arch::x86_64::__m128d;
23
24#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
29#[expect(non_camel_case_types, reason = "matching wasm conventions")]
30#[derive(Copy, Clone)]
31pub(crate) struct i8x16(core::convert::Infallible);
32#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
33#[expect(non_camel_case_types, reason = "matching wasm conventions")]
34#[derive(Copy, Clone)]
35pub(crate) struct f32x4(core::convert::Infallible);
36#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
37#[expect(non_camel_case_types, reason = "matching wasm conventions")]
38#[derive(Copy, Clone)]
39pub(crate) struct f64x2(core::convert::Infallible);
40
41use crate::StoreContextMut;
42use crate::prelude::*;
43use crate::store::{StoreInner, StoreOpaque, StoreResourceLimiter};
44use crate::type_registry::RegisteredType;
45use alloc::sync::Arc;
46use core::fmt;
47use core::ops::{Deref, DerefMut};
48use core::pin::pin;
49use core::ptr::NonNull;
50use core::sync::atomic::{AtomicUsize, Ordering};
51use core::task::{Context, Poll, Waker};
52use wasmtime_environ::error::OutOfMemory;
53use wasmtime_environ::{DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex};
54
55#[cfg(feature = "gc")]
56use wasmtime_environ::ModuleInternedTypeIndex;
57
58mod always_mut;
59#[cfg(feature = "component-model")]
60pub mod component;
61mod export;
62mod gc;
63mod imports;
64mod instance;
65mod memory;
66mod mmap_vec;
67#[cfg(has_virtual_memory)]
68mod pagemap_disabled;
69mod provenance;
70mod send_sync_ptr;
71mod stack_switching;
72mod store_box;
73mod sys;
74mod table;
75#[cfg(feature = "gc")]
76mod throw;
77mod traphandlers;
78mod vmcontext;
79
80#[cfg(feature = "threads")]
81mod parking_spot;
82
83#[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
87pub mod debug_builtins;
88pub mod libcalls;
89pub mod mpk;
90
91#[cfg(feature = "pulley")]
92pub(crate) mod interpreter;
93#[cfg(not(feature = "pulley"))]
94pub(crate) mod interpreter_disabled;
95#[cfg(not(feature = "pulley"))]
96pub(crate) use interpreter_disabled as interpreter;
97
98#[cfg(feature = "component-model-async")]
99pub(crate) use sys::{component_async_tls_get, component_async_tls_set};
100
101#[cfg(feature = "debug-builtins")]
102pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
103
104pub use crate::runtime::vm::always_mut::*;
105pub use crate::runtime::vm::export::*;
106pub use crate::runtime::vm::gc::*;
107pub use crate::runtime::vm::imports::Imports;
108pub use crate::runtime::vm::instance::{
109 GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
110 MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex,
111};
112#[cfg(feature = "pooling-allocator")]
113pub use crate::runtime::vm::instance::{
114 PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,
115};
116pub use crate::runtime::vm::interpreter::*;
117#[cfg(feature = "debug")]
118pub use crate::runtime::vm::memory::WATCH_DEBUG;
119pub use crate::runtime::vm::memory::{Memory, ShadowMemory, SharedMemory};
120pub use crate::runtime::vm::mmap_vec::MmapVec;
121pub use crate::runtime::vm::provenance::*;
122pub use crate::runtime::vm::stack_switching::*;
123pub use crate::runtime::vm::store_box::*;
124#[cfg(feature = "std")]
125pub use crate::runtime::vm::sys::mmap::open_file_for_mmap;
126#[cfg(has_host_compiler_backend)]
127pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
128pub use crate::runtime::vm::table::{Table, TableElementType};
129#[cfg(feature = "gc")]
130pub use crate::runtime::vm::throw::*;
131pub use crate::runtime::vm::traphandlers::*;
132pub use crate::runtime::vm::vmcontext::*;
133#[cfg(has_custom_sync)]
134pub(crate) use sys::capi;
135
136pub use send_sync_ptr::SendSyncPtr;
137pub use wasmtime_unwinder::Unwind;
138
139#[cfg(has_host_compiler_backend)]
140pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer};
141
142mod module_id;
143pub use module_id::CompiledModuleId;
144
145#[cfg(has_virtual_memory)]
146mod byte_count;
147#[cfg(has_virtual_memory)]
148mod cow;
149#[cfg(not(has_virtual_memory))]
150mod cow_disabled;
151#[cfg(has_virtual_memory)]
152mod mmap;
153
154#[allow(unused, reason = "hard to cfg on/off, weird feature interactions")]
155mod send_sync_unsafe_cell;
156#[allow(unused, reason = "hard to cfg on/off, weird feature interactions")]
157pub use send_sync_unsafe_cell::SendSyncUnsafeCell;
158
159cfg_select! {
160 has_virtual_memory => {
161 pub use crate::runtime::vm::byte_count::*;
162 pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
163 pub use self::cow::{MemoryImage, ModuleMemoryImages};
164 #[cfg(feature = "pooling-allocator")]
165 pub use self::cow::MemoryImageSlot;
166 }
167 _ => {
168 pub use self::cow_disabled::{MemoryImage, ModuleMemoryImages};
169 }
170}
171
172pub trait ModuleMemoryImageSource: Send + Sync + 'static {
174 fn wasm_data(&self) -> &[u8];
177
178 fn mmap(&self) -> Option<&MmapVec>;
181}
182
183pub unsafe trait VMStore: 'static {
206 fn resource_limiter_and_store_opaque(
209 &mut self,
210 ) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque);
211
212 #[cfg(feature = "call-hook")]
215 fn call_hook(&mut self, s: crate::CallHook) -> Result<()>;
216
217 #[cfg(target_has_atomic = "64")]
221 fn new_epoch_updated_deadline(&mut self) -> Result<crate::UpdateDeadline>;
222
223 #[cfg(feature = "component-model-async")]
224 fn component_async_store(
225 &mut self,
226 ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore;
227
228 #[cfg(feature = "debug")]
230 fn block_on_debug_handler(&mut self, event: crate::DebugEvent) -> crate::Result<()>;
231}
232
233impl Deref for dyn VMStore + '_ {
234 type Target = StoreOpaque;
235
236 #[inline]
237 fn deref(&self) -> &Self::Target {
238 unsafe { &*(self as *const dyn VMStore as *const StoreOpaque) }
249 }
250}
251
252impl DerefMut for dyn VMStore + '_ {
253 #[inline]
254 fn deref_mut(&mut self) -> &mut Self::Target {
255 unsafe { &mut *(self as *mut dyn VMStore as *mut StoreOpaque) }
257 }
258}
259
260impl dyn VMStore + '_ {
261 pub(crate) unsafe fn unchecked_context_mut<T>(&mut self) -> StoreContextMut<'_, T> {
269 unsafe { StoreContextMut(&mut *(self as *mut dyn VMStore as *mut StoreInner<T>)) }
270 }
271}
272
273#[derive(Copy, Clone)]
288#[repr(transparent)]
289struct VMStoreRawPtr(pub NonNull<dyn VMStore>);
290
291unsafe impl Send for VMStoreRawPtr {}
294unsafe impl Sync for VMStoreRawPtr {}
295
296#[derive(Clone)]
299pub enum ModuleRuntimeInfo {
300 Module(crate::Module),
301 Bare(Arc<BareModuleInfo>),
302}
303
304pub struct BareModuleInfo {
309 module: Arc<wasmtime_environ::Module>,
310 offsets: VMOffsets<HostPtr>,
311 _registered_types: TryVec<RegisteredType>,
312}
313
314impl ModuleRuntimeInfo {
315 pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Result<Self, OutOfMemory> {
316 ModuleRuntimeInfo::new_bare(module, TryVec::new())
317 }
318
319 pub(crate) fn bare_with_registered_types(
327 module: Arc<wasmtime_environ::Module>,
328 engine: &crate::Engine,
329 registered_types: impl IntoIterator<Item = RegisteredType>,
330 ) -> Result<Self> {
331 let mut types = TryVec::new();
332 for ty in registered_types {
333 crate::ensure!(
334 crate::Engine::same(engine, ty.engine()),
335 "type used with wrong engine"
336 );
337 types.push(ty)?;
338 }
339 Ok(ModuleRuntimeInfo::new_bare(module, types)?)
340 }
341
342 fn new_bare(
343 module: Arc<wasmtime_environ::Module>,
344 registered_types: TryVec<RegisteredType>,
345 ) -> Result<Self, OutOfMemory> {
346 let info = try_new(BareModuleInfo {
347 offsets: VMOffsets::new(HostPtr, &module),
348 module,
349 _registered_types: registered_types,
350 })?;
351 Ok(ModuleRuntimeInfo::Bare(info))
352 }
353
354 pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
356 match self {
357 ModuleRuntimeInfo::Module(m) => m.env_module(),
358 ModuleRuntimeInfo::Bare(b) => &b.module,
359 }
360 }
361
362 #[cfg(feature = "gc")]
365 fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
366 match self {
367 ModuleRuntimeInfo::Module(m) => m
368 .engine_code()
369 .signatures()
370 .shared_type(module_index)
371 .expect("bad module-level interned type index"),
372 ModuleRuntimeInfo::Bare(_) => unreachable!(),
373 }
374 }
375
376 fn memory_image(&self, memory: DefinedMemoryIndex) -> crate::Result<Option<&Arc<MemoryImage>>> {
379 match self {
380 ModuleRuntimeInfo::Module(m) => {
381 let images = m.memory_images()?;
382 Ok(images.and_then(|images| images.get_memory_image(memory)))
383 }
384 ModuleRuntimeInfo::Bare(_) => Ok(None),
385 }
386 }
387
388 #[cfg(feature = "pooling-allocator")]
392 fn unique_id(&self) -> Option<CompiledModuleId> {
393 match self {
394 ModuleRuntimeInfo::Module(m) => Some(m.id()),
395 ModuleRuntimeInfo::Bare(_) => None,
396 }
397 }
398
399 fn wasm_data(&self) -> &[u8] {
401 match self {
402 ModuleRuntimeInfo::Module(m) => m.engine_code().wasm_data(),
403 ModuleRuntimeInfo::Bare(_) => &[],
404 }
405 }
406
407 fn type_ids(&self) -> &[VMSharedTypeIndex] {
410 match self {
411 ModuleRuntimeInfo::Module(m) => m
412 .engine_code()
413 .signatures()
414 .as_module_map()
415 .values()
416 .as_slice(),
417 ModuleRuntimeInfo::Bare(_) => &[],
418 }
419 }
420
421 pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
423 match self {
424 ModuleRuntimeInfo::Module(m) => m.offsets(),
425 ModuleRuntimeInfo::Bare(b) => &b.offsets,
426 }
427 }
428}
429
430#[cfg(has_virtual_memory)]
432pub fn host_page_size() -> usize {
433 static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
436
437 return match PAGE_SIZE.load(Ordering::Relaxed) {
438 0 => {
439 let size = sys::vm::get_page_size();
440 assert!(size != 0);
441 PAGE_SIZE.store(size, Ordering::Relaxed);
442 size
443 }
444 n => n,
445 };
446}
447
448#[derive(Copy, Clone, PartialEq, Eq, Debug)]
450pub enum WaitResult {
451 Ok = 0,
454 Mismatch = 1,
457 TimedOut = 2,
460}
461
462#[derive(Debug)]
464pub struct WasmFault {
465 pub memory_size: usize,
467 pub wasm_address: u64,
469}
470
471impl fmt::Display for WasmFault {
472 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
473 write!(
474 f,
475 "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
476 self.wasm_address, self.memory_size,
477 )
478 }
479}
480
481pub fn assert_ready<F: Future>(f: F) -> F::Output {
500 one_poll(f).unwrap()
501}
502
503fn one_poll<F: Future>(f: F) -> Option<F::Output> {
515 let mut context = Context::from_waker(&Waker::noop());
516 match pin!(f).poll(&mut context) {
517 Poll::Ready(output) => Some(output),
518 Poll::Pending => None,
519 }
520}