1#![deny(missing_docs)]
4#![warn(clippy::cast_sign_loss)]
7
8#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
10#[expect(non_camel_case_types, reason = "matching wasm conventions")]
11pub(crate) type i8x16 = core::arch::x86_64::__m128i;
12#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
13#[expect(non_camel_case_types, reason = "matching wasm conventions")]
14pub(crate) type f32x4 = core::arch::x86_64::__m128;
15#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
16#[expect(non_camel_case_types, reason = "matching wasm conventions")]
17pub(crate) type f64x2 = core::arch::x86_64::__m128d;
18
19#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
24#[expect(non_camel_case_types, reason = "matching wasm conventions")]
25#[derive(Copy, Clone)]
26pub(crate) struct i8x16(crate::uninhabited::Uninhabited);
27#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
28#[expect(non_camel_case_types, reason = "matching wasm conventions")]
29#[derive(Copy, Clone)]
30pub(crate) struct f32x4(crate::uninhabited::Uninhabited);
31#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
32#[expect(non_camel_case_types, reason = "matching wasm conventions")]
33#[derive(Copy, Clone)]
34pub(crate) struct f64x2(crate::uninhabited::Uninhabited);
35
36use crate::StoreContextMut;
37use crate::prelude::*;
38use crate::store::StoreInner;
39use crate::store::StoreOpaque;
40use crate::type_registry::RegisteredType;
41use alloc::sync::Arc;
42use core::fmt;
43use core::ops::Deref;
44use core::ops::DerefMut;
45use core::ptr::NonNull;
46use core::sync::atomic::{AtomicUsize, Ordering};
47use wasmtime_environ::{
48 DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex,
49};
50
51#[cfg(feature = "gc")]
52use wasmtime_environ::ModuleInternedTypeIndex;
53
54#[cfg(feature = "component-model")]
55pub mod component;
56mod const_expr;
57mod export;
58mod gc;
59mod imports;
60mod instance;
61mod memory;
62mod mmap_vec;
63mod provenance;
64mod send_sync_ptr;
65mod stack_switching;
66mod store_box;
67mod sys;
68mod table;
69mod traphandlers;
70mod vmcontext;
71
72#[cfg(feature = "threads")]
73mod parking_spot;
74
75#[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
79pub mod debug_builtins;
80pub mod libcalls;
81pub mod mpk;
82
83#[cfg(feature = "pulley")]
84pub(crate) mod interpreter;
85#[cfg(not(feature = "pulley"))]
86pub(crate) mod interpreter_disabled;
87#[cfg(not(feature = "pulley"))]
88pub(crate) use interpreter_disabled as interpreter;
89
90#[cfg(feature = "debug-builtins")]
91pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
92
93pub use crate::runtime::vm::export::*;
94pub use crate::runtime::vm::gc::*;
95pub use crate::runtime::vm::imports::Imports;
96pub use crate::runtime::vm::instance::{
97 GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator,
98 InstanceAllocatorImpl, InstanceAndStore, InstanceHandle, MemoryAllocationIndex,
99 OnDemandInstanceAllocator, StorePtr, TableAllocationIndex, initialize_instance,
100};
101#[cfg(feature = "pooling-allocator")]
102pub use crate::runtime::vm::instance::{
103 InstanceLimits, PoolConcurrencyLimitError, PoolingInstanceAllocator,
104 PoolingInstanceAllocatorConfig,
105};
106pub use crate::runtime::vm::interpreter::*;
107pub use crate::runtime::vm::memory::{
108 Memory, MemoryBase, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory,
109};
110pub use crate::runtime::vm::mmap_vec::MmapVec;
111pub use crate::runtime::vm::provenance::*;
112pub use crate::runtime::vm::stack_switching::*;
113pub use crate::runtime::vm::store_box::*;
114#[cfg(feature = "std")]
115pub use crate::runtime::vm::sys::mmap::open_file_for_mmap;
116#[cfg(has_host_compiler_backend)]
117pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
118pub use crate::runtime::vm::table::{Table, TableElement};
119pub use crate::runtime::vm::traphandlers::*;
120#[cfg(feature = "component-model")]
121pub use crate::runtime::vm::vmcontext::VMTableDefinition;
122pub use crate::runtime::vm::vmcontext::{
123 VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport,
124 VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport,
125 VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
126};
127
128pub use send_sync_ptr::SendSyncPtr;
129pub use wasmtime_unwinder::Unwind;
130
131#[cfg(has_host_compiler_backend)]
132pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer};
133
134mod module_id;
135pub use module_id::CompiledModuleId;
136
137#[cfg(has_virtual_memory)]
138mod byte_count;
139#[cfg(has_virtual_memory)]
140mod cow;
141#[cfg(not(has_virtual_memory))]
142mod cow_disabled;
143#[cfg(has_virtual_memory)]
144mod mmap;
145
146#[cfg(feature = "async")]
147mod async_yield;
148#[cfg(feature = "async")]
149pub use crate::runtime::vm::async_yield::*;
150
151#[cfg(feature = "gc-null")]
152mod send_sync_unsafe_cell;
153#[cfg(feature = "gc-null")]
154pub use send_sync_unsafe_cell::SendSyncUnsafeCell;
155
156cfg_if::cfg_if! {
157 if #[cfg(has_virtual_memory)] {
158 pub use crate::runtime::vm::byte_count::*;
159 pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
160 pub use self::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
161 } else {
162 pub use self::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
163 }
164}
165
166pub unsafe trait VMStore: 'static {
187 fn store_opaque(&self) -> &StoreOpaque;
189
190 fn store_opaque_mut(&mut self) -> &mut StoreOpaque;
192
193 fn memory_growing(
196 &mut self,
197 current: usize,
198 desired: usize,
199 maximum: Option<usize>,
200 ) -> Result<bool, Error>;
201
202 fn memory_grow_failed(&mut self, error: Error) -> Result<()>;
207
208 fn table_growing(
211 &mut self,
212 current: usize,
213 desired: usize,
214 maximum: Option<usize>,
215 ) -> Result<bool, Error>;
216
217 fn table_grow_failed(&mut self, error: Error) -> Result<()>;
222
223 fn out_of_gas(&mut self) -> Result<(), Error>;
227
228 #[cfg(target_has_atomic = "64")]
232 fn new_epoch(&mut self) -> Result<u64, Error>;
233
234 unsafe fn maybe_async_grow_or_collect_gc_heap(
249 &mut self,
250 root: Option<VMGcRef>,
251 bytes_needed: Option<u64>,
252 ) -> Result<Option<VMGcRef>>;
253
254 #[cfg(feature = "component-model")]
256 fn component_calls(&mut self) -> &mut component::CallContexts;
257
258 #[cfg(feature = "component-model-async")]
259 fn component_async_store(
260 &mut self,
261 ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore;
262}
263
264impl Deref for dyn VMStore + '_ {
265 type Target = StoreOpaque;
266
267 fn deref(&self) -> &Self::Target {
268 self.store_opaque()
269 }
270}
271
272impl DerefMut for dyn VMStore + '_ {
273 fn deref_mut(&mut self) -> &mut Self::Target {
274 self.store_opaque_mut()
275 }
276}
277
278impl dyn VMStore + '_ {
279 pub(crate) unsafe fn unchecked_context_mut<T>(&mut self) -> StoreContextMut<'_, T> {
287 StoreContextMut(&mut *(self as *mut dyn VMStore as *mut StoreInner<T>))
288 }
289}
290
291#[derive(Copy, Clone)]
306#[repr(transparent)]
307struct VMStoreRawPtr(pub NonNull<dyn VMStore>);
308
309unsafe impl Send for VMStoreRawPtr {}
312unsafe impl Sync for VMStoreRawPtr {}
313
314#[derive(Clone)]
328pub enum ModuleRuntimeInfo {
329 Module(crate::Module),
330 Bare(Box<BareModuleInfo>),
331}
332
333#[derive(Clone)]
338pub struct BareModuleInfo {
339 module: Arc<wasmtime_environ::Module>,
340 offsets: VMOffsets<HostPtr>,
341 _registered_type: Option<RegisteredType>,
342}
343
344impl ModuleRuntimeInfo {
345 pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Self {
346 ModuleRuntimeInfo::bare_with_registered_type(module, None)
347 }
348
349 pub(crate) fn bare_with_registered_type(
350 module: Arc<wasmtime_environ::Module>,
351 registered_type: Option<RegisteredType>,
352 ) -> Self {
353 ModuleRuntimeInfo::Bare(Box::new(BareModuleInfo {
354 offsets: VMOffsets::new(HostPtr, &module),
355 module,
356 _registered_type: registered_type,
357 }))
358 }
359
360 pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
362 match self {
363 ModuleRuntimeInfo::Module(m) => m.env_module(),
364 ModuleRuntimeInfo::Bare(b) => &b.module,
365 }
366 }
367
368 #[cfg(feature = "gc")]
371 fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
372 match self {
373 ModuleRuntimeInfo::Module(m) => m
374 .code_object()
375 .signatures()
376 .shared_type(module_index)
377 .expect("bad module-level interned type index"),
378 ModuleRuntimeInfo::Bare(_) => unreachable!(),
379 }
380 }
381
382 fn function(&self, index: DefinedFuncIndex) -> NonNull<VMWasmCallFunction> {
384 let module = match self {
385 ModuleRuntimeInfo::Module(m) => m,
386 ModuleRuntimeInfo::Bare(_) => unreachable!(),
387 };
388 let ptr = module
389 .compiled_module()
390 .finished_function(index)
391 .as_ptr()
392 .cast::<VMWasmCallFunction>()
393 .cast_mut();
394 NonNull::new(ptr).unwrap()
395 }
396
397 fn array_to_wasm_trampoline(
403 &self,
404 index: DefinedFuncIndex,
405 ) -> Option<NonNull<VMArrayCallFunction>> {
406 let m = match self {
407 ModuleRuntimeInfo::Module(m) => m,
408 ModuleRuntimeInfo::Bare(_) => unreachable!(),
409 };
410 let ptr = NonNull::from(m.compiled_module().array_to_wasm_trampoline(index)?);
411 Some(ptr.cast())
412 }
413
414 fn memory_image(
417 &self,
418 memory: DefinedMemoryIndex,
419 ) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
420 match self {
421 ModuleRuntimeInfo::Module(m) => {
422 let images = m.memory_images()?;
423 Ok(images.and_then(|images| images.get_memory_image(memory)))
424 }
425 ModuleRuntimeInfo::Bare(_) => Ok(None),
426 }
427 }
428
429 #[cfg(feature = "pooling-allocator")]
433 fn unique_id(&self) -> Option<CompiledModuleId> {
434 match self {
435 ModuleRuntimeInfo::Module(m) => Some(m.id()),
436 ModuleRuntimeInfo::Bare(_) => None,
437 }
438 }
439
440 fn wasm_data(&self) -> &[u8] {
442 match self {
443 ModuleRuntimeInfo::Module(m) => m.compiled_module().code_memory().wasm_data(),
444 ModuleRuntimeInfo::Bare(_) => &[],
445 }
446 }
447
448 fn type_ids(&self) -> &[VMSharedTypeIndex] {
451 match self {
452 ModuleRuntimeInfo::Module(m) => m
453 .code_object()
454 .signatures()
455 .as_module_map()
456 .values()
457 .as_slice(),
458 ModuleRuntimeInfo::Bare(_) => &[],
459 }
460 }
461
462 pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
464 match self {
465 ModuleRuntimeInfo::Module(m) => m.offsets(),
466 ModuleRuntimeInfo::Bare(b) => &b.offsets,
467 }
468 }
469}
470
471#[cfg(has_virtual_memory)]
473pub fn host_page_size() -> usize {
474 static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
477
478 return match PAGE_SIZE.load(Ordering::Relaxed) {
479 0 => {
480 let size = sys::vm::get_page_size();
481 assert!(size != 0);
482 PAGE_SIZE.store(size, Ordering::Relaxed);
483 size
484 }
485 n => n,
486 };
487}
488
489#[derive(Copy, Clone, PartialEq, Eq, Debug)]
491pub enum WaitResult {
492 Ok = 0,
495 Mismatch = 1,
498 TimedOut = 2,
501}
502
503#[derive(Debug)]
505pub struct WasmFault {
506 pub memory_size: usize,
508 pub wasm_address: u64,
510}
511
512impl fmt::Display for WasmFault {
513 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
514 write!(
515 f,
516 "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
517 self.wasm_address, self.memory_size,
518 )
519 }
520}