1#![deny(missing_docs)]
4#![warn(clippy::cast_sign_loss)]
7
8#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
10#[expect(non_camel_case_types, reason = "matching wasm conventions")]
11pub(crate) type i8x16 = core::arch::x86_64::__m128i;
12#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
13#[expect(non_camel_case_types, reason = "matching wasm conventions")]
14pub(crate) type f32x4 = core::arch::x86_64::__m128;
15#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
16#[expect(non_camel_case_types, reason = "matching wasm conventions")]
17pub(crate) type f64x2 = core::arch::x86_64::__m128d;
18
19#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
24#[expect(non_camel_case_types, reason = "matching wasm conventions")]
25#[derive(Copy, Clone)]
26pub(crate) struct i8x16(crate::uninhabited::Uninhabited);
27#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
28#[expect(non_camel_case_types, reason = "matching wasm conventions")]
29#[derive(Copy, Clone)]
30pub(crate) struct f32x4(crate::uninhabited::Uninhabited);
31#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
32#[expect(non_camel_case_types, reason = "matching wasm conventions")]
33#[derive(Copy, Clone)]
34pub(crate) struct f64x2(crate::uninhabited::Uninhabited);
35
36use crate::StoreContextMut;
37use crate::prelude::*;
38use crate::store::{StoreInner, StoreOpaque, StoreResourceLimiter};
39use crate::type_registry::RegisteredType;
40use alloc::sync::Arc;
41use core::fmt;
42use core::ops::{Deref, DerefMut};
43use core::pin::pin;
44use core::ptr::NonNull;
45use core::sync::atomic::{AtomicUsize, Ordering};
46use core::task::{Context, Poll, Waker};
47use wasmtime_environ::{
48 DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex,
49};
50
51#[cfg(feature = "gc")]
52use wasmtime_environ::ModuleInternedTypeIndex;
53
54mod always_mut;
55#[cfg(feature = "component-model")]
56pub mod component;
57mod const_expr;
58mod export;
59mod gc;
60mod imports;
61mod instance;
62mod memory;
63mod mmap_vec;
64#[cfg(has_virtual_memory)]
65mod pagemap_disabled;
66mod provenance;
67mod send_sync_ptr;
68mod stack_switching;
69mod store_box;
70mod sys;
71mod table;
72#[cfg(feature = "gc")]
73mod throw;
74mod traphandlers;
75mod vmcontext;
76
77#[cfg(feature = "threads")]
78mod parking_spot;
79
80#[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
84pub mod debug_builtins;
85pub mod libcalls;
86pub mod mpk;
87
88#[cfg(feature = "pulley")]
89pub(crate) mod interpreter;
90#[cfg(not(feature = "pulley"))]
91pub(crate) mod interpreter_disabled;
92#[cfg(not(feature = "pulley"))]
93pub(crate) use interpreter_disabled as interpreter;
94
95#[cfg(feature = "debug-builtins")]
96pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
97
98pub use crate::runtime::vm::always_mut::*;
99pub use crate::runtime::vm::export::*;
100pub use crate::runtime::vm::gc::*;
101pub use crate::runtime::vm::imports::Imports;
102pub use crate::runtime::vm::instance::{
103 GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
104 MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance,
105};
106#[cfg(feature = "pooling-allocator")]
107pub use crate::runtime::vm::instance::{
108 InstanceLimits, PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,
109 PoolingInstanceAllocatorConfig,
110};
111pub use crate::runtime::vm::interpreter::*;
112pub use crate::runtime::vm::memory::{
113 Memory, MemoryBase, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory,
114};
115pub use crate::runtime::vm::mmap_vec::MmapVec;
116pub use crate::runtime::vm::provenance::*;
117pub use crate::runtime::vm::stack_switching::*;
118pub use crate::runtime::vm::store_box::*;
119#[cfg(feature = "std")]
120pub use crate::runtime::vm::sys::mmap::open_file_for_mmap;
121#[cfg(has_host_compiler_backend)]
122pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
123pub use crate::runtime::vm::table::{Table, TableElementType};
124#[cfg(feature = "gc")]
125pub use crate::runtime::vm::throw::*;
126pub use crate::runtime::vm::traphandlers::*;
127pub use crate::runtime::vm::vmcontext::{
128 VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport,
129 VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport,
130 VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
131};
132
133pub use send_sync_ptr::SendSyncPtr;
134pub use wasmtime_unwinder::Unwind;
135
136#[cfg(has_host_compiler_backend)]
137pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer};
138
139mod module_id;
140pub use module_id::CompiledModuleId;
141
142#[cfg(has_virtual_memory)]
143mod byte_count;
144#[cfg(has_virtual_memory)]
145mod cow;
146#[cfg(not(has_virtual_memory))]
147mod cow_disabled;
148#[cfg(has_virtual_memory)]
149mod mmap;
150
151#[cfg(feature = "async")]
152mod async_yield;
153#[cfg(feature = "async")]
154pub use crate::runtime::vm::async_yield::*;
155
156#[cfg(feature = "gc-null")]
157mod send_sync_unsafe_cell;
158#[cfg(feature = "gc-null")]
159pub use send_sync_unsafe_cell::SendSyncUnsafeCell;
160
161cfg_if::cfg_if! {
162 if #[cfg(has_virtual_memory)] {
163 pub use crate::runtime::vm::byte_count::*;
164 pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
165 pub use self::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
166 } else {
167 pub use self::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
168 }
169}
170
171pub trait ModuleMemoryImageSource: Send + Sync + 'static {
173 fn wasm_data(&self) -> &[u8];
176
177 fn mmap(&self) -> Option<&MmapVec>;
180}
181
182pub unsafe trait VMStore: 'static {
203 fn store_opaque(&self) -> &StoreOpaque;
205
206 fn store_opaque_mut(&mut self) -> &mut StoreOpaque;
208
209 fn resource_limiter_and_store_opaque(
212 &mut self,
213 ) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque);
214
215 #[cfg(target_has_atomic = "64")]
219 fn new_epoch_updated_deadline(&mut self) -> Result<crate::UpdateDeadline>;
220
221 #[cfg(feature = "component-model")]
223 fn component_calls(&mut self) -> &mut component::CallContexts;
224
225 #[cfg(feature = "component-model-async")]
226 fn component_async_store(
227 &mut self,
228 ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore;
229}
230
231impl Deref for dyn VMStore + '_ {
232 type Target = StoreOpaque;
233
234 fn deref(&self) -> &Self::Target {
235 self.store_opaque()
236 }
237}
238
239impl DerefMut for dyn VMStore + '_ {
240 fn deref_mut(&mut self) -> &mut Self::Target {
241 self.store_opaque_mut()
242 }
243}
244
245impl dyn VMStore + '_ {
246 pub(crate) unsafe fn unchecked_context_mut<T>(&mut self) -> StoreContextMut<'_, T> {
254 unsafe { StoreContextMut(&mut *(self as *mut dyn VMStore as *mut StoreInner<T>)) }
255 }
256}
257
258#[derive(Copy, Clone)]
273#[repr(transparent)]
274struct VMStoreRawPtr(pub NonNull<dyn VMStore>);
275
276unsafe impl Send for VMStoreRawPtr {}
279unsafe impl Sync for VMStoreRawPtr {}
280
281#[derive(Clone)]
295pub enum ModuleRuntimeInfo {
296 Module(crate::Module),
297 Bare(Box<BareModuleInfo>),
298}
299
300#[derive(Clone)]
305pub struct BareModuleInfo {
306 module: Arc<wasmtime_environ::Module>,
307 offsets: VMOffsets<HostPtr>,
308 _registered_type: Option<RegisteredType>,
309}
310
311impl ModuleRuntimeInfo {
312 pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Self {
313 ModuleRuntimeInfo::bare_with_registered_type(module, None)
314 }
315
316 pub(crate) fn bare_with_registered_type(
317 module: Arc<wasmtime_environ::Module>,
318 registered_type: Option<RegisteredType>,
319 ) -> Self {
320 ModuleRuntimeInfo::Bare(Box::new(BareModuleInfo {
321 offsets: VMOffsets::new(HostPtr, &module),
322 module,
323 _registered_type: registered_type,
324 }))
325 }
326
327 pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
329 match self {
330 ModuleRuntimeInfo::Module(m) => m.env_module(),
331 ModuleRuntimeInfo::Bare(b) => &b.module,
332 }
333 }
334
335 #[cfg(feature = "gc")]
338 fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
339 match self {
340 ModuleRuntimeInfo::Module(m) => m
341 .code_object()
342 .signatures()
343 .shared_type(module_index)
344 .expect("bad module-level interned type index"),
345 ModuleRuntimeInfo::Bare(_) => unreachable!(),
346 }
347 }
348
349 fn function(&self, index: DefinedFuncIndex) -> NonNull<VMWasmCallFunction> {
351 let module = match self {
352 ModuleRuntimeInfo::Module(m) => m,
353 ModuleRuntimeInfo::Bare(_) => unreachable!(),
354 };
355 let ptr = module
356 .compiled_module()
357 .finished_function(index)
358 .as_ptr()
359 .cast::<VMWasmCallFunction>()
360 .cast_mut();
361 NonNull::new(ptr).unwrap()
362 }
363
364 fn array_to_wasm_trampoline(
370 &self,
371 index: DefinedFuncIndex,
372 ) -> Option<NonNull<VMArrayCallFunction>> {
373 let m = match self {
374 ModuleRuntimeInfo::Module(m) => m,
375 ModuleRuntimeInfo::Bare(_) => unreachable!(),
376 };
377 let ptr = NonNull::from(m.compiled_module().array_to_wasm_trampoline(index)?);
378 Some(ptr.cast())
379 }
380
381 fn memory_image(
384 &self,
385 memory: DefinedMemoryIndex,
386 ) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
387 match self {
388 ModuleRuntimeInfo::Module(m) => {
389 let images = m.memory_images()?;
390 Ok(images.and_then(|images| images.get_memory_image(memory)))
391 }
392 ModuleRuntimeInfo::Bare(_) => Ok(None),
393 }
394 }
395
396 #[cfg(feature = "pooling-allocator")]
400 fn unique_id(&self) -> Option<CompiledModuleId> {
401 match self {
402 ModuleRuntimeInfo::Module(m) => Some(m.id()),
403 ModuleRuntimeInfo::Bare(_) => None,
404 }
405 }
406
407 fn wasm_data(&self) -> &[u8] {
409 match self {
410 ModuleRuntimeInfo::Module(m) => m.compiled_module().code_memory().wasm_data(),
411 ModuleRuntimeInfo::Bare(_) => &[],
412 }
413 }
414
415 fn type_ids(&self) -> &[VMSharedTypeIndex] {
418 match self {
419 ModuleRuntimeInfo::Module(m) => m
420 .code_object()
421 .signatures()
422 .as_module_map()
423 .values()
424 .as_slice(),
425 ModuleRuntimeInfo::Bare(_) => &[],
426 }
427 }
428
429 pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
431 match self {
432 ModuleRuntimeInfo::Module(m) => m.offsets(),
433 ModuleRuntimeInfo::Bare(b) => &b.offsets,
434 }
435 }
436}
437
438#[cfg(has_virtual_memory)]
440pub fn host_page_size() -> usize {
441 static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
444
445 return match PAGE_SIZE.load(Ordering::Relaxed) {
446 0 => {
447 let size = sys::vm::get_page_size();
448 assert!(size != 0);
449 PAGE_SIZE.store(size, Ordering::Relaxed);
450 size
451 }
452 n => n,
453 };
454}
455
456#[derive(Copy, Clone, PartialEq, Eq, Debug)]
458pub enum WaitResult {
459 Ok = 0,
462 Mismatch = 1,
465 TimedOut = 2,
468}
469
470#[derive(Debug)]
472pub struct WasmFault {
473 pub memory_size: usize,
475 pub wasm_address: u64,
477}
478
479impl fmt::Display for WasmFault {
480 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
481 write!(
482 f,
483 "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
484 self.wasm_address, self.memory_size,
485 )
486 }
487}
488
489pub fn assert_ready<F: Future>(f: F) -> F::Output {
502 one_poll(f).unwrap()
503}
504
505pub fn one_poll<F: Future>(f: F) -> Option<F::Output> {
517 let mut context = Context::from_waker(&Waker::noop());
518 match pin!(f).poll(&mut context) {
519 Poll::Ready(output) => Some(output),
520 Poll::Pending => None,
521 }
522}