Skip to main content

wasmtime/runtime/
vm.rs

1//! Runtime library support for Wasmtime.
2
3#![deny(missing_docs)]
4// See documentation in crates/wasmtime/src/runtime.rs for why this is
5// selectively enabled here.
6#![warn(clippy::cast_sign_loss)]
7
8// Whether `wasmtime-versioned-export-macros` is used depends on `build.rs`
9// computed cfgs which `Cargo.toml` can't take into account, so fake a use of it
10// here to force this to be considered used.
11use wasmtime_versioned_export_macros as _;
12
13// Polyfill `std::simd::i8x16` etc. until they're stable.
14#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
15#[expect(non_camel_case_types, reason = "matching wasm conventions")]
16pub(crate) type i8x16 = core::arch::x86_64::__m128i;
17#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
18#[expect(non_camel_case_types, reason = "matching wasm conventions")]
19pub(crate) type f32x4 = core::arch::x86_64::__m128;
20#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
21#[expect(non_camel_case_types, reason = "matching wasm conventions")]
22pub(crate) type f64x2 = core::arch::x86_64::__m128d;
23
24// On platforms other than x86_64, define i8x16 to a non-constructible type;
25// we need a type because we have a lot of macros for defining builtin
26// functions that are awkward to make conditional on the target, but it
27// doesn't need to actually be constructible unless we're on x86_64.
28#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
29#[expect(non_camel_case_types, reason = "matching wasm conventions")]
30#[derive(Copy, Clone)]
31pub(crate) struct i8x16(core::convert::Infallible);
32#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
33#[expect(non_camel_case_types, reason = "matching wasm conventions")]
34#[derive(Copy, Clone)]
35pub(crate) struct f32x4(core::convert::Infallible);
36#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
37#[expect(non_camel_case_types, reason = "matching wasm conventions")]
38#[derive(Copy, Clone)]
39pub(crate) struct f64x2(core::convert::Infallible);
40
41use crate::StoreContextMut;
42use crate::prelude::*;
43use crate::store::{StoreInner, StoreOpaque, StoreResourceLimiter};
44use crate::type_registry::RegisteredType;
45use alloc::sync::Arc;
46use core::fmt;
47use core::ops::{Deref, DerefMut};
48use core::pin::pin;
49use core::ptr::NonNull;
50use core::sync::atomic::{AtomicUsize, Ordering};
51use core::task::{Context, Poll, Waker};
52use wasmtime_environ::error::OutOfMemory;
53use wasmtime_environ::{DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex};
54
55#[cfg(feature = "gc")]
56use wasmtime_environ::ModuleInternedTypeIndex;
57
58mod always_mut;
59#[cfg(feature = "component-model")]
60pub mod component;
61mod export;
62mod gc;
63mod imports;
64mod instance;
65mod memory;
66mod mmap_vec;
67#[cfg(has_virtual_memory)]
68mod pagemap_disabled;
69mod provenance;
70mod send_sync_ptr;
71mod stack_switching;
72mod store_box;
73mod sys;
74mod table;
75#[cfg(feature = "gc")]
76mod throw;
77mod traphandlers;
78mod vmcontext;
79
80#[cfg(feature = "threads")]
81mod parking_spot;
82
83// Note that `debug_builtins` here is disabled with a feature or a lack of a
84// native compilation backend because it's only here to assist in debugging
85// natively compiled code.
86#[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
87pub mod debug_builtins;
88pub mod libcalls;
89pub mod mpk;
90
91#[cfg(feature = "pulley")]
92pub(crate) mod interpreter;
93#[cfg(not(feature = "pulley"))]
94pub(crate) mod interpreter_disabled;
95#[cfg(not(feature = "pulley"))]
96pub(crate) use interpreter_disabled as interpreter;
97
98#[cfg(feature = "component-model-async")]
99pub(crate) use sys::{component_async_tls_get, component_async_tls_set};
100
101#[cfg(feature = "debug-builtins")]
102pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
103
104pub use crate::runtime::vm::always_mut::*;
105pub use crate::runtime::vm::export::*;
106pub use crate::runtime::vm::gc::*;
107pub use crate::runtime::vm::imports::Imports;
108pub use crate::runtime::vm::instance::{
109    GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
110    MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex,
111};
112#[cfg(feature = "pooling-allocator")]
113pub use crate::runtime::vm::instance::{
114    PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,
115};
116pub use crate::runtime::vm::interpreter::*;
117#[cfg(feature = "debug")]
118pub use crate::runtime::vm::memory::WATCH_DEBUG;
119pub use crate::runtime::vm::memory::{Memory, ShadowMemory, SharedMemory};
120pub use crate::runtime::vm::mmap_vec::MmapVec;
121pub use crate::runtime::vm::provenance::*;
122pub use crate::runtime::vm::stack_switching::*;
123pub use crate::runtime::vm::store_box::*;
124#[cfg(feature = "std")]
125pub use crate::runtime::vm::sys::mmap::open_file_for_mmap;
126#[cfg(has_host_compiler_backend)]
127pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
128pub use crate::runtime::vm::table::{Table, TableElementType};
129#[cfg(feature = "gc")]
130pub use crate::runtime::vm::throw::*;
131pub use crate::runtime::vm::traphandlers::*;
132pub use crate::runtime::vm::vmcontext::*;
133#[cfg(has_custom_sync)]
134pub(crate) use sys::capi;
135
136pub use send_sync_ptr::SendSyncPtr;
137pub use wasmtime_unwinder::Unwind;
138
139#[cfg(has_host_compiler_backend)]
140pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer};
141
142mod module_id;
143pub use module_id::CompiledModuleId;
144
145#[cfg(has_virtual_memory)]
146mod byte_count;
147#[cfg(has_virtual_memory)]
148mod cow;
149#[cfg(not(has_virtual_memory))]
150mod cow_disabled;
151#[cfg(has_virtual_memory)]
152mod mmap;
153
154#[allow(unused, reason = "hard to cfg on/off, weird feature interactions")]
155mod send_sync_unsafe_cell;
156#[allow(unused, reason = "hard to cfg on/off, weird feature interactions")]
157pub use send_sync_unsafe_cell::SendSyncUnsafeCell;
158
159cfg_select! {
160    has_virtual_memory => {
161        pub use crate::runtime::vm::byte_count::*;
162        pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
163        pub use self::cow::{MemoryImage, ModuleMemoryImages};
164        #[cfg(feature = "pooling-allocator")]
165        pub use self::cow::MemoryImageSlot;
166    }
167    _ => {
168        pub use self::cow_disabled::{MemoryImage, ModuleMemoryImages};
169    }
170}
171
172/// Source of data used for [`MemoryImage`]
173pub trait ModuleMemoryImageSource: Send + Sync + 'static {
174    /// Returns this image's slice of all wasm data for a module which is then
175    /// further sub-sliced for a particular initialization segment.
176    fn wasm_data(&self) -> &[u8];
177
178    /// Optionally returns the backing mmap. Used for using the backing mmap's
179    /// file to perform other mmaps, for example.
180    fn mmap(&self) -> Option<&MmapVec>;
181}
182
183/// Dynamic runtime functionality needed by this crate throughout the execution
184/// of a wasm instance.
185///
186/// This trait is used to store a raw pointer trait object within each
187/// `VMContext`. This raw pointer trait object points back to the
188/// `wasmtime::Store` internally but is type-erased to avoid needing to
189/// monomorphize the entire runtime on the `T` in `Store<T>`
190///
191/// # Safety
192///
193/// This trait should be implemented by nothing other than `StoreInner<T>` in
194/// this crate. It's not sound to implement it for anything else due to
195/// `unchecked_context_mut` and the `Deref`/`DerefMut` impls below, which
196/// cast the data pointer of a `dyn VMStore` to a `StoreInner<T>` or to its
197/// first field, a `StoreOpaque`.
198///
199/// It's also worth nothing that there are various locations where a `*mut dyn
200/// VMStore` is asserted to be both `Send` and `Sync` which disregards the `T`
201/// that's actually stored in the store itself. It's assume that the high-level
202/// APIs using `Store<T>` are correctly inferring send/sync on the returned
203/// values (e.g. futures) and that internally in the runtime we aren't doing
204/// anything "weird" with threads for example.
205pub unsafe trait VMStore: 'static {
206    /// Returns a split borrow to the limiter plus `StoreOpaque` at the same
207    /// time.
208    fn resource_limiter_and_store_opaque(
209        &mut self,
210    ) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque);
211
212    /// Invoke this store's configured call hook, if any, to notify the
213    /// embedder of a transition between the host and WebAssembly.
214    #[cfg(feature = "call-hook")]
215    fn call_hook(&mut self, s: crate::CallHook) -> Result<()>;
216
217    /// Callback invoked whenever an instance observes a new epoch
218    /// number. Cannot fail; cooperative epoch-based yielding is
219    /// completely semantically transparent. Returns the new deadline.
220    #[cfg(target_has_atomic = "64")]
221    fn new_epoch_updated_deadline(&mut self) -> Result<crate::UpdateDeadline>;
222
223    #[cfg(feature = "component-model-async")]
224    fn component_async_store(
225        &mut self,
226    ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore;
227
228    /// Invoke a debug handler, if present, at a debug event.
229    #[cfg(feature = "debug")]
230    fn block_on_debug_handler(&mut self, event: crate::DebugEvent) -> crate::Result<()>;
231}
232
233impl Deref for dyn VMStore + '_ {
234    type Target = StoreOpaque;
235
236    #[inline]
237    fn deref(&self) -> &Self::Target {
238        // This is a pointer cast rather than a trait method so that
239        // it doesn't require a virtual call, which matters on hot
240        // paths such as libcalls.
241
242        // SAFETY: `VMStore` is only implemented by `StoreInner<T>` (see the
243        // trait's safety documentation), so the data pointer of this trait
244        // object points to a `StoreInner<T>`. That type is `#[repr(C)]` with a
245        // `StoreOpaque` as its first field, so the `StoreOpaque` lives at the
246        // same address. The pointer is derived from `self`, so it retains
247        // provenance over the whole `StoreInner<T>`.
248        unsafe { &*(self as *const dyn VMStore as *const StoreOpaque) }
249    }
250}
251
252impl DerefMut for dyn VMStore + '_ {
253    #[inline]
254    fn deref_mut(&mut self) -> &mut Self::Target {
255        // SAFETY: see `deref` above.
256        unsafe { &mut *(self as *mut dyn VMStore as *mut StoreOpaque) }
257    }
258}
259
260impl dyn VMStore + '_ {
261    /// Asserts that this `VMStore` was originally paired with `StoreInner<T>`
262    /// and then casts to the `StoreContextMut` type.
263    ///
264    /// # Unsafety
265    ///
266    /// This method is not safe as there's no static guarantee that `T` is
267    /// correct for this store.
268    pub(crate) unsafe fn unchecked_context_mut<T>(&mut self) -> StoreContextMut<'_, T> {
269        unsafe { StoreContextMut(&mut *(self as *mut dyn VMStore as *mut StoreInner<T>)) }
270    }
271}
272
273/// A newtype wrapper around `NonNull<dyn VMStore>` intended to be a
274/// self-pointer back to the `Store<T>` within raw data structures like
275/// `VMContext`.
276///
277/// This type exists to manually, and unsafely, implement `Send` and `Sync`.
278/// The `VMStore` trait doesn't require `Send` or `Sync` which means this isn't
279/// naturally either trait (e.g. with `SendSyncPtr` instead). Note that this
280/// means that `Instance` is, for example, mistakenly considered
281/// unconditionally `Send` and `Sync`. This is hopefully ok for now though
282/// because from a user perspective the only type that matters is `Store<T>`.
283/// That type is `Send + Sync` if `T: Send + Sync` already so the internal
284/// storage of `Instance` shouldn't matter as the final result is the same.
285/// Note though that this means we need to be extra vigilant about cross-thread
286/// usage of `Instance` and `ComponentInstance` for example.
287#[derive(Copy, Clone)]
288#[repr(transparent)]
289struct VMStoreRawPtr(pub NonNull<dyn VMStore>);
290
291// SAFETY: this is the purpose of `VMStoreRawPtr`, see docs above about safe
292// usage.
293unsafe impl Send for VMStoreRawPtr {}
294unsafe impl Sync for VMStoreRawPtr {}
295
296/// Functionality required by this crate for a particular module. This is
297/// chiefly needed for lazy initialization of various bits of instance state.
298#[derive(Clone)]
299pub enum ModuleRuntimeInfo {
300    Module(crate::Module),
301    Bare(Arc<BareModuleInfo>),
302}
303
304/// A barebones implementation of ModuleRuntimeInfo that is useful for
305/// cases where a purpose-built environ::Module is used and a full
306/// CompiledModule does not exist (for example, for tests or for the
307/// default-callee instance).
308pub struct BareModuleInfo {
309    module: Arc<wasmtime_environ::Module>,
310    offsets: VMOffsets<HostPtr>,
311    _registered_types: TryVec<RegisteredType>,
312}
313
314impl ModuleRuntimeInfo {
315    pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Result<Self, OutOfMemory> {
316        ModuleRuntimeInfo::new_bare(module, TryVec::new())
317    }
318
319    /// Same as [`ModuleRuntimeInfo::bare`], but additionally keeps
320    /// `registered_types` alive for as long as the resulting instance.
321    ///
322    /// This is the choke point at which a host-allocated table or tag holds
323    /// `VMSharedTypeIndex`es alive on behalf of a store, so it is where we
324    /// check that those types belong to that store's engine. Returns an error
325    /// if any of `registered_types` was not registered with `engine`.
326    pub(crate) fn bare_with_registered_types(
327        module: Arc<wasmtime_environ::Module>,
328        engine: &crate::Engine,
329        registered_types: impl IntoIterator<Item = RegisteredType>,
330    ) -> Result<Self> {
331        let mut types = TryVec::new();
332        for ty in registered_types {
333            crate::ensure!(
334                crate::Engine::same(engine, ty.engine()),
335                "type used with wrong engine"
336            );
337            types.push(ty)?;
338        }
339        Ok(ModuleRuntimeInfo::new_bare(module, types)?)
340    }
341
342    fn new_bare(
343        module: Arc<wasmtime_environ::Module>,
344        registered_types: TryVec<RegisteredType>,
345    ) -> Result<Self, OutOfMemory> {
346        let info = try_new(BareModuleInfo {
347            offsets: VMOffsets::new(HostPtr, &module),
348            module,
349            _registered_types: registered_types,
350        })?;
351        Ok(ModuleRuntimeInfo::Bare(info))
352    }
353
354    /// The underlying Module.
355    pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
356        match self {
357            ModuleRuntimeInfo::Module(m) => m.env_module(),
358            ModuleRuntimeInfo::Bare(b) => &b.module,
359        }
360    }
361
362    /// Translate a module-level interned type index into an engine-level
363    /// interned type index.
364    #[cfg(feature = "gc")]
365    fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
366        match self {
367            ModuleRuntimeInfo::Module(m) => m
368                .engine_code()
369                .signatures()
370                .shared_type(module_index)
371                .expect("bad module-level interned type index"),
372            ModuleRuntimeInfo::Bare(_) => unreachable!(),
373        }
374    }
375
376    /// Returns the `MemoryImage` structure used for copy-on-write
377    /// initialization of the memory, if it's applicable.
378    fn memory_image(&self, memory: DefinedMemoryIndex) -> crate::Result<Option<&Arc<MemoryImage>>> {
379        match self {
380            ModuleRuntimeInfo::Module(m) => {
381                let images = m.memory_images()?;
382                Ok(images.and_then(|images| images.get_memory_image(memory)))
383            }
384            ModuleRuntimeInfo::Bare(_) => Ok(None),
385        }
386    }
387
388    /// A unique ID for this particular module. This can be used to
389    /// allow for fastpaths to optimize a "re-instantiate the same
390    /// module again" case.
391    #[cfg(feature = "pooling-allocator")]
392    fn unique_id(&self) -> Option<CompiledModuleId> {
393        match self {
394            ModuleRuntimeInfo::Module(m) => Some(m.id()),
395            ModuleRuntimeInfo::Bare(_) => None,
396        }
397    }
398
399    /// A slice pointing to all data that is referenced by this instance.
400    fn wasm_data(&self) -> &[u8] {
401        match self {
402            ModuleRuntimeInfo::Module(m) => m.engine_code().wasm_data(),
403            ModuleRuntimeInfo::Bare(_) => &[],
404        }
405    }
406
407    /// Returns an array, indexed by `ModuleInternedTypeIndex` of all
408    /// `VMSharedSignatureIndex` entries corresponding to the `SignatureIndex`.
409    fn type_ids(&self) -> &[VMSharedTypeIndex] {
410        match self {
411            ModuleRuntimeInfo::Module(m) => m
412                .engine_code()
413                .signatures()
414                .as_module_map()
415                .values()
416                .as_slice(),
417            ModuleRuntimeInfo::Bare(_) => &[],
418        }
419    }
420
421    /// Offset information for the current host.
422    pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
423        match self {
424            ModuleRuntimeInfo::Module(m) => m.offsets(),
425            ModuleRuntimeInfo::Bare(b) => &b.offsets,
426        }
427    }
428}
429
430/// Returns the host OS page size, in bytes.
431#[cfg(has_virtual_memory)]
432pub fn host_page_size() -> usize {
433    // NB: this function is duplicated in `crates/fiber/src/unix.rs` so if this
434    // changes that should probably get updated as well.
435    static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
436
437    return match PAGE_SIZE.load(Ordering::Relaxed) {
438        0 => {
439            let size = sys::vm::get_page_size();
440            assert!(size != 0);
441            PAGE_SIZE.store(size, Ordering::Relaxed);
442            size
443        }
444        n => n,
445    };
446}
447
448/// Result of `Memory::atomic_wait32` and `Memory::atomic_wait64`
449#[derive(Copy, Clone, PartialEq, Eq, Debug)]
450pub enum WaitResult {
451    /// Indicates that a `wait` completed by being awoken by a different thread.
452    /// This means the thread went to sleep and didn't time out.
453    Ok = 0,
454    /// Indicates that `wait` did not complete and instead returned due to the
455    /// value in memory not matching the expected value.
456    Mismatch = 1,
457    /// Indicates that `wait` completed with a timeout, meaning that the
458    /// original value matched as expected but nothing ever called `notify`.
459    TimedOut = 2,
460}
461
462/// Description about a fault that occurred in WebAssembly.
463#[derive(Debug)]
464pub struct WasmFault {
465    /// The size of memory, in bytes, at the time of the fault.
466    pub memory_size: usize,
467    /// The WebAssembly address at which the fault occurred.
468    pub wasm_address: u64,
469}
470
471impl fmt::Display for WasmFault {
472    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
473        write!(
474            f,
475            "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
476            self.wasm_address, self.memory_size,
477        )
478    }
479}
480
481/// Asserts that the future `f` is ready and returns its output.
482///
483/// This function is intended to be used with `Store::validate_sync_call`.
484/// Internals of Wasmtime are generally `async` when they optionally can be,
485/// meaning that synchronous entrypoints will invoke this function after
486/// invoking the asynchronous internals. The `validate_sync_call` method
487/// ensures that during this `async` function call there won't actually be any
488/// yield points. If a yield point could possibly happen, then
489/// `validate_sync_call` will fail.
490///
491/// If `validate_sync_call` passes, then this function is an extra assert that
492/// yes, indeed, we coded everything correctly in Wasmtime and there shouldn't
493/// be any yield points in the future provided, so its result should be ready
494/// immediately.
495///
496/// # Panics
497///
498/// Panics if `f` is not yet ready.
499pub fn assert_ready<F: Future>(f: F) -> F::Output {
500    one_poll(f).unwrap()
501}
502
503/// Attempts one poll of `f` to see if its output is available.
504///
505/// This function is intended for a few minor entrypoints into the Wasmtime API
506/// where a synchronous function is documented to work even when `async_support`
507/// is enabled. For example growing a `Memory` can be done with a synchronous
508/// function, but it's documented to panic with an async resource limiter.
509///
510/// This function provides the opportunity to poll `f` once to see if its output
511/// is available. If it isn't then `None` is returned and an appropriate panic
512/// message should be generated recommending to use an async function (e.g.
513/// `grow_async` instead of `grow`).
514fn one_poll<F: Future>(f: F) -> Option<F::Output> {
515    let mut context = Context::from_waker(&Waker::noop());
516    match pin!(f).poll(&mut context) {
517        Poll::Ready(output) => Some(output),
518        Poll::Pending => None,
519    }
520}