wasmtime/runtime/
vm.rs

1//! Runtime library support for Wasmtime.
2
3#![deny(missing_docs)]
4// See documentation in crates/wasmtime/src/runtime.rs for why this is
5// selectively enabled here.
6#![warn(clippy::cast_sign_loss)]
7
8// Polyfill `std::simd::i8x16` etc. until they're stable.
9#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
10#[expect(non_camel_case_types, reason = "matching wasm conventions")]
11pub(crate) type i8x16 = core::arch::x86_64::__m128i;
12#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
13#[expect(non_camel_case_types, reason = "matching wasm conventions")]
14pub(crate) type f32x4 = core::arch::x86_64::__m128;
15#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
16#[expect(non_camel_case_types, reason = "matching wasm conventions")]
17pub(crate) type f64x2 = core::arch::x86_64::__m128d;
18
19// On platforms other than x86_64, define i8x16 to a non-constructible type;
20// we need a type because we have a lot of macros for defining builtin
21// functions that are awkward to make conditional on the target, but it
22// doesn't need to actually be constructible unless we're on x86_64.
23#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
24#[expect(non_camel_case_types, reason = "matching wasm conventions")]
25#[derive(Copy, Clone)]
26pub(crate) struct i8x16(core::convert::Infallible);
27#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
28#[expect(non_camel_case_types, reason = "matching wasm conventions")]
29#[derive(Copy, Clone)]
30pub(crate) struct f32x4(core::convert::Infallible);
31#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
32#[expect(non_camel_case_types, reason = "matching wasm conventions")]
33#[derive(Copy, Clone)]
34pub(crate) struct f64x2(core::convert::Infallible);
35
36use crate::StoreContextMut;
37use crate::prelude::*;
38use crate::store::{StoreInner, StoreOpaque, StoreResourceLimiter};
39use crate::type_registry::RegisteredType;
40use alloc::sync::Arc;
41use core::fmt;
42use core::ops::{Deref, DerefMut};
43use core::pin::pin;
44use core::ptr::NonNull;
45use core::sync::atomic::{AtomicUsize, Ordering};
46use core::task::{Context, Poll, Waker};
47use wasmtime_environ::error::OutOfMemory;
48use wasmtime_environ::{DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex};
49
50#[cfg(feature = "gc")]
51use wasmtime_environ::ModuleInternedTypeIndex;
52
53mod always_mut;
54#[cfg(feature = "component-model")]
55pub mod component;
56mod export;
57mod gc;
58mod imports;
59mod instance;
60mod memory;
61mod mmap_vec;
62#[cfg(has_virtual_memory)]
63mod pagemap_disabled;
64mod provenance;
65mod send_sync_ptr;
66mod stack_switching;
67mod store_box;
68mod sys;
69mod table;
70#[cfg(feature = "gc")]
71mod throw;
72mod traphandlers;
73mod vmcontext;
74
75#[cfg(feature = "threads")]
76mod parking_spot;
77
78// Note that `debug_builtins` here is disabled with a feature or a lack of a
79// native compilation backend because it's only here to assist in debugging
80// natively compiled code.
81#[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
82pub mod debug_builtins;
83pub mod libcalls;
84pub mod mpk;
85
86#[cfg(feature = "pulley")]
87pub(crate) mod interpreter;
88#[cfg(not(feature = "pulley"))]
89pub(crate) mod interpreter_disabled;
90#[cfg(not(feature = "pulley"))]
91pub(crate) use interpreter_disabled as interpreter;
92
93#[cfg(feature = "component-model-async")]
94pub(crate) use sys::{component_async_tls_get, component_async_tls_set};
95
96#[cfg(feature = "debug-builtins")]
97pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
98
99pub use crate::runtime::vm::always_mut::*;
100pub use crate::runtime::vm::export::*;
101pub use crate::runtime::vm::gc::*;
102pub use crate::runtime::vm::imports::Imports;
103pub use crate::runtime::vm::instance::{
104    GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
105    MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex,
106};
107#[cfg(feature = "pooling-allocator")]
108pub use crate::runtime::vm::instance::{
109    PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,
110};
111pub use crate::runtime::vm::interpreter::*;
112pub use crate::runtime::vm::memory::{
113    Memory, MemoryBase, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory,
114};
115pub use crate::runtime::vm::mmap_vec::MmapVec;
116pub use crate::runtime::vm::provenance::*;
117pub use crate::runtime::vm::stack_switching::*;
118pub use crate::runtime::vm::store_box::*;
119#[cfg(feature = "std")]
120pub use crate::runtime::vm::sys::mmap::open_file_for_mmap;
121#[cfg(has_host_compiler_backend)]
122pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
123pub use crate::runtime::vm::table::{Table, TableElementType};
124#[cfg(feature = "gc")]
125pub use crate::runtime::vm::throw::*;
126pub use crate::runtime::vm::traphandlers::*;
127#[cfg(feature = "component-model")]
128pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;
129#[cfg(feature = "gc-copying")]
130pub(crate) use crate::runtime::vm::vmcontext::VMCopyingHeader;
131#[cfg(feature = "gc-copying")]
132pub use crate::runtime::vm::vmcontext::VMCopyingHeapData;
133#[cfg(feature = "gc-drc")]
134pub(crate) use crate::runtime::vm::vmcontext::VMDrcHeader;
135#[cfg(feature = "gc-drc")]
136pub use crate::runtime::vm::vmcontext::VMDrcHeapData;
137#[cfg(feature = "component-model-async")]
138pub use crate::runtime::vm::vmcontext::VMLazyThread;
139#[cfg(feature = "gc-null")]
140pub use crate::runtime::vm::vmcontext::VMNullHeapData;
141pub use crate::runtime::vm::vmcontext::{
142    VMArrayCallHostFuncContext, VMCommonStackInformation, VMContRef, VMContext, VMFuncRef,
143    VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMHostArray,
144    VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMPayloads, VMStackLimits, VMStoreContext,
145    VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
146};
147#[cfg(has_custom_sync)]
148pub(crate) use sys::capi;
149
150pub use send_sync_ptr::SendSyncPtr;
151pub use wasmtime_unwinder::Unwind;
152
153#[cfg(has_host_compiler_backend)]
154pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer};
155
156mod module_id;
157pub use module_id::CompiledModuleId;
158
159#[cfg(has_virtual_memory)]
160mod byte_count;
161#[cfg(has_virtual_memory)]
162mod cow;
163#[cfg(not(has_virtual_memory))]
164mod cow_disabled;
165#[cfg(has_virtual_memory)]
166mod mmap;
167
168#[allow(unused, reason = "hard to cfg on/off, weird feature interactions")]
169mod send_sync_unsafe_cell;
170#[allow(unused, reason = "hard to cfg on/off, weird feature interactions")]
171pub use send_sync_unsafe_cell::SendSyncUnsafeCell;
172
173cfg_select! {
174    has_virtual_memory => {
175        pub use crate::runtime::vm::byte_count::*;
176        pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
177        pub use self::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
178    }
179    _ => {
180        pub use self::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
181    }
182}
183
184/// Source of data used for [`MemoryImage`]
185pub trait ModuleMemoryImageSource: Send + Sync + 'static {
186    /// Returns this image's slice of all wasm data for a module which is then
187    /// further sub-sliced for a particular initialization segment.
188    fn wasm_data(&self) -> &[u8];
189
190    /// Optionally returns the backing mmap. Used for using the backing mmap's
191    /// file to perform other mmaps, for example.
192    fn mmap(&self) -> Option<&MmapVec>;
193}
194
195/// Dynamic runtime functionality needed by this crate throughout the execution
196/// of a wasm instance.
197///
198/// This trait is used to store a raw pointer trait object within each
199/// `VMContext`. This raw pointer trait object points back to the
200/// `wasmtime::Store` internally but is type-erased to avoid needing to
201/// monomorphize the entire runtime on the `T` in `Store<T>`
202///
203/// # Safety
204///
205/// This trait should be implemented by nothing other than `StoreInner<T>` in
206/// this crate. It's not sound to implement it for anything else due to
207/// `unchecked_context_mut` below.
208///
209/// It's also worth nothing that there are various locations where a `*mut dyn
210/// VMStore` is asserted to be both `Send` and `Sync` which disregards the `T`
211/// that's actually stored in the store itself. It's assume that the high-level
212/// APIs using `Store<T>` are correctly inferring send/sync on the returned
213/// values (e.g. futures) and that internally in the runtime we aren't doing
214/// anything "weird" with threads for example.
215pub unsafe trait VMStore: 'static {
216    /// Get a shared borrow of this store's `StoreOpaque`.
217    fn store_opaque(&self) -> &StoreOpaque;
218
219    /// Get an exclusive borrow of this store's `StoreOpaque`.
220    fn store_opaque_mut(&mut self) -> &mut StoreOpaque;
221
222    /// Returns a split borrow to the limiter plus `StoreOpaque` at the same
223    /// time.
224    fn resource_limiter_and_store_opaque(
225        &mut self,
226    ) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque);
227
228    /// Invoke this store's configured call hook, if any, to notify the
229    /// embedder of a transition between the host and WebAssembly.
230    #[cfg(feature = "call-hook")]
231    fn call_hook(&mut self, s: crate::CallHook) -> Result<()>;
232
233    /// Callback invoked whenever an instance observes a new epoch
234    /// number. Cannot fail; cooperative epoch-based yielding is
235    /// completely semantically transparent. Returns the new deadline.
236    #[cfg(target_has_atomic = "64")]
237    fn new_epoch_updated_deadline(&mut self) -> Result<crate::UpdateDeadline>;
238
239    #[cfg(feature = "component-model-async")]
240    fn component_async_store(
241        &mut self,
242    ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore;
243
244    /// Invoke a debug handler, if present, at a debug event.
245    #[cfg(feature = "debug")]
246    fn block_on_debug_handler(&mut self, event: crate::DebugEvent) -> crate::Result<()>;
247}
248
249impl Deref for dyn VMStore + '_ {
250    type Target = StoreOpaque;
251
252    fn deref(&self) -> &Self::Target {
253        self.store_opaque()
254    }
255}
256
257impl DerefMut for dyn VMStore + '_ {
258    fn deref_mut(&mut self) -> &mut Self::Target {
259        self.store_opaque_mut()
260    }
261}
262
263impl dyn VMStore + '_ {
264    /// Asserts that this `VMStore` was originally paired with `StoreInner<T>`
265    /// and then casts to the `StoreContextMut` type.
266    ///
267    /// # Unsafety
268    ///
269    /// This method is not safe as there's no static guarantee that `T` is
270    /// correct for this store.
271    pub(crate) unsafe fn unchecked_context_mut<T>(&mut self) -> StoreContextMut<'_, T> {
272        unsafe { StoreContextMut(&mut *(self as *mut dyn VMStore as *mut StoreInner<T>)) }
273    }
274}
275
276/// A newtype wrapper around `NonNull<dyn VMStore>` intended to be a
277/// self-pointer back to the `Store<T>` within raw data structures like
278/// `VMContext`.
279///
280/// This type exists to manually, and unsafely, implement `Send` and `Sync`.
281/// The `VMStore` trait doesn't require `Send` or `Sync` which means this isn't
282/// naturally either trait (e.g. with `SendSyncPtr` instead). Note that this
283/// means that `Instance` is, for example, mistakenly considered
284/// unconditionally `Send` and `Sync`. This is hopefully ok for now though
285/// because from a user perspective the only type that matters is `Store<T>`.
286/// That type is `Send + Sync` if `T: Send + Sync` already so the internal
287/// storage of `Instance` shouldn't matter as the final result is the same.
288/// Note though that this means we need to be extra vigilant about cross-thread
289/// usage of `Instance` and `ComponentInstance` for example.
290#[derive(Copy, Clone)]
291#[repr(transparent)]
292struct VMStoreRawPtr(pub NonNull<dyn VMStore>);
293
294// SAFETY: this is the purpose of `VMStoreRawPtr`, see docs above about safe
295// usage.
296unsafe impl Send for VMStoreRawPtr {}
297unsafe impl Sync for VMStoreRawPtr {}
298
299/// Functionality required by this crate for a particular module. This is
300/// chiefly needed for lazy initialization of various bits of instance state.
301#[derive(Clone)]
302pub enum ModuleRuntimeInfo {
303    Module(crate::Module),
304    Bare(Arc<BareModuleInfo>),
305}
306
307/// A barebones implementation of ModuleRuntimeInfo that is useful for
308/// cases where a purpose-built environ::Module is used and a full
309/// CompiledModule does not exist (for example, for tests or for the
310/// default-callee instance).
311pub struct BareModuleInfo {
312    module: Arc<wasmtime_environ::Module>,
313    offsets: VMOffsets<HostPtr>,
314    _registered_types: TryVec<RegisteredType>,
315}
316
317impl ModuleRuntimeInfo {
318    pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Result<Self, OutOfMemory> {
319        ModuleRuntimeInfo::new_bare(module, TryVec::new())
320    }
321
322    /// Same as [`ModuleRuntimeInfo::bare`], but additionally keeps
323    /// `registered_types` alive for as long as the resulting instance.
324    ///
325    /// This is the choke point at which a host-allocated table or tag holds
326    /// `VMSharedTypeIndex`es alive on behalf of a store, so it is where we
327    /// check that those types belong to that store's engine. Returns an error
328    /// if any of `registered_types` was not registered with `engine`.
329    pub(crate) fn bare_with_registered_types(
330        module: Arc<wasmtime_environ::Module>,
331        engine: &crate::Engine,
332        registered_types: impl IntoIterator<Item = RegisteredType>,
333    ) -> Result<Self> {
334        let mut types = TryVec::new();
335        for ty in registered_types {
336            crate::ensure!(
337                crate::Engine::same(engine, ty.engine()),
338                "type used with wrong engine"
339            );
340            types.push(ty)?;
341        }
342        Ok(ModuleRuntimeInfo::new_bare(module, types)?)
343    }
344
345    fn new_bare(
346        module: Arc<wasmtime_environ::Module>,
347        registered_types: TryVec<RegisteredType>,
348    ) -> Result<Self, OutOfMemory> {
349        let info = try_new(BareModuleInfo {
350            offsets: VMOffsets::new(HostPtr, &module),
351            module,
352            _registered_types: registered_types,
353        })?;
354        Ok(ModuleRuntimeInfo::Bare(info))
355    }
356
357    /// The underlying Module.
358    pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
359        match self {
360            ModuleRuntimeInfo::Module(m) => m.env_module(),
361            ModuleRuntimeInfo::Bare(b) => &b.module,
362        }
363    }
364
365    /// Translate a module-level interned type index into an engine-level
366    /// interned type index.
367    #[cfg(feature = "gc")]
368    fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
369        match self {
370            ModuleRuntimeInfo::Module(m) => m
371                .engine_code()
372                .signatures()
373                .shared_type(module_index)
374                .expect("bad module-level interned type index"),
375            ModuleRuntimeInfo::Bare(_) => unreachable!(),
376        }
377    }
378
379    /// Returns the `MemoryImage` structure used for copy-on-write
380    /// initialization of the memory, if it's applicable.
381    fn memory_image(&self, memory: DefinedMemoryIndex) -> crate::Result<Option<&Arc<MemoryImage>>> {
382        match self {
383            ModuleRuntimeInfo::Module(m) => {
384                let images = m.memory_images()?;
385                Ok(images.and_then(|images| images.get_memory_image(memory)))
386            }
387            ModuleRuntimeInfo::Bare(_) => Ok(None),
388        }
389    }
390
391    /// A unique ID for this particular module. This can be used to
392    /// allow for fastpaths to optimize a "re-instantiate the same
393    /// module again" case.
394    #[cfg(feature = "pooling-allocator")]
395    fn unique_id(&self) -> Option<CompiledModuleId> {
396        match self {
397            ModuleRuntimeInfo::Module(m) => Some(m.id()),
398            ModuleRuntimeInfo::Bare(_) => None,
399        }
400    }
401
402    /// A slice pointing to all data that is referenced by this instance.
403    fn wasm_data(&self) -> &[u8] {
404        match self {
405            ModuleRuntimeInfo::Module(m) => m.engine_code().wasm_data(),
406            ModuleRuntimeInfo::Bare(_) => &[],
407        }
408    }
409
410    /// Returns an array, indexed by `ModuleInternedTypeIndex` of all
411    /// `VMSharedSignatureIndex` entries corresponding to the `SignatureIndex`.
412    fn type_ids(&self) -> &[VMSharedTypeIndex] {
413        match self {
414            ModuleRuntimeInfo::Module(m) => m
415                .engine_code()
416                .signatures()
417                .as_module_map()
418                .values()
419                .as_slice(),
420            ModuleRuntimeInfo::Bare(_) => &[],
421        }
422    }
423
424    /// Offset information for the current host.
425    pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
426        match self {
427            ModuleRuntimeInfo::Module(m) => m.offsets(),
428            ModuleRuntimeInfo::Bare(b) => &b.offsets,
429        }
430    }
431}
432
433/// Returns the host OS page size, in bytes.
434#[cfg(has_virtual_memory)]
435pub fn host_page_size() -> usize {
436    // NB: this function is duplicated in `crates/fiber/src/unix.rs` so if this
437    // changes that should probably get updated as well.
438    static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
439
440    return match PAGE_SIZE.load(Ordering::Relaxed) {
441        0 => {
442            let size = sys::vm::get_page_size();
443            assert!(size != 0);
444            PAGE_SIZE.store(size, Ordering::Relaxed);
445            size
446        }
447        n => n,
448    };
449}
450
451/// Result of `Memory::atomic_wait32` and `Memory::atomic_wait64`
452#[derive(Copy, Clone, PartialEq, Eq, Debug)]
453pub enum WaitResult {
454    /// Indicates that a `wait` completed by being awoken by a different thread.
455    /// This means the thread went to sleep and didn't time out.
456    Ok = 0,
457    /// Indicates that `wait` did not complete and instead returned due to the
458    /// value in memory not matching the expected value.
459    Mismatch = 1,
460    /// Indicates that `wait` completed with a timeout, meaning that the
461    /// original value matched as expected but nothing ever called `notify`.
462    TimedOut = 2,
463}
464
465/// Description about a fault that occurred in WebAssembly.
466#[derive(Debug)]
467pub struct WasmFault {
468    /// The size of memory, in bytes, at the time of the fault.
469    pub memory_size: usize,
470    /// The WebAssembly address at which the fault occurred.
471    pub wasm_address: u64,
472}
473
474impl fmt::Display for WasmFault {
475    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
476        write!(
477            f,
478            "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
479            self.wasm_address, self.memory_size,
480        )
481    }
482}
483
484/// Asserts that the future `f` is ready and returns its output.
485///
486/// This function is intended to be used with `Store::validate_sync_call`.
487/// Internals of Wasmtime are generally `async` when they optionally can be,
488/// meaning that synchronous entrypoints will invoke this function after
489/// invoking the asynchronous internals. The `validate_sync_call` method
490/// ensures that during this `async` function call there won't actually be any
491/// yield points. If a yield point could possibly happen, then
492/// `validate_sync_call` will fail.
493///
494/// If `validate_sync_call` passes, then this function is an extra assert that
495/// yes, indeed, we coded everything correctly in Wasmtime and there shouldn't
496/// be any yield points in the future provided, so its result should be ready
497/// immediately.
498///
499/// # Panics
500///
501/// Panics if `f` is not yet ready.
502pub fn assert_ready<F: Future>(f: F) -> F::Output {
503    one_poll(f).unwrap()
504}
505
506/// Attempts one poll of `f` to see if its output is available.
507///
508/// This function is intended for a few minor entrypoints into the Wasmtime API
509/// where a synchronous function is documented to work even when `async_support`
510/// is enabled. For example growing a `Memory` can be done with a synchronous
511/// function, but it's documented to panic with an async resource limiter.
512///
513/// This function provides the opportunity to poll `f` once to see if its output
514/// is available. If it isn't then `None` is returned and an appropriate panic
515/// message should be generated recommending to use an async function (e.g.
516/// `grow_async` instead of `grow`).
517fn one_poll<F: Future>(f: F) -> Option<F::Output> {
518    let mut context = Context::from_waker(&Waker::noop());
519    match pin!(f).poll(&mut context) {
520        Poll::Ready(output) => Some(output),
521        Poll::Pending => None,
522    }
523}
wasmtime/runtime/vm.rs

wasmtime/runtime/
vm.rs
