wasmtime/runtime/
vm.rs

1//! Runtime library support for Wasmtime.
2
3#![deny(missing_docs)]
4// See documentation in crates/wasmtime/src/runtime.rs for why this is
5// selectively enabled here.
6#![warn(clippy::cast_sign_loss)]
7
8// Polyfill `std::simd::i8x16` etc. until they're stable.
9#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
10#[expect(non_camel_case_types, reason = "matching wasm conventions")]
11pub(crate) type i8x16 = core::arch::x86_64::__m128i;
12#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
13#[expect(non_camel_case_types, reason = "matching wasm conventions")]
14pub(crate) type f32x4 = core::arch::x86_64::__m128;
15#[cfg(all(target_arch = "x86_64", target_feature = "sse"))]
16#[expect(non_camel_case_types, reason = "matching wasm conventions")]
17pub(crate) type f64x2 = core::arch::x86_64::__m128d;
18
19// On platforms other than x86_64, define i8x16 to a non-constructible type;
20// we need a type because we have a lot of macros for defining builtin
21// functions that are awkward to make conditional on the target, but it
22// doesn't need to actually be constructible unless we're on x86_64.
23#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
24#[expect(non_camel_case_types, reason = "matching wasm conventions")]
25#[derive(Copy, Clone)]
26pub(crate) struct i8x16(crate::uninhabited::Uninhabited);
27#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
28#[expect(non_camel_case_types, reason = "matching wasm conventions")]
29#[derive(Copy, Clone)]
30pub(crate) struct f32x4(crate::uninhabited::Uninhabited);
31#[cfg(not(all(target_arch = "x86_64", target_feature = "sse")))]
32#[expect(non_camel_case_types, reason = "matching wasm conventions")]
33#[derive(Copy, Clone)]
34pub(crate) struct f64x2(crate::uninhabited::Uninhabited);
35
36use crate::StoreContextMut;
37use crate::prelude::*;
38use crate::store::{StoreInner, StoreOpaque, StoreResourceLimiter};
39use crate::type_registry::RegisteredType;
40use alloc::sync::Arc;
41use core::fmt;
42use core::ops::{Deref, DerefMut};
43use core::pin::pin;
44use core::ptr::NonNull;
45use core::sync::atomic::{AtomicUsize, Ordering};
46use core::task::{Context, Poll, Waker};
47use wasmtime_environ::{
48    DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex,
49};
50
51#[cfg(feature = "gc")]
52use wasmtime_environ::ModuleInternedTypeIndex;
53
54mod always_mut;
55#[cfg(feature = "component-model")]
56pub mod component;
57mod const_expr;
58mod export;
59mod gc;
60mod imports;
61mod instance;
62mod memory;
63mod mmap_vec;
64#[cfg(has_virtual_memory)]
65mod pagemap_disabled;
66mod provenance;
67mod send_sync_ptr;
68mod stack_switching;
69mod store_box;
70mod sys;
71mod table;
72#[cfg(feature = "gc")]
73mod throw;
74mod traphandlers;
75mod vmcontext;
76
77#[cfg(feature = "threads")]
78mod parking_spot;
79
80// Note that `debug_builtins` here is disabled with a feature or a lack of a
81// native compilation backend because it's only here to assist in debugging
82// natively compiled code.
83#[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
84pub mod debug_builtins;
85pub mod libcalls;
86pub mod mpk;
87
88#[cfg(feature = "pulley")]
89pub(crate) mod interpreter;
90#[cfg(not(feature = "pulley"))]
91pub(crate) mod interpreter_disabled;
92#[cfg(not(feature = "pulley"))]
93pub(crate) use interpreter_disabled as interpreter;
94
95#[cfg(feature = "debug-builtins")]
96pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
97
98pub use crate::runtime::vm::always_mut::*;
99pub use crate::runtime::vm::export::*;
100pub use crate::runtime::vm::gc::*;
101pub use crate::runtime::vm::imports::Imports;
102pub use crate::runtime::vm::instance::{
103    GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
104    MemoryAllocationIndex, OnDemandInstanceAllocator, TableAllocationIndex, initialize_instance,
105};
106#[cfg(feature = "pooling-allocator")]
107pub use crate::runtime::vm::instance::{
108    InstanceLimits, PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,
109    PoolingInstanceAllocatorConfig,
110};
111pub use crate::runtime::vm::interpreter::*;
112pub use crate::runtime::vm::memory::{
113    Memory, MemoryBase, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory,
114};
115pub use crate::runtime::vm::mmap_vec::MmapVec;
116pub use crate::runtime::vm::provenance::*;
117pub use crate::runtime::vm::stack_switching::*;
118pub use crate::runtime::vm::store_box::*;
119#[cfg(feature = "std")]
120pub use crate::runtime::vm::sys::mmap::open_file_for_mmap;
121#[cfg(has_host_compiler_backend)]
122pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
123pub use crate::runtime::vm::table::{Table, TableElementType};
124#[cfg(feature = "gc")]
125pub use crate::runtime::vm::throw::*;
126pub use crate::runtime::vm::traphandlers::*;
127pub use crate::runtime::vm::vmcontext::{
128    VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport,
129    VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport,
130    VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
131};
132#[cfg(has_custom_sync)]
133pub(crate) use sys::capi;
134
135pub use send_sync_ptr::SendSyncPtr;
136pub use wasmtime_unwinder::Unwind;
137
138#[cfg(has_host_compiler_backend)]
139pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer};
140
141mod module_id;
142pub use module_id::CompiledModuleId;
143
144#[cfg(has_virtual_memory)]
145mod byte_count;
146#[cfg(has_virtual_memory)]
147mod cow;
148#[cfg(not(has_virtual_memory))]
149mod cow_disabled;
150#[cfg(has_virtual_memory)]
151mod mmap;
152
153#[cfg(feature = "async")]
154mod async_yield;
155#[cfg(feature = "async")]
156pub use crate::runtime::vm::async_yield::*;
157
158#[cfg(feature = "gc-null")]
159mod send_sync_unsafe_cell;
160#[cfg(feature = "gc-null")]
161pub use send_sync_unsafe_cell::SendSyncUnsafeCell;
162
163cfg_if::cfg_if! {
164    if #[cfg(has_virtual_memory)] {
165        pub use crate::runtime::vm::byte_count::*;
166        pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
167        pub use self::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
168    } else {
169        pub use self::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
170    }
171}
172
173/// Source of data used for [`MemoryImage`]
174pub trait ModuleMemoryImageSource: Send + Sync + 'static {
175    /// Returns this image's slice of all wasm data for a module which is then
176    /// further sub-sliced for a particular initialization segment.
177    fn wasm_data(&self) -> &[u8];
178
179    /// Optionally returns the backing mmap. Used for using the backing mmap's
180    /// file to perform other mmaps, for example.
181    fn mmap(&self) -> Option<&MmapVec>;
182}
183
184/// Dynamic runtime functionality needed by this crate throughout the execution
185/// of a wasm instance.
186///
187/// This trait is used to store a raw pointer trait object within each
188/// `VMContext`. This raw pointer trait object points back to the
189/// `wasmtime::Store` internally but is type-erased to avoid needing to
190/// monomorphize the entire runtime on the `T` in `Store<T>`
191///
192/// # Safety
193///
194/// This trait should be implemented by nothing other than `StoreInner<T>` in
195/// this crate. It's not sound to implement it for anything else due to
196/// `unchecked_context_mut` below.
197///
198/// It's also worth nothing that there are various locations where a `*mut dyn
199/// VMStore` is asserted to be both `Send` and `Sync` which disregards the `T`
200/// that's actually stored in the store itself. It's assume that the high-level
201/// APIs using `Store<T>` are correctly inferring send/sync on the returned
202/// values (e.g. futures) and that internally in the runtime we aren't doing
203/// anything "weird" with threads for example.
204pub unsafe trait VMStore: 'static {
205    /// Get a shared borrow of this store's `StoreOpaque`.
206    fn store_opaque(&self) -> &StoreOpaque;
207
208    /// Get an exclusive borrow of this store's `StoreOpaque`.
209    fn store_opaque_mut(&mut self) -> &mut StoreOpaque;
210
211    /// Returns a split borrow to the limiter plus `StoreOpaque` at the same
212    /// time.
213    fn resource_limiter_and_store_opaque(
214        &mut self,
215    ) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque);
216
217    /// Callback invoked whenever an instance observes a new epoch
218    /// number. Cannot fail; cooperative epoch-based yielding is
219    /// completely semantically transparent. Returns the new deadline.
220    #[cfg(target_has_atomic = "64")]
221    fn new_epoch_updated_deadline(&mut self) -> Result<crate::UpdateDeadline>;
222
223    /// Metadata required for resources for the component model.
224    #[cfg(feature = "component-model")]
225    fn component_calls(&mut self) -> &mut component::CallContexts;
226
227    #[cfg(feature = "component-model-async")]
228    fn component_async_store(
229        &mut self,
230    ) -> &mut dyn crate::runtime::component::VMComponentAsyncStore;
231
232    /// Invoke a debug handler, if present, at a debug event.
233    #[cfg(feature = "debug")]
234    fn block_on_debug_handler(&mut self, event: crate::DebugEvent) -> anyhow::Result<()>;
235}
236
237impl Deref for dyn VMStore + '_ {
238    type Target = StoreOpaque;
239
240    fn deref(&self) -> &Self::Target {
241        self.store_opaque()
242    }
243}
244
245impl DerefMut for dyn VMStore + '_ {
246    fn deref_mut(&mut self) -> &mut Self::Target {
247        self.store_opaque_mut()
248    }
249}
250
251impl dyn VMStore + '_ {
252    /// Asserts that this `VMStore` was originally paired with `StoreInner<T>`
253    /// and then casts to the `StoreContextMut` type.
254    ///
255    /// # Unsafety
256    ///
257    /// This method is not safe as there's no static guarantee that `T` is
258    /// correct for this store.
259    pub(crate) unsafe fn unchecked_context_mut<T>(&mut self) -> StoreContextMut<'_, T> {
260        unsafe { StoreContextMut(&mut *(self as *mut dyn VMStore as *mut StoreInner<T>)) }
261    }
262}
263
264/// A newtype wrapper around `NonNull<dyn VMStore>` intended to be a
265/// self-pointer back to the `Store<T>` within raw data structures like
266/// `VMContext`.
267///
268/// This type exists to manually, and unsafely, implement `Send` and `Sync`.
269/// The `VMStore` trait doesn't require `Send` or `Sync` which means this isn't
270/// naturally either trait (e.g. with `SendSyncPtr` instead). Note that this
271/// means that `Instance` is, for example, mistakenly considered
272/// unconditionally `Send` and `Sync`. This is hopefully ok for now though
273/// because from a user perspective the only type that matters is `Store<T>`.
274/// That type is `Send + Sync` if `T: Send + Sync` already so the internal
275/// storage of `Instance` shouldn't matter as the final result is the same.
276/// Note though that this means we need to be extra vigilant about cross-thread
277/// usage of `Instance` and `ComponentInstance` for example.
278#[derive(Copy, Clone)]
279#[repr(transparent)]
280struct VMStoreRawPtr(pub NonNull<dyn VMStore>);
281
282// SAFETY: this is the purpose of `VMStoreRawPtr`, see docs above about safe
283// usage.
284unsafe impl Send for VMStoreRawPtr {}
285unsafe impl Sync for VMStoreRawPtr {}
286
287/// Functionality required by this crate for a particular module. This
288/// is chiefly needed for lazy initialization of various bits of
289/// instance state.
290///
291/// When an instance is created, it holds an `Arc<dyn ModuleRuntimeInfo>`
292/// so that it can get to signatures, metadata on functions, memory and
293/// funcref-table images, etc. All of these things are ordinarily known
294/// by the higher-level layers of Wasmtime. Specifically, the main
295/// implementation of this trait is provided by
296/// `wasmtime::module::ModuleInner`.  Since the runtime crate sits at
297/// the bottom of the dependence DAG though, we don't know or care about
298/// that; we just need some implementor of this trait for each
299/// allocation request.
300#[derive(Clone)]
301pub enum ModuleRuntimeInfo {
302    Module(crate::Module),
303    Bare(Box<BareModuleInfo>),
304}
305
306/// A barebones implementation of ModuleRuntimeInfo that is useful for
307/// cases where a purpose-built environ::Module is used and a full
308/// CompiledModule does not exist (for example, for tests or for the
309/// default-callee instance).
310#[derive(Clone)]
311pub struct BareModuleInfo {
312    module: Arc<wasmtime_environ::Module>,
313    offsets: VMOffsets<HostPtr>,
314    _registered_type: Option<RegisteredType>,
315}
316
317impl ModuleRuntimeInfo {
318    pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Self {
319        ModuleRuntimeInfo::bare_with_registered_type(module, None)
320    }
321
322    pub(crate) fn bare_with_registered_type(
323        module: Arc<wasmtime_environ::Module>,
324        registered_type: Option<RegisteredType>,
325    ) -> Self {
326        ModuleRuntimeInfo::Bare(Box::new(BareModuleInfo {
327            offsets: VMOffsets::new(HostPtr, &module),
328            module,
329            _registered_type: registered_type,
330        }))
331    }
332
333    /// The underlying Module.
334    pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
335        match self {
336            ModuleRuntimeInfo::Module(m) => m.env_module(),
337            ModuleRuntimeInfo::Bare(b) => &b.module,
338        }
339    }
340
341    /// Translate a module-level interned type index into an engine-level
342    /// interned type index.
343    #[cfg(feature = "gc")]
344    fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
345        match self {
346            ModuleRuntimeInfo::Module(m) => m
347                .code_object()
348                .signatures()
349                .shared_type(module_index)
350                .expect("bad module-level interned type index"),
351            ModuleRuntimeInfo::Bare(_) => unreachable!(),
352        }
353    }
354
355    /// Returns the address, in memory, that the function `index` resides at.
356    fn function(&self, index: DefinedFuncIndex) -> NonNull<VMWasmCallFunction> {
357        let module = match self {
358            ModuleRuntimeInfo::Module(m) => m,
359            ModuleRuntimeInfo::Bare(_) => unreachable!(),
360        };
361        let ptr = module
362            .compiled_module()
363            .finished_function(index)
364            .as_ptr()
365            .cast::<VMWasmCallFunction>()
366            .cast_mut();
367        NonNull::new(ptr).unwrap()
368    }
369
370    /// Returns the address, in memory, of the trampoline that allows the given
371    /// defined Wasm function to be called by the array calling convention.
372    ///
373    /// Returns `None` for Wasm functions which do not escape, and therefore are
374    /// not callable from outside the Wasm module itself.
375    fn array_to_wasm_trampoline(
376        &self,
377        index: DefinedFuncIndex,
378    ) -> Option<NonNull<VMArrayCallFunction>> {
379        let m = match self {
380            ModuleRuntimeInfo::Module(m) => m,
381            ModuleRuntimeInfo::Bare(_) => unreachable!(),
382        };
383        let ptr = NonNull::from(m.compiled_module().array_to_wasm_trampoline(index)?);
384        Some(ptr.cast())
385    }
386
387    /// Returns the `MemoryImage` structure used for copy-on-write
388    /// initialization of the memory, if it's applicable.
389    fn memory_image(
390        &self,
391        memory: DefinedMemoryIndex,
392    ) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
393        match self {
394            ModuleRuntimeInfo::Module(m) => {
395                let images = m.memory_images()?;
396                Ok(images.and_then(|images| images.get_memory_image(memory)))
397            }
398            ModuleRuntimeInfo::Bare(_) => Ok(None),
399        }
400    }
401
402    /// A unique ID for this particular module. This can be used to
403    /// allow for fastpaths to optimize a "re-instantiate the same
404    /// module again" case.
405    #[cfg(feature = "pooling-allocator")]
406    fn unique_id(&self) -> Option<CompiledModuleId> {
407        match self {
408            ModuleRuntimeInfo::Module(m) => Some(m.id()),
409            ModuleRuntimeInfo::Bare(_) => None,
410        }
411    }
412
413    /// A slice pointing to all data that is referenced by this instance.
414    fn wasm_data(&self) -> &[u8] {
415        match self {
416            ModuleRuntimeInfo::Module(m) => m.compiled_module().code_memory().wasm_data(),
417            ModuleRuntimeInfo::Bare(_) => &[],
418        }
419    }
420
421    /// Returns an array, indexed by `ModuleInternedTypeIndex` of all
422    /// `VMSharedSignatureIndex` entries corresponding to the `SignatureIndex`.
423    fn type_ids(&self) -> &[VMSharedTypeIndex] {
424        match self {
425            ModuleRuntimeInfo::Module(m) => m
426                .code_object()
427                .signatures()
428                .as_module_map()
429                .values()
430                .as_slice(),
431            ModuleRuntimeInfo::Bare(_) => &[],
432        }
433    }
434
435    /// Offset information for the current host.
436    pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
437        match self {
438            ModuleRuntimeInfo::Module(m) => m.offsets(),
439            ModuleRuntimeInfo::Bare(b) => &b.offsets,
440        }
441    }
442}
443
444/// Returns the host OS page size, in bytes.
445#[cfg(has_virtual_memory)]
446pub fn host_page_size() -> usize {
447    // NB: this function is duplicated in `crates/fiber/src/unix.rs` so if this
448    // changes that should probably get updated as well.
449    static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
450
451    return match PAGE_SIZE.load(Ordering::Relaxed) {
452        0 => {
453            let size = sys::vm::get_page_size();
454            assert!(size != 0);
455            PAGE_SIZE.store(size, Ordering::Relaxed);
456            size
457        }
458        n => n,
459    };
460}
461
462/// Result of `Memory::atomic_wait32` and `Memory::atomic_wait64`
463#[derive(Copy, Clone, PartialEq, Eq, Debug)]
464pub enum WaitResult {
465    /// Indicates that a `wait` completed by being awoken by a different thread.
466    /// This means the thread went to sleep and didn't time out.
467    Ok = 0,
468    /// Indicates that `wait` did not complete and instead returned due to the
469    /// value in memory not matching the expected value.
470    Mismatch = 1,
471    /// Indicates that `wait` completed with a timeout, meaning that the
472    /// original value matched as expected but nothing ever called `notify`.
473    TimedOut = 2,
474}
475
476/// Description about a fault that occurred in WebAssembly.
477#[derive(Debug)]
478pub struct WasmFault {
479    /// The size of memory, in bytes, at the time of the fault.
480    pub memory_size: usize,
481    /// The WebAssembly address at which the fault occurred.
482    pub wasm_address: u64,
483}
484
485impl fmt::Display for WasmFault {
486    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
487        write!(
488            f,
489            "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
490            self.wasm_address, self.memory_size,
491        )
492    }
493}
494
495/// Asserts that the future `f` is ready and returns its output.
496///
497/// This function is intended to be used when `async_support` is verified as
498/// disabled. Internals of Wasmtime are generally `async` when they optionally
499/// can be, meaning that synchronous entrypoints will invoke this function
500/// after invoking the asynchronous internals. Due to `async_support` being
501/// disabled there should be no way to introduce a yield point meaning that all
502/// futures built from internal functions should always be ready.
503///
504/// # Panics
505///
506/// Panics if `f` is not yet ready.
507pub fn assert_ready<F: Future>(f: F) -> F::Output {
508    one_poll(f).unwrap()
509}
510
511/// Attempts one poll of `f` to see if its output is available.
512///
513/// This function is intended for a few minor entrypoints into the Wasmtime API
514/// where a synchronous function is documented to work even when `async_support`
515/// is enabled. For example growing a `Memory` can be done with a synchronous
516/// function, but it's documented to panic with an async resource limiter.
517///
518/// This function provides the opportunity to poll `f` once to see if its output
519/// is available. If it isn't then `None` is returned and an appropriate panic
520/// message should be generated recommending to use an async function (e.g.
521/// `grow_async` instead of `grow`).
522pub fn one_poll<F: Future>(f: F) -> Option<F::Output> {
523    let mut context = Context::from_waker(&Waker::noop());
524    match pin!(f).poll(&mut context) {
525        Poll::Ready(output) => Some(output),
526        Poll::Pending => None,
527    }
528}
wasmtime/runtime/vm.rs

wasmtime/runtime/
vm.rs
