wasmtime/runtime/vm/instance.rs
1//! An `Instance` contains all the runtime state used by execution of a
2//! wasm module (except its callstack and register state). An
3//! `InstanceHandle` is a reference-counting handle for an `Instance`.
4
5use crate::code::ModuleWithCode;
6use crate::module::ModuleRegistry;
7use crate::prelude::*;
8use crate::runtime::vm::export::{Export, ExportMemory};
9use crate::runtime::vm::memory::Memory;
10use crate::runtime::vm::table::{Table, TableElementType};
11use crate::runtime::vm::vmcontext::{
12 VMBuiltinFunctionsArray, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,
13 VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStoreContext,
14 VMTableDefinition, VMTableImport, VMTagDefinition, VMTagImport,
15};
16use crate::runtime::vm::{
17 GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, UncaughtException, VMGcRef,
18 VMGlobalKind, VMStore, VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap,
19};
20use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter};
21use crate::vm::{VMWasmCallFunction, ValRaw};
22use alloc::sync::Arc;
23use core::alloc::Layout;
24use core::marker;
25use core::ops::Range;
26use core::pin::Pin;
27use core::ptr::NonNull;
28#[cfg(target_has_atomic = "64")]
29use core::sync::atomic::AtomicU64;
30use core::{mem, ptr};
31#[cfg(feature = "gc")]
32use wasmtime_environ::ModuleInternedTypeIndex;
33use wasmtime_environ::error::OutOfMemory;
34use wasmtime_environ::{
35 Abi, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,
36 DefinedTagIndex, EntityIndex, EntityRef, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,
37 HostPtr, MemoryIndex, MemoryInitialization, ModuleStartup, PassiveElemIndex, PtrSize,
38 RuntimeDataIndex, TableIndex, TagIndex, VMCONTEXT_MAGIC, VMOffsets, VMSharedTypeIndex,
39 WasmRefType, packed_option::ReservedValue,
40};
41#[cfg(feature = "wmemcheck")]
42use wasmtime_wmemcheck::Wmemcheck;
43
44mod allocator;
45pub use allocator::*;
46
47/// A type that roughly corresponds to a WebAssembly instance, but is also used
48/// for host-defined objects.
49///
50/// Instances here can correspond to actual instantiated modules, but it's also
51/// used ubiquitously for host-defined objects. For example creating a
52/// host-defined memory will have a `module` that looks like it exports a single
53/// memory (and similar for other constructs).
54///
55/// This `Instance` type is used as a ubiquitous representation for WebAssembly
56/// values, whether or not they were created on the host or through a module.
57///
58/// # Ownership
59///
60/// This structure is never allocated directly but is instead managed through
61/// an `InstanceHandle`. This structure ends with a `VMContext` which has a
62/// dynamic size corresponding to the `module` configured within. Memory
63/// management of this structure is always done through `InstanceHandle` as the
64/// sole owner of an instance.
65///
66/// # `Instance` and `Pin`
67///
68/// Given an instance it is accompanied with trailing memory for the
69/// appropriate `VMContext`. The `Instance` also holds `runtime_info` and other
70/// information pointing to relevant offsets for the `VMContext`. Thus it is
71/// not sound to mutate `runtime_info` after an instance is created. More
72/// generally it's also not safe to "swap" instances, for example given two
73/// `&mut Instance` values it's not sound to swap them as then the `VMContext`
74/// values are inaccurately described.
75///
76/// To encapsulate this guarantee this type is only ever mutated through Rust's
77/// `Pin` type. All mutable methods here take `self: Pin<&mut Self>` which
78/// statically disallows safe access to `&mut Instance`. There are assorted
79/// "projection methods" to go from `Pin<&mut Instance>` to `&mut T` for
80/// individual fields, for example `memories_mut`. More methods can be added as
81/// necessary or methods may also be added to project multiple fields at a time
82/// if necessary to. The precise ergonomics around getting mutable access to
83/// some fields (but notably not `runtime_info`) is probably going to evolve
84/// over time.
85///
86/// Note that is is not sound to basically ever pass around `&mut Instance`.
87/// That should always instead be `Pin<&mut Instance>`. All usage of
88/// `Pin::new_unchecked` should be here in this module in just a few `unsafe`
89/// locations and it's recommended to use existing helpers if you can.
90#[repr(C)] // ensure that the vmctx field is last.
91pub struct Instance {
92 /// The index, within a `Store` that this instance lives at
93 id: InstanceId,
94
95 /// The runtime info (corresponding to the "compiled module"
96 /// abstraction in higher layers) that is retained and needed for
97 /// lazy initialization. This provides access to the underlying
98 /// Wasm module entities, the compiled JIT code, metadata about
99 /// functions, lazy initialization state, etc.
100 //
101 // SAFETY: this field cannot be overwritten after an instance is created. It
102 // must contain this exact same value for the entire lifetime of this
103 // instance. This enables borrowing the info's `Module` and this instance at
104 // the same time (instance mutably, module not). Additionally it enables
105 // borrowing a store mutably at the same time as a contained instance.
106 runtime_info: ModuleRuntimeInfo,
107
108 /// WebAssembly linear memory data.
109 ///
110 /// This is where all runtime information about defined linear memories in
111 /// this module lives.
112 ///
113 /// The `MemoryAllocationIndex` was given from our `InstanceAllocator` and
114 /// must be given back to the instance allocator when deallocating each
115 /// memory.
116 memories: TryPrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)>,
117
118 /// WebAssembly table data.
119 ///
120 /// Like memories, this is only for defined tables in the module and
121 /// contains all of their runtime state.
122 ///
123 /// The `TableAllocationIndex` was given from our `InstanceAllocator` and
124 /// must be given back to the instance allocator when deallocating each
125 /// table.
126 tables: TryPrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)>,
127
128 /// Evaluated passive element segments.
129 ///
130 /// If an entry is none, then it has been dropped.
131 //
132 // TODO(#12621): This should be a `TrySecondaryMap<PassiveElemIndex, _>`
133 // but that type is currently footgun-y / isn't actually OOM-safe yet.
134 passive_elements: TryVec<PassiveElementSegment>,
135
136 // TODO: add support for multiple memories; `wmemcheck_state` corresponds to
137 // memory 0.
138 #[cfg(feature = "wmemcheck")]
139 pub(crate) wmemcheck_state: Option<Wmemcheck>,
140
141 /// Self-pointer back to `Store<T>` and its functions. Not present for
142 /// the brief time that `Store<T>` is itself being created. Also not
143 /// present for some niche uses that are disconnected from stores (e.g.
144 /// cross-thread stuff used in `InstancePre`)
145 store: Option<VMStoreRawPtr>,
146
147 /// Additional context used by compiled wasm code. This field is last, and
148 /// represents a dynamically-sized array that extends beyond the nominal
149 /// end of the struct (similar to a flexible array member).
150 vmctx: OwnedVMContext<VMContext>,
151}
152
153impl Instance {
154 /// Creates a new owned instance handle from `req`.
155 ///
156 /// The runtime memory/table data structures must have been previously
157 /// allocated and are present within `memories` and `tables`. These values
158 /// are `mem::take`n upon allocation success of an instance, and if the
159 /// instance allocation fails then these are otherwise left in place. This
160 /// enable the pooling allocator to run custom deallocation code for them,
161 /// for example.
162 ///
163 /// # Safety
164 ///
165 /// The `req.imports` field must be appropriately sized/typed for the module
166 /// being allocated according to `req.runtime_info`. Additionally `memories`
167 /// and `tables` must have been allocated for `req.store`.
168 unsafe fn new(
169 req: InstanceAllocationRequest,
170 memories: &mut TryPrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)>,
171 tables: &mut TryPrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)>,
172 ) -> Result<InstanceHandle, OutOfMemory> {
173 let module = req.runtime_info.env_module();
174 let memory_tys = &module.memories;
175 let mut passive_elements = TryVec::with_capacity(module.passive_elements.len())?;
176
177 #[cfg(feature = "wmemcheck")]
178 let wmemcheck_state = if req.store.engine().config().wmemcheck {
179 let size = memory_tys
180 .iter()
181 .next()
182 .map(|memory| memory.1.limits.min)
183 .unwrap_or(0)
184 * 64
185 * 1024;
186 Some(Wmemcheck::new(size.try_into().unwrap()))
187 } else {
188 None
189 };
190 #[cfg(not(feature = "wmemcheck"))]
191 let _ = memory_tys;
192
193 for (_, (ty, len)) in req.runtime_info.env_module().passive_elements.iter() {
194 let len = usize::try_from(*len).unwrap();
195 passive_elements.push(PassiveElementSegment::new(*ty, len)?)?;
196 }
197
198 // Allocate the instance and its `VMContext` with empty memory and table
199 // maps. This is the final fallible allocation in this function; only
200 // after it succeeds do we transfer ownership of the pool-allocated
201 // `memories`/`tables` into the instance (below), so that a failure here
202 // leaves them in the caller's deallocation guard to be freed.
203 let mut ret = OwnedInstance::new(Instance {
204 id: req.id,
205 runtime_info: req.runtime_info.clone(),
206 memories: TryPrimaryMap::default(),
207 tables: TryPrimaryMap::default(),
208 passive_elements,
209 #[cfg(feature = "wmemcheck")]
210 wmemcheck_state,
211 store: None,
212 vmctx: OwnedVMContext::new(),
213 })?;
214
215 // Can't fail any more, so transfer ownership of `memories` and `tables`
216 // to this instance.
217 *ret.get_mut().memories_mut() = mem::take(memories);
218 *ret.get_mut().tables_mut() = mem::take(tables);
219
220 // SAFETY: this vmctx was allocated with the same layout above, so it
221 // should be safe to initialize with the same values here.
222 unsafe {
223 ret.get_mut().initialize_vmctx(req.store, req.imports);
224 }
225
226 Ok(ret)
227 }
228
229 /// Trace element segment GC roots inside this `Instance`.
230 ///
231 /// # Safety
232 ///
233 /// This instance must live for the duration of the associated GC cycle.
234 #[cfg(feature = "gc")]
235 pub(crate) unsafe fn trace_element_segment_roots(
236 self: Pin<&mut Self>,
237 gc_roots: &mut crate::vm::GcRootsList,
238 ) {
239 for segment in self.passive_elements_mut().iter_mut() {
240 if segment.needs_gc_rooting {
241 for e in segment.elements_mut() {
242 if e.get_vmgcref().is_none() {
243 continue;
244 }
245
246 let root: SendSyncPtr<ValRaw> = e.into();
247
248 // Safety: We know this is a type that needs GC rooting and
249 // the lifetime is implied by our safety contract.
250 unsafe {
251 gc_roots.add_val_raw_root(root, "passive element segment");
252 }
253 }
254 }
255 }
256 }
257
258 /// Converts a raw `VMContext` pointer into a raw `Instance` pointer.
259 ///
260 /// # Safety
261 ///
262 /// Calling this function safely requires that `vmctx` is a valid allocation
263 /// of a `VMContext` which is derived from `Instance::new`. To safely
264 /// convert the returned raw pointer into a safe instance pointer callers
265 /// will also want to uphold guarantees such as:
266 ///
267 /// * The instance should not be in use elsewhere. For example you can't
268 /// call this function twice, turn both raw pointers into safe pointers,
269 /// and then use both safe pointers.
270 /// * There should be no other active mutable borrow to any other instance
271 /// within the same store. Note that this is not restricted to just this
272 /// instance pointer, but to all instances in a store. Instances can
273 /// safely traverse to other instances "laterally" meaning that a mutable
274 /// borrow on one is a mutable borrow on all.
275 /// * There should be no active mutable borrow on the store accessible at
276 /// the same time the instance is turned. Instances are owned by a store
277 /// and a store can be used to acquire a safe instance borrow at any time.
278 /// * The lifetime of the usage of the instance should not be unnecessarily
279 /// long, for example it cannot be `'static`.
280 ///
281 /// Other entrypoints exist for converting from a raw `VMContext` to a safe
282 /// pointer such as:
283 ///
284 /// * `Instance::enter_host_from_wasm`
285 /// * `Instance::sibling_vmctx{,_mut}`
286 ///
287 /// These place further restrictions on the API signature to satisfy some of
288 /// the above points.
289 #[inline]
290 pub(crate) unsafe fn from_vmctx(vmctx: NonNull<VMContext>) -> NonNull<Instance> {
291 // SAFETY: The validity of `byte_sub` relies on `vmctx` being a valid
292 // allocation.
293 unsafe {
294 vmctx
295 .byte_sub(mem::size_of::<Instance>())
296 .cast::<Instance>()
297 }
298 }
299
300 /// Encapsulated entrypoint to the host from WebAssembly, converting a raw
301 /// `VMContext` pointer into a `VMStore` plus an `InstanceId`.
302 ///
303 /// This is an entrypoint for core wasm entering back into the host. This is
304 /// used for both host functions and libcalls for example. This will execute
305 /// the closure `f` with safer Internal types than a raw `VMContext`
306 /// pointer.
307 ///
308 /// The closure `f` will have its errors caught, handled, and translated to
309 /// an ABI-safe return value to give back to wasm. This includes both normal
310 /// errors such as traps as well as panics.
311 ///
312 /// # Safety
313 ///
314 /// Callers must ensure that `vmctx` is a valid allocation and is safe to
315 /// dereference at this time. That's generally only true when it's a
316 /// wasm-provided value and this is the first function called after entering
317 /// the host. Otherwise this could unsafely alias the store with a mutable
318 /// pointer, for example.
319 #[inline]
320 pub(crate) unsafe fn enter_host_from_wasm<R>(
321 vmctx: NonNull<VMContext>,
322 f: impl FnOnce(&mut dyn VMStore, InstanceId) -> R,
323 ) -> R::Abi
324 where
325 R: HostResult,
326 {
327 // SAFETY: It's a contract of this function that `vmctx` is a valid
328 // pointer with neither the store nor other instances actively in use
329 // when this is called, so it should be safe to acquire a mutable
330 // pointer to the store and read the instance pointer.
331 let (store, instance) = unsafe {
332 let instance = Instance::from_vmctx(vmctx);
333 let instance = instance.as_ref();
334 let store = &mut *instance.store.unwrap().0.as_ptr();
335 (store, instance.id)
336 };
337
338 // Thread the `store` and `instance` through panic/trap infrastructure
339 // back into `f`.
340 catch_unwind_and_record_trap(store, UncaughtException::Propagate, |store| {
341 f(store, instance)
342 })
343 }
344
345 /// Converts the provided `*mut VMContext` to an `Instance` pointer and
346 /// returns it with the same lifetime as `self`.
347 ///
348 /// This function can be used when traversing a `VMContext` to reach into
349 /// the context needed for imports, optionally.
350 ///
351 /// # Safety
352 ///
353 /// This function requires that the `vmctx` pointer is indeed valid and
354 /// from the store that `self` belongs to.
355 #[inline]
356 unsafe fn sibling_vmctx<'a>(&'a self, vmctx: NonNull<VMContext>) -> &'a Instance {
357 // SAFETY: it's a contract of this function itself that `vmctx` is a
358 // valid pointer. Additionally with `self` being a
359 let ptr = unsafe { Instance::from_vmctx(vmctx) };
360 // SAFETY: it's a contract of this function itself that `vmctx` is a
361 // valid pointer to dereference. Additionally the lifetime of the return
362 // value is constrained to be the same as `self` to avoid granting a
363 // too-long lifetime.
364 unsafe { ptr.as_ref() }
365 }
366
367 /// Same as [`Self::sibling_vmctx`], but the mutable version.
368 ///
369 /// # Safety
370 ///
371 /// This function requires that the `vmctx` pointer is indeed valid and
372 /// from the store that `self` belongs to.
373 ///
374 /// (Note that it is *NOT* required that `vmctx` be distinct from this
375 /// instance's `vmctx`, or that usage of the resulting instance is limited
376 /// to its defined items! The returned borrow has the same lifetime as
377 /// `self`, which means that this instance cannot be used while the
378 /// resulting instance is in use, and we therefore do not need to worry
379 /// about mutable aliasing between this instance and the resulting
380 /// instance.)
381 #[inline]
382 unsafe fn sibling_vmctx_mut<'a>(
383 self: Pin<&'a mut Self>,
384 vmctx: NonNull<VMContext>,
385 ) -> Pin<&'a mut Instance> {
386 // SAFETY: it's a contract of this function itself that `vmctx` is a
387 // valid pointer such that this pointer arithmetic is valid.
388 let mut ptr = unsafe { Instance::from_vmctx(vmctx) };
389
390 // SAFETY: it's a contract of this function itself that `vmctx` is a
391 // valid pointer to dereference. Additionally the lifetime of the return
392 // value is constrained to be the same as `self` to avoid granting a
393 // too-long lifetime. Finally mutable references to an instance are
394 // always through `Pin`, so it's safe to create a pin-pointer here.
395 unsafe { Pin::new_unchecked(ptr.as_mut()) }
396 }
397
398 pub(crate) fn env_module(&self) -> &Arc<wasmtime_environ::Module> {
399 self.runtime_info.env_module()
400 }
401
402 pub(crate) fn runtime_module(&self) -> Option<&crate::Module> {
403 match &self.runtime_info {
404 ModuleRuntimeInfo::Module(m) => Some(m),
405 ModuleRuntimeInfo::Bare(_) => None,
406 }
407 }
408
409 /// Translate a module-level interned type index into an engine-level
410 /// interned type index.
411 #[cfg(feature = "gc")]
412 pub fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
413 self.runtime_info.engine_type_index(module_index)
414 }
415
416 #[inline]
417 fn offsets(&self) -> &VMOffsets<HostPtr> {
418 self.runtime_info.offsets()
419 }
420
421 /// Return the indexed `VMFunctionImport`.
422 fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport {
423 unsafe { self.vmctx_plus_offset(self.offsets().imported_functions().at(index)) }
424 }
425
426 /// Return the index `VMTableImport`.
427 fn imported_table(&self, index: TableIndex) -> &VMTableImport {
428 unsafe { self.vmctx_plus_offset(self.offsets().imported_tables().at(index)) }
429 }
430
431 /// Return the indexed `VMMemoryImport`.
432 fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport {
433 unsafe { self.vmctx_plus_offset(self.offsets().imported_memories().at(index)) }
434 }
435
436 /// Return the indexed `VMGlobalImport`.
437 fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport {
438 unsafe { self.vmctx_plus_offset(self.offsets().imported_globals().at(index)) }
439 }
440
441 /// Return the indexed `VMTagImport`.
442 fn imported_tag(&self, index: TagIndex) -> &VMTagImport {
443 unsafe { self.vmctx_plus_offset(self.offsets().imported_tags().at(index)) }
444 }
445
446 /// Return the indexed `VMTagDefinition`.
447 pub fn tag_ptr(&self, index: DefinedTagIndex) -> NonNull<VMTagDefinition> {
448 unsafe { self.vmctx_plus_offset_raw(self.offsets().tags().at(index)) }
449 }
450
451 /// Return the indexed `VMTableDefinition`.
452 pub fn table(&self, index: DefinedTableIndex) -> VMTableDefinition {
453 unsafe { self.table_ptr(index).read() }
454 }
455
456 /// Updates the value for a defined table to `VMTableDefinition`.
457 fn set_table(self: Pin<&mut Self>, index: DefinedTableIndex, table: VMTableDefinition) {
458 unsafe {
459 self.table_ptr(index).write(table);
460 }
461 }
462
463 /// Return a pointer to the `index`'th table within this instance, stored
464 /// in vmctx memory.
465 pub fn table_ptr(&self, index: DefinedTableIndex) -> NonNull<VMTableDefinition> {
466 unsafe { self.vmctx_plus_offset_raw(self.offsets().tables().at(index)) }
467 }
468
469 /// Get a locally defined or imported memory.
470 #[cfg(all(has_host_compiler_backend, feature = "debug-builtins"))]
471 pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition {
472 if let Some(defined_index) = self.env_module().defined_memory_index(index) {
473 self.memory(defined_index)
474 } else {
475 let import = self.imported_memory(index);
476 unsafe { VMMemoryDefinition::load(import.from.as_ptr()) }
477 }
478 }
479
480 /// Return the indexed `VMMemoryDefinition`, loaded from vmctx memory
481 /// already.
482 #[inline]
483 pub fn memory(&self, index: DefinedMemoryIndex) -> VMMemoryDefinition {
484 unsafe { VMMemoryDefinition::load(self.memory_ptr(index).as_ptr()) }
485 }
486
487 /// Set the indexed memory to `VMMemoryDefinition`.
488 fn set_memory(&self, index: DefinedMemoryIndex, mem: VMMemoryDefinition) {
489 unsafe {
490 self.memory_ptr(index).write(mem);
491 }
492 }
493
494 /// Return the address of the specified memory at `index` within this vmctx.
495 ///
496 /// Note that the returned pointer resides in wasm-code-readable-memory in
497 /// the vmctx.
498 #[inline]
499 pub fn memory_ptr(&self, index: DefinedMemoryIndex) -> NonNull<VMMemoryDefinition> {
500 unsafe {
501 self.vmctx_plus_offset::<VmPtr<_>>(self.offsets().memories().at(index))
502 .as_non_null()
503 }
504 }
505
506 /// Return the indexed `VMGlobalDefinition`.
507 pub fn global_ptr(&self, index: DefinedGlobalIndex) -> NonNull<VMGlobalDefinition> {
508 unsafe { self.vmctx_plus_offset_raw(self.offsets().globals().at(index)) }
509 }
510
511 /// Get all globals within this instance.
512 ///
513 /// Returns both import and defined globals.
514 ///
515 /// Returns both exported and non-exported globals.
516 ///
517 /// Gives access to the full globals space.
518 pub fn all_globals(
519 &self,
520 store: StoreId,
521 ) -> impl ExactSizeIterator<Item = (GlobalIndex, crate::Global)> + '_ {
522 let module = self.env_module();
523 module
524 .globals
525 .keys()
526 .map(move |idx| (idx, self.get_exported_global(store, idx)))
527 }
528
529 /// Get the globals defined in this instance (not imported).
530 pub fn defined_globals(
531 &self,
532 store: StoreId,
533 ) -> impl ExactSizeIterator<Item = (DefinedGlobalIndex, crate::Global)> + '_ {
534 let module = self.env_module();
535 self.all_globals(store)
536 .skip(module.num_imported_globals)
537 .map(move |(i, global)| (module.defined_global_index(i).unwrap(), global))
538 }
539
540 /// Return a pointer to the interrupts structure
541 #[inline]
542 pub fn vm_store_context(&self) -> NonNull<Option<VmPtr<VMStoreContext>>> {
543 unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().store_context()) }
544 }
545
546 /// Return a pointer to the global epoch counter used by this instance.
547 #[cfg(target_has_atomic = "64")]
548 pub fn epoch_ptr(self: Pin<&mut Self>) -> &mut Option<VmPtr<AtomicU64>> {
549 let offset = self.offsets().ptr.vmctx().epoch_ptr();
550 unsafe { self.vmctx_plus_offset_mut(offset) }
551 }
552
553 /// Return a pointer to the collector-specific heap data.
554 pub fn gc_heap_data(self: Pin<&mut Self>) -> &mut Option<VmPtr<u8>> {
555 let offset = self.offsets().ptr.vmctx().gc_heap_data();
556 unsafe { self.vmctx_plus_offset_mut(offset) }
557 }
558
559 pub(crate) unsafe fn set_store(mut self: Pin<&mut Self>, store: &StoreOpaque) {
560 // FIXME: should be more targeted ideally with the `unsafe` than just
561 // throwing this entire function in a large `unsafe` block.
562 unsafe {
563 *self.as_mut().store_mut() = Some(VMStoreRawPtr(store.traitobj()));
564 self.vm_store_context()
565 .write(Some(store.vm_store_context_ptr().into()));
566 #[cfg(target_has_atomic = "64")]
567 {
568 *self.as_mut().epoch_ptr() =
569 Some(NonNull::from(store.engine().epoch_counter()).into());
570 }
571
572 if self.env_module().needs_gc_heap {
573 self.as_mut().set_gc_heap(Some(store.unwrap_gc_store()));
574 } else {
575 self.as_mut().set_gc_heap(None);
576 }
577 }
578 }
579
580 unsafe fn set_gc_heap(self: Pin<&mut Self>, gc_store: Option<&GcStore>) {
581 if let Some(gc_store) = gc_store {
582 *self.gc_heap_data() =
583 Some(unsafe { gc_store.gc_heap.get().vmctx_gc_heap_data().into() });
584 } else {
585 *self.gc_heap_data() = None;
586 }
587 }
588
589 /// Return a reference to the vmctx used by compiled wasm code.
590 #[inline]
591 pub fn vmctx(&self) -> NonNull<VMContext> {
592 InstanceLayout::vmctx(self)
593 }
594
595 /// Lookup a function by index.
596 ///
597 /// # Panics
598 ///
599 /// Panics if `index` is out of bounds for this instance.
600 ///
601 /// # Safety
602 ///
603 /// The `store` parameter must be the store that owns this instance and the
604 /// functions that this instance can reference.
605 pub unsafe fn get_exported_func(
606 self: Pin<&mut Self>,
607 registry: &ModuleRegistry,
608 store: StoreId,
609 index: FuncIndex,
610 ) -> crate::Func {
611 let func_ref = self.get_func_ref(registry, index).unwrap();
612
613 // SAFETY: the validity of `func_ref` is guaranteed by the validity of
614 // `self`, and the contract that `store` must own `func_ref` is a
615 // contract of this function itself.
616 unsafe { crate::Func::from_vm_func_ref(store, func_ref) }
617 }
618
619 /// Returns a `Func` corresponding to the startup function for this
620 /// instance, if generated at compile time.
621 ///
622 /// # Safety
623 ///
624 /// The `store` parameter must be the store that owns this instance and the
625 /// functions that this instance can reference.
626 pub unsafe fn get_startup_func(
627 self: Pin<&mut Self>,
628 registry: &ModuleRegistry,
629 store: StoreId,
630 ) -> Option<crate::Func> {
631 let func_ref = self.get_start_func_ref(registry)?;
632
633 // SAFETY: the validity of `func_ref` is guaranteed by the validity of
634 // `self`, and the contract that `store` must own `func_ref` is a
635 // contract of this function itself.
636 Some(unsafe { crate::Func::from_vm_func_ref(store, func_ref) })
637 }
638
639 /// Lookup a table by index.
640 ///
641 /// # Panics
642 ///
643 /// Panics if `index` is out of bounds for this instance.
644 pub fn get_exported_table(&self, store: StoreId, index: TableIndex) -> crate::Table {
645 let (id, def_index) = if let Some(def_index) = self.env_module().defined_table_index(index)
646 {
647 (self.id, def_index)
648 } else {
649 let import = self.imported_table(index);
650 // SAFETY: validity of this `Instance` guarantees validity of the
651 // `vmctx` pointer being read here to find the transitive
652 // `InstanceId` that the import is associated with.
653 let id = unsafe { self.sibling_vmctx(import.vmctx.as_non_null()).id };
654 (id, import.index)
655 };
656 crate::Table::from_raw(StoreInstanceId::new(store, id), def_index)
657 }
658
659 /// Lookup a memory by index.
660 ///
661 /// # Panics
662 ///
663 /// Panics if `index` is out-of-bounds for this instance.
664 #[cfg_attr(
665 not(feature = "threads"),
666 expect(unused_variables, reason = "definitions cfg'd to dummy",)
667 )]
668 pub fn get_exported_memory(&self, store: StoreId, index: MemoryIndex) -> ExportMemory {
669 let module = self.env_module();
670 if module.memories[index].shared {
671 let (memory, import) =
672 if let Some(def_index) = self.env_module().defined_memory_index(index) {
673 (
674 self.get_defined_memory(def_index),
675 self.get_defined_memory_vmimport(def_index),
676 )
677 } else {
678 let import = self.imported_memory(index);
679 // SAFETY: validity of this `Instance` guarantees validity of
680 // the `vmctx` pointer being read here to find the transitive
681 // `InstanceId` that the import is associated with.
682 let instance = unsafe { self.sibling_vmctx(import.vmctx.as_non_null()) };
683 (instance.get_defined_memory(import.index), *import)
684 };
685
686 let vm = memory.as_shared_memory().unwrap().clone();
687 ExportMemory::Shared(vm, import)
688 } else {
689 let (id, def_index) =
690 if let Some(def_index) = self.env_module().defined_memory_index(index) {
691 (self.id, def_index)
692 } else {
693 let import = self.imported_memory(index);
694 // SAFETY: validity of this `Instance` guarantees validity of the
695 // `vmctx` pointer being read here to find the transitive
696 // `InstanceId` that the import is associated with.
697 let id = unsafe { self.sibling_vmctx(import.vmctx.as_non_null()).id };
698 (id, import.index)
699 };
700
701 // SAFETY: `from_raw` requires that the memory is not shared, which
702 // was tested above in this if/else.
703 let store_id = StoreInstanceId::new(store, id);
704 ExportMemory::Unshared(unsafe { crate::Memory::from_raw(store_id, def_index) })
705 }
706 }
707
708 /// Lookup a global by index.
709 ///
710 /// # Panics
711 ///
712 /// Panics if `index` is out-of-bounds for this instance.
713 pub(crate) fn get_exported_global(&self, store: StoreId, index: GlobalIndex) -> crate::Global {
714 // If this global is defined within this instance, then that's easy to
715 // calculate the `Global`.
716 if let Some(def_index) = self.env_module().defined_global_index(index) {
717 let instance = StoreInstanceId::new(store, self.id);
718 return crate::Global::from_core(instance, def_index);
719 }
720
721 // For imported globals it's required to match on the `kind` to
722 // determine which `Global` constructor is going to be invoked.
723 let import = self.imported_global(index);
724 match import.kind {
725 VMGlobalKind::Host(index) => crate::Global::from_host(store, index),
726 VMGlobalKind::Instance(index) => {
727 // SAFETY: validity of this `&Instance` means validity of its
728 // imports meaning we can read the id of the vmctx within.
729 let id = unsafe {
730 let vmctx = VMContext::from_opaque(import.vmctx.unwrap().as_non_null());
731 self.sibling_vmctx(vmctx).id
732 };
733 crate::Global::from_core(StoreInstanceId::new(store, id), index)
734 }
735 #[cfg(feature = "component-model")]
736 VMGlobalKind::ComponentFlags(index) => {
737 // SAFETY: validity of this `&Instance` means validity of its
738 // imports meaning we can read the id of the vmctx within.
739 let id = unsafe {
740 let vmctx = super::component::VMComponentContext::from_opaque(
741 import.vmctx.unwrap().as_non_null(),
742 );
743 super::component::ComponentInstance::vmctx_instance_id(vmctx)
744 };
745 crate::Global::from_component_flags(
746 crate::component::store::StoreComponentInstanceId::new(store, id),
747 index,
748 )
749 }
750 }
751 }
752
753 /// Get an exported tag by index.
754 ///
755 /// # Panics
756 ///
757 /// Panics if the index is out-of-range.
758 pub fn get_exported_tag(&self, store: StoreId, index: TagIndex) -> crate::Tag {
759 let (id, def_index) = if let Some(def_index) = self.env_module().defined_tag_index(index) {
760 (self.id, def_index)
761 } else {
762 let import = self.imported_tag(index);
763 // SAFETY: validity of this `Instance` guarantees validity of the
764 // `vmctx` pointer being read here to find the transitive
765 // `InstanceId` that the import is associated with.
766 let id = unsafe { self.sibling_vmctx(import.vmctx.as_non_null()).id };
767 (id, import.index)
768 };
769 crate::Tag::from_raw(StoreInstanceId::new(store, id), def_index)
770 }
771
772 /// Grow memory by the specified amount of pages.
773 ///
774 /// Returns `None` if memory can't be grown by the specified amount
775 /// of pages. Returns `Some` with the old size in bytes if growth was
776 /// successful.
777 pub(crate) async fn memory_grow(
778 mut self: Pin<&mut Self>,
779 limiter: Option<&mut StoreResourceLimiter<'_>>,
780 idx: DefinedMemoryIndex,
781 delta: u64,
782 ) -> Result<Option<usize>, Error> {
783 let memory = &mut self.as_mut().memories_mut()[idx].1;
784
785 // SAFETY: this is the safe wrapper around `Memory::grow` because it
786 // automatically updates the `VMMemoryDefinition` in this instance after
787 // a growth operation below.
788 let result = unsafe { memory.grow(delta, limiter).await };
789
790 // Update the state used by a non-shared Wasm memory in case the base
791 // pointer and/or the length changed.
792 if memory.as_shared_memory().is_none() {
793 let vmmemory = memory.vmmemory();
794 self.set_memory(idx, vmmemory);
795 }
796
797 result
798 }
799
800 /// Performs a grow operation on the `table_index` specified using `grow`.
801 ///
802 /// This will handle updating the VMTableDefinition internally as necessary.
803 ///
804 /// # Safety
805 ///
806 /// This function requires that the caller, on success, fills in the table
807 /// elements with an appropriately typed value.
808 pub(crate) async unsafe fn defined_table_grow(
809 mut self: Pin<&mut Self>,
810 table_index: DefinedTableIndex,
811 limiter: Option<&mut StoreResourceLimiter<'_>>,
812 amt: u64,
813 ) -> Result<Option<usize>> {
814 let table = self.as_mut().get_defined_table(table_index);
815 // SAFETY: updating the `VMContext` table pointers and such is done
816 // below, and the responsibility of filling in the new table elements
817 // is forwarded to the caller.
818 let result = unsafe { table.grow(limiter, amt).await? };
819 let element = table.vmtable();
820 self.set_table(table_index, element);
821 Ok(result)
822 }
823
824 fn alloc_layout(offsets: &VMOffsets<HostPtr>) -> Layout {
825 let size = mem::size_of::<Self>()
826 .checked_add(usize::try_from(offsets.size_of_vmctx()).unwrap())
827 .unwrap();
828 let align = mem::align_of::<Self>();
829 Layout::from_size_align(size, align).unwrap()
830 }
831
832 fn type_ids_array(&self) -> NonNull<VmPtr<VMSharedTypeIndex>> {
833 unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().type_ids()) }
834 }
835
836 /// Get a `&VMFuncRef` for the given `FuncIndex`.
837 ///
838 /// Returns `None` if the index is the reserved index value.
839 ///
840 /// The returned reference is a stable reference that won't be moved and can
841 /// be passed into JIT code.
842 pub(crate) fn get_func_ref(
843 self: Pin<&mut Self>,
844 registry: &ModuleRegistry,
845 index: FuncIndex,
846 ) -> Option<NonNull<VMFuncRef>> {
847 if index == FuncIndex::reserved_value() {
848 return None;
849 }
850
851 match self.env_module().defined_func_index(index) {
852 Some(index) => self.initialize_defined_funcref(registry, index),
853 None => {
854 debug_assert!(self.env_module().is_imported_function(index));
855 Some(self.imported_function(index).as_func_ref().into())
856 }
857 }
858 }
859
860 /// Initializes a defined function's `VMFuncRef` in-place and then returns a
861 /// pointer to that location.
862 fn initialize_defined_funcref(
863 self: Pin<&mut Self>,
864 registry: &ModuleRegistry,
865 def_index: DefinedFuncIndex,
866 ) -> Option<NonNull<VMFuncRef>> {
867 let module = self.env_module();
868 let index = module.func_index(def_index);
869 let func = &module.functions[index];
870 let type_index = func.signature.unwrap_engine_type_index();
871 let vmctx_offset = self.offsets().func_refs().at(func.func_ref);
872 let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index);
873 let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index);
874 // SAFETY: the type/offset/keys here are all valid for the defined
875 // function at `def_index`.
876 unsafe {
877 self.initialize_and_return_funcref(
878 registry,
879 type_index,
880 vmctx_offset,
881 array_to_wasm_key,
882 wasm_key,
883 )
884 }
885 }
886
887 fn get_start_func_ref(
888 self: Pin<&mut Self>,
889 registry: &ModuleRegistry,
890 ) -> Option<NonNull<VMFuncRef>> {
891 let module = self.env_module();
892 let type_index = match module.startup {
893 ModuleStartup::None => return None,
894 ModuleStartup::Always(t) | ModuleStartup::IfMemoriesNeedInit(t) => {
895 t.unwrap_engine_type_index()
896 }
897 };
898 let vmctx_offset = self.offsets().startup_func_ref();
899 let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index);
900 let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index);
901 // SAFETY: the type/offset/keys here are all valid for the module
902 // startup function.
903 unsafe {
904 self.initialize_and_return_funcref(
905 registry,
906 type_index,
907 vmctx_offset,
908 array_to_wasm_key,
909 wasm_key,
910 )
911 }
912 }
913
914 /// Common implementation of initializing a `VMFuncRef` stored within this
915 /// instance's `VMContext`.
916 ///
917 /// # Safety
918 ///
919 /// This function requires that `type_index` accurately describes this
920 /// function and `vmctx_offset` is indeed the correct offset for the
921 /// functions here. Effectively all the arguments here must be "logically
922 /// correct" for the `VMFuncRef` being initialized.
923 unsafe fn initialize_and_return_funcref(
924 self: Pin<&mut Self>,
925 registry: &ModuleRegistry,
926 type_index: VMSharedTypeIndex,
927 vmctx_offset: u32,
928 array_to_wasm_key: FuncKey,
929 wasm_key: FuncKey,
930 ) -> Option<NonNull<VMFuncRef>> {
931 // For now, we eagerly initialize an funcref struct in-place whenever
932 // asked for a reference to it. This is mostly fine, because in practice
933 // each funcref is unlikely to be requested more than a few times:
934 // once-ish for funcref tables used for call_indirect (the usual
935 // compilation strategy places each function in the table at most once),
936 // and once or a few times when fetching exports via API. Note that for
937 // any case driven by table accesses, the lazy table init behaves like a
938 // higher-level cache layer that protects this initialization from
939 // happening multiple times, via that particular table at least.
940 //
941 // When `ref.func` becomes more commonly used or if we otherwise see a
942 // use-case where this becomes a hotpath, we can reconsider by using
943 // some state to track "uninitialized" explicitly, for example by
944 // zeroing the funcrefs (perhaps together with other
945 // zeroed-at-instantiate-time state) or using a separate is-initialized
946 // bitmap.
947 //
948 // We arrived at this design because zeroing memory is expensive, so
949 // it's better for instantiation performance if we don't have to track
950 // "is-initialized" state at all!
951 //
952 // Note that this function is duplicated with compiled code generated by
953 // the `translate_ref_defined_func` method and needs to be kept in-sync
954 // with that.
955
956 let module_with_code = ModuleWithCode::in_store(
957 registry,
958 self.runtime_module()
959 .expect("funcref impossible in fake module"),
960 )
961 .expect("module not in store");
962
963 let array_call =
964 VmPtr::from(NonNull::from(module_with_code.function(array_to_wasm_key)).cast());
965
966 let wasm_call = Some(VmPtr::from(
967 NonNull::new(
968 module_with_code
969 .function(wasm_key)
970 .as_ptr()
971 .cast::<VMWasmCallFunction>()
972 .cast_mut(),
973 )
974 .unwrap(),
975 ));
976
977 let vmctx = VMOpaqueContext::from_vmcontext(self.vmctx()).into();
978
979 // SAFETY: the offset calculated here should be correct with
980 // `self.offsets`
981 let func_ref_ptr = unsafe { self.vmctx_plus_offset_raw::<VMFuncRef>(vmctx_offset) };
982
983 // SAFETY: the `func_ref_ptr` should be valid as it's within our
984 // `VMContext` area.
985 unsafe {
986 func_ref_ptr.write(VMFuncRef {
987 array_call,
988 wasm_call,
989 vmctx,
990 type_index,
991 });
992 }
993
994 Some(func_ref_ptr)
995 }
996
997 /// Get the passive elements segment at the given index.
998 pub(crate) fn passive_element_segment(
999 self: Pin<&mut Self>,
1000 passive: PassiveElemIndex,
1001 ) -> &mut [ValRaw] {
1002 self.passive_elements_mut()[passive.index()].elements_mut()
1003 }
1004
1005 pub(crate) fn passive_elements_mut(self: Pin<&mut Self>) -> &mut TryVec<PassiveElementSegment> {
1006 // SAFETY: Not moving data out of `self`.
1007 &mut unsafe { self.get_unchecked_mut() }.passive_elements
1008 }
1009
1010 /// Drop an element.
1011 pub(crate) fn passive_elem_drop(
1012 self: Pin<&mut Self>,
1013 gc_store: Option<&mut GcStore>,
1014 passive_index: PassiveElemIndex,
1015 ) -> Result<(), OutOfMemory> {
1016 self.passive_elements_mut()[passive_index.index()].clear(gc_store);
1017 Ok(())
1018 }
1019
1020 /// Get a locally-defined memory.
1021 pub fn get_defined_memory_mut(self: Pin<&mut Self>, index: DefinedMemoryIndex) -> &mut Memory {
1022 &mut self.memories_mut()[index].1
1023 }
1024
1025 /// Get a locally-defined memory.
1026 pub fn get_defined_memory(&self, index: DefinedMemoryIndex) -> &Memory {
1027 &self.memories[index].1
1028 }
1029
1030 pub fn get_defined_memory_vmimport(&self, index: DefinedMemoryIndex) -> VMMemoryImport {
1031 crate::runtime::vm::VMMemoryImport {
1032 from: self.memory_ptr(index).into(),
1033 vmctx: self.vmctx().into(),
1034 index,
1035 }
1036 }
1037
1038 /// Given an internal storage range of a Wasm data segment (or subset of a
1039 /// Wasm data segment), get the data's raw bytes.
1040 pub(crate) fn wasm_data(&self, range: Range<u32>) -> &[u8] {
1041 let start = usize::try_from(range.start).unwrap();
1042 let end = usize::try_from(range.end).unwrap();
1043 &self.runtime_info.wasm_data()[start..end]
1044 }
1045
1046 /// Returns the data for the runtime segment identified by `index`
1047 ///
1048 /// Does not take into account the dynamic size of the data pointed to by
1049 /// `index`, always returns the raw data from the module itself.
1050 ///
1051 /// # Panics
1052 ///
1053 /// Panics if `index` is out-of-bounds.
1054 fn runtime_data(&self, index: RuntimeDataIndex) -> &[u8] {
1055 let range = self.env_module().runtime_data[index].clone();
1056 self.wasm_data(range)
1057 }
1058
1059 /// Get a table by index regardless of whether it is locally-defined
1060 /// or an imported, foreign table. Ensure that the given range of
1061 /// elements in the table is lazily initialized. We define this
1062 /// operation all-in-one for safety, to ensure the lazy-init
1063 /// happens.
1064 ///
1065 /// Takes an `Iterator` for the index-range to lazy-initialize,
1066 /// for flexibility. This can be a range, single item, or empty
1067 /// sequence, for example. The iterator should return indices in
1068 /// increasing order, so that the break-at-out-of-bounds behavior
1069 /// works correctly.
1070 pub(crate) fn get_table_with_lazy_init(
1071 self: Pin<&mut Self>,
1072 registry: &ModuleRegistry,
1073 table_index: TableIndex,
1074 range: impl Iterator<Item = u64>,
1075 ) -> &mut Table {
1076 let (idx, instance) = self.defined_table_index_and_instance(table_index);
1077 instance.get_defined_table_with_lazy_init(registry, idx, range)
1078 }
1079
1080 /// Gets the raw runtime table data structure owned by this instance
1081 /// given the provided `idx`.
1082 ///
1083 /// The `range` specified is eagerly initialized for funcref tables.
1084 pub fn get_defined_table_with_lazy_init(
1085 mut self: Pin<&mut Self>,
1086 registry: &ModuleRegistry,
1087 idx: DefinedTableIndex,
1088 range: impl IntoIterator<Item = u64>,
1089 ) -> &mut Table {
1090 let elt_ty = self.tables[idx].1.element_type();
1091
1092 if elt_ty == TableElementType::Func {
1093 for i in range {
1094 match self.tables[idx].1.get_func_maybe_init(i) {
1095 // Uninitialized table element.
1096 Ok(None) => {}
1097 // Initialized table element, move on to the next.
1098 Ok(Some(_)) => continue,
1099 // Out-of-bounds; caller will handle by likely
1100 // throwing a trap. No work to do to lazy-init
1101 // beyond the end.
1102 Err(_) => break,
1103 };
1104
1105 // The table element `i` is uninitialized and is now being
1106 // initialized. This must imply that a `precompiled` list of
1107 // function indices is available for this table. The precompiled
1108 // list is extracted and then it is consulted with `i` to
1109 // determine the function that is going to be initialized. Note
1110 // that `i` may be outside the limits of the static
1111 // initialization so it's a fallible `get` instead of an index.
1112 let module = self.env_module();
1113 let precomputed = &module.table_initialization[idx];
1114 // Panicking here helps catch bugs rather than silently truncating by accident.
1115 let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned();
1116 let func_ref = func_index
1117 .and_then(|func_index| self.as_mut().get_func_ref(registry, func_index));
1118 self.as_mut().tables_mut()[idx]
1119 .1
1120 .set_func(i, func_ref)
1121 .expect("Table type should match and index should be in-bounds");
1122 }
1123 }
1124
1125 self.get_defined_table(idx)
1126 }
1127
1128 /// Get a locally-defined table.
1129 pub(crate) fn get_defined_table(self: Pin<&mut Self>, index: DefinedTableIndex) -> &mut Table {
1130 &mut self.tables_mut()[index].1
1131 }
1132
1133 pub(crate) fn defined_table_index_and_instance<'a>(
1134 self: Pin<&'a mut Self>,
1135 index: TableIndex,
1136 ) -> (DefinedTableIndex, Pin<&'a mut Instance>) {
1137 if let Some(defined_table_index) = self.env_module().defined_table_index(index) {
1138 (defined_table_index, self)
1139 } else {
1140 let import = self.imported_table(index);
1141 let index = import.index;
1142 let vmctx = import.vmctx.as_non_null();
1143 // SAFETY: the validity of `self` means that the reachable instances
1144 // should also all be owned by the same store and fully initialized,
1145 // so it's safe to laterally move from a mutable borrow of this
1146 // instance to a mutable borrow of a sibling instance.
1147 let foreign_instance = unsafe { self.sibling_vmctx_mut(vmctx) };
1148 (index, foreign_instance)
1149 }
1150 }
1151
1152 /// Same as `self.runtime_info.env_module()` but additionally returns the
1153 /// `Pin<&mut Self>` with the same original lifetime.
1154 pub fn module_and_self(self: Pin<&mut Self>) -> (&wasmtime_environ::Module, Pin<&mut Self>) {
1155 // SAFETY: this function is projecting both `&Module` and the same
1156 // pointer both connected to the same lifetime. This is safe because
1157 // it's a contract of `Pin<&mut Self>` that the `runtime_info` field is
1158 // never written, meaning it's effectively unsafe to have `&mut Module`
1159 // projected from `Pin<&mut Self>`. Consequently it's safe to have a
1160 // read-only view of the field while still retaining mutable access to
1161 // all other fields.
1162 let module = self.runtime_info.env_module();
1163 let module = &raw const *module;
1164 let module = unsafe { &*module };
1165 (module, self)
1166 }
1167
1168 /// Initialize the VMContext data associated with this Instance.
1169 ///
1170 /// The `VMContext` memory is assumed to be uninitialized; any field
1171 /// that we need in a certain state will be explicitly written by this
1172 /// function.
1173 unsafe fn initialize_vmctx(self: Pin<&mut Self>, store: &StoreOpaque, imports: Imports) {
1174 let (module, mut instance) = self.module_and_self();
1175
1176 // SAFETY: the type of the magic field is indeed `u32` and this function
1177 // is initializing its value.
1178 unsafe {
1179 let offsets = instance.runtime_info.offsets();
1180 instance
1181 .vmctx_plus_offset_raw::<u32>(offsets.ptr.vmctx().magic())
1182 .write(VMCONTEXT_MAGIC);
1183 }
1184
1185 // SAFETY: it's up to the caller to provide a valid store pointer here.
1186 unsafe {
1187 instance.as_mut().set_store(store);
1188 }
1189
1190 // Initialize shared types
1191 //
1192 // SAFETY: validity of the vmctx means it should be safe to write to it
1193 // here.
1194 unsafe {
1195 let types = NonNull::from(instance.runtime_info.type_ids());
1196 instance.type_ids_array().write(types.cast().into());
1197 }
1198
1199 // Initialize the built-in functions
1200 //
1201 // SAFETY: the type of the builtin functions field is indeed a pointer
1202 // and the pointer being filled in here, plus the vmctx is valid to
1203 // write to during initialization.
1204 unsafe {
1205 static BUILTINS: VMBuiltinFunctionsArray = VMBuiltinFunctionsArray::INIT;
1206 let ptr = BUILTINS.expose_provenance();
1207 let offsets = instance.runtime_info.offsets();
1208 instance
1209 .vmctx_plus_offset_raw(offsets.ptr.vmctx().builtin_functions())
1210 .write(VmPtr::from(ptr));
1211 }
1212
1213 // Initialize the imports
1214 //
1215 // SAFETY: the vmctx is safe to initialize during this function and
1216 // validity of each item itself is a contract the caller must uphold.
1217 debug_assert_eq!(imports.functions.len(), module.num_imported_funcs);
1218 unsafe {
1219 let offsets = instance.runtime_info.offsets();
1220 ptr::copy_nonoverlapping(
1221 imports.functions.as_ptr(),
1222 instance
1223 .vmctx_plus_offset_raw(offsets.imported_functions().begin())
1224 .as_ptr(),
1225 imports.functions.len(),
1226 );
1227 debug_assert_eq!(imports.tables.len(), module.num_imported_tables);
1228 ptr::copy_nonoverlapping(
1229 imports.tables.as_ptr(),
1230 instance
1231 .vmctx_plus_offset_raw(offsets.imported_tables().begin())
1232 .as_ptr(),
1233 imports.tables.len(),
1234 );
1235 debug_assert_eq!(imports.memories.len(), module.num_imported_memories);
1236 ptr::copy_nonoverlapping(
1237 imports.memories.as_ptr(),
1238 instance
1239 .vmctx_plus_offset_raw(offsets.imported_memories().begin())
1240 .as_ptr(),
1241 imports.memories.len(),
1242 );
1243 debug_assert_eq!(imports.globals.len(), module.num_imported_globals);
1244 ptr::copy_nonoverlapping(
1245 imports.globals.as_ptr(),
1246 instance
1247 .vmctx_plus_offset_raw(offsets.imported_globals().begin())
1248 .as_ptr(),
1249 imports.globals.len(),
1250 );
1251 debug_assert_eq!(imports.tags.len(), module.num_imported_tags);
1252 ptr::copy_nonoverlapping(
1253 imports.tags.as_ptr(),
1254 instance
1255 .vmctx_plus_offset_raw(offsets.imported_tags().begin())
1256 .as_ptr(),
1257 imports.tags.len(),
1258 );
1259 }
1260
1261 // N.B.: there is no need to initialize the funcrefs array because we
1262 // eagerly construct each element in it whenever asked for a reference
1263 // to that element. In other words, there is no state needed to track
1264 // the lazy-init, so we don't need to initialize any state now.
1265
1266 // Initialize the defined tables
1267 //
1268 // SAFETY: it's safe to initialize these tables during initialization
1269 // here and the various types of pointers and such here should all be
1270 // valid.
1271 unsafe {
1272 let offsets = instance.runtime_info.offsets();
1273 let mut ptr = instance.vmctx_plus_offset_raw(offsets.tables().begin());
1274 let tables = instance.as_mut().tables_mut();
1275 for i in 0..module.num_defined_tables() {
1276 ptr.write(tables[DefinedTableIndex::new(i)].1.vmtable());
1277 ptr = ptr.add(1);
1278 }
1279 }
1280
1281 // Initialize the defined memories. This fills in both the
1282 // `defined_memories` table and the `owned_memories` table at the same
1283 // time. Entries in `defined_memories` hold a pointer to a definition
1284 // (all memories) whereas the `owned_memories` hold the actual
1285 // definitions of memories owned (not shared) in the module.
1286 //
1287 // SAFETY: it's safe to initialize these memories during initialization
1288 // here and the various types of pointers and such here should all be
1289 // valid.
1290 unsafe {
1291 let offsets = instance.runtime_info.offsets();
1292 let mut ptr = instance.vmctx_plus_offset_raw(offsets.memories().begin());
1293 let mut owned_ptr = instance.vmctx_plus_offset_raw(offsets.owned_memories().begin());
1294 let memories = instance.as_mut().memories_mut();
1295 for i in 0..module.num_defined_memories() {
1296 let defined_memory_index = DefinedMemoryIndex::new(i);
1297 let memory_index = module.memory_index(defined_memory_index);
1298 if module.memories[memory_index].shared {
1299 let def_ptr = memories[defined_memory_index]
1300 .1
1301 .as_shared_memory()
1302 .unwrap()
1303 .vmmemory_ptr();
1304 ptr.write(VmPtr::from(def_ptr));
1305 } else {
1306 owned_ptr.write(memories[defined_memory_index].1.vmmemory());
1307 ptr.write(VmPtr::from(owned_ptr));
1308 owned_ptr = owned_ptr.add(1);
1309 }
1310 ptr = ptr.add(1);
1311 }
1312 }
1313
1314 // If this module's code checks memory shadows, point to each memory's
1315 // watchpoint shadow, if it has one. An imported memory's shadow
1316 // belongs to the instance that defines it.
1317 if module.has_shadow_memories {
1318 // SAFETY: the imports were written above and refer to live
1319 // instances.
1320 unsafe {
1321 let offsets = instance.runtime_info.offsets();
1322 let mut ptr = instance.vmctx_plus_offset_raw::<VmPtr<VMMemoryDefinition>>(
1323 offsets.shadow_memories().begin(),
1324 );
1325 for i in 0..module.memories.len() {
1326 let index = MemoryIndex::new(i);
1327 let memory = match module.defined_memory_index(index) {
1328 Some(def_index) => instance.get_defined_memory(def_index),
1329 None => {
1330 let import = instance.imported_memory(index);
1331 instance
1332 .sibling_vmctx(import.vmctx.as_non_null())
1333 .get_defined_memory(import.index)
1334 }
1335 };
1336 let shadow = memory.shadow().map(|s| s.vmmemory_ptr());
1337 ptr.write(VmPtr::from(shadow.unwrap_or(NonNull::dangling())));
1338 ptr = ptr.add(1);
1339 }
1340 }
1341 }
1342
1343 // Zero-initialize the globals so that nothing is uninitialized memory
1344 // after this function returns. The globals are actually initialized
1345 // with their const expression initializers after the instance is fully
1346 // allocated.
1347 //
1348 // SAFETY: it's safe to initialize globals during initialization
1349 // here. Note that while the value being written is not valid for all
1350 // types of globals it's initializing the memory to zero instead of
1351 // being in an undefined state. So it's still unsafe to access globals
1352 // after this, but if it's read then it'd hopefully crash faster than
1353 // leaving this undefined.
1354 unsafe {
1355 for i in 0..module.num_defined_globals() {
1356 let index = DefinedGlobalIndex::new(i);
1357 instance.global_ptr(index).write(VMGlobalDefinition::new());
1358 }
1359 for (index, val) in module.global_initializers.iter() {
1360 let mut def = VMGlobalDefinition::new();
1361 match val {
1362 GlobalConstValue::I32(i) => def.set_i32(*i),
1363 GlobalConstValue::I64(i) => def.set_i64(*i),
1364 GlobalConstValue::F32(i) => def.set_f32_bits(*i),
1365 GlobalConstValue::F64(i) => def.set_f64_bits(*i),
1366 GlobalConstValue::V128(i) => def.set_u128(*i),
1367 }
1368 instance.global_ptr(*index).write(def);
1369 }
1370 }
1371
1372 // Initialize the defined tags
1373 //
1374 // SAFETY: it's safe to initialize these tags during initialization
1375 // here and the various types of pointers and such here should all be
1376 // valid.
1377 unsafe {
1378 let offsets = instance.runtime_info.offsets();
1379 let mut ptr = instance.vmctx_plus_offset_raw(offsets.tags().begin());
1380 for i in 0..module.num_defined_tags() {
1381 let defined_index = DefinedTagIndex::new(i);
1382 let tag_index = module.tag_index(defined_index);
1383 let tag = module.tags[tag_index];
1384 ptr.write(VMTagDefinition::new(
1385 tag.signature.unwrap_engine_type_index(),
1386 ));
1387 ptr = ptr.add(1);
1388 }
1389 }
1390
1391 // Initialize the lengths of runtime data segments.
1392 //
1393 // SAFETY: it's safe to initialize these lengths during initialization
1394 // here and the various types of pointers and such here should all be
1395 // valid.
1396 unsafe {
1397 let offsets = instance.runtime_info.offsets();
1398 let mut lengths =
1399 instance.vmctx_plus_offset_raw(offsets.runtime_data_lengths().begin());
1400 let mut bases = instance.vmctx_plus_offset_raw(offsets.runtime_data_bases().begin());
1401 for i in module.runtime_data.keys() {
1402 let data = instance.runtime_data(i);
1403 lengths.write(u32::try_from(data.len()).unwrap());
1404 lengths = lengths.add(1);
1405 bases.write(VmPtr::from(NonNull::from(data).cast::<u8>()));
1406 bases = bases.add(1);
1407 }
1408 }
1409
1410 // This is the half of the strategy of implementing memory-init-cow data
1411 // segments. Notably the compiled startup function, if present, will
1412 // skip data segments that have a null pointer. Here each linear memory
1413 // is tested to see if it needs initialization. If it does, then the
1414 // data segment is left in-place (and the startup function will
1415 // initialize linear memory). Otherwise the data segment is null'd out.
1416 // If the startup function runs (e.g. something else in the module needs
1417 // it), then the corresponding data segment's initialization will be
1418 // skipped.
1419 if let MemoryInitialization::Static { map } = &module.memory_initialization {
1420 for (memory, init) in map {
1421 let Some(memory) = module.defined_memory_index(memory) else {
1422 continue;
1423 };
1424 if instance.memories[memory].1.needs_init() {
1425 continue;
1426 }
1427 if let Some((_offset, data)) = init {
1428 let offsets = instance.runtime_info.offsets();
1429 unsafe {
1430 instance
1431 .vmctx_plus_offset_raw(offsets.runtime_data_lengths().at(*data))
1432 .write(0u32);
1433 instance
1434 .vmctx_plus_offset_raw(offsets.runtime_data_bases().at(*data))
1435 .write(0usize);
1436 }
1437 }
1438 }
1439 }
1440 }
1441
1442 /// Attempts to convert from the host `addr` specified to a WebAssembly
1443 /// based address recorded in `WasmFault`.
1444 ///
1445 /// This method will check all linear memories that this instance contains
1446 /// to see if any of them contain `addr`. If one does then `Some` is
1447 /// returned with metadata about the wasm fault. Otherwise `None` is
1448 /// returned and `addr` doesn't belong to this instance.
1449 pub fn wasm_fault(&self, addr: usize) -> Option<WasmFault> {
1450 let mut fault = None;
1451 for (_, (_, memory)) in self.memories.iter() {
1452 let accessible = memory.wasm_accessible();
1453 if accessible.start <= addr && addr < accessible.end {
1454 // All linear memories should be disjoint so assert that no
1455 // prior fault has been found.
1456 assert!(fault.is_none());
1457 fault = Some(WasmFault {
1458 memory_size: memory.byte_size(),
1459 wasm_address: u64::try_from(addr - accessible.start).unwrap(),
1460 });
1461 }
1462 }
1463 fault
1464 }
1465
1466 /// Returns the id, within this instance's store, that it's assigned.
1467 pub fn id(&self) -> InstanceId {
1468 self.id
1469 }
1470
1471 /// Get all memories within this instance.
1472 ///
1473 /// Returns both import and defined memories.
1474 ///
1475 /// Returns both exported and non-exported memories.
1476 ///
1477 /// Gives access to the full memories space.
1478 pub fn all_memories(
1479 &self,
1480 store: StoreId,
1481 ) -> impl ExactSizeIterator<Item = (MemoryIndex, ExportMemory)> + '_ {
1482 self.env_module()
1483 .memories
1484 .iter()
1485 .map(move |(i, _)| (i, self.get_exported_memory(store, i)))
1486 }
1487
1488 /// Return the memories defined in this instance (not imported).
1489 pub fn defined_memories<'a>(
1490 &'a self,
1491 store: StoreId,
1492 ) -> impl ExactSizeIterator<Item = ExportMemory> + 'a {
1493 let num_imported = self.env_module().num_imported_memories;
1494 self.all_memories(store)
1495 .skip(num_imported)
1496 .map(|(_i, memory)| memory)
1497 }
1498
1499 /// Lookup an item with the given index.
1500 ///
1501 /// # Panics
1502 ///
1503 /// Panics if `export` is not valid for this instance.
1504 ///
1505 /// # Safety
1506 ///
1507 /// This function requires that `store` is the correct store which owns this
1508 /// instance.
1509 pub unsafe fn get_export_by_index_mut(
1510 self: Pin<&mut Self>,
1511 registry: &ModuleRegistry,
1512 store: StoreId,
1513 export: EntityIndex,
1514 ) -> Export {
1515 match export {
1516 // SAFETY: the contract of `store` owning the this instance is a
1517 // safety requirement of this function itself.
1518 EntityIndex::Function(i) => {
1519 Export::Function(unsafe { self.get_exported_func(registry, store, i) })
1520 }
1521 _ => self.get_export_by_index(store, export).unwrap(),
1522 }
1523 }
1524
1525 /// Like `get_export_by_index_mut`, but with only shared access to the
1526 /// instance, so it returns `None` for functions: looking up a function
1527 /// may need to lazily initialize its `VMFuncRef`, which needs mutable
1528 /// access.
1529 pub fn get_export_by_index(&self, store: StoreId, export: EntityIndex) -> Option<Export> {
1530 Some(match export {
1531 EntityIndex::Function(_) => return None,
1532 EntityIndex::Global(i) => Export::Global(self.get_exported_global(store, i)),
1533 EntityIndex::Table(i) => Export::Table(self.get_exported_table(store, i)),
1534 EntityIndex::Memory(i) => match self.get_exported_memory(store, i) {
1535 ExportMemory::Unshared(m) => Export::Memory(m),
1536 ExportMemory::Shared(m, i) => Export::SharedMemory(m, i),
1537 },
1538 EntityIndex::Tag(i) => Export::Tag(self.get_exported_tag(store, i)),
1539 })
1540 }
1541
1542 fn store_mut(self: Pin<&mut Self>) -> &mut Option<VMStoreRawPtr> {
1543 // SAFETY: this is a pin-projection to get a mutable reference to an
1544 // internal field and is safe so long as the `&mut Self` temporarily
1545 // created is not overwritten, which it isn't here.
1546 unsafe { &mut self.get_unchecked_mut().store }
1547 }
1548
1549 fn memories_mut(
1550 self: Pin<&mut Self>,
1551 ) -> &mut TryPrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)> {
1552 // SAFETY: see `store_mut` above.
1553 unsafe { &mut self.get_unchecked_mut().memories }
1554 }
1555
1556 pub(crate) fn tables_mut(
1557 self: Pin<&mut Self>,
1558 ) -> &mut TryPrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)> {
1559 // SAFETY: see `store_mut` above.
1560 unsafe { &mut self.get_unchecked_mut().tables }
1561 }
1562
1563 #[cfg(feature = "wmemcheck")]
1564 pub(super) fn wmemcheck_state_mut(self: Pin<&mut Self>) -> &mut Option<Wmemcheck> {
1565 // SAFETY: see `store_mut` above.
1566 unsafe { &mut self.get_unchecked_mut().wmemcheck_state }
1567 }
1568
1569 pub(crate) fn needs_startup(&self) -> bool {
1570 match self.env_module().startup {
1571 ModuleStartup::None => false,
1572 ModuleStartup::Always(_) => true,
1573 ModuleStartup::IfMemoriesNeedInit(_) => self
1574 .memories
1575 .iter()
1576 .any(|(_, (_, memory))| memory.needs_init()),
1577 }
1578 }
1579}
1580
1581// SAFETY: `layout` should describe this accurately and `OwnedVMContext` is the
1582// last field of `ComponentInstance`.
1583unsafe impl InstanceLayout for Instance {
1584 const INIT_ZEROED: bool = false;
1585 type VMContext = VMContext;
1586
1587 fn layout(&self) -> Layout {
1588 Self::alloc_layout(self.runtime_info.offsets())
1589 }
1590
1591 fn owned_vmctx(&self) -> &OwnedVMContext<VMContext> {
1592 &self.vmctx
1593 }
1594
1595 fn owned_vmctx_mut(&mut self) -> &mut OwnedVMContext<VMContext> {
1596 &mut self.vmctx
1597 }
1598}
1599
1600pub type InstanceHandle = OwnedInstance<Instance>;
1601
1602/// A handle holding an `Instance` of a WebAssembly module.
1603///
1604/// This structure is an owning handle of the `instance` contained internally.
1605/// When this value goes out of scope it will deallocate the `Instance` and all
1606/// memory associated with it.
1607///
1608/// Note that this lives within a `StoreOpaque` on a list of instances that a
1609/// store is keeping alive.
1610#[derive(Debug)]
1611#[repr(transparent)] // guarantee this is a zero-cost wrapper
1612pub struct OwnedInstance<T: InstanceLayout> {
1613 /// The raw pointer to the instance that was allocated.
1614 ///
1615 /// Note that this is not equivalent to `Box<Instance>` because the
1616 /// allocation here has a `VMContext` trailing after it. Thus the custom
1617 /// destructor to invoke the `dealloc` function with the appropriate
1618 /// layout.
1619 instance: SendSyncPtr<T>,
1620 _marker: marker::PhantomData<Box<(T, OwnedVMContext<T::VMContext>)>>,
1621}
1622
1623/// Structure that must be placed at the end of a type implementing
1624/// `InstanceLayout`.
1625#[repr(align(16))] // match the alignment of VMContext
1626pub struct OwnedVMContext<T> {
1627 /// A pointer to the `vmctx` field at the end of the `structure`.
1628 ///
1629 /// If you're looking at this a reasonable question would be "why do we need
1630 /// a pointer to ourselves?" because after all the pointer's value is
1631 /// trivially derivable from any `&Instance` pointer. The rationale for this
1632 /// field's existence is subtle, but it's required for correctness. The
1633 /// short version is "this makes miri happy".
1634 ///
1635 /// The long version of why this field exists is that the rules that MIRI
1636 /// uses to ensure pointers are used correctly have various conditions on
1637 /// them depend on how pointers are used. More specifically if `*mut T` is
1638 /// derived from `&mut T`, then that invalidates all prior pointers derived
1639 /// from the `&mut T`. This means that while we liberally want to re-acquire
1640 /// a `*mut VMContext` throughout the implementation of `Instance` the
1641 /// trivial way, a function `fn vmctx(Pin<&mut Instance>) -> *mut VMContext`
1642 /// would effectively invalidate all prior `*mut VMContext` pointers
1643 /// acquired. The purpose of this field is to serve as a sort of
1644 /// source-of-truth for where `*mut VMContext` pointers come from.
1645 ///
1646 /// This field is initialized when the `Instance` is created with the
1647 /// original allocation's pointer. That means that the provenance of this
1648 /// pointer contains the entire allocation (both instance and `VMContext`).
1649 /// This provenance bit is then "carried through" where `fn vmctx` will base
1650 /// all returned pointers on this pointer itself. This provides the means of
1651 /// never invalidating this pointer throughout MIRI and additionally being
1652 /// able to still temporarily have `Pin<&mut Instance>` methods and such.
1653 ///
1654 /// It's important to note, though, that this is not here purely for MIRI.
1655 /// The careful construction of the `fn vmctx` method has ramifications on
1656 /// the LLVM IR generated, for example. A historical CVE on Wasmtime,
1657 /// GHSA-ch89-5g45-qwc7, was caused due to relying on undefined behavior. By
1658 /// deriving VMContext pointers from this pointer it specifically hints to
1659 /// LLVM that trickery is afoot and it properly informs `noalias` and such
1660 /// annotations and analysis. More-or-less this pointer is actually loaded
1661 /// in LLVM IR which helps defeat otherwise present aliasing optimizations,
1662 /// which we want, since writes to this should basically never be optimized
1663 /// out.
1664 ///
1665 /// As a final note it's worth pointing out that the machine code generated
1666 /// for accessing `fn vmctx` is still as one would expect. This member isn't
1667 /// actually ever loaded at runtime (or at least shouldn't be). Perhaps in
1668 /// the future if the memory consumption of this field is a problem we could
1669 /// shrink it slightly, but for now one extra pointer per wasm instance
1670 /// seems not too bad.
1671 vmctx_self_reference: SendSyncPtr<T>,
1672
1673 /// This field ensures that going from `Pin<&mut T>` to `&mut T` is not a
1674 /// safe operation.
1675 _marker: core::marker::PhantomPinned,
1676}
1677
1678impl<T> OwnedVMContext<T> {
1679 /// Creates a new blank vmctx to place at the end of an instance.
1680 pub fn new() -> OwnedVMContext<T> {
1681 OwnedVMContext {
1682 vmctx_self_reference: SendSyncPtr::new(NonNull::dangling()),
1683 _marker: core::marker::PhantomPinned,
1684 }
1685 }
1686}
1687
1688/// Helper trait to plumb both core instances and component instances into
1689/// `OwnedInstance` below.
1690///
1691/// # Safety
1692///
1693/// This trait requires `layout` to correctly describe `Self` and appropriately
1694/// allocate space for `Self::VMContext` afterwards. Additionally the field
1695/// returned by `owned_vmctx()` must be the last field in the structure.
1696pub unsafe trait InstanceLayout {
1697 /// Whether or not to allocate this instance with `alloc_zeroed` or `alloc`.
1698 const INIT_ZEROED: bool;
1699
1700 /// The trailing `VMContext` type at the end of this instance.
1701 type VMContext;
1702
1703 /// The memory layout to use to allocate and deallocate this instance.
1704 fn layout(&self) -> Layout;
1705
1706 fn owned_vmctx(&self) -> &OwnedVMContext<Self::VMContext>;
1707 fn owned_vmctx_mut(&mut self) -> &mut OwnedVMContext<Self::VMContext>;
1708
1709 /// Returns the `vmctx_self_reference` set above.
1710 #[inline]
1711 fn vmctx(&self) -> NonNull<Self::VMContext> {
1712 // The definition of this method is subtle but intentional. The goal
1713 // here is that effectively this should return `&mut self.vmctx`, but
1714 // it's not quite so simple. Some more documentation is available on the
1715 // `vmctx_self_reference` field, but the general idea is that we're
1716 // creating a pointer to return with proper provenance. Provenance is
1717 // still in the works in Rust at the time of this writing but the load
1718 // of the `self.vmctx_self_reference` field is important here as it
1719 // affects how LLVM thinks about aliasing with respect to the returned
1720 // pointer.
1721 //
1722 // The intention of this method is to codegen to machine code as `&mut
1723 // self.vmctx`, however. While it doesn't show up like this in LLVM IR
1724 // (there's an actual load of the field) it does look like that by the
1725 // time the backend runs. (that's magic to me, the backend removing
1726 // loads...)
1727 let owned_vmctx = self.owned_vmctx();
1728 let owned_vmctx_raw = NonNull::from(owned_vmctx);
1729 // SAFETY: it's part of the contract of `InstanceLayout` and the usage
1730 // with `OwnedInstance` that this indeed points to the vmctx.
1731 let addr = unsafe { owned_vmctx_raw.add(1) };
1732 owned_vmctx
1733 .vmctx_self_reference
1734 .as_non_null()
1735 .with_addr(addr.addr())
1736 }
1737
1738 /// Helper function to access various locations offset from our `*mut
1739 /// VMContext` object.
1740 ///
1741 /// Note that this method takes `&self` as an argument but returns
1742 /// `NonNull<T>` which is frequently used to mutate said memory. This is an
1743 /// intentional design decision where the safety of the modification of
1744 /// memory is placed as a burden onto the caller. The implementation of this
1745 /// method explicitly does not require `&mut self` to acquire mutable
1746 /// provenance to update the `VMContext` region. Instead all pointers into
1747 /// the `VMContext` area have provenance/permissions to write.
1748 ///
1749 /// Also note though that care must be taken to ensure that reads/writes of
1750 /// memory must only happen where appropriate, for example a non-atomic
1751 /// write (as most are) should never happen concurrently with another read
1752 /// or write. It's generally on the burden of the caller to adhere to this.
1753 ///
1754 /// Also of note is that most of the time the usage of this method falls
1755 /// into one of:
1756 ///
1757 /// * Something in the VMContext is being read or written. In that case use
1758 /// `vmctx_plus_offset` or `vmctx_plus_offset_mut` if possible due to
1759 /// that having a safer lifetime.
1760 ///
1761 /// * A pointer is being created to pass to other VM* data structures. In
1762 /// that situation the lifetime of all VM data structures are typically
1763 /// tied to the `Store<T>` which is what provides the guarantees around
1764 /// concurrency/etc.
1765 ///
1766 /// There's quite a lot of unsafety riding on this method, especially
1767 /// related to the ascription `T` of the byte `offset`. It's hoped that in
1768 /// the future we're able to settle on an in theory safer design.
1769 ///
1770 /// # Safety
1771 ///
1772 /// This method is unsafe because the `offset` must be within bounds of the
1773 /// `VMContext` object trailing this instance. Additionally `T` must be a
1774 /// valid ascription of the value that resides at that location.
1775 unsafe fn vmctx_plus_offset_raw<T: VmSafe>(&self, offset: impl Into<u32>) -> NonNull<T> {
1776 // SAFETY: the safety requirements of `byte_add` are forwarded to this
1777 // method's caller.
1778 unsafe {
1779 self.vmctx()
1780 .byte_add(usize::try_from(offset.into()).unwrap())
1781 .cast()
1782 }
1783 }
1784
1785 /// Helper above `vmctx_plus_offset_raw` which transfers the lifetime of
1786 /// `&self` to the returned reference `&T`.
1787 ///
1788 /// # Safety
1789 ///
1790 /// See the safety documentation of `vmctx_plus_offset_raw`.
1791 unsafe fn vmctx_plus_offset<T: VmSafe>(&self, offset: impl Into<u32>) -> &T {
1792 // SAFETY: this method has the same safety requirements as
1793 // `vmctx_plus_offset_raw`.
1794 unsafe { self.vmctx_plus_offset_raw(offset).as_ref() }
1795 }
1796
1797 /// Helper above `vmctx_plus_offset_raw` which transfers the lifetime of
1798 /// `&mut self` to the returned reference `&mut T`.
1799 ///
1800 /// # Safety
1801 ///
1802 /// See the safety documentation of `vmctx_plus_offset_raw`.
1803 unsafe fn vmctx_plus_offset_mut<T: VmSafe>(
1804 self: Pin<&mut Self>,
1805 offset: impl Into<u32>,
1806 ) -> &mut T {
1807 // SAFETY: this method has the same safety requirements as
1808 // `vmctx_plus_offset_raw`.
1809 unsafe { self.vmctx_plus_offset_raw(offset).as_mut() }
1810 }
1811}
1812
1813impl<T: InstanceLayout> OwnedInstance<T> {
1814 /// Allocates a new `OwnedInstance` and places `instance` inside of it.
1815 ///
1816 /// This will `instance`
1817 pub(super) fn new(mut instance: T) -> Result<OwnedInstance<T>, OutOfMemory> {
1818 let layout = instance.layout();
1819 debug_assert!(layout.size() >= size_of_val(&instance));
1820 debug_assert!(layout.align() >= align_of_val(&instance));
1821
1822 // SAFETY: it's up to us to assert that `layout` has a non-zero size,
1823 // which is asserted here.
1824 let ptr = unsafe {
1825 assert!(layout.size() > 0);
1826 if T::INIT_ZEROED {
1827 alloc::alloc::alloc_zeroed(layout)
1828 } else {
1829 alloc::alloc::alloc(layout)
1830 }
1831 };
1832 let Some(instance_ptr) = NonNull::new(ptr.cast::<T>()) else {
1833 return Err(OutOfMemory::new(layout.size()));
1834 };
1835
1836 // SAFETY: it's part of the unsafe contract of `InstanceLayout` that the
1837 // `add` here is appropriate for the layout allocated.
1838 let vmctx_self_reference = unsafe { instance_ptr.add(1).cast() };
1839 instance.owned_vmctx_mut().vmctx_self_reference = vmctx_self_reference.into();
1840
1841 // SAFETY: we allocated above and it's an unsafe contract of
1842 // `InstanceLayout` that the layout is suitable for writing the
1843 // instance.
1844 unsafe {
1845 instance_ptr.write(instance);
1846 }
1847
1848 let ret = OwnedInstance {
1849 instance: SendSyncPtr::new(instance_ptr),
1850 _marker: marker::PhantomData,
1851 };
1852
1853 // Double-check various vmctx calculations are correct.
1854 debug_assert_eq!(
1855 vmctx_self_reference.addr(),
1856 // SAFETY: `InstanceLayout` should guarantee it's safe to add 1 to
1857 // the last field to get a pointer to 1-byte-past-the-end of an
1858 // object, which should be valid.
1859 unsafe { NonNull::from(ret.get().owned_vmctx()).add(1).addr() }
1860 );
1861 debug_assert_eq!(vmctx_self_reference.addr(), ret.get().vmctx().addr());
1862
1863 Ok(ret)
1864 }
1865
1866 /// Gets the raw underlying `&Instance` from this handle.
1867 pub fn get(&self) -> &T {
1868 // SAFETY: this is an owned instance handle that retains exclusive
1869 // ownership of the `Instance` inside. With `&self` given we know
1870 // this pointer is valid valid and the returned lifetime is connected
1871 // to `self` so that should also be valid.
1872 unsafe { self.instance.as_non_null().as_ref() }
1873 }
1874
1875 /// Same as [`Self::get`] except for mutability.
1876 pub fn get_mut(&mut self) -> Pin<&mut T> {
1877 // SAFETY: The lifetime concerns here are the same as `get` above.
1878 // Otherwise `new_unchecked` is used here to uphold the contract that
1879 // instances are always pinned in memory.
1880 unsafe { Pin::new_unchecked(self.instance.as_non_null().as_mut()) }
1881 }
1882}
1883
1884impl<T: InstanceLayout> Drop for OwnedInstance<T> {
1885 fn drop(&mut self) {
1886 unsafe {
1887 let layout = self.get().layout();
1888 ptr::drop_in_place(self.instance.as_ptr());
1889 alloc::alloc::dealloc(self.instance.as_ptr().cast(), layout);
1890 }
1891 }
1892}
1893
1894#[derive(Debug)]
1895pub(crate) struct PassiveElementSegment {
1896 needs_gc_rooting: bool,
1897 elements: TryVec<ValRaw>,
1898}
1899
1900impl PassiveElementSegment {
1901 /// Create a new passive element segment with the given capacity.
1902 pub(crate) fn new(ty: WasmRefType, capacity: usize) -> Result<Self, OutOfMemory> {
1903 let mut elements = TryVec::with_capacity(capacity)?;
1904 elements.resize_with(capacity, || ValRaw::null())?;
1905 Ok(Self {
1906 needs_gc_rooting: ty.is_vmgcref_type_and_not_i31(),
1907 elements,
1908 })
1909 }
1910
1911 /// Clear this segment's elements.
1912 pub(crate) fn clear(&mut self, mut gc_store: Option<&mut GcStore>) {
1913 let elements = mem::take(&mut self.elements);
1914 if !self.needs_gc_rooting {
1915 return;
1916 }
1917 for val in elements {
1918 // Like above, `anyref` accessors are used here even if this
1919 // element segment has a different type because all of the vmgcref
1920 // types are treated the same way.
1921 let gc_ref = val.get_anyref();
1922 debug_assert_eq!(gc_ref, val.get_exnref());
1923 debug_assert_eq!(gc_ref, val.get_externref());
1924 if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) {
1925 if let Some(gc_store) = gc_store.as_deref_mut() {
1926 let _ = gc_store.drop_gc_ref(gc_ref);
1927 }
1928 }
1929 }
1930 }
1931
1932 /// The elements of this segment.
1933 pub(crate) fn elements_mut(&mut self) -> &mut [ValRaw] {
1934 &mut self.elements
1935 }
1936}