wasmtime_environ/compile/module_environ.rs
1use crate::error::{OutOfMemory, Result, bail};
2use crate::module::{
3 FuncRefIndex, Initializer, MemoryInitialization, Module, TableSegment, TableSegmentElements,
4};
5use crate::prelude::*;
6use crate::{
7 ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex,
8 DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, EntityIndex, EntityType, FuncIndex,
9 FuncKey, GlobalIndex, IndexType, MemoryIndex, MemoryInitializer, ModuleInternedTypeIndex,
10 ModuleStartup, ModuleTypesBuilder, PanicOnOom as _, PassiveElemIndex, PrimaryMap,
11 RuntimeDataIndex, StaticModuleIndex, TableIndex, TableInitialValue, TableInitialization, Tag,
12 TagIndex, Trap, Tunables, TypeConvert, TypeIndex, WasmHeapTopType, WasmHeapType, WasmResult,
13 WasmValType, WasmparserTypeConverter,
14};
15use alloc::borrow::Cow;
16use cranelift_entity::packed_option::ReservedValue;
17use cranelift_entity::{EntitySet, SecondaryMap};
18use std::collections::HashMap;
19use std::mem;
20use std::path::PathBuf;
21use std::sync::Arc;
22use wasmparser::{
23 CustomSectionReader, DataKind, ElementItems, ElementKind, Encoding, ExternalKind,
24 FuncToValidate, FunctionBody, KnownCustom, NameSectionReader, Naming, Parser, Payload, TypeRef,
25 Validator, ValidatorResources, types::Types,
26};
27
28/// Object containing the standalone environment information.
29pub struct ModuleEnvironment<'a, 'data> {
30 /// The current module being translated
31 result: ModuleTranslation<'data>,
32
33 /// Intern'd types for this entire translation, shared by all modules.
34 types: &'a mut ModuleTypesBuilder,
35
36 // Various bits and pieces of configuration
37 validator: &'a mut Validator,
38 tunables: &'a Tunables,
39}
40
41/// Identifies a FACT adapter-module import that the compiler lowers inline when
42/// translating the adapter function.
43#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
44pub enum FactInlineIntrinsic {
45 /// `enter-sync-call`: push a deferred component-model thread inline.
46 EnterSyncCall,
47 /// `exit-sync-call`: pop the deferred thread inline on the fast path, or
48 /// fall back to the out-of-line `exit-sync-call` libcall when the thread
49 /// was promoted.
50 ExitSyncCall,
51 /// `trap`: raise the given trap.
52 Trap(Trap),
53}
54
55/// A statically-known function import.
56#[derive(Clone, Debug)]
57pub enum KnownFunc {
58 /// A function described by the given key.
59 FuncKey(FuncKey),
60 /// An always-inlined FACT intrinsic.
61 FactIntrinsic(FactInlineIntrinsic),
62}
63
64impl From<FuncKey> for KnownFunc {
65 fn from(key: FuncKey) -> Self {
66 Self::FuncKey(key)
67 }
68}
69
70impl From<FactInlineIntrinsic> for KnownFunc {
71 fn from(intrinsic: FactInlineIntrinsic) -> Self {
72 Self::FactIntrinsic(intrinsic)
73 }
74}
75
76/// A statically-known import of a core Wasm global, memory, or table.
77#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
78pub struct KnownEntity<T> {
79 /// The module that defines this entity.
80 pub module: StaticModuleIndex,
81 /// The entity's index in the defining module's defined-entity index space.
82 pub index: T,
83}
84
85/// A statically-known import of a core wasm global.
86#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
87pub enum KnownGlobal {
88 /// A global defined by a module in the same component.
89 Defined(KnownEntity<DefinedGlobalIndex>),
90
91 /// A component instance's runtime-managed flags, such as its `may-leave`
92 /// flag.
93 #[cfg(feature = "component-model")]
94 ComponentInstanceFlags(crate::component::RuntimeComponentInstanceIndex),
95}
96
97/// The result of translating via `ModuleEnvironment`.
98///
99/// Function bodies are not yet translated, and data initializers have not yet
100/// been copied out of the original buffer.
101pub struct ModuleTranslation<'data> {
102 /// Module information.
103 pub module: Module,
104
105 /// The input wasm binary.
106 ///
107 /// This can be useful, for example, when modules are parsed from a
108 /// component and the embedder wants access to the raw wasm modules
109 /// themselves.
110 pub wasm: &'data [u8],
111
112 /// The byte offset of this module's Wasm binary within the outer
113 /// binary (e.g. a component). For standalone modules this is 0.
114 /// This is used to convert component-relative source locations to
115 /// module-relative source locations.
116 pub wasm_module_offset: u64,
117
118 /// References to the function bodies.
119 pub function_body_inputs: PrimaryMap<DefinedFuncIndex, FunctionBodyData<'data>>,
120
121 /// For each imported function, the single statically-known function that
122 /// always satisfies that import, if any.
123 ///
124 /// This is used to turn what would otherwise be indirect calls through the
125 /// imports table into direct calls, when possible.
126 ///
127 /// When filled in, this only ever contains
128 /// `FuncKey::DefinedWasmFunction(..)`s, `FuncKey::Intrinsic(..)`s, and
129 /// `FuncKey::FactInlineIntrinsic`s.
130 pub known_imported_functions: SecondaryMap<FuncIndex, Option<KnownFunc>>,
131
132 /// For each imported global, memory, or table, the single statically-known
133 /// defined entity that always satisfies that import, if any.
134 ///
135 /// This is used to access the entity via the defining module's precise
136 /// `AliasRegionKey::Defined{Global,Memory,Table}` region instead of the
137 /// conservative `AliasRegionKey::Public{Global,Memory,Table}` region that is
138 /// shared by every entity of that kind which crosses a module boundary.
139 ///
140 /// XXX: Being "known" requires more here than it does for functions: it is
141 /// not enough that *this* module's import is always the same entity,
142 /// *every* module that may import that entity must also always import that
143 /// same entity. Otherwise a function from one of those other modules, which
144 /// accesses the entity via the conservative region, could be inlined next
145 /// to one of our accesses via the precise region, and accessing the same
146 /// memory through two different alias regions is invalid.
147 ///
148 /// This extra condition is an artifact of this implementation, and how we
149 /// consume this data to choose the alias region for loads and stores to a
150 /// global/memory/table, not something inherent to knowing exactly which
151 /// entity satisfies a particular import. Really, there are two independent
152 /// axes here:
153 ///
154 /// 1. Is this import always satisfied by the same defined entity?
155 ///
156 /// 2. Is that entity's identity additionally known to *every* other module
157 /// that may import it?
158 ///
159 /// Only alias regions need (2), but other theoretical optimizations could
160 /// be perfectly happy with just (1). For example, if we know that an import
161 /// of an immutable global is always a particular defined global, then we
162 /// could inline that global's value at each `global.get` of the import,
163 /// regardless what any other module does or does not know about that
164 /// global.
165 ///
166 /// TODO(#14164): Actually record (1) and (2) in separate maps, enabling
167 /// optimizations that rely on just (1) but not (2), instead of folding them
168 /// into this same map.
169 pub known_imported_globals: SecondaryMap<GlobalIndex, Option<KnownGlobal>>,
170
171 /// Same as `known_imported_globals`, but for memories.
172 pub known_imported_memories: SecondaryMap<MemoryIndex, Option<KnownEntity<DefinedMemoryIndex>>>,
173
174 /// Same as `known_imported_globals`, but for tables.
175 pub known_imported_tables: SecondaryMap<TableIndex, Option<KnownEntity<DefinedTableIndex>>>,
176
177 /// For each global defined by this module, whether every module that may
178 /// import this global always imports exactly this global.
179 ///
180 /// When this holds, accesses of the global may use its precise
181 /// `AliasRegionKey::DefinedGlobal` region even when the global is exported,
182 /// because every module that can reach it agrees on that same region. This
183 /// is vacuously true of globals that nothing in the component imports.
184 ///
185 /// This can only be determined by looking at the whole component, so it is
186 /// always `false` for standalone modules.
187 pub globals_known_to_importers: EntitySet<DefinedGlobalIndex>,
188
189 /// Same as [`Self::globals_known_to_importers`], but for memories.
190 pub memories_known_to_importers: EntitySet<DefinedMemoryIndex>,
191
192 /// Same as [`Self::globals_known_to_importers`], but for tables.
193 pub tables_known_to_importers: EntitySet<DefinedTableIndex>,
194
195 /// A list of type signatures which are considered exported from this
196 /// module, or those that can possibly be called. This list is sorted, and
197 /// trampolines for each of these signatures are required.
198 pub exported_signatures: Vec<ModuleInternedTypeIndex>,
199
200 /// DWARF debug information, if enabled, parsed from the module.
201 pub debuginfo: DebugInfoData<'data>,
202
203 /// Set if debuginfo was found but it was not parsed due to `Tunables`
204 /// configuration.
205 pub has_unparsed_debuginfo: bool,
206
207 /// The desired alignment of `data` in the final data section of the object
208 /// file that we'll emit.
209 ///
210 /// Note that this is 1 by default but `MemoryInitialization::Static` might
211 /// switch this to a higher alignment to facilitate mmap-ing data from
212 /// an object file into a linear memory.
213 pub data_align: Option<u64>,
214
215 /// Map from a data segment to whether it's a passive data segment or not.
216 pub runtime_data_map: SecondaryMap<DataIndex, Option<RuntimeDataIndex>>,
217
218 /// Map from an elem segment to whether it's a passive elem segment or not.
219 pub passive_elem_map: SecondaryMap<ElemIndex, Option<PassiveElemIndex>>,
220
221 /// List of passive element segments found in this module which will get
222 /// concatenated for the final artifact.
223 pub runtime_data: PrimaryMap<RuntimeDataIndex, Cow<'data, [u8]>>,
224
225 /// Record of all passive data segments that this module contains.
226 ///
227 /// These are processed during [`ModuleTranslation::finalize_memory_init`]
228 /// and eventually moved over into the `runtime_data` list above. Until
229 /// then, however, their `RuntimeDataIndex` is not yet assigned.
230 passive_data: Vec<(DataIndex, &'data [u8])>,
231
232 /// When we're parsing the code section this will be incremented so we know
233 /// which function is currently being defined.
234 code_index: u32,
235
236 /// The type information of the current module made available at the end of the
237 /// validation process.
238 types: Option<Types>,
239
240 /// Per-function [`BranchHintReader`]s from the `metadata.code.branch_hint`
241 /// section, keyed by function index. Populated only when
242 /// [`Tunables::branch_hinting`] is enabled.
243 branch_hints: HashMap<FuncIndex, BranchHintReader<'data>>,
244
245 /// The WebAssembly `start` function, if defined.
246 pub start_func: Option<FuncIndex>,
247
248 /// Initializers for `global` values which aren't considered "simple".
249 ///
250 /// These initializers are later compiled into a "module startup" function.
251 pub global_initializers: Vec<(DefinedGlobalIndex, ConstExpr)>,
252
253 /// Definitions of all passive elements found within a module.
254 ///
255 /// This maps passive element segments to their definition, either functions
256 /// or expressions-basd.
257 pub passive_elements: PrimaryMap<PassiveElemIndex, TableSegmentElements>,
258
259 /// WebAssembly table initialization data, per table.
260 ///
261 /// This keeps track of all per-table initialization (e.g. initial value for
262 /// non-null tables) as well as active element segments. This is processed
263 /// and refined by [`ModuleTranslation::finalize_table_init`] after
264 /// translation.
265 pub table_initialization: TableInitialization,
266
267 /// WebAssembly memory initialization.
268 ///
269 /// This is held here in an `Unprocessed` form during translation, and then
270 /// this is later finished with [`ModuleTranslation::finalize_memory_init`].
271 pub memory_init: MemoryInit<'data>,
272}
273
274/// Different forms of memory initialization that happens for a module.
275pub enum MemoryInit<'a> {
276 /// Raw active data segments that are being applied for an instance.
277 ///
278 /// This list contains the raw data which hasn't yet been processed into
279 /// `RuntimeDataIndex`, for example. This is later processed during
280 /// [`ModuleTranslation::finalize_memory_init`] to optionally shuffle things
281 /// around.
282 Unprocessed(Vec<MemoryInitializer<'a>>),
283
284 /// Finalized memory initialization to be executed after
285 /// [`ModuleTranslation::finalize_memory_init`] has run. This represents
286 /// active data segments which may have been merged from the `Unprocessed`
287 /// list above, and may or may not have statically know offsets.
288 Processed(Vec<(MemoryIndex, MemorySegmentOffset, RuntimeDataIndex)>),
289}
290
291/// Offset within [`MemoryInit::Processed`] which indicates the initial offset
292/// a data segment is applied at.
293pub enum MemorySegmentOffset {
294 /// A "complicated" constant expression deferred to get evaluated at runtime
295 /// with compiled code.
296 Expr(ConstExpr),
297
298 /// A statically known, in-bounds, constant value.
299 Static(u64),
300}
301
302/// Lazy decoder over the branch hints attached to a single function in the
303/// `metadata.code.branch_hint` custom section
304/// ([branch-hinting proposal](https://github.com/WebAssembly/branch-hinting)).
305pub type BranchHintReader<'a> = wasmparser::SectionLimited<'a, wasmparser::BranchHint>;
306
307impl<'data> ModuleTranslation<'data> {
308 /// Create a new translation for the module with the given index.
309 pub fn new(module_index: StaticModuleIndex) -> Self {
310 Self {
311 module: Module::new(module_index),
312 wasm: &[],
313 wasm_module_offset: 0,
314 function_body_inputs: PrimaryMap::default(),
315 known_imported_functions: SecondaryMap::default(),
316 known_imported_globals: SecondaryMap::default(),
317 known_imported_memories: SecondaryMap::default(),
318 known_imported_tables: SecondaryMap::default(),
319 globals_known_to_importers: EntitySet::new(),
320 memories_known_to_importers: EntitySet::new(),
321 tables_known_to_importers: EntitySet::new(),
322 exported_signatures: Vec::default(),
323 debuginfo: DebugInfoData::default(),
324 has_unparsed_debuginfo: false,
325 data_align: None,
326 runtime_data: Default::default(),
327 code_index: 0,
328 types: None,
329 runtime_data_map: Default::default(),
330 passive_elem_map: Default::default(),
331 branch_hints: HashMap::default(),
332 start_func: None,
333 global_initializers: Vec::new(),
334 passive_elements: Default::default(),
335 table_initialization: Default::default(),
336 memory_init: MemoryInit::Unprocessed(Vec::new()),
337 passive_data: Default::default(),
338 }
339 }
340
341 /// Returns the [`BranchHintReader`] for `func`, if the section attached any.
342 pub fn branch_hints(&self, func: FuncIndex) -> Option<BranchHintReader<'data>> {
343 self.branch_hints.get(&func).cloned()
344 }
345
346 /// Returns a reference to the type information of the current module.
347 pub fn get_types(&self) -> &Types {
348 self.types
349 .as_ref()
350 .expect("module type information to be available")
351 }
352
353 /// Get this translation's module's index.
354 pub fn module_index(&self) -> StaticModuleIndex {
355 self.module.module_index
356 }
357}
358
359/// Contains function data: byte code and its offset in the module.
360pub struct FunctionBodyData<'a> {
361 /// The body of the function, containing code and locals.
362 pub body: FunctionBody<'a>,
363 /// Validator for the function body
364 pub validator: FuncToValidate<ValidatorResources>,
365}
366
367#[derive(Debug, Default)]
368#[expect(missing_docs, reason = "self-describing fields")]
369pub struct DebugInfoData<'a> {
370 pub dwarf: Dwarf<'a>,
371 pub name_section: NameSection<'a>,
372 pub wasm_file: WasmFileInfo,
373 pub debug_loc: gimli::DebugLoc<Reader<'a>>,
374 pub debug_loclists: gimli::DebugLocLists<Reader<'a>>,
375 pub debug_ranges: gimli::DebugRanges<Reader<'a>>,
376 pub debug_rnglists: gimli::DebugRngLists<Reader<'a>>,
377 pub debug_cu_index: gimli::DebugCuIndex<Reader<'a>>,
378 pub debug_tu_index: gimli::DebugTuIndex<Reader<'a>>,
379}
380
381#[expect(missing_docs, reason = "self-describing")]
382pub type Dwarf<'input> = gimli::Dwarf<Reader<'input>>;
383
384type Reader<'input> = gimli::EndianSlice<'input, gimli::LittleEndian>;
385
386#[derive(Debug, Default)]
387#[expect(missing_docs, reason = "self-describing fields")]
388pub struct NameSection<'a> {
389 pub module_name: Option<&'a str>,
390 pub func_names: HashMap<FuncIndex, &'a str>,
391 pub locals_names: HashMap<FuncIndex, HashMap<u32, &'a str>>,
392}
393
394#[derive(Debug, Default)]
395#[expect(missing_docs, reason = "self-describing fields")]
396pub struct WasmFileInfo {
397 pub path: Option<PathBuf>,
398 pub code_section_offset: u64,
399 pub imported_func_count: u32,
400 pub funcs: Vec<FunctionMetadata>,
401}
402
403#[derive(Debug)]
404#[expect(missing_docs, reason = "self-describing fields")]
405pub struct FunctionMetadata {
406 pub params: Box<[WasmValType]>,
407 pub locals: Box<[(u32, WasmValType)]>,
408}
409
410impl<'a, 'data> ModuleEnvironment<'a, 'data> {
411 /// Allocates the environment data structures.
412 pub fn new(
413 tunables: &'a Tunables,
414 validator: &'a mut Validator,
415 types: &'a mut ModuleTypesBuilder,
416 module_index: StaticModuleIndex,
417 ) -> Self {
418 Self {
419 result: ModuleTranslation::new(module_index),
420 types,
421 tunables,
422 validator,
423 }
424 }
425
426 /// Translate a wasm module using this environment.
427 ///
428 /// This function will translate the `data` provided with `parser`,
429 /// validating everything along the way with this environment's validator.
430 ///
431 /// The result of translation, [`ModuleTranslation`], contains everything
432 /// necessary to compile functions afterwards as well as learn type
433 /// information about the module at runtime.
434 pub fn translate(
435 mut self,
436 parser: Parser,
437 data: &'data [u8],
438 ) -> Result<ModuleTranslation<'data>> {
439 self.result.wasm = data;
440
441 for payload in parser.parse_all(data) {
442 self.translate_payload(payload?)?;
443 }
444
445 Ok(self.result)
446 }
447
448 fn translate_payload(&mut self, payload: Payload<'data>) -> Result<()> {
449 match payload {
450 Payload::Version {
451 num,
452 encoding,
453 range,
454 } => {
455 self.validator.version(num, encoding, &range)?;
456 match encoding {
457 Encoding::Module => {}
458 Encoding::Component => {
459 bail!("expected a WebAssembly module but was given a WebAssembly component")
460 }
461 }
462 }
463
464 Payload::End(offset) => {
465 self.result.types = Some(self.validator.end(offset)?);
466
467 // With the `escaped_funcs` set of functions finished
468 // we can calculate the set of signatures that are exported as
469 // the set of exported functions' signatures.
470 self.result.exported_signatures = self
471 .result
472 .module
473 .functions
474 .iter()
475 .filter_map(|(_, func)| {
476 if func.is_escaping() {
477 Some(func.signature.unwrap_module_type_index())
478 } else {
479 None
480 }
481 })
482 .collect();
483 self.result.exported_signatures.sort_unstable();
484 self.result.exported_signatures.dedup();
485 }
486
487 Payload::TypeSection(types) => {
488 self.validator.type_section(&types)?;
489
490 let count = self.validator.types(0).unwrap().core_type_count_in_module();
491 log::trace!("interning {count} Wasm types");
492
493 let capacity = usize::try_from(count).unwrap();
494 self.result.module.types.reserve(capacity)?;
495 self.types.reserve_wasm_signatures(capacity);
496
497 // Iterate over each *rec group* -- not type -- defined in the
498 // types section. Rec groups are the unit of canonicalization
499 // and therefore the unit at which we need to process at a
500 // time. `wasmparser` has already done the hard work of
501 // de-duplicating and canonicalizing the rec groups within the
502 // module for us, we just need to translate them into our data
503 // structures. Note that, if the Wasm defines duplicate rec
504 // groups, we need copy the duplicates over (shallowly) as well,
505 // so that our types index space doesn't have holes.
506 let mut type_index = 0;
507 while type_index < count {
508 let validator_types = self.validator.types(0).unwrap();
509
510 // Get the rec group for the current type index, which is
511 // always the first type defined in a rec group.
512 log::trace!("looking up wasmparser type for index {type_index}");
513 let core_type_id = validator_types.core_type_at_in_module(type_index);
514 log::trace!(
515 " --> {core_type_id:?} = {:?}",
516 validator_types[core_type_id],
517 );
518 let rec_group_id = validator_types.rec_group_id_of(core_type_id);
519 debug_assert_eq!(
520 validator_types
521 .rec_group_elements(rec_group_id)
522 .position(|id| id == core_type_id),
523 Some(0)
524 );
525
526 // Intern the rec group and then fill in this module's types
527 // index space.
528 let interned = self.types.intern_rec_group(validator_types, rec_group_id)?;
529 let elems = self.types.rec_group_elements(interned);
530 let len = elems.len();
531 self.result.module.types.reserve(len)?;
532 for ty in elems {
533 self.result.module.types.push(ty.into())?;
534 }
535
536 // Advance `type_index` to the start of the next rec group.
537 type_index += u32::try_from(len).unwrap();
538 }
539 }
540
541 Payload::ImportSection(imports) => {
542 self.validator.import_section(&imports)?;
543
544 let cnt = usize::try_from(imports.count()).unwrap();
545 self.result.module.initializers.reserve(cnt)?;
546
547 for entry in imports.into_imports() {
548 let import = entry?;
549 let ty = match import.ty {
550 TypeRef::Func(index) => {
551 let index = TypeIndex::from_u32(index);
552 let interned_index = self.result.module.types[index];
553 self.result.module.num_imported_funcs += 1;
554 self.result.debuginfo.wasm_file.imported_func_count += 1;
555 EntityType::Function(interned_index)
556 }
557 TypeRef::Memory(ty) => {
558 self.result.module.num_imported_memories += 1;
559 EntityType::Memory(ty.into())
560 }
561 TypeRef::Global(ty) => {
562 self.result.module.num_imported_globals += 1;
563 EntityType::Global(self.convert_global_type(&ty)?)
564 }
565 TypeRef::Table(ty) => {
566 self.result.module.num_imported_tables += 1;
567 EntityType::Table(self.convert_table_type(&ty)?)
568 }
569 TypeRef::Tag(ty) => {
570 let index = TypeIndex::from_u32(ty.func_type_idx);
571 let signature = self.result.module.types[index];
572 let exception = self.types.define_exception_type_for_tag(
573 signature.unwrap_module_type_index(),
574 );
575 let tag = Tag {
576 signature,
577 exception: EngineOrModuleTypeIndex::Module(exception),
578 };
579 self.result.module.num_imported_tags += 1;
580 EntityType::Tag(tag)
581 }
582 TypeRef::FuncExact(_) => {
583 bail!("custom-descriptors proposal not implemented yet");
584 }
585 };
586 self.declare_import(import.module, import.name, ty)?;
587 }
588 }
589
590 Payload::FunctionSection(functions) => {
591 self.validator.function_section(&functions)?;
592
593 let cnt = usize::try_from(functions.count()).unwrap();
594 self.result.module.functions.reserve_exact(cnt)?;
595
596 for entry in functions {
597 let sigindex = entry?;
598 let ty = TypeIndex::from_u32(sigindex);
599 let interned_index = self.result.module.types[ty];
600 self.result.module.push_function(interned_index);
601 }
602 }
603
604 Payload::TableSection(tables) => {
605 self.validator.table_section(&tables)?;
606 let cnt = usize::try_from(tables.count()).unwrap();
607 self.result.module.tables.reserve_exact(cnt)?;
608
609 for entry in tables {
610 let wasmparser::Table { ty, init } = entry?;
611 let table = self.convert_table_type(&ty)?;
612 self.result.module.needs_gc_heap |= table.ref_type.is_vmgcref_type();
613 self.result.module.tables.push(table)?;
614 let init = match init {
615 wasmparser::TableInit::RefNull => TableInitialValue::Null,
616 wasmparser::TableInit::Expr(expr) => {
617 let (init, escaped) = ConstExpr::from_wasmparser(self, expr)?;
618 for f in escaped {
619 self.flag_func_escaped(f);
620 }
621 TableInitialValue::Expr(init)
622 }
623 };
624 self.result.table_initialization.initial_values.push(init)?;
625 self.result
626 .module
627 .table_initialization
628 .push(Default::default())?;
629 }
630 }
631
632 Payload::MemorySection(memories) => {
633 self.validator.memory_section(&memories)?;
634
635 let cnt = usize::try_from(memories.count()).unwrap();
636 self.result.module.memories.reserve_exact(cnt)?;
637
638 for entry in memories {
639 let memory = entry?;
640 self.result.module.memories.push(memory.into())?;
641 }
642 }
643
644 Payload::TagSection(tags) => {
645 self.validator.tag_section(&tags)?;
646
647 for entry in tags {
648 let sigindex = entry?.func_type_idx;
649 let ty = TypeIndex::from_u32(sigindex);
650 let interned_index = self.result.module.types[ty];
651 let exception = self
652 .types
653 .define_exception_type_for_tag(interned_index.unwrap_module_type_index());
654 self.result.module.push_tag(interned_index, exception);
655 }
656 }
657
658 Payload::GlobalSection(globals) => {
659 self.validator.global_section(&globals)?;
660
661 let cnt = usize::try_from(globals.count()).unwrap();
662 self.result.module.globals.reserve_exact(cnt)?;
663
664 for entry in globals {
665 let wasmparser::Global { ty, init_expr } = entry?;
666 let (initializer, escaped) = ConstExpr::from_wasmparser(self, init_expr)?;
667 for f in escaped {
668 self.flag_func_escaped(f);
669 }
670 let ty = self.convert_global_type(&ty)?;
671 let index = self.result.module.globals.push(ty)?;
672 let defined_index = self.result.module.defined_global_index(index).unwrap();
673 match initializer.const_eval() {
674 Some(val) => {
675 self.result
676 .module
677 .global_initializers
678 .push((defined_index, val))?;
679 }
680 None => {
681 // "Complicated" global initializers are deferred
682 // to get evaluated in the startup function.
683 self.require_startup_func();
684 self.result
685 .global_initializers
686 .push((defined_index, initializer));
687 }
688 }
689 }
690 }
691
692 Payload::ExportSection(exports) => {
693 self.validator.export_section(&exports)?;
694
695 let cnt = usize::try_from(exports.count()).unwrap();
696 self.result.module.exports.reserve(cnt)?;
697
698 for entry in exports {
699 let wasmparser::Export { name, kind, index } = entry?;
700 let entity = match kind {
701 ExternalKind::Func | ExternalKind::FuncExact => {
702 let index = FuncIndex::from_u32(index);
703 self.flag_func_escaped(index);
704 EntityIndex::Function(index)
705 }
706 ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(index)),
707 ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(index)),
708 ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(index)),
709 ExternalKind::Tag => EntityIndex::Tag(TagIndex::from_u32(index)),
710 };
711 let name = self.result.module.strings.insert(name)?;
712 self.result.module.exports.insert(name, entity)?;
713 }
714 }
715
716 Payload::StartSection { func, range } => {
717 self.validator.start_section(func, &range)?;
718
719 let func_index = FuncIndex::from_u32(func);
720 debug_assert!(self.result.start_func.is_none());
721 self.result.start_func = Some(func_index);
722
723 // To make startup a bit easier, invoking the `start` function
724 // is a responsibility deferred to the startup function.
725 self.require_startup_func();
726 }
727
728 Payload::ElementSection(elements) => {
729 self.validator.element_section(&elements)?;
730
731 for (index, entry) in elements.into_iter().enumerate() {
732 let wasmparser::Element {
733 kind,
734 items,
735 range: _,
736 } = entry?;
737
738 // Build up a list of `FuncIndex` corresponding to all the
739 // entries listed in this segment. Note that it's not
740 // possible to create anything other than a `ref.null
741 // extern` for externref segments, so those just get
742 // translated to the reserved value of `FuncIndex`.
743 let elements = match items {
744 ElementItems::Functions(funcs) => {
745 let mut elems =
746 Vec::with_capacity(usize::try_from(funcs.count()).unwrap());
747 for func in funcs {
748 let func = FuncIndex::from_u32(func?);
749 self.flag_func_escaped(func);
750 elems.push(func);
751 }
752 TableSegmentElements::Functions(elems.into())
753 }
754 ElementItems::Expressions(ty, items) => {
755 let ty = self.convert_ref_type(ty)?;
756 let mut exprs =
757 Vec::with_capacity(usize::try_from(items.count()).unwrap());
758 for expr in items {
759 let (expr, escaped) = ConstExpr::from_wasmparser(self, expr?)?;
760 exprs.push(expr);
761 for func in escaped {
762 self.flag_func_escaped(func);
763 }
764 }
765 TableSegmentElements::Expressions {
766 ty,
767 exprs: exprs.into(),
768 }
769 }
770 };
771
772 let passive_index = match kind {
773 ElementKind::Active {
774 table_index,
775 offset_expr,
776 } => {
777 let table_index = TableIndex::from_u32(table_index.unwrap_or(0));
778 let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?;
779 debug_assert!(escaped.is_empty());
780
781 self.result
782 .table_initialization
783 .segments
784 .push(TableSegment {
785 table_index,
786 offset,
787 elements,
788 })?;
789 None
790 }
791
792 ElementKind::Passive => {
793 let passive_index = self
794 .result
795 .module
796 .passive_elements
797 .push((elements.ty(), elements.len()))?;
798 self.result.passive_elements.push(elements);
799 // One-time initialization of passive element
800 // segments is deferred to the startup function.
801 self.require_startup_func();
802 Some(passive_index)
803 }
804
805 ElementKind::Declared => None,
806 };
807 let elem_index = ElemIndex::from_u32(index as u32);
808 self.result
809 .passive_elem_map
810 .insert(elem_index, passive_index);
811 }
812 }
813
814 Payload::CodeSectionStart { count, range, .. } => {
815 self.validator.code_section_start(&range)?;
816 let cnt = usize::try_from(count).unwrap();
817 self.result.function_body_inputs.reserve_exact(cnt);
818 self.result.debuginfo.wasm_file.code_section_offset = range.start;
819 }
820
821 Payload::CodeSectionEntry(body) => {
822 let validator = self.validator.code_section_entry(&body)?;
823 let func_index =
824 self.result.code_index + self.result.module.num_imported_funcs as u32;
825 let func_index = FuncIndex::from_u32(func_index);
826
827 if self.tunables.debug_native {
828 let sig_index = self.result.module.functions[func_index]
829 .signature
830 .unwrap_module_type_index();
831 let sig = self.types[sig_index].unwrap_func();
832 let mut locals = Vec::new();
833 for pair in body.get_locals_reader()? {
834 let (cnt, ty) = pair?;
835 let ty = self.convert_valtype(ty)?;
836 locals.push((cnt, ty));
837 }
838 self.result
839 .debuginfo
840 .wasm_file
841 .funcs
842 .push(FunctionMetadata {
843 locals: locals.into_boxed_slice(),
844 params: sig.params().into(),
845 });
846 }
847 if self.tunables.debug_guest {
848 // All functions are potentially reachable and
849 // callable by the guest debugger, so they must
850 // all be flagged as escaping.
851 self.flag_func_escaped(func_index);
852 }
853 self.result
854 .function_body_inputs
855 .push(FunctionBodyData { validator, body });
856 self.result.code_index += 1;
857 }
858
859 Payload::DataSection(data) => {
860 self.validator.data_section(&data)?;
861
862 assert!(self.result.module.memory_initialization.is_segmented());
863
864 for (index, entry) in data.into_iter().enumerate() {
865 let wasmparser::Data {
866 kind,
867 data,
868 range: _,
869 } = entry?;
870 let data_index = DataIndex::from_u32(index.try_into().unwrap());
871 match kind {
872 DataKind::Active {
873 memory_index,
874 offset_expr,
875 } => {
876 let memory_index = MemoryIndex::from_u32(memory_index);
877 let (offset, escaped) = ConstExpr::from_wasmparser(self, offset_expr)?;
878 debug_assert!(escaped.is_empty());
879
880 let MemoryInit::Unprocessed(list) = &mut self.result.memory_init else {
881 panic!("memory initializers should be unprocessed at this point");
882 };
883 list.push(MemoryInitializer {
884 memory_index,
885 offset,
886 data,
887 });
888 }
889 DataKind::Passive => {
890 self.result.passive_data.push((data_index, data));
891 }
892 }
893 }
894 }
895
896 Payload::DataCountSection { count, range } => {
897 self.validator.data_count_section(count, &range)?;
898
899 // Note: the count passed in here is the *total* segment count
900 // There is no way to reserve for just the passive segments as
901 // they are discovered when iterating the data section entries
902 // Given that the total segment count might be much larger than
903 // the passive count, do not reserve anything here.
904 }
905
906 Payload::CustomSection(s)
907 if s.name() == "webidl-bindings" || s.name() == "wasm-interface-types" =>
908 {
909 bail!(
910 "\
911Support for interface types has temporarily been removed from `wasmtime`.
912
913For more information about this temporary change you can read on the issue online:
914
915 https://github.com/bytecodealliance/wasmtime/issues/1271
916
917and for re-adding support for interface types you can see this issue:
918
919 https://github.com/bytecodealliance/wasmtime/issues/677
920"
921 )
922 }
923
924 Payload::CustomSection(s) => {
925 self.register_custom_section(&s);
926 }
927
928 // It's expected that validation will probably reject other
929 // payloads such as `UnknownSection` or those related to the
930 // component model. If, however, something gets past validation then
931 // that's a bug in Wasmtime as we forgot to implement something.
932 other => {
933 self.validator.payload(&other)?;
934 panic!("unimplemented section in wasm file {other:?}");
935 }
936 }
937 Ok(())
938 }
939
940 fn register_custom_section(&mut self, section: &CustomSectionReader<'data>) {
941 match section.as_known() {
942 KnownCustom::Name(name) => {
943 let result = self.name_section(name);
944 if let Err(e) = result {
945 log::warn!("failed to parse name section {e:?}");
946 }
947 }
948 KnownCustom::BranchHints(reader) if self.tunables.branch_hinting => {
949 // Branch hints are advisory and this section is never validated;
950 // it is decoded lazily during compilation, so record only the
951 // per-function sub-readers here. Discard the whole section if any
952 // entry is malformed rather than applying it partially.
953 let mut hints = HashMap::new();
954 let result: wasmparser::Result<()> = reader.into_iter().try_for_each(|func| {
955 let func = func?;
956 // A well-formed section lists each function at most once; keep
957 // the first entry deterministically if it repeats.
958 hints
959 .entry(FuncIndex::from_u32(func.func))
960 .or_insert(func.hints);
961 Ok(())
962 });
963 match result {
964 Ok(()) => self.result.branch_hints = hints,
965 Err(e) => log::warn!("failed to parse branch-hint section {e:?}"),
966 }
967 }
968 _ => {
969 let name = section.name().trim_end_matches(".dwo");
970 if name.starts_with(".debug_") {
971 self.dwarf_section(name, section);
972 }
973 }
974 }
975 }
976
977 fn dwarf_section(&mut self, name: &str, section: &CustomSectionReader<'data>) {
978 if !self.tunables.debug_native && !self.tunables.parse_wasm_debuginfo {
979 self.result.has_unparsed_debuginfo = true;
980 return;
981 }
982 let info = &mut self.result.debuginfo;
983 let dwarf = &mut info.dwarf;
984 let endian = gimli::LittleEndian;
985 let data = section.data();
986 let slice = gimli::EndianSlice::new(data, endian);
987
988 match name {
989 // `gimli::Dwarf` fields.
990 ".debug_abbrev" => dwarf.debug_abbrev = gimli::DebugAbbrev::new(data, endian),
991 ".debug_addr" => dwarf.debug_addr = gimli::DebugAddr::from(slice),
992 ".debug_info" => {
993 dwarf.debug_info = gimli::DebugInfo::new(data, endian);
994 }
995 ".debug_line" => dwarf.debug_line = gimli::DebugLine::new(data, endian),
996 ".debug_line_str" => dwarf.debug_line_str = gimli::DebugLineStr::from(slice),
997 ".debug_str" => dwarf.debug_str = gimli::DebugStr::new(data, endian),
998 ".debug_str_offsets" => dwarf.debug_str_offsets = gimli::DebugStrOffsets::from(slice),
999 ".debug_str_sup" => {
1000 let mut dwarf_sup: Dwarf<'data> = Default::default();
1001 dwarf_sup.debug_str = gimli::DebugStr::from(slice);
1002 dwarf.sup = Some(Arc::new(dwarf_sup));
1003 }
1004 ".debug_types" => dwarf.debug_types = gimli::DebugTypes::from(slice),
1005
1006 // Additional fields.
1007 ".debug_loc" => info.debug_loc = gimli::DebugLoc::from(slice),
1008 ".debug_loclists" => info.debug_loclists = gimli::DebugLocLists::from(slice),
1009 ".debug_ranges" => info.debug_ranges = gimli::DebugRanges::new(data, endian),
1010 ".debug_rnglists" => info.debug_rnglists = gimli::DebugRngLists::new(data, endian),
1011
1012 // DWARF package fields
1013 ".debug_cu_index" => info.debug_cu_index = gimli::DebugCuIndex::new(data, endian),
1014 ".debug_tu_index" => info.debug_tu_index = gimli::DebugTuIndex::new(data, endian),
1015
1016 // We don't use these at the moment.
1017 ".debug_aranges" | ".debug_pubnames" | ".debug_pubtypes" => return,
1018 other => {
1019 log::warn!("unknown debug section `{other}`");
1020 return;
1021 }
1022 }
1023
1024 dwarf.ranges = gimli::RangeLists::new(info.debug_ranges, info.debug_rnglists);
1025 dwarf.locations = gimli::LocationLists::new(info.debug_loc, info.debug_loclists);
1026 }
1027
1028 /// Declares a new import with the `module` and `field` names, importing the
1029 /// `ty` specified.
1030 ///
1031 /// Note that this method is somewhat tricky due to the implementation of
1032 /// the module linking proposal. In the module linking proposal two-level
1033 /// imports are recast as single-level imports of instances. That recasting
1034 /// happens here by recording an import of an instance for the first time
1035 /// we see a two-level import.
1036 ///
1037 /// When the module linking proposal is disabled, however, disregard this
1038 /// logic and instead work directly with two-level imports since no
1039 /// instances are defined.
1040 fn declare_import(
1041 &mut self,
1042 module: &'data str,
1043 field: &'data str,
1044 ty: EntityType,
1045 ) -> Result<(), OutOfMemory> {
1046 let index = self.push_type(ty);
1047 self.result.module.initializers.push(Initializer::Import {
1048 name: self.result.module.strings.insert(module)?,
1049 field: self.result.module.strings.insert(field)?,
1050 index,
1051 })?;
1052 Ok(())
1053 }
1054
1055 fn push_type(&mut self, ty: EntityType) -> EntityIndex {
1056 match ty {
1057 EntityType::Function(ty) => EntityIndex::Function({
1058 let func_index = self
1059 .result
1060 .module
1061 .push_function(ty.unwrap_module_type_index());
1062 // Imported functions can escape; in fact, they've already done
1063 // so to get here.
1064 self.flag_func_escaped(func_index);
1065 func_index
1066 }),
1067 EntityType::Table(ty) => {
1068 EntityIndex::Table(self.result.module.tables.push(ty).panic_on_oom())
1069 }
1070 EntityType::Memory(ty) => {
1071 EntityIndex::Memory(self.result.module.memories.push(ty).panic_on_oom())
1072 }
1073 EntityType::Global(ty) => {
1074 EntityIndex::Global(self.result.module.globals.push(ty).panic_on_oom())
1075 }
1076 EntityType::Tag(ty) => {
1077 EntityIndex::Tag(self.result.module.tags.push(ty).panic_on_oom())
1078 }
1079 }
1080 }
1081
1082 fn flag_func_escaped(&mut self, func: FuncIndex) {
1083 let ty = &mut self.result.module.functions[func];
1084 // If this was already assigned a funcref index no need to re-assign it.
1085 if ty.is_escaping() {
1086 return;
1087 }
1088 let index = self.result.module.num_escaped_funcs as u32;
1089 ty.func_ref = FuncRefIndex::from_u32(index);
1090 self.result.module.num_escaped_funcs += 1;
1091 }
1092
1093 /// Parses the Name section of the wasm module.
1094 fn name_section(&mut self, names: NameSectionReader<'data>) -> WasmResult<()> {
1095 for subsection in names {
1096 match subsection? {
1097 wasmparser::Name::Function(names) => {
1098 for name in names {
1099 let Naming { index, name } = name?;
1100 // Skip this naming if it's naming a function that
1101 // doesn't actually exist.
1102 if (index as usize) >= self.result.module.functions.len() {
1103 continue;
1104 }
1105
1106 // Store the name unconditionally, regardless of
1107 // whether we're parsing debuginfo, since function
1108 // names are almost always present in the
1109 // final compilation artifact.
1110 let index = FuncIndex::from_u32(index);
1111 self.result
1112 .debuginfo
1113 .name_section
1114 .func_names
1115 .insert(index, name);
1116 }
1117 }
1118 wasmparser::Name::Module { name, .. } => {
1119 self.result.module.name =
1120 Some(self.result.module.strings.insert(name).panic_on_oom());
1121 if self.tunables.debug_native {
1122 self.result.debuginfo.name_section.module_name = Some(name);
1123 }
1124 }
1125 wasmparser::Name::Local(reader) => {
1126 if !self.tunables.debug_native {
1127 continue;
1128 }
1129 for f in reader {
1130 let f = f?;
1131 // Skip this naming if it's naming a function that
1132 // doesn't actually exist.
1133 if (f.index as usize) >= self.result.module.functions.len() {
1134 continue;
1135 }
1136 for name in f.names {
1137 let Naming { index, name } = name?;
1138
1139 self.result
1140 .debuginfo
1141 .name_section
1142 .locals_names
1143 .entry(FuncIndex::from_u32(f.index))
1144 .or_insert(HashMap::new())
1145 .insert(index, name);
1146 }
1147 }
1148 }
1149 wasmparser::Name::Label(_)
1150 | wasmparser::Name::Type(_)
1151 | wasmparser::Name::Table(_)
1152 | wasmparser::Name::Global(_)
1153 | wasmparser::Name::Memory(_)
1154 | wasmparser::Name::Element(_)
1155 | wasmparser::Name::Data(_)
1156 | wasmparser::Name::Tag(_)
1157 | wasmparser::Name::Field(_)
1158 | wasmparser::Name::Unknown { .. } => {}
1159 }
1160 }
1161 Ok(())
1162 }
1163
1164 fn require_startup_func(&mut self) {
1165 self.result.require_startup_func(self.types);
1166 }
1167}
1168
1169impl TypeConvert for ModuleEnvironment<'_, '_> {
1170 fn lookup_heap_type(&self, index: wasmparser::UnpackedIndex) -> WasmHeapType {
1171 WasmparserTypeConverter::new(&self.types, |idx| {
1172 self.result.module.types[idx].unwrap_module_type_index()
1173 })
1174 .lookup_heap_type(index)
1175 }
1176
1177 fn lookup_type_index(&self, index: wasmparser::UnpackedIndex) -> EngineOrModuleTypeIndex {
1178 WasmparserTypeConverter::new(&self.types, |idx| {
1179 self.result.module.types[idx].unwrap_module_type_index()
1180 })
1181 .lookup_type_index(index)
1182 }
1183}
1184
1185impl ModuleTranslation<'_> {
1186 /// Called after translation is complete this will finalize the memory
1187 /// initialization strategy for this module.
1188 ///
1189 /// This will notably use `Self::try_static_init` to attempt to massage
1190 /// data segments to being CoW-init-friendly. Afterwards the
1191 /// `self.memory_init` field is transitioned from `Unprocessed` to
1192 /// `Processed`.
1193 pub fn finalize_memory_init(
1194 &mut self,
1195 tunables: &Tunables,
1196 page_size: u64,
1197 max_image_size_always_allowed: u64,
1198 types: &mut ModuleTypesBuilder,
1199 ) {
1200 if tunables.memory_init_cow {
1201 self.try_static_init(page_size, max_image_size_always_allowed);
1202 }
1203
1204 // If any memory is statically initialized, and if that memory has an
1205 // initial data segment, then a startup function is at least
1206 // conditionally needed if the memory needs initialization. Flag as such
1207 // here.
1208 if let MemoryInitialization::Static { map } = &self.module.memory_initialization {
1209 if map.iter().any(|(_, v)| v.is_some()) {
1210 self.require_startup_func_if_memories_need_init(types);
1211 }
1212 }
1213
1214 // If, after `try_static_init`, initializers are still `Unprocessed`
1215 // then this is the catch-all fallback path for initialization. All
1216 // segments are promoted into `self.runtime_data` and then the
1217 // initialization is rewritten to `Processed`.
1218 if let MemoryInit::Unprocessed(list) = &mut self.memory_init {
1219 let segments = mem::take(list);
1220 let mut new_initializers = Vec::new();
1221 for segment in segments {
1222 new_initializers.push((
1223 segment.memory_index,
1224 MemorySegmentOffset::Expr(segment.offset),
1225 self.runtime_data.push(segment.data.into()),
1226 ));
1227 }
1228 if !new_initializers.is_empty() {
1229 self.require_startup_func(types);
1230 }
1231 self.memory_init = MemoryInit::Processed(new_initializers);
1232 }
1233
1234 // At this point append all passive data to the `runtime_data` list.
1235 // This notably occurs after `try_static_init` above to ensure that the
1236 // page-aligned data for static initialization, if applicable, comes
1237 // first.
1238 for (data_index, segment) in self.passive_data.iter() {
1239 let runtime_index = self.runtime_data.push((*segment).into());
1240 self.runtime_data_map
1241 .insert(*data_index, Some(runtime_index));
1242 }
1243
1244 // And, finally, record all chunks from `self.runtime_data` within
1245 // `self.module.runtime_data` as well.
1246 let mut cur = 0;
1247 for (idx, data) in self.runtime_data.iter() {
1248 let len = u32::try_from(data.len()).unwrap();
1249 let i = self.module.runtime_data.push(cur..cur + len).panic_on_oom();
1250 cur += len;
1251 assert_eq!(idx, i);
1252 }
1253 }
1254
1255 /// Attempts to convert segmented memory initialization into static
1256 /// initialization for the module that this translation represents.
1257 ///
1258 /// If this module's memory initialization is not compatible with paged
1259 /// initialization then this won't change anything. Otherwise if it is
1260 /// compatible then the `memory_initialization` field will be updated.
1261 ///
1262 /// Takes a `page_size` argument in order to ensure that all
1263 /// initialization is page-aligned for mmap-ability, and
1264 /// `max_image_size_always_allowed` to control how we decide
1265 /// whether to use static init.
1266 ///
1267 /// We will try to avoid generating very sparse images, which are
1268 /// possible if e.g. a module has an initializer at offset 0 and a
1269 /// very high offset (say, 1 GiB). To avoid this, we use a dual
1270 /// condition: we always allow images less than
1271 /// `max_image_size_always_allowed`, and the embedder of Wasmtime
1272 /// can set this if desired to ensure that static init should
1273 /// always be done if the size of the module or its heaps is
1274 /// otherwise bounded by the system. We also allow images with
1275 /// static init data bigger than that, but only if it is "dense",
1276 /// defined as having at least half (50%) of its pages with some
1277 /// data.
1278 ///
1279 /// We could do something slightly better by building a dense part
1280 /// and keeping a sparse list of outlier/leftover segments (see
1281 /// issue #3820). This would also allow mostly-static init of
1282 /// modules that have some dynamically-placed data segments. But,
1283 /// for now, this is sufficient to allow a system that "knows what
1284 /// it's doing" to always get static init.
1285 fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) {
1286 let segments = match &mut self.memory_init {
1287 MemoryInit::Unprocessed(list) => list,
1288 _ => return,
1289 };
1290
1291 // First a dry run of memory initialization is performed. This
1292 // collects information about the extent of memory initialized for each
1293 // memory as well as the size of all data segments being copied in.
1294 struct Memory<'a> {
1295 data_size: u64,
1296 min_addr: u64,
1297 max_addr: u64,
1298 segments: Vec<(u64, &'a [u8])>,
1299 }
1300 let mut info = PrimaryMap::with_capacity(self.module.memories.len());
1301 for _ in 0..self.module.memories.len() {
1302 info.push(Memory {
1303 data_size: 0,
1304 min_addr: u64::MAX,
1305 max_addr: 0,
1306 segments: Vec::new(),
1307 });
1308 }
1309
1310 for initializer in segments.iter() {
1311 let &MemoryInitializer {
1312 memory_index,
1313 ref offset,
1314 ref data,
1315 } = initializer;
1316
1317 // Currently `Static` only applies to locally-defined memories,
1318 // so if a data segment references an imported memory then
1319 // transitioning to a `Static` memory initializer is not
1320 // possible.
1321 if self.module.defined_memory_index(memory_index).is_none() {
1322 return;
1323 }
1324
1325 // First up determine the start/end range and verify that they're
1326 // in-bounds for the initial size of the memory at `memory_index`.
1327 // Note that this can bail if we don't have access to globals yet
1328 // (e.g. this is a task happening before instantiation at
1329 // compile-time).
1330 let start = match (offset.ops(), self.module.memories[memory_index].idx_type) {
1331 (&[ConstOp::I32Const(offset)], IndexType::I32) => offset.cast_unsigned().into(),
1332 (&[ConstOp::I64Const(offset)], IndexType::I64) => offset.cast_unsigned(),
1333 _ => return,
1334 };
1335 let len = u64::try_from(data.len()).unwrap();
1336 let end = match start.checked_add(len) {
1337 Some(end) => end,
1338 None => return,
1339 };
1340
1341 match self.module.memories[memory_index].minimum_byte_size() {
1342 Ok(max) => {
1343 if end > max {
1344 return;
1345 }
1346 }
1347
1348 // Note that computing the minimum can overflow if the page
1349 // size is the default 64KiB and the memory's minimum size in
1350 // pages is `1 << 48`, the maximum number of minimum pages for
1351 // 64-bit memories. We don't return `false` to signal an error
1352 // here and instead defer the error to runtime, when it will be
1353 // impossible to allocate that much memory anyways.
1354 Err(_) => return,
1355 }
1356
1357 // Skip empty in-bounds data segments.
1358 if data.is_empty() {
1359 continue;
1360 }
1361
1362 let info = &mut info[memory_index];
1363 let len64 = u64::try_from(data.len()).unwrap();
1364 info.data_size += len64;
1365 info.min_addr = info.min_addr.min(start);
1366 info.max_addr = info.max_addr.max(start + len64);
1367 info.segments.push((start, data));
1368 }
1369
1370 // Validate that the memory information collected is indeed valid for
1371 // static memory initialization.
1372 for (i, info) in info.iter().filter(|(_, info)| info.data_size > 0) {
1373 let image_size = info.max_addr - info.min_addr;
1374
1375 // Simplify things for now by bailing out entirely if any memory has
1376 // a page size smaller than the host's page size. This fixes a case
1377 // where currently initializers are created in host-page-size units
1378 // of length which means that a larger-than-the-entire-memory
1379 // initializer can be created. This can be handled technically but
1380 // would require some more changes to help fix the assert elsewhere
1381 // that this protects against.
1382 if self.module.memories[i].page_size() < page_size {
1383 return;
1384 }
1385
1386 // If the range of memory being initialized is less than twice the
1387 // total size of the data itself then it's assumed that static
1388 // initialization is ok. This means we'll at most double memory
1389 // consumption during the memory image creation process, which is
1390 // currently assumed to "probably be ok" but this will likely need
1391 // tweaks over time.
1392 if image_size < info.data_size.saturating_mul(2) {
1393 continue;
1394 }
1395
1396 // If the memory initialization image is larger than the size of all
1397 // data, then we still allow memory initialization if the image will
1398 // be of a relatively modest size, such as 1MB here.
1399 if image_size < max_image_size_always_allowed {
1400 continue;
1401 }
1402
1403 // At this point memory initialization is concluded to be too
1404 // expensive to do at compile time so it's entirely deferred to
1405 // happen at runtime.
1406 return;
1407 }
1408
1409 // Here's where we've now committed to changing to static memory. The
1410 // memory initialization image is built here from the page data and then
1411 // it's converted to a single initializer.
1412 let mut map = TryPrimaryMap::with_capacity(info.len()).panic_on_oom();
1413 let mut new_initializers = Vec::new();
1414 for (memory, info) in info.iter() {
1415 // Create the in-memory `image` which is the initialized contents of
1416 // this linear memory.
1417 let extent = if info.segments.len() > 0 {
1418 (info.max_addr - info.min_addr) as usize
1419 } else {
1420 0
1421 };
1422 let mut image = Vec::with_capacity(extent);
1423 for (offset, data) in info.segments.iter() {
1424 let offset = usize::try_from(*offset - info.min_addr).unwrap();
1425 if image.len() < offset {
1426 image.resize(offset, 0u8);
1427 image.extend_from_slice(data);
1428 } else {
1429 image.splice(
1430 offset..(offset + data.len()).min(image.len()),
1431 data.iter().copied(),
1432 );
1433 }
1434 }
1435 assert_eq!(image.len(), extent);
1436 assert_eq!(image.capacity(), extent);
1437 let mut offset = if info.segments.len() > 0 {
1438 info.min_addr
1439 } else {
1440 0
1441 };
1442
1443 // Chop off trailing zeros from the image as memory is already
1444 // zero-initialized. Note that `i` is the position of a nonzero
1445 // entry here, so to not lose it we truncate to `i + 1`.
1446 if let Some(i) = image.iter().rposition(|i| *i != 0) {
1447 image.truncate(i + 1);
1448 }
1449
1450 // Also chop off leading zeros, if any.
1451 if let Some(i) = image.iter().position(|i| *i != 0) {
1452 offset += i as u64;
1453 image.drain(..i);
1454 }
1455 let mut len = u64::try_from(image.len()).unwrap();
1456
1457 // The goal is to enable mapping this image directly into memory, so
1458 // the offset into linear memory must be a multiple of the page
1459 // size. If that's not already the case then the image is padded at
1460 // the front and back with extra zeros as necessary
1461 if offset % page_size != 0 {
1462 let zero_padding = offset % page_size;
1463 image.splice(0..0, std::iter::repeat(0).take(zero_padding as usize));
1464 offset -= zero_padding;
1465 len += zero_padding;
1466 }
1467 if len % page_size != 0 {
1468 let zero_padding = page_size - (len % page_size);
1469 image.extend(std::iter::repeat(0).take(zero_padding as usize));
1470 len += zero_padding;
1471 }
1472 let runtime_index = if image.is_empty() {
1473 None
1474 } else {
1475 Some(self.runtime_data.push(image.into()))
1476 };
1477
1478 // Offset/length should now always be page-aligned.
1479 assert!(offset % page_size == 0);
1480 assert!(len % page_size == 0);
1481
1482 // Record the static memory initializer which describes this image,
1483 // only needed if the image is actually present and has a nonzero
1484 // length. The `offset` has been calculates above, originally
1485 // sourced from `info.min_addr`. The `data` field is the extent
1486 // within the final data segment we'll emit to an ELF image, which
1487 // is the concatenation of `self.data`, so here it's the size of
1488 // the section-so-far plus the current segment we're appending.
1489 let idx = map.push(runtime_index.map(|i| (offset, i))).panic_on_oom();
1490 assert_eq!(idx, memory);
1491 if let Some(runtime_index) = runtime_index {
1492 new_initializers.push((idx, MemorySegmentOffset::Static(offset), runtime_index));
1493 }
1494 }
1495 self.data_align = Some(page_size);
1496 self.module.memory_initialization = MemoryInitialization::Static { map };
1497 self.memory_init = MemoryInit::Processed(new_initializers);
1498 }
1499
1500 /// Finalizes the initialization of tables.
1501 ///
1502 /// This is invoked after translation and notably uses
1503 /// `Self::try_func_table_init` to attempt to optimize initialization of
1504 /// tables into static precomputed images.
1505 pub fn finalize_table_init(&mut self, tunables: &Tunables, types: &mut ModuleTypesBuilder) {
1506 if tunables.table_lazy_init {
1507 self.try_func_table_init();
1508 }
1509
1510 // If any table has a non-null initializers, or if there's any active
1511 // data segments, then a startup function is unconditionally required to
1512 // configure the table.
1513 if self
1514 .table_initialization
1515 .initial_values
1516 .iter()
1517 .any(|(_, v)| !matches!(v, TableInitialValue::Null))
1518 || !self.table_initialization.segments.is_empty()
1519 {
1520 self.require_startup_func(types);
1521 }
1522 }
1523
1524 /// Attempts to convert the module's table initializers to
1525 /// FuncTable form where possible. This enables lazy table
1526 /// initialization later by providing a one-to-one map of initial
1527 /// table values, without having to parse all segments.
1528 fn try_func_table_init(&mut self) {
1529 // This should be large enough to support very large Wasm
1530 // modules with huge funcref tables, but small enough to avoid
1531 // OOMs or DoS on truly sparse tables.
1532 const MAX_FUNC_TABLE_SIZE: u64 = 1024 * 1024;
1533
1534 // First convert any element-initialized tables to images of just that
1535 // single function if the minimum size of the table allows doing so.
1536 for ((i, init), (_, table)) in self.table_initialization.initial_values.iter_mut().zip(
1537 self.module
1538 .tables
1539 .iter()
1540 .skip(self.module.num_imported_tables),
1541 ) {
1542 let table_size = table.limits.min;
1543 if table_size > MAX_FUNC_TABLE_SIZE {
1544 continue;
1545 }
1546 if let TableInitialValue::Expr(expr) = init {
1547 if let [ConstOp::RefFunc(f)] = expr.ops() {
1548 assert!(self.module.table_initialization[i].is_empty());
1549 self.module.table_initialization[i] =
1550 try_vec![*f; table_size as usize].panic_on_oom();
1551 *init = TableInitialValue::Null;
1552 }
1553 }
1554 }
1555
1556 let mut segments = mem::take(&mut self.table_initialization.segments)
1557 .into_iter()
1558 .peekable();
1559
1560 // The goal of this loop is to interpret a table segment and apply it
1561 // "statically" to a local table. This will iterate over segments and
1562 // apply them one-by-one to each table.
1563 //
1564 // If any segment can't be applied, however, then this loop exits and
1565 // all remaining segments are placed back into the segment list. This is
1566 // because segments are supposed to be initialized one-at-a-time which
1567 // means that intermediate state is visible with respect to traps. If
1568 // anything isn't statically known to not trap it's pessimistically
1569 // assumed to trap meaning all further segment initializers must be
1570 // applied manually at instantiation time.
1571 while let Some(segment) = segments.peek() {
1572 let defined_index = match self.module.defined_table_index(segment.table_index) {
1573 Some(index) => index,
1574 // Skip imported tables: we can't provide a preconstructed
1575 // table for them, because their values depend on the
1576 // imported table overlaid with whatever segments we have.
1577 None => break,
1578 };
1579
1580 // If the base of this segment is dynamic, then we can't
1581 // include it in the statically-built array of initial
1582 // contents.
1583 let offset = match segment.offset.ops() {
1584 &[ConstOp::I32Const(offset)] => u64::from(offset.cast_unsigned()),
1585 &[ConstOp::I64Const(offset)] => offset.cast_unsigned(),
1586 _ => break,
1587 };
1588
1589 // Get the end of this segment. If out-of-bounds, or too
1590 // large for our dense table representation, then skip the
1591 // segment.
1592 let top = match offset.checked_add(segment.elements.len()) {
1593 Some(top) => top,
1594 None => break,
1595 };
1596 let table_size = self.module.tables[segment.table_index].limits.min;
1597 if top > table_size || top > MAX_FUNC_TABLE_SIZE {
1598 break;
1599 }
1600
1601 match self.module.tables[segment.table_index]
1602 .ref_type
1603 .heap_type
1604 .top()
1605 {
1606 WasmHeapTopType::Func => {}
1607 // If this is not a funcref table, then we can't support a
1608 // pre-computed table of function indices. Technically this
1609 // initializer won't trap so we could continue processing
1610 // segments, but that's left as a future optimization if
1611 // necessary.
1612 WasmHeapTopType::Any
1613 | WasmHeapTopType::Extern
1614 | WasmHeapTopType::Cont
1615 | WasmHeapTopType::Exn => break,
1616 }
1617
1618 // Function indices can be optimized here, but fully general
1619 // expressions are deferred to get evaluated at runtime.
1620 let function_elements = match &segment.elements {
1621 TableSegmentElements::Functions(indices) => indices,
1622 TableSegmentElements::Expressions { .. } => break,
1623 };
1624
1625 match &self.table_initialization.initial_values[defined_index] {
1626 TableInitialValue::Null => {}
1627
1628 // If this table is still listed as an initial value here
1629 // then that means the initial size of the table doesn't
1630 // support a precomputed function list, so skip this.
1631 // Technically this won't trap so it's possible to process
1632 // further initializers, but that's left as a future
1633 // optimization.
1634 TableInitialValue::Expr(_) => break,
1635 }
1636 let precomputed = &mut self.module.table_initialization[defined_index];
1637
1638 // At this point we're committing to pre-initializing the table
1639 // with the `segment` that's being iterated over. This segment is
1640 // applied to the `precomputed` list for the table by ensuring
1641 // it's large enough to hold the segment and then copying the
1642 // segment into the precomputed list.
1643 if precomputed.len() < top as usize {
1644 precomputed
1645 .resize(top as usize, FuncIndex::reserved_value())
1646 .panic_on_oom();
1647 }
1648 let dst = &mut precomputed[offset as usize..top as usize];
1649 dst.copy_from_slice(&function_elements);
1650
1651 // advance the iterator to see the next segment
1652 let _ = segments.next();
1653 }
1654 self.table_initialization.segments = segments.try_collect().panic_on_oom();
1655 }
1656
1657 /// Helper function to ratchet the `startup` function for this module as
1658 /// `Always`.
1659 fn require_startup_func(&mut self, types: &mut ModuleTypesBuilder) {
1660 let ty = match self.module.startup {
1661 ModuleStartup::None => types.startup_func_type().into(),
1662 ModuleStartup::Always(_) => return,
1663 ModuleStartup::IfMemoriesNeedInit(ty) => ty,
1664 };
1665 self.module.startup = ModuleStartup::Always(ty);
1666 }
1667
1668 /// Helper function to ratchet the `startup` function for this module as
1669 /// `IfMemoriesNeedInit`.
1670 fn require_startup_func_if_memories_need_init(&mut self, types: &mut ModuleTypesBuilder) {
1671 let ty = match self.module.startup {
1672 ModuleStartup::None => types.startup_func_type().into(),
1673 ModuleStartup::Always(_) | ModuleStartup::IfMemoriesNeedInit(_) => return,
1674 };
1675 self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty);
1676 }
1677}