cranelift_codegen/ir/
instructions.rs

1//! Instruction formats and opcodes.
2//!
3//! The `instructions` module contains definitions for instruction formats, opcodes, and the
4//! in-memory representation of IR instructions.
5//!
6//! A large part of this module is auto-generated from the instruction descriptions in the meta
7//! directory.
8
9use crate::constant_hash::Table;
10use alloc::vec::Vec;
11use core::fmt::{self, Display, Formatter};
12use core::ops::{Deref, DerefMut};
13use core::str::FromStr;
14
15#[cfg(feature = "enable-serde")]
16use serde_derive::{Deserialize, Serialize};
17
18use crate::bitset::ScalarBitSet;
19use crate::entity;
20use crate::ir::{
21    self, Block, ExceptionTable, ExceptionTables, FuncRef, MemFlags, SigRef, StackSlot, Type,
22    Value,
23    condcodes::{FloatCC, IntCC},
24    trapcode::TrapCode,
25    types,
26};
27
28/// Some instructions use an external list of argument values because there is not enough space in
29/// the 16-byte `InstructionData` struct. These value lists are stored in a memory pool in
30/// `dfg.value_lists`.
31pub type ValueList = entity::EntityList<Value>;
32
33/// Memory pool for holding value lists. See `ValueList`.
34pub type ValueListPool = entity::ListPool<Value>;
35
36/// A pair of a Block and its arguments, stored in a single EntityList internally.
37///
38/// Block arguments are semantically a `BlockArg`.
39///
40/// NOTE: We don't expose either value_to_block or block_to_value outside of this module because
41/// this operation is not generally safe. However, as the two share the same underlying layout,
42/// they can be stored in the same value pool.
43///
44/// BlockCall makes use of this shared layout by storing all of its contents (a block and its
45/// argument) in a single EntityList. This is a bit better than introducing a new entity type for
46/// the pair of a block name and the arguments entity list, as we don't pay any indirection penalty
47/// to get to the argument values -- they're stored in-line with the block in the same list.
48///
49/// The BlockCall::new function guarantees this layout by requiring a block argument that's written
50/// in as the first element of the EntityList. Any subsequent entries are always assumed to be real
51/// Values.
52#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
53#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
54pub struct BlockCall {
55    /// The underlying storage for the BlockCall. The first element of the values EntityList is
56    /// guaranteed to always be a Block encoded as a Value via BlockCall::block_to_value.
57    /// Consequently, the values entity list is never empty.
58    values: entity::EntityList<Value>,
59}
60
61impl BlockCall {
62    // NOTE: the only uses of this function should be internal to BlockCall. See the block comment
63    // on BlockCall for more context.
64    fn value_to_block(val: Value) -> Block {
65        Block::from_u32(val.as_u32())
66    }
67
68    // NOTE: the only uses of this function should be internal to BlockCall. See the block comment
69    // on BlockCall for more context.
70    fn block_to_value(block: Block) -> Value {
71        Value::from_u32(block.as_u32())
72    }
73
74    /// Construct a BlockCall with the given block and arguments.
75    pub fn new(
76        block: Block,
77        args: impl IntoIterator<Item = BlockArg>,
78        pool: &mut ValueListPool,
79    ) -> Self {
80        let mut values = ValueList::default();
81        values.push(Self::block_to_value(block), pool);
82        values.extend(args.into_iter().map(|arg| arg.encode_as_value()), pool);
83        Self { values }
84    }
85
86    /// Return the block for this BlockCall.
87    pub fn block(&self, pool: &ValueListPool) -> Block {
88        let val = self.values.first(pool).unwrap();
89        Self::value_to_block(val)
90    }
91
92    /// Replace the block for this BlockCall.
93    pub fn set_block(&mut self, block: Block, pool: &mut ValueListPool) {
94        *self.values.get_mut(0, pool).unwrap() = Self::block_to_value(block);
95    }
96
97    /// Append an argument to the block args.
98    pub fn append_argument(&mut self, arg: impl Into<BlockArg>, pool: &mut ValueListPool) {
99        self.values.push(arg.into().encode_as_value(), pool);
100    }
101
102    /// Return the length of the argument list.
103    pub fn len(&self, pool: &ValueListPool) -> usize {
104        self.values.len(pool) - 1
105    }
106
107    /// Return an iterator over the arguments of this block.
108    pub fn args<'a>(
109        &self,
110        pool: &'a ValueListPool,
111    ) -> impl ExactSizeIterator<Item = BlockArg> + DoubleEndedIterator<Item = BlockArg> + use<'a>
112    {
113        self.values.as_slice(pool)[1..]
114            .iter()
115            .map(|value| BlockArg::decode_from_value(*value))
116    }
117
118    /// Traverse the arguments with a closure that can mutate them.
119    pub fn update_args<F: FnMut(BlockArg) -> BlockArg>(
120        &mut self,
121        pool: &mut ValueListPool,
122        mut f: F,
123    ) {
124        for raw in self.values.as_mut_slice(pool)[1..].iter_mut() {
125            let new = f(BlockArg::decode_from_value(*raw));
126            *raw = new.encode_as_value();
127        }
128    }
129
130    /// Remove the argument at ix from the argument list.
131    pub fn remove(&mut self, ix: usize, pool: &mut ValueListPool) {
132        self.values.remove(1 + ix, pool)
133    }
134
135    /// Clear out the arguments list.
136    pub fn clear(&mut self, pool: &mut ValueListPool) {
137        self.values.truncate(1, pool)
138    }
139
140    /// Appends multiple elements to the arguments.
141    pub fn extend<I, T>(&mut self, elements: I, pool: &mut ValueListPool)
142    where
143        I: IntoIterator<Item = T>,
144        T: Into<BlockArg>,
145    {
146        self.values.extend(
147            elements
148                .into_iter()
149                .map(|elem| elem.into().encode_as_value()),
150            pool,
151        )
152    }
153
154    /// Return a value that can display this block call.
155    pub fn display<'a>(&self, pool: &'a ValueListPool) -> DisplayBlockCall<'a> {
156        DisplayBlockCall { block: *self, pool }
157    }
158
159    /// Deep-clone the underlying list in the same pool. The returned
160    /// list will have identical contents but changes to this list
161    /// will not change its contents or vice-versa.
162    pub fn deep_clone(&self, pool: &mut ValueListPool) -> Self {
163        Self {
164            values: self.values.deep_clone(pool),
165        }
166    }
167}
168
169/// Wrapper for the context needed to display a [BlockCall] value.
170pub struct DisplayBlockCall<'a> {
171    block: BlockCall,
172    pool: &'a ValueListPool,
173}
174
175impl<'a> Display for DisplayBlockCall<'a> {
176    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
177        write!(f, "{}", self.block.block(&self.pool))?;
178        if self.block.len(self.pool) > 0 {
179            write!(f, "(")?;
180            for (ix, arg) in self.block.args(self.pool).enumerate() {
181                if ix > 0 {
182                    write!(f, ", ")?;
183                }
184                write!(f, "{arg}")?;
185            }
186            write!(f, ")")?;
187        }
188        Ok(())
189    }
190}
191
192/// A `BlockArg` is a sum type of `Value`, `TryCallRet`, and
193/// `TryCallExn`. The latter two are values that are generated "on the
194/// edge" out of a `try_call` instruction into a successor block. We
195/// use special arguments rather than special values for these because
196/// they are not definable as SSA values at a certain program point --
197/// only when the `BlockCall` is executed.
198#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
199pub enum BlockArg {
200    /// An ordinary value, usable at the branch instruction using this
201    /// `BlockArg`, whose value is passed as an argument.
202    Value(Value),
203
204    /// A return value of a `try_call`'s called function. Signatures
205    /// allow multiple return values, so this carries an index. This
206    /// may be used only on the normal (non-exceptional) `BlockCall`
207    /// out of a `try_call` or `try_call_indirect` instruction.
208    TryCallRet(u32),
209
210    /// An exception payload value of a `try_call`. Some ABIs may
211    /// allow multiple payload values, so this carries an index. Its
212    /// type is defined by the ABI of the called function. This may be
213    /// used only on an exceptional `BlockCall` out of a `try_call` or
214    /// `try_call_indirect` instruction.
215    TryCallExn(u32),
216}
217
218impl BlockArg {
219    /// Encode this block argument as a `Value` for storage in the
220    /// value pool. Internal to `BlockCall`, must not be used
221    /// elsewhere to avoid exposing the raw bit encoding.
222    fn encode_as_value(&self) -> Value {
223        let (tag, payload) = match *self {
224            BlockArg::Value(v) => (0, v.as_bits()),
225            BlockArg::TryCallRet(i) => (1, i),
226            BlockArg::TryCallExn(i) => (2, i),
227        };
228        assert!(payload < (1 << 30));
229        let raw = (tag << 30) | payload;
230        Value::from_bits(raw)
231    }
232
233    /// Decode a raw `Value` encoding of this block argument.
234    fn decode_from_value(v: Value) -> Self {
235        let raw = v.as_u32();
236        let tag = raw >> 30;
237        let payload = raw & ((1 << 30) - 1);
238        match tag {
239            0 => BlockArg::Value(Value::from_bits(payload)),
240            1 => BlockArg::TryCallRet(payload),
241            2 => BlockArg::TryCallExn(payload),
242            _ => unreachable!(),
243        }
244    }
245
246    /// Return this argument as a `Value`, if it is one, or `None`
247    /// otherwise.
248    pub fn as_value(&self) -> Option<Value> {
249        match *self {
250            BlockArg::Value(v) => Some(v),
251            _ => None,
252        }
253    }
254
255    /// Update the contained value, if any.
256    pub fn map_value<F: FnMut(Value) -> Value>(&self, mut f: F) -> Self {
257        match *self {
258            BlockArg::Value(v) => BlockArg::Value(f(v)),
259            other => other,
260        }
261    }
262}
263
264impl Display for BlockArg {
265    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
266        match self {
267            BlockArg::Value(v) => write!(f, "{v}"),
268            BlockArg::TryCallRet(i) => write!(f, "ret{i}"),
269            BlockArg::TryCallExn(i) => write!(f, "exn{i}"),
270        }
271    }
272}
273
274impl From<Value> for BlockArg {
275    fn from(value: Value) -> BlockArg {
276        BlockArg::Value(value)
277    }
278}
279
280// Include code generated by `cranelift-codegen/meta/src/gen_inst.rs`. This file contains:
281//
282// - The `pub enum InstructionFormat` enum with all the instruction formats.
283// - The `pub enum InstructionData` enum with all the instruction data fields.
284// - The `pub enum Opcode` definition with all known opcodes,
285// - The `const OPCODE_FORMAT: [InstructionFormat; N]` table.
286// - The private `fn opcode_name(Opcode) -> &'static str` function, and
287// - The hash table `const OPCODE_HASH_TABLE: [Opcode; N]`.
288//
289// For value type constraints:
290//
291// - The `const OPCODE_CONSTRAINTS : [OpcodeConstraints; N]` table.
292// - The `const TYPE_SETS : [ValueTypeSet; N]` table.
293// - The `const OPERAND_CONSTRAINTS : [OperandConstraint; N]` table.
294//
295include!(concat!(env!("OUT_DIR"), "/opcodes.rs"));
296
297impl Display for Opcode {
298    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
299        write!(f, "{}", opcode_name(*self))
300    }
301}
302
303impl Opcode {
304    /// Get the instruction format for this opcode.
305    pub fn format(self) -> InstructionFormat {
306        OPCODE_FORMAT[self as usize - 1]
307    }
308
309    /// Get the constraint descriptor for this opcode.
310    /// Panic if this is called on `NotAnOpcode`.
311    pub fn constraints(self) -> OpcodeConstraints {
312        OPCODE_CONSTRAINTS[self as usize - 1]
313    }
314
315    /// Is this instruction a GC safepoint?
316    ///
317    /// Safepoints are all kinds of calls, except for tail calls.
318    #[inline]
319    pub fn is_safepoint(self) -> bool {
320        self.is_call() && !self.is_return()
321    }
322}
323
324// This trait really belongs in cranelift-reader where it is used by the `.clif` file parser, but since
325// it critically depends on the `opcode_name()` function which is needed here anyway, it lives in
326// this module. This also saves us from running the build script twice to generate code for the two
327// separate crates.
328impl FromStr for Opcode {
329    type Err = &'static str;
330
331    /// Parse an Opcode name from a string.
332    fn from_str(s: &str) -> Result<Self, &'static str> {
333        use crate::constant_hash::{probe, simple_hash};
334
335        match probe::<&str, [Option<Self>]>(&OPCODE_HASH_TABLE, s, simple_hash(s)) {
336            Err(_) => Err("Unknown opcode"),
337            // We unwrap here because probe() should have ensured that the entry
338            // at this index is not None.
339            Ok(i) => Ok(OPCODE_HASH_TABLE[i].unwrap()),
340        }
341    }
342}
343
344impl<'a> Table<&'a str> for [Option<Opcode>] {
345    fn len(&self) -> usize {
346        self.len()
347    }
348
349    fn key(&self, idx: usize) -> Option<&'a str> {
350        self[idx].map(opcode_name)
351    }
352}
353
354/// A variable list of `Value` operands used for function call arguments and passing arguments to
355/// basic blocks.
356#[derive(Clone, Debug)]
357pub struct VariableArgs(Vec<Value>);
358
359impl VariableArgs {
360    /// Create an empty argument list.
361    pub fn new() -> Self {
362        Self(Vec::new())
363    }
364
365    /// Add an argument to the end.
366    pub fn push(&mut self, v: Value) {
367        self.0.push(v)
368    }
369
370    /// Check if the list is empty.
371    pub fn is_empty(&self) -> bool {
372        self.0.is_empty()
373    }
374
375    /// Convert this to a value list in `pool` with `fixed` prepended.
376    pub fn into_value_list(self, fixed: &[Value], pool: &mut ValueListPool) -> ValueList {
377        let mut vlist = ValueList::default();
378        vlist.extend(fixed.iter().cloned(), pool);
379        vlist.extend(self.0, pool);
380        vlist
381    }
382}
383
384// Coerce `VariableArgs` into a `&[Value]` slice.
385impl Deref for VariableArgs {
386    type Target = [Value];
387
388    fn deref(&self) -> &[Value] {
389        &self.0
390    }
391}
392
393impl DerefMut for VariableArgs {
394    fn deref_mut(&mut self) -> &mut [Value] {
395        &mut self.0
396    }
397}
398
399impl Display for VariableArgs {
400    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
401        for (i, val) in self.0.iter().enumerate() {
402            if i == 0 {
403                write!(fmt, "{val}")?;
404            } else {
405                write!(fmt, ", {val}")?;
406            }
407        }
408        Ok(())
409    }
410}
411
412impl Default for VariableArgs {
413    fn default() -> Self {
414        Self::new()
415    }
416}
417
418/// Analyzing an instruction.
419///
420/// Avoid large matches on instruction formats by using the methods defined here to examine
421/// instructions.
422impl InstructionData {
423    /// Get the destinations of this instruction, if it's a branch.
424    ///
425    /// `br_table` returns the empty slice.
426    pub fn branch_destination<'a>(
427        &'a self,
428        jump_tables: &'a ir::JumpTables,
429        exception_tables: &'a ir::ExceptionTables,
430    ) -> &'a [BlockCall] {
431        match self {
432            Self::Jump { destination, .. } => std::slice::from_ref(destination),
433            Self::Brif { blocks, .. } => blocks.as_slice(),
434            Self::BranchTable { table, .. } => jump_tables.get(*table).unwrap().all_branches(),
435            Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => {
436                exception_tables.get(*exception).unwrap().all_branches()
437            }
438            _ => {
439                debug_assert!(!self.opcode().is_branch());
440                &[]
441            }
442        }
443    }
444
445    /// Get a mutable slice of the destinations of this instruction, if it's a branch.
446    ///
447    /// `br_table` returns the empty slice.
448    pub fn branch_destination_mut<'a>(
449        &'a mut self,
450        jump_tables: &'a mut ir::JumpTables,
451        exception_tables: &'a mut ir::ExceptionTables,
452    ) -> &'a mut [BlockCall] {
453        match self {
454            Self::Jump { destination, .. } => std::slice::from_mut(destination),
455            Self::Brif { blocks, .. } => blocks.as_mut_slice(),
456            Self::BranchTable { table, .. } => {
457                jump_tables.get_mut(*table).unwrap().all_branches_mut()
458            }
459            Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => {
460                exception_tables
461                    .get_mut(*exception)
462                    .unwrap()
463                    .all_branches_mut()
464            }
465            _ => {
466                debug_assert!(!self.opcode().is_branch());
467                &mut []
468            }
469        }
470    }
471
472    /// Replace the values used in this instruction according to the given
473    /// function.
474    pub fn map_values(
475        &mut self,
476        pool: &mut ValueListPool,
477        jump_tables: &mut ir::JumpTables,
478        exception_tables: &mut ir::ExceptionTables,
479        mut f: impl FnMut(Value) -> Value,
480    ) {
481        // Map all normal operator args.
482        for arg in self.arguments_mut(pool) {
483            *arg = f(*arg);
484        }
485
486        // Map all BlockCall args.
487        for block in self.branch_destination_mut(jump_tables, exception_tables) {
488            block.update_args(pool, |arg| arg.map_value(|val| f(val)));
489        }
490
491        // Map all context items.
492        if let Some(et) = self.exception_table() {
493            for ctx in exception_tables[et].contexts_mut() {
494                *ctx = f(*ctx);
495            }
496        }
497    }
498
499    /// If this is a trapping instruction, get its trap code. Otherwise, return
500    /// `None`.
501    pub fn trap_code(&self) -> Option<TrapCode> {
502        match *self {
503            Self::CondTrap { code, .. }
504            | Self::IntAddTrap { code, .. }
505            | Self::Trap { code, .. } => Some(code),
506            _ => None,
507        }
508    }
509
510    /// If this is a control-flow instruction depending on an integer condition, gets its
511    /// condition.  Otherwise, return `None`.
512    pub fn cond_code(&self) -> Option<IntCC> {
513        match self {
514            &InstructionData::IntCompare { cond, .. }
515            | &InstructionData::IntCompareImm { cond, .. } => Some(cond),
516            _ => None,
517        }
518    }
519
520    /// If this is a control-flow instruction depending on a floating-point condition, gets its
521    /// condition.  Otherwise, return `None`.
522    pub fn fp_cond_code(&self) -> Option<FloatCC> {
523        match self {
524            &InstructionData::FloatCompare { cond, .. } => Some(cond),
525            _ => None,
526        }
527    }
528
529    /// If this is a trapping instruction, get an exclusive reference to its
530    /// trap code. Otherwise, return `None`.
531    pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> {
532        match self {
533            Self::CondTrap { code, .. }
534            | Self::IntAddTrap { code, .. }
535            | Self::Trap { code, .. } => Some(code),
536            _ => None,
537        }
538    }
539
540    /// If this is an atomic read/modify/write instruction, return its subopcode.
541    pub fn atomic_rmw_op(&self) -> Option<ir::AtomicRmwOp> {
542        match self {
543            &InstructionData::AtomicRmw { op, .. } => Some(op),
544            _ => None,
545        }
546    }
547
548    /// If this is a load/store instruction, returns its immediate offset.
549    pub fn load_store_offset(&self) -> Option<i32> {
550        match self {
551            &InstructionData::Load { offset, .. }
552            | &InstructionData::StackLoad { offset, .. }
553            | &InstructionData::Store { offset, .. }
554            | &InstructionData::StackStore { offset, .. } => Some(offset.into()),
555            _ => None,
556        }
557    }
558
559    /// If this is a load/store instruction, return its memory flags.
560    pub fn memflags(&self) -> Option<MemFlags> {
561        match self {
562            &InstructionData::Load { flags, .. }
563            | &InstructionData::LoadNoOffset { flags, .. }
564            | &InstructionData::Store { flags, .. }
565            | &InstructionData::StoreNoOffset { flags, .. }
566            | &InstructionData::AtomicCas { flags, .. }
567            | &InstructionData::AtomicRmw { flags, .. } => Some(flags),
568            _ => None,
569        }
570    }
571
572    /// If this instruction references a stack slot, return it
573    pub fn stack_slot(&self) -> Option<StackSlot> {
574        match self {
575            &InstructionData::StackStore { stack_slot, .. }
576            | &InstructionData::StackLoad { stack_slot, .. } => Some(stack_slot),
577            _ => None,
578        }
579    }
580
581    /// Return information about a call instruction.
582    ///
583    /// Any instruction that can call another function reveals its call signature here.
584    pub fn analyze_call<'a>(
585        &'a self,
586        pool: &'a ValueListPool,
587        exception_tables: &ExceptionTables,
588    ) -> CallInfo<'a> {
589        match *self {
590            Self::Call {
591                func_ref, ref args, ..
592            } => CallInfo::Direct(func_ref, args.as_slice(pool)),
593            Self::CallIndirect {
594                sig_ref, ref args, ..
595            } => CallInfo::Indirect(sig_ref, &args.as_slice(pool)[1..]),
596            Self::TryCall {
597                func_ref,
598                ref args,
599                exception,
600                ..
601            } => {
602                let exdata = &exception_tables[exception];
603                CallInfo::DirectWithSig(func_ref, exdata.signature(), args.as_slice(pool))
604            }
605            Self::TryCallIndirect {
606                exception,
607                ref args,
608                ..
609            } => {
610                let exdata = &exception_tables[exception];
611                CallInfo::Indirect(exdata.signature(), &args.as_slice(pool)[1..])
612            }
613            Self::Ternary {
614                opcode: Opcode::StackSwitch,
615                ..
616            } => {
617                // `StackSwitch` is not actually a call, but has the .call() side
618                // effect as it continues execution elsewhere.
619                CallInfo::NotACall
620            }
621            _ => {
622                debug_assert!(!self.opcode().is_call());
623                CallInfo::NotACall
624            }
625        }
626    }
627
628    #[inline]
629    pub(crate) fn mask_immediates(&mut self, ctrl_typevar: Type) {
630        if ctrl_typevar.is_invalid() {
631            return;
632        }
633
634        let bit_width = ctrl_typevar.bits();
635
636        match self {
637            Self::UnaryImm { opcode: _, imm } => {
638                *imm = imm.mask_to_width(bit_width);
639            }
640            Self::BinaryImm64 {
641                opcode,
642                arg: _,
643                imm,
644            } => {
645                if *opcode == Opcode::SdivImm || *opcode == Opcode::SremImm {
646                    *imm = imm.mask_to_width(bit_width);
647                }
648            }
649            Self::IntCompareImm {
650                opcode,
651                arg: _,
652                cond,
653                imm,
654            } => {
655                debug_assert_eq!(*opcode, Opcode::IcmpImm);
656                if cond.unsigned() != *cond {
657                    *imm = imm.mask_to_width(bit_width);
658                }
659            }
660            _ => {}
661        }
662    }
663
664    /// Get the exception table, if any, associated with this instruction.
665    pub fn exception_table(&self) -> Option<ExceptionTable> {
666        match self {
667            Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => {
668                Some(*exception)
669            }
670            _ => None,
671        }
672    }
673}
674
675/// Information about call instructions.
676pub enum CallInfo<'a> {
677    /// This is not a call instruction.
678    NotACall,
679
680    /// This is a direct call to an external function declared in the preamble. See
681    /// `DataFlowGraph.ext_funcs`.
682    Direct(FuncRef, &'a [Value]),
683
684    /// This is an indirect call with the specified signature. See `DataFlowGraph.signatures`.
685    Indirect(SigRef, &'a [Value]),
686
687    /// This is a direct call to an external function declared in the
688    /// preamble, but the signature is also known by other means:
689    /// e.g., from an exception table entry.
690    DirectWithSig(FuncRef, SigRef, &'a [Value]),
691}
692
693/// Value type constraints for a given opcode.
694///
695/// The `InstructionFormat` determines the constraints on most operands, but `Value` operands and
696/// results are not determined by the format. Every `Opcode` has an associated
697/// `OpcodeConstraints` object that provides the missing details.
698#[derive(Clone, Copy)]
699pub struct OpcodeConstraints {
700    /// Flags for this opcode encoded as a bit field:
701    ///
702    /// Bits 0-2:
703    ///     Number of fixed result values. This does not include `variable_args` results as are
704    ///     produced by call instructions.
705    ///
706    /// Bit 3:
707    ///     This opcode is polymorphic and the controlling type variable can be inferred from the
708    ///     designated input operand. This is the `typevar_operand` index given to the
709    ///     `InstructionFormat` meta language object. When this bit is not set, the controlling
710    ///     type variable must be the first output value instead.
711    ///
712    /// Bit 4:
713    ///     This opcode is polymorphic and the controlling type variable does *not* appear as the
714    ///     first result type.
715    ///
716    /// Bits 5-7:
717    ///     Number of fixed value arguments. The minimum required number of value operands.
718    flags: u8,
719
720    /// Permitted set of types for the controlling type variable as an index into `TYPE_SETS`.
721    typeset_offset: u8,
722
723    /// Offset into `OPERAND_CONSTRAINT` table of the descriptors for this opcode. The first
724    /// `num_fixed_results()` entries describe the result constraints, then follows constraints for
725    /// the fixed `Value` input operands. (`num_fixed_value_arguments()` of them).
726    constraint_offset: u16,
727}
728
729impl OpcodeConstraints {
730    /// Can the controlling type variable for this opcode be inferred from the designated value
731    /// input operand?
732    /// This also implies that this opcode is polymorphic.
733    pub fn use_typevar_operand(self) -> bool {
734        (self.flags & 0x8) != 0
735    }
736
737    /// Is it necessary to look at the designated value input operand in order to determine the
738    /// controlling type variable, or is it good enough to use the first return type?
739    ///
740    /// Most polymorphic instructions produce a single result with the type of the controlling type
741    /// variable. A few polymorphic instructions either don't produce any results, or produce
742    /// results with a fixed type. These instructions return `true`.
743    pub fn requires_typevar_operand(self) -> bool {
744        (self.flags & 0x10) != 0
745    }
746
747    /// Get the number of *fixed* result values produced by this opcode.
748    /// This does not include `variable_args` produced by calls.
749    pub fn num_fixed_results(self) -> usize {
750        (self.flags & 0x7) as usize
751    }
752
753    /// Get the number of *fixed* input values required by this opcode.
754    ///
755    /// This does not include `variable_args` arguments on call and branch instructions.
756    ///
757    /// The number of fixed input values is usually implied by the instruction format, but
758    /// instruction formats that use a `ValueList` put both fixed and variable arguments in the
759    /// list. This method returns the *minimum* number of values required in the value list.
760    pub fn num_fixed_value_arguments(self) -> usize {
761        ((self.flags >> 5) & 0x7) as usize
762    }
763
764    /// Get the offset into `TYPE_SETS` for the controlling type variable.
765    /// Returns `None` if the instruction is not polymorphic.
766    fn typeset_offset(self) -> Option<usize> {
767        let offset = usize::from(self.typeset_offset);
768        if offset < TYPE_SETS.len() {
769            Some(offset)
770        } else {
771            None
772        }
773    }
774
775    /// Get the offset into OPERAND_CONSTRAINTS where the descriptors for this opcode begin.
776    fn constraint_offset(self) -> usize {
777        self.constraint_offset as usize
778    }
779
780    /// Get the value type of result number `n`, having resolved the controlling type variable to
781    /// `ctrl_type`.
782    pub fn result_type(self, n: usize, ctrl_type: Type) -> Type {
783        debug_assert!(n < self.num_fixed_results(), "Invalid result index");
784        match OPERAND_CONSTRAINTS[self.constraint_offset() + n].resolve(ctrl_type) {
785            ResolvedConstraint::Bound(t) => t,
786            ResolvedConstraint::Free(ts) => panic!("Result constraints can't be free: {ts:?}"),
787        }
788    }
789
790    /// Get the value type of input value number `n`, having resolved the controlling type variable
791    /// to `ctrl_type`.
792    ///
793    /// Unlike results, it is possible for some input values to vary freely within a specific
794    /// `ValueTypeSet`. This is represented with the `ArgumentConstraint::Free` variant.
795    pub fn value_argument_constraint(self, n: usize, ctrl_type: Type) -> ResolvedConstraint {
796        debug_assert!(
797            n < self.num_fixed_value_arguments(),
798            "Invalid value argument index"
799        );
800        let offset = self.constraint_offset() + self.num_fixed_results();
801        OPERAND_CONSTRAINTS[offset + n].resolve(ctrl_type)
802    }
803
804    /// Get the typeset of allowed types for the controlling type variable in a polymorphic
805    /// instruction.
806    pub fn ctrl_typeset(self) -> Option<ValueTypeSet> {
807        self.typeset_offset().map(|offset| TYPE_SETS[offset])
808    }
809
810    /// Is this instruction polymorphic?
811    pub fn is_polymorphic(self) -> bool {
812        self.ctrl_typeset().is_some()
813    }
814}
815
816type BitSet8 = ScalarBitSet<u8>;
817type BitSet16 = ScalarBitSet<u16>;
818
819/// A value type set describes the permitted set of types for a type variable.
820#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
821pub struct ValueTypeSet {
822    /// Allowed lane sizes
823    pub lanes: BitSet16,
824    /// Allowed int widths
825    pub ints: BitSet8,
826    /// Allowed float widths
827    pub floats: BitSet8,
828    /// Allowed dynamic vectors minimum lane sizes
829    pub dynamic_lanes: BitSet16,
830}
831
832impl ValueTypeSet {
833    /// Is `scalar` part of the base type set?
834    ///
835    /// Note that the base type set does not have to be included in the type set proper.
836    fn is_base_type(self, scalar: Type) -> bool {
837        let l2b = u8::try_from(scalar.log2_lane_bits()).unwrap();
838        if scalar.is_int() {
839            self.ints.contains(l2b)
840        } else if scalar.is_float() {
841            self.floats.contains(l2b)
842        } else {
843            false
844        }
845    }
846
847    /// Does `typ` belong to this set?
848    pub fn contains(self, typ: Type) -> bool {
849        if typ.is_dynamic_vector() {
850            let l2l = u8::try_from(typ.log2_min_lane_count()).unwrap();
851            self.dynamic_lanes.contains(l2l) && self.is_base_type(typ.lane_type())
852        } else {
853            let l2l = u8::try_from(typ.log2_lane_count()).unwrap();
854            self.lanes.contains(l2l) && self.is_base_type(typ.lane_type())
855        }
856    }
857
858    /// Get an example member of this type set.
859    ///
860    /// This is used for error messages to avoid suggesting invalid types.
861    pub fn example(self) -> Type {
862        let t = if self.ints.max().unwrap_or(0) > 5 {
863            types::I32
864        } else if self.floats.max().unwrap_or(0) > 5 {
865            types::F32
866        } else {
867            types::I8
868        };
869        t.by(1 << self.lanes.min().unwrap()).unwrap()
870    }
871}
872
873/// Operand constraints. This describes the value type constraints on a single `Value` operand.
874enum OperandConstraint {
875    /// This operand has a concrete value type.
876    Concrete(Type),
877
878    /// This operand can vary freely within the given type set.
879    /// The type set is identified by its index into the TYPE_SETS constant table.
880    Free(u8),
881
882    /// This operand is the same type as the controlling type variable.
883    Same,
884
885    /// This operand is `ctrlType.lane_of()`.
886    LaneOf,
887
888    /// This operand is `ctrlType.as_truthy()`.
889    AsTruthy,
890
891    /// This operand is `ctrlType.half_width()`.
892    HalfWidth,
893
894    /// This operand is `ctrlType.double_width()`.
895    DoubleWidth,
896
897    /// This operand is `ctrlType.split_lanes()`.
898    SplitLanes,
899
900    /// This operand is `ctrlType.merge_lanes()`.
901    MergeLanes,
902
903    /// This operands is `ctrlType.dynamic_to_vector()`.
904    DynamicToVector,
905
906    /// This operand is `ctrlType.narrower()`.
907    Narrower,
908
909    /// This operand is `ctrlType.wider()`.
910    Wider,
911}
912
913impl OperandConstraint {
914    /// Resolve this operand constraint into a concrete value type, given the value of the
915    /// controlling type variable.
916    pub fn resolve(&self, ctrl_type: Type) -> ResolvedConstraint {
917        use self::OperandConstraint::*;
918        use self::ResolvedConstraint::Bound;
919        match *self {
920            Concrete(t) => Bound(t),
921            Free(vts) => ResolvedConstraint::Free(TYPE_SETS[vts as usize]),
922            Same => Bound(ctrl_type),
923            LaneOf => Bound(ctrl_type.lane_of()),
924            AsTruthy => Bound(ctrl_type.as_truthy()),
925            HalfWidth => Bound(ctrl_type.half_width().expect("invalid type for half_width")),
926            DoubleWidth => Bound(
927                ctrl_type
928                    .double_width()
929                    .expect("invalid type for double_width"),
930            ),
931            SplitLanes => {
932                if ctrl_type.is_dynamic_vector() {
933                    Bound(
934                        ctrl_type
935                            .dynamic_to_vector()
936                            .expect("invalid type for dynamic_to_vector")
937                            .split_lanes()
938                            .expect("invalid type for split_lanes")
939                            .vector_to_dynamic()
940                            .expect("invalid dynamic type"),
941                    )
942                } else {
943                    Bound(
944                        ctrl_type
945                            .split_lanes()
946                            .expect("invalid type for split_lanes"),
947                    )
948                }
949            }
950            MergeLanes => {
951                if ctrl_type.is_dynamic_vector() {
952                    Bound(
953                        ctrl_type
954                            .dynamic_to_vector()
955                            .expect("invalid type for dynamic_to_vector")
956                            .merge_lanes()
957                            .expect("invalid type for merge_lanes")
958                            .vector_to_dynamic()
959                            .expect("invalid dynamic type"),
960                    )
961                } else {
962                    Bound(
963                        ctrl_type
964                            .merge_lanes()
965                            .expect("invalid type for merge_lanes"),
966                    )
967                }
968            }
969            DynamicToVector => Bound(
970                ctrl_type
971                    .dynamic_to_vector()
972                    .expect("invalid type for dynamic_to_vector"),
973            ),
974            Narrower => {
975                let ctrl_type_bits = ctrl_type.log2_lane_bits();
976                let mut tys = ValueTypeSet::default();
977
978                // We're testing scalar values, only.
979                tys.lanes = ScalarBitSet::from_range(0, 1);
980
981                if ctrl_type.is_int() {
982                    // The upper bound in from_range is exclusive, and we want to exclude the
983                    // control type to construct the interval of [I8, ctrl_type).
984                    tys.ints = BitSet8::from_range(3, ctrl_type_bits as u8);
985                } else if ctrl_type.is_float() {
986                    // The upper bound in from_range is exclusive, and we want to exclude the
987                    // control type to construct the interval of [F16, ctrl_type).
988                    tys.floats = BitSet8::from_range(4, ctrl_type_bits as u8);
989                } else {
990                    panic!(
991                        "The Narrower constraint only operates on floats or ints, got {ctrl_type:?}"
992                    );
993                }
994                ResolvedConstraint::Free(tys)
995            }
996            Wider => {
997                let ctrl_type_bits = ctrl_type.log2_lane_bits();
998                let mut tys = ValueTypeSet::default();
999
1000                // We're testing scalar values, only.
1001                tys.lanes = ScalarBitSet::from_range(0, 1);
1002
1003                if ctrl_type.is_int() {
1004                    let lower_bound = ctrl_type_bits as u8 + 1;
1005                    // The largest integer type we can represent in `BitSet8` is I128, which is
1006                    // represented by bit 7 in the bit set. Adding one to exclude I128 from the
1007                    // lower bound would overflow as 2^8 doesn't fit in a u8, but this would
1008                    // already describe the empty set so instead we leave `ints` in its default
1009                    // empty state.
1010                    if lower_bound < BitSet8::capacity() {
1011                        // The interval should include all types wider than `ctrl_type`, so we use
1012                        // `2^8` as the upper bound, and add one to the bits of `ctrl_type` to define
1013                        // the interval `(ctrl_type, I128]`.
1014                        tys.ints = BitSet8::from_range(lower_bound, 8);
1015                    }
1016                } else if ctrl_type.is_float() {
1017                    // Same as above but for `tys.floats`, as the largest float type is F128.
1018                    let lower_bound = ctrl_type_bits as u8 + 1;
1019                    if lower_bound < BitSet8::capacity() {
1020                        tys.floats = BitSet8::from_range(lower_bound, 8);
1021                    }
1022                } else {
1023                    panic!(
1024                        "The Wider constraint only operates on floats or ints, got {ctrl_type:?}"
1025                    );
1026                }
1027
1028                ResolvedConstraint::Free(tys)
1029            }
1030        }
1031    }
1032}
1033
1034/// The type constraint on a value argument once the controlling type variable is known.
1035#[derive(Copy, Clone, Debug, PartialEq, Eq)]
1036pub enum ResolvedConstraint {
1037    /// The operand is bound to a known type.
1038    Bound(Type),
1039    /// The operand type can vary freely within the given set.
1040    Free(ValueTypeSet),
1041}
1042
1043/// A trait to map some functions over each of the entities within an
1044/// instruction, when paired with `InstructionData::map`.
1045pub trait InstructionMapper {
1046    /// Map a function over a `Value`.
1047    fn map_value(&mut self, value: Value) -> Value;
1048
1049    /// Map a function over a `ValueList`.
1050    fn map_value_list(&mut self, value_list: ValueList) -> ValueList;
1051
1052    /// Map a function over a `GlobalValue`.
1053    fn map_global_value(&mut self, global_value: ir::GlobalValue) -> ir::GlobalValue;
1054
1055    /// Map a function over a `JumpTable`.
1056    fn map_jump_table(&mut self, jump_table: ir::JumpTable) -> ir::JumpTable;
1057
1058    /// Map a function over an `ExceptionTable`.
1059    fn map_exception_table(&mut self, exception_table: ExceptionTable) -> ExceptionTable;
1060
1061    /// Map a function over a `BlockCall`.
1062    fn map_block_call(&mut self, block_call: BlockCall) -> BlockCall;
1063
1064    /// Map a function over a `FuncRef`.
1065    fn map_func_ref(&mut self, func_ref: FuncRef) -> FuncRef;
1066
1067    /// Map a function over a `SigRef`.
1068    fn map_sig_ref(&mut self, sig_ref: SigRef) -> SigRef;
1069
1070    /// Map a function over a `StackSlot`.
1071    fn map_stack_slot(&mut self, stack_slot: StackSlot) -> StackSlot;
1072
1073    /// Map a function over a `DynamicStackSlot`.
1074    fn map_dynamic_stack_slot(
1075        &mut self,
1076        dynamic_stack_slot: ir::DynamicStackSlot,
1077    ) -> ir::DynamicStackSlot;
1078
1079    /// Map a function over a `Constant`.
1080    fn map_constant(&mut self, constant: ir::Constant) -> ir::Constant;
1081
1082    /// Map a function over an `Immediate`.
1083    fn map_immediate(&mut self, immediate: ir::Immediate) -> ir::Immediate;
1084}
1085
1086impl<'a, T> InstructionMapper for &'a mut T
1087where
1088    T: InstructionMapper,
1089{
1090    fn map_value(&mut self, value: Value) -> Value {
1091        (**self).map_value(value)
1092    }
1093
1094    fn map_value_list(&mut self, value_list: ValueList) -> ValueList {
1095        (**self).map_value_list(value_list)
1096    }
1097
1098    fn map_global_value(&mut self, global_value: ir::GlobalValue) -> ir::GlobalValue {
1099        (**self).map_global_value(global_value)
1100    }
1101
1102    fn map_jump_table(&mut self, jump_table: ir::JumpTable) -> ir::JumpTable {
1103        (**self).map_jump_table(jump_table)
1104    }
1105
1106    fn map_exception_table(&mut self, exception_table: ExceptionTable) -> ExceptionTable {
1107        (**self).map_exception_table(exception_table)
1108    }
1109
1110    fn map_block_call(&mut self, block_call: BlockCall) -> BlockCall {
1111        (**self).map_block_call(block_call)
1112    }
1113
1114    fn map_func_ref(&mut self, func_ref: FuncRef) -> FuncRef {
1115        (**self).map_func_ref(func_ref)
1116    }
1117
1118    fn map_sig_ref(&mut self, sig_ref: SigRef) -> SigRef {
1119        (**self).map_sig_ref(sig_ref)
1120    }
1121
1122    fn map_stack_slot(&mut self, stack_slot: StackSlot) -> StackSlot {
1123        (**self).map_stack_slot(stack_slot)
1124    }
1125
1126    fn map_dynamic_stack_slot(
1127        &mut self,
1128        dynamic_stack_slot: ir::DynamicStackSlot,
1129    ) -> ir::DynamicStackSlot {
1130        (**self).map_dynamic_stack_slot(dynamic_stack_slot)
1131    }
1132
1133    fn map_constant(&mut self, constant: ir::Constant) -> ir::Constant {
1134        (**self).map_constant(constant)
1135    }
1136
1137    fn map_immediate(&mut self, immediate: ir::Immediate) -> ir::Immediate {
1138        (**self).map_immediate(immediate)
1139    }
1140}
1141
1142#[cfg(test)]
1143mod tests {
1144    use super::*;
1145    use alloc::string::ToString;
1146    use ir::{DynamicStackSlot, GlobalValue, JumpTable};
1147
1148    #[test]
1149    fn inst_data_is_copy() {
1150        fn is_copy<T: Copy>() {}
1151        is_copy::<InstructionData>();
1152    }
1153
1154    #[test]
1155    fn inst_data_size() {
1156        // The size of `InstructionData` is performance sensitive, so make sure
1157        // we don't regress it unintentionally.
1158        assert_eq!(std::mem::size_of::<InstructionData>(), 16);
1159    }
1160
1161    #[test]
1162    fn opcodes() {
1163        use core::mem;
1164
1165        let x = Opcode::Iadd;
1166        let mut y = Opcode::Isub;
1167
1168        assert!(x != y);
1169        y = Opcode::Iadd;
1170        assert_eq!(x, y);
1171        assert_eq!(x.format(), InstructionFormat::Binary);
1172
1173        assert_eq!(format!("{:?}", Opcode::IaddImm), "IaddImm");
1174        assert_eq!(Opcode::IaddImm.to_string(), "iadd_imm");
1175
1176        // Check the matcher.
1177        assert_eq!("iadd".parse::<Opcode>(), Ok(Opcode::Iadd));
1178        assert_eq!("iadd_imm".parse::<Opcode>(), Ok(Opcode::IaddImm));
1179        assert_eq!("iadd\0".parse::<Opcode>(), Err("Unknown opcode"));
1180        assert_eq!("".parse::<Opcode>(), Err("Unknown opcode"));
1181        assert_eq!("\0".parse::<Opcode>(), Err("Unknown opcode"));
1182
1183        // Opcode is a single byte, and because Option<Opcode> originally came to 2 bytes, early on
1184        // Opcode included a variant NotAnOpcode to avoid the unnecessary bloat. Since then the Rust
1185        // compiler has brought in NonZero optimization, meaning that an enum not using the 0 value
1186        // can be optional for no size cost. We want to ensure Option<Opcode> remains small.
1187        assert_eq!(mem::size_of::<Opcode>(), mem::size_of::<Option<Opcode>>());
1188    }
1189
1190    #[test]
1191    fn instruction_data() {
1192        use core::mem;
1193        // The size of the `InstructionData` enum is important for performance. It should not
1194        // exceed 16 bytes. Use `Box<FooData>` out-of-line payloads for instruction formats that
1195        // require more space than that. It would be fine with a data structure smaller than 16
1196        // bytes, but what are the odds of that?
1197        assert_eq!(mem::size_of::<InstructionData>(), 16);
1198    }
1199
1200    #[test]
1201    fn constraints() {
1202        let a = Opcode::Iadd.constraints();
1203        assert!(a.use_typevar_operand());
1204        assert!(!a.requires_typevar_operand());
1205        assert_eq!(a.num_fixed_results(), 1);
1206        assert_eq!(a.num_fixed_value_arguments(), 2);
1207        assert_eq!(a.result_type(0, types::I32), types::I32);
1208        assert_eq!(a.result_type(0, types::I8), types::I8);
1209        assert_eq!(
1210            a.value_argument_constraint(0, types::I32),
1211            ResolvedConstraint::Bound(types::I32)
1212        );
1213        assert_eq!(
1214            a.value_argument_constraint(1, types::I32),
1215            ResolvedConstraint::Bound(types::I32)
1216        );
1217
1218        let b = Opcode::Bitcast.constraints();
1219        assert!(!b.use_typevar_operand());
1220        assert!(!b.requires_typevar_operand());
1221        assert_eq!(b.num_fixed_results(), 1);
1222        assert_eq!(b.num_fixed_value_arguments(), 1);
1223        assert_eq!(b.result_type(0, types::I32), types::I32);
1224        assert_eq!(b.result_type(0, types::I8), types::I8);
1225        match b.value_argument_constraint(0, types::I32) {
1226            ResolvedConstraint::Free(vts) => assert!(vts.contains(types::F32)),
1227            _ => panic!("Unexpected constraint from value_argument_constraint"),
1228        }
1229
1230        let c = Opcode::Call.constraints();
1231        assert_eq!(c.num_fixed_results(), 0);
1232        assert_eq!(c.num_fixed_value_arguments(), 0);
1233
1234        let i = Opcode::CallIndirect.constraints();
1235        assert_eq!(i.num_fixed_results(), 0);
1236        assert_eq!(i.num_fixed_value_arguments(), 1);
1237
1238        let cmp = Opcode::Icmp.constraints();
1239        assert!(cmp.use_typevar_operand());
1240        assert!(cmp.requires_typevar_operand());
1241        assert_eq!(cmp.num_fixed_results(), 1);
1242        assert_eq!(cmp.num_fixed_value_arguments(), 2);
1243        assert_eq!(cmp.result_type(0, types::I64), types::I8);
1244    }
1245
1246    #[test]
1247    fn value_set() {
1248        use crate::ir::types::*;
1249
1250        let vts = ValueTypeSet {
1251            lanes: BitSet16::from_range(0, 8),
1252            ints: BitSet8::from_range(4, 7),
1253            floats: BitSet8::from_range(0, 0),
1254            dynamic_lanes: BitSet16::from_range(0, 4),
1255        };
1256        assert!(!vts.contains(I8));
1257        assert!(vts.contains(I32));
1258        assert!(vts.contains(I64));
1259        assert!(vts.contains(I32X4));
1260        assert!(vts.contains(I32X4XN));
1261        assert!(!vts.contains(F16));
1262        assert!(!vts.contains(F32));
1263        assert!(!vts.contains(F128));
1264        assert_eq!(vts.example().to_string(), "i32");
1265
1266        let vts = ValueTypeSet {
1267            lanes: BitSet16::from_range(0, 8),
1268            ints: BitSet8::from_range(0, 0),
1269            floats: BitSet8::from_range(5, 7),
1270            dynamic_lanes: BitSet16::from_range(0, 8),
1271        };
1272        assert_eq!(vts.example().to_string(), "f32");
1273
1274        let vts = ValueTypeSet {
1275            lanes: BitSet16::from_range(1, 8),
1276            ints: BitSet8::from_range(0, 0),
1277            floats: BitSet8::from_range(5, 7),
1278            dynamic_lanes: BitSet16::from_range(0, 8),
1279        };
1280        assert_eq!(vts.example().to_string(), "f32x2");
1281
1282        let vts = ValueTypeSet {
1283            lanes: BitSet16::from_range(2, 8),
1284            ints: BitSet8::from_range(3, 7),
1285            floats: BitSet8::from_range(0, 0),
1286            dynamic_lanes: BitSet16::from_range(0, 8),
1287        };
1288        assert_eq!(vts.example().to_string(), "i32x4");
1289
1290        let vts = ValueTypeSet {
1291            // TypeSet(lanes=(1, 256), ints=(8, 64))
1292            lanes: BitSet16::from_range(0, 9),
1293            ints: BitSet8::from_range(3, 7),
1294            floats: BitSet8::from_range(0, 0),
1295            dynamic_lanes: BitSet16::from_range(0, 8),
1296        };
1297        assert!(vts.contains(I32));
1298        assert!(vts.contains(I32X4));
1299    }
1300
1301    #[test]
1302    fn instruction_data_map() {
1303        struct TestMapper;
1304
1305        impl InstructionMapper for TestMapper {
1306            fn map_value(&mut self, value: Value) -> Value {
1307                Value::from_u32(value.as_u32() + 1)
1308            }
1309
1310            fn map_value_list(&mut self, _value_list: ValueList) -> ValueList {
1311                ValueList::new()
1312            }
1313
1314            fn map_global_value(&mut self, global_value: ir::GlobalValue) -> ir::GlobalValue {
1315                GlobalValue::from_u32(global_value.as_u32() + 1)
1316            }
1317
1318            fn map_jump_table(&mut self, jump_table: ir::JumpTable) -> ir::JumpTable {
1319                JumpTable::from_u32(jump_table.as_u32() + 1)
1320            }
1321
1322            fn map_exception_table(&mut self, exception_table: ExceptionTable) -> ExceptionTable {
1323                ExceptionTable::from_u32(exception_table.as_u32() + 1)
1324            }
1325
1326            fn map_block_call(&mut self, _block_call: BlockCall) -> BlockCall {
1327                let block = Block::from_u32(42);
1328                let mut pool = ValueListPool::new();
1329                BlockCall::new(block, [], &mut pool)
1330            }
1331
1332            fn map_func_ref(&mut self, func_ref: FuncRef) -> FuncRef {
1333                FuncRef::from_u32(func_ref.as_u32() + 1)
1334            }
1335
1336            fn map_sig_ref(&mut self, sig_ref: SigRef) -> SigRef {
1337                SigRef::from_u32(sig_ref.as_u32() + 1)
1338            }
1339
1340            fn map_stack_slot(&mut self, stack_slot: StackSlot) -> StackSlot {
1341                StackSlot::from_u32(stack_slot.as_u32() + 1)
1342            }
1343
1344            fn map_dynamic_stack_slot(
1345                &mut self,
1346                dynamic_stack_slot: ir::DynamicStackSlot,
1347            ) -> ir::DynamicStackSlot {
1348                DynamicStackSlot::from_u32(dynamic_stack_slot.as_u32() + 1)
1349            }
1350
1351            fn map_constant(&mut self, constant: ir::Constant) -> ir::Constant {
1352                ir::Constant::from_u32(constant.as_u32() + 1)
1353            }
1354
1355            fn map_immediate(&mut self, immediate: ir::Immediate) -> ir::Immediate {
1356                ir::Immediate::from_u32(immediate.as_u32() + 1)
1357            }
1358        }
1359
1360        let mut pool = ValueListPool::new();
1361        let map = |inst: InstructionData| inst.map(TestMapper);
1362
1363        // Mapping `Value`s.
1364        assert_eq!(
1365            map(InstructionData::Binary {
1366                opcode: Opcode::Iadd,
1367                args: [Value::from_u32(10), Value::from_u32(20)]
1368            }),
1369            InstructionData::Binary {
1370                opcode: Opcode::Iadd,
1371                args: [Value::from_u32(11), Value::from_u32(21)]
1372            }
1373        );
1374
1375        // Mapping `ValueList`s and `FuncRef`s.
1376        let mut args = ValueList::new();
1377        args.push(Value::from_u32(42), &mut pool);
1378        let func_ref = FuncRef::from_u32(99);
1379        let inst = map(InstructionData::Call {
1380            opcode: Opcode::Call,
1381            args,
1382            func_ref,
1383        });
1384        let InstructionData::Call {
1385            opcode: Opcode::Call,
1386            args,
1387            func_ref,
1388        } = inst
1389        else {
1390            panic!()
1391        };
1392        assert!(args.is_empty());
1393        assert_eq!(func_ref, FuncRef::from_u32(100));
1394
1395        // Mapping `GlobalValue`s.
1396        assert_eq!(
1397            map(InstructionData::UnaryGlobalValue {
1398                opcode: Opcode::GlobalValue,
1399                global_value: GlobalValue::from_u32(4),
1400            }),
1401            InstructionData::UnaryGlobalValue {
1402                opcode: Opcode::GlobalValue,
1403                global_value: GlobalValue::from_u32(5),
1404            }
1405        );
1406
1407        // Mapping `JumpTable`s.
1408        assert_eq!(
1409            map(InstructionData::BranchTable {
1410                opcode: Opcode::BrTable,
1411                arg: Value::from_u32(0),
1412                table: JumpTable::from_u32(1),
1413            }),
1414            InstructionData::BranchTable {
1415                opcode: Opcode::BrTable,
1416                arg: Value::from_u32(1),
1417                table: JumpTable::from_u32(2),
1418            }
1419        );
1420
1421        // Mapping `ExceptionTable`s.
1422        assert_eq!(
1423            map(InstructionData::TryCall {
1424                opcode: Opcode::TryCall,
1425                args,
1426                func_ref: FuncRef::from_u32(0),
1427                exception: ExceptionTable::from_u32(1),
1428            }),
1429            InstructionData::TryCall {
1430                opcode: Opcode::TryCall,
1431                args,
1432                func_ref: FuncRef::from_u32(1),
1433                exception: ExceptionTable::from_u32(2),
1434            }
1435        );
1436
1437        // Mapping `BlockCall`s.
1438        assert_eq!(
1439            map(InstructionData::Jump {
1440                opcode: Opcode::Jump,
1441                destination: BlockCall::new(Block::from_u32(99), [], &mut pool),
1442            }),
1443            map(InstructionData::Jump {
1444                opcode: Opcode::Jump,
1445                destination: BlockCall::new(Block::from_u32(42), [], &mut pool),
1446            })
1447        );
1448
1449        // Mapping `SigRef`s.
1450        assert_eq!(
1451            map(InstructionData::CallIndirect {
1452                opcode: Opcode::CallIndirect,
1453                args,
1454                sig_ref: SigRef::from_u32(11)
1455            }),
1456            InstructionData::CallIndirect {
1457                opcode: Opcode::CallIndirect,
1458                args: ValueList::new(),
1459                sig_ref: SigRef::from_u32(12)
1460            }
1461        );
1462
1463        // Mapping `StackSlot`s.
1464        assert_eq!(
1465            map(InstructionData::StackLoad {
1466                opcode: Opcode::StackLoad,
1467                stack_slot: StackSlot::from_u32(0),
1468                offset: 0.into()
1469            }),
1470            InstructionData::StackLoad {
1471                opcode: Opcode::StackLoad,
1472                stack_slot: StackSlot::from_u32(1),
1473                offset: 0.into()
1474            },
1475        );
1476
1477        // Mapping `DynamicStackSlot`s.
1478        assert_eq!(
1479            map(InstructionData::DynamicStackLoad {
1480                opcode: Opcode::DynamicStackLoad,
1481                dynamic_stack_slot: DynamicStackSlot::from_u32(0),
1482            }),
1483            InstructionData::DynamicStackLoad {
1484                opcode: Opcode::DynamicStackLoad,
1485                dynamic_stack_slot: DynamicStackSlot::from_u32(1),
1486            },
1487        );
1488
1489        // Mapping `Constant`s
1490        assert_eq!(
1491            map(InstructionData::UnaryConst {
1492                opcode: ir::Opcode::Vconst,
1493                constant_handle: ir::Constant::from_u32(2)
1494            }),
1495            InstructionData::UnaryConst {
1496                opcode: ir::Opcode::Vconst,
1497                constant_handle: ir::Constant::from_u32(3)
1498            },
1499        );
1500
1501        // Mapping `Immediate`s
1502        assert_eq!(
1503            map(InstructionData::Shuffle {
1504                opcode: ir::Opcode::Shuffle,
1505                args: [Value::from_u32(0), Value::from_u32(1)],
1506                imm: ir::Immediate::from_u32(41),
1507            }),
1508            InstructionData::Shuffle {
1509                opcode: ir::Opcode::Shuffle,
1510                args: [Value::from_u32(1), Value::from_u32(2)],
1511                imm: ir::Immediate::from_u32(42),
1512            },
1513        );
1514    }
1515}