Skip to main content

cranelift_codegen/isa/aarch64/inst/
mod.rs

1//! This module defines aarch64-specific machine instruction types.
2
3use crate::binemit::{Addend, CodeOffset, Reloc};
4use crate::ir::types::{F16, F32, F64, F128, I8, I8X16, I16, I32, I64, I128};
5use crate::ir::{MemFlagsData, Type, types};
6use crate::isa::{CallConv, FunctionAlignment};
7use crate::machinst::*;
8use crate::{CodegenError, CodegenResult, settings};
9
10use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
11
12use alloc::string::{String, ToString};
13use alloc::vec::Vec;
14use core::fmt::Write;
15use core::slice;
16use smallvec::{SmallVec, smallvec};
17
18pub(crate) mod regs;
19pub use self::regs::*;
20pub mod imms;
21pub use self::imms::*;
22pub mod args;
23pub use self::args::*;
24pub mod emit;
25pub(crate) use self::emit::*;
26use crate::isa::aarch64::abi::AArch64MachineDeps;
27
28pub(crate) mod unwind;
29
30#[cfg(test)]
31mod emit_tests;
32
33//=============================================================================
34// Instructions (top level): definition
35
36pub use crate::isa::aarch64::lower::isle::generated_code::{
37    ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BfmOp, BitOp, BranchTargetType,
38    FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp,
39    VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp,
40    VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,
41};
42
43/// A floating-point unit (FPU) operation with two args, a register and an immediate.
44#[derive(Copy, Clone, Debug)]
45pub enum FPUOpRI {
46    /// Unsigned right shift. Rd = Rn << #imm
47    UShr32(FPURightShiftImm),
48    /// Unsigned right shift. Rd = Rn << #imm
49    UShr64(FPURightShiftImm),
50}
51
52/// A floating-point unit (FPU) operation with two args, a register and
53/// an immediate that modifies its dest (so takes that input value as a
54/// separate virtual register).
55#[derive(Copy, Clone, Debug)]
56pub enum FPUOpRIMod {
57    /// Shift left and insert. Rd |= Rn << #imm
58    Sli32(FPULeftShiftImm),
59    /// Shift left and insert. Rd |= Rn << #imm
60    Sli64(FPULeftShiftImm),
61}
62
63impl BfmOp {
64    /// Get the assembly mnemonic for this opcode.
65    pub fn op_str(&self) -> &'static str {
66        match self {
67            BfmOp::UBfm => "ubfm",
68            BfmOp::SBfm => "sbfm",
69        }
70    }
71}
72
73impl BitOp {
74    /// Get the assembly mnemonic for this opcode.
75    pub fn op_str(&self) -> &'static str {
76        match self {
77            BitOp::RBit => "rbit",
78            BitOp::Clz => "clz",
79            BitOp::Cls => "cls",
80            BitOp::Rev16 => "rev16",
81            BitOp::Rev32 => "rev32",
82            BitOp::Rev64 => "rev64",
83        }
84    }
85}
86
87/// Additional information for `return_call[_ind]` instructions, left out of
88/// line to lower the size of the `Inst` enum.
89#[derive(Clone, Debug)]
90pub struct ReturnCallInfo<T> {
91    /// Where this call is going to
92    pub dest: T,
93    /// Arguments to the call instruction.
94    pub uses: CallArgList,
95    /// The size of the new stack frame's stack arguments. This is necessary
96    /// for copying the frame over our current frame. It must already be
97    /// allocated on the stack.
98    pub new_stack_arg_size: u32,
99    /// API key to use to restore the return address, if any.
100    pub key: Option<APIKey>,
101    /// Whether pointer-auth return addresses are signed even without frame setup.
102    pub sign_return_address_all: bool,
103}
104
105fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
106    let mut count = 0;
107    for _ in 0..num_half_words {
108        if value & 0xffff == 0 {
109            count += 1;
110        }
111        value >>= 16;
112    }
113
114    count
115}
116
117impl Inst {
118    /// Create an instruction that loads a constant, using one of several options (MOVZ, MOVN,
119    /// logical immediate, or constant pool).
120    pub fn load_constant(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]> {
121        // NB: this is duplicated in `lower/isle.rs` and `inst.isle` right now,
122        // if modifications are made here before this is deleted after moving to
123        // ISLE then those locations should be updated as well.
124
125        if let Some(imm) = MoveWideConst::maybe_from_u64(value) {
126            // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ
127            smallvec![Inst::MovWide {
128                op: MoveWideOp::MovZ,
129                rd,
130                imm,
131                size: OperandSize::Size64
132            }]
133        } else if let Some(imm) = MoveWideConst::maybe_from_u64(!value) {
134            // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVN
135            smallvec![Inst::MovWide {
136                op: MoveWideOp::MovN,
137                rd,
138                imm,
139                size: OperandSize::Size64
140            }]
141        } else if let Some(imml) = ImmLogic::maybe_from_u64(value, I64) {
142            // Weird logical-instruction immediate in ORI using zero register
143            smallvec![Inst::AluRRImmLogic {
144                alu_op: ALUOp::Orr,
145                size: OperandSize::Size64,
146                rd,
147                rn: zero_reg(),
148                imml,
149            }]
150        } else {
151            let mut insts = smallvec![];
152
153            // If the top 32 bits are zero, use 32-bit `mov` operations.
154            let (num_half_words, size, negated) = if value >> 32 == 0 {
155                (2, OperandSize::Size32, (!value << 32) >> 32)
156            } else {
157                (4, OperandSize::Size64, !value)
158            };
159
160            // If the number of 0xffff half words is greater than the number of 0x0000 half words
161            // it is more efficient to use `movn` for the first instruction.
162            let first_is_inverted = count_zero_half_words(negated, num_half_words)
163                > count_zero_half_words(value, num_half_words);
164
165            // Either 0xffff or 0x0000 half words can be skipped, depending on the first
166            // instruction used.
167            let ignored_halfword = if first_is_inverted { 0xffff } else { 0 };
168
169            let halfwords: SmallVec<[_; 4]> = (0..num_half_words)
170                .filter_map(|i| {
171                    let imm16 = (value >> (16 * i)) & 0xffff;
172                    if imm16 == ignored_halfword {
173                        None
174                    } else {
175                        Some((i, imm16))
176                    }
177                })
178                .collect();
179
180            let mut prev_result = None;
181            for (i, imm16) in halfwords {
182                let shift = i * 16;
183
184                if let Some(rn) = prev_result {
185                    let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
186                    insts.push(Inst::MovK { rd, rn, imm, size });
187                } else {
188                    if first_is_inverted {
189                        let imm =
190                            MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, shift)
191                                .unwrap();
192                        insts.push(Inst::MovWide {
193                            op: MoveWideOp::MovN,
194                            rd,
195                            imm,
196                            size,
197                        });
198                    } else {
199                        let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
200                        insts.push(Inst::MovWide {
201                            op: MoveWideOp::MovZ,
202                            rd,
203                            imm,
204                            size,
205                        });
206                    }
207                }
208
209                prev_result = Some(rd.to_reg());
210            }
211
212            assert!(prev_result.is_some());
213
214            insts
215        }
216    }
217
218    /// Generic constructor for a load (zero-extending where appropriate).
219    pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlagsData) -> Inst {
220        match ty {
221            I8 => Inst::ULoad8 {
222                rd: into_reg,
223                mem,
224                flags,
225            },
226            I16 => Inst::ULoad16 {
227                rd: into_reg,
228                mem,
229                flags,
230            },
231            I32 => Inst::ULoad32 {
232                rd: into_reg,
233                mem,
234                flags,
235            },
236            I64 => Inst::ULoad64 {
237                rd: into_reg,
238                mem,
239                flags,
240            },
241            _ => {
242                if ty.is_vector() || ty.is_float() {
243                    let bits = ty_bits(ty);
244                    let rd = into_reg;
245
246                    match bits {
247                        128 => Inst::FpuLoad128 { rd, mem, flags },
248                        64 => Inst::FpuLoad64 { rd, mem, flags },
249                        32 => Inst::FpuLoad32 { rd, mem, flags },
250                        16 => Inst::FpuLoad16 { rd, mem, flags },
251                        _ => unimplemented!("gen_load({})", ty),
252                    }
253                } else {
254                    unimplemented!("gen_load({})", ty);
255                }
256            }
257        }
258    }
259
260    /// Generic constructor for a store.
261    pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlagsData) -> Inst {
262        match ty {
263            I8 => Inst::Store8 {
264                rd: from_reg,
265                mem,
266                flags,
267            },
268            I16 => Inst::Store16 {
269                rd: from_reg,
270                mem,
271                flags,
272            },
273            I32 => Inst::Store32 {
274                rd: from_reg,
275                mem,
276                flags,
277            },
278            I64 => Inst::Store64 {
279                rd: from_reg,
280                mem,
281                flags,
282            },
283            _ => {
284                if ty.is_vector() || ty.is_float() {
285                    let bits = ty_bits(ty);
286                    let rd = from_reg;
287
288                    match bits {
289                        128 => Inst::FpuStore128 { rd, mem, flags },
290                        64 => Inst::FpuStore64 { rd, mem, flags },
291                        32 => Inst::FpuStore32 { rd, mem, flags },
292                        16 => Inst::FpuStore16 { rd, mem, flags },
293                        _ => unimplemented!("gen_store({})", ty),
294                    }
295                } else {
296                    unimplemented!("gen_store({})", ty);
297                }
298            }
299        }
300    }
301
302    /// Zero- or sign-extend `from_reg` as described by `op` into a `to`-sized result.
303    pub fn gen_extend(to_reg: Writable<Reg>, from_reg: Reg, op: ExtendOp, to: OperandSize) -> Inst {
304        assert!(op.from_bits() < to.bits());
305
306        let bfm = |size, bfm_op| Inst::BitfieldMove {
307            size,
308            bfm_op,
309            rd: to_reg,
310            rn: from_reg,
311            immr: UImm6::maybe_from_u8(0).unwrap(),
312            imms: UImm6::maybe_from_u8(op.from_bits() - 1).unwrap(),
313        };
314
315        match op {
316            // A 32-bit move zeroes the upper half of the destination.
317            ExtendOp::UXTW => Inst::Mov {
318                size: OperandSize::Size32,
319                rd: to_reg,
320                rm: from_reg,
321            },
322            // 32-bit UBFM already zeroes the upper half, so it's equivalent to the
323            // 64-bit form; using it preserves the encoding emitted previously.
324            ExtendOp::UXTB | ExtendOp::UXTH => bfm(OperandSize::Size32, BfmOp::UBfm),
325            // SBFM must run at the destination width to sign-fill it.
326            ExtendOp::SXTB | ExtendOp::SXTH | ExtendOp::SXTW => bfm(to, BfmOp::SBfm),
327            // These cases are excluded by the assertion above.
328            ExtendOp::UXTX | ExtendOp::SXTX => unreachable!(),
329        }
330    }
331
332    /// What type does this load or store instruction access in memory? When
333    /// uimm12 encoding is used, the size of this type is the amount that
334    /// immediate offsets are scaled by.
335    pub fn mem_type(&self) -> Option<Type> {
336        match self {
337            Inst::ULoad8 { .. } => Some(I8),
338            Inst::SLoad8 { .. } => Some(I8),
339            Inst::ULoad16 { .. } => Some(I16),
340            Inst::SLoad16 { .. } => Some(I16),
341            Inst::ULoad32 { .. } => Some(I32),
342            Inst::SLoad32 { .. } => Some(I32),
343            Inst::ULoad64 { .. } => Some(I64),
344            Inst::FpuLoad16 { .. } => Some(F16),
345            Inst::FpuLoad32 { .. } => Some(F32),
346            Inst::FpuLoad64 { .. } => Some(F64),
347            Inst::FpuLoad128 { .. } => Some(I8X16),
348            Inst::Store8 { .. } => Some(I8),
349            Inst::Store16 { .. } => Some(I16),
350            Inst::Store32 { .. } => Some(I32),
351            Inst::Store64 { .. } => Some(I64),
352            Inst::FpuStore16 { .. } => Some(F16),
353            Inst::FpuStore32 { .. } => Some(F32),
354            Inst::FpuStore64 { .. } => Some(F64),
355            Inst::FpuStore128 { .. } => Some(I8X16),
356            _ => None,
357        }
358    }
359}
360
361//=============================================================================
362// Instructions: get_regs
363
364fn memarg_operands(memarg: &mut AMode, collector: &mut impl OperandVisitor) {
365    match memarg {
366        AMode::Unscaled { rn, .. } | AMode::UnsignedOffset { rn, .. } => {
367            collector.reg_use(rn);
368        }
369        AMode::RegReg { rn, rm, .. }
370        | AMode::RegScaled { rn, rm, .. }
371        | AMode::RegScaledExtended { rn, rm, .. }
372        | AMode::RegExtended { rn, rm, .. } => {
373            collector.reg_use(rn);
374            collector.reg_use(rm);
375        }
376        AMode::Label { .. } => {}
377        AMode::SPPreIndexed { .. } | AMode::SPPostIndexed { .. } => {}
378        AMode::FPOffset { .. } | AMode::IncomingArg { .. } => {}
379        AMode::SPOffset { .. } | AMode::SlotOffset { .. } => {}
380        AMode::RegOffset { rn, .. } => {
381            collector.reg_use(rn);
382        }
383        AMode::Const { .. } => {}
384    }
385}
386
387fn pairmemarg_operands(pairmemarg: &mut PairAMode, collector: &mut impl OperandVisitor) {
388    match pairmemarg {
389        PairAMode::SignedOffset { reg, .. } => {
390            collector.reg_use(reg);
391        }
392        PairAMode::SPPreIndexed { .. } | PairAMode::SPPostIndexed { .. } => {}
393    }
394}
395
396fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
397    match inst {
398        Inst::AluRRR { rd, rn, rm, .. } => {
399            collector.reg_def(rd);
400            collector.reg_use(rn);
401            collector.reg_use(rm);
402        }
403        Inst::AluRRRR { rd, rn, rm, ra, .. } => {
404            collector.reg_def(rd);
405            collector.reg_use(rn);
406            collector.reg_use(rm);
407            collector.reg_use(ra);
408        }
409        Inst::AluRRImm12 { rd, rn, .. } => {
410            collector.reg_def(rd);
411            collector.reg_use(rn);
412        }
413        Inst::AluRRImmLogic { rd, rn, .. } => {
414            collector.reg_def(rd);
415            collector.reg_use(rn);
416        }
417        Inst::AluRRImmShift { rd, rn, .. } => {
418            collector.reg_def(rd);
419            collector.reg_use(rn);
420        }
421        Inst::AluRRRShift { rd, rn, rm, .. } => {
422            collector.reg_def(rd);
423            collector.reg_use(rn);
424            collector.reg_use(rm);
425        }
426        Inst::AluRRRExtend { rd, rn, rm, .. } => {
427            collector.reg_def(rd);
428            collector.reg_use(rn);
429            collector.reg_use(rm);
430        }
431        Inst::BitRR { rd, rn, .. } => {
432            collector.reg_def(rd);
433            collector.reg_use(rn);
434        }
435        Inst::ULoad8 { rd, mem, .. }
436        | Inst::SLoad8 { rd, mem, .. }
437        | Inst::ULoad16 { rd, mem, .. }
438        | Inst::SLoad16 { rd, mem, .. }
439        | Inst::ULoad32 { rd, mem, .. }
440        | Inst::SLoad32 { rd, mem, .. }
441        | Inst::ULoad64 { rd, mem, .. } => {
442            collector.reg_def(rd);
443            memarg_operands(mem, collector);
444        }
445        Inst::Store8 { rd, mem, .. }
446        | Inst::Store16 { rd, mem, .. }
447        | Inst::Store32 { rd, mem, .. }
448        | Inst::Store64 { rd, mem, .. } => {
449            collector.reg_use(rd);
450            memarg_operands(mem, collector);
451        }
452        Inst::StoreP64 { rt, rt2, mem, .. } => {
453            collector.reg_use(rt);
454            collector.reg_use(rt2);
455            pairmemarg_operands(mem, collector);
456        }
457        Inst::LoadP64 { rt, rt2, mem, .. } => {
458            collector.reg_def(rt);
459            collector.reg_def(rt2);
460            pairmemarg_operands(mem, collector);
461        }
462        Inst::Mov { rd, rm, .. } => {
463            collector.reg_def(rd);
464            collector.reg_use(rm);
465        }
466        Inst::MovFromPReg { rd, rm } => {
467            debug_assert!(rd.to_reg().is_virtual());
468            collector.reg_def(rd);
469            collector.reg_fixed_nonallocatable(*rm);
470        }
471        Inst::MovToPReg { rd, rm } => {
472            debug_assert!(rm.is_virtual());
473            collector.reg_fixed_nonallocatable(*rd);
474            collector.reg_use(rm);
475        }
476        Inst::MovK { rd, rn, .. } => {
477            collector.reg_use(rn);
478            collector.reg_reuse_def(rd, 0); // `rn` == `rd`.
479        }
480        Inst::MovWide { rd, .. } => {
481            collector.reg_def(rd);
482        }
483        Inst::CSel { rd, rn, rm, .. } => {
484            collector.reg_def(rd);
485            collector.reg_use(rn);
486            collector.reg_use(rm);
487        }
488        Inst::CSNeg { rd, rn, rm, .. } => {
489            collector.reg_def(rd);
490            collector.reg_use(rn);
491            collector.reg_use(rm);
492        }
493        Inst::CSet { rd, .. } | Inst::CSetm { rd, .. } => {
494            collector.reg_def(rd);
495        }
496        Inst::CCmp { rn, rm, .. } => {
497            collector.reg_use(rn);
498            collector.reg_use(rm);
499        }
500        Inst::CCmpImm { rn, .. } => {
501            collector.reg_use(rn);
502        }
503        Inst::AtomicRMWLoop {
504            op,
505            addr,
506            operand,
507            oldval,
508            scratch1,
509            scratch2,
510            ..
511        } => {
512            collector.reg_fixed_use(addr, xreg(25));
513            collector.reg_fixed_use(operand, xreg(26));
514            collector.reg_fixed_def(oldval, xreg(27));
515            collector.reg_fixed_def(scratch1, xreg(24));
516            if *op != AtomicRMWLoopOp::Xchg {
517                collector.reg_fixed_def(scratch2, xreg(28));
518            }
519        }
520        Inst::AtomicRMW128Loop {
521            op,
522            addr,
523            operand_lo,
524            operand_hi,
525            oldval_lo,
526            oldval_hi,
527            scratch1,
528            scratch2,
529            scratch3,
530            ..
531        } => {
532            collector.reg_fixed_use(addr, xreg(25));
533            collector.reg_fixed_use(operand_lo, xreg(26));
534            collector.reg_fixed_use(operand_hi, xreg(22));
535            collector.reg_fixed_def(oldval_lo, xreg(27));
536            collector.reg_fixed_def(oldval_hi, xreg(23));
537            collector.reg_fixed_def(scratch1, xreg(24));
538            if *op != AtomicRMWLoopOp::Xchg {
539                collector.reg_fixed_def(scratch2, xreg(28));
540                collector.reg_fixed_def(scratch3, xreg(21));
541            }
542        }
543        Inst::AtomicRMW { rs, rt, rn, .. } => {
544            collector.reg_use(rs);
545            collector.reg_def(rt);
546            collector.reg_use(rn);
547        }
548        Inst::AtomicCAS { rd, rs, rt, rn, .. } => {
549            collector.reg_reuse_def(rd, 1); // reuse `rs`.
550            collector.reg_use(rs);
551            collector.reg_use(rt);
552            collector.reg_use(rn);
553        }
554        Inst::AtomicCAS128 { args } => {
555            let AtomicCAS128Args {
556                rd_lo,
557                rd_hi,
558                rs_lo,
559                rs_hi,
560                rt_lo,
561                rt_hi,
562                rn,
563                flags: _,
564            } = &mut **args;
565            // `casp` requires two consecutive even-aligned register pairs,
566            // which regalloc2 cannot express, so pin everything down.
567            collector.reg_fixed_use(rs_lo, xreg(24));
568            collector.reg_fixed_use(rs_hi, xreg(25));
569            collector.reg_fixed_def(rd_lo, xreg(24));
570            collector.reg_fixed_def(rd_hi, xreg(25));
571            collector.reg_fixed_use(rt_lo, xreg(26));
572            collector.reg_fixed_use(rt_hi, xreg(27));
573            collector.reg_fixed_use(rn, xreg(28));
574        }
575        Inst::AtomicCASLoop {
576            addr,
577            expected,
578            replacement,
579            oldval,
580            scratch,
581            ..
582        } => {
583            collector.reg_fixed_use(addr, xreg(25));
584            collector.reg_fixed_use(expected, xreg(26));
585            collector.reg_fixed_use(replacement, xreg(28));
586            collector.reg_fixed_def(oldval, xreg(27));
587            collector.reg_fixed_def(scratch, xreg(24));
588        }
589        Inst::AtomicCAS128Loop {
590            addr,
591            expected_lo,
592            expected_hi,
593            replacement_lo,
594            replacement_hi,
595            oldval_lo,
596            oldval_hi,
597            scratch,
598            ..
599        } => {
600            collector.reg_fixed_use(addr, xreg(25));
601            collector.reg_fixed_use(expected_lo, xreg(26));
602            collector.reg_fixed_use(expected_hi, xreg(23));
603            collector.reg_fixed_use(replacement_lo, xreg(28));
604            collector.reg_fixed_use(replacement_hi, xreg(22));
605            collector.reg_fixed_def(oldval_lo, xreg(27));
606            collector.reg_fixed_def(oldval_hi, xreg(21));
607            collector.reg_fixed_def(scratch, xreg(24));
608        }
609        Inst::LoadAcquire { rt, rn, .. } => {
610            collector.reg_use(rn);
611            collector.reg_def(rt);
612        }
613        Inst::LoadAcquire128 {
614            rt1,
615            rt2,
616            rn,
617            scratch,
618            ..
619        } => {
620            collector.reg_use(rn);
621            // These registers are written to in a retry loop so their register
622            // assignment can't reuse `rn`, thus they are early-defs.
623            collector.reg_early_def(rt1);
624            collector.reg_early_def(rt2);
625            collector.reg_early_def(scratch);
626        }
627        Inst::StoreRelease { rt, rn, .. } => {
628            collector.reg_use(rn);
629            collector.reg_use(rt);
630        }
631        Inst::StoreRelease128 {
632            rt1,
633            rt2,
634            rn,
635            scratch,
636            ..
637        } => {
638            collector.reg_use(rn);
639            collector.reg_use(rt1);
640            collector.reg_use(rt2);
641            collector.reg_early_def(scratch);
642        }
643        Inst::Fence {} | Inst::Csdb {} => {}
644        Inst::FpuMove32 { rd, rn } => {
645            collector.reg_def(rd);
646            collector.reg_use(rn);
647        }
648        Inst::FpuMove64 { rd, rn } => {
649            collector.reg_def(rd);
650            collector.reg_use(rn);
651        }
652        Inst::FpuMove128 { rd, rn } => {
653            collector.reg_def(rd);
654            collector.reg_use(rn);
655        }
656        Inst::FpuMoveFromVec { rd, rn, .. } => {
657            collector.reg_def(rd);
658            collector.reg_use(rn);
659        }
660        Inst::FpuExtend { rd, rn, .. } => {
661            collector.reg_def(rd);
662            collector.reg_use(rn);
663        }
664        Inst::FpuRR { rd, rn, .. } => {
665            collector.reg_def(rd);
666            collector.reg_use(rn);
667        }
668        Inst::FpuRRR { rd, rn, rm, .. } => {
669            collector.reg_def(rd);
670            collector.reg_use(rn);
671            collector.reg_use(rm);
672        }
673        Inst::FpuRRI { rd, rn, .. } => {
674            collector.reg_def(rd);
675            collector.reg_use(rn);
676        }
677        Inst::FpuRRIMod { rd, ri, rn, .. } => {
678            collector.reg_reuse_def(rd, 1); // reuse `ri`.
679            collector.reg_use(ri);
680            collector.reg_use(rn);
681        }
682        Inst::FpuRRRR { rd, rn, rm, ra, .. } => {
683            collector.reg_def(rd);
684            collector.reg_use(rn);
685            collector.reg_use(rm);
686            collector.reg_use(ra);
687        }
688        Inst::VecMisc { rd, rn, .. } => {
689            collector.reg_def(rd);
690            collector.reg_use(rn);
691        }
692
693        Inst::VecLanes { rd, rn, .. } => {
694            collector.reg_def(rd);
695            collector.reg_use(rn);
696        }
697        Inst::VecShiftImm { rd, rn, .. } => {
698            collector.reg_def(rd);
699            collector.reg_use(rn);
700        }
701        Inst::VecShiftImmMod { rd, ri, rn, .. } => {
702            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
703            collector.reg_use(ri);
704            collector.reg_use(rn);
705        }
706        Inst::VecExtract { rd, rn, rm, .. } => {
707            collector.reg_def(rd);
708            collector.reg_use(rn);
709            collector.reg_use(rm);
710        }
711        Inst::VecTbl { rd, rn, rm } => {
712            collector.reg_use(rn);
713            collector.reg_use(rm);
714            collector.reg_def(rd);
715        }
716        Inst::VecTblExt { rd, ri, rn, rm } => {
717            collector.reg_use(rn);
718            collector.reg_use(rm);
719            collector.reg_reuse_def(rd, 3); // `rd` == `ri`.
720            collector.reg_use(ri);
721        }
722
723        Inst::VecTbl2 { rd, rn, rn2, rm } => {
724            // Constrain to v30 / v31 so that we satisfy the "adjacent
725            // registers" constraint without use of pinned vregs in
726            // lowering.
727            collector.reg_fixed_use(rn, vreg(30));
728            collector.reg_fixed_use(rn2, vreg(31));
729            collector.reg_use(rm);
730            collector.reg_def(rd);
731        }
732        Inst::VecTbl2Ext {
733            rd,
734            ri,
735            rn,
736            rn2,
737            rm,
738        } => {
739            // Constrain to v30 / v31 so that we satisfy the "adjacent
740            // registers" constraint without use of pinned vregs in
741            // lowering.
742            collector.reg_fixed_use(rn, vreg(30));
743            collector.reg_fixed_use(rn2, vreg(31));
744            collector.reg_use(rm);
745            collector.reg_reuse_def(rd, 4); // `rd` == `ri`.
746            collector.reg_use(ri);
747        }
748        Inst::VecLoadReplicate { rd, rn, .. } => {
749            collector.reg_def(rd);
750            collector.reg_use(rn);
751        }
752        Inst::VecCSel { rd, rn, rm, .. } => {
753            collector.reg_def(rd);
754            collector.reg_use(rn);
755            collector.reg_use(rm);
756        }
757        Inst::FpuCmp { rn, rm, .. } => {
758            collector.reg_use(rn);
759            collector.reg_use(rm);
760        }
761        Inst::FpuLoad16 { rd, mem, .. } => {
762            collector.reg_def(rd);
763            memarg_operands(mem, collector);
764        }
765        Inst::FpuLoad32 { rd, mem, .. } => {
766            collector.reg_def(rd);
767            memarg_operands(mem, collector);
768        }
769        Inst::FpuLoad64 { rd, mem, .. } => {
770            collector.reg_def(rd);
771            memarg_operands(mem, collector);
772        }
773        Inst::FpuLoad128 { rd, mem, .. } => {
774            collector.reg_def(rd);
775            memarg_operands(mem, collector);
776        }
777        Inst::FpuStore16 { rd, mem, .. } => {
778            collector.reg_use(rd);
779            memarg_operands(mem, collector);
780        }
781        Inst::FpuStore32 { rd, mem, .. } => {
782            collector.reg_use(rd);
783            memarg_operands(mem, collector);
784        }
785        Inst::FpuStore64 { rd, mem, .. } => {
786            collector.reg_use(rd);
787            memarg_operands(mem, collector);
788        }
789        Inst::FpuStore128 { rd, mem, .. } => {
790            collector.reg_use(rd);
791            memarg_operands(mem, collector);
792        }
793        Inst::FpuLoadP64 { rt, rt2, mem, .. } => {
794            collector.reg_def(rt);
795            collector.reg_def(rt2);
796            pairmemarg_operands(mem, collector);
797        }
798        Inst::FpuStoreP64 { rt, rt2, mem, .. } => {
799            collector.reg_use(rt);
800            collector.reg_use(rt2);
801            pairmemarg_operands(mem, collector);
802        }
803        Inst::FpuLoadP128 { rt, rt2, mem, .. } => {
804            collector.reg_def(rt);
805            collector.reg_def(rt2);
806            pairmemarg_operands(mem, collector);
807        }
808        Inst::FpuStoreP128 { rt, rt2, mem, .. } => {
809            collector.reg_use(rt);
810            collector.reg_use(rt2);
811            pairmemarg_operands(mem, collector);
812        }
813        Inst::FpuToInt { rd, rn, .. } => {
814            collector.reg_def(rd);
815            collector.reg_use(rn);
816        }
817        Inst::IntToFpu { rd, rn, .. } => {
818            collector.reg_def(rd);
819            collector.reg_use(rn);
820        }
821        Inst::FpuCSel16 { rd, rn, rm, .. }
822        | Inst::FpuCSel32 { rd, rn, rm, .. }
823        | Inst::FpuCSel64 { rd, rn, rm, .. } => {
824            collector.reg_def(rd);
825            collector.reg_use(rn);
826            collector.reg_use(rm);
827        }
828        Inst::FpuRound { rd, rn, .. } => {
829            collector.reg_def(rd);
830            collector.reg_use(rn);
831        }
832        Inst::MovToFpu { rd, rn, .. } => {
833            collector.reg_def(rd);
834            collector.reg_use(rn);
835        }
836        Inst::FpuMoveFPImm { rd, .. } => {
837            collector.reg_def(rd);
838        }
839        Inst::MovToVec { rd, ri, rn, .. } => {
840            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
841            collector.reg_use(ri);
842            collector.reg_use(rn);
843        }
844        Inst::MovFromVec { rd, rn, .. } | Inst::MovFromVecSigned { rd, rn, .. } => {
845            collector.reg_def(rd);
846            collector.reg_use(rn);
847        }
848        Inst::VecDup { rd, rn, .. } => {
849            collector.reg_def(rd);
850            collector.reg_use(rn);
851        }
852        Inst::VecDupFromFpu { rd, rn, .. } => {
853            collector.reg_def(rd);
854            collector.reg_use(rn);
855        }
856        Inst::VecDupFPImm { rd, .. } => {
857            collector.reg_def(rd);
858        }
859        Inst::VecDupImm { rd, .. } => {
860            collector.reg_def(rd);
861        }
862        Inst::VecExtend { rd, rn, .. } => {
863            collector.reg_def(rd);
864            collector.reg_use(rn);
865        }
866        Inst::VecMovElement { rd, ri, rn, .. } => {
867            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
868            collector.reg_use(ri);
869            collector.reg_use(rn);
870        }
871        Inst::VecRRLong { rd, rn, .. } => {
872            collector.reg_def(rd);
873            collector.reg_use(rn);
874        }
875        Inst::VecRRNarrowLow { rd, rn, .. } => {
876            collector.reg_use(rn);
877            collector.reg_def(rd);
878        }
879        Inst::VecRRNarrowHigh { rd, ri, rn, .. } => {
880            collector.reg_use(rn);
881            collector.reg_reuse_def(rd, 2); // `rd` == `ri`.
882            collector.reg_use(ri);
883        }
884        Inst::VecRRPair { rd, rn, .. } => {
885            collector.reg_def(rd);
886            collector.reg_use(rn);
887        }
888        Inst::VecRRRLong { rd, rn, rm, .. } => {
889            collector.reg_def(rd);
890            collector.reg_use(rn);
891            collector.reg_use(rm);
892        }
893        Inst::VecRRRLongMod { rd, ri, rn, rm, .. } => {
894            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
895            collector.reg_use(ri);
896            collector.reg_use(rn);
897            collector.reg_use(rm);
898        }
899        Inst::VecRRPairLong { rd, rn, .. } => {
900            collector.reg_def(rd);
901            collector.reg_use(rn);
902        }
903        Inst::VecRRR { rd, rn, rm, .. } => {
904            collector.reg_def(rd);
905            collector.reg_use(rn);
906            collector.reg_use(rm);
907        }
908        Inst::VecRRRMod { rd, ri, rn, rm, .. } | Inst::VecFmlaElem { rd, ri, rn, rm, .. } => {
909            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
910            collector.reg_use(ri);
911            collector.reg_use(rn);
912            collector.reg_use(rm);
913        }
914        Inst::MovToNZCV { rn } => {
915            collector.reg_use(rn);
916        }
917        Inst::MovFromNZCV { rd } => {
918            collector.reg_def(rd);
919        }
920        Inst::BitfieldMove { rd, rn, .. } => {
921            // The UBFM and SBFM instructions overwrite all bits in `rd`,
922            // unlike BFM which is represented as `BitfieldMoveMod` instead.
923            collector.reg_def(rd);
924            collector.reg_use(rn);
925        }
926        Inst::BitfieldMoveMod { rd, ri, rn, .. } => {
927            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
928            collector.reg_use(ri);
929            collector.reg_use(rn);
930        }
931        Inst::Args { args } => {
932            for ArgPair { vreg, preg } in args {
933                collector.reg_fixed_def(vreg, *preg);
934            }
935        }
936        Inst::Rets { rets } => {
937            for RetPair { vreg, preg } in rets {
938                collector.reg_fixed_use(vreg, *preg);
939            }
940        }
941        Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}
942        Inst::Jump { .. } => {}
943        Inst::Call { info, .. } => {
944            let CallInfo { uses, defs, .. } = &mut **info;
945            for CallArgPair { vreg, preg } in uses {
946                collector.reg_fixed_use(vreg, *preg);
947            }
948            for CallRetPair { vreg, location } in defs {
949                match location {
950                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
951                    RetLocation::Stack(..) => collector.any_def(vreg),
952                }
953            }
954            collector.reg_clobbers(info.clobbers);
955            if let Some(try_call_info) = &mut info.try_call_info {
956                try_call_info.collect_operands(collector);
957            }
958        }
959        Inst::CallInd { info, .. } => {
960            let CallInfo {
961                dest, uses, defs, ..
962            } = &mut **info;
963            collector.reg_use(dest);
964            for CallArgPair { vreg, preg } in uses {
965                collector.reg_fixed_use(vreg, *preg);
966            }
967            for CallRetPair { vreg, location } in defs {
968                match location {
969                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
970                    RetLocation::Stack(..) => collector.any_def(vreg),
971                }
972            }
973            collector.reg_clobbers(info.clobbers);
974            if let Some(try_call_info) = &mut info.try_call_info {
975                try_call_info.collect_operands(collector);
976            }
977        }
978        Inst::ReturnCall { info } => {
979            for CallArgPair { vreg, preg } in &mut info.uses {
980                collector.reg_fixed_use(vreg, *preg);
981            }
982        }
983        Inst::ReturnCallInd { info } => {
984            // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
985            // This shouldn't be a fixed register constraint, but it's not clear how to pick a
986            // register that won't be clobbered by the callee-save restore code emitted with a
987            // return_call_indirect.
988            collector.reg_fixed_use(&mut info.dest, xreg(1));
989            for CallArgPair { vreg, preg } in &mut info.uses {
990                collector.reg_fixed_use(vreg, *preg);
991            }
992        }
993        Inst::CondBr { kind, .. } => match kind {
994            CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
995            CondBrKind::Cond(_) => {}
996        },
997        Inst::TestBitAndBranch { rn, .. } => {
998            collector.reg_use(rn);
999        }
1000        Inst::IndirectBr { rn, .. } => {
1001            collector.reg_use(rn);
1002        }
1003        Inst::Nop0 | Inst::Nop4 => {}
1004        Inst::Brk => {}
1005        Inst::Udf { .. } => {}
1006        Inst::TrapIf { kind, .. } => match kind {
1007            CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
1008            CondBrKind::Cond(_) => {}
1009        },
1010        Inst::Adr { rd, .. } | Inst::Adrp { rd, .. } => {
1011            collector.reg_def(rd);
1012        }
1013        Inst::Word4 { .. } | Inst::Word8 { .. } => {}
1014        Inst::JTSequence {
1015            ridx, rtmp1, rtmp2, ..
1016        } => {
1017            collector.reg_use(ridx);
1018            collector.reg_early_def(rtmp1);
1019            collector.reg_early_def(rtmp2);
1020        }
1021        Inst::LoadExtNameGot { rd, .. }
1022        | Inst::LoadExtNameNear { rd, .. }
1023        | Inst::LoadExtNameFar { rd, .. } => {
1024            collector.reg_def(rd);
1025        }
1026        Inst::LoadAddr { rd, mem } => {
1027            collector.reg_def(rd);
1028            memarg_operands(mem, collector);
1029        }
1030        Inst::Paci { .. } | Inst::Xpaclri => {
1031            // Neither LR nor SP is an allocatable register, so there is no need
1032            // to do anything.
1033        }
1034        Inst::Bti { .. } => {}
1035
1036        Inst::ElfTlsGetAddr { rd, tmp, .. } => {
1037            // TLSDESC has a very neat calling convention. It is required to preserve
1038            // all registers except x0 and x30. X30 is non allocatable in cranelift since
1039            // its the link register.
1040            //
1041            // Additionally we need a second register as a temporary register for the
1042            // TLSDESC sequence. This register can be any register other than x0 (and x30).
1043            collector.reg_fixed_def(rd, regs::xreg(0));
1044            collector.reg_early_def(tmp);
1045        }
1046        Inst::MachOTlsGetAddr { rd, .. } => {
1047            collector.reg_fixed_def(rd, regs::xreg(0));
1048            let mut clobbers =
1049                AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64, false);
1050            clobbers.remove(regs::xreg_preg(0));
1051            collector.reg_clobbers(clobbers);
1052        }
1053        Inst::Unwind { .. } => {}
1054        Inst::EmitIsland { .. } => {}
1055        Inst::DummyUse { reg } => {
1056            collector.reg_use(reg);
1057        }
1058        Inst::LabelAddress { dst, .. } => {
1059            collector.reg_def(dst);
1060        }
1061        Inst::SequencePoint { .. } => {}
1062        Inst::StackProbeLoop { start, end, .. } => {
1063            collector.reg_early_def(start);
1064            collector.reg_use(end);
1065        }
1066        Inst::InterruptPoll {
1067            dst,
1068            load_ptr,
1069            context,
1070            trap_code: _,
1071            next_load_ptr,
1072        } => {
1073            // `load_ptr` is an input param. It is pinned to x10 so an update of
1074            // `next_load_ptr` updates this as well. x10 is chosen because it is
1075            // a caller-saved scratch reg with no special role.
1076            collector.reg_fixed_use(load_ptr, regs::xreg(10));
1077            // Demand `context` (the vmctx) go into x0.
1078            collector.reg_fixed_use(context, regs::xreg(0));
1079            // Reserve x9 as scratch space. x9 is caller saved and has no
1080            // special role.
1081            //
1082            // Define it, so we can use it as the destination of the dead load
1083            // rather than consuming another arbitrary reg.
1084            collector.reg_fixed_def(dst, regs::xreg(9));
1085            // `next_load_ptr` is pinned so embedders know where to write to
1086            // fill it. It takes on the value of `load_ptr` unless a signal
1087            // handler invoked at the load overwrites it. We constrain
1088            // `next_load_ptr` and `load_ptr` to the same register so this
1089            // dataflow happens without a move.
1090            collector.reg_fixed_def(next_load_ptr, regs::xreg(10));
1091        }
1092    }
1093}
1094
1095//=============================================================================
1096// Instructions: misc functions and external interface
1097
1098impl MachInst for Inst {
1099    type ABIMachineSpec = AArch64MachineDeps;
1100    type LabelUse = LabelUse;
1101
1102    // "CLIF" in hex, to make the trap recognizable during
1103    // debugging.
1104    const TRAP_OPCODE: &'static [u8] = &0xc11f_u32.to_le_bytes();
1105
1106    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
1107        aarch64_get_operands(self, collector);
1108    }
1109
1110    fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
1111        match self {
1112            &Inst::Mov {
1113                size: OperandSize::Size64,
1114                rd,
1115                rm,
1116            } => Some((rd, rm)),
1117            &Inst::FpuMove64 { rd, rn } => Some((rd, rn)),
1118            &Inst::FpuMove128 { rd, rn } => Some((rd, rn)),
1119            _ => None,
1120        }
1121    }
1122
1123    fn is_included_in_clobbers(&self) -> bool {
1124        let (caller, callee, is_exception) = match self {
1125            Inst::Args { .. } => return false,
1126            Inst::Call { info } => (
1127                info.caller_conv,
1128                info.callee_conv,
1129                info.try_call_info.is_some(),
1130            ),
1131            Inst::CallInd { info } => (
1132                info.caller_conv,
1133                info.callee_conv,
1134                info.try_call_info.is_some(),
1135            ),
1136            _ => return true,
1137        };
1138
1139        // We exclude call instructions from the clobber-set when they are calls
1140        // from caller to callee that both clobber the same register (such as
1141        // using the same or similar ABIs). Such calls cannot possibly force any
1142        // new registers to be saved in the prologue, because anything that the
1143        // callee clobbers, the caller is also allowed to clobber. This both
1144        // saves work and enables us to more precisely follow the
1145        // half-caller-save, half-callee-save SysV ABI for some vector
1146        // registers.
1147        //
1148        // See the note in [crate::isa::aarch64::abi::is_caller_save_reg] for
1149        // more information on this ABI-implementation hack.
1150        let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller, false);
1151        let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee, is_exception);
1152
1153        let mut all_clobbers = caller_clobbers;
1154        all_clobbers.union_from(callee_clobbers);
1155        all_clobbers != caller_clobbers
1156    }
1157
1158    fn is_trap(&self) -> bool {
1159        match self {
1160            Self::Udf { .. } => true,
1161            _ => false,
1162        }
1163    }
1164
1165    fn is_args(&self) -> bool {
1166        match self {
1167            Self::Args { .. } => true,
1168            _ => false,
1169        }
1170    }
1171
1172    fn call_type(&self) -> CallType {
1173        match self {
1174            Inst::Call { .. }
1175            | Inst::CallInd { .. }
1176            | Inst::ElfTlsGetAddr { .. }
1177            | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
1178
1179            Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
1180
1181            _ => CallType::None,
1182        }
1183    }
1184
1185    fn is_term(&self) -> MachTerminator {
1186        match self {
1187            &Inst::Rets { .. } => MachTerminator::Ret,
1188            &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
1189            &Inst::Jump { .. } => MachTerminator::Branch,
1190            &Inst::CondBr { .. } => MachTerminator::Branch,
1191            &Inst::TestBitAndBranch { .. } => MachTerminator::Branch,
1192            &Inst::IndirectBr { .. } => MachTerminator::Branch,
1193            &Inst::JTSequence { .. } => MachTerminator::Branch,
1194            &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1195            &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1196            _ => MachTerminator::None,
1197        }
1198    }
1199
1200    fn is_mem_access(&self) -> bool {
1201        match self {
1202            &Inst::ULoad8 { .. }
1203            | &Inst::SLoad8 { .. }
1204            | &Inst::ULoad16 { .. }
1205            | &Inst::SLoad16 { .. }
1206            | &Inst::ULoad32 { .. }
1207            | &Inst::SLoad32 { .. }
1208            | &Inst::ULoad64 { .. }
1209            | &Inst::LoadP64 { .. }
1210            | &Inst::FpuLoad16 { .. }
1211            | &Inst::FpuLoad32 { .. }
1212            | &Inst::FpuLoad64 { .. }
1213            | &Inst::FpuLoad128 { .. }
1214            | &Inst::FpuLoadP64 { .. }
1215            | &Inst::FpuLoadP128 { .. }
1216            | &Inst::Store8 { .. }
1217            | &Inst::Store16 { .. }
1218            | &Inst::Store32 { .. }
1219            | &Inst::Store64 { .. }
1220            | &Inst::StoreP64 { .. }
1221            | &Inst::FpuStore16 { .. }
1222            | &Inst::FpuStore32 { .. }
1223            | &Inst::FpuStore64 { .. }
1224            | &Inst::FpuStore128 { .. } => true,
1225            // TODO: verify this carefully
1226            _ => false,
1227        }
1228    }
1229
1230    fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
1231        let bits = ty.bits();
1232
1233        assert!(bits <= 128);
1234        assert!(to_reg.to_reg().class() == from_reg.class());
1235        match from_reg.class() {
1236            RegClass::Int => Inst::Mov {
1237                size: OperandSize::Size64,
1238                rd: to_reg,
1239                rm: from_reg,
1240            },
1241            RegClass::Float => {
1242                if bits > 64 {
1243                    Inst::FpuMove128 {
1244                        rd: to_reg,
1245                        rn: from_reg,
1246                    }
1247                } else {
1248                    Inst::FpuMove64 {
1249                        rd: to_reg,
1250                        rn: from_reg,
1251                    }
1252                }
1253            }
1254            RegClass::Vector => unreachable!(),
1255        }
1256    }
1257
1258    fn is_safepoint(&self) -> bool {
1259        match self {
1260            Inst::Call { .. } | Inst::CallInd { .. } => true,
1261            _ => false,
1262        }
1263    }
1264
1265    fn gen_dummy_use(reg: Reg) -> Inst {
1266        Inst::DummyUse { reg }
1267    }
1268
1269    fn gen_nop(preferred_size: usize) -> Inst {
1270        if preferred_size == 0 {
1271            return Inst::Nop0;
1272        }
1273        // We can't give a NOP (or any insn) < 4 bytes.
1274        assert!(preferred_size >= 4);
1275        Inst::Nop4
1276    }
1277
1278    fn gen_nop_units() -> Vec<Vec<u8>> {
1279        vec![vec![0x1f, 0x20, 0x03, 0xd5]]
1280    }
1281
1282    fn rc_for_type(ty: &Type) -> CodegenResult<(&[RegClass], &[Type])> {
1283        match *ty {
1284            I8 | I16 | I32 | I64 => Ok((&[RegClass::Int], slice::from_ref(ty))),
1285            F16 | F32 | F64 | F128 => Ok((&[RegClass::Float], slice::from_ref(ty))),
1286            I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
1287            _ if ty.is_vector() && ty.bits() <= 128 => {
1288                let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1289                Ok((
1290                    &[RegClass::Float],
1291                    slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1292                ))
1293            }
1294            _ if ty.is_dynamic_vector() => Ok((&[RegClass::Float], &[I8X16])),
1295            _ => Err(CodegenError::Unsupported(format!(
1296                "Unexpected SSA-value type: {ty}"
1297            ))),
1298        }
1299    }
1300
1301    fn canonical_type_for_rc(rc: RegClass) -> Type {
1302        match rc {
1303            RegClass::Float => types::I8X16,
1304            RegClass::Int => types::I64,
1305            RegClass::Vector => unreachable!(),
1306        }
1307    }
1308
1309    fn gen_jump(target: MachLabel) -> Inst {
1310        Inst::Jump {
1311            dest: BranchTarget::Label(target),
1312        }
1313    }
1314
1315    fn worst_case_size() -> CodeOffset {
1316        // The maximum size, in bytes, of any `Inst`'s emitted code. We have at least one case of
1317        // an 8-instruction sequence (saturating int-to-float conversions) with three embedded
1318        // 64-bit f64 constants.
1319        //
1320        // Note that inline jump-tables handle island/pool insertion separately, so we do not need
1321        // to account for them here (otherwise the worst case would be 2^31 * 4, clearly not
1322        // feasible for other reasons).
1323        44
1324    }
1325
1326    fn worst_case_island_growth() -> CodeOffset {
1327        // A single `Inst` may add to the buffer's pending-island state:
1328        //
1329        // - Up to three 8-byte constants (the saturating int-to-float sequence
1330        //   noted above); count alignment padding into each.
1331        // - Up to one deferred trap (TrapIf and similar), 4 bytes.
1332        // - Up to one fixup per emitted instruction word, each contributing at
1333        //   most `worst_case_veneer_size()` (= 20) bytes of veneer.
1334        //
1335        // We pick a conservative bound that comfortably covers these.
1336        128
1337    }
1338
1339    fn gen_block_start(
1340        is_indirect_branch_target: bool,
1341        is_forward_edge_cfi_enabled: bool,
1342    ) -> Option<Self> {
1343        if is_indirect_branch_target && is_forward_edge_cfi_enabled {
1344            Some(Inst::Bti {
1345                targets: BranchTargetType::J,
1346            })
1347        } else {
1348            None
1349        }
1350    }
1351
1352    fn function_alignment() -> FunctionAlignment {
1353        // We use 32-byte alignment for performance reasons, but for correctness
1354        // we would only need 4-byte alignment.
1355        FunctionAlignment {
1356            minimum: 4,
1357            preferred: 32,
1358        }
1359    }
1360}
1361
1362//=============================================================================
1363// Pretty-printing of instructions.
1364
1365fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (String, String) {
1366    let (mem_insts, mem) = mem_finalize(None, mem, access_ty, state);
1367    let mut mem_str = mem_insts
1368        .into_iter()
1369        .map(|inst| inst.print_with_state(&mut EmitState::default()))
1370        .collect::<Vec<_>>()
1371        .join(" ; ");
1372    if !mem_str.is_empty() {
1373        mem_str += " ; ";
1374    }
1375
1376    let mem = mem.pretty_print(access_ty.bytes() as u8);
1377    (mem_str, mem)
1378}
1379
1380fn pretty_print_try_call(info: &TryCallInfo) -> String {
1381    format!(
1382        "; b {:?}; catch [{}]",
1383        info.continuation,
1384        info.pretty_print_dests()
1385    )
1386}
1387
1388impl Inst {
1389    #[expect(
1390        missing_docs,
1391        reason = "exposed for cranelift-isle/veri pretty-printing"
1392    )]
1393    pub fn print_with_state(&self, state: &mut EmitState) -> String {
1394        fn op_name(alu_op: ALUOp) -> &'static str {
1395            match alu_op {
1396                ALUOp::Add => "add",
1397                ALUOp::Sub => "sub",
1398                ALUOp::Orr => "orr",
1399                ALUOp::And => "and",
1400                ALUOp::AndS => "ands",
1401                ALUOp::Eor => "eor",
1402                ALUOp::AddS => "adds",
1403                ALUOp::SubS => "subs",
1404                ALUOp::SMulH => "smulh",
1405                ALUOp::UMulH => "umulh",
1406                ALUOp::SDiv => "sdiv",
1407                ALUOp::UDiv => "udiv",
1408                ALUOp::AndNot => "bic",
1409                ALUOp::OrrNot => "orn",
1410                ALUOp::EorNot => "eon",
1411                ALUOp::Extr => "extr",
1412                ALUOp::Lsr => "lsr",
1413                ALUOp::Asr => "asr",
1414                ALUOp::Lsl => "lsl",
1415                ALUOp::Adc => "adc",
1416                ALUOp::AdcS => "adcs",
1417                ALUOp::Sbc => "sbc",
1418                ALUOp::SbcS => "sbcs",
1419            }
1420        }
1421
1422        match self {
1423            &Inst::Nop0 => "nop-zero-len".to_string(),
1424            &Inst::Nop4 => "nop".to_string(),
1425            &Inst::AluRRR {
1426                alu_op,
1427                size,
1428                rd,
1429                rn,
1430                rm,
1431            } => {
1432                let op = op_name(alu_op);
1433                let rd = pretty_print_ireg(rd.to_reg(), size);
1434                let rn = pretty_print_ireg(rn, size);
1435                let rm = pretty_print_ireg(rm, size);
1436                format!("{op} {rd}, {rn}, {rm}")
1437            }
1438            &Inst::AluRRRR {
1439                alu_op,
1440                size,
1441                rd,
1442                rn,
1443                rm,
1444                ra,
1445            } => {
1446                let (op, da_size) = match alu_op {
1447                    ALUOp3::MAdd => ("madd", size),
1448                    ALUOp3::MSub => ("msub", size),
1449                    ALUOp3::UMAddL => ("umaddl", OperandSize::Size64),
1450                    ALUOp3::SMAddL => ("smaddl", OperandSize::Size64),
1451                };
1452                let rd = pretty_print_ireg(rd.to_reg(), da_size);
1453                let rn = pretty_print_ireg(rn, size);
1454                let rm = pretty_print_ireg(rm, size);
1455                let ra = pretty_print_ireg(ra, da_size);
1456
1457                format!("{op} {rd}, {rn}, {rm}, {ra}")
1458            }
1459            &Inst::AluRRImm12 {
1460                alu_op,
1461                size,
1462                rd,
1463                rn,
1464                ref imm12,
1465            } => {
1466                let op = op_name(alu_op);
1467                let rd = pretty_print_ireg(rd.to_reg(), size);
1468                let rn = pretty_print_ireg(rn, size);
1469
1470                if imm12.bits == 0 && alu_op == ALUOp::Add && size.is64() {
1471                    // special-case MOV (used for moving into SP).
1472                    format!("mov {rd}, {rn}")
1473                } else {
1474                    let imm12 = imm12.pretty_print(0);
1475                    format!("{op} {rd}, {rn}, {imm12}")
1476                }
1477            }
1478            &Inst::AluRRImmLogic {
1479                alu_op,
1480                size,
1481                rd,
1482                rn,
1483                ref imml,
1484            } => {
1485                let op = op_name(alu_op);
1486                let rd = pretty_print_ireg(rd.to_reg(), size);
1487                let rn = pretty_print_ireg(rn, size);
1488                let imml = imml.pretty_print(0);
1489                format!("{op} {rd}, {rn}, {imml}")
1490            }
1491            &Inst::AluRRImmShift {
1492                alu_op,
1493                size,
1494                rd,
1495                rn,
1496                ref immshift,
1497            } => {
1498                let op = op_name(alu_op);
1499                let rd = pretty_print_ireg(rd.to_reg(), size);
1500                let rn = pretty_print_ireg(rn, size);
1501                let immshift = immshift.pretty_print(0);
1502                format!("{op} {rd}, {rn}, {immshift}")
1503            }
1504            &Inst::AluRRRShift {
1505                alu_op,
1506                size,
1507                rd,
1508                rn,
1509                rm,
1510                ref shiftop,
1511            } => {
1512                let op = op_name(alu_op);
1513                let rd = pretty_print_ireg(rd.to_reg(), size);
1514                let rn = pretty_print_ireg(rn, size);
1515                let rm = pretty_print_ireg(rm, size);
1516                let shiftop = shiftop.pretty_print(0);
1517                format!("{op} {rd}, {rn}, {rm}, {shiftop}")
1518            }
1519            &Inst::AluRRRExtend {
1520                alu_op,
1521                size,
1522                rd,
1523                rn,
1524                rm,
1525                ref extendop,
1526            } => {
1527                let op = op_name(alu_op);
1528                let rd = pretty_print_ireg(rd.to_reg(), size);
1529                let rn = pretty_print_ireg(rn, size);
1530                let rm = pretty_print_ireg(rm, size);
1531                let extendop = extendop.pretty_print(0);
1532                format!("{op} {rd}, {rn}, {rm}, {extendop}")
1533            }
1534            &Inst::BitRR { op, size, rd, rn } => {
1535                let op = op.op_str();
1536                let rd = pretty_print_ireg(rd.to_reg(), size);
1537                let rn = pretty_print_ireg(rn, size);
1538                format!("{op} {rd}, {rn}")
1539            }
1540            &Inst::ULoad8 { rd, ref mem, .. }
1541            | &Inst::SLoad8 { rd, ref mem, .. }
1542            | &Inst::ULoad16 { rd, ref mem, .. }
1543            | &Inst::SLoad16 { rd, ref mem, .. }
1544            | &Inst::ULoad32 { rd, ref mem, .. }
1545            | &Inst::SLoad32 { rd, ref mem, .. }
1546            | &Inst::ULoad64 { rd, ref mem, .. } => {
1547                let is_unscaled = match &mem {
1548                    &AMode::Unscaled { .. } => true,
1549                    _ => false,
1550                };
1551                let (op, size) = match (self, is_unscaled) {
1552                    (&Inst::ULoad8 { .. }, false) => ("ldrb", OperandSize::Size32),
1553                    (&Inst::ULoad8 { .. }, true) => ("ldurb", OperandSize::Size32),
1554                    (&Inst::SLoad8 { .. }, false) => ("ldrsb", OperandSize::Size64),
1555                    (&Inst::SLoad8 { .. }, true) => ("ldursb", OperandSize::Size64),
1556                    (&Inst::ULoad16 { .. }, false) => ("ldrh", OperandSize::Size32),
1557                    (&Inst::ULoad16 { .. }, true) => ("ldurh", OperandSize::Size32),
1558                    (&Inst::SLoad16 { .. }, false) => ("ldrsh", OperandSize::Size64),
1559                    (&Inst::SLoad16 { .. }, true) => ("ldursh", OperandSize::Size64),
1560                    (&Inst::ULoad32 { .. }, false) => ("ldr", OperandSize::Size32),
1561                    (&Inst::ULoad32 { .. }, true) => ("ldur", OperandSize::Size32),
1562                    (&Inst::SLoad32 { .. }, false) => ("ldrsw", OperandSize::Size64),
1563                    (&Inst::SLoad32 { .. }, true) => ("ldursw", OperandSize::Size64),
1564                    (&Inst::ULoad64 { .. }, false) => ("ldr", OperandSize::Size64),
1565                    (&Inst::ULoad64 { .. }, true) => ("ldur", OperandSize::Size64),
1566                    _ => unreachable!(),
1567                };
1568
1569                let rd = pretty_print_ireg(rd.to_reg(), size);
1570                let mem = mem.clone();
1571                let access_ty = self.mem_type().unwrap();
1572                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1573
1574                format!("{mem_str}{op} {rd}, {mem}")
1575            }
1576            &Inst::Store8 { rd, ref mem, .. }
1577            | &Inst::Store16 { rd, ref mem, .. }
1578            | &Inst::Store32 { rd, ref mem, .. }
1579            | &Inst::Store64 { rd, ref mem, .. } => {
1580                let is_unscaled = match &mem {
1581                    &AMode::Unscaled { .. } => true,
1582                    _ => false,
1583                };
1584                let (op, size) = match (self, is_unscaled) {
1585                    (&Inst::Store8 { .. }, false) => ("strb", OperandSize::Size32),
1586                    (&Inst::Store8 { .. }, true) => ("sturb", OperandSize::Size32),
1587                    (&Inst::Store16 { .. }, false) => ("strh", OperandSize::Size32),
1588                    (&Inst::Store16 { .. }, true) => ("sturh", OperandSize::Size32),
1589                    (&Inst::Store32 { .. }, false) => ("str", OperandSize::Size32),
1590                    (&Inst::Store32 { .. }, true) => ("stur", OperandSize::Size32),
1591                    (&Inst::Store64 { .. }, false) => ("str", OperandSize::Size64),
1592                    (&Inst::Store64 { .. }, true) => ("stur", OperandSize::Size64),
1593                    _ => unreachable!(),
1594                };
1595
1596                let rd = pretty_print_ireg(rd, size);
1597                let mem = mem.clone();
1598                let access_ty = self.mem_type().unwrap();
1599                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1600
1601                format!("{mem_str}{op} {rd}, {mem}")
1602            }
1603            &Inst::StoreP64 {
1604                rt, rt2, ref mem, ..
1605            } => {
1606                let rt = pretty_print_ireg(rt, OperandSize::Size64);
1607                let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);
1608                let mem = mem.clone();
1609                let mem = mem.pretty_print_default();
1610                format!("stp {rt}, {rt2}, {mem}")
1611            }
1612            &Inst::LoadP64 {
1613                rt, rt2, ref mem, ..
1614            } => {
1615                let rt = pretty_print_ireg(rt.to_reg(), OperandSize::Size64);
1616                let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);
1617                let mem = mem.clone();
1618                let mem = mem.pretty_print_default();
1619                format!("ldp {rt}, {rt2}, {mem}")
1620            }
1621            &Inst::Mov { size, rd, rm } => {
1622                let rd = pretty_print_ireg(rd.to_reg(), size);
1623                let rm = pretty_print_ireg(rm, size);
1624                format!("mov {rd}, {rm}")
1625            }
1626            &Inst::MovFromPReg { rd, rm } => {
1627                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1628                let rm = show_ireg_sized(rm.into(), OperandSize::Size64);
1629                format!("mov {rd}, {rm}")
1630            }
1631            &Inst::MovToPReg { rd, rm } => {
1632                let rd = show_ireg_sized(rd.into(), OperandSize::Size64);
1633                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1634                format!("mov {rd}, {rm}")
1635            }
1636            &Inst::MovWide {
1637                op,
1638                rd,
1639                ref imm,
1640                size,
1641            } => {
1642                let op_str = match op {
1643                    MoveWideOp::MovZ => "movz",
1644                    MoveWideOp::MovN => "movn",
1645                };
1646                let rd = pretty_print_ireg(rd.to_reg(), size);
1647                let imm = imm.pretty_print(0);
1648                format!("{op_str} {rd}, {imm}")
1649            }
1650            &Inst::MovK {
1651                rd,
1652                rn,
1653                ref imm,
1654                size,
1655            } => {
1656                let rn = pretty_print_ireg(rn, size);
1657                let rd = pretty_print_ireg(rd.to_reg(), size);
1658                let imm = imm.pretty_print(0);
1659                format!("movk {rd}, {rn}, {imm}")
1660            }
1661            &Inst::CSel { rd, rn, rm, cond } => {
1662                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1663                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1664                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1665                let cond = cond.pretty_print(0);
1666                format!("csel {rd}, {rn}, {rm}, {cond}")
1667            }
1668            &Inst::CSNeg { rd, rn, rm, cond } => {
1669                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1670                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1671                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1672                let cond = cond.pretty_print(0);
1673                format!("csneg {rd}, {rn}, {rm}, {cond}")
1674            }
1675            &Inst::CSet { rd, cond } => {
1676                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1677                let cond = cond.pretty_print(0);
1678                format!("cset {rd}, {cond}")
1679            }
1680            &Inst::CSetm { rd, cond } => {
1681                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1682                let cond = cond.pretty_print(0);
1683                format!("csetm {rd}, {cond}")
1684            }
1685            &Inst::CCmp {
1686                size,
1687                rn,
1688                rm,
1689                nzcv,
1690                cond,
1691            } => {
1692                let rn = pretty_print_ireg(rn, size);
1693                let rm = pretty_print_ireg(rm, size);
1694                let nzcv = nzcv.pretty_print(0);
1695                let cond = cond.pretty_print(0);
1696                format!("ccmp {rn}, {rm}, {nzcv}, {cond}")
1697            }
1698            &Inst::CCmpImm {
1699                size,
1700                rn,
1701                imm,
1702                nzcv,
1703                cond,
1704            } => {
1705                let rn = pretty_print_ireg(rn, size);
1706                let imm = imm.pretty_print(0);
1707                let nzcv = nzcv.pretty_print(0);
1708                let cond = cond.pretty_print(0);
1709                format!("ccmp {rn}, {imm}, {nzcv}, {cond}")
1710            }
1711            &Inst::AtomicRMW {
1712                rs, rt, rn, ty, op, ..
1713            } => {
1714                let op = match op {
1715                    AtomicRMWOp::Add => "ldaddal",
1716                    AtomicRMWOp::Clr => "ldclral",
1717                    AtomicRMWOp::Eor => "ldeoral",
1718                    AtomicRMWOp::Set => "ldsetal",
1719                    AtomicRMWOp::Smax => "ldsmaxal",
1720                    AtomicRMWOp::Umax => "ldumaxal",
1721                    AtomicRMWOp::Smin => "ldsminal",
1722                    AtomicRMWOp::Umin => "lduminal",
1723                    AtomicRMWOp::Swp => "swpal",
1724                };
1725
1726                let size = OperandSize::from_ty(ty);
1727                let rs = pretty_print_ireg(rs, size);
1728                let rt = pretty_print_ireg(rt.to_reg(), size);
1729                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1730
1731                let ty_suffix = match ty {
1732                    I8 => "b",
1733                    I16 => "h",
1734                    _ => "",
1735                };
1736                format!("{op}{ty_suffix} {rs}, {rt}, [{rn}]")
1737            }
1738            &Inst::AtomicRMWLoop {
1739                ty,
1740                op,
1741                addr,
1742                operand,
1743                oldval,
1744                scratch1,
1745                scratch2,
1746                ..
1747            } => {
1748                let op = match op {
1749                    AtomicRMWLoopOp::Add => "add",
1750                    AtomicRMWLoopOp::Sub => "sub",
1751                    AtomicRMWLoopOp::Eor => "eor",
1752                    AtomicRMWLoopOp::Orr => "orr",
1753                    AtomicRMWLoopOp::And => "and",
1754                    AtomicRMWLoopOp::Nand => "nand",
1755                    AtomicRMWLoopOp::Smin => "smin",
1756                    AtomicRMWLoopOp::Smax => "smax",
1757                    AtomicRMWLoopOp::Umin => "umin",
1758                    AtomicRMWLoopOp::Umax => "umax",
1759                    AtomicRMWLoopOp::Xchg => "xchg",
1760                };
1761                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1762                let operand = pretty_print_ireg(operand, OperandSize::Size64);
1763                let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1764                let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);
1765                let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);
1766                format!(
1767                    "atomic_rmw_loop_{}_{} addr={} operand={} oldval={} scratch1={} scratch2={}",
1768                    op,
1769                    ty.bits(),
1770                    addr,
1771                    operand,
1772                    oldval,
1773                    scratch1,
1774                    scratch2,
1775                )
1776            }
1777            &Inst::AtomicRMW128Loop {
1778                op,
1779                addr,
1780                operand_lo,
1781                operand_hi,
1782                oldval_lo,
1783                oldval_hi,
1784                scratch1,
1785                scratch2,
1786                scratch3,
1787                ..
1788            } => {
1789                let op = match op {
1790                    AtomicRMWLoopOp::Add => "add",
1791                    AtomicRMWLoopOp::Sub => "sub",
1792                    AtomicRMWLoopOp::Eor => "eor",
1793                    AtomicRMWLoopOp::Orr => "orr",
1794                    AtomicRMWLoopOp::And => "and",
1795                    AtomicRMWLoopOp::Nand => "nand",
1796                    AtomicRMWLoopOp::Smin => "smin",
1797                    AtomicRMWLoopOp::Smax => "smax",
1798                    AtomicRMWLoopOp::Umin => "umin",
1799                    AtomicRMWLoopOp::Umax => "umax",
1800                    AtomicRMWLoopOp::Xchg => "xchg",
1801                };
1802                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1803                let operand_lo = pretty_print_ireg(operand_lo, OperandSize::Size64);
1804                let operand_hi = pretty_print_ireg(operand_hi, OperandSize::Size64);
1805                let oldval_lo = pretty_print_ireg(oldval_lo.to_reg(), OperandSize::Size64);
1806                let oldval_hi = pretty_print_ireg(oldval_hi.to_reg(), OperandSize::Size64);
1807                let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);
1808                let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);
1809                let scratch3 = pretty_print_ireg(scratch3.to_reg(), OperandSize::Size64);
1810                format!(
1811                    "atomic_rmw_128_loop_{op} addr={addr} operand_lo={operand_lo} operand_hi={operand_hi} oldval_lo={oldval_lo} oldval_hi={oldval_hi} scratch1={scratch1} scratch2={scratch2} scratch3={scratch3}",
1812                )
1813            }
1814            &Inst::AtomicCAS {
1815                rd, rs, rt, rn, ty, ..
1816            } => {
1817                let op = match ty {
1818                    I8 => "casalb",
1819                    I16 => "casalh",
1820                    I32 | I64 => "casal",
1821                    _ => panic!("Unsupported type: {ty}"),
1822                };
1823                let size = OperandSize::from_ty(ty);
1824                let rd = pretty_print_ireg(rd.to_reg(), size);
1825                let rs = pretty_print_ireg(rs, size);
1826                let rt = pretty_print_ireg(rt, size);
1827                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1828
1829                format!("{op} {rd}, {rs}, {rt}, [{rn}]")
1830            }
1831            Inst::AtomicCAS128 { args } => {
1832                let &AtomicCAS128Args {
1833                    rd_lo,
1834                    rd_hi,
1835                    rs_lo,
1836                    rs_hi,
1837                    rt_lo,
1838                    rt_hi,
1839                    rn,
1840                    flags: _,
1841                } = &**args;
1842                let size = OperandSize::Size64;
1843                let rd_lo = pretty_print_ireg(rd_lo.to_reg(), size);
1844                let rd_hi = pretty_print_ireg(rd_hi.to_reg(), size);
1845                let rs_lo = pretty_print_ireg(rs_lo, size);
1846                let rs_hi = pretty_print_ireg(rs_hi, size);
1847                let rt_lo = pretty_print_ireg(rt_lo, size);
1848                let rt_hi = pretty_print_ireg(rt_hi, size);
1849                let rn = pretty_print_ireg(rn, size);
1850
1851                format!("caspal {rd_lo}, {rd_hi}, {rs_lo}, {rs_hi}, {rt_lo}, {rt_hi}, [{rn}]")
1852            }
1853            &Inst::AtomicCASLoop {
1854                ty,
1855                addr,
1856                expected,
1857                replacement,
1858                oldval,
1859                scratch,
1860                ..
1861            } => {
1862                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1863                let expected = pretty_print_ireg(expected, OperandSize::Size64);
1864                let replacement = pretty_print_ireg(replacement, OperandSize::Size64);
1865                let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1866                let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1867                format!(
1868                    "atomic_cas_loop_{} addr={}, expect={}, replacement={}, oldval={}, scratch={}",
1869                    ty.bits(),
1870                    addr,
1871                    expected,
1872                    replacement,
1873                    oldval,
1874                    scratch,
1875                )
1876            }
1877            &Inst::AtomicCAS128Loop {
1878                addr,
1879                expected_lo,
1880                expected_hi,
1881                replacement_lo,
1882                replacement_hi,
1883                oldval_lo,
1884                oldval_hi,
1885                scratch,
1886                ..
1887            } => {
1888                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1889                let expected_lo = pretty_print_ireg(expected_lo, OperandSize::Size64);
1890                let expected_hi = pretty_print_ireg(expected_hi, OperandSize::Size64);
1891                let replacement_lo = pretty_print_ireg(replacement_lo, OperandSize::Size64);
1892                let replacement_hi = pretty_print_ireg(replacement_hi, OperandSize::Size64);
1893                let oldval_lo = pretty_print_ireg(oldval_lo.to_reg(), OperandSize::Size64);
1894                let oldval_hi = pretty_print_ireg(oldval_hi.to_reg(), OperandSize::Size64);
1895                let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1896                format!(
1897                    "atomic_cas_128_loop addr={addr}, expected_lo={expected_lo}, expected_hi={expected_hi}, replacement_lo={replacement_lo}, replacement_hi={replacement_hi}, oldval_lo={oldval_lo}, oldval_hi={oldval_hi}, scratch={scratch}",
1898                )
1899            }
1900            &Inst::LoadAcquire {
1901                access_ty, rt, rn, ..
1902            } => {
1903                let (op, ty) = match access_ty {
1904                    I8 => ("ldarb", I32),
1905                    I16 => ("ldarh", I32),
1906                    I32 => ("ldar", I32),
1907                    I64 => ("ldar", I64),
1908                    _ => panic!("Unsupported type: {access_ty}"),
1909                };
1910                let size = OperandSize::from_ty(ty);
1911                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1912                let rt = pretty_print_ireg(rt.to_reg(), size);
1913                format!("{op} {rt}, [{rn}]")
1914            }
1915            &Inst::LoadAcquire128 {
1916                rt1,
1917                rt2,
1918                rn,
1919                scratch,
1920                ..
1921            } => {
1922                let rt1 = pretty_print_ireg(rt1.to_reg(), OperandSize::Size64);
1923                let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);
1924                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1925                let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1926                format!("load_acquire_128 {rt1}, {rt2}, [{rn}], scratch={scratch}")
1927            }
1928            &Inst::StoreRelease {
1929                access_ty, rt, rn, ..
1930            } => {
1931                let (op, ty) = match access_ty {
1932                    I8 => ("stlrb", I32),
1933                    I16 => ("stlrh", I32),
1934                    I32 => ("stlr", I32),
1935                    I64 => ("stlr", I64),
1936                    _ => panic!("Unsupported type: {access_ty}"),
1937                };
1938                let size = OperandSize::from_ty(ty);
1939                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1940                let rt = pretty_print_ireg(rt, size);
1941                format!("{op} {rt}, [{rn}]")
1942            }
1943            &Inst::StoreRelease128 {
1944                rt1,
1945                rt2,
1946                rn,
1947                scratch,
1948                ..
1949            } => {
1950                let rt1 = pretty_print_ireg(rt1, OperandSize::Size64);
1951                let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);
1952                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1953                let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1954                format!("store_release_128 {rt1}, {rt2}, [{rn}], scratch={scratch}")
1955            }
1956            &Inst::Fence {} => {
1957                format!("dmb ish")
1958            }
1959            &Inst::Csdb {} => {
1960                format!("csdb")
1961            }
1962            &Inst::FpuMove32 { rd, rn } => {
1963                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1964                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
1965                format!("fmov {rd}, {rn}")
1966            }
1967            &Inst::FpuMove64 { rd, rn } => {
1968                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1969                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
1970                format!("fmov {rd}, {rn}")
1971            }
1972            &Inst::FpuMove128 { rd, rn } => {
1973                let rd = pretty_print_reg(rd.to_reg());
1974                let rn = pretty_print_reg(rn);
1975                format!("mov {rd}.16b, {rn}.16b")
1976            }
1977            &Inst::FpuMoveFromVec { rd, rn, idx, size } => {
1978                let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
1979                let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
1980                format!("mov {rd}, {rn}")
1981            }
1982            &Inst::FpuExtend { rd, rn, size } => {
1983                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1984                let rn = pretty_print_vreg_scalar(rn, size);
1985                format!("fmov {rd}, {rn}")
1986            }
1987            &Inst::FpuRR {
1988                fpu_op,
1989                size,
1990                rd,
1991                rn,
1992            } => {
1993                let op = match fpu_op {
1994                    FPUOp1::Abs => "fabs",
1995                    FPUOp1::Neg => "fneg",
1996                    FPUOp1::Sqrt => "fsqrt",
1997                    FPUOp1::Cvt16To32 | FPUOp1::Cvt32To64 | FPUOp1::Cvt64To32 => "fcvt",
1998                };
1999                let dst_size = match fpu_op {
2000                    FPUOp1::Cvt32To64 => ScalarSize::Size64,
2001                    FPUOp1::Cvt16To32 | FPUOp1::Cvt64To32 => ScalarSize::Size32,
2002                    _ => size,
2003                };
2004                let rd = pretty_print_vreg_scalar(rd.to_reg(), dst_size);
2005                let rn = pretty_print_vreg_scalar(rn, size);
2006                format!("{op} {rd}, {rn}")
2007            }
2008            &Inst::FpuRRR {
2009                fpu_op,
2010                size,
2011                rd,
2012                rn,
2013                rm,
2014            } => {
2015                let op = match fpu_op {
2016                    FPUOp2::Add => "fadd",
2017                    FPUOp2::Sub => "fsub",
2018                    FPUOp2::Mul => "fmul",
2019                    FPUOp2::Div => "fdiv",
2020                    FPUOp2::Max => "fmax",
2021                    FPUOp2::Min => "fmin",
2022                };
2023                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2024                let rn = pretty_print_vreg_scalar(rn, size);
2025                let rm = pretty_print_vreg_scalar(rm, size);
2026                format!("{op} {rd}, {rn}, {rm}")
2027            }
2028            &Inst::FpuRRI { fpu_op, rd, rn } => {
2029                let (op, imm, vector) = match fpu_op {
2030                    FPUOpRI::UShr32(imm) => ("ushr", imm.pretty_print(0), true),
2031                    FPUOpRI::UShr64(imm) => ("ushr", imm.pretty_print(0), false),
2032                };
2033
2034                let (rd, rn) = if vector {
2035                    (
2036                        pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
2037                        pretty_print_vreg_vector(rn, VectorSize::Size32x2),
2038                    )
2039                } else {
2040                    (
2041                        pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
2042                        pretty_print_vreg_scalar(rn, ScalarSize::Size64),
2043                    )
2044                };
2045                format!("{op} {rd}, {rn}, {imm}")
2046            }
2047            &Inst::FpuRRIMod { fpu_op, rd, ri, rn } => {
2048                let (op, imm, vector) = match fpu_op {
2049                    FPUOpRIMod::Sli32(imm) => ("sli", imm.pretty_print(0), true),
2050                    FPUOpRIMod::Sli64(imm) => ("sli", imm.pretty_print(0), false),
2051                };
2052
2053                let (rd, ri, rn) = if vector {
2054                    (
2055                        pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
2056                        pretty_print_vreg_vector(ri, VectorSize::Size32x2),
2057                        pretty_print_vreg_vector(rn, VectorSize::Size32x2),
2058                    )
2059                } else {
2060                    (
2061                        pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
2062                        pretty_print_vreg_scalar(ri, ScalarSize::Size64),
2063                        pretty_print_vreg_scalar(rn, ScalarSize::Size64),
2064                    )
2065                };
2066                format!("{op} {rd}, {ri}, {rn}, {imm}")
2067            }
2068            &Inst::FpuRRRR {
2069                fpu_op,
2070                size,
2071                rd,
2072                rn,
2073                rm,
2074                ra,
2075            } => {
2076                let op = match fpu_op {
2077                    FPUOp3::MAdd => "fmadd",
2078                    FPUOp3::MSub => "fmsub",
2079                    FPUOp3::NMAdd => "fnmadd",
2080                    FPUOp3::NMSub => "fnmsub",
2081                };
2082                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2083                let rn = pretty_print_vreg_scalar(rn, size);
2084                let rm = pretty_print_vreg_scalar(rm, size);
2085                let ra = pretty_print_vreg_scalar(ra, size);
2086                format!("{op} {rd}, {rn}, {rm}, {ra}")
2087            }
2088            &Inst::FpuCmp { size, rn, rm } => {
2089                let rn = pretty_print_vreg_scalar(rn, size);
2090                let rm = pretty_print_vreg_scalar(rm, size);
2091                format!("fcmp {rn}, {rm}")
2092            }
2093            &Inst::FpuLoad16 { rd, ref mem, .. } => {
2094                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
2095                let mem = mem.clone();
2096                let access_ty = self.mem_type().unwrap();
2097                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2098                format!("{mem_str}ldr {rd}, {mem}")
2099            }
2100            &Inst::FpuLoad32 { rd, ref mem, .. } => {
2101                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
2102                let mem = mem.clone();
2103                let access_ty = self.mem_type().unwrap();
2104                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2105                format!("{mem_str}ldr {rd}, {mem}")
2106            }
2107            &Inst::FpuLoad64 { rd, ref mem, .. } => {
2108                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2109                let mem = mem.clone();
2110                let access_ty = self.mem_type().unwrap();
2111                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2112                format!("{mem_str}ldr {rd}, {mem}")
2113            }
2114            &Inst::FpuLoad128 { rd, ref mem, .. } => {
2115                let rd = pretty_print_reg(rd.to_reg());
2116                let rd = "q".to_string() + &rd[1..];
2117                let mem = mem.clone();
2118                let access_ty = self.mem_type().unwrap();
2119                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2120                format!("{mem_str}ldr {rd}, {mem}")
2121            }
2122            &Inst::FpuStore16 { rd, ref mem, .. } => {
2123                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size16);
2124                let mem = mem.clone();
2125                let access_ty = self.mem_type().unwrap();
2126                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2127                format!("{mem_str}str {rd}, {mem}")
2128            }
2129            &Inst::FpuStore32 { rd, ref mem, .. } => {
2130                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size32);
2131                let mem = mem.clone();
2132                let access_ty = self.mem_type().unwrap();
2133                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2134                format!("{mem_str}str {rd}, {mem}")
2135            }
2136            &Inst::FpuStore64 { rd, ref mem, .. } => {
2137                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size64);
2138                let mem = mem.clone();
2139                let access_ty = self.mem_type().unwrap();
2140                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2141                format!("{mem_str}str {rd}, {mem}")
2142            }
2143            &Inst::FpuStore128 { rd, ref mem, .. } => {
2144                let rd = pretty_print_reg(rd);
2145                let rd = "q".to_string() + &rd[1..];
2146                let mem = mem.clone();
2147                let access_ty = self.mem_type().unwrap();
2148                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
2149                format!("{mem_str}str {rd}, {mem}")
2150            }
2151            &Inst::FpuLoadP64 {
2152                rt, rt2, ref mem, ..
2153            } => {
2154                let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size64);
2155                let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size64);
2156                let mem = mem.clone();
2157                let mem = mem.pretty_print_default();
2158
2159                format!("ldp {rt}, {rt2}, {mem}")
2160            }
2161            &Inst::FpuStoreP64 {
2162                rt, rt2, ref mem, ..
2163            } => {
2164                let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size64);
2165                let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size64);
2166                let mem = mem.clone();
2167                let mem = mem.pretty_print_default();
2168
2169                format!("stp {rt}, {rt2}, {mem}")
2170            }
2171            &Inst::FpuLoadP128 {
2172                rt, rt2, ref mem, ..
2173            } => {
2174                let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size128);
2175                let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size128);
2176                let mem = mem.clone();
2177                let mem = mem.pretty_print_default();
2178
2179                format!("ldp {rt}, {rt2}, {mem}")
2180            }
2181            &Inst::FpuStoreP128 {
2182                rt, rt2, ref mem, ..
2183            } => {
2184                let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size128);
2185                let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size128);
2186                let mem = mem.clone();
2187                let mem = mem.pretty_print_default();
2188
2189                format!("stp {rt}, {rt2}, {mem}")
2190            }
2191            &Inst::FpuToInt { op, rd, rn } => {
2192                let (op, sizesrc, sizedest) = match op {
2193                    FpuToIntOp::F32ToI32 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size32),
2194                    FpuToIntOp::F32ToU32 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size32),
2195                    FpuToIntOp::F32ToI64 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size64),
2196                    FpuToIntOp::F32ToU64 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size64),
2197                    FpuToIntOp::F64ToI32 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size32),
2198                    FpuToIntOp::F64ToU32 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size32),
2199                    FpuToIntOp::F64ToI64 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size64),
2200                    FpuToIntOp::F64ToU64 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size64),
2201                };
2202                let rd = pretty_print_ireg(rd.to_reg(), sizedest);
2203                let rn = pretty_print_vreg_scalar(rn, sizesrc);
2204                format!("{op} {rd}, {rn}")
2205            }
2206            &Inst::IntToFpu { op, rd, rn } => {
2207                let (op, sizesrc, sizedest) = match op {
2208                    IntToFpuOp::I32ToF32 => ("scvtf", OperandSize::Size32, ScalarSize::Size32),
2209                    IntToFpuOp::U32ToF32 => ("ucvtf", OperandSize::Size32, ScalarSize::Size32),
2210                    IntToFpuOp::I64ToF32 => ("scvtf", OperandSize::Size64, ScalarSize::Size32),
2211                    IntToFpuOp::U64ToF32 => ("ucvtf", OperandSize::Size64, ScalarSize::Size32),
2212                    IntToFpuOp::I32ToF64 => ("scvtf", OperandSize::Size32, ScalarSize::Size64),
2213                    IntToFpuOp::U32ToF64 => ("ucvtf", OperandSize::Size32, ScalarSize::Size64),
2214                    IntToFpuOp::I64ToF64 => ("scvtf", OperandSize::Size64, ScalarSize::Size64),
2215                    IntToFpuOp::U64ToF64 => ("ucvtf", OperandSize::Size64, ScalarSize::Size64),
2216                };
2217                let rd = pretty_print_vreg_scalar(rd.to_reg(), sizedest);
2218                let rn = pretty_print_ireg(rn, sizesrc);
2219                format!("{op} {rd}, {rn}")
2220            }
2221            &Inst::FpuCSel16 { rd, rn, rm, cond } => {
2222                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
2223                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size16);
2224                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size16);
2225                let cond = cond.pretty_print(0);
2226                format!("fcsel {rd}, {rn}, {rm}, {cond}")
2227            }
2228            &Inst::FpuCSel32 { rd, rn, rm, cond } => {
2229                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
2230                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
2231                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size32);
2232                let cond = cond.pretty_print(0);
2233                format!("fcsel {rd}, {rn}, {rm}, {cond}")
2234            }
2235            &Inst::FpuCSel64 { rd, rn, rm, cond } => {
2236                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2237                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
2238                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size64);
2239                let cond = cond.pretty_print(0);
2240                format!("fcsel {rd}, {rn}, {rm}, {cond}")
2241            }
2242            &Inst::FpuRound { op, rd, rn } => {
2243                let (inst, size) = match op {
2244                    FpuRoundMode::Minus32 => ("frintm", ScalarSize::Size32),
2245                    FpuRoundMode::Minus64 => ("frintm", ScalarSize::Size64),
2246                    FpuRoundMode::Plus32 => ("frintp", ScalarSize::Size32),
2247                    FpuRoundMode::Plus64 => ("frintp", ScalarSize::Size64),
2248                    FpuRoundMode::Zero32 => ("frintz", ScalarSize::Size32),
2249                    FpuRoundMode::Zero64 => ("frintz", ScalarSize::Size64),
2250                    FpuRoundMode::Nearest32 => ("frintn", ScalarSize::Size32),
2251                    FpuRoundMode::Nearest64 => ("frintn", ScalarSize::Size64),
2252                };
2253                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2254                let rn = pretty_print_vreg_scalar(rn, size);
2255                format!("{inst} {rd}, {rn}")
2256            }
2257            &Inst::MovToFpu { rd, rn, size } => {
2258                let operand_size = size.operand_size();
2259                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2260                let rn = pretty_print_ireg(rn, operand_size);
2261                format!("fmov {rd}, {rn}")
2262            }
2263            &Inst::FpuMoveFPImm { rd, imm, size } => {
2264                let imm = imm.pretty_print(0);
2265                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2266
2267                format!("fmov {rd}, {imm}")
2268            }
2269            &Inst::MovToVec {
2270                rd,
2271                ri,
2272                rn,
2273                idx,
2274                size,
2275            } => {
2276                let rd = pretty_print_vreg_element(rd.to_reg(), idx as usize, size.lane_size());
2277                let ri = pretty_print_vreg_element(ri, idx as usize, size.lane_size());
2278                let rn = pretty_print_ireg(rn, size.operand_size());
2279                format!("mov {rd}, {ri}, {rn}")
2280            }
2281            &Inst::MovFromVec { rd, rn, idx, size } => {
2282                let op = match size {
2283                    ScalarSize::Size8 => "umov",
2284                    ScalarSize::Size16 => "umov",
2285                    ScalarSize::Size32 => "mov",
2286                    ScalarSize::Size64 => "mov",
2287                    _ => unimplemented!(),
2288                };
2289                let rd = pretty_print_ireg(rd.to_reg(), size.operand_size());
2290                let rn = pretty_print_vreg_element(rn, idx as usize, size);
2291                format!("{op} {rd}, {rn}")
2292            }
2293            &Inst::MovFromVecSigned {
2294                rd,
2295                rn,
2296                idx,
2297                size,
2298                scalar_size,
2299            } => {
2300                let rd = pretty_print_ireg(rd.to_reg(), scalar_size);
2301                let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
2302                format!("smov {rd}, {rn}")
2303            }
2304            &Inst::VecDup { rd, rn, size } => {
2305                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2306                let rn = pretty_print_ireg(rn, size.operand_size());
2307                format!("dup {rd}, {rn}")
2308            }
2309            &Inst::VecDupFromFpu { rd, rn, size, lane } => {
2310                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2311                let rn = pretty_print_vreg_element(rn, lane.into(), size.lane_size());
2312                format!("dup {rd}, {rn}")
2313            }
2314            &Inst::VecDupFPImm { rd, imm, size } => {
2315                let imm = imm.pretty_print(0);
2316                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2317
2318                format!("fmov {rd}, {imm}")
2319            }
2320            &Inst::VecDupImm {
2321                rd,
2322                imm,
2323                invert,
2324                size,
2325            } => {
2326                let imm = imm.pretty_print(0);
2327                let op = if invert { "mvni" } else { "movi" };
2328                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2329
2330                format!("{op} {rd}, {imm}")
2331            }
2332            &Inst::VecExtend {
2333                t,
2334                rd,
2335                rn,
2336                high_half,
2337                lane_size,
2338            } => {
2339                let vec64 = VectorSize::from_lane_size(lane_size.narrow(), false);
2340                let vec128 = VectorSize::from_lane_size(lane_size.narrow(), true);
2341                let rd_size = VectorSize::from_lane_size(lane_size, true);
2342                let (op, rn_size) = match (t, high_half) {
2343                    (VecExtendOp::Sxtl, false) => ("sxtl", vec64),
2344                    (VecExtendOp::Sxtl, true) => ("sxtl2", vec128),
2345                    (VecExtendOp::Uxtl, false) => ("uxtl", vec64),
2346                    (VecExtendOp::Uxtl, true) => ("uxtl2", vec128),
2347                };
2348                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2349                let rn = pretty_print_vreg_vector(rn, rn_size);
2350                format!("{op} {rd}, {rn}")
2351            }
2352            &Inst::VecMovElement {
2353                rd,
2354                ri,
2355                rn,
2356                dest_idx,
2357                src_idx,
2358                size,
2359            } => {
2360                let rd =
2361                    pretty_print_vreg_element(rd.to_reg(), dest_idx as usize, size.lane_size());
2362                let ri = pretty_print_vreg_element(ri, dest_idx as usize, size.lane_size());
2363                let rn = pretty_print_vreg_element(rn, src_idx as usize, size.lane_size());
2364                format!("mov {rd}, {ri}, {rn}")
2365            }
2366            &Inst::VecRRLong {
2367                op,
2368                rd,
2369                rn,
2370                high_half,
2371            } => {
2372                let (op, rd_size, size, suffix) = match (op, high_half) {
2373                    (VecRRLongOp::Fcvtl16, false) => {
2374                        ("fcvtl", VectorSize::Size32x4, VectorSize::Size16x4, "")
2375                    }
2376                    (VecRRLongOp::Fcvtl16, true) => {
2377                        ("fcvtl2", VectorSize::Size32x4, VectorSize::Size16x8, "")
2378                    }
2379                    (VecRRLongOp::Fcvtl32, false) => {
2380                        ("fcvtl", VectorSize::Size64x2, VectorSize::Size32x2, "")
2381                    }
2382                    (VecRRLongOp::Fcvtl32, true) => {
2383                        ("fcvtl2", VectorSize::Size64x2, VectorSize::Size32x4, "")
2384                    }
2385                    (VecRRLongOp::Shll8, false) => {
2386                        ("shll", VectorSize::Size16x8, VectorSize::Size8x8, ", #8")
2387                    }
2388                    (VecRRLongOp::Shll8, true) => {
2389                        ("shll2", VectorSize::Size16x8, VectorSize::Size8x16, ", #8")
2390                    }
2391                    (VecRRLongOp::Shll16, false) => {
2392                        ("shll", VectorSize::Size32x4, VectorSize::Size16x4, ", #16")
2393                    }
2394                    (VecRRLongOp::Shll16, true) => {
2395                        ("shll2", VectorSize::Size32x4, VectorSize::Size16x8, ", #16")
2396                    }
2397                    (VecRRLongOp::Shll32, false) => {
2398                        ("shll", VectorSize::Size64x2, VectorSize::Size32x2, ", #32")
2399                    }
2400                    (VecRRLongOp::Shll32, true) => {
2401                        ("shll2", VectorSize::Size64x2, VectorSize::Size32x4, ", #32")
2402                    }
2403                };
2404                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2405                let rn = pretty_print_vreg_vector(rn, size);
2406
2407                format!("{op} {rd}, {rn}{suffix}")
2408            }
2409            &Inst::VecRRNarrowLow {
2410                op,
2411                rd,
2412                rn,
2413                lane_size,
2414                ..
2415            }
2416            | &Inst::VecRRNarrowHigh {
2417                op,
2418                rd,
2419                rn,
2420                lane_size,
2421                ..
2422            } => {
2423                let vec64 = VectorSize::from_lane_size(lane_size, false);
2424                let vec128 = VectorSize::from_lane_size(lane_size, true);
2425                let rn_size = VectorSize::from_lane_size(lane_size.widen(), true);
2426                let high_half = match self {
2427                    &Inst::VecRRNarrowLow { .. } => false,
2428                    &Inst::VecRRNarrowHigh { .. } => true,
2429                    _ => unreachable!(),
2430                };
2431                let (op, rd_size) = match (op, high_half) {
2432                    (VecRRNarrowOp::Xtn, false) => ("xtn", vec64),
2433                    (VecRRNarrowOp::Xtn, true) => ("xtn2", vec128),
2434                    (VecRRNarrowOp::Sqxtn, false) => ("sqxtn", vec64),
2435                    (VecRRNarrowOp::Sqxtn, true) => ("sqxtn2", vec128),
2436                    (VecRRNarrowOp::Sqxtun, false) => ("sqxtun", vec64),
2437                    (VecRRNarrowOp::Sqxtun, true) => ("sqxtun2", vec128),
2438                    (VecRRNarrowOp::Uqxtn, false) => ("uqxtn", vec64),
2439                    (VecRRNarrowOp::Uqxtn, true) => ("uqxtn2", vec128),
2440                    (VecRRNarrowOp::Fcvtn, false) => ("fcvtn", vec64),
2441                    (VecRRNarrowOp::Fcvtn, true) => ("fcvtn2", vec128),
2442                };
2443                let rn = pretty_print_vreg_vector(rn, rn_size);
2444                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2445                let ri = match self {
2446                    &Inst::VecRRNarrowLow { .. } => "".to_string(),
2447                    &Inst::VecRRNarrowHigh { ri, .. } => {
2448                        format!("{}, ", pretty_print_vreg_vector(ri, rd_size))
2449                    }
2450                    _ => unreachable!(),
2451                };
2452
2453                format!("{op} {rd}, {ri}{rn}")
2454            }
2455            &Inst::VecRRPair { op, rd, rn } => {
2456                let op = match op {
2457                    VecPairOp::Addp => "addp",
2458                };
2459                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2460                let rn = pretty_print_vreg_vector(rn, VectorSize::Size64x2);
2461
2462                format!("{op} {rd}, {rn}")
2463            }
2464            &Inst::VecRRPairLong { op, rd, rn } => {
2465                let (op, dest, src) = match op {
2466                    VecRRPairLongOp::Saddlp8 => {
2467                        ("saddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2468                    }
2469                    VecRRPairLongOp::Saddlp16 => {
2470                        ("saddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2471                    }
2472                    VecRRPairLongOp::Uaddlp8 => {
2473                        ("uaddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2474                    }
2475                    VecRRPairLongOp::Uaddlp16 => {
2476                        ("uaddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2477                    }
2478                };
2479                let rd = pretty_print_vreg_vector(rd.to_reg(), dest);
2480                let rn = pretty_print_vreg_vector(rn, src);
2481
2482                format!("{op} {rd}, {rn}")
2483            }
2484            &Inst::VecRRR {
2485                rd,
2486                rn,
2487                rm,
2488                alu_op,
2489                size,
2490            } => {
2491                let (op, size) = match alu_op {
2492                    VecALUOp::Sqadd => ("sqadd", size),
2493                    VecALUOp::Uqadd => ("uqadd", size),
2494                    VecALUOp::Sqsub => ("sqsub", size),
2495                    VecALUOp::Uqsub => ("uqsub", size),
2496                    VecALUOp::Cmeq => ("cmeq", size),
2497                    VecALUOp::Cmge => ("cmge", size),
2498                    VecALUOp::Cmgt => ("cmgt", size),
2499                    VecALUOp::Cmhs => ("cmhs", size),
2500                    VecALUOp::Cmhi => ("cmhi", size),
2501                    VecALUOp::Fcmeq => ("fcmeq", size),
2502                    VecALUOp::Fcmgt => ("fcmgt", size),
2503                    VecALUOp::Fcmge => ("fcmge", size),
2504                    VecALUOp::Umaxp => ("umaxp", size),
2505                    VecALUOp::Add => ("add", size),
2506                    VecALUOp::Sub => ("sub", size),
2507                    VecALUOp::Mul => ("mul", size),
2508                    VecALUOp::Sshl => ("sshl", size),
2509                    VecALUOp::Ushl => ("ushl", size),
2510                    VecALUOp::Umin => ("umin", size),
2511                    VecALUOp::Smin => ("smin", size),
2512                    VecALUOp::Umax => ("umax", size),
2513                    VecALUOp::Smax => ("smax", size),
2514                    VecALUOp::Urhadd => ("urhadd", size),
2515                    VecALUOp::Fadd => ("fadd", size),
2516                    VecALUOp::Fsub => ("fsub", size),
2517                    VecALUOp::Fdiv => ("fdiv", size),
2518                    VecALUOp::Fmax => ("fmax", size),
2519                    VecALUOp::Fmin => ("fmin", size),
2520                    VecALUOp::Fmul => ("fmul", size),
2521                    VecALUOp::Addp => ("addp", size),
2522                    VecALUOp::Zip1 => ("zip1", size),
2523                    VecALUOp::Zip2 => ("zip2", size),
2524                    VecALUOp::Sqrdmulh => ("sqrdmulh", size),
2525                    VecALUOp::Uzp1 => ("uzp1", size),
2526                    VecALUOp::Uzp2 => ("uzp2", size),
2527                    VecALUOp::Trn1 => ("trn1", size),
2528                    VecALUOp::Trn2 => ("trn2", size),
2529
2530                    // Lane division does not affect bitwise operations.
2531                    // However, when printing, use 8-bit lane division to conform to ARM formatting.
2532                    VecALUOp::And => ("and", size.as_scalar8_vector()),
2533                    VecALUOp::Bic => ("bic", size.as_scalar8_vector()),
2534                    VecALUOp::Orr => ("orr", size.as_scalar8_vector()),
2535                    VecALUOp::Orn => ("orn", size.as_scalar8_vector()),
2536                    VecALUOp::Eor => ("eor", size.as_scalar8_vector()),
2537                };
2538                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2539                let rn = pretty_print_vreg_vector(rn, size);
2540                let rm = pretty_print_vreg_vector(rm, size);
2541                format!("{op} {rd}, {rn}, {rm}")
2542            }
2543            &Inst::VecRRRMod {
2544                rd,
2545                ri,
2546                rn,
2547                rm,
2548                alu_op,
2549                size,
2550            } => {
2551                let (op, size) = match alu_op {
2552                    VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
2553                    VecALUModOp::Fmla => ("fmla", size),
2554                    VecALUModOp::Fmls => ("fmls", size),
2555                    // Note: the real operand arrangement is .4s, .16b, .16b;
2556                    // this debug print renders all lanes as .4s.
2557                    VecALUModOp::Sdot => ("sdot", VectorSize::Size32x4),
2558                    VecALUModOp::Usdot => ("usdot", VectorSize::Size32x4),
2559                };
2560                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2561                let ri = pretty_print_vreg_vector(ri, size);
2562                let rn = pretty_print_vreg_vector(rn, size);
2563                let rm = pretty_print_vreg_vector(rm, size);
2564                format!("{op} {rd}, {ri}, {rn}, {rm}")
2565            }
2566            &Inst::VecFmlaElem {
2567                rd,
2568                ri,
2569                rn,
2570                rm,
2571                alu_op,
2572                size,
2573                idx,
2574            } => {
2575                let (op, size) = match alu_op {
2576                    VecALUModOp::Fmla => ("fmla", size),
2577                    VecALUModOp::Fmls => ("fmls", size),
2578                    _ => unreachable!(),
2579                };
2580                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2581                let ri = pretty_print_vreg_vector(ri, size);
2582                let rn = pretty_print_vreg_vector(rn, size);
2583                let rm = pretty_print_vreg_element(rm, idx.into(), size.lane_size());
2584                format!("{op} {rd}, {ri}, {rn}, {rm}")
2585            }
2586            &Inst::VecRRRLong {
2587                rd,
2588                rn,
2589                rm,
2590                alu_op,
2591                high_half,
2592            } => {
2593                let (op, dest_size, src_size) = match (alu_op, high_half) {
2594                    (VecRRRLongOp::Smull8, false) => {
2595                        ("smull", VectorSize::Size16x8, VectorSize::Size8x8)
2596                    }
2597                    (VecRRRLongOp::Smull8, true) => {
2598                        ("smull2", VectorSize::Size16x8, VectorSize::Size8x16)
2599                    }
2600                    (VecRRRLongOp::Smull16, false) => {
2601                        ("smull", VectorSize::Size32x4, VectorSize::Size16x4)
2602                    }
2603                    (VecRRRLongOp::Smull16, true) => {
2604                        ("smull2", VectorSize::Size32x4, VectorSize::Size16x8)
2605                    }
2606                    (VecRRRLongOp::Smull32, false) => {
2607                        ("smull", VectorSize::Size64x2, VectorSize::Size32x2)
2608                    }
2609                    (VecRRRLongOp::Smull32, true) => {
2610                        ("smull2", VectorSize::Size64x2, VectorSize::Size32x4)
2611                    }
2612                    (VecRRRLongOp::Umull8, false) => {
2613                        ("umull", VectorSize::Size16x8, VectorSize::Size8x8)
2614                    }
2615                    (VecRRRLongOp::Umull8, true) => {
2616                        ("umull2", VectorSize::Size16x8, VectorSize::Size8x16)
2617                    }
2618                    (VecRRRLongOp::Umull16, false) => {
2619                        ("umull", VectorSize::Size32x4, VectorSize::Size16x4)
2620                    }
2621                    (VecRRRLongOp::Umull16, true) => {
2622                        ("umull2", VectorSize::Size32x4, VectorSize::Size16x8)
2623                    }
2624                    (VecRRRLongOp::Umull32, false) => {
2625                        ("umull", VectorSize::Size64x2, VectorSize::Size32x2)
2626                    }
2627                    (VecRRRLongOp::Umull32, true) => {
2628                        ("umull2", VectorSize::Size64x2, VectorSize::Size32x4)
2629                    }
2630                };
2631                let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2632                let rn = pretty_print_vreg_vector(rn, src_size);
2633                let rm = pretty_print_vreg_vector(rm, src_size);
2634                format!("{op} {rd}, {rn}, {rm}")
2635            }
2636            &Inst::VecRRRLongMod {
2637                rd,
2638                ri,
2639                rn,
2640                rm,
2641                alu_op,
2642                high_half,
2643            } => {
2644                let (op, dest_size, src_size) = match (alu_op, high_half) {
2645                    (VecRRRLongModOp::Umlal8, false) => {
2646                        ("umlal", VectorSize::Size16x8, VectorSize::Size8x8)
2647                    }
2648                    (VecRRRLongModOp::Umlal8, true) => {
2649                        ("umlal2", VectorSize::Size16x8, VectorSize::Size8x16)
2650                    }
2651                    (VecRRRLongModOp::Umlal16, false) => {
2652                        ("umlal", VectorSize::Size32x4, VectorSize::Size16x4)
2653                    }
2654                    (VecRRRLongModOp::Umlal16, true) => {
2655                        ("umlal2", VectorSize::Size32x4, VectorSize::Size16x8)
2656                    }
2657                    (VecRRRLongModOp::Umlal32, false) => {
2658                        ("umlal", VectorSize::Size64x2, VectorSize::Size32x2)
2659                    }
2660                    (VecRRRLongModOp::Umlal32, true) => {
2661                        ("umlal2", VectorSize::Size64x2, VectorSize::Size32x4)
2662                    }
2663                };
2664                let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2665                let ri = pretty_print_vreg_vector(ri, dest_size);
2666                let rn = pretty_print_vreg_vector(rn, src_size);
2667                let rm = pretty_print_vreg_vector(rm, src_size);
2668                format!("{op} {rd}, {ri}, {rn}, {rm}")
2669            }
2670            &Inst::VecMisc { op, rd, rn, size } => {
2671                let (op, size, suffix) = match op {
2672                    VecMisc2::Neg => ("neg", size, ""),
2673                    VecMisc2::Abs => ("abs", size, ""),
2674                    VecMisc2::Fabs => ("fabs", size, ""),
2675                    VecMisc2::Fneg => ("fneg", size, ""),
2676                    VecMisc2::Fsqrt => ("fsqrt", size, ""),
2677                    VecMisc2::Rev16 => ("rev16", size, ""),
2678                    VecMisc2::Rev32 => ("rev32", size, ""),
2679                    VecMisc2::Rev64 => ("rev64", size, ""),
2680                    VecMisc2::Fcvtzs => ("fcvtzs", size, ""),
2681                    VecMisc2::Fcvtzu => ("fcvtzu", size, ""),
2682                    VecMisc2::Scvtf => ("scvtf", size, ""),
2683                    VecMisc2::Ucvtf => ("ucvtf", size, ""),
2684                    VecMisc2::Frintn => ("frintn", size, ""),
2685                    VecMisc2::Frintz => ("frintz", size, ""),
2686                    VecMisc2::Frintm => ("frintm", size, ""),
2687                    VecMisc2::Frintp => ("frintp", size, ""),
2688                    VecMisc2::Cnt => ("cnt", size, ""),
2689                    VecMisc2::Cmeq0 => ("cmeq", size, ", #0"),
2690                    VecMisc2::Cmge0 => ("cmge", size, ", #0"),
2691                    VecMisc2::Cmgt0 => ("cmgt", size, ", #0"),
2692                    VecMisc2::Cmle0 => ("cmle", size, ", #0"),
2693                    VecMisc2::Cmlt0 => ("cmlt", size, ", #0"),
2694                    VecMisc2::Fcmeq0 => ("fcmeq", size, ", #0.0"),
2695                    VecMisc2::Fcmge0 => ("fcmge", size, ", #0.0"),
2696                    VecMisc2::Fcmgt0 => ("fcmgt", size, ", #0.0"),
2697                    VecMisc2::Fcmle0 => ("fcmle", size, ", #0.0"),
2698                    VecMisc2::Fcmlt0 => ("fcmlt", size, ", #0.0"),
2699
2700                    // Lane division does not affect bitwise operations.
2701                    // However, when printing, use 8-bit lane division to conform to ARM formatting.
2702                    VecMisc2::Not => ("mvn", size.as_scalar8_vector(), ""),
2703                };
2704                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2705                let rn = pretty_print_vreg_vector(rn, size);
2706                format!("{op} {rd}, {rn}{suffix}")
2707            }
2708            &Inst::VecLanes { op, rd, rn, size } => {
2709                let op = match op {
2710                    VecLanesOp::Uminv => "uminv",
2711                    VecLanesOp::Addv => "addv",
2712                };
2713                let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
2714                let rn = pretty_print_vreg_vector(rn, size);
2715                format!("{op} {rd}, {rn}")
2716            }
2717            &Inst::VecShiftImm {
2718                op,
2719                rd,
2720                rn,
2721                size,
2722                imm,
2723            } => {
2724                let op = match op {
2725                    VecShiftImmOp::Shl => "shl",
2726                    VecShiftImmOp::Ushr => "ushr",
2727                    VecShiftImmOp::Sshr => "sshr",
2728                };
2729                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2730                let rn = pretty_print_vreg_vector(rn, size);
2731                format!("{op} {rd}, {rn}, #{imm}")
2732            }
2733            &Inst::VecShiftImmMod {
2734                op,
2735                rd,
2736                ri,
2737                rn,
2738                size,
2739                imm,
2740            } => {
2741                let op = match op {
2742                    VecShiftImmModOp::Sli => "sli",
2743                };
2744                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2745                let ri = pretty_print_vreg_vector(ri, size);
2746                let rn = pretty_print_vreg_vector(rn, size);
2747                format!("{op} {rd}, {ri}, {rn}, #{imm}")
2748            }
2749            &Inst::VecExtract { rd, rn, rm, imm4 } => {
2750                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2751                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2752                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2753                format!("ext {rd}, {rn}, {rm}, #{imm4}")
2754            }
2755            &Inst::VecTbl { rd, rn, rm } => {
2756                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2757                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2758                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2759                format!("tbl {rd}, {{ {rn} }}, {rm}")
2760            }
2761            &Inst::VecTblExt { rd, ri, rn, rm } => {
2762                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2763                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2764                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2765                let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2766                format!("tbx {rd}, {ri}, {{ {rn} }}, {rm}")
2767            }
2768            &Inst::VecTbl2 { rd, rn, rn2, rm } => {
2769                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2770                let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2771                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2772                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2773                format!("tbl {rd}, {{ {rn}, {rn2} }}, {rm}")
2774            }
2775            &Inst::VecTbl2Ext {
2776                rd,
2777                ri,
2778                rn,
2779                rn2,
2780                rm,
2781            } => {
2782                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2783                let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2784                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2785                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2786                let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2787                format!("tbx {rd}, {ri}, {{ {rn}, {rn2} }}, {rm}")
2788            }
2789            &Inst::VecLoadReplicate { rd, rn, size, .. } => {
2790                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2791                let rn = pretty_print_reg(rn);
2792
2793                format!("ld1r {{ {rd} }}, [{rn}]")
2794            }
2795            &Inst::VecCSel { rd, rn, rm, cond } => {
2796                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2797                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2798                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2799                let cond = cond.pretty_print(0);
2800                format!("vcsel {rd}, {rn}, {rm}, {cond} (if-then-else diamond)")
2801            }
2802            &Inst::MovToNZCV { rn } => {
2803                let rn = pretty_print_reg(rn);
2804                format!("msr nzcv, {rn}")
2805            }
2806            &Inst::MovFromNZCV { rd } => {
2807                let rd = pretty_print_reg(rd.to_reg());
2808                format!("mrs {rd}, nzcv")
2809            }
2810            // Special case when `immr == 0` so that zero- and sign-extending
2811            // operations use the appropriate aliases, preserving the behavior
2812            // of the old `Inst::Extend` instruction.
2813            &Inst::BitfieldMove {
2814                size,
2815                bfm_op,
2816                rd,
2817                rn,
2818                immr: UImm6 { imm: 0 },
2819                imms,
2820            } => {
2821                let from_bits = imms.value() + 1;
2822                let op = match (bfm_op, size, from_bits) {
2823                    (BfmOp::UBfm, OperandSize::Size32, 8) => "uxtb",
2824                    (BfmOp::UBfm, OperandSize::Size32, 16) => "uxth",
2825                    (BfmOp::SBfm, _, 8) => "sxtb",
2826                    (BfmOp::SBfm, _, 16) => "sxth",
2827                    (BfmOp::SBfm, OperandSize::Size64, 32) => "sxtw",
2828                    (BfmOp::UBfm, _, _) => "ubfx",
2829                    (BfmOp::SBfm, _, _) => "sbfx",
2830                };
2831                if op == "sbfx" || op == "ubfx" {
2832                    let rd = pretty_print_ireg(rd.to_reg(), size);
2833                    let rn = pretty_print_ireg(rn, size);
2834                    format!("{op} {rd}, {rn}, #0, #{from_bits}")
2835                } else {
2836                    let rd = pretty_print_ireg(rd.to_reg(), size);
2837                    let rn = pretty_print_ireg(rn, OperandSize::Size32);
2838                    format!("{op} {rd}, {rn}")
2839                }
2840            }
2841            &Inst::BitfieldMove {
2842                size,
2843                bfm_op,
2844                rd,
2845                rn,
2846                immr,
2847                imms,
2848            } => {
2849                let op = bfm_op.op_str();
2850                let rd = pretty_print_ireg(rd.to_reg(), size);
2851                let rn = pretty_print_ireg(rn, size);
2852                let immr = immr.pretty_print(0);
2853                let imms = imms.pretty_print(0);
2854                format!("{op} {rd}, {rn}, {immr}, {imms}")
2855            }
2856            &Inst::BitfieldMoveMod {
2857                size,
2858                rd,
2859                ri,
2860                rn,
2861                immr,
2862                imms,
2863            } => {
2864                let rd = pretty_print_ireg(rd.to_reg(), size);
2865                let ri = pretty_print_ireg(ri, size);
2866                let rn = pretty_print_ireg(rn, size);
2867                let immr = immr.pretty_print(0);
2868                let imms = imms.pretty_print(0);
2869                format!("bfm {rd}, {ri}, {rn}, {immr}, {imms}")
2870            }
2871            &Inst::Call { ref info } => {
2872                let try_call = info
2873                    .try_call_info
2874                    .as_ref()
2875                    .map(|tci| pretty_print_try_call(tci))
2876                    .unwrap_or_default();
2877                format!("bl 0{try_call}")
2878            }
2879            &Inst::CallInd { ref info } => {
2880                let rn = pretty_print_reg(info.dest);
2881                let try_call = info
2882                    .try_call_info
2883                    .as_ref()
2884                    .map(|tci| pretty_print_try_call(tci))
2885                    .unwrap_or_default();
2886                format!("blr {rn}{try_call}")
2887            }
2888            &Inst::ReturnCall { ref info } => {
2889                let mut s = format!(
2890                    "return_call {:?} new_stack_arg_size:{}",
2891                    info.dest, info.new_stack_arg_size
2892                );
2893                for ret in &info.uses {
2894                    let preg = pretty_print_reg(ret.preg);
2895                    let vreg = pretty_print_reg(ret.vreg);
2896                    write!(&mut s, " {vreg}={preg}").unwrap();
2897                }
2898                s
2899            }
2900            &Inst::ReturnCallInd { ref info } => {
2901                let callee = pretty_print_reg(info.dest);
2902                let mut s = format!(
2903                    "return_call_ind {callee} new_stack_arg_size:{}",
2904                    info.new_stack_arg_size
2905                );
2906                for ret in &info.uses {
2907                    let preg = pretty_print_reg(ret.preg);
2908                    let vreg = pretty_print_reg(ret.vreg);
2909                    write!(&mut s, " {vreg}={preg}").unwrap();
2910                }
2911                s
2912            }
2913            &Inst::Args { ref args } => {
2914                let mut s = "args".to_string();
2915                for arg in args {
2916                    let preg = pretty_print_reg(arg.preg);
2917                    let def = pretty_print_reg(arg.vreg.to_reg());
2918                    write!(&mut s, " {def}={preg}").unwrap();
2919                }
2920                s
2921            }
2922            &Inst::Rets { ref rets } => {
2923                let mut s = "rets".to_string();
2924                for ret in rets {
2925                    let preg = pretty_print_reg(ret.preg);
2926                    let vreg = pretty_print_reg(ret.vreg);
2927                    write!(&mut s, " {vreg}={preg}").unwrap();
2928                }
2929                s
2930            }
2931            &Inst::Ret {} => "ret".to_string(),
2932            &Inst::AuthenticatedRet { key, is_hint } => {
2933                let key = match key {
2934                    APIKey::AZ => "az",
2935                    APIKey::BZ => "bz",
2936                    APIKey::ASP => "asp",
2937                    APIKey::BSP => "bsp",
2938                };
2939                match is_hint {
2940                    false => format!("reta{key}"),
2941                    true => format!("auti{key} ; ret"),
2942                }
2943            }
2944            &Inst::Jump { ref dest } => {
2945                let dest = dest.pretty_print(0);
2946                format!("b {dest}")
2947            }
2948            &Inst::CondBr {
2949                ref taken,
2950                ref not_taken,
2951                ref kind,
2952            } => {
2953                let taken = taken.pretty_print(0);
2954                let not_taken = not_taken.pretty_print(0);
2955                match kind {
2956                    &CondBrKind::Zero(reg, size) => {
2957                        let reg = pretty_print_reg_sized(reg, size);
2958                        format!("cbz {reg}, {taken} ; b {not_taken}")
2959                    }
2960                    &CondBrKind::NotZero(reg, size) => {
2961                        let reg = pretty_print_reg_sized(reg, size);
2962                        format!("cbnz {reg}, {taken} ; b {not_taken}")
2963                    }
2964                    &CondBrKind::Cond(c) => {
2965                        let c = c.pretty_print(0);
2966                        format!("b.{c} {taken} ; b {not_taken}")
2967                    }
2968                }
2969            }
2970            &Inst::TestBitAndBranch {
2971                kind,
2972                ref taken,
2973                ref not_taken,
2974                rn,
2975                bit,
2976            } => {
2977                let cond = match kind {
2978                    TestBitAndBranchKind::Z => "z",
2979                    TestBitAndBranchKind::NZ => "nz",
2980                };
2981                let taken = taken.pretty_print(0);
2982                let not_taken = not_taken.pretty_print(0);
2983                let rn = pretty_print_reg(rn);
2984                format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
2985            }
2986            &Inst::IndirectBr { rn, .. } => {
2987                let rn = pretty_print_reg(rn);
2988                format!("br {rn}")
2989            }
2990            &Inst::Brk => "brk #0xf000".to_string(),
2991            &Inst::Udf { .. } => "udf #0xc11f".to_string(),
2992            &Inst::TrapIf {
2993                ref kind,
2994                trap_code,
2995            } => match kind {
2996                &CondBrKind::Zero(reg, size) => {
2997                    let reg = pretty_print_reg_sized(reg, size);
2998                    format!("cbz {reg}, #trap={trap_code}")
2999                }
3000                &CondBrKind::NotZero(reg, size) => {
3001                    let reg = pretty_print_reg_sized(reg, size);
3002                    format!("cbnz {reg}, #trap={trap_code}")
3003                }
3004                &CondBrKind::Cond(c) => {
3005                    let c = c.pretty_print(0);
3006                    format!("b.{c} #trap={trap_code}")
3007                }
3008            },
3009            &Inst::Adr { rd, off } => {
3010                let rd = pretty_print_reg(rd.to_reg());
3011                format!("adr {rd}, pc+{off}")
3012            }
3013            &Inst::Adrp { rd, off } => {
3014                let rd = pretty_print_reg(rd.to_reg());
3015                // This instruction addresses 4KiB pages, so multiply it by the page size.
3016                let byte_offset = off * 4096;
3017                format!("adrp {rd}, pc+{byte_offset}")
3018            }
3019            &Inst::Word4 { data } => format!("data.i32 {data}"),
3020            &Inst::Word8 { data } => format!("data.i64 {data}"),
3021            &Inst::JTSequence {
3022                default,
3023                ref targets,
3024                ridx,
3025                rtmp1,
3026                rtmp2,
3027                ..
3028            } => {
3029                let ridx = pretty_print_reg(ridx);
3030                let rtmp1 = pretty_print_reg(rtmp1.to_reg());
3031                let rtmp2 = pretty_print_reg(rtmp2.to_reg());
3032                let default_target = BranchTarget::Label(default).pretty_print(0);
3033                let num_targets = targets.len();
3034                format!(
3035                    "mov {rtmp2}, #{num_targets} ; \
3036                     csel {rtmp2}, {ridx}, {rtmp2}, lo ; \
3037                     csdb ; \
3038                     adr {rtmp1}, pc+16 ; \
3039                     ldrsw {rtmp2}, [{rtmp1}, {rtmp2}, uxtw #2] ; \
3040                     add {rtmp1}, {rtmp1}, {rtmp2} ; \
3041                     br {rtmp1} ; \
3042                     jt_entries {targets:?} ; \
3043                     default {default_target}"
3044                )
3045            }
3046            &Inst::LoadExtNameGot { rd, ref name } => {
3047                let rd = pretty_print_reg(rd.to_reg());
3048                format!("load_ext_name_got {rd}, {name:?}")
3049            }
3050            &Inst::LoadExtNameNear {
3051                rd,
3052                ref name,
3053                offset,
3054            } => {
3055                let rd = pretty_print_reg(rd.to_reg());
3056                format!("load_ext_name_near {rd}, {name:?}+{offset}")
3057            }
3058            &Inst::LoadExtNameFar {
3059                rd,
3060                ref name,
3061                offset,
3062            } => {
3063                let rd = pretty_print_reg(rd.to_reg());
3064                format!("load_ext_name_far {rd}, {name:?}+{offset}")
3065            }
3066            &Inst::LoadAddr { rd, ref mem } => {
3067                // TODO: we really should find a better way to avoid duplication of
3068                // this logic between `emit()` and `show_rru()` -- a separate 1-to-N
3069                // expansion stage (i.e., legalization, but without the slow edit-in-place
3070                // of the existing legalization framework).
3071                let mem = mem.clone();
3072                let (mem_insts, mem) = mem_finalize(None, &mem, I8, state);
3073                let mut ret = String::new();
3074                for inst in mem_insts.into_iter() {
3075                    ret.push_str(&inst.print_with_state(&mut EmitState::default()));
3076                }
3077                let (reg, index_reg, offset) = match mem {
3078                    AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0),
3079                    AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()),
3080                    AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32),
3081                    _ => panic!("Unsupported case for LoadAddr: {mem:?}"),
3082                };
3083                let abs_offset = if offset < 0 {
3084                    -offset as u64
3085                } else {
3086                    offset as u64
3087                };
3088                let alu_op = if offset < 0 { ALUOp::Sub } else { ALUOp::Add };
3089
3090                if let Some((idx, extendop)) = index_reg {
3091                    let add = Inst::AluRRRExtend {
3092                        alu_op: ALUOp::Add,
3093                        size: OperandSize::Size64,
3094                        rd,
3095                        rn: reg,
3096                        rm: idx,
3097                        extendop,
3098                    };
3099
3100                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
3101                } else if offset == 0 {
3102                    let mov = Inst::gen_move(rd, reg, I64);
3103                    ret.push_str(&mov.print_with_state(&mut EmitState::default()));
3104                } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
3105                    let add = Inst::AluRRImm12 {
3106                        alu_op,
3107                        size: OperandSize::Size64,
3108                        rd,
3109                        rn: reg,
3110                        imm12,
3111                    };
3112                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
3113                } else {
3114                    let tmp = writable_spilltmp_reg();
3115                    for inst in Inst::load_constant(tmp, abs_offset).into_iter() {
3116                        ret.push_str(&inst.print_with_state(&mut EmitState::default()));
3117                    }
3118                    let add = Inst::AluRRR {
3119                        alu_op,
3120                        size: OperandSize::Size64,
3121                        rd,
3122                        rn: reg,
3123                        rm: tmp.to_reg(),
3124                    };
3125                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
3126                }
3127                ret
3128            }
3129            &Inst::Paci { key } => {
3130                let key = match key {
3131                    APIKey::AZ => "az",
3132                    APIKey::BZ => "bz",
3133                    APIKey::ASP => "asp",
3134                    APIKey::BSP => "bsp",
3135                };
3136
3137                "paci".to_string() + key
3138            }
3139            &Inst::Xpaclri => "xpaclri".to_string(),
3140            &Inst::Bti { targets } => {
3141                let targets = match targets {
3142                    BranchTargetType::None => "",
3143                    BranchTargetType::C => " c",
3144                    BranchTargetType::J => " j",
3145                    BranchTargetType::JC => " jc",
3146                };
3147
3148                "bti".to_string() + targets
3149            }
3150            &Inst::EmitIsland { needed_space } => format!("emit_island {needed_space}"),
3151
3152            &Inst::ElfTlsGetAddr {
3153                ref symbol,
3154                rd,
3155                tmp,
3156            } => {
3157                let rd = pretty_print_reg(rd.to_reg());
3158                let tmp = pretty_print_reg(tmp.to_reg());
3159                format!("elf_tls_get_addr {}, {}, {}", rd, tmp, symbol.display(None))
3160            }
3161            &Inst::MachOTlsGetAddr { ref symbol, rd } => {
3162                let rd = pretty_print_reg(rd.to_reg());
3163                format!("macho_tls_get_addr {}, {}", rd, symbol.display(None))
3164            }
3165            &Inst::Unwind { ref inst } => {
3166                format!("unwind {inst:?}")
3167            }
3168            &Inst::DummyUse { reg } => {
3169                let reg = pretty_print_reg(reg);
3170                format!("dummy_use {reg}")
3171            }
3172            &Inst::LabelAddress { dst, label } => {
3173                let dst = pretty_print_reg(dst.to_reg());
3174                format!("label_address {dst}, {label:?}")
3175            }
3176            &Inst::SequencePoint {} => {
3177                format!("sequence_point")
3178            }
3179            &Inst::StackProbeLoop { start, end, step } => {
3180                let start = pretty_print_reg(start.to_reg());
3181                let end = pretty_print_reg(end);
3182                let step = step.pretty_print(0);
3183                format!("stack_probe_loop {start}, {end}, {step}")
3184            }
3185            &Inst::InterruptPoll {
3186                dst,
3187                load_ptr,
3188                context,
3189                trap_code,
3190                next_load_ptr,
3191            } => {
3192                let dst = pretty_print_reg(dst.to_reg());
3193                let load_ptr = pretty_print_reg(load_ptr);
3194                let context = pretty_print_reg(context);
3195                let next_load_ptr = pretty_print_reg(next_load_ptr.to_reg());
3196                format!(
3197                    "{next_load_ptr} = interrupt_poll load_ptr={load_ptr}, context={context}, scratch={dst} #trap={trap_code}"
3198                )
3199            }
3200        }
3201    }
3202}
3203
3204//=============================================================================
3205// Label fixups and jump veneers.
3206
3207/// Different forms of label references for different instruction formats.
3208#[derive(Clone, Copy, Debug, PartialEq, Eq)]
3209pub enum LabelUse {
3210    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
3211    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
3212    Branch14,
3213    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
3214    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
3215    Branch19,
3216    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
3217    /// signed bits, in bits 25:0. Used by b, bl.
3218    Branch26,
3219    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
3220    /// in bits 23:5.
3221    Ldr19,
3222    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
3223    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
3224    Adr21,
3225    /// 32-bit PC relative constant offset (from address of constant itself),
3226    /// signed. Used in jump tables.
3227    PCRel32,
3228    /// 21-bit page offset for ADRP. The immediate is the signed difference
3229    /// between the 4KiB page of the label and the 4KiB page of the ADRP
3230    /// instruction, with high 19 bits in bits 23:5 and low 2 bits in bits
3231    /// 30:29. Equivalent to the `R_AARCH64_ADR_PREL_PG_HI21` relocation.
3232    AdrpPage21,
3233    /// Low 12 bits of the label's address in the 12-bit immediate of an ADD
3234    /// (immediate), bits 21:10. Equivalent to the `R_AARCH64_ADD_ABS_LO12_NC`
3235    /// relocation. This has the same 4KiB-alignment requirement as
3236    /// `AdrpPage21`, with which it is paired.
3237    AddLo12,
3238}
3239
3240impl MachInstLabelUse for LabelUse {
3241    /// Alignment for veneer code. Every AArch64 instruction must be 4-byte-aligned.
3242    const ALIGN: CodeOffset = 4;
3243
3244    /// Maximum PC-relative range (positive), inclusive.
3245    fn max_pos_range(self) -> CodeOffset {
3246        match self {
3247            // N-bit immediate, left-shifted by 2, for (N+2) bits of total
3248            // range. Signed, so +2^(N+1) from zero. Likewise for two other
3249            // shifted cases below.
3250            LabelUse::Branch14 => (1 << 15) - 1,
3251            LabelUse::Branch19 => (1 << 20) - 1,
3252            LabelUse::Branch26 => (1 << 27) - 1,
3253            LabelUse::Ldr19 => (1 << 20) - 1,
3254            // Adr does not shift its immediate, so the 21-bit immediate gives 21 bits of total
3255            // range.
3256            LabelUse::Adr21 => (1 << 20) - 1,
3257            LabelUse::PCRel32 => 0x7fffffff,
3258            // `adrp` technically has +/-4GiB of range, but cap it to the same
3259            // range as other 32-bit uses to keep offset arithmetic within
3260            // `u32`.
3261            LabelUse::AdrpPage21 | LabelUse::AddLo12 => 0x7fffffff,
3262        }
3263    }
3264
3265    /// Maximum PC-relative range (negative).
3266    fn max_neg_range(self) -> CodeOffset {
3267        // All forms are twos-complement signed offsets, so negative limit is one more than
3268        // positive limit.
3269        self.max_pos_range() + 1
3270    }
3271
3272    /// Size of window into code needed to do the patch.
3273    fn patch_size(self) -> CodeOffset {
3274        // Patch is on one instruction only for all of these label reference types.
3275        4
3276    }
3277
3278    /// Perform the patch.
3279    fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
3280        let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
3281        match self {
3282            LabelUse::AdrpPage21 => {
3283                let page_delta = i64::from(label_offset >> 12) - i64::from(use_offset >> 12);
3284                let imm = page_delta as u32;
3285                let insn_word = (insn_word & !0x60ffffe0) | (imm & 0x1ffffc) << 3 | (imm & 3) << 29;
3286                buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3287                return;
3288            }
3289            LabelUse::AddLo12 => {
3290                let imm12 = label_offset & 0xfff;
3291                let insn_word = (insn_word & !(0xfff << 10)) | imm12 << 10;
3292                buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3293                return;
3294            }
3295            _ => {}
3296        }
3297        let pc_rel = (label_offset as i64) - (use_offset as i64);
3298        assert!(pc_rel <= self.max_pos_range() as i64);
3299        assert!(pc_rel >= -(self.max_neg_range() as i64));
3300        let pc_rel = pc_rel as u32;
3301        let mask = match self {
3302            LabelUse::Branch14 => 0x0007ffe0, // bits 18..5 inclusive
3303            LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
3304            LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
3305            LabelUse::Ldr19 => 0x00ffffe0,    // bits 23..5 inclusive
3306            LabelUse::Adr21 => 0x60ffffe0,    // bits 30..29, 25..5 inclusive
3307            LabelUse::PCRel32 => 0xffffffff,
3308            LabelUse::AdrpPage21 | LabelUse::AddLo12 => unreachable!(),
3309        };
3310        let pc_rel_shifted = match self {
3311            LabelUse::Adr21 | LabelUse::PCRel32 => pc_rel,
3312            _ => {
3313                debug_assert!(pc_rel & 3 == 0);
3314                pc_rel >> 2
3315            }
3316        };
3317        let pc_rel_inserted = match self {
3318            LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
3319            LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
3320            LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
3321            // Note: the *low* two bits of offset are put in the
3322            // *high* bits (30, 29).
3323            LabelUse::Adr21 => (pc_rel_shifted & 0x1ffffc) << 3 | (pc_rel_shifted & 3) << 29,
3324            LabelUse::PCRel32 => pc_rel_shifted,
3325            LabelUse::AdrpPage21 | LabelUse::AddLo12 => unreachable!(),
3326        };
3327        let is_add = match self {
3328            LabelUse::PCRel32 => true,
3329            _ => false,
3330        };
3331        let insn_word = if is_add {
3332            insn_word.wrapping_add(pc_rel_inserted)
3333        } else {
3334            (insn_word & !mask) | pc_rel_inserted
3335        };
3336        buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3337    }
3338
3339    /// Is a veneer supported for this label reference type?
3340    fn supports_veneer(self) -> bool {
3341        match self {
3342            LabelUse::Branch14 | LabelUse::Branch19 => true, // veneer is a Branch26
3343            LabelUse::Branch26 => true,                      // veneer is a PCRel32
3344            _ => false,
3345        }
3346    }
3347
3348    /// How large is the veneer, if supported?
3349    fn veneer_size(self) -> CodeOffset {
3350        match self {
3351            LabelUse::Branch14 | LabelUse::Branch19 => 4,
3352            LabelUse::Branch26 => 20,
3353            _ => unreachable!(),
3354        }
3355    }
3356
3357    fn worst_case_veneer_size() -> CodeOffset {
3358        20
3359    }
3360
3361    /// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return
3362    /// an offset and label-use for the veneer's use of the original label.
3363    fn generate_veneer(
3364        self,
3365        buffer: &mut [u8],
3366        veneer_offset: CodeOffset,
3367    ) -> (CodeOffset, LabelUse) {
3368        match self {
3369            LabelUse::Branch14 | LabelUse::Branch19 => {
3370                // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
3371                // bother with constructing an Inst.
3372                let insn_word = 0b000101 << 26;
3373                buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3374                (veneer_offset, LabelUse::Branch26)
3375            }
3376
3377            // This is promoting a 26-bit call/jump to a 32-bit call/jump to
3378            // get a further range. This jump translates to a jump to a
3379            // relative location based on the address of the constant loaded
3380            // from here.
3381            //
3382            // If this path is taken from a call instruction then caller-saved
3383            // registers are available (minus arguments), so x16/x17 are
3384            // available. Otherwise for intra-function jumps we also reserve
3385            // x16/x17 as spill-style registers. In both cases these are
3386            // available for us to use.
3387            LabelUse::Branch26 => {
3388                let tmp1 = regs::spilltmp_reg();
3389                let tmp1_w = regs::writable_spilltmp_reg();
3390                let tmp2 = regs::tmp2_reg();
3391                let tmp2_w = regs::writable_tmp2_reg();
3392                // ldrsw x16, 16
3393                let ldr = emit::enc_ldst_imm19(0b1001_1000, 16 / 4, tmp1);
3394                // adr x17, 12
3395                let adr = emit::enc_adr(12, tmp2_w);
3396                // add x16, x16, x17
3397                let add = emit::enc_arith_rrr(0b10001011_000, 0, tmp1_w, tmp1, tmp2);
3398                // br x16
3399                let br = emit::enc_br(tmp1);
3400                buffer[0..4].clone_from_slice(&u32::to_le_bytes(ldr));
3401                buffer[4..8].clone_from_slice(&u32::to_le_bytes(adr));
3402                buffer[8..12].clone_from_slice(&u32::to_le_bytes(add));
3403                buffer[12..16].clone_from_slice(&u32::to_le_bytes(br));
3404                // the 4-byte signed immediate we'll load is after these
3405                // instructions, 16-bytes in.
3406                (veneer_offset + 16, LabelUse::PCRel32)
3407            }
3408
3409            _ => panic!("Unsupported label-reference type for veneer generation!"),
3410        }
3411    }
3412
3413    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {
3414        match (reloc, addend) {
3415            (Reloc::Arm64Call, 0) => Some(LabelUse::Branch26),
3416            (Reloc::Aarch64AdrPrelPgHi21, 0) => Some(LabelUse::AdrpPage21),
3417            (Reloc::Aarch64AddAbsLo12Nc, 0) => Some(LabelUse::AddLo12),
3418            _ => None,
3419        }
3420    }
3421}
3422
3423#[test]
3424#[cfg(target_pointer_width = "64")]
3425fn inst_size_test() {
3426    // This test will help with unintentionally growing the size
3427    // of the Inst enum.
3428    assert_eq!(48, core::mem::size_of::<Inst>());
3429}