Skip to main content

cranelift_codegen/isa/aarch64/inst/
mod.rs

1//! This module defines aarch64-specific machine instruction types.
2
3use crate::binemit::{Addend, CodeOffset, Reloc};
4use crate::ir::types::{F16, F32, F64, F128, I8, I8X16, I16, I32, I64, I128};
5use crate::ir::{MemFlagsData, Type, types};
6use crate::isa::{CallConv, FunctionAlignment};
7use crate::machinst::*;
8use crate::{CodegenError, CodegenResult, settings};
9
10use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
11
12use alloc::string::{String, ToString};
13use alloc::vec::Vec;
14use core::fmt::Write;
15use core::slice;
16use smallvec::{SmallVec, smallvec};
17
18pub(crate) mod regs;
19pub use self::regs::*;
20pub mod imms;
21pub use self::imms::*;
22pub mod args;
23pub use self::args::*;
24pub mod emit;
25pub(crate) use self::emit::*;
26use crate::isa::aarch64::abi::AArch64MachineDeps;
27
28pub(crate) mod unwind;
29
30#[cfg(test)]
31mod emit_tests;
32
33//=============================================================================
34// Instructions (top level): definition
35
36pub use crate::isa::aarch64::lower::isle::generated_code::{
37    ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BfmOp, BitOp, BranchTargetType,
38    FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp,
39    VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp,
40    VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,
41};
42
43/// A floating-point unit (FPU) operation with two args, a register and an immediate.
44#[derive(Copy, Clone, Debug)]
45pub enum FPUOpRI {
46    /// Unsigned right shift. Rd = Rn << #imm
47    UShr32(FPURightShiftImm),
48    /// Unsigned right shift. Rd = Rn << #imm
49    UShr64(FPURightShiftImm),
50}
51
52/// A floating-point unit (FPU) operation with two args, a register and
53/// an immediate that modifies its dest (so takes that input value as a
54/// separate virtual register).
55#[derive(Copy, Clone, Debug)]
56pub enum FPUOpRIMod {
57    /// Shift left and insert. Rd |= Rn << #imm
58    Sli32(FPULeftShiftImm),
59    /// Shift left and insert. Rd |= Rn << #imm
60    Sli64(FPULeftShiftImm),
61}
62
63impl BfmOp {
64    /// Get the assembly mnemonic for this opcode.
65    pub fn op_str(&self) -> &'static str {
66        match self {
67            BfmOp::UBfm => "ubfm",
68            BfmOp::SBfm => "sbfm",
69        }
70    }
71}
72
73impl BitOp {
74    /// Get the assembly mnemonic for this opcode.
75    pub fn op_str(&self) -> &'static str {
76        match self {
77            BitOp::RBit => "rbit",
78            BitOp::Clz => "clz",
79            BitOp::Cls => "cls",
80            BitOp::Rev16 => "rev16",
81            BitOp::Rev32 => "rev32",
82            BitOp::Rev64 => "rev64",
83        }
84    }
85}
86
87/// Additional information for `return_call[_ind]` instructions, left out of
88/// line to lower the size of the `Inst` enum.
89#[derive(Clone, Debug)]
90pub struct ReturnCallInfo<T> {
91    /// Where this call is going to
92    pub dest: T,
93    /// Arguments to the call instruction.
94    pub uses: CallArgList,
95    /// The size of the new stack frame's stack arguments. This is necessary
96    /// for copying the frame over our current frame. It must already be
97    /// allocated on the stack.
98    pub new_stack_arg_size: u32,
99    /// API key to use to restore the return address, if any.
100    pub key: Option<APIKey>,
101    /// Whether pointer-auth return addresses are signed even without frame setup.
102    pub sign_return_address_all: bool,
103}
104
105fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
106    let mut count = 0;
107    for _ in 0..num_half_words {
108        if value & 0xffff == 0 {
109            count += 1;
110        }
111        value >>= 16;
112    }
113
114    count
115}
116
117impl Inst {
118    /// Create an instruction that loads a constant, using one of several options (MOVZ, MOVN,
119    /// logical immediate, or constant pool).
120    pub fn load_constant(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]> {
121        // NB: this is duplicated in `lower/isle.rs` and `inst.isle` right now,
122        // if modifications are made here before this is deleted after moving to
123        // ISLE then those locations should be updated as well.
124
125        if let Some(imm) = MoveWideConst::maybe_from_u64(value) {
126            // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ
127            smallvec![Inst::MovWide {
128                op: MoveWideOp::MovZ,
129                rd,
130                imm,
131                size: OperandSize::Size64
132            }]
133        } else if let Some(imm) = MoveWideConst::maybe_from_u64(!value) {
134            // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVN
135            smallvec![Inst::MovWide {
136                op: MoveWideOp::MovN,
137                rd,
138                imm,
139                size: OperandSize::Size64
140            }]
141        } else if let Some(imml) = ImmLogic::maybe_from_u64(value, I64) {
142            // Weird logical-instruction immediate in ORI using zero register
143            smallvec![Inst::AluRRImmLogic {
144                alu_op: ALUOp::Orr,
145                size: OperandSize::Size64,
146                rd,
147                rn: zero_reg(),
148                imml,
149            }]
150        } else {
151            let mut insts = smallvec![];
152
153            // If the top 32 bits are zero, use 32-bit `mov` operations.
154            let (num_half_words, size, negated) = if value >> 32 == 0 {
155                (2, OperandSize::Size32, (!value << 32) >> 32)
156            } else {
157                (4, OperandSize::Size64, !value)
158            };
159
160            // If the number of 0xffff half words is greater than the number of 0x0000 half words
161            // it is more efficient to use `movn` for the first instruction.
162            let first_is_inverted = count_zero_half_words(negated, num_half_words)
163                > count_zero_half_words(value, num_half_words);
164
165            // Either 0xffff or 0x0000 half words can be skipped, depending on the first
166            // instruction used.
167            let ignored_halfword = if first_is_inverted { 0xffff } else { 0 };
168
169            let halfwords: SmallVec<[_; 4]> = (0..num_half_words)
170                .filter_map(|i| {
171                    let imm16 = (value >> (16 * i)) & 0xffff;
172                    if imm16 == ignored_halfword {
173                        None
174                    } else {
175                        Some((i, imm16))
176                    }
177                })
178                .collect();
179
180            let mut prev_result = None;
181            for (i, imm16) in halfwords {
182                let shift = i * 16;
183
184                if let Some(rn) = prev_result {
185                    let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
186                    insts.push(Inst::MovK { rd, rn, imm, size });
187                } else {
188                    if first_is_inverted {
189                        let imm =
190                            MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, shift)
191                                .unwrap();
192                        insts.push(Inst::MovWide {
193                            op: MoveWideOp::MovN,
194                            rd,
195                            imm,
196                            size,
197                        });
198                    } else {
199                        let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
200                        insts.push(Inst::MovWide {
201                            op: MoveWideOp::MovZ,
202                            rd,
203                            imm,
204                            size,
205                        });
206                    }
207                }
208
209                prev_result = Some(rd.to_reg());
210            }
211
212            assert!(prev_result.is_some());
213
214            insts
215        }
216    }
217
218    /// Generic constructor for a load (zero-extending where appropriate).
219    pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlagsData) -> Inst {
220        match ty {
221            I8 => Inst::ULoad8 {
222                rd: into_reg,
223                mem,
224                flags,
225            },
226            I16 => Inst::ULoad16 {
227                rd: into_reg,
228                mem,
229                flags,
230            },
231            I32 => Inst::ULoad32 {
232                rd: into_reg,
233                mem,
234                flags,
235            },
236            I64 => Inst::ULoad64 {
237                rd: into_reg,
238                mem,
239                flags,
240            },
241            _ => {
242                if ty.is_vector() || ty.is_float() {
243                    let bits = ty_bits(ty);
244                    let rd = into_reg;
245
246                    match bits {
247                        128 => Inst::FpuLoad128 { rd, mem, flags },
248                        64 => Inst::FpuLoad64 { rd, mem, flags },
249                        32 => Inst::FpuLoad32 { rd, mem, flags },
250                        16 => Inst::FpuLoad16 { rd, mem, flags },
251                        _ => unimplemented!("gen_load({})", ty),
252                    }
253                } else {
254                    unimplemented!("gen_load({})", ty);
255                }
256            }
257        }
258    }
259
260    /// Generic constructor for a store.
261    pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlagsData) -> Inst {
262        match ty {
263            I8 => Inst::Store8 {
264                rd: from_reg,
265                mem,
266                flags,
267            },
268            I16 => Inst::Store16 {
269                rd: from_reg,
270                mem,
271                flags,
272            },
273            I32 => Inst::Store32 {
274                rd: from_reg,
275                mem,
276                flags,
277            },
278            I64 => Inst::Store64 {
279                rd: from_reg,
280                mem,
281                flags,
282            },
283            _ => {
284                if ty.is_vector() || ty.is_float() {
285                    let bits = ty_bits(ty);
286                    let rd = from_reg;
287
288                    match bits {
289                        128 => Inst::FpuStore128 { rd, mem, flags },
290                        64 => Inst::FpuStore64 { rd, mem, flags },
291                        32 => Inst::FpuStore32 { rd, mem, flags },
292                        16 => Inst::FpuStore16 { rd, mem, flags },
293                        _ => unimplemented!("gen_store({})", ty),
294                    }
295                } else {
296                    unimplemented!("gen_store({})", ty);
297                }
298            }
299        }
300    }
301
302    /// What type does this load or store instruction access in memory? When
303    /// uimm12 encoding is used, the size of this type is the amount that
304    /// immediate offsets are scaled by.
305    pub fn mem_type(&self) -> Option<Type> {
306        match self {
307            Inst::ULoad8 { .. } => Some(I8),
308            Inst::SLoad8 { .. } => Some(I8),
309            Inst::ULoad16 { .. } => Some(I16),
310            Inst::SLoad16 { .. } => Some(I16),
311            Inst::ULoad32 { .. } => Some(I32),
312            Inst::SLoad32 { .. } => Some(I32),
313            Inst::ULoad64 { .. } => Some(I64),
314            Inst::FpuLoad16 { .. } => Some(F16),
315            Inst::FpuLoad32 { .. } => Some(F32),
316            Inst::FpuLoad64 { .. } => Some(F64),
317            Inst::FpuLoad128 { .. } => Some(I8X16),
318            Inst::Store8 { .. } => Some(I8),
319            Inst::Store16 { .. } => Some(I16),
320            Inst::Store32 { .. } => Some(I32),
321            Inst::Store64 { .. } => Some(I64),
322            Inst::FpuStore16 { .. } => Some(F16),
323            Inst::FpuStore32 { .. } => Some(F32),
324            Inst::FpuStore64 { .. } => Some(F64),
325            Inst::FpuStore128 { .. } => Some(I8X16),
326            _ => None,
327        }
328    }
329}
330
331//=============================================================================
332// Instructions: get_regs
333
334fn memarg_operands(memarg: &mut AMode, collector: &mut impl OperandVisitor) {
335    match memarg {
336        AMode::Unscaled { rn, .. } | AMode::UnsignedOffset { rn, .. } => {
337            collector.reg_use(rn);
338        }
339        AMode::RegReg { rn, rm, .. }
340        | AMode::RegScaled { rn, rm, .. }
341        | AMode::RegScaledExtended { rn, rm, .. }
342        | AMode::RegExtended { rn, rm, .. } => {
343            collector.reg_use(rn);
344            collector.reg_use(rm);
345        }
346        AMode::Label { .. } => {}
347        AMode::SPPreIndexed { .. } | AMode::SPPostIndexed { .. } => {}
348        AMode::FPOffset { .. } | AMode::IncomingArg { .. } => {}
349        AMode::SPOffset { .. } | AMode::SlotOffset { .. } => {}
350        AMode::RegOffset { rn, .. } => {
351            collector.reg_use(rn);
352        }
353        AMode::Const { .. } => {}
354    }
355}
356
357fn pairmemarg_operands(pairmemarg: &mut PairAMode, collector: &mut impl OperandVisitor) {
358    match pairmemarg {
359        PairAMode::SignedOffset { reg, .. } => {
360            collector.reg_use(reg);
361        }
362        PairAMode::SPPreIndexed { .. } | PairAMode::SPPostIndexed { .. } => {}
363    }
364}
365
366fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
367    match inst {
368        Inst::AluRRR { rd, rn, rm, .. } => {
369            collector.reg_def(rd);
370            collector.reg_use(rn);
371            collector.reg_use(rm);
372        }
373        Inst::AluRRRR { rd, rn, rm, ra, .. } => {
374            collector.reg_def(rd);
375            collector.reg_use(rn);
376            collector.reg_use(rm);
377            collector.reg_use(ra);
378        }
379        Inst::AluRRImm12 { rd, rn, .. } => {
380            collector.reg_def(rd);
381            collector.reg_use(rn);
382        }
383        Inst::AluRRImmLogic { rd, rn, .. } => {
384            collector.reg_def(rd);
385            collector.reg_use(rn);
386        }
387        Inst::AluRRImmShift { rd, rn, .. } => {
388            collector.reg_def(rd);
389            collector.reg_use(rn);
390        }
391        Inst::AluRRRShift { rd, rn, rm, .. } => {
392            collector.reg_def(rd);
393            collector.reg_use(rn);
394            collector.reg_use(rm);
395        }
396        Inst::AluRRRExtend { rd, rn, rm, .. } => {
397            collector.reg_def(rd);
398            collector.reg_use(rn);
399            collector.reg_use(rm);
400        }
401        Inst::BitRR { rd, rn, .. } => {
402            collector.reg_def(rd);
403            collector.reg_use(rn);
404        }
405        Inst::ULoad8 { rd, mem, .. }
406        | Inst::SLoad8 { rd, mem, .. }
407        | Inst::ULoad16 { rd, mem, .. }
408        | Inst::SLoad16 { rd, mem, .. }
409        | Inst::ULoad32 { rd, mem, .. }
410        | Inst::SLoad32 { rd, mem, .. }
411        | Inst::ULoad64 { rd, mem, .. } => {
412            collector.reg_def(rd);
413            memarg_operands(mem, collector);
414        }
415        Inst::Store8 { rd, mem, .. }
416        | Inst::Store16 { rd, mem, .. }
417        | Inst::Store32 { rd, mem, .. }
418        | Inst::Store64 { rd, mem, .. } => {
419            collector.reg_use(rd);
420            memarg_operands(mem, collector);
421        }
422        Inst::StoreP64 { rt, rt2, mem, .. } => {
423            collector.reg_use(rt);
424            collector.reg_use(rt2);
425            pairmemarg_operands(mem, collector);
426        }
427        Inst::LoadP64 { rt, rt2, mem, .. } => {
428            collector.reg_def(rt);
429            collector.reg_def(rt2);
430            pairmemarg_operands(mem, collector);
431        }
432        Inst::Mov { rd, rm, .. } => {
433            collector.reg_def(rd);
434            collector.reg_use(rm);
435        }
436        Inst::MovFromPReg { rd, rm } => {
437            debug_assert!(rd.to_reg().is_virtual());
438            collector.reg_def(rd);
439            collector.reg_fixed_nonallocatable(*rm);
440        }
441        Inst::MovToPReg { rd, rm } => {
442            debug_assert!(rm.is_virtual());
443            collector.reg_fixed_nonallocatable(*rd);
444            collector.reg_use(rm);
445        }
446        Inst::MovK { rd, rn, .. } => {
447            collector.reg_use(rn);
448            collector.reg_reuse_def(rd, 0); // `rn` == `rd`.
449        }
450        Inst::MovWide { rd, .. } => {
451            collector.reg_def(rd);
452        }
453        Inst::CSel { rd, rn, rm, .. } => {
454            collector.reg_def(rd);
455            collector.reg_use(rn);
456            collector.reg_use(rm);
457        }
458        Inst::CSNeg { rd, rn, rm, .. } => {
459            collector.reg_def(rd);
460            collector.reg_use(rn);
461            collector.reg_use(rm);
462        }
463        Inst::CSet { rd, .. } | Inst::CSetm { rd, .. } => {
464            collector.reg_def(rd);
465        }
466        Inst::CCmp { rn, rm, .. } => {
467            collector.reg_use(rn);
468            collector.reg_use(rm);
469        }
470        Inst::CCmpImm { rn, .. } => {
471            collector.reg_use(rn);
472        }
473        Inst::AtomicRMWLoop {
474            op,
475            addr,
476            operand,
477            oldval,
478            scratch1,
479            scratch2,
480            ..
481        } => {
482            collector.reg_fixed_use(addr, xreg(25));
483            collector.reg_fixed_use(operand, xreg(26));
484            collector.reg_fixed_def(oldval, xreg(27));
485            collector.reg_fixed_def(scratch1, xreg(24));
486            if *op != AtomicRMWLoopOp::Xchg {
487                collector.reg_fixed_def(scratch2, xreg(28));
488            }
489        }
490        Inst::AtomicRMW { rs, rt, rn, .. } => {
491            collector.reg_use(rs);
492            collector.reg_def(rt);
493            collector.reg_use(rn);
494        }
495        Inst::AtomicCAS { rd, rs, rt, rn, .. } => {
496            collector.reg_reuse_def(rd, 1); // reuse `rs`.
497            collector.reg_use(rs);
498            collector.reg_use(rt);
499            collector.reg_use(rn);
500        }
501        Inst::AtomicCAS128 { args } => {
502            let AtomicCAS128Args {
503                rd_lo,
504                rd_hi,
505                rs_lo,
506                rs_hi,
507                rt_lo,
508                rt_hi,
509                rn,
510                flags: _,
511            } = &mut **args;
512            // `casp` requires two consecutive even-aligned register pairs,
513            // which regalloc2 cannot express, so pin everything down.
514            collector.reg_fixed_use(rs_lo, xreg(24));
515            collector.reg_fixed_use(rs_hi, xreg(25));
516            collector.reg_fixed_def(rd_lo, xreg(24));
517            collector.reg_fixed_def(rd_hi, xreg(25));
518            collector.reg_fixed_use(rt_lo, xreg(26));
519            collector.reg_fixed_use(rt_hi, xreg(27));
520            collector.reg_fixed_use(rn, xreg(28));
521        }
522        Inst::AtomicCASLoop {
523            addr,
524            expected,
525            replacement,
526            oldval,
527            scratch,
528            ..
529        } => {
530            collector.reg_fixed_use(addr, xreg(25));
531            collector.reg_fixed_use(expected, xreg(26));
532            collector.reg_fixed_use(replacement, xreg(28));
533            collector.reg_fixed_def(oldval, xreg(27));
534            collector.reg_fixed_def(scratch, xreg(24));
535        }
536        Inst::LoadAcquire { rt, rn, .. } => {
537            collector.reg_use(rn);
538            collector.reg_def(rt);
539        }
540        Inst::StoreRelease { rt, rn, .. } => {
541            collector.reg_use(rn);
542            collector.reg_use(rt);
543        }
544        Inst::Fence {} | Inst::Csdb {} => {}
545        Inst::FpuMove32 { rd, rn } => {
546            collector.reg_def(rd);
547            collector.reg_use(rn);
548        }
549        Inst::FpuMove64 { rd, rn } => {
550            collector.reg_def(rd);
551            collector.reg_use(rn);
552        }
553        Inst::FpuMove128 { rd, rn } => {
554            collector.reg_def(rd);
555            collector.reg_use(rn);
556        }
557        Inst::FpuMoveFromVec { rd, rn, .. } => {
558            collector.reg_def(rd);
559            collector.reg_use(rn);
560        }
561        Inst::FpuExtend { rd, rn, .. } => {
562            collector.reg_def(rd);
563            collector.reg_use(rn);
564        }
565        Inst::FpuRR { rd, rn, .. } => {
566            collector.reg_def(rd);
567            collector.reg_use(rn);
568        }
569        Inst::FpuRRR { rd, rn, rm, .. } => {
570            collector.reg_def(rd);
571            collector.reg_use(rn);
572            collector.reg_use(rm);
573        }
574        Inst::FpuRRI { rd, rn, .. } => {
575            collector.reg_def(rd);
576            collector.reg_use(rn);
577        }
578        Inst::FpuRRIMod { rd, ri, rn, .. } => {
579            collector.reg_reuse_def(rd, 1); // reuse `ri`.
580            collector.reg_use(ri);
581            collector.reg_use(rn);
582        }
583        Inst::FpuRRRR { rd, rn, rm, ra, .. } => {
584            collector.reg_def(rd);
585            collector.reg_use(rn);
586            collector.reg_use(rm);
587            collector.reg_use(ra);
588        }
589        Inst::VecMisc { rd, rn, .. } => {
590            collector.reg_def(rd);
591            collector.reg_use(rn);
592        }
593
594        Inst::VecLanes { rd, rn, .. } => {
595            collector.reg_def(rd);
596            collector.reg_use(rn);
597        }
598        Inst::VecShiftImm { rd, rn, .. } => {
599            collector.reg_def(rd);
600            collector.reg_use(rn);
601        }
602        Inst::VecShiftImmMod { rd, ri, rn, .. } => {
603            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
604            collector.reg_use(ri);
605            collector.reg_use(rn);
606        }
607        Inst::VecExtract { rd, rn, rm, .. } => {
608            collector.reg_def(rd);
609            collector.reg_use(rn);
610            collector.reg_use(rm);
611        }
612        Inst::VecTbl { rd, rn, rm } => {
613            collector.reg_use(rn);
614            collector.reg_use(rm);
615            collector.reg_def(rd);
616        }
617        Inst::VecTblExt { rd, ri, rn, rm } => {
618            collector.reg_use(rn);
619            collector.reg_use(rm);
620            collector.reg_reuse_def(rd, 3); // `rd` == `ri`.
621            collector.reg_use(ri);
622        }
623
624        Inst::VecTbl2 { rd, rn, rn2, rm } => {
625            // Constrain to v30 / v31 so that we satisfy the "adjacent
626            // registers" constraint without use of pinned vregs in
627            // lowering.
628            collector.reg_fixed_use(rn, vreg(30));
629            collector.reg_fixed_use(rn2, vreg(31));
630            collector.reg_use(rm);
631            collector.reg_def(rd);
632        }
633        Inst::VecTbl2Ext {
634            rd,
635            ri,
636            rn,
637            rn2,
638            rm,
639        } => {
640            // Constrain to v30 / v31 so that we satisfy the "adjacent
641            // registers" constraint without use of pinned vregs in
642            // lowering.
643            collector.reg_fixed_use(rn, vreg(30));
644            collector.reg_fixed_use(rn2, vreg(31));
645            collector.reg_use(rm);
646            collector.reg_reuse_def(rd, 4); // `rd` == `ri`.
647            collector.reg_use(ri);
648        }
649        Inst::VecLoadReplicate { rd, rn, .. } => {
650            collector.reg_def(rd);
651            collector.reg_use(rn);
652        }
653        Inst::VecCSel { rd, rn, rm, .. } => {
654            collector.reg_def(rd);
655            collector.reg_use(rn);
656            collector.reg_use(rm);
657        }
658        Inst::FpuCmp { rn, rm, .. } => {
659            collector.reg_use(rn);
660            collector.reg_use(rm);
661        }
662        Inst::FpuLoad16 { rd, mem, .. } => {
663            collector.reg_def(rd);
664            memarg_operands(mem, collector);
665        }
666        Inst::FpuLoad32 { rd, mem, .. } => {
667            collector.reg_def(rd);
668            memarg_operands(mem, collector);
669        }
670        Inst::FpuLoad64 { rd, mem, .. } => {
671            collector.reg_def(rd);
672            memarg_operands(mem, collector);
673        }
674        Inst::FpuLoad128 { rd, mem, .. } => {
675            collector.reg_def(rd);
676            memarg_operands(mem, collector);
677        }
678        Inst::FpuStore16 { rd, mem, .. } => {
679            collector.reg_use(rd);
680            memarg_operands(mem, collector);
681        }
682        Inst::FpuStore32 { rd, mem, .. } => {
683            collector.reg_use(rd);
684            memarg_operands(mem, collector);
685        }
686        Inst::FpuStore64 { rd, mem, .. } => {
687            collector.reg_use(rd);
688            memarg_operands(mem, collector);
689        }
690        Inst::FpuStore128 { rd, mem, .. } => {
691            collector.reg_use(rd);
692            memarg_operands(mem, collector);
693        }
694        Inst::FpuLoadP64 { rt, rt2, mem, .. } => {
695            collector.reg_def(rt);
696            collector.reg_def(rt2);
697            pairmemarg_operands(mem, collector);
698        }
699        Inst::FpuStoreP64 { rt, rt2, mem, .. } => {
700            collector.reg_use(rt);
701            collector.reg_use(rt2);
702            pairmemarg_operands(mem, collector);
703        }
704        Inst::FpuLoadP128 { rt, rt2, mem, .. } => {
705            collector.reg_def(rt);
706            collector.reg_def(rt2);
707            pairmemarg_operands(mem, collector);
708        }
709        Inst::FpuStoreP128 { rt, rt2, mem, .. } => {
710            collector.reg_use(rt);
711            collector.reg_use(rt2);
712            pairmemarg_operands(mem, collector);
713        }
714        Inst::FpuToInt { rd, rn, .. } => {
715            collector.reg_def(rd);
716            collector.reg_use(rn);
717        }
718        Inst::IntToFpu { rd, rn, .. } => {
719            collector.reg_def(rd);
720            collector.reg_use(rn);
721        }
722        Inst::FpuCSel16 { rd, rn, rm, .. }
723        | Inst::FpuCSel32 { rd, rn, rm, .. }
724        | Inst::FpuCSel64 { rd, rn, rm, .. } => {
725            collector.reg_def(rd);
726            collector.reg_use(rn);
727            collector.reg_use(rm);
728        }
729        Inst::FpuRound { rd, rn, .. } => {
730            collector.reg_def(rd);
731            collector.reg_use(rn);
732        }
733        Inst::MovToFpu { rd, rn, .. } => {
734            collector.reg_def(rd);
735            collector.reg_use(rn);
736        }
737        Inst::FpuMoveFPImm { rd, .. } => {
738            collector.reg_def(rd);
739        }
740        Inst::MovToVec { rd, ri, rn, .. } => {
741            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
742            collector.reg_use(ri);
743            collector.reg_use(rn);
744        }
745        Inst::MovFromVec { rd, rn, .. } | Inst::MovFromVecSigned { rd, rn, .. } => {
746            collector.reg_def(rd);
747            collector.reg_use(rn);
748        }
749        Inst::VecDup { rd, rn, .. } => {
750            collector.reg_def(rd);
751            collector.reg_use(rn);
752        }
753        Inst::VecDupFromFpu { rd, rn, .. } => {
754            collector.reg_def(rd);
755            collector.reg_use(rn);
756        }
757        Inst::VecDupFPImm { rd, .. } => {
758            collector.reg_def(rd);
759        }
760        Inst::VecDupImm { rd, .. } => {
761            collector.reg_def(rd);
762        }
763        Inst::VecExtend { rd, rn, .. } => {
764            collector.reg_def(rd);
765            collector.reg_use(rn);
766        }
767        Inst::VecMovElement { rd, ri, rn, .. } => {
768            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
769            collector.reg_use(ri);
770            collector.reg_use(rn);
771        }
772        Inst::VecRRLong { rd, rn, .. } => {
773            collector.reg_def(rd);
774            collector.reg_use(rn);
775        }
776        Inst::VecRRNarrowLow { rd, rn, .. } => {
777            collector.reg_use(rn);
778            collector.reg_def(rd);
779        }
780        Inst::VecRRNarrowHigh { rd, ri, rn, .. } => {
781            collector.reg_use(rn);
782            collector.reg_reuse_def(rd, 2); // `rd` == `ri`.
783            collector.reg_use(ri);
784        }
785        Inst::VecRRPair { rd, rn, .. } => {
786            collector.reg_def(rd);
787            collector.reg_use(rn);
788        }
789        Inst::VecRRRLong { rd, rn, rm, .. } => {
790            collector.reg_def(rd);
791            collector.reg_use(rn);
792            collector.reg_use(rm);
793        }
794        Inst::VecRRRLongMod { rd, ri, rn, rm, .. } => {
795            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
796            collector.reg_use(ri);
797            collector.reg_use(rn);
798            collector.reg_use(rm);
799        }
800        Inst::VecRRPairLong { rd, rn, .. } => {
801            collector.reg_def(rd);
802            collector.reg_use(rn);
803        }
804        Inst::VecRRR { rd, rn, rm, .. } => {
805            collector.reg_def(rd);
806            collector.reg_use(rn);
807            collector.reg_use(rm);
808        }
809        Inst::VecRRRMod { rd, ri, rn, rm, .. } | Inst::VecFmlaElem { rd, ri, rn, rm, .. } => {
810            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
811            collector.reg_use(ri);
812            collector.reg_use(rn);
813            collector.reg_use(rm);
814        }
815        Inst::MovToNZCV { rn } => {
816            collector.reg_use(rn);
817        }
818        Inst::MovFromNZCV { rd } => {
819            collector.reg_def(rd);
820        }
821        Inst::Extend { rd, rn, .. } => {
822            collector.reg_def(rd);
823            collector.reg_use(rn);
824        }
825        Inst::BitfieldMove { rd, rn, .. } => {
826            // The UBFM and SBFM instructions overwrite all bits in `rd`,
827            // unlike BFM which is represented as `BitfieldMoveMod` instead.
828            collector.reg_def(rd);
829            collector.reg_use(rn);
830        }
831        Inst::BitfieldMoveMod { rd, ri, rn, .. } => {
832            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
833            collector.reg_use(ri);
834            collector.reg_use(rn);
835        }
836        Inst::Args { args } => {
837            for ArgPair { vreg, preg } in args {
838                collector.reg_fixed_def(vreg, *preg);
839            }
840        }
841        Inst::Rets { rets } => {
842            for RetPair { vreg, preg } in rets {
843                collector.reg_fixed_use(vreg, *preg);
844            }
845        }
846        Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}
847        Inst::Jump { .. } => {}
848        Inst::Call { info, .. } => {
849            let CallInfo { uses, defs, .. } = &mut **info;
850            for CallArgPair { vreg, preg } in uses {
851                collector.reg_fixed_use(vreg, *preg);
852            }
853            for CallRetPair { vreg, location } in defs {
854                match location {
855                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
856                    RetLocation::Stack(..) => collector.any_def(vreg),
857                }
858            }
859            collector.reg_clobbers(info.clobbers);
860            if let Some(try_call_info) = &mut info.try_call_info {
861                try_call_info.collect_operands(collector);
862            }
863        }
864        Inst::CallInd { info, .. } => {
865            let CallInfo {
866                dest, uses, defs, ..
867            } = &mut **info;
868            collector.reg_use(dest);
869            for CallArgPair { vreg, preg } in uses {
870                collector.reg_fixed_use(vreg, *preg);
871            }
872            for CallRetPair { vreg, location } in defs {
873                match location {
874                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
875                    RetLocation::Stack(..) => collector.any_def(vreg),
876                }
877            }
878            collector.reg_clobbers(info.clobbers);
879            if let Some(try_call_info) = &mut info.try_call_info {
880                try_call_info.collect_operands(collector);
881            }
882        }
883        Inst::ReturnCall { info } => {
884            for CallArgPair { vreg, preg } in &mut info.uses {
885                collector.reg_fixed_use(vreg, *preg);
886            }
887        }
888        Inst::ReturnCallInd { info } => {
889            // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
890            // This shouldn't be a fixed register constraint, but it's not clear how to pick a
891            // register that won't be clobbered by the callee-save restore code emitted with a
892            // return_call_indirect.
893            collector.reg_fixed_use(&mut info.dest, xreg(1));
894            for CallArgPair { vreg, preg } in &mut info.uses {
895                collector.reg_fixed_use(vreg, *preg);
896            }
897        }
898        Inst::CondBr { kind, .. } => match kind {
899            CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
900            CondBrKind::Cond(_) => {}
901        },
902        Inst::TestBitAndBranch { rn, .. } => {
903            collector.reg_use(rn);
904        }
905        Inst::IndirectBr { rn, .. } => {
906            collector.reg_use(rn);
907        }
908        Inst::Nop0 | Inst::Nop4 => {}
909        Inst::Brk => {}
910        Inst::Udf { .. } => {}
911        Inst::TrapIf { kind, .. } => match kind {
912            CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
913            CondBrKind::Cond(_) => {}
914        },
915        Inst::Adr { rd, .. } | Inst::Adrp { rd, .. } => {
916            collector.reg_def(rd);
917        }
918        Inst::Word4 { .. } | Inst::Word8 { .. } => {}
919        Inst::JTSequence {
920            ridx, rtmp1, rtmp2, ..
921        } => {
922            collector.reg_use(ridx);
923            collector.reg_early_def(rtmp1);
924            collector.reg_early_def(rtmp2);
925        }
926        Inst::LoadExtNameGot { rd, .. }
927        | Inst::LoadExtNameNear { rd, .. }
928        | Inst::LoadExtNameFar { rd, .. } => {
929            collector.reg_def(rd);
930        }
931        Inst::LoadAddr { rd, mem } => {
932            collector.reg_def(rd);
933            memarg_operands(mem, collector);
934        }
935        Inst::Paci { .. } | Inst::Xpaclri => {
936            // Neither LR nor SP is an allocatable register, so there is no need
937            // to do anything.
938        }
939        Inst::Bti { .. } => {}
940
941        Inst::ElfTlsGetAddr { rd, tmp, .. } => {
942            // TLSDESC has a very neat calling convention. It is required to preserve
943            // all registers except x0 and x30. X30 is non allocatable in cranelift since
944            // its the link register.
945            //
946            // Additionally we need a second register as a temporary register for the
947            // TLSDESC sequence. This register can be any register other than x0 (and x30).
948            collector.reg_fixed_def(rd, regs::xreg(0));
949            collector.reg_early_def(tmp);
950        }
951        Inst::MachOTlsGetAddr { rd, .. } => {
952            collector.reg_fixed_def(rd, regs::xreg(0));
953            let mut clobbers =
954                AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64, false);
955            clobbers.remove(regs::xreg_preg(0));
956            collector.reg_clobbers(clobbers);
957        }
958        Inst::Unwind { .. } => {}
959        Inst::EmitIsland { .. } => {}
960        Inst::DummyUse { reg } => {
961            collector.reg_use(reg);
962        }
963        Inst::LabelAddress { dst, .. } => {
964            collector.reg_def(dst);
965        }
966        Inst::SequencePoint { .. } => {}
967        Inst::StackProbeLoop { start, end, .. } => {
968            collector.reg_early_def(start);
969            collector.reg_use(end);
970        }
971    }
972}
973
974//=============================================================================
975// Instructions: misc functions and external interface
976
977impl MachInst for Inst {
978    type ABIMachineSpec = AArch64MachineDeps;
979    type LabelUse = LabelUse;
980
981    // "CLIF" in hex, to make the trap recognizable during
982    // debugging.
983    const TRAP_OPCODE: &'static [u8] = &0xc11f_u32.to_le_bytes();
984
985    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
986        aarch64_get_operands(self, collector);
987    }
988
989    fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
990        match self {
991            &Inst::Mov {
992                size: OperandSize::Size64,
993                rd,
994                rm,
995            } => Some((rd, rm)),
996            &Inst::FpuMove64 { rd, rn } => Some((rd, rn)),
997            &Inst::FpuMove128 { rd, rn } => Some((rd, rn)),
998            _ => None,
999        }
1000    }
1001
1002    fn is_included_in_clobbers(&self) -> bool {
1003        let (caller, callee, is_exception) = match self {
1004            Inst::Args { .. } => return false,
1005            Inst::Call { info } => (
1006                info.caller_conv,
1007                info.callee_conv,
1008                info.try_call_info.is_some(),
1009            ),
1010            Inst::CallInd { info } => (
1011                info.caller_conv,
1012                info.callee_conv,
1013                info.try_call_info.is_some(),
1014            ),
1015            _ => return true,
1016        };
1017
1018        // We exclude call instructions from the clobber-set when they are calls
1019        // from caller to callee that both clobber the same register (such as
1020        // using the same or similar ABIs). Such calls cannot possibly force any
1021        // new registers to be saved in the prologue, because anything that the
1022        // callee clobbers, the caller is also allowed to clobber. This both
1023        // saves work and enables us to more precisely follow the
1024        // half-caller-save, half-callee-save SysV ABI for some vector
1025        // registers.
1026        //
1027        // See the note in [crate::isa::aarch64::abi::is_caller_save_reg] for
1028        // more information on this ABI-implementation hack.
1029        let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller, false);
1030        let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee, is_exception);
1031
1032        let mut all_clobbers = caller_clobbers;
1033        all_clobbers.union_from(callee_clobbers);
1034        all_clobbers != caller_clobbers
1035    }
1036
1037    fn is_trap(&self) -> bool {
1038        match self {
1039            Self::Udf { .. } => true,
1040            _ => false,
1041        }
1042    }
1043
1044    fn is_args(&self) -> bool {
1045        match self {
1046            Self::Args { .. } => true,
1047            _ => false,
1048        }
1049    }
1050
1051    fn call_type(&self) -> CallType {
1052        match self {
1053            Inst::Call { .. }
1054            | Inst::CallInd { .. }
1055            | Inst::ElfTlsGetAddr { .. }
1056            | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
1057
1058            Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
1059
1060            _ => CallType::None,
1061        }
1062    }
1063
1064    fn is_term(&self) -> MachTerminator {
1065        match self {
1066            &Inst::Rets { .. } => MachTerminator::Ret,
1067            &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
1068            &Inst::Jump { .. } => MachTerminator::Branch,
1069            &Inst::CondBr { .. } => MachTerminator::Branch,
1070            &Inst::TestBitAndBranch { .. } => MachTerminator::Branch,
1071            &Inst::IndirectBr { .. } => MachTerminator::Branch,
1072            &Inst::JTSequence { .. } => MachTerminator::Branch,
1073            &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1074            &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1075            _ => MachTerminator::None,
1076        }
1077    }
1078
1079    fn is_mem_access(&self) -> bool {
1080        match self {
1081            &Inst::ULoad8 { .. }
1082            | &Inst::SLoad8 { .. }
1083            | &Inst::ULoad16 { .. }
1084            | &Inst::SLoad16 { .. }
1085            | &Inst::ULoad32 { .. }
1086            | &Inst::SLoad32 { .. }
1087            | &Inst::ULoad64 { .. }
1088            | &Inst::LoadP64 { .. }
1089            | &Inst::FpuLoad16 { .. }
1090            | &Inst::FpuLoad32 { .. }
1091            | &Inst::FpuLoad64 { .. }
1092            | &Inst::FpuLoad128 { .. }
1093            | &Inst::FpuLoadP64 { .. }
1094            | &Inst::FpuLoadP128 { .. }
1095            | &Inst::Store8 { .. }
1096            | &Inst::Store16 { .. }
1097            | &Inst::Store32 { .. }
1098            | &Inst::Store64 { .. }
1099            | &Inst::StoreP64 { .. }
1100            | &Inst::FpuStore16 { .. }
1101            | &Inst::FpuStore32 { .. }
1102            | &Inst::FpuStore64 { .. }
1103            | &Inst::FpuStore128 { .. } => true,
1104            // TODO: verify this carefully
1105            _ => false,
1106        }
1107    }
1108
1109    fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
1110        let bits = ty.bits();
1111
1112        assert!(bits <= 128);
1113        assert!(to_reg.to_reg().class() == from_reg.class());
1114        match from_reg.class() {
1115            RegClass::Int => Inst::Mov {
1116                size: OperandSize::Size64,
1117                rd: to_reg,
1118                rm: from_reg,
1119            },
1120            RegClass::Float => {
1121                if bits > 64 {
1122                    Inst::FpuMove128 {
1123                        rd: to_reg,
1124                        rn: from_reg,
1125                    }
1126                } else {
1127                    Inst::FpuMove64 {
1128                        rd: to_reg,
1129                        rn: from_reg,
1130                    }
1131                }
1132            }
1133            RegClass::Vector => unreachable!(),
1134        }
1135    }
1136
1137    fn is_safepoint(&self) -> bool {
1138        match self {
1139            Inst::Call { .. } | Inst::CallInd { .. } => true,
1140            _ => false,
1141        }
1142    }
1143
1144    fn gen_dummy_use(reg: Reg) -> Inst {
1145        Inst::DummyUse { reg }
1146    }
1147
1148    fn gen_nop(preferred_size: usize) -> Inst {
1149        if preferred_size == 0 {
1150            return Inst::Nop0;
1151        }
1152        // We can't give a NOP (or any insn) < 4 bytes.
1153        assert!(preferred_size >= 4);
1154        Inst::Nop4
1155    }
1156
1157    fn gen_nop_units() -> Vec<Vec<u8>> {
1158        vec![vec![0x1f, 0x20, 0x03, 0xd5]]
1159    }
1160
1161    fn rc_for_type(ty: &Type) -> CodegenResult<(&[RegClass], &[Type])> {
1162        match *ty {
1163            I8 | I16 | I32 | I64 => Ok((&[RegClass::Int], slice::from_ref(ty))),
1164            F16 | F32 | F64 | F128 => Ok((&[RegClass::Float], slice::from_ref(ty))),
1165            I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
1166            _ if ty.is_vector() && ty.bits() <= 128 => {
1167                let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1168                Ok((
1169                    &[RegClass::Float],
1170                    slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1171                ))
1172            }
1173            _ if ty.is_dynamic_vector() => Ok((&[RegClass::Float], &[I8X16])),
1174            _ => Err(CodegenError::Unsupported(format!(
1175                "Unexpected SSA-value type: {ty}"
1176            ))),
1177        }
1178    }
1179
1180    fn canonical_type_for_rc(rc: RegClass) -> Type {
1181        match rc {
1182            RegClass::Float => types::I8X16,
1183            RegClass::Int => types::I64,
1184            RegClass::Vector => unreachable!(),
1185        }
1186    }
1187
1188    fn gen_jump(target: MachLabel) -> Inst {
1189        Inst::Jump {
1190            dest: BranchTarget::Label(target),
1191        }
1192    }
1193
1194    fn worst_case_size() -> CodeOffset {
1195        // The maximum size, in bytes, of any `Inst`'s emitted code. We have at least one case of
1196        // an 8-instruction sequence (saturating int-to-float conversions) with three embedded
1197        // 64-bit f64 constants.
1198        //
1199        // Note that inline jump-tables handle island/pool insertion separately, so we do not need
1200        // to account for them here (otherwise the worst case would be 2^31 * 4, clearly not
1201        // feasible for other reasons).
1202        44
1203    }
1204
1205    fn worst_case_island_growth() -> CodeOffset {
1206        // A single `Inst` may add to the buffer's pending-island state:
1207        //
1208        // - Up to three 8-byte constants (the saturating int-to-float sequence
1209        //   noted above); count alignment padding into each.
1210        // - Up to one deferred trap (TrapIf and similar), 4 bytes.
1211        // - Up to one fixup per emitted instruction word, each contributing at
1212        //   most `worst_case_veneer_size()` (= 20) bytes of veneer.
1213        //
1214        // We pick a conservative bound that comfortably covers these.
1215        128
1216    }
1217
1218    fn gen_block_start(
1219        is_indirect_branch_target: bool,
1220        is_forward_edge_cfi_enabled: bool,
1221    ) -> Option<Self> {
1222        if is_indirect_branch_target && is_forward_edge_cfi_enabled {
1223            Some(Inst::Bti {
1224                targets: BranchTargetType::J,
1225            })
1226        } else {
1227            None
1228        }
1229    }
1230
1231    fn function_alignment() -> FunctionAlignment {
1232        // We use 32-byte alignment for performance reasons, but for correctness
1233        // we would only need 4-byte alignment.
1234        FunctionAlignment {
1235            minimum: 4,
1236            preferred: 32,
1237        }
1238    }
1239}
1240
1241//=============================================================================
1242// Pretty-printing of instructions.
1243
1244fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (String, String) {
1245    let (mem_insts, mem) = mem_finalize(None, mem, access_ty, state);
1246    let mut mem_str = mem_insts
1247        .into_iter()
1248        .map(|inst| inst.print_with_state(&mut EmitState::default()))
1249        .collect::<Vec<_>>()
1250        .join(" ; ");
1251    if !mem_str.is_empty() {
1252        mem_str += " ; ";
1253    }
1254
1255    let mem = mem.pretty_print(access_ty.bytes() as u8);
1256    (mem_str, mem)
1257}
1258
1259fn pretty_print_try_call(info: &TryCallInfo) -> String {
1260    format!(
1261        "; b {:?}; catch [{}]",
1262        info.continuation,
1263        info.pretty_print_dests()
1264    )
1265}
1266
1267impl Inst {
1268    #[expect(
1269        missing_docs,
1270        reason = "exposed for cranelift-isle/veri pretty-printing"
1271    )]
1272    pub fn print_with_state(&self, state: &mut EmitState) -> String {
1273        fn op_name(alu_op: ALUOp) -> &'static str {
1274            match alu_op {
1275                ALUOp::Add => "add",
1276                ALUOp::Sub => "sub",
1277                ALUOp::Orr => "orr",
1278                ALUOp::And => "and",
1279                ALUOp::AndS => "ands",
1280                ALUOp::Eor => "eor",
1281                ALUOp::AddS => "adds",
1282                ALUOp::SubS => "subs",
1283                ALUOp::SMulH => "smulh",
1284                ALUOp::UMulH => "umulh",
1285                ALUOp::SDiv => "sdiv",
1286                ALUOp::UDiv => "udiv",
1287                ALUOp::AndNot => "bic",
1288                ALUOp::OrrNot => "orn",
1289                ALUOp::EorNot => "eon",
1290                ALUOp::Extr => "extr",
1291                ALUOp::Lsr => "lsr",
1292                ALUOp::Asr => "asr",
1293                ALUOp::Lsl => "lsl",
1294                ALUOp::Adc => "adc",
1295                ALUOp::AdcS => "adcs",
1296                ALUOp::Sbc => "sbc",
1297                ALUOp::SbcS => "sbcs",
1298            }
1299        }
1300
1301        match self {
1302            &Inst::Nop0 => "nop-zero-len".to_string(),
1303            &Inst::Nop4 => "nop".to_string(),
1304            &Inst::AluRRR {
1305                alu_op,
1306                size,
1307                rd,
1308                rn,
1309                rm,
1310            } => {
1311                let op = op_name(alu_op);
1312                let rd = pretty_print_ireg(rd.to_reg(), size);
1313                let rn = pretty_print_ireg(rn, size);
1314                let rm = pretty_print_ireg(rm, size);
1315                format!("{op} {rd}, {rn}, {rm}")
1316            }
1317            &Inst::AluRRRR {
1318                alu_op,
1319                size,
1320                rd,
1321                rn,
1322                rm,
1323                ra,
1324            } => {
1325                let (op, da_size) = match alu_op {
1326                    ALUOp3::MAdd => ("madd", size),
1327                    ALUOp3::MSub => ("msub", size),
1328                    ALUOp3::UMAddL => ("umaddl", OperandSize::Size64),
1329                    ALUOp3::SMAddL => ("smaddl", OperandSize::Size64),
1330                };
1331                let rd = pretty_print_ireg(rd.to_reg(), da_size);
1332                let rn = pretty_print_ireg(rn, size);
1333                let rm = pretty_print_ireg(rm, size);
1334                let ra = pretty_print_ireg(ra, da_size);
1335
1336                format!("{op} {rd}, {rn}, {rm}, {ra}")
1337            }
1338            &Inst::AluRRImm12 {
1339                alu_op,
1340                size,
1341                rd,
1342                rn,
1343                ref imm12,
1344            } => {
1345                let op = op_name(alu_op);
1346                let rd = pretty_print_ireg(rd.to_reg(), size);
1347                let rn = pretty_print_ireg(rn, size);
1348
1349                if imm12.bits == 0 && alu_op == ALUOp::Add && size.is64() {
1350                    // special-case MOV (used for moving into SP).
1351                    format!("mov {rd}, {rn}")
1352                } else {
1353                    let imm12 = imm12.pretty_print(0);
1354                    format!("{op} {rd}, {rn}, {imm12}")
1355                }
1356            }
1357            &Inst::AluRRImmLogic {
1358                alu_op,
1359                size,
1360                rd,
1361                rn,
1362                ref imml,
1363            } => {
1364                let op = op_name(alu_op);
1365                let rd = pretty_print_ireg(rd.to_reg(), size);
1366                let rn = pretty_print_ireg(rn, size);
1367                let imml = imml.pretty_print(0);
1368                format!("{op} {rd}, {rn}, {imml}")
1369            }
1370            &Inst::AluRRImmShift {
1371                alu_op,
1372                size,
1373                rd,
1374                rn,
1375                ref immshift,
1376            } => {
1377                let op = op_name(alu_op);
1378                let rd = pretty_print_ireg(rd.to_reg(), size);
1379                let rn = pretty_print_ireg(rn, size);
1380                let immshift = immshift.pretty_print(0);
1381                format!("{op} {rd}, {rn}, {immshift}")
1382            }
1383            &Inst::AluRRRShift {
1384                alu_op,
1385                size,
1386                rd,
1387                rn,
1388                rm,
1389                ref shiftop,
1390            } => {
1391                let op = op_name(alu_op);
1392                let rd = pretty_print_ireg(rd.to_reg(), size);
1393                let rn = pretty_print_ireg(rn, size);
1394                let rm = pretty_print_ireg(rm, size);
1395                let shiftop = shiftop.pretty_print(0);
1396                format!("{op} {rd}, {rn}, {rm}, {shiftop}")
1397            }
1398            &Inst::AluRRRExtend {
1399                alu_op,
1400                size,
1401                rd,
1402                rn,
1403                rm,
1404                ref extendop,
1405            } => {
1406                let op = op_name(alu_op);
1407                let rd = pretty_print_ireg(rd.to_reg(), size);
1408                let rn = pretty_print_ireg(rn, size);
1409                let rm = pretty_print_ireg(rm, size);
1410                let extendop = extendop.pretty_print(0);
1411                format!("{op} {rd}, {rn}, {rm}, {extendop}")
1412            }
1413            &Inst::BitRR { op, size, rd, rn } => {
1414                let op = op.op_str();
1415                let rd = pretty_print_ireg(rd.to_reg(), size);
1416                let rn = pretty_print_ireg(rn, size);
1417                format!("{op} {rd}, {rn}")
1418            }
1419            &Inst::ULoad8 { rd, ref mem, .. }
1420            | &Inst::SLoad8 { rd, ref mem, .. }
1421            | &Inst::ULoad16 { rd, ref mem, .. }
1422            | &Inst::SLoad16 { rd, ref mem, .. }
1423            | &Inst::ULoad32 { rd, ref mem, .. }
1424            | &Inst::SLoad32 { rd, ref mem, .. }
1425            | &Inst::ULoad64 { rd, ref mem, .. } => {
1426                let is_unscaled = match &mem {
1427                    &AMode::Unscaled { .. } => true,
1428                    _ => false,
1429                };
1430                let (op, size) = match (self, is_unscaled) {
1431                    (&Inst::ULoad8 { .. }, false) => ("ldrb", OperandSize::Size32),
1432                    (&Inst::ULoad8 { .. }, true) => ("ldurb", OperandSize::Size32),
1433                    (&Inst::SLoad8 { .. }, false) => ("ldrsb", OperandSize::Size64),
1434                    (&Inst::SLoad8 { .. }, true) => ("ldursb", OperandSize::Size64),
1435                    (&Inst::ULoad16 { .. }, false) => ("ldrh", OperandSize::Size32),
1436                    (&Inst::ULoad16 { .. }, true) => ("ldurh", OperandSize::Size32),
1437                    (&Inst::SLoad16 { .. }, false) => ("ldrsh", OperandSize::Size64),
1438                    (&Inst::SLoad16 { .. }, true) => ("ldursh", OperandSize::Size64),
1439                    (&Inst::ULoad32 { .. }, false) => ("ldr", OperandSize::Size32),
1440                    (&Inst::ULoad32 { .. }, true) => ("ldur", OperandSize::Size32),
1441                    (&Inst::SLoad32 { .. }, false) => ("ldrsw", OperandSize::Size64),
1442                    (&Inst::SLoad32 { .. }, true) => ("ldursw", OperandSize::Size64),
1443                    (&Inst::ULoad64 { .. }, false) => ("ldr", OperandSize::Size64),
1444                    (&Inst::ULoad64 { .. }, true) => ("ldur", OperandSize::Size64),
1445                    _ => unreachable!(),
1446                };
1447
1448                let rd = pretty_print_ireg(rd.to_reg(), size);
1449                let mem = mem.clone();
1450                let access_ty = self.mem_type().unwrap();
1451                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1452
1453                format!("{mem_str}{op} {rd}, {mem}")
1454            }
1455            &Inst::Store8 { rd, ref mem, .. }
1456            | &Inst::Store16 { rd, ref mem, .. }
1457            | &Inst::Store32 { rd, ref mem, .. }
1458            | &Inst::Store64 { rd, ref mem, .. } => {
1459                let is_unscaled = match &mem {
1460                    &AMode::Unscaled { .. } => true,
1461                    _ => false,
1462                };
1463                let (op, size) = match (self, is_unscaled) {
1464                    (&Inst::Store8 { .. }, false) => ("strb", OperandSize::Size32),
1465                    (&Inst::Store8 { .. }, true) => ("sturb", OperandSize::Size32),
1466                    (&Inst::Store16 { .. }, false) => ("strh", OperandSize::Size32),
1467                    (&Inst::Store16 { .. }, true) => ("sturh", OperandSize::Size32),
1468                    (&Inst::Store32 { .. }, false) => ("str", OperandSize::Size32),
1469                    (&Inst::Store32 { .. }, true) => ("stur", OperandSize::Size32),
1470                    (&Inst::Store64 { .. }, false) => ("str", OperandSize::Size64),
1471                    (&Inst::Store64 { .. }, true) => ("stur", OperandSize::Size64),
1472                    _ => unreachable!(),
1473                };
1474
1475                let rd = pretty_print_ireg(rd, size);
1476                let mem = mem.clone();
1477                let access_ty = self.mem_type().unwrap();
1478                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1479
1480                format!("{mem_str}{op} {rd}, {mem}")
1481            }
1482            &Inst::StoreP64 {
1483                rt, rt2, ref mem, ..
1484            } => {
1485                let rt = pretty_print_ireg(rt, OperandSize::Size64);
1486                let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);
1487                let mem = mem.clone();
1488                let mem = mem.pretty_print_default();
1489                format!("stp {rt}, {rt2}, {mem}")
1490            }
1491            &Inst::LoadP64 {
1492                rt, rt2, ref mem, ..
1493            } => {
1494                let rt = pretty_print_ireg(rt.to_reg(), OperandSize::Size64);
1495                let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);
1496                let mem = mem.clone();
1497                let mem = mem.pretty_print_default();
1498                format!("ldp {rt}, {rt2}, {mem}")
1499            }
1500            &Inst::Mov { size, rd, rm } => {
1501                let rd = pretty_print_ireg(rd.to_reg(), size);
1502                let rm = pretty_print_ireg(rm, size);
1503                format!("mov {rd}, {rm}")
1504            }
1505            &Inst::MovFromPReg { rd, rm } => {
1506                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1507                let rm = show_ireg_sized(rm.into(), OperandSize::Size64);
1508                format!("mov {rd}, {rm}")
1509            }
1510            &Inst::MovToPReg { rd, rm } => {
1511                let rd = show_ireg_sized(rd.into(), OperandSize::Size64);
1512                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1513                format!("mov {rd}, {rm}")
1514            }
1515            &Inst::MovWide {
1516                op,
1517                rd,
1518                ref imm,
1519                size,
1520            } => {
1521                let op_str = match op {
1522                    MoveWideOp::MovZ => "movz",
1523                    MoveWideOp::MovN => "movn",
1524                };
1525                let rd = pretty_print_ireg(rd.to_reg(), size);
1526                let imm = imm.pretty_print(0);
1527                format!("{op_str} {rd}, {imm}")
1528            }
1529            &Inst::MovK {
1530                rd,
1531                rn,
1532                ref imm,
1533                size,
1534            } => {
1535                let rn = pretty_print_ireg(rn, size);
1536                let rd = pretty_print_ireg(rd.to_reg(), size);
1537                let imm = imm.pretty_print(0);
1538                format!("movk {rd}, {rn}, {imm}")
1539            }
1540            &Inst::CSel { rd, rn, rm, cond } => {
1541                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1542                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1543                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1544                let cond = cond.pretty_print(0);
1545                format!("csel {rd}, {rn}, {rm}, {cond}")
1546            }
1547            &Inst::CSNeg { rd, rn, rm, cond } => {
1548                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1549                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1550                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1551                let cond = cond.pretty_print(0);
1552                format!("csneg {rd}, {rn}, {rm}, {cond}")
1553            }
1554            &Inst::CSet { rd, cond } => {
1555                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1556                let cond = cond.pretty_print(0);
1557                format!("cset {rd}, {cond}")
1558            }
1559            &Inst::CSetm { rd, cond } => {
1560                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1561                let cond = cond.pretty_print(0);
1562                format!("csetm {rd}, {cond}")
1563            }
1564            &Inst::CCmp {
1565                size,
1566                rn,
1567                rm,
1568                nzcv,
1569                cond,
1570            } => {
1571                let rn = pretty_print_ireg(rn, size);
1572                let rm = pretty_print_ireg(rm, size);
1573                let nzcv = nzcv.pretty_print(0);
1574                let cond = cond.pretty_print(0);
1575                format!("ccmp {rn}, {rm}, {nzcv}, {cond}")
1576            }
1577            &Inst::CCmpImm {
1578                size,
1579                rn,
1580                imm,
1581                nzcv,
1582                cond,
1583            } => {
1584                let rn = pretty_print_ireg(rn, size);
1585                let imm = imm.pretty_print(0);
1586                let nzcv = nzcv.pretty_print(0);
1587                let cond = cond.pretty_print(0);
1588                format!("ccmp {rn}, {imm}, {nzcv}, {cond}")
1589            }
1590            &Inst::AtomicRMW {
1591                rs, rt, rn, ty, op, ..
1592            } => {
1593                let op = match op {
1594                    AtomicRMWOp::Add => "ldaddal",
1595                    AtomicRMWOp::Clr => "ldclral",
1596                    AtomicRMWOp::Eor => "ldeoral",
1597                    AtomicRMWOp::Set => "ldsetal",
1598                    AtomicRMWOp::Smax => "ldsmaxal",
1599                    AtomicRMWOp::Umax => "ldumaxal",
1600                    AtomicRMWOp::Smin => "ldsminal",
1601                    AtomicRMWOp::Umin => "lduminal",
1602                    AtomicRMWOp::Swp => "swpal",
1603                };
1604
1605                let size = OperandSize::from_ty(ty);
1606                let rs = pretty_print_ireg(rs, size);
1607                let rt = pretty_print_ireg(rt.to_reg(), size);
1608                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1609
1610                let ty_suffix = match ty {
1611                    I8 => "b",
1612                    I16 => "h",
1613                    _ => "",
1614                };
1615                format!("{op}{ty_suffix} {rs}, {rt}, [{rn}]")
1616            }
1617            &Inst::AtomicRMWLoop {
1618                ty,
1619                op,
1620                addr,
1621                operand,
1622                oldval,
1623                scratch1,
1624                scratch2,
1625                ..
1626            } => {
1627                let op = match op {
1628                    AtomicRMWLoopOp::Add => "add",
1629                    AtomicRMWLoopOp::Sub => "sub",
1630                    AtomicRMWLoopOp::Eor => "eor",
1631                    AtomicRMWLoopOp::Orr => "orr",
1632                    AtomicRMWLoopOp::And => "and",
1633                    AtomicRMWLoopOp::Nand => "nand",
1634                    AtomicRMWLoopOp::Smin => "smin",
1635                    AtomicRMWLoopOp::Smax => "smax",
1636                    AtomicRMWLoopOp::Umin => "umin",
1637                    AtomicRMWLoopOp::Umax => "umax",
1638                    AtomicRMWLoopOp::Xchg => "xchg",
1639                };
1640                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1641                let operand = pretty_print_ireg(operand, OperandSize::Size64);
1642                let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1643                let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);
1644                let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);
1645                format!(
1646                    "atomic_rmw_loop_{}_{} addr={} operand={} oldval={} scratch1={} scratch2={}",
1647                    op,
1648                    ty.bits(),
1649                    addr,
1650                    operand,
1651                    oldval,
1652                    scratch1,
1653                    scratch2,
1654                )
1655            }
1656            &Inst::AtomicCAS {
1657                rd, rs, rt, rn, ty, ..
1658            } => {
1659                let op = match ty {
1660                    I8 => "casalb",
1661                    I16 => "casalh",
1662                    I32 | I64 => "casal",
1663                    _ => panic!("Unsupported type: {ty}"),
1664                };
1665                let size = OperandSize::from_ty(ty);
1666                let rd = pretty_print_ireg(rd.to_reg(), size);
1667                let rs = pretty_print_ireg(rs, size);
1668                let rt = pretty_print_ireg(rt, size);
1669                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1670
1671                format!("{op} {rd}, {rs}, {rt}, [{rn}]")
1672            }
1673            Inst::AtomicCAS128 { args } => {
1674                let &AtomicCAS128Args {
1675                    rd_lo,
1676                    rd_hi,
1677                    rs_lo,
1678                    rs_hi,
1679                    rt_lo,
1680                    rt_hi,
1681                    rn,
1682                    flags: _,
1683                } = &**args;
1684                let size = OperandSize::Size64;
1685                let rd_lo = pretty_print_ireg(rd_lo.to_reg(), size);
1686                let rd_hi = pretty_print_ireg(rd_hi.to_reg(), size);
1687                let rs_lo = pretty_print_ireg(rs_lo, size);
1688                let rs_hi = pretty_print_ireg(rs_hi, size);
1689                let rt_lo = pretty_print_ireg(rt_lo, size);
1690                let rt_hi = pretty_print_ireg(rt_hi, size);
1691                let rn = pretty_print_ireg(rn, size);
1692
1693                format!("caspal {rd_lo}, {rd_hi}, {rs_lo}, {rs_hi}, {rt_lo}, {rt_hi}, [{rn}]")
1694            }
1695            &Inst::AtomicCASLoop {
1696                ty,
1697                addr,
1698                expected,
1699                replacement,
1700                oldval,
1701                scratch,
1702                ..
1703            } => {
1704                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1705                let expected = pretty_print_ireg(expected, OperandSize::Size64);
1706                let replacement = pretty_print_ireg(replacement, OperandSize::Size64);
1707                let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1708                let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1709                format!(
1710                    "atomic_cas_loop_{} addr={}, expect={}, replacement={}, oldval={}, scratch={}",
1711                    ty.bits(),
1712                    addr,
1713                    expected,
1714                    replacement,
1715                    oldval,
1716                    scratch,
1717                )
1718            }
1719            &Inst::LoadAcquire {
1720                access_ty, rt, rn, ..
1721            } => {
1722                let (op, ty) = match access_ty {
1723                    I8 => ("ldarb", I32),
1724                    I16 => ("ldarh", I32),
1725                    I32 => ("ldar", I32),
1726                    I64 => ("ldar", I64),
1727                    _ => panic!("Unsupported type: {access_ty}"),
1728                };
1729                let size = OperandSize::from_ty(ty);
1730                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1731                let rt = pretty_print_ireg(rt.to_reg(), size);
1732                format!("{op} {rt}, [{rn}]")
1733            }
1734            &Inst::StoreRelease {
1735                access_ty, rt, rn, ..
1736            } => {
1737                let (op, ty) = match access_ty {
1738                    I8 => ("stlrb", I32),
1739                    I16 => ("stlrh", I32),
1740                    I32 => ("stlr", I32),
1741                    I64 => ("stlr", I64),
1742                    _ => panic!("Unsupported type: {access_ty}"),
1743                };
1744                let size = OperandSize::from_ty(ty);
1745                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1746                let rt = pretty_print_ireg(rt, size);
1747                format!("{op} {rt}, [{rn}]")
1748            }
1749            &Inst::Fence {} => {
1750                format!("dmb ish")
1751            }
1752            &Inst::Csdb {} => {
1753                format!("csdb")
1754            }
1755            &Inst::FpuMove32 { rd, rn } => {
1756                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1757                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
1758                format!("fmov {rd}, {rn}")
1759            }
1760            &Inst::FpuMove64 { rd, rn } => {
1761                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1762                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
1763                format!("fmov {rd}, {rn}")
1764            }
1765            &Inst::FpuMove128 { rd, rn } => {
1766                let rd = pretty_print_reg(rd.to_reg());
1767                let rn = pretty_print_reg(rn);
1768                format!("mov {rd}.16b, {rn}.16b")
1769            }
1770            &Inst::FpuMoveFromVec { rd, rn, idx, size } => {
1771                let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
1772                let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
1773                format!("mov {rd}, {rn}")
1774            }
1775            &Inst::FpuExtend { rd, rn, size } => {
1776                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1777                let rn = pretty_print_vreg_scalar(rn, size);
1778                format!("fmov {rd}, {rn}")
1779            }
1780            &Inst::FpuRR {
1781                fpu_op,
1782                size,
1783                rd,
1784                rn,
1785            } => {
1786                let op = match fpu_op {
1787                    FPUOp1::Abs => "fabs",
1788                    FPUOp1::Neg => "fneg",
1789                    FPUOp1::Sqrt => "fsqrt",
1790                    FPUOp1::Cvt32To64 | FPUOp1::Cvt64To32 => "fcvt",
1791                };
1792                let dst_size = match fpu_op {
1793                    FPUOp1::Cvt32To64 => ScalarSize::Size64,
1794                    FPUOp1::Cvt64To32 => ScalarSize::Size32,
1795                    _ => size,
1796                };
1797                let rd = pretty_print_vreg_scalar(rd.to_reg(), dst_size);
1798                let rn = pretty_print_vreg_scalar(rn, size);
1799                format!("{op} {rd}, {rn}")
1800            }
1801            &Inst::FpuRRR {
1802                fpu_op,
1803                size,
1804                rd,
1805                rn,
1806                rm,
1807            } => {
1808                let op = match fpu_op {
1809                    FPUOp2::Add => "fadd",
1810                    FPUOp2::Sub => "fsub",
1811                    FPUOp2::Mul => "fmul",
1812                    FPUOp2::Div => "fdiv",
1813                    FPUOp2::Max => "fmax",
1814                    FPUOp2::Min => "fmin",
1815                };
1816                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1817                let rn = pretty_print_vreg_scalar(rn, size);
1818                let rm = pretty_print_vreg_scalar(rm, size);
1819                format!("{op} {rd}, {rn}, {rm}")
1820            }
1821            &Inst::FpuRRI { fpu_op, rd, rn } => {
1822                let (op, imm, vector) = match fpu_op {
1823                    FPUOpRI::UShr32(imm) => ("ushr", imm.pretty_print(0), true),
1824                    FPUOpRI::UShr64(imm) => ("ushr", imm.pretty_print(0), false),
1825                };
1826
1827                let (rd, rn) = if vector {
1828                    (
1829                        pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
1830                        pretty_print_vreg_vector(rn, VectorSize::Size32x2),
1831                    )
1832                } else {
1833                    (
1834                        pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
1835                        pretty_print_vreg_scalar(rn, ScalarSize::Size64),
1836                    )
1837                };
1838                format!("{op} {rd}, {rn}, {imm}")
1839            }
1840            &Inst::FpuRRIMod { fpu_op, rd, ri, rn } => {
1841                let (op, imm, vector) = match fpu_op {
1842                    FPUOpRIMod::Sli32(imm) => ("sli", imm.pretty_print(0), true),
1843                    FPUOpRIMod::Sli64(imm) => ("sli", imm.pretty_print(0), false),
1844                };
1845
1846                let (rd, ri, rn) = if vector {
1847                    (
1848                        pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
1849                        pretty_print_vreg_vector(ri, VectorSize::Size32x2),
1850                        pretty_print_vreg_vector(rn, VectorSize::Size32x2),
1851                    )
1852                } else {
1853                    (
1854                        pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
1855                        pretty_print_vreg_scalar(ri, ScalarSize::Size64),
1856                        pretty_print_vreg_scalar(rn, ScalarSize::Size64),
1857                    )
1858                };
1859                format!("{op} {rd}, {ri}, {rn}, {imm}")
1860            }
1861            &Inst::FpuRRRR {
1862                fpu_op,
1863                size,
1864                rd,
1865                rn,
1866                rm,
1867                ra,
1868            } => {
1869                let op = match fpu_op {
1870                    FPUOp3::MAdd => "fmadd",
1871                    FPUOp3::MSub => "fmsub",
1872                    FPUOp3::NMAdd => "fnmadd",
1873                    FPUOp3::NMSub => "fnmsub",
1874                };
1875                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1876                let rn = pretty_print_vreg_scalar(rn, size);
1877                let rm = pretty_print_vreg_scalar(rm, size);
1878                let ra = pretty_print_vreg_scalar(ra, size);
1879                format!("{op} {rd}, {rn}, {rm}, {ra}")
1880            }
1881            &Inst::FpuCmp { size, rn, rm } => {
1882                let rn = pretty_print_vreg_scalar(rn, size);
1883                let rm = pretty_print_vreg_scalar(rm, size);
1884                format!("fcmp {rn}, {rm}")
1885            }
1886            &Inst::FpuLoad16 { rd, ref mem, .. } => {
1887                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
1888                let mem = mem.clone();
1889                let access_ty = self.mem_type().unwrap();
1890                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1891                format!("{mem_str}ldr {rd}, {mem}")
1892            }
1893            &Inst::FpuLoad32 { rd, ref mem, .. } => {
1894                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1895                let mem = mem.clone();
1896                let access_ty = self.mem_type().unwrap();
1897                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1898                format!("{mem_str}ldr {rd}, {mem}")
1899            }
1900            &Inst::FpuLoad64 { rd, ref mem, .. } => {
1901                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1902                let mem = mem.clone();
1903                let access_ty = self.mem_type().unwrap();
1904                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1905                format!("{mem_str}ldr {rd}, {mem}")
1906            }
1907            &Inst::FpuLoad128 { rd, ref mem, .. } => {
1908                let rd = pretty_print_reg(rd.to_reg());
1909                let rd = "q".to_string() + &rd[1..];
1910                let mem = mem.clone();
1911                let access_ty = self.mem_type().unwrap();
1912                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1913                format!("{mem_str}ldr {rd}, {mem}")
1914            }
1915            &Inst::FpuStore16 { rd, ref mem, .. } => {
1916                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size16);
1917                let mem = mem.clone();
1918                let access_ty = self.mem_type().unwrap();
1919                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1920                format!("{mem_str}str {rd}, {mem}")
1921            }
1922            &Inst::FpuStore32 { rd, ref mem, .. } => {
1923                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size32);
1924                let mem = mem.clone();
1925                let access_ty = self.mem_type().unwrap();
1926                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1927                format!("{mem_str}str {rd}, {mem}")
1928            }
1929            &Inst::FpuStore64 { rd, ref mem, .. } => {
1930                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size64);
1931                let mem = mem.clone();
1932                let access_ty = self.mem_type().unwrap();
1933                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1934                format!("{mem_str}str {rd}, {mem}")
1935            }
1936            &Inst::FpuStore128 { rd, ref mem, .. } => {
1937                let rd = pretty_print_reg(rd);
1938                let rd = "q".to_string() + &rd[1..];
1939                let mem = mem.clone();
1940                let access_ty = self.mem_type().unwrap();
1941                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1942                format!("{mem_str}str {rd}, {mem}")
1943            }
1944            &Inst::FpuLoadP64 {
1945                rt, rt2, ref mem, ..
1946            } => {
1947                let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size64);
1948                let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size64);
1949                let mem = mem.clone();
1950                let mem = mem.pretty_print_default();
1951
1952                format!("ldp {rt}, {rt2}, {mem}")
1953            }
1954            &Inst::FpuStoreP64 {
1955                rt, rt2, ref mem, ..
1956            } => {
1957                let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size64);
1958                let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size64);
1959                let mem = mem.clone();
1960                let mem = mem.pretty_print_default();
1961
1962                format!("stp {rt}, {rt2}, {mem}")
1963            }
1964            &Inst::FpuLoadP128 {
1965                rt, rt2, ref mem, ..
1966            } => {
1967                let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size128);
1968                let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size128);
1969                let mem = mem.clone();
1970                let mem = mem.pretty_print_default();
1971
1972                format!("ldp {rt}, {rt2}, {mem}")
1973            }
1974            &Inst::FpuStoreP128 {
1975                rt, rt2, ref mem, ..
1976            } => {
1977                let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size128);
1978                let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size128);
1979                let mem = mem.clone();
1980                let mem = mem.pretty_print_default();
1981
1982                format!("stp {rt}, {rt2}, {mem}")
1983            }
1984            &Inst::FpuToInt { op, rd, rn } => {
1985                let (op, sizesrc, sizedest) = match op {
1986                    FpuToIntOp::F32ToI32 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size32),
1987                    FpuToIntOp::F32ToU32 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size32),
1988                    FpuToIntOp::F32ToI64 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size64),
1989                    FpuToIntOp::F32ToU64 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size64),
1990                    FpuToIntOp::F64ToI32 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size32),
1991                    FpuToIntOp::F64ToU32 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size32),
1992                    FpuToIntOp::F64ToI64 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size64),
1993                    FpuToIntOp::F64ToU64 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size64),
1994                };
1995                let rd = pretty_print_ireg(rd.to_reg(), sizedest);
1996                let rn = pretty_print_vreg_scalar(rn, sizesrc);
1997                format!("{op} {rd}, {rn}")
1998            }
1999            &Inst::IntToFpu { op, rd, rn } => {
2000                let (op, sizesrc, sizedest) = match op {
2001                    IntToFpuOp::I32ToF32 => ("scvtf", OperandSize::Size32, ScalarSize::Size32),
2002                    IntToFpuOp::U32ToF32 => ("ucvtf", OperandSize::Size32, ScalarSize::Size32),
2003                    IntToFpuOp::I64ToF32 => ("scvtf", OperandSize::Size64, ScalarSize::Size32),
2004                    IntToFpuOp::U64ToF32 => ("ucvtf", OperandSize::Size64, ScalarSize::Size32),
2005                    IntToFpuOp::I32ToF64 => ("scvtf", OperandSize::Size32, ScalarSize::Size64),
2006                    IntToFpuOp::U32ToF64 => ("ucvtf", OperandSize::Size32, ScalarSize::Size64),
2007                    IntToFpuOp::I64ToF64 => ("scvtf", OperandSize::Size64, ScalarSize::Size64),
2008                    IntToFpuOp::U64ToF64 => ("ucvtf", OperandSize::Size64, ScalarSize::Size64),
2009                };
2010                let rd = pretty_print_vreg_scalar(rd.to_reg(), sizedest);
2011                let rn = pretty_print_ireg(rn, sizesrc);
2012                format!("{op} {rd}, {rn}")
2013            }
2014            &Inst::FpuCSel16 { rd, rn, rm, cond } => {
2015                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
2016                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size16);
2017                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size16);
2018                let cond = cond.pretty_print(0);
2019                format!("fcsel {rd}, {rn}, {rm}, {cond}")
2020            }
2021            &Inst::FpuCSel32 { rd, rn, rm, cond } => {
2022                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
2023                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
2024                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size32);
2025                let cond = cond.pretty_print(0);
2026                format!("fcsel {rd}, {rn}, {rm}, {cond}")
2027            }
2028            &Inst::FpuCSel64 { rd, rn, rm, cond } => {
2029                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2030                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
2031                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size64);
2032                let cond = cond.pretty_print(0);
2033                format!("fcsel {rd}, {rn}, {rm}, {cond}")
2034            }
2035            &Inst::FpuRound { op, rd, rn } => {
2036                let (inst, size) = match op {
2037                    FpuRoundMode::Minus32 => ("frintm", ScalarSize::Size32),
2038                    FpuRoundMode::Minus64 => ("frintm", ScalarSize::Size64),
2039                    FpuRoundMode::Plus32 => ("frintp", ScalarSize::Size32),
2040                    FpuRoundMode::Plus64 => ("frintp", ScalarSize::Size64),
2041                    FpuRoundMode::Zero32 => ("frintz", ScalarSize::Size32),
2042                    FpuRoundMode::Zero64 => ("frintz", ScalarSize::Size64),
2043                    FpuRoundMode::Nearest32 => ("frintn", ScalarSize::Size32),
2044                    FpuRoundMode::Nearest64 => ("frintn", ScalarSize::Size64),
2045                };
2046                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2047                let rn = pretty_print_vreg_scalar(rn, size);
2048                format!("{inst} {rd}, {rn}")
2049            }
2050            &Inst::MovToFpu { rd, rn, size } => {
2051                let operand_size = size.operand_size();
2052                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2053                let rn = pretty_print_ireg(rn, operand_size);
2054                format!("fmov {rd}, {rn}")
2055            }
2056            &Inst::FpuMoveFPImm { rd, imm, size } => {
2057                let imm = imm.pretty_print(0);
2058                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
2059
2060                format!("fmov {rd}, {imm}")
2061            }
2062            &Inst::MovToVec {
2063                rd,
2064                ri,
2065                rn,
2066                idx,
2067                size,
2068            } => {
2069                let rd = pretty_print_vreg_element(rd.to_reg(), idx as usize, size.lane_size());
2070                let ri = pretty_print_vreg_element(ri, idx as usize, size.lane_size());
2071                let rn = pretty_print_ireg(rn, size.operand_size());
2072                format!("mov {rd}, {ri}, {rn}")
2073            }
2074            &Inst::MovFromVec { rd, rn, idx, size } => {
2075                let op = match size {
2076                    ScalarSize::Size8 => "umov",
2077                    ScalarSize::Size16 => "umov",
2078                    ScalarSize::Size32 => "mov",
2079                    ScalarSize::Size64 => "mov",
2080                    _ => unimplemented!(),
2081                };
2082                let rd = pretty_print_ireg(rd.to_reg(), size.operand_size());
2083                let rn = pretty_print_vreg_element(rn, idx as usize, size);
2084                format!("{op} {rd}, {rn}")
2085            }
2086            &Inst::MovFromVecSigned {
2087                rd,
2088                rn,
2089                idx,
2090                size,
2091                scalar_size,
2092            } => {
2093                let rd = pretty_print_ireg(rd.to_reg(), scalar_size);
2094                let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
2095                format!("smov {rd}, {rn}")
2096            }
2097            &Inst::VecDup { rd, rn, size } => {
2098                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2099                let rn = pretty_print_ireg(rn, size.operand_size());
2100                format!("dup {rd}, {rn}")
2101            }
2102            &Inst::VecDupFromFpu { rd, rn, size, lane } => {
2103                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2104                let rn = pretty_print_vreg_element(rn, lane.into(), size.lane_size());
2105                format!("dup {rd}, {rn}")
2106            }
2107            &Inst::VecDupFPImm { rd, imm, size } => {
2108                let imm = imm.pretty_print(0);
2109                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2110
2111                format!("fmov {rd}, {imm}")
2112            }
2113            &Inst::VecDupImm {
2114                rd,
2115                imm,
2116                invert,
2117                size,
2118            } => {
2119                let imm = imm.pretty_print(0);
2120                let op = if invert { "mvni" } else { "movi" };
2121                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2122
2123                format!("{op} {rd}, {imm}")
2124            }
2125            &Inst::VecExtend {
2126                t,
2127                rd,
2128                rn,
2129                high_half,
2130                lane_size,
2131            } => {
2132                let vec64 = VectorSize::from_lane_size(lane_size.narrow(), false);
2133                let vec128 = VectorSize::from_lane_size(lane_size.narrow(), true);
2134                let rd_size = VectorSize::from_lane_size(lane_size, true);
2135                let (op, rn_size) = match (t, high_half) {
2136                    (VecExtendOp::Sxtl, false) => ("sxtl", vec64),
2137                    (VecExtendOp::Sxtl, true) => ("sxtl2", vec128),
2138                    (VecExtendOp::Uxtl, false) => ("uxtl", vec64),
2139                    (VecExtendOp::Uxtl, true) => ("uxtl2", vec128),
2140                };
2141                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2142                let rn = pretty_print_vreg_vector(rn, rn_size);
2143                format!("{op} {rd}, {rn}")
2144            }
2145            &Inst::VecMovElement {
2146                rd,
2147                ri,
2148                rn,
2149                dest_idx,
2150                src_idx,
2151                size,
2152            } => {
2153                let rd =
2154                    pretty_print_vreg_element(rd.to_reg(), dest_idx as usize, size.lane_size());
2155                let ri = pretty_print_vreg_element(ri, dest_idx as usize, size.lane_size());
2156                let rn = pretty_print_vreg_element(rn, src_idx as usize, size.lane_size());
2157                format!("mov {rd}, {ri}, {rn}")
2158            }
2159            &Inst::VecRRLong {
2160                op,
2161                rd,
2162                rn,
2163                high_half,
2164            } => {
2165                let (op, rd_size, size, suffix) = match (op, high_half) {
2166                    (VecRRLongOp::Fcvtl16, false) => {
2167                        ("fcvtl", VectorSize::Size32x4, VectorSize::Size16x4, "")
2168                    }
2169                    (VecRRLongOp::Fcvtl16, true) => {
2170                        ("fcvtl2", VectorSize::Size32x4, VectorSize::Size16x8, "")
2171                    }
2172                    (VecRRLongOp::Fcvtl32, false) => {
2173                        ("fcvtl", VectorSize::Size64x2, VectorSize::Size32x2, "")
2174                    }
2175                    (VecRRLongOp::Fcvtl32, true) => {
2176                        ("fcvtl2", VectorSize::Size64x2, VectorSize::Size32x4, "")
2177                    }
2178                    (VecRRLongOp::Shll8, false) => {
2179                        ("shll", VectorSize::Size16x8, VectorSize::Size8x8, ", #8")
2180                    }
2181                    (VecRRLongOp::Shll8, true) => {
2182                        ("shll2", VectorSize::Size16x8, VectorSize::Size8x16, ", #8")
2183                    }
2184                    (VecRRLongOp::Shll16, false) => {
2185                        ("shll", VectorSize::Size32x4, VectorSize::Size16x4, ", #16")
2186                    }
2187                    (VecRRLongOp::Shll16, true) => {
2188                        ("shll2", VectorSize::Size32x4, VectorSize::Size16x8, ", #16")
2189                    }
2190                    (VecRRLongOp::Shll32, false) => {
2191                        ("shll", VectorSize::Size64x2, VectorSize::Size32x2, ", #32")
2192                    }
2193                    (VecRRLongOp::Shll32, true) => {
2194                        ("shll2", VectorSize::Size64x2, VectorSize::Size32x4, ", #32")
2195                    }
2196                };
2197                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2198                let rn = pretty_print_vreg_vector(rn, size);
2199
2200                format!("{op} {rd}, {rn}{suffix}")
2201            }
2202            &Inst::VecRRNarrowLow {
2203                op,
2204                rd,
2205                rn,
2206                lane_size,
2207                ..
2208            }
2209            | &Inst::VecRRNarrowHigh {
2210                op,
2211                rd,
2212                rn,
2213                lane_size,
2214                ..
2215            } => {
2216                let vec64 = VectorSize::from_lane_size(lane_size, false);
2217                let vec128 = VectorSize::from_lane_size(lane_size, true);
2218                let rn_size = VectorSize::from_lane_size(lane_size.widen(), true);
2219                let high_half = match self {
2220                    &Inst::VecRRNarrowLow { .. } => false,
2221                    &Inst::VecRRNarrowHigh { .. } => true,
2222                    _ => unreachable!(),
2223                };
2224                let (op, rd_size) = match (op, high_half) {
2225                    (VecRRNarrowOp::Xtn, false) => ("xtn", vec64),
2226                    (VecRRNarrowOp::Xtn, true) => ("xtn2", vec128),
2227                    (VecRRNarrowOp::Sqxtn, false) => ("sqxtn", vec64),
2228                    (VecRRNarrowOp::Sqxtn, true) => ("sqxtn2", vec128),
2229                    (VecRRNarrowOp::Sqxtun, false) => ("sqxtun", vec64),
2230                    (VecRRNarrowOp::Sqxtun, true) => ("sqxtun2", vec128),
2231                    (VecRRNarrowOp::Uqxtn, false) => ("uqxtn", vec64),
2232                    (VecRRNarrowOp::Uqxtn, true) => ("uqxtn2", vec128),
2233                    (VecRRNarrowOp::Fcvtn, false) => ("fcvtn", vec64),
2234                    (VecRRNarrowOp::Fcvtn, true) => ("fcvtn2", vec128),
2235                };
2236                let rn = pretty_print_vreg_vector(rn, rn_size);
2237                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2238                let ri = match self {
2239                    &Inst::VecRRNarrowLow { .. } => "".to_string(),
2240                    &Inst::VecRRNarrowHigh { ri, .. } => {
2241                        format!("{}, ", pretty_print_vreg_vector(ri, rd_size))
2242                    }
2243                    _ => unreachable!(),
2244                };
2245
2246                format!("{op} {rd}, {ri}{rn}")
2247            }
2248            &Inst::VecRRPair { op, rd, rn } => {
2249                let op = match op {
2250                    VecPairOp::Addp => "addp",
2251                };
2252                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2253                let rn = pretty_print_vreg_vector(rn, VectorSize::Size64x2);
2254
2255                format!("{op} {rd}, {rn}")
2256            }
2257            &Inst::VecRRPairLong { op, rd, rn } => {
2258                let (op, dest, src) = match op {
2259                    VecRRPairLongOp::Saddlp8 => {
2260                        ("saddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2261                    }
2262                    VecRRPairLongOp::Saddlp16 => {
2263                        ("saddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2264                    }
2265                    VecRRPairLongOp::Uaddlp8 => {
2266                        ("uaddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2267                    }
2268                    VecRRPairLongOp::Uaddlp16 => {
2269                        ("uaddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2270                    }
2271                };
2272                let rd = pretty_print_vreg_vector(rd.to_reg(), dest);
2273                let rn = pretty_print_vreg_vector(rn, src);
2274
2275                format!("{op} {rd}, {rn}")
2276            }
2277            &Inst::VecRRR {
2278                rd,
2279                rn,
2280                rm,
2281                alu_op,
2282                size,
2283            } => {
2284                let (op, size) = match alu_op {
2285                    VecALUOp::Sqadd => ("sqadd", size),
2286                    VecALUOp::Uqadd => ("uqadd", size),
2287                    VecALUOp::Sqsub => ("sqsub", size),
2288                    VecALUOp::Uqsub => ("uqsub", size),
2289                    VecALUOp::Cmeq => ("cmeq", size),
2290                    VecALUOp::Cmge => ("cmge", size),
2291                    VecALUOp::Cmgt => ("cmgt", size),
2292                    VecALUOp::Cmhs => ("cmhs", size),
2293                    VecALUOp::Cmhi => ("cmhi", size),
2294                    VecALUOp::Fcmeq => ("fcmeq", size),
2295                    VecALUOp::Fcmgt => ("fcmgt", size),
2296                    VecALUOp::Fcmge => ("fcmge", size),
2297                    VecALUOp::Umaxp => ("umaxp", size),
2298                    VecALUOp::Add => ("add", size),
2299                    VecALUOp::Sub => ("sub", size),
2300                    VecALUOp::Mul => ("mul", size),
2301                    VecALUOp::Sshl => ("sshl", size),
2302                    VecALUOp::Ushl => ("ushl", size),
2303                    VecALUOp::Umin => ("umin", size),
2304                    VecALUOp::Smin => ("smin", size),
2305                    VecALUOp::Umax => ("umax", size),
2306                    VecALUOp::Smax => ("smax", size),
2307                    VecALUOp::Urhadd => ("urhadd", size),
2308                    VecALUOp::Fadd => ("fadd", size),
2309                    VecALUOp::Fsub => ("fsub", size),
2310                    VecALUOp::Fdiv => ("fdiv", size),
2311                    VecALUOp::Fmax => ("fmax", size),
2312                    VecALUOp::Fmin => ("fmin", size),
2313                    VecALUOp::Fmul => ("fmul", size),
2314                    VecALUOp::Addp => ("addp", size),
2315                    VecALUOp::Zip1 => ("zip1", size),
2316                    VecALUOp::Zip2 => ("zip2", size),
2317                    VecALUOp::Sqrdmulh => ("sqrdmulh", size),
2318                    VecALUOp::Uzp1 => ("uzp1", size),
2319                    VecALUOp::Uzp2 => ("uzp2", size),
2320                    VecALUOp::Trn1 => ("trn1", size),
2321                    VecALUOp::Trn2 => ("trn2", size),
2322
2323                    // Lane division does not affect bitwise operations.
2324                    // However, when printing, use 8-bit lane division to conform to ARM formatting.
2325                    VecALUOp::And => ("and", size.as_scalar8_vector()),
2326                    VecALUOp::Bic => ("bic", size.as_scalar8_vector()),
2327                    VecALUOp::Orr => ("orr", size.as_scalar8_vector()),
2328                    VecALUOp::Orn => ("orn", size.as_scalar8_vector()),
2329                    VecALUOp::Eor => ("eor", size.as_scalar8_vector()),
2330                };
2331                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2332                let rn = pretty_print_vreg_vector(rn, size);
2333                let rm = pretty_print_vreg_vector(rm, size);
2334                format!("{op} {rd}, {rn}, {rm}")
2335            }
2336            &Inst::VecRRRMod {
2337                rd,
2338                ri,
2339                rn,
2340                rm,
2341                alu_op,
2342                size,
2343            } => {
2344                let (op, size) = match alu_op {
2345                    VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
2346                    VecALUModOp::Fmla => ("fmla", size),
2347                    VecALUModOp::Fmls => ("fmls", size),
2348                    // Note: the real operand arrangement is .4s, .16b, .16b;
2349                    // this debug print renders all lanes as .4s.
2350                    VecALUModOp::Sdot => ("sdot", VectorSize::Size32x4),
2351                    VecALUModOp::Usdot => ("usdot", VectorSize::Size32x4),
2352                };
2353                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2354                let ri = pretty_print_vreg_vector(ri, size);
2355                let rn = pretty_print_vreg_vector(rn, size);
2356                let rm = pretty_print_vreg_vector(rm, size);
2357                format!("{op} {rd}, {ri}, {rn}, {rm}")
2358            }
2359            &Inst::VecFmlaElem {
2360                rd,
2361                ri,
2362                rn,
2363                rm,
2364                alu_op,
2365                size,
2366                idx,
2367            } => {
2368                let (op, size) = match alu_op {
2369                    VecALUModOp::Fmla => ("fmla", size),
2370                    VecALUModOp::Fmls => ("fmls", size),
2371                    _ => unreachable!(),
2372                };
2373                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2374                let ri = pretty_print_vreg_vector(ri, size);
2375                let rn = pretty_print_vreg_vector(rn, size);
2376                let rm = pretty_print_vreg_element(rm, idx.into(), size.lane_size());
2377                format!("{op} {rd}, {ri}, {rn}, {rm}")
2378            }
2379            &Inst::VecRRRLong {
2380                rd,
2381                rn,
2382                rm,
2383                alu_op,
2384                high_half,
2385            } => {
2386                let (op, dest_size, src_size) = match (alu_op, high_half) {
2387                    (VecRRRLongOp::Smull8, false) => {
2388                        ("smull", VectorSize::Size16x8, VectorSize::Size8x8)
2389                    }
2390                    (VecRRRLongOp::Smull8, true) => {
2391                        ("smull2", VectorSize::Size16x8, VectorSize::Size8x16)
2392                    }
2393                    (VecRRRLongOp::Smull16, false) => {
2394                        ("smull", VectorSize::Size32x4, VectorSize::Size16x4)
2395                    }
2396                    (VecRRRLongOp::Smull16, true) => {
2397                        ("smull2", VectorSize::Size32x4, VectorSize::Size16x8)
2398                    }
2399                    (VecRRRLongOp::Smull32, false) => {
2400                        ("smull", VectorSize::Size64x2, VectorSize::Size32x2)
2401                    }
2402                    (VecRRRLongOp::Smull32, true) => {
2403                        ("smull2", VectorSize::Size64x2, VectorSize::Size32x4)
2404                    }
2405                    (VecRRRLongOp::Umull8, false) => {
2406                        ("umull", VectorSize::Size16x8, VectorSize::Size8x8)
2407                    }
2408                    (VecRRRLongOp::Umull8, true) => {
2409                        ("umull2", VectorSize::Size16x8, VectorSize::Size8x16)
2410                    }
2411                    (VecRRRLongOp::Umull16, false) => {
2412                        ("umull", VectorSize::Size32x4, VectorSize::Size16x4)
2413                    }
2414                    (VecRRRLongOp::Umull16, true) => {
2415                        ("umull2", VectorSize::Size32x4, VectorSize::Size16x8)
2416                    }
2417                    (VecRRRLongOp::Umull32, false) => {
2418                        ("umull", VectorSize::Size64x2, VectorSize::Size32x2)
2419                    }
2420                    (VecRRRLongOp::Umull32, true) => {
2421                        ("umull2", VectorSize::Size64x2, VectorSize::Size32x4)
2422                    }
2423                };
2424                let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2425                let rn = pretty_print_vreg_vector(rn, src_size);
2426                let rm = pretty_print_vreg_vector(rm, src_size);
2427                format!("{op} {rd}, {rn}, {rm}")
2428            }
2429            &Inst::VecRRRLongMod {
2430                rd,
2431                ri,
2432                rn,
2433                rm,
2434                alu_op,
2435                high_half,
2436            } => {
2437                let (op, dest_size, src_size) = match (alu_op, high_half) {
2438                    (VecRRRLongModOp::Umlal8, false) => {
2439                        ("umlal", VectorSize::Size16x8, VectorSize::Size8x8)
2440                    }
2441                    (VecRRRLongModOp::Umlal8, true) => {
2442                        ("umlal2", VectorSize::Size16x8, VectorSize::Size8x16)
2443                    }
2444                    (VecRRRLongModOp::Umlal16, false) => {
2445                        ("umlal", VectorSize::Size32x4, VectorSize::Size16x4)
2446                    }
2447                    (VecRRRLongModOp::Umlal16, true) => {
2448                        ("umlal2", VectorSize::Size32x4, VectorSize::Size16x8)
2449                    }
2450                    (VecRRRLongModOp::Umlal32, false) => {
2451                        ("umlal", VectorSize::Size64x2, VectorSize::Size32x2)
2452                    }
2453                    (VecRRRLongModOp::Umlal32, true) => {
2454                        ("umlal2", VectorSize::Size64x2, VectorSize::Size32x4)
2455                    }
2456                };
2457                let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2458                let ri = pretty_print_vreg_vector(ri, dest_size);
2459                let rn = pretty_print_vreg_vector(rn, src_size);
2460                let rm = pretty_print_vreg_vector(rm, src_size);
2461                format!("{op} {rd}, {ri}, {rn}, {rm}")
2462            }
2463            &Inst::VecMisc { op, rd, rn, size } => {
2464                let (op, size, suffix) = match op {
2465                    VecMisc2::Neg => ("neg", size, ""),
2466                    VecMisc2::Abs => ("abs", size, ""),
2467                    VecMisc2::Fabs => ("fabs", size, ""),
2468                    VecMisc2::Fneg => ("fneg", size, ""),
2469                    VecMisc2::Fsqrt => ("fsqrt", size, ""),
2470                    VecMisc2::Rev16 => ("rev16", size, ""),
2471                    VecMisc2::Rev32 => ("rev32", size, ""),
2472                    VecMisc2::Rev64 => ("rev64", size, ""),
2473                    VecMisc2::Fcvtzs => ("fcvtzs", size, ""),
2474                    VecMisc2::Fcvtzu => ("fcvtzu", size, ""),
2475                    VecMisc2::Scvtf => ("scvtf", size, ""),
2476                    VecMisc2::Ucvtf => ("ucvtf", size, ""),
2477                    VecMisc2::Frintn => ("frintn", size, ""),
2478                    VecMisc2::Frintz => ("frintz", size, ""),
2479                    VecMisc2::Frintm => ("frintm", size, ""),
2480                    VecMisc2::Frintp => ("frintp", size, ""),
2481                    VecMisc2::Cnt => ("cnt", size, ""),
2482                    VecMisc2::Cmeq0 => ("cmeq", size, ", #0"),
2483                    VecMisc2::Cmge0 => ("cmge", size, ", #0"),
2484                    VecMisc2::Cmgt0 => ("cmgt", size, ", #0"),
2485                    VecMisc2::Cmle0 => ("cmle", size, ", #0"),
2486                    VecMisc2::Cmlt0 => ("cmlt", size, ", #0"),
2487                    VecMisc2::Fcmeq0 => ("fcmeq", size, ", #0.0"),
2488                    VecMisc2::Fcmge0 => ("fcmge", size, ", #0.0"),
2489                    VecMisc2::Fcmgt0 => ("fcmgt", size, ", #0.0"),
2490                    VecMisc2::Fcmle0 => ("fcmle", size, ", #0.0"),
2491                    VecMisc2::Fcmlt0 => ("fcmlt", size, ", #0.0"),
2492
2493                    // Lane division does not affect bitwise operations.
2494                    // However, when printing, use 8-bit lane division to conform to ARM formatting.
2495                    VecMisc2::Not => ("mvn", size.as_scalar8_vector(), ""),
2496                };
2497                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2498                let rn = pretty_print_vreg_vector(rn, size);
2499                format!("{op} {rd}, {rn}{suffix}")
2500            }
2501            &Inst::VecLanes { op, rd, rn, size } => {
2502                let op = match op {
2503                    VecLanesOp::Uminv => "uminv",
2504                    VecLanesOp::Addv => "addv",
2505                };
2506                let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
2507                let rn = pretty_print_vreg_vector(rn, size);
2508                format!("{op} {rd}, {rn}")
2509            }
2510            &Inst::VecShiftImm {
2511                op,
2512                rd,
2513                rn,
2514                size,
2515                imm,
2516            } => {
2517                let op = match op {
2518                    VecShiftImmOp::Shl => "shl",
2519                    VecShiftImmOp::Ushr => "ushr",
2520                    VecShiftImmOp::Sshr => "sshr",
2521                };
2522                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2523                let rn = pretty_print_vreg_vector(rn, size);
2524                format!("{op} {rd}, {rn}, #{imm}")
2525            }
2526            &Inst::VecShiftImmMod {
2527                op,
2528                rd,
2529                ri,
2530                rn,
2531                size,
2532                imm,
2533            } => {
2534                let op = match op {
2535                    VecShiftImmModOp::Sli => "sli",
2536                };
2537                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2538                let ri = pretty_print_vreg_vector(ri, size);
2539                let rn = pretty_print_vreg_vector(rn, size);
2540                format!("{op} {rd}, {ri}, {rn}, #{imm}")
2541            }
2542            &Inst::VecExtract { rd, rn, rm, imm4 } => {
2543                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2544                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2545                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2546                format!("ext {rd}, {rn}, {rm}, #{imm4}")
2547            }
2548            &Inst::VecTbl { rd, rn, rm } => {
2549                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2550                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2551                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2552                format!("tbl {rd}, {{ {rn} }}, {rm}")
2553            }
2554            &Inst::VecTblExt { rd, ri, rn, rm } => {
2555                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2556                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2557                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2558                let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2559                format!("tbx {rd}, {ri}, {{ {rn} }}, {rm}")
2560            }
2561            &Inst::VecTbl2 { rd, rn, rn2, rm } => {
2562                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2563                let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2564                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2565                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2566                format!("tbl {rd}, {{ {rn}, {rn2} }}, {rm}")
2567            }
2568            &Inst::VecTbl2Ext {
2569                rd,
2570                ri,
2571                rn,
2572                rn2,
2573                rm,
2574            } => {
2575                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2576                let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2577                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2578                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2579                let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2580                format!("tbx {rd}, {ri}, {{ {rn}, {rn2} }}, {rm}")
2581            }
2582            &Inst::VecLoadReplicate { rd, rn, size, .. } => {
2583                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2584                let rn = pretty_print_reg(rn);
2585
2586                format!("ld1r {{ {rd} }}, [{rn}]")
2587            }
2588            &Inst::VecCSel { rd, rn, rm, cond } => {
2589                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2590                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2591                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2592                let cond = cond.pretty_print(0);
2593                format!("vcsel {rd}, {rn}, {rm}, {cond} (if-then-else diamond)")
2594            }
2595            &Inst::MovToNZCV { rn } => {
2596                let rn = pretty_print_reg(rn);
2597                format!("msr nzcv, {rn}")
2598            }
2599            &Inst::MovFromNZCV { rd } => {
2600                let rd = pretty_print_reg(rd.to_reg());
2601                format!("mrs {rd}, nzcv")
2602            }
2603            &Inst::Extend {
2604                rd,
2605                rn,
2606                signed: false,
2607                from_bits: 1,
2608                ..
2609            } => {
2610                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size32);
2611                let rn = pretty_print_ireg(rn, OperandSize::Size32);
2612                format!("and {rd}, {rn}, #1")
2613            }
2614            &Inst::Extend {
2615                rd,
2616                rn,
2617                signed: false,
2618                from_bits: 32,
2619                to_bits: 64,
2620            } => {
2621                // The case of a zero extension from 32 to 64 bits, is implemented
2622                // with a "mov" to a 32-bit (W-reg) dest, because this zeroes
2623                // the top 32 bits.
2624                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size32);
2625                let rn = pretty_print_ireg(rn, OperandSize::Size32);
2626                format!("mov {rd}, {rn}")
2627            }
2628            &Inst::Extend {
2629                rd,
2630                rn,
2631                signed,
2632                from_bits,
2633                to_bits,
2634            } => {
2635                assert!(from_bits <= to_bits);
2636                let op = match (signed, from_bits) {
2637                    (false, 8) => "uxtb",
2638                    (true, 8) => "sxtb",
2639                    (false, 16) => "uxth",
2640                    (true, 16) => "sxth",
2641                    (true, 32) => "sxtw",
2642                    (true, _) => "sbfx",
2643                    (false, _) => "ubfx",
2644                };
2645                if op == "sbfx" || op == "ubfx" {
2646                    let dest_size = OperandSize::from_bits(to_bits);
2647                    let rd = pretty_print_ireg(rd.to_reg(), dest_size);
2648                    let rn = pretty_print_ireg(rn, dest_size);
2649                    format!("{op} {rd}, {rn}, #0, #{from_bits}")
2650                } else {
2651                    let dest_size = if signed {
2652                        OperandSize::from_bits(to_bits)
2653                    } else {
2654                        OperandSize::Size32
2655                    };
2656                    let rd = pretty_print_ireg(rd.to_reg(), dest_size);
2657                    let rn = pretty_print_ireg(rn, OperandSize::from_bits(from_bits));
2658                    format!("{op} {rd}, {rn}")
2659                }
2660            }
2661            &Inst::BitfieldMove {
2662                size,
2663                bfm_op,
2664                rd,
2665                rn,
2666                immr,
2667                imms,
2668            } => {
2669                let op = bfm_op.op_str();
2670                let rd = pretty_print_ireg(rd.to_reg(), size);
2671                let rn = pretty_print_ireg(rn, size);
2672                let immr = immr.pretty_print(0);
2673                let imms = imms.pretty_print(0);
2674                format!("{op} {rd}, {rn}, {immr}, {imms}")
2675            }
2676            &Inst::BitfieldMoveMod {
2677                size,
2678                rd,
2679                ri,
2680                rn,
2681                immr,
2682                imms,
2683            } => {
2684                let rd = pretty_print_ireg(rd.to_reg(), size);
2685                let ri = pretty_print_ireg(ri, size);
2686                let rn = pretty_print_ireg(rn, size);
2687                let immr = immr.pretty_print(0);
2688                let imms = imms.pretty_print(0);
2689                format!("bfm {rd}, {ri}, {rn}, {immr}, {imms}")
2690            }
2691            &Inst::Call { ref info } => {
2692                let try_call = info
2693                    .try_call_info
2694                    .as_ref()
2695                    .map(|tci| pretty_print_try_call(tci))
2696                    .unwrap_or_default();
2697                format!("bl 0{try_call}")
2698            }
2699            &Inst::CallInd { ref info } => {
2700                let rn = pretty_print_reg(info.dest);
2701                let try_call = info
2702                    .try_call_info
2703                    .as_ref()
2704                    .map(|tci| pretty_print_try_call(tci))
2705                    .unwrap_or_default();
2706                format!("blr {rn}{try_call}")
2707            }
2708            &Inst::ReturnCall { ref info } => {
2709                let mut s = format!(
2710                    "return_call {:?} new_stack_arg_size:{}",
2711                    info.dest, info.new_stack_arg_size
2712                );
2713                for ret in &info.uses {
2714                    let preg = pretty_print_reg(ret.preg);
2715                    let vreg = pretty_print_reg(ret.vreg);
2716                    write!(&mut s, " {vreg}={preg}").unwrap();
2717                }
2718                s
2719            }
2720            &Inst::ReturnCallInd { ref info } => {
2721                let callee = pretty_print_reg(info.dest);
2722                let mut s = format!(
2723                    "return_call_ind {callee} new_stack_arg_size:{}",
2724                    info.new_stack_arg_size
2725                );
2726                for ret in &info.uses {
2727                    let preg = pretty_print_reg(ret.preg);
2728                    let vreg = pretty_print_reg(ret.vreg);
2729                    write!(&mut s, " {vreg}={preg}").unwrap();
2730                }
2731                s
2732            }
2733            &Inst::Args { ref args } => {
2734                let mut s = "args".to_string();
2735                for arg in args {
2736                    let preg = pretty_print_reg(arg.preg);
2737                    let def = pretty_print_reg(arg.vreg.to_reg());
2738                    write!(&mut s, " {def}={preg}").unwrap();
2739                }
2740                s
2741            }
2742            &Inst::Rets { ref rets } => {
2743                let mut s = "rets".to_string();
2744                for ret in rets {
2745                    let preg = pretty_print_reg(ret.preg);
2746                    let vreg = pretty_print_reg(ret.vreg);
2747                    write!(&mut s, " {vreg}={preg}").unwrap();
2748                }
2749                s
2750            }
2751            &Inst::Ret {} => "ret".to_string(),
2752            &Inst::AuthenticatedRet { key, is_hint } => {
2753                let key = match key {
2754                    APIKey::AZ => "az",
2755                    APIKey::BZ => "bz",
2756                    APIKey::ASP => "asp",
2757                    APIKey::BSP => "bsp",
2758                };
2759                match is_hint {
2760                    false => format!("reta{key}"),
2761                    true => format!("auti{key} ; ret"),
2762                }
2763            }
2764            &Inst::Jump { ref dest } => {
2765                let dest = dest.pretty_print(0);
2766                format!("b {dest}")
2767            }
2768            &Inst::CondBr {
2769                ref taken,
2770                ref not_taken,
2771                ref kind,
2772            } => {
2773                let taken = taken.pretty_print(0);
2774                let not_taken = not_taken.pretty_print(0);
2775                match kind {
2776                    &CondBrKind::Zero(reg, size) => {
2777                        let reg = pretty_print_reg_sized(reg, size);
2778                        format!("cbz {reg}, {taken} ; b {not_taken}")
2779                    }
2780                    &CondBrKind::NotZero(reg, size) => {
2781                        let reg = pretty_print_reg_sized(reg, size);
2782                        format!("cbnz {reg}, {taken} ; b {not_taken}")
2783                    }
2784                    &CondBrKind::Cond(c) => {
2785                        let c = c.pretty_print(0);
2786                        format!("b.{c} {taken} ; b {not_taken}")
2787                    }
2788                }
2789            }
2790            &Inst::TestBitAndBranch {
2791                kind,
2792                ref taken,
2793                ref not_taken,
2794                rn,
2795                bit,
2796            } => {
2797                let cond = match kind {
2798                    TestBitAndBranchKind::Z => "z",
2799                    TestBitAndBranchKind::NZ => "nz",
2800                };
2801                let taken = taken.pretty_print(0);
2802                let not_taken = not_taken.pretty_print(0);
2803                let rn = pretty_print_reg(rn);
2804                format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
2805            }
2806            &Inst::IndirectBr { rn, .. } => {
2807                let rn = pretty_print_reg(rn);
2808                format!("br {rn}")
2809            }
2810            &Inst::Brk => "brk #0xf000".to_string(),
2811            &Inst::Udf { .. } => "udf #0xc11f".to_string(),
2812            &Inst::TrapIf {
2813                ref kind,
2814                trap_code,
2815            } => match kind {
2816                &CondBrKind::Zero(reg, size) => {
2817                    let reg = pretty_print_reg_sized(reg, size);
2818                    format!("cbz {reg}, #trap={trap_code}")
2819                }
2820                &CondBrKind::NotZero(reg, size) => {
2821                    let reg = pretty_print_reg_sized(reg, size);
2822                    format!("cbnz {reg}, #trap={trap_code}")
2823                }
2824                &CondBrKind::Cond(c) => {
2825                    let c = c.pretty_print(0);
2826                    format!("b.{c} #trap={trap_code}")
2827                }
2828            },
2829            &Inst::Adr { rd, off } => {
2830                let rd = pretty_print_reg(rd.to_reg());
2831                format!("adr {rd}, pc+{off}")
2832            }
2833            &Inst::Adrp { rd, off } => {
2834                let rd = pretty_print_reg(rd.to_reg());
2835                // This instruction addresses 4KiB pages, so multiply it by the page size.
2836                let byte_offset = off * 4096;
2837                format!("adrp {rd}, pc+{byte_offset}")
2838            }
2839            &Inst::Word4 { data } => format!("data.i32 {data}"),
2840            &Inst::Word8 { data } => format!("data.i64 {data}"),
2841            &Inst::JTSequence {
2842                default,
2843                ref targets,
2844                ridx,
2845                rtmp1,
2846                rtmp2,
2847                ..
2848            } => {
2849                let ridx = pretty_print_reg(ridx);
2850                let rtmp1 = pretty_print_reg(rtmp1.to_reg());
2851                let rtmp2 = pretty_print_reg(rtmp2.to_reg());
2852                let default_target = BranchTarget::Label(default).pretty_print(0);
2853                format!(
2854                    concat!(
2855                        "b.hs {} ; ",
2856                        "csel {}, xzr, {}, hs ; ",
2857                        "csdb ; ",
2858                        "adr {}, pc+16 ; ",
2859                        "ldrsw {}, [{}, {}, uxtw #2] ; ",
2860                        "add {}, {}, {} ; ",
2861                        "br {} ; ",
2862                        "jt_entries {:?}"
2863                    ),
2864                    default_target,
2865                    rtmp2,
2866                    ridx,
2867                    rtmp1,
2868                    rtmp2,
2869                    rtmp1,
2870                    rtmp2,
2871                    rtmp1,
2872                    rtmp1,
2873                    rtmp2,
2874                    rtmp1,
2875                    targets
2876                )
2877            }
2878            &Inst::LoadExtNameGot { rd, ref name } => {
2879                let rd = pretty_print_reg(rd.to_reg());
2880                format!("load_ext_name_got {rd}, {name:?}")
2881            }
2882            &Inst::LoadExtNameNear {
2883                rd,
2884                ref name,
2885                offset,
2886            } => {
2887                let rd = pretty_print_reg(rd.to_reg());
2888                format!("load_ext_name_near {rd}, {name:?}+{offset}")
2889            }
2890            &Inst::LoadExtNameFar {
2891                rd,
2892                ref name,
2893                offset,
2894            } => {
2895                let rd = pretty_print_reg(rd.to_reg());
2896                format!("load_ext_name_far {rd}, {name:?}+{offset}")
2897            }
2898            &Inst::LoadAddr { rd, ref mem } => {
2899                // TODO: we really should find a better way to avoid duplication of
2900                // this logic between `emit()` and `show_rru()` -- a separate 1-to-N
2901                // expansion stage (i.e., legalization, but without the slow edit-in-place
2902                // of the existing legalization framework).
2903                let mem = mem.clone();
2904                let (mem_insts, mem) = mem_finalize(None, &mem, I8, state);
2905                let mut ret = String::new();
2906                for inst in mem_insts.into_iter() {
2907                    ret.push_str(&inst.print_with_state(&mut EmitState::default()));
2908                }
2909                let (reg, index_reg, offset) = match mem {
2910                    AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0),
2911                    AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()),
2912                    AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32),
2913                    _ => panic!("Unsupported case for LoadAddr: {mem:?}"),
2914                };
2915                let abs_offset = if offset < 0 {
2916                    -offset as u64
2917                } else {
2918                    offset as u64
2919                };
2920                let alu_op = if offset < 0 { ALUOp::Sub } else { ALUOp::Add };
2921
2922                if let Some((idx, extendop)) = index_reg {
2923                    let add = Inst::AluRRRExtend {
2924                        alu_op: ALUOp::Add,
2925                        size: OperandSize::Size64,
2926                        rd,
2927                        rn: reg,
2928                        rm: idx,
2929                        extendop,
2930                    };
2931
2932                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
2933                } else if offset == 0 {
2934                    let mov = Inst::gen_move(rd, reg, I64);
2935                    ret.push_str(&mov.print_with_state(&mut EmitState::default()));
2936                } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
2937                    let add = Inst::AluRRImm12 {
2938                        alu_op,
2939                        size: OperandSize::Size64,
2940                        rd,
2941                        rn: reg,
2942                        imm12,
2943                    };
2944                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
2945                } else {
2946                    let tmp = writable_spilltmp_reg();
2947                    for inst in Inst::load_constant(tmp, abs_offset).into_iter() {
2948                        ret.push_str(&inst.print_with_state(&mut EmitState::default()));
2949                    }
2950                    let add = Inst::AluRRR {
2951                        alu_op,
2952                        size: OperandSize::Size64,
2953                        rd,
2954                        rn: reg,
2955                        rm: tmp.to_reg(),
2956                    };
2957                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
2958                }
2959                ret
2960            }
2961            &Inst::Paci { key } => {
2962                let key = match key {
2963                    APIKey::AZ => "az",
2964                    APIKey::BZ => "bz",
2965                    APIKey::ASP => "asp",
2966                    APIKey::BSP => "bsp",
2967                };
2968
2969                "paci".to_string() + key
2970            }
2971            &Inst::Xpaclri => "xpaclri".to_string(),
2972            &Inst::Bti { targets } => {
2973                let targets = match targets {
2974                    BranchTargetType::None => "",
2975                    BranchTargetType::C => " c",
2976                    BranchTargetType::J => " j",
2977                    BranchTargetType::JC => " jc",
2978                };
2979
2980                "bti".to_string() + targets
2981            }
2982            &Inst::EmitIsland { needed_space } => format!("emit_island {needed_space}"),
2983
2984            &Inst::ElfTlsGetAddr {
2985                ref symbol,
2986                rd,
2987                tmp,
2988            } => {
2989                let rd = pretty_print_reg(rd.to_reg());
2990                let tmp = pretty_print_reg(tmp.to_reg());
2991                format!("elf_tls_get_addr {}, {}, {}", rd, tmp, symbol.display(None))
2992            }
2993            &Inst::MachOTlsGetAddr { ref symbol, rd } => {
2994                let rd = pretty_print_reg(rd.to_reg());
2995                format!("macho_tls_get_addr {}, {}", rd, symbol.display(None))
2996            }
2997            &Inst::Unwind { ref inst } => {
2998                format!("unwind {inst:?}")
2999            }
3000            &Inst::DummyUse { reg } => {
3001                let reg = pretty_print_reg(reg);
3002                format!("dummy_use {reg}")
3003            }
3004            &Inst::LabelAddress { dst, label } => {
3005                let dst = pretty_print_reg(dst.to_reg());
3006                format!("label_address {dst}, {label:?}")
3007            }
3008            &Inst::SequencePoint {} => {
3009                format!("sequence_point")
3010            }
3011            &Inst::StackProbeLoop { start, end, step } => {
3012                let start = pretty_print_reg(start.to_reg());
3013                let end = pretty_print_reg(end);
3014                let step = step.pretty_print(0);
3015                format!("stack_probe_loop {start}, {end}, {step}")
3016            }
3017        }
3018    }
3019}
3020
3021//=============================================================================
3022// Label fixups and jump veneers.
3023
3024/// Different forms of label references for different instruction formats.
3025#[derive(Clone, Copy, Debug, PartialEq, Eq)]
3026pub enum LabelUse {
3027    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
3028    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
3029    Branch14,
3030    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
3031    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
3032    Branch19,
3033    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
3034    /// signed bits, in bits 25:0. Used by b, bl.
3035    Branch26,
3036    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
3037    /// in bits 23:5.
3038    Ldr19,
3039    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
3040    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
3041    Adr21,
3042    /// 32-bit PC relative constant offset (from address of constant itself),
3043    /// signed. Used in jump tables.
3044    PCRel32,
3045}
3046
3047impl MachInstLabelUse for LabelUse {
3048    /// Alignment for veneer code. Every AArch64 instruction must be 4-byte-aligned.
3049    const ALIGN: CodeOffset = 4;
3050
3051    /// Maximum PC-relative range (positive), inclusive.
3052    fn max_pos_range(self) -> CodeOffset {
3053        match self {
3054            // N-bit immediate, left-shifted by 2, for (N+2) bits of total
3055            // range. Signed, so +2^(N+1) from zero. Likewise for two other
3056            // shifted cases below.
3057            LabelUse::Branch14 => (1 << 15) - 1,
3058            LabelUse::Branch19 => (1 << 20) - 1,
3059            LabelUse::Branch26 => (1 << 27) - 1,
3060            LabelUse::Ldr19 => (1 << 20) - 1,
3061            // Adr does not shift its immediate, so the 21-bit immediate gives 21 bits of total
3062            // range.
3063            LabelUse::Adr21 => (1 << 20) - 1,
3064            LabelUse::PCRel32 => 0x7fffffff,
3065        }
3066    }
3067
3068    /// Maximum PC-relative range (negative).
3069    fn max_neg_range(self) -> CodeOffset {
3070        // All forms are twos-complement signed offsets, so negative limit is one more than
3071        // positive limit.
3072        self.max_pos_range() + 1
3073    }
3074
3075    /// Size of window into code needed to do the patch.
3076    fn patch_size(self) -> CodeOffset {
3077        // Patch is on one instruction only for all of these label reference types.
3078        4
3079    }
3080
3081    /// Perform the patch.
3082    fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
3083        let pc_rel = (label_offset as i64) - (use_offset as i64);
3084        debug_assert!(pc_rel <= self.max_pos_range() as i64);
3085        debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
3086        let pc_rel = pc_rel as u32;
3087        let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
3088        let mask = match self {
3089            LabelUse::Branch14 => 0x0007ffe0, // bits 18..5 inclusive
3090            LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
3091            LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
3092            LabelUse::Ldr19 => 0x00ffffe0,    // bits 23..5 inclusive
3093            LabelUse::Adr21 => 0x60ffffe0,    // bits 30..29, 25..5 inclusive
3094            LabelUse::PCRel32 => 0xffffffff,
3095        };
3096        let pc_rel_shifted = match self {
3097            LabelUse::Adr21 | LabelUse::PCRel32 => pc_rel,
3098            _ => {
3099                debug_assert!(pc_rel & 3 == 0);
3100                pc_rel >> 2
3101            }
3102        };
3103        let pc_rel_inserted = match self {
3104            LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
3105            LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
3106            LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
3107            // Note: the *low* two bits of offset are put in the
3108            // *high* bits (30, 29).
3109            LabelUse::Adr21 => (pc_rel_shifted & 0x1ffffc) << 3 | (pc_rel_shifted & 3) << 29,
3110            LabelUse::PCRel32 => pc_rel_shifted,
3111        };
3112        let is_add = match self {
3113            LabelUse::PCRel32 => true,
3114            _ => false,
3115        };
3116        let insn_word = if is_add {
3117            insn_word.wrapping_add(pc_rel_inserted)
3118        } else {
3119            (insn_word & !mask) | pc_rel_inserted
3120        };
3121        buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3122    }
3123
3124    /// Is a veneer supported for this label reference type?
3125    fn supports_veneer(self) -> bool {
3126        match self {
3127            LabelUse::Branch14 | LabelUse::Branch19 => true, // veneer is a Branch26
3128            LabelUse::Branch26 => true,                      // veneer is a PCRel32
3129            _ => false,
3130        }
3131    }
3132
3133    /// How large is the veneer, if supported?
3134    fn veneer_size(self) -> CodeOffset {
3135        match self {
3136            LabelUse::Branch14 | LabelUse::Branch19 => 4,
3137            LabelUse::Branch26 => 20,
3138            _ => unreachable!(),
3139        }
3140    }
3141
3142    fn worst_case_veneer_size() -> CodeOffset {
3143        20
3144    }
3145
3146    /// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return
3147    /// an offset and label-use for the veneer's use of the original label.
3148    fn generate_veneer(
3149        self,
3150        buffer: &mut [u8],
3151        veneer_offset: CodeOffset,
3152    ) -> (CodeOffset, LabelUse) {
3153        match self {
3154            LabelUse::Branch14 | LabelUse::Branch19 => {
3155                // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
3156                // bother with constructing an Inst.
3157                let insn_word = 0b000101 << 26;
3158                buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3159                (veneer_offset, LabelUse::Branch26)
3160            }
3161
3162            // This is promoting a 26-bit call/jump to a 32-bit call/jump to
3163            // get a further range. This jump translates to a jump to a
3164            // relative location based on the address of the constant loaded
3165            // from here.
3166            //
3167            // If this path is taken from a call instruction then caller-saved
3168            // registers are available (minus arguments), so x16/x17 are
3169            // available. Otherwise for intra-function jumps we also reserve
3170            // x16/x17 as spill-style registers. In both cases these are
3171            // available for us to use.
3172            LabelUse::Branch26 => {
3173                let tmp1 = regs::spilltmp_reg();
3174                let tmp1_w = regs::writable_spilltmp_reg();
3175                let tmp2 = regs::tmp2_reg();
3176                let tmp2_w = regs::writable_tmp2_reg();
3177                // ldrsw x16, 16
3178                let ldr = emit::enc_ldst_imm19(0b1001_1000, 16 / 4, tmp1);
3179                // adr x17, 12
3180                let adr = emit::enc_adr(12, tmp2_w);
3181                // add x16, x16, x17
3182                let add = emit::enc_arith_rrr(0b10001011_000, 0, tmp1_w, tmp1, tmp2);
3183                // br x16
3184                let br = emit::enc_br(tmp1);
3185                buffer[0..4].clone_from_slice(&u32::to_le_bytes(ldr));
3186                buffer[4..8].clone_from_slice(&u32::to_le_bytes(adr));
3187                buffer[8..12].clone_from_slice(&u32::to_le_bytes(add));
3188                buffer[12..16].clone_from_slice(&u32::to_le_bytes(br));
3189                // the 4-byte signed immediate we'll load is after these
3190                // instructions, 16-bytes in.
3191                (veneer_offset + 16, LabelUse::PCRel32)
3192            }
3193
3194            _ => panic!("Unsupported label-reference type for veneer generation!"),
3195        }
3196    }
3197
3198    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {
3199        match (reloc, addend) {
3200            (Reloc::Arm64Call, 0) => Some(LabelUse::Branch26),
3201            _ => None,
3202        }
3203    }
3204}
3205
3206#[cfg(test)]
3207mod tests {
3208    use super::*;
3209
3210    #[test]
3211    fn inst_size_test() {
3212        // This test will help with unintentionally growing the size
3213        // of the Inst enum.
3214        assert_eq!(32, core::mem::size_of::<Inst>());
3215    }
3216}