cranelift_codegen/isa/aarch64/inst/
mod.rs

1//! This module defines aarch64-specific machine instruction types.
2
3use crate::binemit::{Addend, CodeOffset, Reloc};
4use crate::ir::types::{F16, F32, F64, F128, I8, I8X16, I16, I32, I64, I128};
5use crate::ir::{MemFlags, Type, types};
6use crate::isa::{CallConv, FunctionAlignment};
7use crate::machinst::*;
8use crate::{CodegenError, CodegenResult, settings};
9
10use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
11
12use alloc::vec::Vec;
13use core::slice;
14use smallvec::{SmallVec, smallvec};
15use std::fmt::Write;
16use std::string::{String, ToString};
17
18pub(crate) mod regs;
19pub(crate) use self::regs::*;
20pub mod imms;
21pub use self::imms::*;
22pub mod args;
23pub use self::args::*;
24pub mod emit;
25pub(crate) use self::emit::*;
26use crate::isa::aarch64::abi::AArch64MachineDeps;
27
28pub(crate) mod unwind;
29
30#[cfg(test)]
31mod emit_tests;
32
33//=============================================================================
34// Instructions (top level): definition
35
36pub use crate::isa::aarch64::lower::isle::generated_code::{
37    ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, BranchTargetType, FPUOp1,
38    FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp,
39    VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp,
40    VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,
41};
42
43/// A floating-point unit (FPU) operation with two args, a register and an immediate.
44#[derive(Copy, Clone, Debug)]
45pub enum FPUOpRI {
46    /// Unsigned right shift. Rd = Rn << #imm
47    UShr32(FPURightShiftImm),
48    /// Unsigned right shift. Rd = Rn << #imm
49    UShr64(FPURightShiftImm),
50}
51
52/// A floating-point unit (FPU) operation with two args, a register and
53/// an immediate that modifies its dest (so takes that input value as a
54/// separate virtual register).
55#[derive(Copy, Clone, Debug)]
56pub enum FPUOpRIMod {
57    /// Shift left and insert. Rd |= Rn << #imm
58    Sli32(FPULeftShiftImm),
59    /// Shift left and insert. Rd |= Rn << #imm
60    Sli64(FPULeftShiftImm),
61}
62
63impl BitOp {
64    /// Get the assembly mnemonic for this opcode.
65    pub fn op_str(&self) -> &'static str {
66        match self {
67            BitOp::RBit => "rbit",
68            BitOp::Clz => "clz",
69            BitOp::Cls => "cls",
70            BitOp::Rev16 => "rev16",
71            BitOp::Rev32 => "rev32",
72            BitOp::Rev64 => "rev64",
73        }
74    }
75}
76
77/// Additional information for `return_call[_ind]` instructions, left out of
78/// line to lower the size of the `Inst` enum.
79#[derive(Clone, Debug)]
80pub struct ReturnCallInfo<T> {
81    /// Where this call is going to
82    pub dest: T,
83    /// Arguments to the call instruction.
84    pub uses: CallArgList,
85    /// The size of the new stack frame's stack arguments. This is necessary
86    /// for copying the frame over our current frame. It must already be
87    /// allocated on the stack.
88    pub new_stack_arg_size: u32,
89    /// API key to use to restore the return address, if any.
90    pub key: Option<APIKey>,
91}
92
93fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
94    let mut count = 0;
95    for _ in 0..num_half_words {
96        if value & 0xffff == 0 {
97            count += 1;
98        }
99        value >>= 16;
100    }
101
102    count
103}
104
105impl Inst {
106    /// Create an instruction that loads a constant, using one of several options (MOVZ, MOVN,
107    /// logical immediate, or constant pool).
108    pub fn load_constant(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]> {
109        // NB: this is duplicated in `lower/isle.rs` and `inst.isle` right now,
110        // if modifications are made here before this is deleted after moving to
111        // ISLE then those locations should be updated as well.
112
113        if let Some(imm) = MoveWideConst::maybe_from_u64(value) {
114            // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVZ
115            smallvec![Inst::MovWide {
116                op: MoveWideOp::MovZ,
117                rd,
118                imm,
119                size: OperandSize::Size64
120            }]
121        } else if let Some(imm) = MoveWideConst::maybe_from_u64(!value) {
122            // 16-bit immediate (shifted by 0, 16, 32 or 48 bits) in MOVN
123            smallvec![Inst::MovWide {
124                op: MoveWideOp::MovN,
125                rd,
126                imm,
127                size: OperandSize::Size64
128            }]
129        } else if let Some(imml) = ImmLogic::maybe_from_u64(value, I64) {
130            // Weird logical-instruction immediate in ORI using zero register
131            smallvec![Inst::AluRRImmLogic {
132                alu_op: ALUOp::Orr,
133                size: OperandSize::Size64,
134                rd,
135                rn: zero_reg(),
136                imml,
137            }]
138        } else {
139            let mut insts = smallvec![];
140
141            // If the top 32 bits are zero, use 32-bit `mov` operations.
142            let (num_half_words, size, negated) = if value >> 32 == 0 {
143                (2, OperandSize::Size32, (!value << 32) >> 32)
144            } else {
145                (4, OperandSize::Size64, !value)
146            };
147
148            // If the number of 0xffff half words is greater than the number of 0x0000 half words
149            // it is more efficient to use `movn` for the first instruction.
150            let first_is_inverted = count_zero_half_words(negated, num_half_words)
151                > count_zero_half_words(value, num_half_words);
152
153            // Either 0xffff or 0x0000 half words can be skipped, depending on the first
154            // instruction used.
155            let ignored_halfword = if first_is_inverted { 0xffff } else { 0 };
156
157            let halfwords: SmallVec<[_; 4]> = (0..num_half_words)
158                .filter_map(|i| {
159                    let imm16 = (value >> (16 * i)) & 0xffff;
160                    if imm16 == ignored_halfword {
161                        None
162                    } else {
163                        Some((i, imm16))
164                    }
165                })
166                .collect();
167
168            let mut prev_result = None;
169            for (i, imm16) in halfwords {
170                let shift = i * 16;
171
172                if let Some(rn) = prev_result {
173                    let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
174                    insts.push(Inst::MovK { rd, rn, imm, size });
175                } else {
176                    if first_is_inverted {
177                        let imm =
178                            MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, shift)
179                                .unwrap();
180                        insts.push(Inst::MovWide {
181                            op: MoveWideOp::MovN,
182                            rd,
183                            imm,
184                            size,
185                        });
186                    } else {
187                        let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
188                        insts.push(Inst::MovWide {
189                            op: MoveWideOp::MovZ,
190                            rd,
191                            imm,
192                            size,
193                        });
194                    }
195                }
196
197                prev_result = Some(rd.to_reg());
198            }
199
200            assert!(prev_result.is_some());
201
202            insts
203        }
204    }
205
206    /// Generic constructor for a load (zero-extending where appropriate).
207    pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
208        match ty {
209            I8 => Inst::ULoad8 {
210                rd: into_reg,
211                mem,
212                flags,
213            },
214            I16 => Inst::ULoad16 {
215                rd: into_reg,
216                mem,
217                flags,
218            },
219            I32 => Inst::ULoad32 {
220                rd: into_reg,
221                mem,
222                flags,
223            },
224            I64 => Inst::ULoad64 {
225                rd: into_reg,
226                mem,
227                flags,
228            },
229            _ => {
230                if ty.is_vector() || ty.is_float() {
231                    let bits = ty_bits(ty);
232                    let rd = into_reg;
233
234                    match bits {
235                        128 => Inst::FpuLoad128 { rd, mem, flags },
236                        64 => Inst::FpuLoad64 { rd, mem, flags },
237                        32 => Inst::FpuLoad32 { rd, mem, flags },
238                        16 => Inst::FpuLoad16 { rd, mem, flags },
239                        _ => unimplemented!("gen_load({})", ty),
240                    }
241                } else {
242                    unimplemented!("gen_load({})", ty);
243                }
244            }
245        }
246    }
247
248    /// Generic constructor for a store.
249    pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {
250        match ty {
251            I8 => Inst::Store8 {
252                rd: from_reg,
253                mem,
254                flags,
255            },
256            I16 => Inst::Store16 {
257                rd: from_reg,
258                mem,
259                flags,
260            },
261            I32 => Inst::Store32 {
262                rd: from_reg,
263                mem,
264                flags,
265            },
266            I64 => Inst::Store64 {
267                rd: from_reg,
268                mem,
269                flags,
270            },
271            _ => {
272                if ty.is_vector() || ty.is_float() {
273                    let bits = ty_bits(ty);
274                    let rd = from_reg;
275
276                    match bits {
277                        128 => Inst::FpuStore128 { rd, mem, flags },
278                        64 => Inst::FpuStore64 { rd, mem, flags },
279                        32 => Inst::FpuStore32 { rd, mem, flags },
280                        16 => Inst::FpuStore16 { rd, mem, flags },
281                        _ => unimplemented!("gen_store({})", ty),
282                    }
283                } else {
284                    unimplemented!("gen_store({})", ty);
285                }
286            }
287        }
288    }
289
290    /// What type does this load or store instruction access in memory? When
291    /// uimm12 encoding is used, the size of this type is the amount that
292    /// immediate offsets are scaled by.
293    pub fn mem_type(&self) -> Option<Type> {
294        match self {
295            Inst::ULoad8 { .. } => Some(I8),
296            Inst::SLoad8 { .. } => Some(I8),
297            Inst::ULoad16 { .. } => Some(I16),
298            Inst::SLoad16 { .. } => Some(I16),
299            Inst::ULoad32 { .. } => Some(I32),
300            Inst::SLoad32 { .. } => Some(I32),
301            Inst::ULoad64 { .. } => Some(I64),
302            Inst::FpuLoad16 { .. } => Some(F16),
303            Inst::FpuLoad32 { .. } => Some(F32),
304            Inst::FpuLoad64 { .. } => Some(F64),
305            Inst::FpuLoad128 { .. } => Some(I8X16),
306            Inst::Store8 { .. } => Some(I8),
307            Inst::Store16 { .. } => Some(I16),
308            Inst::Store32 { .. } => Some(I32),
309            Inst::Store64 { .. } => Some(I64),
310            Inst::FpuStore16 { .. } => Some(F16),
311            Inst::FpuStore32 { .. } => Some(F32),
312            Inst::FpuStore64 { .. } => Some(F64),
313            Inst::FpuStore128 { .. } => Some(I8X16),
314            _ => None,
315        }
316    }
317}
318
319//=============================================================================
320// Instructions: get_regs
321
322fn memarg_operands(memarg: &mut AMode, collector: &mut impl OperandVisitor) {
323    match memarg {
324        AMode::Unscaled { rn, .. } | AMode::UnsignedOffset { rn, .. } => {
325            collector.reg_use(rn);
326        }
327        AMode::RegReg { rn, rm, .. }
328        | AMode::RegScaled { rn, rm, .. }
329        | AMode::RegScaledExtended { rn, rm, .. }
330        | AMode::RegExtended { rn, rm, .. } => {
331            collector.reg_use(rn);
332            collector.reg_use(rm);
333        }
334        AMode::Label { .. } => {}
335        AMode::SPPreIndexed { .. } | AMode::SPPostIndexed { .. } => {}
336        AMode::FPOffset { .. } | AMode::IncomingArg { .. } => {}
337        AMode::SPOffset { .. } | AMode::SlotOffset { .. } => {}
338        AMode::RegOffset { rn, .. } => {
339            collector.reg_use(rn);
340        }
341        AMode::Const { .. } => {}
342    }
343}
344
345fn pairmemarg_operands(pairmemarg: &mut PairAMode, collector: &mut impl OperandVisitor) {
346    match pairmemarg {
347        PairAMode::SignedOffset { reg, .. } => {
348            collector.reg_use(reg);
349        }
350        PairAMode::SPPreIndexed { .. } | PairAMode::SPPostIndexed { .. } => {}
351    }
352}
353
354fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
355    match inst {
356        Inst::AluRRR { rd, rn, rm, .. } => {
357            collector.reg_def(rd);
358            collector.reg_use(rn);
359            collector.reg_use(rm);
360        }
361        Inst::AluRRRR { rd, rn, rm, ra, .. } => {
362            collector.reg_def(rd);
363            collector.reg_use(rn);
364            collector.reg_use(rm);
365            collector.reg_use(ra);
366        }
367        Inst::AluRRImm12 { rd, rn, .. } => {
368            collector.reg_def(rd);
369            collector.reg_use(rn);
370        }
371        Inst::AluRRImmLogic { rd, rn, .. } => {
372            collector.reg_def(rd);
373            collector.reg_use(rn);
374        }
375        Inst::AluRRImmShift { rd, rn, .. } => {
376            collector.reg_def(rd);
377            collector.reg_use(rn);
378        }
379        Inst::AluRRRShift { rd, rn, rm, .. } => {
380            collector.reg_def(rd);
381            collector.reg_use(rn);
382            collector.reg_use(rm);
383        }
384        Inst::AluRRRExtend { rd, rn, rm, .. } => {
385            collector.reg_def(rd);
386            collector.reg_use(rn);
387            collector.reg_use(rm);
388        }
389        Inst::BitRR { rd, rn, .. } => {
390            collector.reg_def(rd);
391            collector.reg_use(rn);
392        }
393        Inst::ULoad8 { rd, mem, .. }
394        | Inst::SLoad8 { rd, mem, .. }
395        | Inst::ULoad16 { rd, mem, .. }
396        | Inst::SLoad16 { rd, mem, .. }
397        | Inst::ULoad32 { rd, mem, .. }
398        | Inst::SLoad32 { rd, mem, .. }
399        | Inst::ULoad64 { rd, mem, .. } => {
400            collector.reg_def(rd);
401            memarg_operands(mem, collector);
402        }
403        Inst::Store8 { rd, mem, .. }
404        | Inst::Store16 { rd, mem, .. }
405        | Inst::Store32 { rd, mem, .. }
406        | Inst::Store64 { rd, mem, .. } => {
407            collector.reg_use(rd);
408            memarg_operands(mem, collector);
409        }
410        Inst::StoreP64 { rt, rt2, mem, .. } => {
411            collector.reg_use(rt);
412            collector.reg_use(rt2);
413            pairmemarg_operands(mem, collector);
414        }
415        Inst::LoadP64 { rt, rt2, mem, .. } => {
416            collector.reg_def(rt);
417            collector.reg_def(rt2);
418            pairmemarg_operands(mem, collector);
419        }
420        Inst::Mov { rd, rm, .. } => {
421            collector.reg_def(rd);
422            collector.reg_use(rm);
423        }
424        Inst::MovFromPReg { rd, rm } => {
425            debug_assert!(rd.to_reg().is_virtual());
426            collector.reg_def(rd);
427            collector.reg_fixed_nonallocatable(*rm);
428        }
429        Inst::MovToPReg { rd, rm } => {
430            debug_assert!(rm.is_virtual());
431            collector.reg_fixed_nonallocatable(*rd);
432            collector.reg_use(rm);
433        }
434        Inst::MovK { rd, rn, .. } => {
435            collector.reg_use(rn);
436            collector.reg_reuse_def(rd, 0); // `rn` == `rd`.
437        }
438        Inst::MovWide { rd, .. } => {
439            collector.reg_def(rd);
440        }
441        Inst::CSel { rd, rn, rm, .. } => {
442            collector.reg_def(rd);
443            collector.reg_use(rn);
444            collector.reg_use(rm);
445        }
446        Inst::CSNeg { rd, rn, rm, .. } => {
447            collector.reg_def(rd);
448            collector.reg_use(rn);
449            collector.reg_use(rm);
450        }
451        Inst::CSet { rd, .. } | Inst::CSetm { rd, .. } => {
452            collector.reg_def(rd);
453        }
454        Inst::CCmp { rn, rm, .. } => {
455            collector.reg_use(rn);
456            collector.reg_use(rm);
457        }
458        Inst::CCmpImm { rn, .. } => {
459            collector.reg_use(rn);
460        }
461        Inst::AtomicRMWLoop {
462            op,
463            addr,
464            operand,
465            oldval,
466            scratch1,
467            scratch2,
468            ..
469        } => {
470            collector.reg_fixed_use(addr, xreg(25));
471            collector.reg_fixed_use(operand, xreg(26));
472            collector.reg_fixed_def(oldval, xreg(27));
473            collector.reg_fixed_def(scratch1, xreg(24));
474            if *op != AtomicRMWLoopOp::Xchg {
475                collector.reg_fixed_def(scratch2, xreg(28));
476            }
477        }
478        Inst::AtomicRMW { rs, rt, rn, .. } => {
479            collector.reg_use(rs);
480            collector.reg_def(rt);
481            collector.reg_use(rn);
482        }
483        Inst::AtomicCAS { rd, rs, rt, rn, .. } => {
484            collector.reg_reuse_def(rd, 1); // reuse `rs`.
485            collector.reg_use(rs);
486            collector.reg_use(rt);
487            collector.reg_use(rn);
488        }
489        Inst::AtomicCASLoop {
490            addr,
491            expected,
492            replacement,
493            oldval,
494            scratch,
495            ..
496        } => {
497            collector.reg_fixed_use(addr, xreg(25));
498            collector.reg_fixed_use(expected, xreg(26));
499            collector.reg_fixed_use(replacement, xreg(28));
500            collector.reg_fixed_def(oldval, xreg(27));
501            collector.reg_fixed_def(scratch, xreg(24));
502        }
503        Inst::LoadAcquire { rt, rn, .. } => {
504            collector.reg_use(rn);
505            collector.reg_def(rt);
506        }
507        Inst::StoreRelease { rt, rn, .. } => {
508            collector.reg_use(rn);
509            collector.reg_use(rt);
510        }
511        Inst::Fence {} | Inst::Csdb {} => {}
512        Inst::FpuMove32 { rd, rn } => {
513            collector.reg_def(rd);
514            collector.reg_use(rn);
515        }
516        Inst::FpuMove64 { rd, rn } => {
517            collector.reg_def(rd);
518            collector.reg_use(rn);
519        }
520        Inst::FpuMove128 { rd, rn } => {
521            collector.reg_def(rd);
522            collector.reg_use(rn);
523        }
524        Inst::FpuMoveFromVec { rd, rn, .. } => {
525            collector.reg_def(rd);
526            collector.reg_use(rn);
527        }
528        Inst::FpuExtend { rd, rn, .. } => {
529            collector.reg_def(rd);
530            collector.reg_use(rn);
531        }
532        Inst::FpuRR { rd, rn, .. } => {
533            collector.reg_def(rd);
534            collector.reg_use(rn);
535        }
536        Inst::FpuRRR { rd, rn, rm, .. } => {
537            collector.reg_def(rd);
538            collector.reg_use(rn);
539            collector.reg_use(rm);
540        }
541        Inst::FpuRRI { rd, rn, .. } => {
542            collector.reg_def(rd);
543            collector.reg_use(rn);
544        }
545        Inst::FpuRRIMod { rd, ri, rn, .. } => {
546            collector.reg_reuse_def(rd, 1); // reuse `ri`.
547            collector.reg_use(ri);
548            collector.reg_use(rn);
549        }
550        Inst::FpuRRRR { rd, rn, rm, ra, .. } => {
551            collector.reg_def(rd);
552            collector.reg_use(rn);
553            collector.reg_use(rm);
554            collector.reg_use(ra);
555        }
556        Inst::VecMisc { rd, rn, .. } => {
557            collector.reg_def(rd);
558            collector.reg_use(rn);
559        }
560
561        Inst::VecLanes { rd, rn, .. } => {
562            collector.reg_def(rd);
563            collector.reg_use(rn);
564        }
565        Inst::VecShiftImm { rd, rn, .. } => {
566            collector.reg_def(rd);
567            collector.reg_use(rn);
568        }
569        Inst::VecShiftImmMod { rd, ri, rn, .. } => {
570            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
571            collector.reg_use(ri);
572            collector.reg_use(rn);
573        }
574        Inst::VecExtract { rd, rn, rm, .. } => {
575            collector.reg_def(rd);
576            collector.reg_use(rn);
577            collector.reg_use(rm);
578        }
579        Inst::VecTbl { rd, rn, rm } => {
580            collector.reg_use(rn);
581            collector.reg_use(rm);
582            collector.reg_def(rd);
583        }
584        Inst::VecTblExt { rd, ri, rn, rm } => {
585            collector.reg_use(rn);
586            collector.reg_use(rm);
587            collector.reg_reuse_def(rd, 3); // `rd` == `ri`.
588            collector.reg_use(ri);
589        }
590
591        Inst::VecTbl2 { rd, rn, rn2, rm } => {
592            // Constrain to v30 / v31 so that we satisfy the "adjacent
593            // registers" constraint without use of pinned vregs in
594            // lowering.
595            collector.reg_fixed_use(rn, vreg(30));
596            collector.reg_fixed_use(rn2, vreg(31));
597            collector.reg_use(rm);
598            collector.reg_def(rd);
599        }
600        Inst::VecTbl2Ext {
601            rd,
602            ri,
603            rn,
604            rn2,
605            rm,
606        } => {
607            // Constrain to v30 / v31 so that we satisfy the "adjacent
608            // registers" constraint without use of pinned vregs in
609            // lowering.
610            collector.reg_fixed_use(rn, vreg(30));
611            collector.reg_fixed_use(rn2, vreg(31));
612            collector.reg_use(rm);
613            collector.reg_reuse_def(rd, 4); // `rd` == `ri`.
614            collector.reg_use(ri);
615        }
616        Inst::VecLoadReplicate { rd, rn, .. } => {
617            collector.reg_def(rd);
618            collector.reg_use(rn);
619        }
620        Inst::VecCSel { rd, rn, rm, .. } => {
621            collector.reg_def(rd);
622            collector.reg_use(rn);
623            collector.reg_use(rm);
624        }
625        Inst::FpuCmp { rn, rm, .. } => {
626            collector.reg_use(rn);
627            collector.reg_use(rm);
628        }
629        Inst::FpuLoad16 { rd, mem, .. } => {
630            collector.reg_def(rd);
631            memarg_operands(mem, collector);
632        }
633        Inst::FpuLoad32 { rd, mem, .. } => {
634            collector.reg_def(rd);
635            memarg_operands(mem, collector);
636        }
637        Inst::FpuLoad64 { rd, mem, .. } => {
638            collector.reg_def(rd);
639            memarg_operands(mem, collector);
640        }
641        Inst::FpuLoad128 { rd, mem, .. } => {
642            collector.reg_def(rd);
643            memarg_operands(mem, collector);
644        }
645        Inst::FpuStore16 { rd, mem, .. } => {
646            collector.reg_use(rd);
647            memarg_operands(mem, collector);
648        }
649        Inst::FpuStore32 { rd, mem, .. } => {
650            collector.reg_use(rd);
651            memarg_operands(mem, collector);
652        }
653        Inst::FpuStore64 { rd, mem, .. } => {
654            collector.reg_use(rd);
655            memarg_operands(mem, collector);
656        }
657        Inst::FpuStore128 { rd, mem, .. } => {
658            collector.reg_use(rd);
659            memarg_operands(mem, collector);
660        }
661        Inst::FpuLoadP64 { rt, rt2, mem, .. } => {
662            collector.reg_def(rt);
663            collector.reg_def(rt2);
664            pairmemarg_operands(mem, collector);
665        }
666        Inst::FpuStoreP64 { rt, rt2, mem, .. } => {
667            collector.reg_use(rt);
668            collector.reg_use(rt2);
669            pairmemarg_operands(mem, collector);
670        }
671        Inst::FpuLoadP128 { rt, rt2, mem, .. } => {
672            collector.reg_def(rt);
673            collector.reg_def(rt2);
674            pairmemarg_operands(mem, collector);
675        }
676        Inst::FpuStoreP128 { rt, rt2, mem, .. } => {
677            collector.reg_use(rt);
678            collector.reg_use(rt2);
679            pairmemarg_operands(mem, collector);
680        }
681        Inst::FpuToInt { rd, rn, .. } => {
682            collector.reg_def(rd);
683            collector.reg_use(rn);
684        }
685        Inst::IntToFpu { rd, rn, .. } => {
686            collector.reg_def(rd);
687            collector.reg_use(rn);
688        }
689        Inst::FpuCSel16 { rd, rn, rm, .. }
690        | Inst::FpuCSel32 { rd, rn, rm, .. }
691        | Inst::FpuCSel64 { rd, rn, rm, .. } => {
692            collector.reg_def(rd);
693            collector.reg_use(rn);
694            collector.reg_use(rm);
695        }
696        Inst::FpuRound { rd, rn, .. } => {
697            collector.reg_def(rd);
698            collector.reg_use(rn);
699        }
700        Inst::MovToFpu { rd, rn, .. } => {
701            collector.reg_def(rd);
702            collector.reg_use(rn);
703        }
704        Inst::FpuMoveFPImm { rd, .. } => {
705            collector.reg_def(rd);
706        }
707        Inst::MovToVec { rd, ri, rn, .. } => {
708            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
709            collector.reg_use(ri);
710            collector.reg_use(rn);
711        }
712        Inst::MovFromVec { rd, rn, .. } | Inst::MovFromVecSigned { rd, rn, .. } => {
713            collector.reg_def(rd);
714            collector.reg_use(rn);
715        }
716        Inst::VecDup { rd, rn, .. } => {
717            collector.reg_def(rd);
718            collector.reg_use(rn);
719        }
720        Inst::VecDupFromFpu { rd, rn, .. } => {
721            collector.reg_def(rd);
722            collector.reg_use(rn);
723        }
724        Inst::VecDupFPImm { rd, .. } => {
725            collector.reg_def(rd);
726        }
727        Inst::VecDupImm { rd, .. } => {
728            collector.reg_def(rd);
729        }
730        Inst::VecExtend { rd, rn, .. } => {
731            collector.reg_def(rd);
732            collector.reg_use(rn);
733        }
734        Inst::VecMovElement { rd, ri, rn, .. } => {
735            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
736            collector.reg_use(ri);
737            collector.reg_use(rn);
738        }
739        Inst::VecRRLong { rd, rn, .. } => {
740            collector.reg_def(rd);
741            collector.reg_use(rn);
742        }
743        Inst::VecRRNarrowLow { rd, rn, .. } => {
744            collector.reg_use(rn);
745            collector.reg_def(rd);
746        }
747        Inst::VecRRNarrowHigh { rd, ri, rn, .. } => {
748            collector.reg_use(rn);
749            collector.reg_reuse_def(rd, 2); // `rd` == `ri`.
750            collector.reg_use(ri);
751        }
752        Inst::VecRRPair { rd, rn, .. } => {
753            collector.reg_def(rd);
754            collector.reg_use(rn);
755        }
756        Inst::VecRRRLong { rd, rn, rm, .. } => {
757            collector.reg_def(rd);
758            collector.reg_use(rn);
759            collector.reg_use(rm);
760        }
761        Inst::VecRRRLongMod { rd, ri, rn, rm, .. } => {
762            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
763            collector.reg_use(ri);
764            collector.reg_use(rn);
765            collector.reg_use(rm);
766        }
767        Inst::VecRRPairLong { rd, rn, .. } => {
768            collector.reg_def(rd);
769            collector.reg_use(rn);
770        }
771        Inst::VecRRR { rd, rn, rm, .. } => {
772            collector.reg_def(rd);
773            collector.reg_use(rn);
774            collector.reg_use(rm);
775        }
776        Inst::VecRRRMod { rd, ri, rn, rm, .. } | Inst::VecFmlaElem { rd, ri, rn, rm, .. } => {
777            collector.reg_reuse_def(rd, 1); // `rd` == `ri`.
778            collector.reg_use(ri);
779            collector.reg_use(rn);
780            collector.reg_use(rm);
781        }
782        Inst::MovToNZCV { rn } => {
783            collector.reg_use(rn);
784        }
785        Inst::MovFromNZCV { rd } => {
786            collector.reg_def(rd);
787        }
788        Inst::Extend { rd, rn, .. } => {
789            collector.reg_def(rd);
790            collector.reg_use(rn);
791        }
792        Inst::Args { args } => {
793            for ArgPair { vreg, preg } in args {
794                collector.reg_fixed_def(vreg, *preg);
795            }
796        }
797        Inst::Rets { rets } => {
798            for RetPair { vreg, preg } in rets {
799                collector.reg_fixed_use(vreg, *preg);
800            }
801        }
802        Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}
803        Inst::Jump { .. } => {}
804        Inst::Call { info, .. } => {
805            let CallInfo { uses, defs, .. } = &mut **info;
806            for CallArgPair { vreg, preg } in uses {
807                collector.reg_fixed_use(vreg, *preg);
808            }
809            for CallRetPair { vreg, location } in defs {
810                match location {
811                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
812                    RetLocation::Stack(..) => collector.any_def(vreg),
813                }
814            }
815            collector.reg_clobbers(info.clobbers);
816            if let Some(try_call_info) = &mut info.try_call_info {
817                try_call_info.collect_operands(collector);
818            }
819        }
820        Inst::CallInd { info, .. } => {
821            let CallInfo {
822                dest, uses, defs, ..
823            } = &mut **info;
824            collector.reg_use(dest);
825            for CallArgPair { vreg, preg } in uses {
826                collector.reg_fixed_use(vreg, *preg);
827            }
828            for CallRetPair { vreg, location } in defs {
829                match location {
830                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
831                    RetLocation::Stack(..) => collector.any_def(vreg),
832                }
833            }
834            collector.reg_clobbers(info.clobbers);
835            if let Some(try_call_info) = &mut info.try_call_info {
836                try_call_info.collect_operands(collector);
837            }
838        }
839        Inst::ReturnCall { info } => {
840            for CallArgPair { vreg, preg } in &mut info.uses {
841                collector.reg_fixed_use(vreg, *preg);
842            }
843        }
844        Inst::ReturnCallInd { info } => {
845            // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
846            // This shouldn't be a fixed register constraint, but it's not clear how to pick a
847            // register that won't be clobbered by the callee-save restore code emitted with a
848            // return_call_indirect.
849            collector.reg_fixed_use(&mut info.dest, xreg(1));
850            for CallArgPair { vreg, preg } in &mut info.uses {
851                collector.reg_fixed_use(vreg, *preg);
852            }
853        }
854        Inst::CondBr { kind, .. } => match kind {
855            CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
856            CondBrKind::Cond(_) => {}
857        },
858        Inst::TestBitAndBranch { rn, .. } => {
859            collector.reg_use(rn);
860        }
861        Inst::IndirectBr { rn, .. } => {
862            collector.reg_use(rn);
863        }
864        Inst::Nop0 | Inst::Nop4 => {}
865        Inst::Brk => {}
866        Inst::Udf { .. } => {}
867        Inst::TrapIf { kind, .. } => match kind {
868            CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
869            CondBrKind::Cond(_) => {}
870        },
871        Inst::Adr { rd, .. } | Inst::Adrp { rd, .. } => {
872            collector.reg_def(rd);
873        }
874        Inst::Word4 { .. } | Inst::Word8 { .. } => {}
875        Inst::JTSequence {
876            ridx, rtmp1, rtmp2, ..
877        } => {
878            collector.reg_use(ridx);
879            collector.reg_early_def(rtmp1);
880            collector.reg_early_def(rtmp2);
881        }
882        Inst::LoadExtName { rd, .. } => {
883            collector.reg_def(rd);
884        }
885        Inst::LoadAddr { rd, mem } => {
886            collector.reg_def(rd);
887            memarg_operands(mem, collector);
888        }
889        Inst::Paci { .. } | Inst::Xpaclri => {
890            // Neither LR nor SP is an allocatable register, so there is no need
891            // to do anything.
892        }
893        Inst::Bti { .. } => {}
894
895        Inst::ElfTlsGetAddr { rd, tmp, .. } => {
896            // TLSDESC has a very neat calling convention. It is required to preserve
897            // all registers except x0 and x30. X30 is non allocatable in cranelift since
898            // its the link register.
899            //
900            // Additionally we need a second register as a temporary register for the
901            // TLSDESC sequence. This register can be any register other than x0 (and x30).
902            collector.reg_fixed_def(rd, regs::xreg(0));
903            collector.reg_early_def(tmp);
904        }
905        Inst::MachOTlsGetAddr { rd, .. } => {
906            collector.reg_fixed_def(rd, regs::xreg(0));
907            let mut clobbers =
908                AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64, false);
909            clobbers.remove(regs::xreg_preg(0));
910            collector.reg_clobbers(clobbers);
911        }
912        Inst::Unwind { .. } => {}
913        Inst::EmitIsland { .. } => {}
914        Inst::DummyUse { reg } => {
915            collector.reg_use(reg);
916        }
917        Inst::StackProbeLoop { start, end, .. } => {
918            collector.reg_early_def(start);
919            collector.reg_use(end);
920        }
921    }
922}
923
924//=============================================================================
925// Instructions: misc functions and external interface
926
927impl MachInst for Inst {
928    type ABIMachineSpec = AArch64MachineDeps;
929    type LabelUse = LabelUse;
930
931    // "CLIF" in hex, to make the trap recognizable during
932    // debugging.
933    const TRAP_OPCODE: &'static [u8] = &0xc11f_u32.to_le_bytes();
934
935    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
936        aarch64_get_operands(self, collector);
937    }
938
939    fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
940        match self {
941            &Inst::Mov {
942                size: OperandSize::Size64,
943                rd,
944                rm,
945            } => Some((rd, rm)),
946            &Inst::FpuMove64 { rd, rn } => Some((rd, rn)),
947            &Inst::FpuMove128 { rd, rn } => Some((rd, rn)),
948            _ => None,
949        }
950    }
951
952    fn is_included_in_clobbers(&self) -> bool {
953        let (caller, callee, is_exception) = match self {
954            Inst::Args { .. } => return false,
955            Inst::Call { info } => (
956                info.caller_conv,
957                info.callee_conv,
958                info.try_call_info.is_some(),
959            ),
960            Inst::CallInd { info } => (
961                info.caller_conv,
962                info.callee_conv,
963                info.try_call_info.is_some(),
964            ),
965            _ => return true,
966        };
967
968        // We exclude call instructions from the clobber-set when they are calls
969        // from caller to callee that both clobber the same register (such as
970        // using the same or similar ABIs). Such calls cannot possibly force any
971        // new registers to be saved in the prologue, because anything that the
972        // callee clobbers, the caller is also allowed to clobber. This both
973        // saves work and enables us to more precisely follow the
974        // half-caller-save, half-callee-save SysV ABI for some vector
975        // registers.
976        //
977        // See the note in [crate::isa::aarch64::abi::is_caller_save_reg] for
978        // more information on this ABI-implementation hack.
979        let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller, is_exception);
980        let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee, is_exception);
981
982        let mut all_clobbers = caller_clobbers;
983        all_clobbers.union_from(callee_clobbers);
984        all_clobbers != caller_clobbers
985    }
986
987    fn is_trap(&self) -> bool {
988        match self {
989            Self::Udf { .. } => true,
990            _ => false,
991        }
992    }
993
994    fn is_args(&self) -> bool {
995        match self {
996            Self::Args { .. } => true,
997            _ => false,
998        }
999    }
1000
1001    fn is_term(&self) -> MachTerminator {
1002        match self {
1003            &Inst::Rets { .. } => MachTerminator::Ret,
1004            &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
1005            &Inst::Jump { .. } => MachTerminator::Branch,
1006            &Inst::CondBr { .. } => MachTerminator::Branch,
1007            &Inst::TestBitAndBranch { .. } => MachTerminator::Branch,
1008            &Inst::IndirectBr { .. } => MachTerminator::Branch,
1009            &Inst::JTSequence { .. } => MachTerminator::Branch,
1010            &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1011            &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1012            _ => MachTerminator::None,
1013        }
1014    }
1015
1016    fn is_mem_access(&self) -> bool {
1017        match self {
1018            &Inst::ULoad8 { .. }
1019            | &Inst::SLoad8 { .. }
1020            | &Inst::ULoad16 { .. }
1021            | &Inst::SLoad16 { .. }
1022            | &Inst::ULoad32 { .. }
1023            | &Inst::SLoad32 { .. }
1024            | &Inst::ULoad64 { .. }
1025            | &Inst::LoadP64 { .. }
1026            | &Inst::FpuLoad16 { .. }
1027            | &Inst::FpuLoad32 { .. }
1028            | &Inst::FpuLoad64 { .. }
1029            | &Inst::FpuLoad128 { .. }
1030            | &Inst::FpuLoadP64 { .. }
1031            | &Inst::FpuLoadP128 { .. }
1032            | &Inst::Store8 { .. }
1033            | &Inst::Store16 { .. }
1034            | &Inst::Store32 { .. }
1035            | &Inst::Store64 { .. }
1036            | &Inst::StoreP64 { .. }
1037            | &Inst::FpuStore16 { .. }
1038            | &Inst::FpuStore32 { .. }
1039            | &Inst::FpuStore64 { .. }
1040            | &Inst::FpuStore128 { .. } => true,
1041            // TODO: verify this carefully
1042            _ => false,
1043        }
1044    }
1045
1046    fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
1047        let bits = ty.bits();
1048
1049        assert!(bits <= 128);
1050        assert!(to_reg.to_reg().class() == from_reg.class());
1051        match from_reg.class() {
1052            RegClass::Int => Inst::Mov {
1053                size: OperandSize::Size64,
1054                rd: to_reg,
1055                rm: from_reg,
1056            },
1057            RegClass::Float => {
1058                if bits > 64 {
1059                    Inst::FpuMove128 {
1060                        rd: to_reg,
1061                        rn: from_reg,
1062                    }
1063                } else {
1064                    Inst::FpuMove64 {
1065                        rd: to_reg,
1066                        rn: from_reg,
1067                    }
1068                }
1069            }
1070            RegClass::Vector => unreachable!(),
1071        }
1072    }
1073
1074    fn is_safepoint(&self) -> bool {
1075        match self {
1076            Inst::Call { .. } | Inst::CallInd { .. } => true,
1077            _ => false,
1078        }
1079    }
1080
1081    fn gen_dummy_use(reg: Reg) -> Inst {
1082        Inst::DummyUse { reg }
1083    }
1084
1085    fn gen_nop(preferred_size: usize) -> Inst {
1086        if preferred_size == 0 {
1087            return Inst::Nop0;
1088        }
1089        // We can't give a NOP (or any insn) < 4 bytes.
1090        assert!(preferred_size >= 4);
1091        Inst::Nop4
1092    }
1093
1094    fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
1095        match ty {
1096            I8 => Ok((&[RegClass::Int], &[I8])),
1097            I16 => Ok((&[RegClass::Int], &[I16])),
1098            I32 => Ok((&[RegClass::Int], &[I32])),
1099            I64 => Ok((&[RegClass::Int], &[I64])),
1100            F16 => Ok((&[RegClass::Float], &[F16])),
1101            F32 => Ok((&[RegClass::Float], &[F32])),
1102            F64 => Ok((&[RegClass::Float], &[F64])),
1103            F128 => Ok((&[RegClass::Float], &[F128])),
1104            I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
1105            _ if ty.is_vector() && ty.bits() <= 128 => {
1106                let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1107                Ok((
1108                    &[RegClass::Float],
1109                    slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1110                ))
1111            }
1112            _ if ty.is_dynamic_vector() => Ok((&[RegClass::Float], &[I8X16])),
1113            _ => Err(CodegenError::Unsupported(format!(
1114                "Unexpected SSA-value type: {ty}"
1115            ))),
1116        }
1117    }
1118
1119    fn canonical_type_for_rc(rc: RegClass) -> Type {
1120        match rc {
1121            RegClass::Float => types::I8X16,
1122            RegClass::Int => types::I64,
1123            RegClass::Vector => unreachable!(),
1124        }
1125    }
1126
1127    fn gen_jump(target: MachLabel) -> Inst {
1128        Inst::Jump {
1129            dest: BranchTarget::Label(target),
1130        }
1131    }
1132
1133    fn worst_case_size() -> CodeOffset {
1134        // The maximum size, in bytes, of any `Inst`'s emitted code. We have at least one case of
1135        // an 8-instruction sequence (saturating int-to-float conversions) with three embedded
1136        // 64-bit f64 constants.
1137        //
1138        // Note that inline jump-tables handle island/pool insertion separately, so we do not need
1139        // to account for them here (otherwise the worst case would be 2^31 * 4, clearly not
1140        // feasible for other reasons).
1141        44
1142    }
1143
1144    fn ref_type_regclass(_: &settings::Flags) -> RegClass {
1145        RegClass::Int
1146    }
1147
1148    fn gen_block_start(
1149        is_indirect_branch_target: bool,
1150        is_forward_edge_cfi_enabled: bool,
1151    ) -> Option<Self> {
1152        if is_indirect_branch_target && is_forward_edge_cfi_enabled {
1153            Some(Inst::Bti {
1154                targets: BranchTargetType::J,
1155            })
1156        } else {
1157            None
1158        }
1159    }
1160
1161    fn function_alignment() -> FunctionAlignment {
1162        // We use 32-byte alignment for performance reasons, but for correctness
1163        // we would only need 4-byte alignment.
1164        FunctionAlignment {
1165            minimum: 4,
1166            preferred: 32,
1167        }
1168    }
1169}
1170
1171//=============================================================================
1172// Pretty-printing of instructions.
1173
1174fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (String, String) {
1175    let (mem_insts, mem) = mem_finalize(None, mem, access_ty, state);
1176    let mut mem_str = mem_insts
1177        .into_iter()
1178        .map(|inst| inst.print_with_state(&mut EmitState::default()))
1179        .collect::<Vec<_>>()
1180        .join(" ; ");
1181    if !mem_str.is_empty() {
1182        mem_str += " ; ";
1183    }
1184
1185    let mem = mem.pretty_print(access_ty.bytes() as u8);
1186    (mem_str, mem)
1187}
1188
1189fn pretty_print_try_call(info: &TryCallInfo) -> String {
1190    format!(
1191        "; b {:?}; catch [{}]",
1192        info.continuation,
1193        info.pretty_print_dests()
1194    )
1195}
1196
1197impl Inst {
1198    fn print_with_state(&self, state: &mut EmitState) -> String {
1199        fn op_name(alu_op: ALUOp) -> &'static str {
1200            match alu_op {
1201                ALUOp::Add => "add",
1202                ALUOp::Sub => "sub",
1203                ALUOp::Orr => "orr",
1204                ALUOp::And => "and",
1205                ALUOp::AndS => "ands",
1206                ALUOp::Eor => "eor",
1207                ALUOp::AddS => "adds",
1208                ALUOp::SubS => "subs",
1209                ALUOp::SMulH => "smulh",
1210                ALUOp::UMulH => "umulh",
1211                ALUOp::SDiv => "sdiv",
1212                ALUOp::UDiv => "udiv",
1213                ALUOp::AndNot => "bic",
1214                ALUOp::OrrNot => "orn",
1215                ALUOp::EorNot => "eon",
1216                ALUOp::Extr => "extr",
1217                ALUOp::Lsr => "lsr",
1218                ALUOp::Asr => "asr",
1219                ALUOp::Lsl => "lsl",
1220                ALUOp::Adc => "adc",
1221                ALUOp::AdcS => "adcs",
1222                ALUOp::Sbc => "sbc",
1223                ALUOp::SbcS => "sbcs",
1224            }
1225        }
1226
1227        match self {
1228            &Inst::Nop0 => "nop-zero-len".to_string(),
1229            &Inst::Nop4 => "nop".to_string(),
1230            &Inst::AluRRR {
1231                alu_op,
1232                size,
1233                rd,
1234                rn,
1235                rm,
1236            } => {
1237                let op = op_name(alu_op);
1238                let rd = pretty_print_ireg(rd.to_reg(), size);
1239                let rn = pretty_print_ireg(rn, size);
1240                let rm = pretty_print_ireg(rm, size);
1241                format!("{op} {rd}, {rn}, {rm}")
1242            }
1243            &Inst::AluRRRR {
1244                alu_op,
1245                size,
1246                rd,
1247                rn,
1248                rm,
1249                ra,
1250            } => {
1251                let (op, da_size) = match alu_op {
1252                    ALUOp3::MAdd => ("madd", size),
1253                    ALUOp3::MSub => ("msub", size),
1254                    ALUOp3::UMAddL => ("umaddl", OperandSize::Size64),
1255                    ALUOp3::SMAddL => ("smaddl", OperandSize::Size64),
1256                };
1257                let rd = pretty_print_ireg(rd.to_reg(), da_size);
1258                let rn = pretty_print_ireg(rn, size);
1259                let rm = pretty_print_ireg(rm, size);
1260                let ra = pretty_print_ireg(ra, da_size);
1261
1262                format!("{op} {rd}, {rn}, {rm}, {ra}")
1263            }
1264            &Inst::AluRRImm12 {
1265                alu_op,
1266                size,
1267                rd,
1268                rn,
1269                ref imm12,
1270            } => {
1271                let op = op_name(alu_op);
1272                let rd = pretty_print_ireg(rd.to_reg(), size);
1273                let rn = pretty_print_ireg(rn, size);
1274
1275                if imm12.bits == 0 && alu_op == ALUOp::Add && size.is64() {
1276                    // special-case MOV (used for moving into SP).
1277                    format!("mov {rd}, {rn}")
1278                } else {
1279                    let imm12 = imm12.pretty_print(0);
1280                    format!("{op} {rd}, {rn}, {imm12}")
1281                }
1282            }
1283            &Inst::AluRRImmLogic {
1284                alu_op,
1285                size,
1286                rd,
1287                rn,
1288                ref imml,
1289            } => {
1290                let op = op_name(alu_op);
1291                let rd = pretty_print_ireg(rd.to_reg(), size);
1292                let rn = pretty_print_ireg(rn, size);
1293                let imml = imml.pretty_print(0);
1294                format!("{op} {rd}, {rn}, {imml}")
1295            }
1296            &Inst::AluRRImmShift {
1297                alu_op,
1298                size,
1299                rd,
1300                rn,
1301                ref immshift,
1302            } => {
1303                let op = op_name(alu_op);
1304                let rd = pretty_print_ireg(rd.to_reg(), size);
1305                let rn = pretty_print_ireg(rn, size);
1306                let immshift = immshift.pretty_print(0);
1307                format!("{op} {rd}, {rn}, {immshift}")
1308            }
1309            &Inst::AluRRRShift {
1310                alu_op,
1311                size,
1312                rd,
1313                rn,
1314                rm,
1315                ref shiftop,
1316            } => {
1317                let op = op_name(alu_op);
1318                let rd = pretty_print_ireg(rd.to_reg(), size);
1319                let rn = pretty_print_ireg(rn, size);
1320                let rm = pretty_print_ireg(rm, size);
1321                let shiftop = shiftop.pretty_print(0);
1322                format!("{op} {rd}, {rn}, {rm}, {shiftop}")
1323            }
1324            &Inst::AluRRRExtend {
1325                alu_op,
1326                size,
1327                rd,
1328                rn,
1329                rm,
1330                ref extendop,
1331            } => {
1332                let op = op_name(alu_op);
1333                let rd = pretty_print_ireg(rd.to_reg(), size);
1334                let rn = pretty_print_ireg(rn, size);
1335                let rm = pretty_print_ireg(rm, size);
1336                let extendop = extendop.pretty_print(0);
1337                format!("{op} {rd}, {rn}, {rm}, {extendop}")
1338            }
1339            &Inst::BitRR { op, size, rd, rn } => {
1340                let op = op.op_str();
1341                let rd = pretty_print_ireg(rd.to_reg(), size);
1342                let rn = pretty_print_ireg(rn, size);
1343                format!("{op} {rd}, {rn}")
1344            }
1345            &Inst::ULoad8 { rd, ref mem, .. }
1346            | &Inst::SLoad8 { rd, ref mem, .. }
1347            | &Inst::ULoad16 { rd, ref mem, .. }
1348            | &Inst::SLoad16 { rd, ref mem, .. }
1349            | &Inst::ULoad32 { rd, ref mem, .. }
1350            | &Inst::SLoad32 { rd, ref mem, .. }
1351            | &Inst::ULoad64 { rd, ref mem, .. } => {
1352                let is_unscaled = match &mem {
1353                    &AMode::Unscaled { .. } => true,
1354                    _ => false,
1355                };
1356                let (op, size) = match (self, is_unscaled) {
1357                    (&Inst::ULoad8 { .. }, false) => ("ldrb", OperandSize::Size32),
1358                    (&Inst::ULoad8 { .. }, true) => ("ldurb", OperandSize::Size32),
1359                    (&Inst::SLoad8 { .. }, false) => ("ldrsb", OperandSize::Size64),
1360                    (&Inst::SLoad8 { .. }, true) => ("ldursb", OperandSize::Size64),
1361                    (&Inst::ULoad16 { .. }, false) => ("ldrh", OperandSize::Size32),
1362                    (&Inst::ULoad16 { .. }, true) => ("ldurh", OperandSize::Size32),
1363                    (&Inst::SLoad16 { .. }, false) => ("ldrsh", OperandSize::Size64),
1364                    (&Inst::SLoad16 { .. }, true) => ("ldursh", OperandSize::Size64),
1365                    (&Inst::ULoad32 { .. }, false) => ("ldr", OperandSize::Size32),
1366                    (&Inst::ULoad32 { .. }, true) => ("ldur", OperandSize::Size32),
1367                    (&Inst::SLoad32 { .. }, false) => ("ldrsw", OperandSize::Size64),
1368                    (&Inst::SLoad32 { .. }, true) => ("ldursw", OperandSize::Size64),
1369                    (&Inst::ULoad64 { .. }, false) => ("ldr", OperandSize::Size64),
1370                    (&Inst::ULoad64 { .. }, true) => ("ldur", OperandSize::Size64),
1371                    _ => unreachable!(),
1372                };
1373
1374                let rd = pretty_print_ireg(rd.to_reg(), size);
1375                let mem = mem.clone();
1376                let access_ty = self.mem_type().unwrap();
1377                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1378
1379                format!("{mem_str}{op} {rd}, {mem}")
1380            }
1381            &Inst::Store8 { rd, ref mem, .. }
1382            | &Inst::Store16 { rd, ref mem, .. }
1383            | &Inst::Store32 { rd, ref mem, .. }
1384            | &Inst::Store64 { rd, ref mem, .. } => {
1385                let is_unscaled = match &mem {
1386                    &AMode::Unscaled { .. } => true,
1387                    _ => false,
1388                };
1389                let (op, size) = match (self, is_unscaled) {
1390                    (&Inst::Store8 { .. }, false) => ("strb", OperandSize::Size32),
1391                    (&Inst::Store8 { .. }, true) => ("sturb", OperandSize::Size32),
1392                    (&Inst::Store16 { .. }, false) => ("strh", OperandSize::Size32),
1393                    (&Inst::Store16 { .. }, true) => ("sturh", OperandSize::Size32),
1394                    (&Inst::Store32 { .. }, false) => ("str", OperandSize::Size32),
1395                    (&Inst::Store32 { .. }, true) => ("stur", OperandSize::Size32),
1396                    (&Inst::Store64 { .. }, false) => ("str", OperandSize::Size64),
1397                    (&Inst::Store64 { .. }, true) => ("stur", OperandSize::Size64),
1398                    _ => unreachable!(),
1399                };
1400
1401                let rd = pretty_print_ireg(rd, size);
1402                let mem = mem.clone();
1403                let access_ty = self.mem_type().unwrap();
1404                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1405
1406                format!("{mem_str}{op} {rd}, {mem}")
1407            }
1408            &Inst::StoreP64 {
1409                rt, rt2, ref mem, ..
1410            } => {
1411                let rt = pretty_print_ireg(rt, OperandSize::Size64);
1412                let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);
1413                let mem = mem.clone();
1414                let mem = mem.pretty_print_default();
1415                format!("stp {rt}, {rt2}, {mem}")
1416            }
1417            &Inst::LoadP64 {
1418                rt, rt2, ref mem, ..
1419            } => {
1420                let rt = pretty_print_ireg(rt.to_reg(), OperandSize::Size64);
1421                let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);
1422                let mem = mem.clone();
1423                let mem = mem.pretty_print_default();
1424                format!("ldp {rt}, {rt2}, {mem}")
1425            }
1426            &Inst::Mov { size, rd, rm } => {
1427                let rd = pretty_print_ireg(rd.to_reg(), size);
1428                let rm = pretty_print_ireg(rm, size);
1429                format!("mov {rd}, {rm}")
1430            }
1431            &Inst::MovFromPReg { rd, rm } => {
1432                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1433                let rm = show_ireg_sized(rm.into(), OperandSize::Size64);
1434                format!("mov {rd}, {rm}")
1435            }
1436            &Inst::MovToPReg { rd, rm } => {
1437                let rd = show_ireg_sized(rd.into(), OperandSize::Size64);
1438                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1439                format!("mov {rd}, {rm}")
1440            }
1441            &Inst::MovWide {
1442                op,
1443                rd,
1444                ref imm,
1445                size,
1446            } => {
1447                let op_str = match op {
1448                    MoveWideOp::MovZ => "movz",
1449                    MoveWideOp::MovN => "movn",
1450                };
1451                let rd = pretty_print_ireg(rd.to_reg(), size);
1452                let imm = imm.pretty_print(0);
1453                format!("{op_str} {rd}, {imm}")
1454            }
1455            &Inst::MovK {
1456                rd,
1457                rn,
1458                ref imm,
1459                size,
1460            } => {
1461                let rn = pretty_print_ireg(rn, size);
1462                let rd = pretty_print_ireg(rd.to_reg(), size);
1463                let imm = imm.pretty_print(0);
1464                format!("movk {rd}, {rn}, {imm}")
1465            }
1466            &Inst::CSel { rd, rn, rm, cond } => {
1467                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1468                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1469                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1470                let cond = cond.pretty_print(0);
1471                format!("csel {rd}, {rn}, {rm}, {cond}")
1472            }
1473            &Inst::CSNeg { rd, rn, rm, cond } => {
1474                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1475                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1476                let rm = pretty_print_ireg(rm, OperandSize::Size64);
1477                let cond = cond.pretty_print(0);
1478                format!("csneg {rd}, {rn}, {rm}, {cond}")
1479            }
1480            &Inst::CSet { rd, cond } => {
1481                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1482                let cond = cond.pretty_print(0);
1483                format!("cset {rd}, {cond}")
1484            }
1485            &Inst::CSetm { rd, cond } => {
1486                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1487                let cond = cond.pretty_print(0);
1488                format!("csetm {rd}, {cond}")
1489            }
1490            &Inst::CCmp {
1491                size,
1492                rn,
1493                rm,
1494                nzcv,
1495                cond,
1496            } => {
1497                let rn = pretty_print_ireg(rn, size);
1498                let rm = pretty_print_ireg(rm, size);
1499                let nzcv = nzcv.pretty_print(0);
1500                let cond = cond.pretty_print(0);
1501                format!("ccmp {rn}, {rm}, {nzcv}, {cond}")
1502            }
1503            &Inst::CCmpImm {
1504                size,
1505                rn,
1506                imm,
1507                nzcv,
1508                cond,
1509            } => {
1510                let rn = pretty_print_ireg(rn, size);
1511                let imm = imm.pretty_print(0);
1512                let nzcv = nzcv.pretty_print(0);
1513                let cond = cond.pretty_print(0);
1514                format!("ccmp {rn}, {imm}, {nzcv}, {cond}")
1515            }
1516            &Inst::AtomicRMW {
1517                rs, rt, rn, ty, op, ..
1518            } => {
1519                let op = match op {
1520                    AtomicRMWOp::Add => "ldaddal",
1521                    AtomicRMWOp::Clr => "ldclral",
1522                    AtomicRMWOp::Eor => "ldeoral",
1523                    AtomicRMWOp::Set => "ldsetal",
1524                    AtomicRMWOp::Smax => "ldsmaxal",
1525                    AtomicRMWOp::Umax => "ldumaxal",
1526                    AtomicRMWOp::Smin => "ldsminal",
1527                    AtomicRMWOp::Umin => "lduminal",
1528                    AtomicRMWOp::Swp => "swpal",
1529                };
1530
1531                let size = OperandSize::from_ty(ty);
1532                let rs = pretty_print_ireg(rs, size);
1533                let rt = pretty_print_ireg(rt.to_reg(), size);
1534                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1535
1536                let ty_suffix = match ty {
1537                    I8 => "b",
1538                    I16 => "h",
1539                    _ => "",
1540                };
1541                format!("{op}{ty_suffix} {rs}, {rt}, [{rn}]")
1542            }
1543            &Inst::AtomicRMWLoop {
1544                ty,
1545                op,
1546                addr,
1547                operand,
1548                oldval,
1549                scratch1,
1550                scratch2,
1551                ..
1552            } => {
1553                let op = match op {
1554                    AtomicRMWLoopOp::Add => "add",
1555                    AtomicRMWLoopOp::Sub => "sub",
1556                    AtomicRMWLoopOp::Eor => "eor",
1557                    AtomicRMWLoopOp::Orr => "orr",
1558                    AtomicRMWLoopOp::And => "and",
1559                    AtomicRMWLoopOp::Nand => "nand",
1560                    AtomicRMWLoopOp::Smin => "smin",
1561                    AtomicRMWLoopOp::Smax => "smax",
1562                    AtomicRMWLoopOp::Umin => "umin",
1563                    AtomicRMWLoopOp::Umax => "umax",
1564                    AtomicRMWLoopOp::Xchg => "xchg",
1565                };
1566                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1567                let operand = pretty_print_ireg(operand, OperandSize::Size64);
1568                let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1569                let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);
1570                let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);
1571                format!(
1572                    "atomic_rmw_loop_{}_{} addr={} operand={} oldval={} scratch1={} scratch2={}",
1573                    op,
1574                    ty.bits(),
1575                    addr,
1576                    operand,
1577                    oldval,
1578                    scratch1,
1579                    scratch2,
1580                )
1581            }
1582            &Inst::AtomicCAS {
1583                rd, rs, rt, rn, ty, ..
1584            } => {
1585                let op = match ty {
1586                    I8 => "casalb",
1587                    I16 => "casalh",
1588                    I32 | I64 => "casal",
1589                    _ => panic!("Unsupported type: {ty}"),
1590                };
1591                let size = OperandSize::from_ty(ty);
1592                let rd = pretty_print_ireg(rd.to_reg(), size);
1593                let rs = pretty_print_ireg(rs, size);
1594                let rt = pretty_print_ireg(rt, size);
1595                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1596
1597                format!("{op} {rd}, {rs}, {rt}, [{rn}]")
1598            }
1599            &Inst::AtomicCASLoop {
1600                ty,
1601                addr,
1602                expected,
1603                replacement,
1604                oldval,
1605                scratch,
1606                ..
1607            } => {
1608                let addr = pretty_print_ireg(addr, OperandSize::Size64);
1609                let expected = pretty_print_ireg(expected, OperandSize::Size64);
1610                let replacement = pretty_print_ireg(replacement, OperandSize::Size64);
1611                let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1612                let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1613                format!(
1614                    "atomic_cas_loop_{} addr={}, expect={}, replacement={}, oldval={}, scratch={}",
1615                    ty.bits(),
1616                    addr,
1617                    expected,
1618                    replacement,
1619                    oldval,
1620                    scratch,
1621                )
1622            }
1623            &Inst::LoadAcquire {
1624                access_ty, rt, rn, ..
1625            } => {
1626                let (op, ty) = match access_ty {
1627                    I8 => ("ldarb", I32),
1628                    I16 => ("ldarh", I32),
1629                    I32 => ("ldar", I32),
1630                    I64 => ("ldar", I64),
1631                    _ => panic!("Unsupported type: {access_ty}"),
1632                };
1633                let size = OperandSize::from_ty(ty);
1634                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1635                let rt = pretty_print_ireg(rt.to_reg(), size);
1636                format!("{op} {rt}, [{rn}]")
1637            }
1638            &Inst::StoreRelease {
1639                access_ty, rt, rn, ..
1640            } => {
1641                let (op, ty) = match access_ty {
1642                    I8 => ("stlrb", I32),
1643                    I16 => ("stlrh", I32),
1644                    I32 => ("stlr", I32),
1645                    I64 => ("stlr", I64),
1646                    _ => panic!("Unsupported type: {access_ty}"),
1647                };
1648                let size = OperandSize::from_ty(ty);
1649                let rn = pretty_print_ireg(rn, OperandSize::Size64);
1650                let rt = pretty_print_ireg(rt, size);
1651                format!("{op} {rt}, [{rn}]")
1652            }
1653            &Inst::Fence {} => {
1654                format!("dmb ish")
1655            }
1656            &Inst::Csdb {} => {
1657                format!("csdb")
1658            }
1659            &Inst::FpuMove32 { rd, rn } => {
1660                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1661                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
1662                format!("fmov {rd}, {rn}")
1663            }
1664            &Inst::FpuMove64 { rd, rn } => {
1665                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1666                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
1667                format!("fmov {rd}, {rn}")
1668            }
1669            &Inst::FpuMove128 { rd, rn } => {
1670                let rd = pretty_print_reg(rd.to_reg());
1671                let rn = pretty_print_reg(rn);
1672                format!("mov {rd}.16b, {rn}.16b")
1673            }
1674            &Inst::FpuMoveFromVec { rd, rn, idx, size } => {
1675                let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
1676                let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
1677                format!("mov {rd}, {rn}")
1678            }
1679            &Inst::FpuExtend { rd, rn, size } => {
1680                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1681                let rn = pretty_print_vreg_scalar(rn, size);
1682                format!("fmov {rd}, {rn}")
1683            }
1684            &Inst::FpuRR {
1685                fpu_op,
1686                size,
1687                rd,
1688                rn,
1689            } => {
1690                let op = match fpu_op {
1691                    FPUOp1::Abs => "fabs",
1692                    FPUOp1::Neg => "fneg",
1693                    FPUOp1::Sqrt => "fsqrt",
1694                    FPUOp1::Cvt32To64 | FPUOp1::Cvt64To32 => "fcvt",
1695                };
1696                let dst_size = match fpu_op {
1697                    FPUOp1::Cvt32To64 => ScalarSize::Size64,
1698                    FPUOp1::Cvt64To32 => ScalarSize::Size32,
1699                    _ => size,
1700                };
1701                let rd = pretty_print_vreg_scalar(rd.to_reg(), dst_size);
1702                let rn = pretty_print_vreg_scalar(rn, size);
1703                format!("{op} {rd}, {rn}")
1704            }
1705            &Inst::FpuRRR {
1706                fpu_op,
1707                size,
1708                rd,
1709                rn,
1710                rm,
1711            } => {
1712                let op = match fpu_op {
1713                    FPUOp2::Add => "fadd",
1714                    FPUOp2::Sub => "fsub",
1715                    FPUOp2::Mul => "fmul",
1716                    FPUOp2::Div => "fdiv",
1717                    FPUOp2::Max => "fmax",
1718                    FPUOp2::Min => "fmin",
1719                };
1720                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1721                let rn = pretty_print_vreg_scalar(rn, size);
1722                let rm = pretty_print_vreg_scalar(rm, size);
1723                format!("{op} {rd}, {rn}, {rm}")
1724            }
1725            &Inst::FpuRRI { fpu_op, rd, rn } => {
1726                let (op, imm, vector) = match fpu_op {
1727                    FPUOpRI::UShr32(imm) => ("ushr", imm.pretty_print(0), true),
1728                    FPUOpRI::UShr64(imm) => ("ushr", imm.pretty_print(0), false),
1729                };
1730
1731                let (rd, rn) = if vector {
1732                    (
1733                        pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
1734                        pretty_print_vreg_vector(rn, VectorSize::Size32x2),
1735                    )
1736                } else {
1737                    (
1738                        pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
1739                        pretty_print_vreg_scalar(rn, ScalarSize::Size64),
1740                    )
1741                };
1742                format!("{op} {rd}, {rn}, {imm}")
1743            }
1744            &Inst::FpuRRIMod { fpu_op, rd, ri, rn } => {
1745                let (op, imm, vector) = match fpu_op {
1746                    FPUOpRIMod::Sli32(imm) => ("sli", imm.pretty_print(0), true),
1747                    FPUOpRIMod::Sli64(imm) => ("sli", imm.pretty_print(0), false),
1748                };
1749
1750                let (rd, ri, rn) = if vector {
1751                    (
1752                        pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
1753                        pretty_print_vreg_vector(ri, VectorSize::Size32x2),
1754                        pretty_print_vreg_vector(rn, VectorSize::Size32x2),
1755                    )
1756                } else {
1757                    (
1758                        pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
1759                        pretty_print_vreg_scalar(ri, ScalarSize::Size64),
1760                        pretty_print_vreg_scalar(rn, ScalarSize::Size64),
1761                    )
1762                };
1763                format!("{op} {rd}, {ri}, {rn}, {imm}")
1764            }
1765            &Inst::FpuRRRR {
1766                fpu_op,
1767                size,
1768                rd,
1769                rn,
1770                rm,
1771                ra,
1772            } => {
1773                let op = match fpu_op {
1774                    FPUOp3::MAdd => "fmadd",
1775                    FPUOp3::MSub => "fmsub",
1776                    FPUOp3::NMAdd => "fnmadd",
1777                    FPUOp3::NMSub => "fnmsub",
1778                };
1779                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1780                let rn = pretty_print_vreg_scalar(rn, size);
1781                let rm = pretty_print_vreg_scalar(rm, size);
1782                let ra = pretty_print_vreg_scalar(ra, size);
1783                format!("{op} {rd}, {rn}, {rm}, {ra}")
1784            }
1785            &Inst::FpuCmp { size, rn, rm } => {
1786                let rn = pretty_print_vreg_scalar(rn, size);
1787                let rm = pretty_print_vreg_scalar(rm, size);
1788                format!("fcmp {rn}, {rm}")
1789            }
1790            &Inst::FpuLoad16 { rd, ref mem, .. } => {
1791                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
1792                let mem = mem.clone();
1793                let access_ty = self.mem_type().unwrap();
1794                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1795                format!("{mem_str}ldr {rd}, {mem}")
1796            }
1797            &Inst::FpuLoad32 { rd, ref mem, .. } => {
1798                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1799                let mem = mem.clone();
1800                let access_ty = self.mem_type().unwrap();
1801                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1802                format!("{mem_str}ldr {rd}, {mem}")
1803            }
1804            &Inst::FpuLoad64 { rd, ref mem, .. } => {
1805                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1806                let mem = mem.clone();
1807                let access_ty = self.mem_type().unwrap();
1808                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1809                format!("{mem_str}ldr {rd}, {mem}")
1810            }
1811            &Inst::FpuLoad128 { rd, ref mem, .. } => {
1812                let rd = pretty_print_reg(rd.to_reg());
1813                let rd = "q".to_string() + &rd[1..];
1814                let mem = mem.clone();
1815                let access_ty = self.mem_type().unwrap();
1816                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1817                format!("{mem_str}ldr {rd}, {mem}")
1818            }
1819            &Inst::FpuStore16 { rd, ref mem, .. } => {
1820                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size16);
1821                let mem = mem.clone();
1822                let access_ty = self.mem_type().unwrap();
1823                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1824                format!("{mem_str}str {rd}, {mem}")
1825            }
1826            &Inst::FpuStore32 { rd, ref mem, .. } => {
1827                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size32);
1828                let mem = mem.clone();
1829                let access_ty = self.mem_type().unwrap();
1830                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1831                format!("{mem_str}str {rd}, {mem}")
1832            }
1833            &Inst::FpuStore64 { rd, ref mem, .. } => {
1834                let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size64);
1835                let mem = mem.clone();
1836                let access_ty = self.mem_type().unwrap();
1837                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1838                format!("{mem_str}str {rd}, {mem}")
1839            }
1840            &Inst::FpuStore128 { rd, ref mem, .. } => {
1841                let rd = pretty_print_reg(rd);
1842                let rd = "q".to_string() + &rd[1..];
1843                let mem = mem.clone();
1844                let access_ty = self.mem_type().unwrap();
1845                let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1846                format!("{mem_str}str {rd}, {mem}")
1847            }
1848            &Inst::FpuLoadP64 {
1849                rt, rt2, ref mem, ..
1850            } => {
1851                let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size64);
1852                let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size64);
1853                let mem = mem.clone();
1854                let mem = mem.pretty_print_default();
1855
1856                format!("ldp {rt}, {rt2}, {mem}")
1857            }
1858            &Inst::FpuStoreP64 {
1859                rt, rt2, ref mem, ..
1860            } => {
1861                let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size64);
1862                let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size64);
1863                let mem = mem.clone();
1864                let mem = mem.pretty_print_default();
1865
1866                format!("stp {rt}, {rt2}, {mem}")
1867            }
1868            &Inst::FpuLoadP128 {
1869                rt, rt2, ref mem, ..
1870            } => {
1871                let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size128);
1872                let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size128);
1873                let mem = mem.clone();
1874                let mem = mem.pretty_print_default();
1875
1876                format!("ldp {rt}, {rt2}, {mem}")
1877            }
1878            &Inst::FpuStoreP128 {
1879                rt, rt2, ref mem, ..
1880            } => {
1881                let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size128);
1882                let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size128);
1883                let mem = mem.clone();
1884                let mem = mem.pretty_print_default();
1885
1886                format!("stp {rt}, {rt2}, {mem}")
1887            }
1888            &Inst::FpuToInt { op, rd, rn } => {
1889                let (op, sizesrc, sizedest) = match op {
1890                    FpuToIntOp::F32ToI32 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size32),
1891                    FpuToIntOp::F32ToU32 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size32),
1892                    FpuToIntOp::F32ToI64 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size64),
1893                    FpuToIntOp::F32ToU64 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size64),
1894                    FpuToIntOp::F64ToI32 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size32),
1895                    FpuToIntOp::F64ToU32 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size32),
1896                    FpuToIntOp::F64ToI64 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size64),
1897                    FpuToIntOp::F64ToU64 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size64),
1898                };
1899                let rd = pretty_print_ireg(rd.to_reg(), sizedest);
1900                let rn = pretty_print_vreg_scalar(rn, sizesrc);
1901                format!("{op} {rd}, {rn}")
1902            }
1903            &Inst::IntToFpu { op, rd, rn } => {
1904                let (op, sizesrc, sizedest) = match op {
1905                    IntToFpuOp::I32ToF32 => ("scvtf", OperandSize::Size32, ScalarSize::Size32),
1906                    IntToFpuOp::U32ToF32 => ("ucvtf", OperandSize::Size32, ScalarSize::Size32),
1907                    IntToFpuOp::I64ToF32 => ("scvtf", OperandSize::Size64, ScalarSize::Size32),
1908                    IntToFpuOp::U64ToF32 => ("ucvtf", OperandSize::Size64, ScalarSize::Size32),
1909                    IntToFpuOp::I32ToF64 => ("scvtf", OperandSize::Size32, ScalarSize::Size64),
1910                    IntToFpuOp::U32ToF64 => ("ucvtf", OperandSize::Size32, ScalarSize::Size64),
1911                    IntToFpuOp::I64ToF64 => ("scvtf", OperandSize::Size64, ScalarSize::Size64),
1912                    IntToFpuOp::U64ToF64 => ("ucvtf", OperandSize::Size64, ScalarSize::Size64),
1913                };
1914                let rd = pretty_print_vreg_scalar(rd.to_reg(), sizedest);
1915                let rn = pretty_print_ireg(rn, sizesrc);
1916                format!("{op} {rd}, {rn}")
1917            }
1918            &Inst::FpuCSel16 { rd, rn, rm, cond } => {
1919                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
1920                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size16);
1921                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size16);
1922                let cond = cond.pretty_print(0);
1923                format!("fcsel {rd}, {rn}, {rm}, {cond}")
1924            }
1925            &Inst::FpuCSel32 { rd, rn, rm, cond } => {
1926                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1927                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
1928                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size32);
1929                let cond = cond.pretty_print(0);
1930                format!("fcsel {rd}, {rn}, {rm}, {cond}")
1931            }
1932            &Inst::FpuCSel64 { rd, rn, rm, cond } => {
1933                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1934                let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
1935                let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size64);
1936                let cond = cond.pretty_print(0);
1937                format!("fcsel {rd}, {rn}, {rm}, {cond}")
1938            }
1939            &Inst::FpuRound { op, rd, rn } => {
1940                let (inst, size) = match op {
1941                    FpuRoundMode::Minus32 => ("frintm", ScalarSize::Size32),
1942                    FpuRoundMode::Minus64 => ("frintm", ScalarSize::Size64),
1943                    FpuRoundMode::Plus32 => ("frintp", ScalarSize::Size32),
1944                    FpuRoundMode::Plus64 => ("frintp", ScalarSize::Size64),
1945                    FpuRoundMode::Zero32 => ("frintz", ScalarSize::Size32),
1946                    FpuRoundMode::Zero64 => ("frintz", ScalarSize::Size64),
1947                    FpuRoundMode::Nearest32 => ("frintn", ScalarSize::Size32),
1948                    FpuRoundMode::Nearest64 => ("frintn", ScalarSize::Size64),
1949                };
1950                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1951                let rn = pretty_print_vreg_scalar(rn, size);
1952                format!("{inst} {rd}, {rn}")
1953            }
1954            &Inst::MovToFpu { rd, rn, size } => {
1955                let operand_size = size.operand_size();
1956                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1957                let rn = pretty_print_ireg(rn, operand_size);
1958                format!("fmov {rd}, {rn}")
1959            }
1960            &Inst::FpuMoveFPImm { rd, imm, size } => {
1961                let imm = imm.pretty_print(0);
1962                let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1963
1964                format!("fmov {rd}, {imm}")
1965            }
1966            &Inst::MovToVec {
1967                rd,
1968                ri,
1969                rn,
1970                idx,
1971                size,
1972            } => {
1973                let rd = pretty_print_vreg_element(rd.to_reg(), idx as usize, size.lane_size());
1974                let ri = pretty_print_vreg_element(ri, idx as usize, size.lane_size());
1975                let rn = pretty_print_ireg(rn, size.operand_size());
1976                format!("mov {rd}, {ri}, {rn}")
1977            }
1978            &Inst::MovFromVec { rd, rn, idx, size } => {
1979                let op = match size {
1980                    ScalarSize::Size8 => "umov",
1981                    ScalarSize::Size16 => "umov",
1982                    ScalarSize::Size32 => "mov",
1983                    ScalarSize::Size64 => "mov",
1984                    _ => unimplemented!(),
1985                };
1986                let rd = pretty_print_ireg(rd.to_reg(), size.operand_size());
1987                let rn = pretty_print_vreg_element(rn, idx as usize, size);
1988                format!("{op} {rd}, {rn}")
1989            }
1990            &Inst::MovFromVecSigned {
1991                rd,
1992                rn,
1993                idx,
1994                size,
1995                scalar_size,
1996            } => {
1997                let rd = pretty_print_ireg(rd.to_reg(), scalar_size);
1998                let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
1999                format!("smov {rd}, {rn}")
2000            }
2001            &Inst::VecDup { rd, rn, size } => {
2002                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2003                let rn = pretty_print_ireg(rn, size.operand_size());
2004                format!("dup {rd}, {rn}")
2005            }
2006            &Inst::VecDupFromFpu { rd, rn, size, lane } => {
2007                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2008                let rn = pretty_print_vreg_element(rn, lane.into(), size.lane_size());
2009                format!("dup {rd}, {rn}")
2010            }
2011            &Inst::VecDupFPImm { rd, imm, size } => {
2012                let imm = imm.pretty_print(0);
2013                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2014
2015                format!("fmov {rd}, {imm}")
2016            }
2017            &Inst::VecDupImm {
2018                rd,
2019                imm,
2020                invert,
2021                size,
2022            } => {
2023                let imm = imm.pretty_print(0);
2024                let op = if invert { "mvni" } else { "movi" };
2025                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2026
2027                format!("{op} {rd}, {imm}")
2028            }
2029            &Inst::VecExtend {
2030                t,
2031                rd,
2032                rn,
2033                high_half,
2034                lane_size,
2035            } => {
2036                let vec64 = VectorSize::from_lane_size(lane_size.narrow(), false);
2037                let vec128 = VectorSize::from_lane_size(lane_size.narrow(), true);
2038                let rd_size = VectorSize::from_lane_size(lane_size, true);
2039                let (op, rn_size) = match (t, high_half) {
2040                    (VecExtendOp::Sxtl, false) => ("sxtl", vec64),
2041                    (VecExtendOp::Sxtl, true) => ("sxtl2", vec128),
2042                    (VecExtendOp::Uxtl, false) => ("uxtl", vec64),
2043                    (VecExtendOp::Uxtl, true) => ("uxtl2", vec128),
2044                };
2045                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2046                let rn = pretty_print_vreg_vector(rn, rn_size);
2047                format!("{op} {rd}, {rn}")
2048            }
2049            &Inst::VecMovElement {
2050                rd,
2051                ri,
2052                rn,
2053                dest_idx,
2054                src_idx,
2055                size,
2056            } => {
2057                let rd =
2058                    pretty_print_vreg_element(rd.to_reg(), dest_idx as usize, size.lane_size());
2059                let ri = pretty_print_vreg_element(ri, dest_idx as usize, size.lane_size());
2060                let rn = pretty_print_vreg_element(rn, src_idx as usize, size.lane_size());
2061                format!("mov {rd}, {ri}, {rn}")
2062            }
2063            &Inst::VecRRLong {
2064                op,
2065                rd,
2066                rn,
2067                high_half,
2068            } => {
2069                let (op, rd_size, size, suffix) = match (op, high_half) {
2070                    (VecRRLongOp::Fcvtl16, false) => {
2071                        ("fcvtl", VectorSize::Size32x4, VectorSize::Size16x4, "")
2072                    }
2073                    (VecRRLongOp::Fcvtl16, true) => {
2074                        ("fcvtl2", VectorSize::Size32x4, VectorSize::Size16x8, "")
2075                    }
2076                    (VecRRLongOp::Fcvtl32, false) => {
2077                        ("fcvtl", VectorSize::Size64x2, VectorSize::Size32x2, "")
2078                    }
2079                    (VecRRLongOp::Fcvtl32, true) => {
2080                        ("fcvtl2", VectorSize::Size64x2, VectorSize::Size32x4, "")
2081                    }
2082                    (VecRRLongOp::Shll8, false) => {
2083                        ("shll", VectorSize::Size16x8, VectorSize::Size8x8, ", #8")
2084                    }
2085                    (VecRRLongOp::Shll8, true) => {
2086                        ("shll2", VectorSize::Size16x8, VectorSize::Size8x16, ", #8")
2087                    }
2088                    (VecRRLongOp::Shll16, false) => {
2089                        ("shll", VectorSize::Size32x4, VectorSize::Size16x4, ", #16")
2090                    }
2091                    (VecRRLongOp::Shll16, true) => {
2092                        ("shll2", VectorSize::Size32x4, VectorSize::Size16x8, ", #16")
2093                    }
2094                    (VecRRLongOp::Shll32, false) => {
2095                        ("shll", VectorSize::Size64x2, VectorSize::Size32x2, ", #32")
2096                    }
2097                    (VecRRLongOp::Shll32, true) => {
2098                        ("shll2", VectorSize::Size64x2, VectorSize::Size32x4, ", #32")
2099                    }
2100                };
2101                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2102                let rn = pretty_print_vreg_vector(rn, size);
2103
2104                format!("{op} {rd}, {rn}{suffix}")
2105            }
2106            &Inst::VecRRNarrowLow {
2107                op,
2108                rd,
2109                rn,
2110                lane_size,
2111                ..
2112            }
2113            | &Inst::VecRRNarrowHigh {
2114                op,
2115                rd,
2116                rn,
2117                lane_size,
2118                ..
2119            } => {
2120                let vec64 = VectorSize::from_lane_size(lane_size, false);
2121                let vec128 = VectorSize::from_lane_size(lane_size, true);
2122                let rn_size = VectorSize::from_lane_size(lane_size.widen(), true);
2123                let high_half = match self {
2124                    &Inst::VecRRNarrowLow { .. } => false,
2125                    &Inst::VecRRNarrowHigh { .. } => true,
2126                    _ => unreachable!(),
2127                };
2128                let (op, rd_size) = match (op, high_half) {
2129                    (VecRRNarrowOp::Xtn, false) => ("xtn", vec64),
2130                    (VecRRNarrowOp::Xtn, true) => ("xtn2", vec128),
2131                    (VecRRNarrowOp::Sqxtn, false) => ("sqxtn", vec64),
2132                    (VecRRNarrowOp::Sqxtn, true) => ("sqxtn2", vec128),
2133                    (VecRRNarrowOp::Sqxtun, false) => ("sqxtun", vec64),
2134                    (VecRRNarrowOp::Sqxtun, true) => ("sqxtun2", vec128),
2135                    (VecRRNarrowOp::Uqxtn, false) => ("uqxtn", vec64),
2136                    (VecRRNarrowOp::Uqxtn, true) => ("uqxtn2", vec128),
2137                    (VecRRNarrowOp::Fcvtn, false) => ("fcvtn", vec64),
2138                    (VecRRNarrowOp::Fcvtn, true) => ("fcvtn2", vec128),
2139                };
2140                let rn = pretty_print_vreg_vector(rn, rn_size);
2141                let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2142                let ri = match self {
2143                    &Inst::VecRRNarrowLow { .. } => "".to_string(),
2144                    &Inst::VecRRNarrowHigh { ri, .. } => {
2145                        format!("{}, ", pretty_print_vreg_vector(ri, rd_size))
2146                    }
2147                    _ => unreachable!(),
2148                };
2149
2150                format!("{op} {rd}, {ri}{rn}")
2151            }
2152            &Inst::VecRRPair { op, rd, rn } => {
2153                let op = match op {
2154                    VecPairOp::Addp => "addp",
2155                };
2156                let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2157                let rn = pretty_print_vreg_vector(rn, VectorSize::Size64x2);
2158
2159                format!("{op} {rd}, {rn}")
2160            }
2161            &Inst::VecRRPairLong { op, rd, rn } => {
2162                let (op, dest, src) = match op {
2163                    VecRRPairLongOp::Saddlp8 => {
2164                        ("saddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2165                    }
2166                    VecRRPairLongOp::Saddlp16 => {
2167                        ("saddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2168                    }
2169                    VecRRPairLongOp::Uaddlp8 => {
2170                        ("uaddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2171                    }
2172                    VecRRPairLongOp::Uaddlp16 => {
2173                        ("uaddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2174                    }
2175                };
2176                let rd = pretty_print_vreg_vector(rd.to_reg(), dest);
2177                let rn = pretty_print_vreg_vector(rn, src);
2178
2179                format!("{op} {rd}, {rn}")
2180            }
2181            &Inst::VecRRR {
2182                rd,
2183                rn,
2184                rm,
2185                alu_op,
2186                size,
2187            } => {
2188                let (op, size) = match alu_op {
2189                    VecALUOp::Sqadd => ("sqadd", size),
2190                    VecALUOp::Uqadd => ("uqadd", size),
2191                    VecALUOp::Sqsub => ("sqsub", size),
2192                    VecALUOp::Uqsub => ("uqsub", size),
2193                    VecALUOp::Cmeq => ("cmeq", size),
2194                    VecALUOp::Cmge => ("cmge", size),
2195                    VecALUOp::Cmgt => ("cmgt", size),
2196                    VecALUOp::Cmhs => ("cmhs", size),
2197                    VecALUOp::Cmhi => ("cmhi", size),
2198                    VecALUOp::Fcmeq => ("fcmeq", size),
2199                    VecALUOp::Fcmgt => ("fcmgt", size),
2200                    VecALUOp::Fcmge => ("fcmge", size),
2201                    VecALUOp::And => ("and", VectorSize::Size8x16),
2202                    VecALUOp::Bic => ("bic", VectorSize::Size8x16),
2203                    VecALUOp::Orr => ("orr", VectorSize::Size8x16),
2204                    VecALUOp::Eor => ("eor", VectorSize::Size8x16),
2205                    VecALUOp::Umaxp => ("umaxp", size),
2206                    VecALUOp::Add => ("add", size),
2207                    VecALUOp::Sub => ("sub", size),
2208                    VecALUOp::Mul => ("mul", size),
2209                    VecALUOp::Sshl => ("sshl", size),
2210                    VecALUOp::Ushl => ("ushl", size),
2211                    VecALUOp::Umin => ("umin", size),
2212                    VecALUOp::Smin => ("smin", size),
2213                    VecALUOp::Umax => ("umax", size),
2214                    VecALUOp::Smax => ("smax", size),
2215                    VecALUOp::Urhadd => ("urhadd", size),
2216                    VecALUOp::Fadd => ("fadd", size),
2217                    VecALUOp::Fsub => ("fsub", size),
2218                    VecALUOp::Fdiv => ("fdiv", size),
2219                    VecALUOp::Fmax => ("fmax", size),
2220                    VecALUOp::Fmin => ("fmin", size),
2221                    VecALUOp::Fmul => ("fmul", size),
2222                    VecALUOp::Addp => ("addp", size),
2223                    VecALUOp::Zip1 => ("zip1", size),
2224                    VecALUOp::Zip2 => ("zip2", size),
2225                    VecALUOp::Sqrdmulh => ("sqrdmulh", size),
2226                    VecALUOp::Uzp1 => ("uzp1", size),
2227                    VecALUOp::Uzp2 => ("uzp2", size),
2228                    VecALUOp::Trn1 => ("trn1", size),
2229                    VecALUOp::Trn2 => ("trn2", size),
2230                };
2231                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2232                let rn = pretty_print_vreg_vector(rn, size);
2233                let rm = pretty_print_vreg_vector(rm, size);
2234                format!("{op} {rd}, {rn}, {rm}")
2235            }
2236            &Inst::VecRRRMod {
2237                rd,
2238                ri,
2239                rn,
2240                rm,
2241                alu_op,
2242                size,
2243            } => {
2244                let (op, size) = match alu_op {
2245                    VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
2246                    VecALUModOp::Fmla => ("fmla", size),
2247                    VecALUModOp::Fmls => ("fmls", size),
2248                };
2249                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2250                let ri = pretty_print_vreg_vector(ri, size);
2251                let rn = pretty_print_vreg_vector(rn, size);
2252                let rm = pretty_print_vreg_vector(rm, size);
2253                format!("{op} {rd}, {ri}, {rn}, {rm}")
2254            }
2255            &Inst::VecFmlaElem {
2256                rd,
2257                ri,
2258                rn,
2259                rm,
2260                alu_op,
2261                size,
2262                idx,
2263            } => {
2264                let (op, size) = match alu_op {
2265                    VecALUModOp::Fmla => ("fmla", size),
2266                    VecALUModOp::Fmls => ("fmls", size),
2267                    _ => unreachable!(),
2268                };
2269                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2270                let ri = pretty_print_vreg_vector(ri, size);
2271                let rn = pretty_print_vreg_vector(rn, size);
2272                let rm = pretty_print_vreg_element(rm, idx.into(), size.lane_size());
2273                format!("{op} {rd}, {ri}, {rn}, {rm}")
2274            }
2275            &Inst::VecRRRLong {
2276                rd,
2277                rn,
2278                rm,
2279                alu_op,
2280                high_half,
2281            } => {
2282                let (op, dest_size, src_size) = match (alu_op, high_half) {
2283                    (VecRRRLongOp::Smull8, false) => {
2284                        ("smull", VectorSize::Size16x8, VectorSize::Size8x8)
2285                    }
2286                    (VecRRRLongOp::Smull8, true) => {
2287                        ("smull2", VectorSize::Size16x8, VectorSize::Size8x16)
2288                    }
2289                    (VecRRRLongOp::Smull16, false) => {
2290                        ("smull", VectorSize::Size32x4, VectorSize::Size16x4)
2291                    }
2292                    (VecRRRLongOp::Smull16, true) => {
2293                        ("smull2", VectorSize::Size32x4, VectorSize::Size16x8)
2294                    }
2295                    (VecRRRLongOp::Smull32, false) => {
2296                        ("smull", VectorSize::Size64x2, VectorSize::Size32x2)
2297                    }
2298                    (VecRRRLongOp::Smull32, true) => {
2299                        ("smull2", VectorSize::Size64x2, VectorSize::Size32x4)
2300                    }
2301                    (VecRRRLongOp::Umull8, false) => {
2302                        ("umull", VectorSize::Size16x8, VectorSize::Size8x8)
2303                    }
2304                    (VecRRRLongOp::Umull8, true) => {
2305                        ("umull2", VectorSize::Size16x8, VectorSize::Size8x16)
2306                    }
2307                    (VecRRRLongOp::Umull16, false) => {
2308                        ("umull", VectorSize::Size32x4, VectorSize::Size16x4)
2309                    }
2310                    (VecRRRLongOp::Umull16, true) => {
2311                        ("umull2", VectorSize::Size32x4, VectorSize::Size16x8)
2312                    }
2313                    (VecRRRLongOp::Umull32, false) => {
2314                        ("umull", VectorSize::Size64x2, VectorSize::Size32x2)
2315                    }
2316                    (VecRRRLongOp::Umull32, true) => {
2317                        ("umull2", VectorSize::Size64x2, VectorSize::Size32x4)
2318                    }
2319                };
2320                let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2321                let rn = pretty_print_vreg_vector(rn, src_size);
2322                let rm = pretty_print_vreg_vector(rm, src_size);
2323                format!("{op} {rd}, {rn}, {rm}")
2324            }
2325            &Inst::VecRRRLongMod {
2326                rd,
2327                ri,
2328                rn,
2329                rm,
2330                alu_op,
2331                high_half,
2332            } => {
2333                let (op, dest_size, src_size) = match (alu_op, high_half) {
2334                    (VecRRRLongModOp::Umlal8, false) => {
2335                        ("umlal", VectorSize::Size16x8, VectorSize::Size8x8)
2336                    }
2337                    (VecRRRLongModOp::Umlal8, true) => {
2338                        ("umlal2", VectorSize::Size16x8, VectorSize::Size8x16)
2339                    }
2340                    (VecRRRLongModOp::Umlal16, false) => {
2341                        ("umlal", VectorSize::Size32x4, VectorSize::Size16x4)
2342                    }
2343                    (VecRRRLongModOp::Umlal16, true) => {
2344                        ("umlal2", VectorSize::Size32x4, VectorSize::Size16x8)
2345                    }
2346                    (VecRRRLongModOp::Umlal32, false) => {
2347                        ("umlal", VectorSize::Size64x2, VectorSize::Size32x2)
2348                    }
2349                    (VecRRRLongModOp::Umlal32, true) => {
2350                        ("umlal2", VectorSize::Size64x2, VectorSize::Size32x4)
2351                    }
2352                };
2353                let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2354                let ri = pretty_print_vreg_vector(ri, dest_size);
2355                let rn = pretty_print_vreg_vector(rn, src_size);
2356                let rm = pretty_print_vreg_vector(rm, src_size);
2357                format!("{op} {rd}, {ri}, {rn}, {rm}")
2358            }
2359            &Inst::VecMisc { op, rd, rn, size } => {
2360                let (op, size, suffix) = match op {
2361                    VecMisc2::Not => (
2362                        "mvn",
2363                        if size.is_128bits() {
2364                            VectorSize::Size8x16
2365                        } else {
2366                            VectorSize::Size8x8
2367                        },
2368                        "",
2369                    ),
2370                    VecMisc2::Neg => ("neg", size, ""),
2371                    VecMisc2::Abs => ("abs", size, ""),
2372                    VecMisc2::Fabs => ("fabs", size, ""),
2373                    VecMisc2::Fneg => ("fneg", size, ""),
2374                    VecMisc2::Fsqrt => ("fsqrt", size, ""),
2375                    VecMisc2::Rev16 => ("rev16", size, ""),
2376                    VecMisc2::Rev32 => ("rev32", size, ""),
2377                    VecMisc2::Rev64 => ("rev64", size, ""),
2378                    VecMisc2::Fcvtzs => ("fcvtzs", size, ""),
2379                    VecMisc2::Fcvtzu => ("fcvtzu", size, ""),
2380                    VecMisc2::Scvtf => ("scvtf", size, ""),
2381                    VecMisc2::Ucvtf => ("ucvtf", size, ""),
2382                    VecMisc2::Frintn => ("frintn", size, ""),
2383                    VecMisc2::Frintz => ("frintz", size, ""),
2384                    VecMisc2::Frintm => ("frintm", size, ""),
2385                    VecMisc2::Frintp => ("frintp", size, ""),
2386                    VecMisc2::Cnt => ("cnt", size, ""),
2387                    VecMisc2::Cmeq0 => ("cmeq", size, ", #0"),
2388                    VecMisc2::Cmge0 => ("cmge", size, ", #0"),
2389                    VecMisc2::Cmgt0 => ("cmgt", size, ", #0"),
2390                    VecMisc2::Cmle0 => ("cmle", size, ", #0"),
2391                    VecMisc2::Cmlt0 => ("cmlt", size, ", #0"),
2392                    VecMisc2::Fcmeq0 => ("fcmeq", size, ", #0.0"),
2393                    VecMisc2::Fcmge0 => ("fcmge", size, ", #0.0"),
2394                    VecMisc2::Fcmgt0 => ("fcmgt", size, ", #0.0"),
2395                    VecMisc2::Fcmle0 => ("fcmle", size, ", #0.0"),
2396                    VecMisc2::Fcmlt0 => ("fcmlt", size, ", #0.0"),
2397                };
2398                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2399                let rn = pretty_print_vreg_vector(rn, size);
2400                format!("{op} {rd}, {rn}{suffix}")
2401            }
2402            &Inst::VecLanes { op, rd, rn, size } => {
2403                let op = match op {
2404                    VecLanesOp::Uminv => "uminv",
2405                    VecLanesOp::Addv => "addv",
2406                };
2407                let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
2408                let rn = pretty_print_vreg_vector(rn, size);
2409                format!("{op} {rd}, {rn}")
2410            }
2411            &Inst::VecShiftImm {
2412                op,
2413                rd,
2414                rn,
2415                size,
2416                imm,
2417            } => {
2418                let op = match op {
2419                    VecShiftImmOp::Shl => "shl",
2420                    VecShiftImmOp::Ushr => "ushr",
2421                    VecShiftImmOp::Sshr => "sshr",
2422                };
2423                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2424                let rn = pretty_print_vreg_vector(rn, size);
2425                format!("{op} {rd}, {rn}, #{imm}")
2426            }
2427            &Inst::VecShiftImmMod {
2428                op,
2429                rd,
2430                ri,
2431                rn,
2432                size,
2433                imm,
2434            } => {
2435                let op = match op {
2436                    VecShiftImmModOp::Sli => "sli",
2437                };
2438                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2439                let ri = pretty_print_vreg_vector(ri, size);
2440                let rn = pretty_print_vreg_vector(rn, size);
2441                format!("{op} {rd}, {ri}, {rn}, #{imm}")
2442            }
2443            &Inst::VecExtract { rd, rn, rm, imm4 } => {
2444                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2445                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2446                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2447                format!("ext {rd}, {rn}, {rm}, #{imm4}")
2448            }
2449            &Inst::VecTbl { rd, rn, rm } => {
2450                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2451                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2452                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2453                format!("tbl {rd}, {{ {rn} }}, {rm}")
2454            }
2455            &Inst::VecTblExt { rd, ri, rn, rm } => {
2456                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2457                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2458                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2459                let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2460                format!("tbx {rd}, {ri}, {{ {rn} }}, {rm}")
2461            }
2462            &Inst::VecTbl2 { rd, rn, rn2, rm } => {
2463                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2464                let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2465                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2466                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2467                format!("tbl {rd}, {{ {rn}, {rn2} }}, {rm}")
2468            }
2469            &Inst::VecTbl2Ext {
2470                rd,
2471                ri,
2472                rn,
2473                rn2,
2474                rm,
2475            } => {
2476                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2477                let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2478                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2479                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2480                let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2481                format!("tbx {rd}, {ri}, {{ {rn}, {rn2} }}, {rm}")
2482            }
2483            &Inst::VecLoadReplicate { rd, rn, size, .. } => {
2484                let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2485                let rn = pretty_print_reg(rn);
2486
2487                format!("ld1r {{ {rd} }}, [{rn}]")
2488            }
2489            &Inst::VecCSel { rd, rn, rm, cond } => {
2490                let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2491                let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2492                let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2493                let cond = cond.pretty_print(0);
2494                format!("vcsel {rd}, {rn}, {rm}, {cond} (if-then-else diamond)")
2495            }
2496            &Inst::MovToNZCV { rn } => {
2497                let rn = pretty_print_reg(rn);
2498                format!("msr nzcv, {rn}")
2499            }
2500            &Inst::MovFromNZCV { rd } => {
2501                let rd = pretty_print_reg(rd.to_reg());
2502                format!("mrs {rd}, nzcv")
2503            }
2504            &Inst::Extend {
2505                rd,
2506                rn,
2507                signed: false,
2508                from_bits: 1,
2509                ..
2510            } => {
2511                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size32);
2512                let rn = pretty_print_ireg(rn, OperandSize::Size32);
2513                format!("and {rd}, {rn}, #1")
2514            }
2515            &Inst::Extend {
2516                rd,
2517                rn,
2518                signed: false,
2519                from_bits: 32,
2520                to_bits: 64,
2521            } => {
2522                // The case of a zero extension from 32 to 64 bits, is implemented
2523                // with a "mov" to a 32-bit (W-reg) dest, because this zeroes
2524                // the top 32 bits.
2525                let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size32);
2526                let rn = pretty_print_ireg(rn, OperandSize::Size32);
2527                format!("mov {rd}, {rn}")
2528            }
2529            &Inst::Extend {
2530                rd,
2531                rn,
2532                signed,
2533                from_bits,
2534                to_bits,
2535            } => {
2536                assert!(from_bits <= to_bits);
2537                let op = match (signed, from_bits) {
2538                    (false, 8) => "uxtb",
2539                    (true, 8) => "sxtb",
2540                    (false, 16) => "uxth",
2541                    (true, 16) => "sxth",
2542                    (true, 32) => "sxtw",
2543                    (true, _) => "sbfx",
2544                    (false, _) => "ubfx",
2545                };
2546                if op == "sbfx" || op == "ubfx" {
2547                    let dest_size = OperandSize::from_bits(to_bits);
2548                    let rd = pretty_print_ireg(rd.to_reg(), dest_size);
2549                    let rn = pretty_print_ireg(rn, dest_size);
2550                    format!("{op} {rd}, {rn}, #0, #{from_bits}")
2551                } else {
2552                    let dest_size = if signed {
2553                        OperandSize::from_bits(to_bits)
2554                    } else {
2555                        OperandSize::Size32
2556                    };
2557                    let rd = pretty_print_ireg(rd.to_reg(), dest_size);
2558                    let rn = pretty_print_ireg(rn, OperandSize::from_bits(from_bits));
2559                    format!("{op} {rd}, {rn}")
2560                }
2561            }
2562            &Inst::Call { ref info } => {
2563                let try_call = info
2564                    .try_call_info
2565                    .as_ref()
2566                    .map(|tci| pretty_print_try_call(tci))
2567                    .unwrap_or_default();
2568                format!("bl 0{try_call}")
2569            }
2570            &Inst::CallInd { ref info } => {
2571                let rn = pretty_print_reg(info.dest);
2572                let try_call = info
2573                    .try_call_info
2574                    .as_ref()
2575                    .map(|tci| pretty_print_try_call(tci))
2576                    .unwrap_or_default();
2577                format!("blr {rn}{try_call}")
2578            }
2579            &Inst::ReturnCall { ref info } => {
2580                let mut s = format!(
2581                    "return_call {:?} new_stack_arg_size:{}",
2582                    info.dest, info.new_stack_arg_size
2583                );
2584                for ret in &info.uses {
2585                    let preg = pretty_print_reg(ret.preg);
2586                    let vreg = pretty_print_reg(ret.vreg);
2587                    write!(&mut s, " {vreg}={preg}").unwrap();
2588                }
2589                s
2590            }
2591            &Inst::ReturnCallInd { ref info } => {
2592                let callee = pretty_print_reg(info.dest);
2593                let mut s = format!(
2594                    "return_call_ind {callee} new_stack_arg_size:{}",
2595                    info.new_stack_arg_size
2596                );
2597                for ret in &info.uses {
2598                    let preg = pretty_print_reg(ret.preg);
2599                    let vreg = pretty_print_reg(ret.vreg);
2600                    write!(&mut s, " {vreg}={preg}").unwrap();
2601                }
2602                s
2603            }
2604            &Inst::Args { ref args } => {
2605                let mut s = "args".to_string();
2606                for arg in args {
2607                    let preg = pretty_print_reg(arg.preg);
2608                    let def = pretty_print_reg(arg.vreg.to_reg());
2609                    write!(&mut s, " {def}={preg}").unwrap();
2610                }
2611                s
2612            }
2613            &Inst::Rets { ref rets } => {
2614                let mut s = "rets".to_string();
2615                for ret in rets {
2616                    let preg = pretty_print_reg(ret.preg);
2617                    let vreg = pretty_print_reg(ret.vreg);
2618                    write!(&mut s, " {vreg}={preg}").unwrap();
2619                }
2620                s
2621            }
2622            &Inst::Ret {} => "ret".to_string(),
2623            &Inst::AuthenticatedRet { key, is_hint } => {
2624                let key = match key {
2625                    APIKey::AZ => "az",
2626                    APIKey::BZ => "bz",
2627                    APIKey::ASP => "asp",
2628                    APIKey::BSP => "bsp",
2629                };
2630                match is_hint {
2631                    false => format!("reta{key}"),
2632                    true => format!("auti{key} ; ret"),
2633                }
2634            }
2635            &Inst::Jump { ref dest } => {
2636                let dest = dest.pretty_print(0);
2637                format!("b {dest}")
2638            }
2639            &Inst::CondBr {
2640                ref taken,
2641                ref not_taken,
2642                ref kind,
2643            } => {
2644                let taken = taken.pretty_print(0);
2645                let not_taken = not_taken.pretty_print(0);
2646                match kind {
2647                    &CondBrKind::Zero(reg, size) => {
2648                        let reg = pretty_print_reg_sized(reg, size);
2649                        format!("cbz {reg}, {taken} ; b {not_taken}")
2650                    }
2651                    &CondBrKind::NotZero(reg, size) => {
2652                        let reg = pretty_print_reg_sized(reg, size);
2653                        format!("cbnz {reg}, {taken} ; b {not_taken}")
2654                    }
2655                    &CondBrKind::Cond(c) => {
2656                        let c = c.pretty_print(0);
2657                        format!("b.{c} {taken} ; b {not_taken}")
2658                    }
2659                }
2660            }
2661            &Inst::TestBitAndBranch {
2662                kind,
2663                ref taken,
2664                ref not_taken,
2665                rn,
2666                bit,
2667            } => {
2668                let cond = match kind {
2669                    TestBitAndBranchKind::Z => "z",
2670                    TestBitAndBranchKind::NZ => "nz",
2671                };
2672                let taken = taken.pretty_print(0);
2673                let not_taken = not_taken.pretty_print(0);
2674                let rn = pretty_print_reg(rn);
2675                format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
2676            }
2677            &Inst::IndirectBr { rn, .. } => {
2678                let rn = pretty_print_reg(rn);
2679                format!("br {rn}")
2680            }
2681            &Inst::Brk => "brk #0xf000".to_string(),
2682            &Inst::Udf { .. } => "udf #0xc11f".to_string(),
2683            &Inst::TrapIf {
2684                ref kind,
2685                trap_code,
2686            } => match kind {
2687                &CondBrKind::Zero(reg, size) => {
2688                    let reg = pretty_print_reg_sized(reg, size);
2689                    format!("cbz {reg}, #trap={trap_code}")
2690                }
2691                &CondBrKind::NotZero(reg, size) => {
2692                    let reg = pretty_print_reg_sized(reg, size);
2693                    format!("cbnz {reg}, #trap={trap_code}")
2694                }
2695                &CondBrKind::Cond(c) => {
2696                    let c = c.pretty_print(0);
2697                    format!("b.{c} #trap={trap_code}")
2698                }
2699            },
2700            &Inst::Adr { rd, off } => {
2701                let rd = pretty_print_reg(rd.to_reg());
2702                format!("adr {rd}, pc+{off}")
2703            }
2704            &Inst::Adrp { rd, off } => {
2705                let rd = pretty_print_reg(rd.to_reg());
2706                // This instruction addresses 4KiB pages, so multiply it by the page size.
2707                let byte_offset = off * 4096;
2708                format!("adrp {rd}, pc+{byte_offset}")
2709            }
2710            &Inst::Word4 { data } => format!("data.i32 {data}"),
2711            &Inst::Word8 { data } => format!("data.i64 {data}"),
2712            &Inst::JTSequence {
2713                default,
2714                ref targets,
2715                ridx,
2716                rtmp1,
2717                rtmp2,
2718                ..
2719            } => {
2720                let ridx = pretty_print_reg(ridx);
2721                let rtmp1 = pretty_print_reg(rtmp1.to_reg());
2722                let rtmp2 = pretty_print_reg(rtmp2.to_reg());
2723                let default_target = BranchTarget::Label(default).pretty_print(0);
2724                format!(
2725                    concat!(
2726                        "b.hs {} ; ",
2727                        "csel {}, xzr, {}, hs ; ",
2728                        "csdb ; ",
2729                        "adr {}, pc+16 ; ",
2730                        "ldrsw {}, [{}, {}, uxtw #2] ; ",
2731                        "add {}, {}, {} ; ",
2732                        "br {} ; ",
2733                        "jt_entries {:?}"
2734                    ),
2735                    default_target,
2736                    rtmp2,
2737                    ridx,
2738                    rtmp1,
2739                    rtmp2,
2740                    rtmp1,
2741                    rtmp2,
2742                    rtmp1,
2743                    rtmp1,
2744                    rtmp2,
2745                    rtmp1,
2746                    targets
2747                )
2748            }
2749            &Inst::LoadExtName {
2750                rd,
2751                ref name,
2752                offset,
2753            } => {
2754                let rd = pretty_print_reg(rd.to_reg());
2755                format!("load_ext_name {rd}, {name:?}+{offset}")
2756            }
2757            &Inst::LoadAddr { rd, ref mem } => {
2758                // TODO: we really should find a better way to avoid duplication of
2759                // this logic between `emit()` and `show_rru()` -- a separate 1-to-N
2760                // expansion stage (i.e., legalization, but without the slow edit-in-place
2761                // of the existing legalization framework).
2762                let mem = mem.clone();
2763                let (mem_insts, mem) = mem_finalize(None, &mem, I8, state);
2764                let mut ret = String::new();
2765                for inst in mem_insts.into_iter() {
2766                    ret.push_str(&inst.print_with_state(&mut EmitState::default()));
2767                }
2768                let (reg, index_reg, offset) = match mem {
2769                    AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0),
2770                    AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()),
2771                    AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32),
2772                    _ => panic!("Unsupported case for LoadAddr: {mem:?}"),
2773                };
2774                let abs_offset = if offset < 0 {
2775                    -offset as u64
2776                } else {
2777                    offset as u64
2778                };
2779                let alu_op = if offset < 0 { ALUOp::Sub } else { ALUOp::Add };
2780
2781                if let Some((idx, extendop)) = index_reg {
2782                    let add = Inst::AluRRRExtend {
2783                        alu_op: ALUOp::Add,
2784                        size: OperandSize::Size64,
2785                        rd,
2786                        rn: reg,
2787                        rm: idx,
2788                        extendop,
2789                    };
2790
2791                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
2792                } else if offset == 0 {
2793                    let mov = Inst::gen_move(rd, reg, I64);
2794                    ret.push_str(&mov.print_with_state(&mut EmitState::default()));
2795                } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
2796                    let add = Inst::AluRRImm12 {
2797                        alu_op,
2798                        size: OperandSize::Size64,
2799                        rd,
2800                        rn: reg,
2801                        imm12,
2802                    };
2803                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
2804                } else {
2805                    let tmp = writable_spilltmp_reg();
2806                    for inst in Inst::load_constant(tmp, abs_offset).into_iter() {
2807                        ret.push_str(&inst.print_with_state(&mut EmitState::default()));
2808                    }
2809                    let add = Inst::AluRRR {
2810                        alu_op,
2811                        size: OperandSize::Size64,
2812                        rd,
2813                        rn: reg,
2814                        rm: tmp.to_reg(),
2815                    };
2816                    ret.push_str(&add.print_with_state(&mut EmitState::default()));
2817                }
2818                ret
2819            }
2820            &Inst::Paci { key } => {
2821                let key = match key {
2822                    APIKey::AZ => "az",
2823                    APIKey::BZ => "bz",
2824                    APIKey::ASP => "asp",
2825                    APIKey::BSP => "bsp",
2826                };
2827
2828                "paci".to_string() + key
2829            }
2830            &Inst::Xpaclri => "xpaclri".to_string(),
2831            &Inst::Bti { targets } => {
2832                let targets = match targets {
2833                    BranchTargetType::None => "",
2834                    BranchTargetType::C => " c",
2835                    BranchTargetType::J => " j",
2836                    BranchTargetType::JC => " jc",
2837                };
2838
2839                "bti".to_string() + targets
2840            }
2841            &Inst::EmitIsland { needed_space } => format!("emit_island {needed_space}"),
2842
2843            &Inst::ElfTlsGetAddr {
2844                ref symbol,
2845                rd,
2846                tmp,
2847            } => {
2848                let rd = pretty_print_reg(rd.to_reg());
2849                let tmp = pretty_print_reg(tmp.to_reg());
2850                format!("elf_tls_get_addr {}, {}, {}", rd, tmp, symbol.display(None))
2851            }
2852            &Inst::MachOTlsGetAddr { ref symbol, rd } => {
2853                let rd = pretty_print_reg(rd.to_reg());
2854                format!("macho_tls_get_addr {}, {}", rd, symbol.display(None))
2855            }
2856            &Inst::Unwind { ref inst } => {
2857                format!("unwind {inst:?}")
2858            }
2859            &Inst::DummyUse { reg } => {
2860                let reg = pretty_print_reg(reg);
2861                format!("dummy_use {reg}")
2862            }
2863            &Inst::StackProbeLoop { start, end, step } => {
2864                let start = pretty_print_reg(start.to_reg());
2865                let end = pretty_print_reg(end);
2866                let step = step.pretty_print(0);
2867                format!("stack_probe_loop {start}, {end}, {step}")
2868            }
2869        }
2870    }
2871}
2872
2873//=============================================================================
2874// Label fixups and jump veneers.
2875
2876/// Different forms of label references for different instruction formats.
2877#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2878pub enum LabelUse {
2879    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
2880    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
2881    Branch14,
2882    /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
2883    /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
2884    Branch19,
2885    /// 26-bit branch offset (unconditional branches). PC-rel, offset is imm << 2. Immediate is 26
2886    /// signed bits, in bits 25:0. Used by b, bl.
2887    Branch26,
2888    /// 19-bit offset for LDR (load literal). PC-rel, offset is imm << 2. Immediate is 19 signed bits,
2889    /// in bits 23:5.
2890    Ldr19,
2891    /// 21-bit offset for ADR (get address of label). PC-rel, offset is not shifted. Immediate is
2892    /// 21 signed bits, with high 19 bits in bits 23:5 and low 2 bits in bits 30:29.
2893    Adr21,
2894    /// 32-bit PC relative constant offset (from address of constant itself),
2895    /// signed. Used in jump tables.
2896    PCRel32,
2897}
2898
2899impl MachInstLabelUse for LabelUse {
2900    /// Alignment for veneer code. Every AArch64 instruction must be 4-byte-aligned.
2901    const ALIGN: CodeOffset = 4;
2902
2903    /// Maximum PC-relative range (positive), inclusive.
2904    fn max_pos_range(self) -> CodeOffset {
2905        match self {
2906            // N-bit immediate, left-shifted by 2, for (N+2) bits of total
2907            // range. Signed, so +2^(N+1) from zero. Likewise for two other
2908            // shifted cases below.
2909            LabelUse::Branch14 => (1 << 15) - 1,
2910            LabelUse::Branch19 => (1 << 20) - 1,
2911            LabelUse::Branch26 => (1 << 27) - 1,
2912            LabelUse::Ldr19 => (1 << 20) - 1,
2913            // Adr does not shift its immediate, so the 21-bit immediate gives 21 bits of total
2914            // range.
2915            LabelUse::Adr21 => (1 << 20) - 1,
2916            LabelUse::PCRel32 => 0x7fffffff,
2917        }
2918    }
2919
2920    /// Maximum PC-relative range (negative).
2921    fn max_neg_range(self) -> CodeOffset {
2922        // All forms are twos-complement signed offsets, so negative limit is one more than
2923        // positive limit.
2924        self.max_pos_range() + 1
2925    }
2926
2927    /// Size of window into code needed to do the patch.
2928    fn patch_size(self) -> CodeOffset {
2929        // Patch is on one instruction only for all of these label reference types.
2930        4
2931    }
2932
2933    /// Perform the patch.
2934    fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2935        let pc_rel = (label_offset as i64) - (use_offset as i64);
2936        debug_assert!(pc_rel <= self.max_pos_range() as i64);
2937        debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2938        let pc_rel = pc_rel as u32;
2939        let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2940        let mask = match self {
2941            LabelUse::Branch14 => 0x0007ffe0, // bits 18..5 inclusive
2942            LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
2943            LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
2944            LabelUse::Ldr19 => 0x00ffffe0,    // bits 23..5 inclusive
2945            LabelUse::Adr21 => 0x60ffffe0,    // bits 30..29, 25..5 inclusive
2946            LabelUse::PCRel32 => 0xffffffff,
2947        };
2948        let pc_rel_shifted = match self {
2949            LabelUse::Adr21 | LabelUse::PCRel32 => pc_rel,
2950            _ => {
2951                debug_assert!(pc_rel & 3 == 0);
2952                pc_rel >> 2
2953            }
2954        };
2955        let pc_rel_inserted = match self {
2956            LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
2957            LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
2958            LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
2959            LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
2960            LabelUse::PCRel32 => pc_rel_shifted,
2961        };
2962        let is_add = match self {
2963            LabelUse::PCRel32 => true,
2964            _ => false,
2965        };
2966        let insn_word = if is_add {
2967            insn_word.wrapping_add(pc_rel_inserted)
2968        } else {
2969            (insn_word & !mask) | pc_rel_inserted
2970        };
2971        buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
2972    }
2973
2974    /// Is a veneer supported for this label reference type?
2975    fn supports_veneer(self) -> bool {
2976        match self {
2977            LabelUse::Branch14 | LabelUse::Branch19 => true, // veneer is a Branch26
2978            LabelUse::Branch26 => true,                      // veneer is a PCRel32
2979            _ => false,
2980        }
2981    }
2982
2983    /// How large is the veneer, if supported?
2984    fn veneer_size(self) -> CodeOffset {
2985        match self {
2986            LabelUse::Branch14 | LabelUse::Branch19 => 4,
2987            LabelUse::Branch26 => 20,
2988            _ => unreachable!(),
2989        }
2990    }
2991
2992    fn worst_case_veneer_size() -> CodeOffset {
2993        20
2994    }
2995
2996    /// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return
2997    /// an offset and label-use for the veneer's use of the original label.
2998    fn generate_veneer(
2999        self,
3000        buffer: &mut [u8],
3001        veneer_offset: CodeOffset,
3002    ) -> (CodeOffset, LabelUse) {
3003        match self {
3004            LabelUse::Branch14 | LabelUse::Branch19 => {
3005                // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
3006                // bother with constructing an Inst.
3007                let insn_word = 0b000101 << 26;
3008                buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3009                (veneer_offset, LabelUse::Branch26)
3010            }
3011
3012            // This is promoting a 26-bit call/jump to a 32-bit call/jump to
3013            // get a further range. This jump translates to a jump to a
3014            // relative location based on the address of the constant loaded
3015            // from here.
3016            //
3017            // If this path is taken from a call instruction then caller-saved
3018            // registers are available (minus arguments), so x16/x17 are
3019            // available. Otherwise for intra-function jumps we also reserve
3020            // x16/x17 as spill-style registers. In both cases these are
3021            // available for us to use.
3022            LabelUse::Branch26 => {
3023                let tmp1 = regs::spilltmp_reg();
3024                let tmp1_w = regs::writable_spilltmp_reg();
3025                let tmp2 = regs::tmp2_reg();
3026                let tmp2_w = regs::writable_tmp2_reg();
3027                // ldrsw x16, 16
3028                let ldr = emit::enc_ldst_imm19(0b1001_1000, 16 / 4, tmp1);
3029                // adr x17, 12
3030                let adr = emit::enc_adr(12, tmp2_w);
3031                // add x16, x16, x17
3032                let add = emit::enc_arith_rrr(0b10001011_000, 0, tmp1_w, tmp1, tmp2);
3033                // br x16
3034                let br = emit::enc_br(tmp1);
3035                buffer[0..4].clone_from_slice(&u32::to_le_bytes(ldr));
3036                buffer[4..8].clone_from_slice(&u32::to_le_bytes(adr));
3037                buffer[8..12].clone_from_slice(&u32::to_le_bytes(add));
3038                buffer[12..16].clone_from_slice(&u32::to_le_bytes(br));
3039                // the 4-byte signed immediate we'll load is after these
3040                // instructions, 16-bytes in.
3041                (veneer_offset + 16, LabelUse::PCRel32)
3042            }
3043
3044            _ => panic!("Unsupported label-reference type for veneer generation!"),
3045        }
3046    }
3047
3048    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {
3049        match (reloc, addend) {
3050            (Reloc::Arm64Call, 0) => Some(LabelUse::Branch26),
3051            _ => None,
3052        }
3053    }
3054}
3055
3056#[cfg(test)]
3057mod tests {
3058    use super::*;
3059
3060    #[test]
3061    fn inst_size_test() {
3062        // This test will help with unintentionally growing the size
3063        // of the Inst enum.
3064        let expected = if cfg!(target_pointer_width = "32") && !cfg!(target_arch = "arm") {
3065            28
3066        } else {
3067            32
3068        };
3069        assert_eq!(expected, std::mem::size_of::<Inst>());
3070    }
3071}