Skip to main content

cranelift_codegen/isa/aarch64/inst/
emit.rs

1//! AArch64 ISA: binary code emission.
2
3use cranelift_control::ControlPlane;
4
5use crate::ir::types::*;
6use crate::isa::aarch64;
7use crate::isa::aarch64::inst::*;
8use crate::trace;
9
10/// Memory addressing mode finalization: convert "special" modes (e.g.,
11/// generic arbitrary stack offset) into real addressing modes, possibly by
12/// emitting some helper instructions that come immediately before the use
13/// of this amode.
14pub fn mem_finalize(
15    sink: Option<&mut MachBuffer<Inst>>,
16    mem: &AMode,
17    access_ty: Type,
18    state: &EmitState,
19) -> (SmallVec<[Inst; 4]>, AMode) {
20    match mem {
21        &AMode::RegOffset { off, .. }
22        | &AMode::SPOffset { off }
23        | &AMode::FPOffset { off }
24        | &AMode::IncomingArg { off }
25        | &AMode::SlotOffset { off } => {
26            let basereg = match mem {
27                &AMode::RegOffset { rn, .. } => rn,
28                &AMode::SPOffset { .. }
29                | &AMode::SlotOffset { .. }
30                | &AMode::IncomingArg { .. } => stack_reg(),
31                &AMode::FPOffset { .. } => fp_reg(),
32                _ => unreachable!(),
33            };
34            let off = match mem {
35                &AMode::IncomingArg { .. } => {
36                    let frame_layout = state.frame_layout();
37                    i64::from(
38                        frame_layout.setup_area_size
39                            + frame_layout.tail_args_size
40                            + frame_layout.clobber_size
41                            + frame_layout.fixed_frame_storage_size
42                            + frame_layout.outgoing_args_size,
43                    ) - off
44                }
45                &AMode::SlotOffset { .. } => {
46                    let adj = i64::from(state.frame_layout().outgoing_args_size);
47                    trace!(
48                        "mem_finalize: slot offset {} + adj {} -> {}",
49                        off,
50                        adj,
51                        off + adj
52                    );
53                    off + adj
54                }
55                _ => off,
56            };
57
58            if let Some(simm9) = SImm9::maybe_from_i64(off) {
59                let mem = AMode::Unscaled { rn: basereg, simm9 };
60                (smallvec![], mem)
61            } else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, access_ty) {
62                let mem = AMode::UnsignedOffset {
63                    rn: basereg,
64                    uimm12,
65                };
66                (smallvec![], mem)
67            } else {
68                let tmp = if basereg == spilltmp_reg() {
69                    writable_tmp2_reg()
70                } else {
71                    writable_spilltmp_reg()
72                };
73                (
74                    Inst::load_constant(tmp, off as u64),
75                    AMode::RegExtended {
76                        rn: basereg,
77                        rm: tmp.to_reg(),
78                        extendop: ExtendOp::SXTX,
79                    },
80                )
81            }
82        }
83
84        AMode::Const { addr } => {
85            let sink = match sink {
86                Some(sink) => sink,
87                None => return (smallvec![], mem.clone()),
88            };
89            let label = sink.get_label_for_constant(*addr);
90            let label = MemLabel::Mach(label);
91            (smallvec![], AMode::Label { label })
92        }
93
94        _ => (smallvec![], mem.clone()),
95    }
96}
97
98//=============================================================================
99// Instructions and subcomponents: emission
100
101pub(crate) fn machreg_to_gpr(m: Reg) -> u32 {
102    assert_eq!(m.class(), RegClass::Int);
103    u32::from(m.to_real_reg().unwrap().hw_enc() & 31)
104}
105
106pub(crate) fn machreg_to_vec(m: Reg) -> u32 {
107    assert_eq!(m.class(), RegClass::Float);
108    u32::from(m.to_real_reg().unwrap().hw_enc())
109}
110
111fn machreg_to_gpr_or_vec(m: Reg) -> u32 {
112    u32::from(m.to_real_reg().unwrap().hw_enc() & 31)
113}
114
115/// Encode a 3-register aeithmeric instruction.
116pub fn enc_arith_rrr(bits_31_21: u32, bits_15_10: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
117    (bits_31_21 << 21)
118        | (bits_15_10 << 10)
119        | machreg_to_gpr(rd.to_reg())
120        | (machreg_to_gpr(rn) << 5)
121        | (machreg_to_gpr(rm) << 16)
122}
123
124fn enc_arith_rr_imm12(
125    bits_31_24: u32,
126    immshift: u32,
127    imm12: u32,
128    rn: Reg,
129    rd: Writable<Reg>,
130) -> u32 {
131    (bits_31_24 << 24)
132        | (immshift << 22)
133        | (imm12 << 10)
134        | (machreg_to_gpr(rn) << 5)
135        | machreg_to_gpr(rd.to_reg())
136}
137
138fn enc_arith_rr_imml(bits_31_23: u32, imm_bits: u32, rn: Reg, rd: Writable<Reg>) -> u32 {
139    (bits_31_23 << 23) | (imm_bits << 10) | (machreg_to_gpr(rn) << 5) | machreg_to_gpr(rd.to_reg())
140}
141
142fn enc_arith_rrrr(top11: u32, rm: Reg, bit15: u32, ra: Reg, rn: Reg, rd: Writable<Reg>) -> u32 {
143    (top11 << 21)
144        | (machreg_to_gpr(rm) << 16)
145        | (bit15 << 15)
146        | (machreg_to_gpr(ra) << 10)
147        | (machreg_to_gpr(rn) << 5)
148        | machreg_to_gpr(rd.to_reg())
149}
150
151fn enc_jump26(op_31_26: u32, off_26_0: u32) -> u32 {
152    assert!(off_26_0 < (1 << 26));
153    (op_31_26 << 26) | off_26_0
154}
155
156fn enc_cmpbr(op_31_24: u32, off_18_0: u32, reg: Reg) -> u32 {
157    assert!(off_18_0 < (1 << 19));
158    (op_31_24 << 24) | (off_18_0 << 5) | machreg_to_gpr(reg)
159}
160
161fn enc_cbr(op_31_24: u32, off_18_0: u32, op_4: u32, cond: u32) -> u32 {
162    assert!(off_18_0 < (1 << 19));
163    assert!(cond < (1 << 4));
164    (op_31_24 << 24) | (off_18_0 << 5) | (op_4 << 4) | cond
165}
166
167/// Set the size bit of an instruction.
168fn enc_op_size(op: u32, size: OperandSize) -> u32 {
169    (op & !(1 << 31)) | (size.sf_bit() << 31)
170}
171
172fn enc_conditional_br(taken: BranchTarget, kind: CondBrKind) -> u32 {
173    match kind {
174        CondBrKind::Zero(reg, size) => enc_op_size(
175            enc_cmpbr(0b0_011010_0, taken.as_offset19_or_zero(), reg),
176            size,
177        ),
178        CondBrKind::NotZero(reg, size) => enc_op_size(
179            enc_cmpbr(0b0_011010_1, taken.as_offset19_or_zero(), reg),
180            size,
181        ),
182        CondBrKind::Cond(c) => enc_cbr(0b01010100, taken.as_offset19_or_zero(), 0b0, c.bits()),
183    }
184}
185
186fn enc_test_bit_and_branch(
187    kind: TestBitAndBranchKind,
188    taken: BranchTarget,
189    reg: Reg,
190    bit: u8,
191) -> u32 {
192    assert!(bit < 64);
193    let op_31 = u32::from(bit >> 5);
194    let op_23_19 = u32::from(bit & 0b11111);
195    let op_30_24 = 0b0110110
196        | match kind {
197            TestBitAndBranchKind::Z => 0,
198            TestBitAndBranchKind::NZ => 1,
199        };
200    (op_31 << 31)
201        | (op_30_24 << 24)
202        | (op_23_19 << 19)
203        | (taken.as_offset14_or_zero() << 5)
204        | machreg_to_gpr(reg)
205}
206
207/// Encode a move-wide instruction.
208pub fn enc_move_wide(
209    op: MoveWideOp,
210    rd: Writable<Reg>,
211    imm: MoveWideConst,
212    size: OperandSize,
213) -> u32 {
214    assert!(imm.shift <= 0b11);
215    let op = match op {
216        MoveWideOp::MovN => 0b00,
217        MoveWideOp::MovZ => 0b10,
218    };
219    0x12800000
220        | size.sf_bit() << 31
221        | op << 29
222        | u32::from(imm.shift) << 21
223        | u32::from(imm.bits) << 5
224        | machreg_to_gpr(rd.to_reg())
225}
226
227/// Encode a move-keep immediate instruction.
228pub fn enc_movk(rd: Writable<Reg>, imm: MoveWideConst, size: OperandSize) -> u32 {
229    assert!(imm.shift <= 0b11);
230    0x72800000
231        | size.sf_bit() << 31
232        | u32::from(imm.shift) << 21
233        | u32::from(imm.bits) << 5
234        | machreg_to_gpr(rd.to_reg())
235}
236
237fn enc_ldst_pair(op_31_22: u32, simm7: SImm7Scaled, rn: Reg, rt: Reg, rt2: Reg) -> u32 {
238    (op_31_22 << 22)
239        | (simm7.bits() << 15)
240        | (machreg_to_gpr(rt2) << 10)
241        | (machreg_to_gpr(rn) << 5)
242        | machreg_to_gpr(rt)
243}
244
245fn enc_ldst_simm9(op_31_22: u32, simm9: SImm9, op_11_10: u32, rn: Reg, rd: Reg) -> u32 {
246    (op_31_22 << 22)
247        | (simm9.bits() << 12)
248        | (op_11_10 << 10)
249        | (machreg_to_gpr(rn) << 5)
250        | machreg_to_gpr_or_vec(rd)
251}
252
253fn enc_ldst_uimm12(op_31_22: u32, uimm12: UImm12Scaled, rn: Reg, rd: Reg) -> u32 {
254    (op_31_22 << 22)
255        | (0b1 << 24)
256        | (uimm12.bits() << 10)
257        | (machreg_to_gpr(rn) << 5)
258        | machreg_to_gpr_or_vec(rd)
259}
260
261fn enc_ldst_reg(
262    op_31_22: u32,
263    rn: Reg,
264    rm: Reg,
265    s_bit: bool,
266    extendop: Option<ExtendOp>,
267    rd: Reg,
268) -> u32 {
269    let s_bit = if s_bit { 1 } else { 0 };
270    let extend_bits = match extendop {
271        Some(ExtendOp::UXTW) => 0b010,
272        Some(ExtendOp::SXTW) => 0b110,
273        Some(ExtendOp::SXTX) => 0b111,
274        None => 0b011, // LSL
275        _ => panic!("bad extend mode for ld/st AMode"),
276    };
277    (op_31_22 << 22)
278        | (1 << 21)
279        | (machreg_to_gpr(rm) << 16)
280        | (extend_bits << 13)
281        | (s_bit << 12)
282        | (0b10 << 10)
283        | (machreg_to_gpr(rn) << 5)
284        | machreg_to_gpr_or_vec(rd)
285}
286
287pub(crate) fn enc_ldst_imm19(op_31_24: u32, imm19: u32, rd: Reg) -> u32 {
288    (op_31_24 << 24) | (imm19 << 5) | machreg_to_gpr_or_vec(rd)
289}
290
291fn enc_ldst_vec(q: u32, size: u32, rn: Reg, rt: Writable<Reg>) -> u32 {
292    debug_assert_eq!(q & 0b1, q);
293    debug_assert_eq!(size & 0b11, size);
294    0b0_0_0011010_10_00000_110_0_00_00000_00000
295        | q << 30
296        | size << 10
297        | machreg_to_gpr(rn) << 5
298        | machreg_to_vec(rt.to_reg())
299}
300
301fn enc_ldst_vec_pair(
302    opc: u32,
303    amode: u32,
304    is_load: bool,
305    simm7: SImm7Scaled,
306    rn: Reg,
307    rt: Reg,
308    rt2: Reg,
309) -> u32 {
310    debug_assert_eq!(opc & 0b11, opc);
311    debug_assert_eq!(amode & 0b11, amode);
312
313    0b00_10110_00_0_0000000_00000_00000_00000
314        | opc << 30
315        | amode << 23
316        | (is_load as u32) << 22
317        | simm7.bits() << 15
318        | machreg_to_vec(rt2) << 10
319        | machreg_to_gpr(rn) << 5
320        | machreg_to_vec(rt)
321}
322
323fn enc_vec_rrr(top11: u32, rm: Reg, bit15_10: u32, rn: Reg, rd: Writable<Reg>) -> u32 {
324    (top11 << 21)
325        | (machreg_to_vec(rm) << 16)
326        | (bit15_10 << 10)
327        | (machreg_to_vec(rn) << 5)
328        | machreg_to_vec(rd.to_reg())
329}
330
331fn enc_vec_rrr_long(
332    q: u32,
333    u: u32,
334    size: u32,
335    bit14: u32,
336    rm: Reg,
337    rn: Reg,
338    rd: Writable<Reg>,
339) -> u32 {
340    debug_assert_eq!(q & 0b1, q);
341    debug_assert_eq!(u & 0b1, u);
342    debug_assert_eq!(size & 0b11, size);
343    debug_assert_eq!(bit14 & 0b1, bit14);
344
345    0b0_0_0_01110_00_1_00000_100000_00000_00000
346        | q << 30
347        | u << 29
348        | size << 22
349        | bit14 << 14
350        | (machreg_to_vec(rm) << 16)
351        | (machreg_to_vec(rn) << 5)
352        | machreg_to_vec(rd.to_reg())
353}
354
355fn enc_bit_rr(size: u32, opcode2: u32, opcode1: u32, rn: Reg, rd: Writable<Reg>) -> u32 {
356    (0b01011010110 << 21)
357        | size << 31
358        | opcode2 << 16
359        | opcode1 << 10
360        | machreg_to_gpr(rn) << 5
361        | machreg_to_gpr(rd.to_reg())
362}
363
364pub(crate) fn enc_br(rn: Reg) -> u32 {
365    0b1101011_0000_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5)
366}
367
368pub(crate) fn enc_adr_inst(opcode: u32, off: i32, rd: Writable<Reg>) -> u32 {
369    let off = u32::try_from(off).unwrap();
370    let immlo = off & 3;
371    let immhi = (off >> 2) & ((1 << 19) - 1);
372    opcode | (immlo << 29) | (immhi << 5) | machreg_to_gpr(rd.to_reg())
373}
374
375pub(crate) fn enc_adr(off: i32, rd: Writable<Reg>) -> u32 {
376    let opcode = 0b00010000 << 24;
377    enc_adr_inst(opcode, off, rd)
378}
379
380pub(crate) fn enc_adrp(off: i32, rd: Writable<Reg>) -> u32 {
381    let opcode = 0b10010000 << 24;
382    enc_adr_inst(opcode, off, rd)
383}
384
385fn enc_csel(rd: Writable<Reg>, rn: Reg, rm: Reg, cond: Cond, op: u32, o2: u32) -> u32 {
386    debug_assert_eq!(op & 0b1, op);
387    debug_assert_eq!(o2 & 0b1, o2);
388    0b100_11010100_00000_0000_00_00000_00000
389        | (op << 30)
390        | (machreg_to_gpr(rm) << 16)
391        | (cond.bits() << 12)
392        | (o2 << 10)
393        | (machreg_to_gpr(rn) << 5)
394        | machreg_to_gpr(rd.to_reg())
395}
396
397fn enc_fcsel(rd: Writable<Reg>, rn: Reg, rm: Reg, cond: Cond, size: ScalarSize) -> u32 {
398    0b000_11110_00_1_00000_0000_11_00000_00000
399        | (size.ftype() << 22)
400        | (machreg_to_vec(rm) << 16)
401        | (machreg_to_vec(rn) << 5)
402        | machreg_to_vec(rd.to_reg())
403        | (cond.bits() << 12)
404}
405
406fn enc_ccmp(size: OperandSize, rn: Reg, rm: Reg, nzcv: NZCV, cond: Cond) -> u32 {
407    0b0_1_1_11010010_00000_0000_00_00000_0_0000
408        | size.sf_bit() << 31
409        | machreg_to_gpr(rm) << 16
410        | cond.bits() << 12
411        | machreg_to_gpr(rn) << 5
412        | nzcv.bits()
413}
414
415fn enc_ccmp_imm(size: OperandSize, rn: Reg, imm: UImm5, nzcv: NZCV, cond: Cond) -> u32 {
416    0b0_1_1_11010010_00000_0000_10_00000_0_0000
417        | size.sf_bit() << 31
418        | imm.bits() << 16
419        | cond.bits() << 12
420        | machreg_to_gpr(rn) << 5
421        | nzcv.bits()
422}
423
424impl BfmOp {
425    fn opc(self) -> u8 {
426        match self {
427            BfmOp::UBfm => 0b10,
428            BfmOp::SBfm => 0b00,
429        }
430    }
431}
432
433fn enc_bfm(opc: u8, size: OperandSize, rd: Writable<Reg>, rn: Reg, immr: u8, imms: u8) -> u32 {
434    match size {
435        OperandSize::Size64 => {
436            debug_assert!(immr <= 63);
437            debug_assert!(imms <= 63);
438        }
439        OperandSize::Size32 => {
440            debug_assert!(immr <= 31);
441            debug_assert!(imms <= 31);
442        }
443    }
444    debug_assert_eq!(opc & 0b11, opc);
445    let n_bit = size.sf_bit();
446    0b0_00_100110_0_000000_000000_00000_00000
447        | size.sf_bit() << 31
448        | u32::from(opc) << 29
449        | n_bit << 22
450        | u32::from(immr) << 16
451        | u32::from(imms) << 10
452        | machreg_to_gpr(rn) << 5
453        | machreg_to_gpr(rd.to_reg())
454}
455
456fn enc_vecmov(is_16b: bool, rd: Writable<Reg>, rn: Reg) -> u32 {
457    0b00001110_101_00000_00011_1_00000_00000
458        | ((is_16b as u32) << 30)
459        | machreg_to_vec(rd.to_reg())
460        | (machreg_to_vec(rn) << 16)
461        | (machreg_to_vec(rn) << 5)
462}
463
464fn enc_fpurr(top22: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
465    (top22 << 10) | (machreg_to_vec(rn) << 5) | machreg_to_vec(rd.to_reg())
466}
467
468fn enc_fpurrr(top22: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
469    (top22 << 10)
470        | (machreg_to_vec(rm) << 16)
471        | (machreg_to_vec(rn) << 5)
472        | machreg_to_vec(rd.to_reg())
473}
474
475fn enc_fpurrrr(top17: u32, rd: Writable<Reg>, rn: Reg, rm: Reg, ra: Reg) -> u32 {
476    (top17 << 15)
477        | (machreg_to_vec(rm) << 16)
478        | (machreg_to_vec(ra) << 10)
479        | (machreg_to_vec(rn) << 5)
480        | machreg_to_vec(rd.to_reg())
481}
482
483fn enc_fcmp(size: ScalarSize, rn: Reg, rm: Reg) -> u32 {
484    0b000_11110_00_1_00000_00_1000_00000_00000
485        | (size.ftype() << 22)
486        | (machreg_to_vec(rm) << 16)
487        | (machreg_to_vec(rn) << 5)
488}
489
490fn enc_fputoint(top16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
491    (top16 << 16) | (machreg_to_vec(rn) << 5) | machreg_to_gpr(rd.to_reg())
492}
493
494fn enc_inttofpu(top16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
495    (top16 << 16) | (machreg_to_gpr(rn) << 5) | machreg_to_vec(rd.to_reg())
496}
497
498fn enc_fround(top22: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
499    (top22 << 10) | (machreg_to_vec(rn) << 5) | machreg_to_vec(rd.to_reg())
500}
501
502fn enc_vec_rr_misc(qu: u32, size: u32, bits_12_16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
503    debug_assert_eq!(qu & 0b11, qu);
504    debug_assert_eq!(size & 0b11, size);
505    debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16);
506    let bits = 0b0_00_01110_00_10000_00000_10_00000_00000;
507    bits | qu << 29
508        | size << 22
509        | bits_12_16 << 12
510        | machreg_to_vec(rn) << 5
511        | machreg_to_vec(rd.to_reg())
512}
513
514fn enc_vec_rr_pair(bits_12_16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
515    debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16);
516
517    0b010_11110_11_11000_11011_10_00000_00000
518        | bits_12_16 << 12
519        | machreg_to_vec(rn) << 5
520        | machreg_to_vec(rd.to_reg())
521}
522
523fn enc_vec_rr_pair_long(u: u32, enc_size: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
524    debug_assert_eq!(u & 0b1, u);
525    debug_assert_eq!(enc_size & 0b1, enc_size);
526
527    0b0_1_0_01110_00_10000_00_0_10_10_00000_00000
528        | u << 29
529        | enc_size << 22
530        | machreg_to_vec(rn) << 5
531        | machreg_to_vec(rd.to_reg())
532}
533
534fn enc_vec_lanes(q: u32, u: u32, size: u32, opcode: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
535    debug_assert_eq!(q & 0b1, q);
536    debug_assert_eq!(u & 0b1, u);
537    debug_assert_eq!(size & 0b11, size);
538    debug_assert_eq!(opcode & 0b11111, opcode);
539    0b0_0_0_01110_00_11000_0_0000_10_00000_00000
540        | q << 30
541        | u << 29
542        | size << 22
543        | opcode << 12
544        | machreg_to_vec(rn) << 5
545        | machreg_to_vec(rd.to_reg())
546}
547
548fn enc_tbl(is_extension: bool, len: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
549    debug_assert_eq!(len & 0b11, len);
550    0b0_1_001110_000_00000_0_00_0_00_00000_00000
551        | (machreg_to_vec(rm) << 16)
552        | len << 13
553        | (is_extension as u32) << 12
554        | (machreg_to_vec(rn) << 5)
555        | machreg_to_vec(rd.to_reg())
556}
557
558fn enc_dmb_ish() -> u32 {
559    0xD5033BBF
560}
561
562fn enc_acq_rel(ty: Type, op: AtomicRMWOp, rs: Reg, rt: Writable<Reg>, rn: Reg) -> u32 {
563    assert!(machreg_to_gpr(rt.to_reg()) != 31);
564    let sz = match ty {
565        I64 => 0b11,
566        I32 => 0b10,
567        I16 => 0b01,
568        I8 => 0b00,
569        _ => unreachable!(),
570    };
571    let bit15 = match op {
572        AtomicRMWOp::Swp => 0b1,
573        _ => 0b0,
574    };
575    let op = match op {
576        AtomicRMWOp::Add => 0b000,
577        AtomicRMWOp::Clr => 0b001,
578        AtomicRMWOp::Eor => 0b010,
579        AtomicRMWOp::Set => 0b011,
580        AtomicRMWOp::Smax => 0b100,
581        AtomicRMWOp::Smin => 0b101,
582        AtomicRMWOp::Umax => 0b110,
583        AtomicRMWOp::Umin => 0b111,
584        AtomicRMWOp::Swp => 0b000,
585    };
586    0b00_111_000_111_00000_0_000_00_00000_00000
587        | (sz << 30)
588        | (machreg_to_gpr(rs) << 16)
589        | bit15 << 15
590        | (op << 12)
591        | (machreg_to_gpr(rn) << 5)
592        | machreg_to_gpr(rt.to_reg())
593}
594
595fn enc_ldar(ty: Type, rt: Writable<Reg>, rn: Reg) -> u32 {
596    let sz = match ty {
597        I64 => 0b11,
598        I32 => 0b10,
599        I16 => 0b01,
600        I8 => 0b00,
601        _ => unreachable!(),
602    };
603    0b00_001000_1_1_0_11111_1_11111_00000_00000
604        | (sz << 30)
605        | (machreg_to_gpr(rn) << 5)
606        | machreg_to_gpr(rt.to_reg())
607}
608
609fn enc_stlr(ty: Type, rt: Reg, rn: Reg) -> u32 {
610    let sz = match ty {
611        I64 => 0b11,
612        I32 => 0b10,
613        I16 => 0b01,
614        I8 => 0b00,
615        _ => unreachable!(),
616    };
617    0b00_001000_100_11111_1_11111_00000_00000
618        | (sz << 30)
619        | (machreg_to_gpr(rn) << 5)
620        | machreg_to_gpr(rt)
621}
622
623fn enc_ldaxr(ty: Type, rt: Writable<Reg>, rn: Reg) -> u32 {
624    let sz = match ty {
625        I64 => 0b11,
626        I32 => 0b10,
627        I16 => 0b01,
628        I8 => 0b00,
629        _ => unreachable!(),
630    };
631    0b00_001000_0_1_0_11111_1_11111_00000_00000
632        | (sz << 30)
633        | (machreg_to_gpr(rn) << 5)
634        | machreg_to_gpr(rt.to_reg())
635}
636
637fn enc_ldaxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {
638    let sz = match ty {
639        I64 => 0b1,
640        I32 => 0b0,
641        _ => unreachable!(),
642    };
643    0b10_0010000_1_1_11111_1_00000_00000_00000
644        | (sz << 30)
645        | (machreg_to_gpr(rt2.to_reg()) << 10)
646        | (machreg_to_gpr(rn) << 5)
647        | machreg_to_gpr(rt.to_reg())
648}
649
650fn enc_ldxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {
651    let sz = match ty {
652        I64 => 0b1,
653        I32 => 0b0,
654        _ => unreachable!(),
655    };
656    0b10_0010000_1_1_11111_0_00000_00000_00000
657        | (sz << 30)
658        | (machreg_to_gpr(rt2.to_reg()) << 10)
659        | (machreg_to_gpr(rn) << 5)
660        | machreg_to_gpr(rt.to_reg())
661}
662
663fn enc_stlxr(ty: Type, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
664    let sz = match ty {
665        I64 => 0b11,
666        I32 => 0b10,
667        I16 => 0b01,
668        I8 => 0b00,
669        _ => unreachable!(),
670    };
671    0b00_001000_000_00000_1_11111_00000_00000
672        | (sz << 30)
673        | (machreg_to_gpr(rs.to_reg()) << 16)
674        | (machreg_to_gpr(rn) << 5)
675        | machreg_to_gpr(rt)
676}
677
678fn enc_stlxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {
679    let sz = match ty {
680        I64 => 0b1,
681        I32 => 0b0,
682        _ => unreachable!(),
683    };
684    0b10_0010000_0_1_00000_1_00000_00000_00000
685        | (sz << 30)
686        | (machreg_to_gpr(rs.to_reg()) << 16)
687        | (machreg_to_gpr(rt2) << 10)
688        | (machreg_to_gpr(rn) << 5)
689        | machreg_to_gpr(rt)
690}
691
692fn enc_stxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {
693    let sz = match ty {
694        I64 => 0b1,
695        I32 => 0b0,
696        _ => unreachable!(),
697    };
698    0b10_0010000_0_1_00000_0_00000_00000_00000
699        | (sz << 30)
700        | (machreg_to_gpr(rs.to_reg()) << 16)
701        | (machreg_to_gpr(rt2) << 10)
702        | (machreg_to_gpr(rn) << 5)
703        | machreg_to_gpr(rt)
704}
705
706fn enc_cas(size: u32, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
707    debug_assert_eq!(size & 0b11, size);
708
709    0b00_0010001_1_1_00000_1_11111_00000_00000
710        | size << 30
711        | machreg_to_gpr(rs.to_reg()) << 16
712        | machreg_to_gpr(rn) << 5
713        | machreg_to_gpr(rt)
714}
715
716fn enc_casp(rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
717    debug_assert_eq!(machreg_to_gpr(rs.to_reg()) & 1, 0);
718    debug_assert_eq!(machreg_to_gpr(rt) & 1, 0);
719
720    0b0_1_0010000_1_1_00000_1_11111_00000_00000
721        | machreg_to_gpr(rs.to_reg()) << 16
722        | machreg_to_gpr(rn) << 5
723        | machreg_to_gpr(rt)
724}
725
726fn enc_asimd_mod_imm(rd: Writable<Reg>, q_op: u32, cmode: u32, imm: u8) -> u32 {
727    let abc = (imm >> 5) as u32;
728    let defgh = (imm & 0b11111) as u32;
729
730    debug_assert_eq!(cmode & 0b1111, cmode);
731    debug_assert_eq!(q_op & 0b11, q_op);
732
733    0b0_0_0_0111100000_000_0000_01_00000_00000
734        | (q_op << 29)
735        | (abc << 16)
736        | (cmode << 12)
737        | (defgh << 5)
738        | machreg_to_vec(rd.to_reg())
739}
740
741/// State carried between emissions of a sequence of instructions.
742#[derive(Default, Clone, Debug)]
743pub struct EmitState {
744    /// Tracks user stack maps for safepoint instructions.
745    stack_maps: StackMapEmitter<Inst>,
746
747    /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
748    /// optimized away at compiletime. See [cranelift_control].
749    ctrl_plane: ControlPlane,
750
751    frame_layout: FrameLayout,
752}
753
754impl MachInstEmitState<Inst> for EmitState {
755    fn new(abi: &Callee<AArch64MachineDeps>, ctrl_plane: ControlPlane) -> Self {
756        EmitState {
757            stack_maps: StackMapEmitter::default(),
758            ctrl_plane,
759            frame_layout: abi.frame_layout().clone(),
760        }
761    }
762
763    fn stack_map_emitter(&mut self) -> &mut StackMapEmitter<Inst> {
764        &mut self.stack_maps
765    }
766
767    fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
768        &mut self.ctrl_plane
769    }
770
771    fn take_ctrl_plane(self) -> ControlPlane {
772        self.ctrl_plane
773    }
774
775    fn frame_layout(&self) -> &FrameLayout {
776        &self.frame_layout
777    }
778}
779
780/// Constant state used during function compilation.
781pub struct EmitInfo {
782    flags: settings::Flags,
783    isa_flags: aarch64::settings::Flags,
784}
785
786impl EmitInfo {
787    /// Create a constant state for emission of instructions.
788    pub fn new(flags: settings::Flags, isa_flags: aarch64::settings::Flags) -> Self {
789        Self { flags, isa_flags }
790    }
791}
792
793impl MachInstEmit for Inst {
794    type State = EmitState;
795    type Info = EmitInfo;
796
797    fn emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState) {
798        // N.B.: we *must* not exceed the "worst-case size" used to compute
799        // where to insert islands, except when islands are explicitly triggered
800        // (with an `EmitIsland`). We check this in debug builds. This is `mut`
801        // to allow disabling the check for `JTSequence`, which is always
802        // emitted following an `EmitIsland`.
803        let mut start_off = sink.cur_offset();
804
805        match self {
806            &Inst::AluRRR {
807                alu_op,
808                size,
809                rd,
810                rn,
811                rm,
812            } => {
813                debug_assert!(match alu_op {
814                    ALUOp::SMulH | ALUOp::UMulH => size == OperandSize::Size64,
815                    _ => true,
816                });
817                let top11 = match alu_op {
818                    ALUOp::Add => 0b00001011_000,
819                    ALUOp::Adc => 0b00011010_000,
820                    ALUOp::AdcS => 0b00111010_000,
821                    ALUOp::Sub => 0b01001011_000,
822                    ALUOp::Sbc => 0b01011010_000,
823                    ALUOp::SbcS => 0b01111010_000,
824                    ALUOp::Orr => 0b00101010_000,
825                    ALUOp::And => 0b00001010_000,
826                    ALUOp::AndS => 0b01101010_000,
827                    ALUOp::Eor => 0b01001010_000,
828                    ALUOp::OrrNot => 0b00101010_001,
829                    ALUOp::AndNot => 0b00001010_001,
830                    ALUOp::EorNot => 0b01001010_001,
831                    ALUOp::AddS => 0b00101011_000,
832                    ALUOp::SubS => 0b01101011_000,
833                    ALUOp::SDiv | ALUOp::UDiv => 0b00011010_110,
834                    ALUOp::Extr | ALUOp::Lsr | ALUOp::Asr | ALUOp::Lsl => 0b00011010_110,
835                    ALUOp::SMulH => 0b10011011_010,
836                    ALUOp::UMulH => 0b10011011_110,
837                };
838
839                let top11 = top11 | size.sf_bit() << 10;
840                let bit15_10 = match alu_op {
841                    ALUOp::SDiv => 0b000011,
842                    ALUOp::UDiv => 0b000010,
843                    ALUOp::Extr => 0b001011,
844                    ALUOp::Lsr => 0b001001,
845                    ALUOp::Asr => 0b001010,
846                    ALUOp::Lsl => 0b001000,
847                    ALUOp::SMulH | ALUOp::UMulH => 0b011111,
848                    _ => 0b000000,
849                };
850                debug_assert_ne!(writable_stack_reg(), rd);
851                // The stack pointer is the zero register in this context, so this might be an
852                // indication that something is wrong.
853                debug_assert_ne!(stack_reg(), rn);
854                debug_assert_ne!(stack_reg(), rm);
855                sink.put4(enc_arith_rrr(top11, bit15_10, rd, rn, rm));
856            }
857            &Inst::AluRRRR {
858                alu_op,
859                size,
860                rd,
861                rm,
862                rn,
863                ra,
864            } => {
865                let (top11, bit15) = match alu_op {
866                    ALUOp3::MAdd => (0b0_00_11011_000, 0),
867                    ALUOp3::MSub => (0b0_00_11011_000, 1),
868                    ALUOp3::UMAddL => {
869                        debug_assert!(size == OperandSize::Size32);
870                        (0b1_00_11011_1_01, 0)
871                    }
872                    ALUOp3::SMAddL => {
873                        debug_assert!(size == OperandSize::Size32);
874                        (0b1_00_11011_0_01, 0)
875                    }
876                };
877                let top11 = top11 | size.sf_bit() << 10;
878                sink.put4(enc_arith_rrrr(top11, rm, bit15, ra, rn, rd));
879            }
880            &Inst::AluRRImm12 {
881                alu_op,
882                size,
883                rd,
884                rn,
885                ref imm12,
886            } => {
887                let top8 = match alu_op {
888                    ALUOp::Add => 0b000_10001,
889                    ALUOp::Sub => 0b010_10001,
890                    ALUOp::AddS => 0b001_10001,
891                    ALUOp::SubS => 0b011_10001,
892                    _ => unimplemented!("{:?}", alu_op),
893                };
894                let top8 = top8 | size.sf_bit() << 7;
895                sink.put4(enc_arith_rr_imm12(
896                    top8,
897                    imm12.shift_bits(),
898                    imm12.imm_bits(),
899                    rn,
900                    rd,
901                ));
902            }
903            &Inst::AluRRImmLogic {
904                alu_op,
905                size,
906                rd,
907                rn,
908                ref imml,
909            } => {
910                let (top9, inv) = match alu_op {
911                    ALUOp::Orr => (0b001_100100, false),
912                    ALUOp::And => (0b000_100100, false),
913                    ALUOp::AndS => (0b011_100100, false),
914                    ALUOp::Eor => (0b010_100100, false),
915                    ALUOp::OrrNot => (0b001_100100, true),
916                    ALUOp::AndNot => (0b000_100100, true),
917                    ALUOp::EorNot => (0b010_100100, true),
918                    _ => unimplemented!("{:?}", alu_op),
919                };
920                let top9 = top9 | size.sf_bit() << 8;
921                let imml = if inv { imml.invert() } else { *imml };
922                sink.put4(enc_arith_rr_imml(top9, imml.enc_bits(), rn, rd));
923            }
924
925            &Inst::AluRRImmShift {
926                alu_op,
927                size,
928                rd,
929                rn,
930                ref immshift,
931            } => {
932                let amt = immshift.value();
933                let (top10, immr, imms) = match alu_op {
934                    ALUOp::Extr => (0b0001001110, machreg_to_gpr(rn), u32::from(amt)),
935                    ALUOp::Lsr => (0b0101001100, u32::from(amt), 0b011111),
936                    ALUOp::Asr => (0b0001001100, u32::from(amt), 0b011111),
937                    ALUOp::Lsl => {
938                        let bits = if size.is64() { 64 } else { 32 };
939                        (
940                            0b0101001100,
941                            u32::from((bits - amt) % bits),
942                            u32::from(bits - 1 - amt),
943                        )
944                    }
945                    _ => unimplemented!("{:?}", alu_op),
946                };
947                let top10 = top10 | size.sf_bit() << 9 | size.sf_bit();
948                let imms = match alu_op {
949                    ALUOp::Lsr | ALUOp::Asr => imms | size.sf_bit() << 5,
950                    _ => imms,
951                };
952                sink.put4(
953                    (top10 << 22)
954                        | (immr << 16)
955                        | (imms << 10)
956                        | (machreg_to_gpr(rn) << 5)
957                        | machreg_to_gpr(rd.to_reg()),
958                );
959            }
960
961            &Inst::AluRRRShift {
962                alu_op,
963                size,
964                rd,
965                rn,
966                rm,
967                ref shiftop,
968            } => {
969                let top11: u32 = match alu_op {
970                    ALUOp::Add => 0b000_01011000,
971                    ALUOp::AddS => 0b001_01011000,
972                    ALUOp::Sub => 0b010_01011000,
973                    ALUOp::SubS => 0b011_01011000,
974                    ALUOp::Orr => 0b001_01010000,
975                    ALUOp::And => 0b000_01010000,
976                    ALUOp::AndS => 0b011_01010000,
977                    ALUOp::Eor => 0b010_01010000,
978                    ALUOp::OrrNot => 0b001_01010001,
979                    ALUOp::EorNot => 0b010_01010001,
980                    ALUOp::AndNot => 0b000_01010001,
981                    ALUOp::Extr => 0b000_10011100,
982                    _ => unimplemented!("{:?}", alu_op),
983                };
984                let top11 = top11 | size.sf_bit() << 10;
985                let top11 = top11 | (u32::from(shiftop.op().bits()) << 1);
986                let bits_15_10 = u32::from(shiftop.amt().value());
987                sink.put4(enc_arith_rrr(top11, bits_15_10, rd, rn, rm));
988            }
989
990            &Inst::AluRRRExtend {
991                alu_op,
992                size,
993                rd,
994                rn,
995                rm,
996                extendop,
997            } => {
998                let top11: u32 = match alu_op {
999                    ALUOp::Add => 0b00001011001,
1000                    ALUOp::Sub => 0b01001011001,
1001                    ALUOp::AddS => 0b00101011001,
1002                    ALUOp::SubS => 0b01101011001,
1003                    _ => unimplemented!("{:?}", alu_op),
1004                };
1005                let top11 = top11 | size.sf_bit() << 10;
1006                let bits_15_10 = u32::from(extendop.bits()) << 3;
1007                sink.put4(enc_arith_rrr(top11, bits_15_10, rd, rn, rm));
1008            }
1009
1010            &Inst::BitRR {
1011                op, size, rd, rn, ..
1012            } => {
1013                let (op1, op2) = match op {
1014                    BitOp::RBit => (0b00000, 0b000000),
1015                    BitOp::Clz => (0b00000, 0b000100),
1016                    BitOp::Cls => (0b00000, 0b000101),
1017                    BitOp::Rev16 => (0b00000, 0b000001),
1018                    BitOp::Rev32 => (0b00000, 0b000010),
1019                    BitOp::Rev64 => (0b00000, 0b000011),
1020                };
1021                sink.put4(enc_bit_rr(size.sf_bit(), op1, op2, rn, rd))
1022            }
1023
1024            &Inst::ULoad8 { rd, ref mem, flags }
1025            | &Inst::SLoad8 { rd, ref mem, flags }
1026            | &Inst::ULoad16 { rd, ref mem, flags }
1027            | &Inst::SLoad16 { rd, ref mem, flags }
1028            | &Inst::ULoad32 { rd, ref mem, flags }
1029            | &Inst::SLoad32 { rd, ref mem, flags }
1030            | &Inst::ULoad64 {
1031                rd, ref mem, flags, ..
1032            }
1033            | &Inst::FpuLoad16 { rd, ref mem, flags }
1034            | &Inst::FpuLoad32 { rd, ref mem, flags }
1035            | &Inst::FpuLoad64 { rd, ref mem, flags }
1036            | &Inst::FpuLoad128 { rd, ref mem, flags } => {
1037                let mem = mem.clone();
1038                let access_ty = self.mem_type().unwrap();
1039                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, access_ty, state);
1040
1041                for inst in mem_insts.into_iter() {
1042                    inst.emit(sink, emit_info, state);
1043                }
1044
1045                // ldst encoding helpers take Reg, not Writable<Reg>.
1046                let rd = rd.to_reg();
1047
1048                // This is the base opcode (top 10 bits) for the "unscaled
1049                // immediate" form (Unscaled). Other addressing modes will OR in
1050                // other values for bits 24/25 (bits 1/2 of this constant).
1051                let op = match self {
1052                    Inst::ULoad8 { .. } => 0b0011100001,
1053                    Inst::SLoad8 { .. } => 0b0011100010,
1054                    Inst::ULoad16 { .. } => 0b0111100001,
1055                    Inst::SLoad16 { .. } => 0b0111100010,
1056                    Inst::ULoad32 { .. } => 0b1011100001,
1057                    Inst::SLoad32 { .. } => 0b1011100010,
1058                    Inst::ULoad64 { .. } => 0b1111100001,
1059                    Inst::FpuLoad16 { .. } => 0b0111110001,
1060                    Inst::FpuLoad32 { .. } => 0b1011110001,
1061                    Inst::FpuLoad64 { .. } => 0b1111110001,
1062                    Inst::FpuLoad128 { .. } => 0b0011110011,
1063                    _ => unreachable!(),
1064                };
1065
1066                if let Some(trap_code) = flags.trap_code() {
1067                    // Register the offset at which the actual load instruction starts.
1068                    sink.add_trap(trap_code);
1069                }
1070
1071                match &mem {
1072                    &AMode::Unscaled { rn, simm9 } => {
1073                        let reg = rn;
1074                        sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
1075                    }
1076                    &AMode::UnsignedOffset { rn, uimm12 } => {
1077                        let reg = rn;
1078                        sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
1079                    }
1080                    &AMode::RegReg { rn, rm } => {
1081                        let r1 = rn;
1082                        let r2 = rm;
1083                        sink.put4(enc_ldst_reg(
1084                            op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
1085                        ));
1086                    }
1087                    &AMode::RegScaled { rn, rm } | &AMode::RegScaledExtended { rn, rm, .. } => {
1088                        let r1 = rn;
1089                        let r2 = rm;
1090                        let extendop = match &mem {
1091                            &AMode::RegScaled { .. } => None,
1092                            &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
1093                            _ => unreachable!(),
1094                        };
1095                        sink.put4(enc_ldst_reg(
1096                            op, r1, r2, /* scaled = */ true, extendop, rd,
1097                        ));
1098                    }
1099                    &AMode::RegExtended { rn, rm, extendop } => {
1100                        let r1 = rn;
1101                        let r2 = rm;
1102                        sink.put4(enc_ldst_reg(
1103                            op,
1104                            r1,
1105                            r2,
1106                            /* scaled = */ false,
1107                            Some(extendop),
1108                            rd,
1109                        ));
1110                    }
1111                    &AMode::Label { ref label } => {
1112                        let offset = match label {
1113                            // cast i32 to u32 (two's-complement)
1114                            MemLabel::PCRel(off) => *off as u32,
1115                            // Emit a relocation into the `MachBuffer`
1116                            // for the label that's being loaded from and
1117                            // encode an address of 0 in its place which will
1118                            // get filled in by relocation resolution later on.
1119                            MemLabel::Mach(label) => {
1120                                sink.use_label_at_offset(
1121                                    sink.cur_offset(),
1122                                    *label,
1123                                    LabelUse::Ldr19,
1124                                );
1125                                0
1126                            }
1127                        } / 4;
1128                        assert!(offset < (1 << 19));
1129                        match self {
1130                            &Inst::ULoad32 { .. } => {
1131                                sink.put4(enc_ldst_imm19(0b00011000, offset, rd));
1132                            }
1133                            &Inst::SLoad32 { .. } => {
1134                                sink.put4(enc_ldst_imm19(0b10011000, offset, rd));
1135                            }
1136                            &Inst::FpuLoad32 { .. } => {
1137                                sink.put4(enc_ldst_imm19(0b00011100, offset, rd));
1138                            }
1139                            &Inst::ULoad64 { .. } => {
1140                                sink.put4(enc_ldst_imm19(0b01011000, offset, rd));
1141                            }
1142                            &Inst::FpuLoad64 { .. } => {
1143                                sink.put4(enc_ldst_imm19(0b01011100, offset, rd));
1144                            }
1145                            &Inst::FpuLoad128 { .. } => {
1146                                sink.put4(enc_ldst_imm19(0b10011100, offset, rd));
1147                            }
1148                            _ => panic!("Unsupported size for LDR from constant pool!"),
1149                        }
1150                    }
1151                    &AMode::SPPreIndexed { simm9 } => {
1152                        let reg = stack_reg();
1153                        sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
1154                    }
1155                    &AMode::SPPostIndexed { simm9 } => {
1156                        let reg = stack_reg();
1157                        sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
1158                    }
1159                    // Eliminated by `mem_finalize()` above.
1160                    &AMode::SPOffset { .. }
1161                    | &AMode::FPOffset { .. }
1162                    | &AMode::IncomingArg { .. }
1163                    | &AMode::SlotOffset { .. }
1164                    | &AMode::Const { .. }
1165                    | &AMode::RegOffset { .. } => {
1166                        panic!("Should not see {mem:?} here!")
1167                    }
1168                }
1169            }
1170
1171            &Inst::Store8 { rd, ref mem, flags }
1172            | &Inst::Store16 { rd, ref mem, flags }
1173            | &Inst::Store32 { rd, ref mem, flags }
1174            | &Inst::Store64 { rd, ref mem, flags }
1175            | &Inst::FpuStore16 { rd, ref mem, flags }
1176            | &Inst::FpuStore32 { rd, ref mem, flags }
1177            | &Inst::FpuStore64 { rd, ref mem, flags }
1178            | &Inst::FpuStore128 { rd, ref mem, flags } => {
1179                let mem = mem.clone();
1180                let access_ty = self.mem_type().unwrap();
1181                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, access_ty, state);
1182
1183                for inst in mem_insts.into_iter() {
1184                    inst.emit(sink, emit_info, state);
1185                }
1186
1187                let op = match self {
1188                    Inst::Store8 { .. } => 0b0011100000,
1189                    Inst::Store16 { .. } => 0b0111100000,
1190                    Inst::Store32 { .. } => 0b1011100000,
1191                    Inst::Store64 { .. } => 0b1111100000,
1192                    Inst::FpuStore16 { .. } => 0b0111110000,
1193                    Inst::FpuStore32 { .. } => 0b1011110000,
1194                    Inst::FpuStore64 { .. } => 0b1111110000,
1195                    Inst::FpuStore128 { .. } => 0b0011110010,
1196                    _ => unreachable!(),
1197                };
1198
1199                if let Some(trap_code) = flags.trap_code() {
1200                    // Register the offset at which the actual store instruction starts.
1201                    sink.add_trap(trap_code);
1202                }
1203
1204                match &mem {
1205                    &AMode::Unscaled { rn, simm9 } => {
1206                        let reg = rn;
1207                        sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
1208                    }
1209                    &AMode::UnsignedOffset { rn, uimm12 } => {
1210                        let reg = rn;
1211                        sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
1212                    }
1213                    &AMode::RegReg { rn, rm } => {
1214                        let r1 = rn;
1215                        let r2 = rm;
1216                        sink.put4(enc_ldst_reg(
1217                            op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
1218                        ));
1219                    }
1220                    &AMode::RegScaled { rn, rm } | &AMode::RegScaledExtended { rn, rm, .. } => {
1221                        let r1 = rn;
1222                        let r2 = rm;
1223                        let extendop = match &mem {
1224                            &AMode::RegScaled { .. } => None,
1225                            &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
1226                            _ => unreachable!(),
1227                        };
1228                        sink.put4(enc_ldst_reg(
1229                            op, r1, r2, /* scaled = */ true, extendop, rd,
1230                        ));
1231                    }
1232                    &AMode::RegExtended { rn, rm, extendop } => {
1233                        let r1 = rn;
1234                        let r2 = rm;
1235                        sink.put4(enc_ldst_reg(
1236                            op,
1237                            r1,
1238                            r2,
1239                            /* scaled = */ false,
1240                            Some(extendop),
1241                            rd,
1242                        ));
1243                    }
1244                    &AMode::Label { .. } => {
1245                        panic!("Store to a MemLabel not implemented!");
1246                    }
1247                    &AMode::SPPreIndexed { simm9 } => {
1248                        let reg = stack_reg();
1249                        sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
1250                    }
1251                    &AMode::SPPostIndexed { simm9 } => {
1252                        let reg = stack_reg();
1253                        sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
1254                    }
1255                    // Eliminated by `mem_finalize()` above.
1256                    &AMode::SPOffset { .. }
1257                    | &AMode::FPOffset { .. }
1258                    | &AMode::IncomingArg { .. }
1259                    | &AMode::SlotOffset { .. }
1260                    | &AMode::Const { .. }
1261                    | &AMode::RegOffset { .. } => {
1262                        panic!("Should not see {mem:?} here!")
1263                    }
1264                }
1265            }
1266
1267            &Inst::StoreP64 {
1268                rt,
1269                rt2,
1270                ref mem,
1271                flags,
1272            } => {
1273                let mem = mem.clone();
1274                if let Some(trap_code) = flags.trap_code() {
1275                    // Register the offset at which the actual store instruction starts.
1276                    sink.add_trap(trap_code);
1277                }
1278                match &mem {
1279                    &PairAMode::SignedOffset { reg, simm7 } => {
1280                        assert_eq!(simm7.scale_ty, I64);
1281                        sink.put4(enc_ldst_pair(0b1010100100, simm7, reg, rt, rt2));
1282                    }
1283                    &PairAMode::SPPreIndexed { simm7 } => {
1284                        assert_eq!(simm7.scale_ty, I64);
1285                        let reg = stack_reg();
1286                        sink.put4(enc_ldst_pair(0b1010100110, simm7, reg, rt, rt2));
1287                    }
1288                    &PairAMode::SPPostIndexed { simm7 } => {
1289                        assert_eq!(simm7.scale_ty, I64);
1290                        let reg = stack_reg();
1291                        sink.put4(enc_ldst_pair(0b1010100010, simm7, reg, rt, rt2));
1292                    }
1293                }
1294            }
1295            &Inst::LoadP64 {
1296                rt,
1297                rt2,
1298                ref mem,
1299                flags,
1300            } => {
1301                let rt = rt.to_reg();
1302                let rt2 = rt2.to_reg();
1303                let mem = mem.clone();
1304                if let Some(trap_code) = flags.trap_code() {
1305                    // Register the offset at which the actual load instruction starts.
1306                    sink.add_trap(trap_code);
1307                }
1308
1309                match &mem {
1310                    &PairAMode::SignedOffset { reg, simm7 } => {
1311                        assert_eq!(simm7.scale_ty, I64);
1312                        sink.put4(enc_ldst_pair(0b1010100101, simm7, reg, rt, rt2));
1313                    }
1314                    &PairAMode::SPPreIndexed { simm7 } => {
1315                        assert_eq!(simm7.scale_ty, I64);
1316                        let reg = stack_reg();
1317                        sink.put4(enc_ldst_pair(0b1010100111, simm7, reg, rt, rt2));
1318                    }
1319                    &PairAMode::SPPostIndexed { simm7 } => {
1320                        assert_eq!(simm7.scale_ty, I64);
1321                        let reg = stack_reg();
1322                        sink.put4(enc_ldst_pair(0b1010100011, simm7, reg, rt, rt2));
1323                    }
1324                }
1325            }
1326            &Inst::FpuLoadP64 {
1327                rt,
1328                rt2,
1329                ref mem,
1330                flags,
1331            }
1332            | &Inst::FpuLoadP128 {
1333                rt,
1334                rt2,
1335                ref mem,
1336                flags,
1337            } => {
1338                let rt = rt.to_reg();
1339                let rt2 = rt2.to_reg();
1340                let mem = mem.clone();
1341
1342                if let Some(trap_code) = flags.trap_code() {
1343                    // Register the offset at which the actual load instruction starts.
1344                    sink.add_trap(trap_code);
1345                }
1346
1347                let opc = match self {
1348                    &Inst::FpuLoadP64 { .. } => 0b01,
1349                    &Inst::FpuLoadP128 { .. } => 0b10,
1350                    _ => unreachable!(),
1351                };
1352
1353                match &mem {
1354                    &PairAMode::SignedOffset { reg, simm7 } => {
1355                        assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1356                        sink.put4(enc_ldst_vec_pair(opc, 0b10, true, simm7, reg, rt, rt2));
1357                    }
1358                    &PairAMode::SPPreIndexed { simm7 } => {
1359                        assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1360                        let reg = stack_reg();
1361                        sink.put4(enc_ldst_vec_pair(opc, 0b11, true, simm7, reg, rt, rt2));
1362                    }
1363                    &PairAMode::SPPostIndexed { simm7 } => {
1364                        assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1365                        let reg = stack_reg();
1366                        sink.put4(enc_ldst_vec_pair(opc, 0b01, true, simm7, reg, rt, rt2));
1367                    }
1368                }
1369            }
1370            &Inst::FpuStoreP64 {
1371                rt,
1372                rt2,
1373                ref mem,
1374                flags,
1375            }
1376            | &Inst::FpuStoreP128 {
1377                rt,
1378                rt2,
1379                ref mem,
1380                flags,
1381            } => {
1382                let mem = mem.clone();
1383
1384                if let Some(trap_code) = flags.trap_code() {
1385                    // Register the offset at which the actual store instruction starts.
1386                    sink.add_trap(trap_code);
1387                }
1388
1389                let opc = match self {
1390                    &Inst::FpuStoreP64 { .. } => 0b01,
1391                    &Inst::FpuStoreP128 { .. } => 0b10,
1392                    _ => unreachable!(),
1393                };
1394
1395                match &mem {
1396                    &PairAMode::SignedOffset { reg, simm7 } => {
1397                        assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1398                        sink.put4(enc_ldst_vec_pair(opc, 0b10, false, simm7, reg, rt, rt2));
1399                    }
1400                    &PairAMode::SPPreIndexed { simm7 } => {
1401                        assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1402                        let reg = stack_reg();
1403                        sink.put4(enc_ldst_vec_pair(opc, 0b11, false, simm7, reg, rt, rt2));
1404                    }
1405                    &PairAMode::SPPostIndexed { simm7 } => {
1406                        assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1407                        let reg = stack_reg();
1408                        sink.put4(enc_ldst_vec_pair(opc, 0b01, false, simm7, reg, rt, rt2));
1409                    }
1410                }
1411            }
1412            &Inst::Mov { size, rd, rm } => {
1413                assert!(rd.to_reg().class() == rm.class());
1414                assert!(rm.class() == RegClass::Int);
1415
1416                match size {
1417                    OperandSize::Size64 => {
1418                        // MOV to SP is interpreted as MOV to XZR instead. And our codegen
1419                        // should never MOV to XZR.
1420                        assert!(rd.to_reg() != stack_reg());
1421
1422                        if rm == stack_reg() {
1423                            // We can't use ORR here, so use an `add rd, sp, #0` instead.
1424                            let imm12 = Imm12::maybe_from_u64(0).unwrap();
1425                            sink.put4(enc_arith_rr_imm12(
1426                                0b100_10001,
1427                                imm12.shift_bits(),
1428                                imm12.imm_bits(),
1429                                rm,
1430                                rd,
1431                            ));
1432                        } else {
1433                            // Encoded as ORR rd, rm, zero.
1434                            sink.put4(enc_arith_rrr(0b10101010_000, 0b000_000, rd, zero_reg(), rm));
1435                        }
1436                    }
1437                    OperandSize::Size32 => {
1438                        // MOV to SP is interpreted as MOV to XZR instead. And our codegen
1439                        // should never MOV to XZR.
1440                        assert!(machreg_to_gpr(rd.to_reg()) != 31);
1441                        // Encoded as ORR rd, rm, zero.
1442                        sink.put4(enc_arith_rrr(0b00101010_000, 0b000_000, rd, zero_reg(), rm));
1443                    }
1444                }
1445            }
1446            &Inst::MovFromPReg { rd, rm } => {
1447                let rm: Reg = rm.into();
1448                debug_assert!(
1449                    [
1450                        regs::fp_reg(),
1451                        regs::stack_reg(),
1452                        regs::link_reg(),
1453                        regs::pinned_reg()
1454                    ]
1455                    .contains(&rm)
1456                );
1457                assert!(rm.class() == RegClass::Int);
1458                assert!(rd.to_reg().class() == rm.class());
1459                let size = OperandSize::Size64;
1460                Inst::Mov { size, rd, rm }.emit(sink, emit_info, state);
1461            }
1462            &Inst::MovToPReg { rd, rm } => {
1463                let rd: Writable<Reg> = Writable::from_reg(rd.into());
1464                debug_assert!(
1465                    [
1466                        regs::fp_reg(),
1467                        regs::stack_reg(),
1468                        regs::link_reg(),
1469                        regs::pinned_reg()
1470                    ]
1471                    .contains(&rd.to_reg())
1472                );
1473                assert!(rd.to_reg().class() == RegClass::Int);
1474                assert!(rm.class() == rd.to_reg().class());
1475                let size = OperandSize::Size64;
1476                Inst::Mov { size, rd, rm }.emit(sink, emit_info, state);
1477            }
1478            &Inst::MovWide { op, rd, imm, size } => {
1479                sink.put4(enc_move_wide(op, rd, imm, size));
1480            }
1481            &Inst::MovK { rd, rn, imm, size } => {
1482                debug_assert_eq!(rn, rd.to_reg());
1483                sink.put4(enc_movk(rd, imm, size));
1484            }
1485            &Inst::CSel { rd, rn, rm, cond } => {
1486                sink.put4(enc_csel(rd, rn, rm, cond, 0, 0));
1487            }
1488            &Inst::CSNeg { rd, rn, rm, cond } => {
1489                sink.put4(enc_csel(rd, rn, rm, cond, 1, 1));
1490            }
1491            &Inst::CSet { rd, cond } => {
1492                sink.put4(enc_csel(rd, zero_reg(), zero_reg(), cond.invert(), 0, 1));
1493            }
1494            &Inst::CSetm { rd, cond } => {
1495                sink.put4(enc_csel(rd, zero_reg(), zero_reg(), cond.invert(), 1, 0));
1496            }
1497            &Inst::CCmp {
1498                size,
1499                rn,
1500                rm,
1501                nzcv,
1502                cond,
1503            } => {
1504                sink.put4(enc_ccmp(size, rn, rm, nzcv, cond));
1505            }
1506            &Inst::CCmpImm {
1507                size,
1508                rn,
1509                imm,
1510                nzcv,
1511                cond,
1512            } => {
1513                sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
1514            }
1515            &Inst::AtomicRMW {
1516                ty,
1517                op,
1518                rs,
1519                rt,
1520                rn,
1521                flags,
1522            } => {
1523                if let Some(trap_code) = flags.trap_code() {
1524                    sink.add_trap(trap_code);
1525                }
1526
1527                sink.put4(enc_acq_rel(ty, op, rs, rt, rn));
1528            }
1529            &Inst::AtomicRMWLoop { ty, op, flags, .. } => {
1530                /* Emit this:
1531                     again:
1532                      ldaxr{,b,h}  x/w27, [x25]
1533                      // maybe sign extend
1534                      op          x28, x27, x26 // op is add,sub,and,orr,eor
1535                      stlxr{,b,h}  w24, x/w28, [x25]
1536                      cbnz        x24, again
1537
1538                   Operand conventions:
1539                      IN:  x25 (addr), x26 (2nd arg for op)
1540                      OUT: x27 (old value), x24 (trashed), x28 (trashed)
1541
1542                   It is unfortunate that, per the ARM documentation, x28 cannot be used for
1543                   both the store-data and success-flag operands of stlxr.  This causes the
1544                   instruction's behaviour to be "CONSTRAINED UNPREDICTABLE", so we use x24
1545                   instead for the success-flag.
1546                */
1547                // TODO: We should not hardcode registers here, a better idea would be to
1548                // pass some scratch registers in the AtomicRMWLoop pseudo-instruction, and use those
1549                let xzr = zero_reg();
1550                let x24 = xreg(24);
1551                let x25 = xreg(25);
1552                let x26 = xreg(26);
1553                let x27 = xreg(27);
1554                let x28 = xreg(28);
1555                let x24wr = writable_xreg(24);
1556                let x27wr = writable_xreg(27);
1557                let x28wr = writable_xreg(28);
1558                let again_label = sink.get_label();
1559
1560                // again:
1561                sink.bind_label(again_label, &mut state.ctrl_plane);
1562
1563                if let Some(trap_code) = flags.trap_code() {
1564                    sink.add_trap(trap_code);
1565                }
1566
1567                sink.put4(enc_ldaxr(ty, x27wr, x25)); // ldaxr x27, [x25]
1568                let size = OperandSize::from_ty(ty);
1569                let sign_ext = match op {
1570                    AtomicRMWLoopOp::Smin | AtomicRMWLoopOp::Smax => match ty {
1571                        I16 => Some(ExtendOp::SXTH),
1572                        I8 => Some(ExtendOp::SXTB),
1573                        _ => None,
1574                    },
1575                    _ => None,
1576                };
1577                let zero_ext = match op {
1578                    AtomicRMWLoopOp::Umin | AtomicRMWLoopOp::Umax => match ty {
1579                        I16 => Some(ExtendOp::UXTH),
1580                        I8 => Some(ExtendOp::UXTB),
1581                        _ => None,
1582                    },
1583                    _ => None,
1584                };
1585                // sxt{b|h} the loaded result if necessary.
1586                if let Some(op) = sign_ext {
1587                    Inst::gen_extend(x27wr, x27, op, size).emit(sink, emit_info, state);
1588                }
1589
1590                match op {
1591                    AtomicRMWLoopOp::Xchg => {} // do nothing
1592                    AtomicRMWLoopOp::Nand => {
1593                        // and x28, x27, x26
1594                        // mvn x28, x28
1595
1596                        Inst::AluRRR {
1597                            alu_op: ALUOp::And,
1598                            size,
1599                            rd: x28wr,
1600                            rn: x27,
1601                            rm: x26,
1602                        }
1603                        .emit(sink, emit_info, state);
1604
1605                        Inst::AluRRR {
1606                            alu_op: ALUOp::OrrNot,
1607                            size,
1608                            rd: x28wr,
1609                            rn: xzr,
1610                            rm: x28,
1611                        }
1612                        .emit(sink, emit_info, state);
1613                    }
1614                    AtomicRMWLoopOp::Umin
1615                    | AtomicRMWLoopOp::Umax
1616                    | AtomicRMWLoopOp::Smin
1617                    | AtomicRMWLoopOp::Smax => {
1618                        // cmp x27, x26 {?sxt}
1619                        // csel.op x28, x27, x26
1620
1621                        let cond = match op {
1622                            AtomicRMWLoopOp::Umin => Cond::Lo,
1623                            AtomicRMWLoopOp::Umax => Cond::Hi,
1624                            AtomicRMWLoopOp::Smin => Cond::Lt,
1625                            AtomicRMWLoopOp::Smax => Cond::Gt,
1626                            _ => unreachable!(),
1627                        };
1628
1629                        if let Some(extendop) = sign_ext.or(zero_ext) {
1630                            Inst::AluRRRExtend {
1631                                alu_op: ALUOp::SubS,
1632                                size,
1633                                rd: writable_zero_reg(),
1634                                rn: x27,
1635                                rm: x26,
1636                                extendop,
1637                            }
1638                            .emit(sink, emit_info, state);
1639                        } else {
1640                            Inst::AluRRR {
1641                                alu_op: ALUOp::SubS,
1642                                size,
1643                                rd: writable_zero_reg(),
1644                                rn: x27,
1645                                rm: x26,
1646                            }
1647                            .emit(sink, emit_info, state);
1648                        }
1649
1650                        Inst::CSel {
1651                            cond,
1652                            rd: x28wr,
1653                            rn: x27,
1654                            rm: x26,
1655                        }
1656                        .emit(sink, emit_info, state);
1657                    }
1658                    _ => {
1659                        // add/sub/and/orr/eor x28, x27, x26
1660                        let alu_op = match op {
1661                            AtomicRMWLoopOp::Add => ALUOp::Add,
1662                            AtomicRMWLoopOp::Sub => ALUOp::Sub,
1663                            AtomicRMWLoopOp::And => ALUOp::And,
1664                            AtomicRMWLoopOp::Orr => ALUOp::Orr,
1665                            AtomicRMWLoopOp::Eor => ALUOp::Eor,
1666                            AtomicRMWLoopOp::Nand
1667                            | AtomicRMWLoopOp::Umin
1668                            | AtomicRMWLoopOp::Umax
1669                            | AtomicRMWLoopOp::Smin
1670                            | AtomicRMWLoopOp::Smax
1671                            | AtomicRMWLoopOp::Xchg => unreachable!(),
1672                        };
1673
1674                        Inst::AluRRR {
1675                            alu_op,
1676                            size,
1677                            rd: x28wr,
1678                            rn: x27,
1679                            rm: x26,
1680                        }
1681                        .emit(sink, emit_info, state);
1682                    }
1683                }
1684
1685                if let Some(trap_code) = flags.trap_code() {
1686                    sink.add_trap(trap_code);
1687                }
1688                if op == AtomicRMWLoopOp::Xchg {
1689                    sink.put4(enc_stlxr(ty, x24wr, x26, x25)); // stlxr w24, x26, [x25]
1690                } else {
1691                    sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25]
1692                }
1693
1694                // cbnz w24, again
1695                // Note, we're actually testing x24, and relying on the default zero-high-half
1696                // rule in the assignment that `stlxr` does.
1697                let br_offset = sink.cur_offset();
1698                sink.put4(enc_conditional_br(
1699                    BranchTarget::Label(again_label),
1700                    CondBrKind::NotZero(x24, OperandSize::Size64),
1701                ));
1702                sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
1703            }
1704            &Inst::AtomicRMW128Loop { op, flags, .. } => {
1705                /* Emit this:
1706                     again:
1707                      ldaxp       x27, x23, [x25]
1708                      op          x28, x21, x27, x23, x26, x22 // op is adds,subc,and,orr,eor
1709                      stlxp       w24, x28, [x25]
1710                      cbnz        x24, again
1711
1712                   Operand conventions:
1713                      IN:  x25 (addr), x26 (low bytes of op), x22 (high bytes of op)
1714                      OUT: x27 (low bytes of old value), x23 (high bytes of old value), x24 (trashed), x28 (trashed), x21 (trashed)
1715
1716                   It is unfortunate that, per the ARM documentation, x28 cannot be used for
1717                   both the store-data and success-flag operands of stlxr.  This causes the
1718                   instruction's behaviour to be "CONSTRAINED UNPREDICTABLE", so we use x24
1719                   instead for the success-flag.
1720                */
1721                // TODO: We should not hardcode registers here, a better idea would be to
1722                // pass some scratch registers in the AtomicRMWLoop pseudo-instruction, and use those
1723                let xzr = zero_reg();
1724                let x21 = xreg(21);
1725                let x22 = xreg(22);
1726                let x23 = xreg(23);
1727                let x24 = xreg(24);
1728                let x25 = xreg(25);
1729                let x26 = xreg(26);
1730                let x27 = xreg(27);
1731                let x28 = xreg(28);
1732                let x21wr = writable_xreg(21);
1733                let x23wr = writable_xreg(23);
1734                let x24wr = writable_xreg(24);
1735                let x27wr = writable_xreg(27);
1736                let x28wr = writable_xreg(28);
1737                let again_label = sink.get_label();
1738
1739                // again:
1740                sink.bind_label(again_label, &mut state.ctrl_plane);
1741
1742                if let Some(trap_code) = flags.trap_code() {
1743                    sink.add_trap(trap_code);
1744                }
1745
1746                sink.put4(enc_ldaxp(I64, x27wr, x23wr, x25)); // ldaxp x27, x23, [x25]
1747
1748                match op {
1749                    AtomicRMWLoopOp::Xchg => {} // do nothing
1750                    AtomicRMWLoopOp::Smax
1751                    | AtomicRMWLoopOp::Umax
1752                    | AtomicRMWLoopOp::Smin
1753                    | AtomicRMWLoopOp::Umin => {
1754                        // cmp x27, x26
1755                        // sbcs xzr, x23, x22
1756                        // csel.op x21, x22, x23
1757                        // csel.op x28, x26, x27
1758
1759                        let cond = match op {
1760                            AtomicRMWLoopOp::Smax => Cond::Lt,
1761                            AtomicRMWLoopOp::Umax => Cond::Lo,
1762                            AtomicRMWLoopOp::Smin => Cond::Ge,
1763                            AtomicRMWLoopOp::Umin => Cond::Hs,
1764                            _ => unreachable!(),
1765                        };
1766
1767                        Inst::AluRRR {
1768                            alu_op: ALUOp::SubS,
1769                            size: OperandSize::Size64,
1770                            rd: writable_zero_reg(),
1771                            rn: x27,
1772                            rm: x26,
1773                        }
1774                        .emit(sink, emit_info, state);
1775
1776                        Inst::AluRRR {
1777                            alu_op: ALUOp::SbcS,
1778                            size: OperandSize::Size64,
1779                            rd: writable_zero_reg(),
1780                            rn: x23,
1781                            rm: x22,
1782                        }
1783                        .emit(sink, emit_info, state);
1784
1785                        Inst::CSel {
1786                            cond,
1787                            rd: x21wr,
1788                            rn: x22,
1789                            rm: x23,
1790                        }
1791                        .emit(sink, emit_info, state);
1792
1793                        Inst::CSel {
1794                            cond,
1795                            rd: x28wr,
1796                            rn: x26,
1797                            rm: x27,
1798                        }
1799                        .emit(sink, emit_info, state);
1800                    }
1801                    _ => {
1802                        let (op_lo, op_hi) = match op {
1803                            // adds x28, x27, x26
1804                            // adc x21, x23, x22
1805                            AtomicRMWLoopOp::Add => (ALUOp::AddS, ALUOp::Adc),
1806                            // subs x28, x27, x26
1807                            // sbc x21, x23, x22
1808                            AtomicRMWLoopOp::Sub => (ALUOp::SubS, ALUOp::Sbc),
1809                            // and x28, x27, x26
1810                            // and x21, x23, x22
1811                            AtomicRMWLoopOp::And | AtomicRMWLoopOp::Nand => {
1812                                (ALUOp::And, ALUOp::And)
1813                            }
1814                            // orr x28, x27, x26
1815                            // orr x21, x23, x22
1816                            AtomicRMWLoopOp::Orr => (ALUOp::Orr, ALUOp::Orr),
1817                            // eor x28, x27, x26
1818                            // eor x21, x23, x22
1819                            AtomicRMWLoopOp::Eor => (ALUOp::Eor, ALUOp::Eor),
1820                            _ => unreachable!(),
1821                        };
1822
1823                        Inst::AluRRR {
1824                            alu_op: op_lo,
1825                            size: OperandSize::Size64,
1826                            rd: x28wr,
1827                            rn: x27,
1828                            rm: x26,
1829                        }
1830                        .emit(sink, emit_info, state);
1831
1832                        Inst::AluRRR {
1833                            alu_op: op_hi,
1834                            size: OperandSize::Size64,
1835                            rd: x21wr,
1836                            rn: x23,
1837                            rm: x22,
1838                        }
1839                        .emit(sink, emit_info, state);
1840
1841                        if op == AtomicRMWLoopOp::Nand {
1842                            // mvn x28, x28
1843                            // mvn x21, x21
1844
1845                            Inst::AluRRR {
1846                                alu_op: ALUOp::OrrNot,
1847                                size: OperandSize::Size64,
1848                                rd: x28wr,
1849                                rn: xzr,
1850                                rm: x28,
1851                            }
1852                            .emit(sink, emit_info, state);
1853
1854                            Inst::AluRRR {
1855                                alu_op: ALUOp::OrrNot,
1856                                size: OperandSize::Size64,
1857                                rd: x21wr,
1858                                rn: xzr,
1859                                rm: x21,
1860                            }
1861                            .emit(sink, emit_info, state);
1862                        }
1863                    }
1864                }
1865
1866                if let Some(trap_code) = flags.trap_code() {
1867                    sink.add_trap(trap_code);
1868                }
1869
1870                if op == AtomicRMWLoopOp::Xchg {
1871                    sink.put4(enc_stlxp(I64, x24wr, x26, x22, x25)); // stlxp w24, x26, x22, [x25]
1872                } else {
1873                    sink.put4(enc_stlxp(I64, x24wr, x28, x21, x25)); // stlxp w24, x28, x21, [x25]
1874                }
1875
1876                // cbnz w24, again
1877                // Note, we're actually testing x24, and relying on the default zero-high-half
1878                // rule in the assignment that `stlxp` does.
1879                let br_offset = sink.cur_offset();
1880                sink.put4(enc_conditional_br(
1881                    BranchTarget::Label(again_label),
1882                    CondBrKind::NotZero(x24, OperandSize::Size64),
1883                ));
1884                sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
1885            }
1886            &Inst::AtomicCAS {
1887                rd,
1888                rs,
1889                rt,
1890                rn,
1891                ty,
1892                flags,
1893            } => {
1894                debug_assert_eq!(rd.to_reg(), rs);
1895                let size = match ty {
1896                    I8 => 0b00,
1897                    I16 => 0b01,
1898                    I32 => 0b10,
1899                    I64 => 0b11,
1900                    _ => panic!("Unsupported type: {ty}"),
1901                };
1902
1903                if let Some(trap_code) = flags.trap_code() {
1904                    sink.add_trap(trap_code);
1905                }
1906
1907                sink.put4(enc_cas(size, rd, rt, rn));
1908            }
1909            Inst::AtomicCAS128 { args } => {
1910                let &AtomicCAS128Args {
1911                    rd_lo,
1912                    rd_hi,
1913                    rs_lo,
1914                    rs_hi,
1915                    rt_lo,
1916                    rt_hi,
1917                    rn,
1918                    flags,
1919                } = &**args;
1920                debug_assert_eq!(rd_lo.to_reg(), rs_lo);
1921                debug_assert_eq!(rd_hi.to_reg(), rs_hi);
1922
1923                // These should be pinned to pairs that `casp` requires.
1924                debug_assert_eq!(rs_hi, xreg(machreg_to_gpr(rs_lo) as u8 + 1));
1925                debug_assert_eq!(rt_hi, xreg(machreg_to_gpr(rt_lo) as u8 + 1));
1926
1927                if let Some(trap_code) = flags.trap_code() {
1928                    sink.add_trap(trap_code);
1929                }
1930
1931                sink.put4(enc_casp(rd_lo, rt_lo, rn));
1932            }
1933            &Inst::AtomicCASLoop { ty, flags, .. } => {
1934                /* Emit this:
1935                    again:
1936                     ldaxr{,b,h} x/w27, [x25]
1937                     cmp         x/w27, x/w26 uxt{b,h}
1938                     b.ne        out
1939                     stlxr{,b,h} w24, x/w28, [x25]
1940                     cbnz        x24, again
1941                    out:
1942
1943                  Operand conventions:
1944                     IN:  x25 (addr), x26 (expected value), x28 (replacement value)
1945                     OUT: x27 (old value), x24 (trashed)
1946                */
1947                let x24 = xreg(24);
1948                let x25 = xreg(25);
1949                let x26 = xreg(26);
1950                let x27 = xreg(27);
1951                let x28 = xreg(28);
1952                let xzrwr = writable_zero_reg();
1953                let x24wr = writable_xreg(24);
1954                let x27wr = writable_xreg(27);
1955                let again_label = sink.get_label();
1956                let out_label = sink.get_label();
1957
1958                // again:
1959                sink.bind_label(again_label, &mut state.ctrl_plane);
1960
1961                if let Some(trap_code) = flags.trap_code() {
1962                    sink.add_trap(trap_code);
1963                }
1964
1965                // ldaxr x27, [x25]
1966                sink.put4(enc_ldaxr(ty, x27wr, x25));
1967
1968                // The top 32-bits are zero-extended by the ldaxr so for I64
1969                // we don't have to use UXTW, just the x-form of the
1970                // register. I8/I16 use UXTB/UXTH to only compare their
1971                // respective widths, and I32 uses the w-form of the
1972                // register (`sf` cleared) to only compare the low 32 bits,
1973                // since the expected value may have garbage in its upper
1974                // 32 bits (e.g. left behind by an `ireduce`).
1975                let (sf, bit21, extend_op) = match ty {
1976                    I8 => (0b1, 0b1, 0b000000),
1977                    I16 => (0b1, 0b1, 0b001000),
1978                    I32 => (0b0, 0b0, 0b000000),
1979                    _ => (0b1, 0b0, 0b000000),
1980                };
1981                let bits_31_21 = (sf << 10) | 0b011_01011_000 | bit21;
1982                // cmp x27, x26 (== subs xzr, x27, x26)
1983                sink.put4(enc_arith_rrr(bits_31_21, extend_op, xzrwr, x27, x26));
1984
1985                // b.ne out
1986                let br_out_offset = sink.cur_offset();
1987                sink.put4(enc_conditional_br(
1988                    BranchTarget::Label(out_label),
1989                    CondBrKind::Cond(Cond::Ne),
1990                ));
1991                sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);
1992
1993                if let Some(trap_code) = flags.trap_code() {
1994                    sink.add_trap(trap_code);
1995                }
1996
1997                sink.put4(enc_stlxr(ty, x24wr, x28, x25)); // stlxr w24, x28, [x25]
1998
1999                // cbnz w24, again.
2000                // Note, we're actually testing x24, and relying on the default zero-high-half
2001                // rule in the assignment that `stlxr` does.
2002                let br_again_offset = sink.cur_offset();
2003                sink.put4(enc_conditional_br(
2004                    BranchTarget::Label(again_label),
2005                    CondBrKind::NotZero(x24, OperandSize::Size64),
2006                ));
2007                sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2008
2009                // out:
2010                sink.bind_label(out_label, &mut state.ctrl_plane);
2011            }
2012            &Inst::AtomicCAS128Loop { flags, .. } => {
2013                /* Emit this:
2014                    again:
2015                     ldaxp       x27, x21, [x25]
2016                     cmp         x27, x26
2017                     b.ne        keep
2018                     cmp         x21, x23
2019                     b.ne        keep
2020                     stlxp       w24, x28, x22, [x25]
2021                     cbnz        w24, again
2022                     b           out
2023                    keep:
2024                     stlxp       w24, x27, x21, [x25]
2025                     cbnz        w24, again
2026                    out:
2027
2028                  Operand conventions:
2029                     IN:  x25 (addr), x26 (low bytes of expected value), x23 (high bytes of expected value), x28 (low bytes of replacement value), x22 (high bytes of replacement value)
2030                     OUT: x27 (low bytes of old value), x21 (high bytes of old value), x24 (trashed)
2031                */
2032                let x21 = xreg(21);
2033                let x22 = xreg(22);
2034                let x23 = xreg(23);
2035                let x24 = xreg(24);
2036                let x25 = xreg(25);
2037                let x26 = xreg(26);
2038                let x27 = xreg(27);
2039                let x28 = xreg(28);
2040                let xzrwr = writable_zero_reg();
2041                let x21wr = writable_xreg(21);
2042                let x24wr = writable_xreg(24);
2043                let x27wr = writable_xreg(27);
2044                let again_label = sink.get_label();
2045                let keep_label = sink.get_label();
2046                let out_label = sink.get_label();
2047
2048                // again:
2049                sink.bind_label(again_label, &mut state.ctrl_plane);
2050
2051                if let Some(trap_code) = flags.trap_code() {
2052                    sink.add_trap(trap_code);
2053                }
2054
2055                // ldaxp x27, x21, [x25]
2056                sink.put4(enc_ldaxp(I64, x27wr, x21wr, x25));
2057
2058                // cmp x27, x26
2059                Inst::AluRRR {
2060                    alu_op: ALUOp::SubS,
2061                    size: OperandSize::Size64,
2062                    rd: xzrwr,
2063                    rn: x27,
2064                    rm: x26,
2065                }
2066                .emit(sink, emit_info, state);
2067
2068                // b.ne keep
2069                let br_keep_offset = sink.cur_offset();
2070                sink.put4(enc_conditional_br(
2071                    BranchTarget::Label(keep_label),
2072                    CondBrKind::Cond(Cond::Ne),
2073                ));
2074                sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);
2075
2076                // cmp x21, x23
2077                Inst::AluRRR {
2078                    alu_op: ALUOp::SubS,
2079                    size: OperandSize::Size64,
2080                    rd: xzrwr,
2081                    rn: x21,
2082                    rm: x23,
2083                }
2084                .emit(sink, emit_info, state);
2085
2086                // b.ne keep
2087                let br_keep_offset = sink.cur_offset();
2088                sink.put4(enc_conditional_br(
2089                    BranchTarget::Label(keep_label),
2090                    CondBrKind::Cond(Cond::Ne),
2091                ));
2092                sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);
2093
2094                if let Some(trap_code) = flags.trap_code() {
2095                    sink.add_trap(trap_code);
2096                }
2097
2098                // stlxp w24, x28, x22, [x25]
2099                sink.put4(enc_stlxp(I64, x24wr, x28, x22, x25));
2100
2101                // cbnz w24, again
2102                let br_again_offset = sink.cur_offset();
2103                sink.put4(enc_conditional_br(
2104                    BranchTarget::Label(again_label),
2105                    CondBrKind::NotZero(x24, OperandSize::Size64),
2106                ));
2107                sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2108
2109                // b out
2110                let b_out_offset = sink.cur_offset();
2111                sink.put4(enc_jump26(
2112                    0b000101,
2113                    BranchTarget::Label(out_label).as_offset26_or_zero(),
2114                ));
2115                sink.use_label_at_offset(b_out_offset, out_label, LabelUse::Branch26);
2116
2117                // keep:
2118                sink.bind_label(keep_label, &mut state.ctrl_plane);
2119
2120                if let Some(trap_code) = flags.trap_code() {
2121                    sink.add_trap(trap_code);
2122                }
2123
2124                // stlxp w24, x27, x21, [x25]
2125                sink.put4(enc_stlxp(I64, x24wr, x27, x21, x25));
2126
2127                // cbnz w24, again
2128                let br_again_offset = sink.cur_offset();
2129                sink.put4(enc_conditional_br(
2130                    BranchTarget::Label(again_label),
2131                    CondBrKind::NotZero(x24, OperandSize::Size64),
2132                ));
2133                sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2134
2135                // out:
2136                sink.bind_label(out_label, &mut state.ctrl_plane);
2137            }
2138            &Inst::LoadAcquire {
2139                access_ty,
2140                rt,
2141                rn,
2142                flags,
2143            } => {
2144                if let Some(trap_code) = flags.trap_code() {
2145                    sink.add_trap(trap_code);
2146                }
2147
2148                sink.put4(enc_ldar(access_ty, rt, rn));
2149            }
2150            &Inst::LoadAcquire128 {
2151                rt1,
2152                rt2,
2153                rn,
2154                scratch,
2155                flags,
2156            } => {
2157                let again_label = sink.get_label();
2158
2159                // again:
2160                sink.bind_label(again_label, &mut state.ctrl_plane);
2161
2162                if let Some(trap_code) = flags.trap_code() {
2163                    sink.add_trap(trap_code);
2164                }
2165
2166                // ldaxp rt1, rt2, [rn]
2167                sink.put4(enc_ldaxp(I64, rt1, rt2, rn));
2168
2169                if let Some(trap_code) = flags.trap_code() {
2170                    sink.add_trap(trap_code);
2171                }
2172
2173                // stxp scratch, rt1, rt2, [rn]
2174                sink.put4(enc_stxp(I64, scratch, rt1.to_reg(), rt2.to_reg(), rn));
2175
2176                // cbnz scratch, again.
2177                let br_again_offset = sink.cur_offset();
2178                sink.put4(enc_conditional_br(
2179                    BranchTarget::Label(again_label),
2180                    CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),
2181                ));
2182                sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2183            }
2184            &Inst::StoreRelease {
2185                access_ty,
2186                rt,
2187                rn,
2188                flags,
2189            } => {
2190                if let Some(trap_code) = flags.trap_code() {
2191                    sink.add_trap(trap_code);
2192                }
2193
2194                sink.put4(enc_stlr(access_ty, rt, rn));
2195            }
2196            &Inst::StoreRelease128 {
2197                rt1,
2198                rt2,
2199                rn,
2200                scratch,
2201                flags,
2202            } => {
2203                let again_label = sink.get_label();
2204
2205                // again:
2206                sink.bind_label(again_label, &mut state.ctrl_plane);
2207
2208                if let Some(trap_code) = flags.trap_code() {
2209                    sink.add_trap(trap_code);
2210                }
2211
2212                // ldxp xzr, scratch, [rn]
2213                sink.put4(enc_ldxp(I64, writable_zero_reg(), scratch, rn));
2214
2215                if let Some(trap_code) = flags.trap_code() {
2216                    sink.add_trap(trap_code);
2217                }
2218
2219                // stlxp scratch, rt1, rt2, [rn]
2220                sink.put4(enc_stlxp(I64, scratch, rt1, rt2, rn));
2221
2222                // cbnz scratch, again.
2223                let br_again_offset = sink.cur_offset();
2224                sink.put4(enc_conditional_br(
2225                    BranchTarget::Label(again_label),
2226                    CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),
2227                ));
2228                sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2229            }
2230            &Inst::Fence {} => {
2231                sink.put4(enc_dmb_ish()); // dmb ish
2232            }
2233            &Inst::Csdb {} => {
2234                sink.put4(0xd503229f);
2235            }
2236            &Inst::FpuMove32 { rd, rn } => {
2237                sink.put4(enc_fpurr(0b000_11110_00_1_000000_10000, rd, rn));
2238            }
2239            &Inst::FpuMove64 { rd, rn } => {
2240                sink.put4(enc_fpurr(0b000_11110_01_1_000000_10000, rd, rn));
2241            }
2242            &Inst::FpuMove128 { rd, rn } => {
2243                sink.put4(enc_vecmov(/* 16b = */ true, rd, rn));
2244            }
2245            &Inst::FpuMoveFromVec { rd, rn, idx, size } => {
2246                let (imm5, shift, mask) = match size.lane_size() {
2247                    ScalarSize::Size32 => (0b00100, 3, 0b011),
2248                    ScalarSize::Size64 => (0b01000, 4, 0b001),
2249                    _ => unimplemented!(),
2250                };
2251                debug_assert_eq!(idx & mask, idx);
2252                let imm5 = imm5 | ((idx as u32) << shift);
2253                sink.put4(
2254                    0b010_11110000_00000_000001_00000_00000
2255                        | (imm5 << 16)
2256                        | (machreg_to_vec(rn) << 5)
2257                        | machreg_to_vec(rd.to_reg()),
2258                );
2259            }
2260            &Inst::FpuExtend { rd, rn, size } => {
2261                sink.put4(enc_fpurr(
2262                    0b000_11110_00_1_000000_10000 | (size.ftype() << 12),
2263                    rd,
2264                    rn,
2265                ));
2266            }
2267            &Inst::FpuRR {
2268                fpu_op,
2269                size,
2270                rd,
2271                rn,
2272            } => {
2273                let top22 = match fpu_op {
2274                    FPUOp1::Abs => 0b000_11110_00_1_000001_10000,
2275                    FPUOp1::Neg => 0b000_11110_00_1_000010_10000,
2276                    FPUOp1::Sqrt => 0b000_11110_00_1_000011_10000,
2277                    FPUOp1::Cvt16To32 => {
2278                        debug_assert_eq!(size, ScalarSize::Size16);
2279                        0b000_11110_11_1_000100_10000
2280                    }
2281                    FPUOp1::Cvt32To64 => {
2282                        debug_assert_eq!(size, ScalarSize::Size32);
2283                        0b000_11110_00_1_000101_10000
2284                    }
2285                    FPUOp1::Cvt64To32 => {
2286                        debug_assert_eq!(size, ScalarSize::Size64);
2287                        0b000_11110_01_1_000100_10000
2288                    }
2289                };
2290                let top22 = top22 | size.ftype() << 12;
2291                sink.put4(enc_fpurr(top22, rd, rn));
2292            }
2293            &Inst::FpuRRR {
2294                fpu_op,
2295                size,
2296                rd,
2297                rn,
2298                rm,
2299            } => {
2300                let top22 = match fpu_op {
2301                    FPUOp2::Add => 0b000_11110_00_1_00000_001010,
2302                    FPUOp2::Sub => 0b000_11110_00_1_00000_001110,
2303                    FPUOp2::Mul => 0b000_11110_00_1_00000_000010,
2304                    FPUOp2::Div => 0b000_11110_00_1_00000_000110,
2305                    FPUOp2::Max => 0b000_11110_00_1_00000_010010,
2306                    FPUOp2::Min => 0b000_11110_00_1_00000_010110,
2307                };
2308                let top22 = top22 | size.ftype() << 12;
2309                sink.put4(enc_fpurrr(top22, rd, rn, rm));
2310            }
2311            &Inst::FpuRRI { fpu_op, rd, rn } => match fpu_op {
2312                FPUOpRI::UShr32(imm) => {
2313                    debug_assert_eq!(32, imm.lane_size_in_bits);
2314                    sink.put4(
2315                        0b0_0_1_011110_0000000_00_0_0_0_1_00000_00000
2316                            | imm.enc() << 16
2317                            | machreg_to_vec(rn) << 5
2318                            | machreg_to_vec(rd.to_reg()),
2319                    )
2320                }
2321                FPUOpRI::UShr64(imm) => {
2322                    debug_assert_eq!(64, imm.lane_size_in_bits);
2323                    sink.put4(
2324                        0b01_1_111110_0000000_00_0_0_0_1_00000_00000
2325                            | imm.enc() << 16
2326                            | machreg_to_vec(rn) << 5
2327                            | machreg_to_vec(rd.to_reg()),
2328                    )
2329                }
2330            },
2331            &Inst::FpuRRIMod { fpu_op, rd, ri, rn } => {
2332                debug_assert_eq!(rd.to_reg(), ri);
2333                match fpu_op {
2334                    FPUOpRIMod::Sli64(imm) => {
2335                        debug_assert_eq!(64, imm.lane_size_in_bits);
2336                        sink.put4(
2337                            0b01_1_111110_0000000_010101_00000_00000
2338                                | imm.enc() << 16
2339                                | machreg_to_vec(rn) << 5
2340                                | machreg_to_vec(rd.to_reg()),
2341                        )
2342                    }
2343                    FPUOpRIMod::Sli32(imm) => {
2344                        debug_assert_eq!(32, imm.lane_size_in_bits);
2345                        sink.put4(
2346                            0b0_0_1_011110_0000000_010101_00000_00000
2347                                | imm.enc() << 16
2348                                | machreg_to_vec(rn) << 5
2349                                | machreg_to_vec(rd.to_reg()),
2350                        )
2351                    }
2352                }
2353            }
2354            &Inst::FpuRRRR {
2355                fpu_op,
2356                size,
2357                rd,
2358                rn,
2359                rm,
2360                ra,
2361            } => {
2362                let top17 = match fpu_op {
2363                    FPUOp3::MAdd => 0b000_11111_00_0_00000_0,
2364                    FPUOp3::MSub => 0b000_11111_00_0_00000_1,
2365                    FPUOp3::NMAdd => 0b000_11111_00_1_00000_0,
2366                    FPUOp3::NMSub => 0b000_11111_00_1_00000_1,
2367                };
2368                let top17 = top17 | size.ftype() << 7;
2369                sink.put4(enc_fpurrrr(top17, rd, rn, rm, ra));
2370            }
2371            &Inst::VecMisc { op, rd, rn, size } => {
2372                let (q, enc_size) = size.enc_size();
2373                let (u, bits_12_16, size) = match op {
2374                    VecMisc2::Not => (0b1, 0b00101, 0b00),
2375                    VecMisc2::Neg => (0b1, 0b01011, enc_size),
2376                    VecMisc2::Abs => (0b0, 0b01011, enc_size),
2377                    VecMisc2::Fabs => {
2378                        debug_assert!(
2379                            size == VectorSize::Size32x2
2380                                || size == VectorSize::Size32x4
2381                                || size == VectorSize::Size64x2
2382                        );
2383                        (0b0, 0b01111, enc_size)
2384                    }
2385                    VecMisc2::Fneg => {
2386                        debug_assert!(
2387                            size == VectorSize::Size32x2
2388                                || size == VectorSize::Size32x4
2389                                || size == VectorSize::Size64x2
2390                        );
2391                        (0b1, 0b01111, enc_size)
2392                    }
2393                    VecMisc2::Fsqrt => {
2394                        debug_assert!(
2395                            size == VectorSize::Size32x2
2396                                || size == VectorSize::Size32x4
2397                                || size == VectorSize::Size64x2
2398                        );
2399                        (0b1, 0b11111, enc_size)
2400                    }
2401                    VecMisc2::Rev16 => {
2402                        debug_assert_eq!(size, VectorSize::Size8x16);
2403                        (0b0, 0b00001, enc_size)
2404                    }
2405                    VecMisc2::Rev32 => {
2406                        debug_assert!(size == VectorSize::Size8x16 || size == VectorSize::Size16x8);
2407                        (0b1, 0b00000, enc_size)
2408                    }
2409                    VecMisc2::Rev64 => {
2410                        debug_assert!(
2411                            size == VectorSize::Size8x16
2412                                || size == VectorSize::Size16x8
2413                                || size == VectorSize::Size32x4
2414                        );
2415                        (0b0, 0b00000, enc_size)
2416                    }
2417                    VecMisc2::Fcvtzs => {
2418                        debug_assert!(
2419                            size == VectorSize::Size32x2
2420                                || size == VectorSize::Size32x4
2421                                || size == VectorSize::Size64x2
2422                        );
2423                        (0b0, 0b11011, enc_size)
2424                    }
2425                    VecMisc2::Fcvtzu => {
2426                        debug_assert!(
2427                            size == VectorSize::Size32x2
2428                                || size == VectorSize::Size32x4
2429                                || size == VectorSize::Size64x2
2430                        );
2431                        (0b1, 0b11011, enc_size)
2432                    }
2433                    VecMisc2::Scvtf => {
2434                        debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2);
2435                        (0b0, 0b11101, enc_size & 0b1)
2436                    }
2437                    VecMisc2::Ucvtf => {
2438                        debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2);
2439                        (0b1, 0b11101, enc_size & 0b1)
2440                    }
2441                    VecMisc2::Frintn => {
2442                        debug_assert!(
2443                            size == VectorSize::Size32x2
2444                                || size == VectorSize::Size32x4
2445                                || size == VectorSize::Size64x2
2446                        );
2447                        (0b0, 0b11000, enc_size & 0b01)
2448                    }
2449                    VecMisc2::Frintz => {
2450                        debug_assert!(
2451                            size == VectorSize::Size32x2
2452                                || size == VectorSize::Size32x4
2453                                || size == VectorSize::Size64x2
2454                        );
2455                        (0b0, 0b11001, enc_size)
2456                    }
2457                    VecMisc2::Frintm => {
2458                        debug_assert!(
2459                            size == VectorSize::Size32x2
2460                                || size == VectorSize::Size32x4
2461                                || size == VectorSize::Size64x2
2462                        );
2463                        (0b0, 0b11001, enc_size & 0b01)
2464                    }
2465                    VecMisc2::Frintp => {
2466                        debug_assert!(
2467                            size == VectorSize::Size32x2
2468                                || size == VectorSize::Size32x4
2469                                || size == VectorSize::Size64x2
2470                        );
2471                        (0b0, 0b11000, enc_size)
2472                    }
2473                    VecMisc2::Cnt => {
2474                        debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16);
2475                        (0b0, 0b00101, enc_size)
2476                    }
2477                    VecMisc2::Cmeq0 => (0b0, 0b01001, enc_size),
2478                    VecMisc2::Cmge0 => (0b1, 0b01000, enc_size),
2479                    VecMisc2::Cmgt0 => (0b0, 0b01000, enc_size),
2480                    VecMisc2::Cmle0 => (0b1, 0b01001, enc_size),
2481                    VecMisc2::Cmlt0 => (0b0, 0b01010, enc_size),
2482                    VecMisc2::Fcmeq0 => {
2483                        debug_assert!(
2484                            size == VectorSize::Size32x2
2485                                || size == VectorSize::Size32x4
2486                                || size == VectorSize::Size64x2
2487                        );
2488                        (0b0, 0b01101, enc_size)
2489                    }
2490                    VecMisc2::Fcmge0 => {
2491                        debug_assert!(
2492                            size == VectorSize::Size32x2
2493                                || size == VectorSize::Size32x4
2494                                || size == VectorSize::Size64x2
2495                        );
2496                        (0b1, 0b01100, enc_size)
2497                    }
2498                    VecMisc2::Fcmgt0 => {
2499                        debug_assert!(
2500                            size == VectorSize::Size32x2
2501                                || size == VectorSize::Size32x4
2502                                || size == VectorSize::Size64x2
2503                        );
2504                        (0b0, 0b01100, enc_size)
2505                    }
2506                    VecMisc2::Fcmle0 => {
2507                        debug_assert!(
2508                            size == VectorSize::Size32x2
2509                                || size == VectorSize::Size32x4
2510                                || size == VectorSize::Size64x2
2511                        );
2512                        (0b1, 0b01101, enc_size)
2513                    }
2514                    VecMisc2::Fcmlt0 => {
2515                        debug_assert!(
2516                            size == VectorSize::Size32x2
2517                                || size == VectorSize::Size32x4
2518                                || size == VectorSize::Size64x2
2519                        );
2520                        (0b0, 0b01110, enc_size)
2521                    }
2522                };
2523                sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn));
2524            }
2525            &Inst::VecLanes { op, rd, rn, size } => {
2526                let (q, size) = match size {
2527                    VectorSize::Size8x8 => (0b0, 0b00),
2528                    VectorSize::Size8x16 => (0b1, 0b00),
2529                    VectorSize::Size16x4 => (0b0, 0b01),
2530                    VectorSize::Size16x8 => (0b1, 0b01),
2531                    VectorSize::Size32x4 => (0b1, 0b10),
2532                    _ => unreachable!(),
2533                };
2534                let (u, opcode) = match op {
2535                    VecLanesOp::Uminv => (0b1, 0b11010),
2536                    VecLanesOp::Addv => (0b0, 0b11011),
2537                };
2538                sink.put4(enc_vec_lanes(q, u, size, opcode, rd, rn));
2539            }
2540            &Inst::VecShiftImm {
2541                op,
2542                rd,
2543                rn,
2544                size,
2545                imm,
2546            } => {
2547                let (is_shr, mut template) = match op {
2548                    VecShiftImmOp::Ushr => (true, 0b_001_011110_0000_000_000001_00000_00000_u32),
2549                    VecShiftImmOp::Sshr => (true, 0b_000_011110_0000_000_000001_00000_00000_u32),
2550                    VecShiftImmOp::Shl => (false, 0b_000_011110_0000_000_010101_00000_00000_u32),
2551                };
2552                if size.is_128bits() {
2553                    template |= 0b1 << 30;
2554                }
2555                let imm = imm as u32;
2556                // Deal with the somewhat strange encoding scheme for, and limits on,
2557                // the shift amount.
2558                let immh_immb = match (size.lane_size(), is_shr) {
2559                    (ScalarSize::Size64, true) if imm >= 1 && imm <= 64 => {
2560                        0b_1000_000_u32 | (64 - imm)
2561                    }
2562                    (ScalarSize::Size32, true) if imm >= 1 && imm <= 32 => {
2563                        0b_0100_000_u32 | (32 - imm)
2564                    }
2565                    (ScalarSize::Size16, true) if imm >= 1 && imm <= 16 => {
2566                        0b_0010_000_u32 | (16 - imm)
2567                    }
2568                    (ScalarSize::Size8, true) if imm >= 1 && imm <= 8 => {
2569                        0b_0001_000_u32 | (8 - imm)
2570                    }
2571                    (ScalarSize::Size64, false) if imm <= 63 => 0b_1000_000_u32 | imm,
2572                    (ScalarSize::Size32, false) if imm <= 31 => 0b_0100_000_u32 | imm,
2573                    (ScalarSize::Size16, false) if imm <= 15 => 0b_0010_000_u32 | imm,
2574                    (ScalarSize::Size8, false) if imm <= 7 => 0b_0001_000_u32 | imm,
2575                    _ => panic!(
2576                        "aarch64: Inst::VecShiftImm: emit: invalid op/size/imm {op:?}, {size:?}, {imm:?}"
2577                    ),
2578                };
2579                let rn_enc = machreg_to_vec(rn);
2580                let rd_enc = machreg_to_vec(rd.to_reg());
2581                sink.put4(template | (immh_immb << 16) | (rn_enc << 5) | rd_enc);
2582            }
2583            &Inst::VecShiftImmMod {
2584                op,
2585                rd,
2586                ri,
2587                rn,
2588                size,
2589                imm,
2590            } => {
2591                debug_assert_eq!(rd.to_reg(), ri);
2592                let (is_shr, mut template) = match op {
2593                    VecShiftImmModOp::Sli => (false, 0b_001_011110_0000_000_010101_00000_00000_u32),
2594                };
2595                if size.is_128bits() {
2596                    template |= 0b1 << 30;
2597                }
2598                let imm = imm as u32;
2599                // Deal with the somewhat strange encoding scheme for, and limits on,
2600                // the shift amount.
2601                let immh_immb = match (size.lane_size(), is_shr) {
2602                    (ScalarSize::Size64, true) if imm >= 1 && imm <= 64 => {
2603                        0b_1000_000_u32 | (64 - imm)
2604                    }
2605                    (ScalarSize::Size32, true) if imm >= 1 && imm <= 32 => {
2606                        0b_0100_000_u32 | (32 - imm)
2607                    }
2608                    (ScalarSize::Size16, true) if imm >= 1 && imm <= 16 => {
2609                        0b_0010_000_u32 | (16 - imm)
2610                    }
2611                    (ScalarSize::Size8, true) if imm >= 1 && imm <= 8 => {
2612                        0b_0001_000_u32 | (8 - imm)
2613                    }
2614                    (ScalarSize::Size64, false) if imm <= 63 => 0b_1000_000_u32 | imm,
2615                    (ScalarSize::Size32, false) if imm <= 31 => 0b_0100_000_u32 | imm,
2616                    (ScalarSize::Size16, false) if imm <= 15 => 0b_0010_000_u32 | imm,
2617                    (ScalarSize::Size8, false) if imm <= 7 => 0b_0001_000_u32 | imm,
2618                    _ => panic!(
2619                        "aarch64: Inst::VecShiftImmMod: emit: invalid op/size/imm {op:?}, {size:?}, {imm:?}"
2620                    ),
2621                };
2622                let rn_enc = machreg_to_vec(rn);
2623                let rd_enc = machreg_to_vec(rd.to_reg());
2624                sink.put4(template | (immh_immb << 16) | (rn_enc << 5) | rd_enc);
2625            }
2626            &Inst::VecExtract { rd, rn, rm, imm4 } => {
2627                if imm4 < 16 {
2628                    let template = 0b_01_101110_000_00000_0_0000_0_00000_00000_u32;
2629                    let rm_enc = machreg_to_vec(rm);
2630                    let rn_enc = machreg_to_vec(rn);
2631                    let rd_enc = machreg_to_vec(rd.to_reg());
2632                    sink.put4(
2633                        template | (rm_enc << 16) | ((imm4 as u32) << 11) | (rn_enc << 5) | rd_enc,
2634                    );
2635                } else {
2636                    panic!("aarch64: Inst::VecExtract: emit: invalid extract index {imm4}");
2637                }
2638            }
2639            &Inst::VecTbl { rd, rn, rm } => {
2640                sink.put4(enc_tbl(/* is_extension = */ false, 0b00, rd, rn, rm));
2641            }
2642            &Inst::VecTblExt { rd, ri, rn, rm } => {
2643                debug_assert_eq!(rd.to_reg(), ri);
2644                sink.put4(enc_tbl(/* is_extension = */ true, 0b00, rd, rn, rm));
2645            }
2646            &Inst::VecTbl2 { rd, rn, rn2, rm } => {
2647                assert_eq!(machreg_to_vec(rn2), (machreg_to_vec(rn) + 1) % 32);
2648                sink.put4(enc_tbl(/* is_extension = */ false, 0b01, rd, rn, rm));
2649            }
2650            &Inst::VecTbl2Ext {
2651                rd,
2652                ri,
2653                rn,
2654                rn2,
2655                rm,
2656            } => {
2657                debug_assert_eq!(rd.to_reg(), ri);
2658                assert_eq!(machreg_to_vec(rn2), (machreg_to_vec(rn) + 1) % 32);
2659                sink.put4(enc_tbl(/* is_extension = */ true, 0b01, rd, rn, rm));
2660            }
2661            &Inst::FpuCmp { size, rn, rm } => {
2662                sink.put4(enc_fcmp(size, rn, rm));
2663            }
2664            &Inst::FpuToInt { op, rd, rn } => {
2665                let top16 = match op {
2666                    // FCVTZS (32/32-bit)
2667                    FpuToIntOp::F32ToI32 => 0b000_11110_00_1_11_000,
2668                    // FCVTZU (32/32-bit)
2669                    FpuToIntOp::F32ToU32 => 0b000_11110_00_1_11_001,
2670                    // FCVTZS (32/64-bit)
2671                    FpuToIntOp::F32ToI64 => 0b100_11110_00_1_11_000,
2672                    // FCVTZU (32/64-bit)
2673                    FpuToIntOp::F32ToU64 => 0b100_11110_00_1_11_001,
2674                    // FCVTZS (64/32-bit)
2675                    FpuToIntOp::F64ToI32 => 0b000_11110_01_1_11_000,
2676                    // FCVTZU (64/32-bit)
2677                    FpuToIntOp::F64ToU32 => 0b000_11110_01_1_11_001,
2678                    // FCVTZS (64/64-bit)
2679                    FpuToIntOp::F64ToI64 => 0b100_11110_01_1_11_000,
2680                    // FCVTZU (64/64-bit)
2681                    FpuToIntOp::F64ToU64 => 0b100_11110_01_1_11_001,
2682                };
2683                sink.put4(enc_fputoint(top16, rd, rn));
2684            }
2685            &Inst::IntToFpu { op, rd, rn } => {
2686                let top16 = match op {
2687                    // SCVTF (32/32-bit)
2688                    IntToFpuOp::I32ToF32 => 0b000_11110_00_1_00_010,
2689                    // UCVTF (32/32-bit)
2690                    IntToFpuOp::U32ToF32 => 0b000_11110_00_1_00_011,
2691                    // SCVTF (64/32-bit)
2692                    IntToFpuOp::I64ToF32 => 0b100_11110_00_1_00_010,
2693                    // UCVTF (64/32-bit)
2694                    IntToFpuOp::U64ToF32 => 0b100_11110_00_1_00_011,
2695                    // SCVTF (32/64-bit)
2696                    IntToFpuOp::I32ToF64 => 0b000_11110_01_1_00_010,
2697                    // UCVTF (32/64-bit)
2698                    IntToFpuOp::U32ToF64 => 0b000_11110_01_1_00_011,
2699                    // SCVTF (64/64-bit)
2700                    IntToFpuOp::I64ToF64 => 0b100_11110_01_1_00_010,
2701                    // UCVTF (64/64-bit)
2702                    IntToFpuOp::U64ToF64 => 0b100_11110_01_1_00_011,
2703                };
2704                sink.put4(enc_inttofpu(top16, rd, rn));
2705            }
2706            &Inst::FpuCSel16 { rd, rn, rm, cond } => {
2707                sink.put4(enc_fcsel(rd, rn, rm, cond, ScalarSize::Size16));
2708            }
2709            &Inst::FpuCSel32 { rd, rn, rm, cond } => {
2710                sink.put4(enc_fcsel(rd, rn, rm, cond, ScalarSize::Size32));
2711            }
2712            &Inst::FpuCSel64 { rd, rn, rm, cond } => {
2713                sink.put4(enc_fcsel(rd, rn, rm, cond, ScalarSize::Size64));
2714            }
2715            &Inst::FpuRound { op, rd, rn } => {
2716                let top22 = match op {
2717                    FpuRoundMode::Minus32 => 0b000_11110_00_1_001_010_10000,
2718                    FpuRoundMode::Minus64 => 0b000_11110_01_1_001_010_10000,
2719                    FpuRoundMode::Plus32 => 0b000_11110_00_1_001_001_10000,
2720                    FpuRoundMode::Plus64 => 0b000_11110_01_1_001_001_10000,
2721                    FpuRoundMode::Zero32 => 0b000_11110_00_1_001_011_10000,
2722                    FpuRoundMode::Zero64 => 0b000_11110_01_1_001_011_10000,
2723                    FpuRoundMode::Nearest32 => 0b000_11110_00_1_001_000_10000,
2724                    FpuRoundMode::Nearest64 => 0b000_11110_01_1_001_000_10000,
2725                };
2726                sink.put4(enc_fround(top22, rd, rn));
2727            }
2728            &Inst::MovToFpu { rd, rn, size } => {
2729                let template = match size {
2730                    ScalarSize::Size16 => 0b000_11110_11_1_00_111_000000_00000_00000,
2731                    ScalarSize::Size32 => 0b000_11110_00_1_00_111_000000_00000_00000,
2732                    ScalarSize::Size64 => 0b100_11110_01_1_00_111_000000_00000_00000,
2733                    _ => unreachable!(),
2734                };
2735                sink.put4(template | (machreg_to_gpr(rn) << 5) | machreg_to_vec(rd.to_reg()));
2736            }
2737            &Inst::FpuMoveFPImm { rd, imm, size } => {
2738                sink.put4(
2739                    0b000_11110_00_1_00_000_000100_00000_00000
2740                        | size.ftype() << 22
2741                        | ((imm.enc_bits() as u32) << 13)
2742                        | machreg_to_vec(rd.to_reg()),
2743                );
2744            }
2745            &Inst::MovToVec {
2746                rd,
2747                ri,
2748                rn,
2749                idx,
2750                size,
2751            } => {
2752                debug_assert_eq!(rd.to_reg(), ri);
2753                let (imm5, shift) = match size.lane_size() {
2754                    ScalarSize::Size8 => (0b00001, 1),
2755                    ScalarSize::Size16 => (0b00010, 2),
2756                    ScalarSize::Size32 => (0b00100, 3),
2757                    ScalarSize::Size64 => (0b01000, 4),
2758                    _ => unreachable!(),
2759                };
2760                debug_assert_eq!(idx & (0b11111 >> shift), idx);
2761                let imm5 = imm5 | ((idx as u32) << shift);
2762                sink.put4(
2763                    0b010_01110000_00000_0_0011_1_00000_00000
2764                        | (imm5 << 16)
2765                        | (machreg_to_gpr(rn) << 5)
2766                        | machreg_to_vec(rd.to_reg()),
2767                );
2768            }
2769            &Inst::MovFromVec { rd, rn, idx, size } => {
2770                let (q, imm5, shift, mask) = match size {
2771                    ScalarSize::Size8 => (0b0, 0b00001, 1, 0b1111),
2772                    ScalarSize::Size16 => (0b0, 0b00010, 2, 0b0111),
2773                    ScalarSize::Size32 => (0b0, 0b00100, 3, 0b0011),
2774                    ScalarSize::Size64 => (0b1, 0b01000, 4, 0b0001),
2775                    _ => panic!("Unexpected scalar FP operand size: {size:?}"),
2776                };
2777                debug_assert_eq!(idx & mask, idx);
2778                let imm5 = imm5 | ((idx as u32) << shift);
2779                sink.put4(
2780                    0b000_01110000_00000_0_0111_1_00000_00000
2781                        | (q << 30)
2782                        | (imm5 << 16)
2783                        | (machreg_to_vec(rn) << 5)
2784                        | machreg_to_gpr(rd.to_reg()),
2785                );
2786            }
2787            &Inst::MovFromVecSigned {
2788                rd,
2789                rn,
2790                idx,
2791                size,
2792                scalar_size,
2793            } => {
2794                let (imm5, shift, half) = match size {
2795                    VectorSize::Size8x8 => (0b00001, 1, true),
2796                    VectorSize::Size8x16 => (0b00001, 1, false),
2797                    VectorSize::Size16x4 => (0b00010, 2, true),
2798                    VectorSize::Size16x8 => (0b00010, 2, false),
2799                    VectorSize::Size32x2 => {
2800                        debug_assert_ne!(scalar_size, OperandSize::Size32);
2801                        (0b00100, 3, true)
2802                    }
2803                    VectorSize::Size32x4 => {
2804                        debug_assert_ne!(scalar_size, OperandSize::Size32);
2805                        (0b00100, 3, false)
2806                    }
2807                    _ => panic!("Unexpected vector operand size"),
2808                };
2809                debug_assert_eq!(idx & (0b11111 >> (half as u32 + shift)), idx);
2810                let imm5 = imm5 | ((idx as u32) << shift);
2811                sink.put4(
2812                    0b000_01110000_00000_0_0101_1_00000_00000
2813                        | (scalar_size.is64() as u32) << 30
2814                        | (imm5 << 16)
2815                        | (machreg_to_vec(rn) << 5)
2816                        | machreg_to_gpr(rd.to_reg()),
2817                );
2818            }
2819            &Inst::VecDup { rd, rn, size } => {
2820                let q = size.is_128bits() as u32;
2821                let imm5 = match size.lane_size() {
2822                    ScalarSize::Size8 => 0b00001,
2823                    ScalarSize::Size16 => 0b00010,
2824                    ScalarSize::Size32 => 0b00100,
2825                    ScalarSize::Size64 => 0b01000,
2826                    _ => unreachable!(),
2827                };
2828                sink.put4(
2829                    0b0_0_0_01110000_00000_000011_00000_00000
2830                        | (q << 30)
2831                        | (imm5 << 16)
2832                        | (machreg_to_gpr(rn) << 5)
2833                        | machreg_to_vec(rd.to_reg()),
2834                );
2835            }
2836            &Inst::VecDupFromFpu { rd, rn, size, lane } => {
2837                let q = size.is_128bits() as u32;
2838                let imm5 = match size.lane_size() {
2839                    ScalarSize::Size8 => {
2840                        assert!(lane < 16);
2841                        0b00001 | (u32::from(lane) << 1)
2842                    }
2843                    ScalarSize::Size16 => {
2844                        assert!(lane < 8);
2845                        0b00010 | (u32::from(lane) << 2)
2846                    }
2847                    ScalarSize::Size32 => {
2848                        assert!(lane < 4);
2849                        0b00100 | (u32::from(lane) << 3)
2850                    }
2851                    ScalarSize::Size64 => {
2852                        assert!(lane < 2);
2853                        0b01000 | (u32::from(lane) << 4)
2854                    }
2855                    _ => unimplemented!(),
2856                };
2857                sink.put4(
2858                    0b000_01110000_00000_000001_00000_00000
2859                        | (q << 30)
2860                        | (imm5 << 16)
2861                        | (machreg_to_vec(rn) << 5)
2862                        | machreg_to_vec(rd.to_reg()),
2863                );
2864            }
2865            &Inst::VecDupFPImm { rd, imm, size } => {
2866                let imm = imm.enc_bits();
2867                let op = match size.lane_size() {
2868                    ScalarSize::Size32 => 0,
2869                    ScalarSize::Size64 => 1,
2870                    _ => unimplemented!(),
2871                };
2872                let q_op = op | ((size.is_128bits() as u32) << 1);
2873
2874                sink.put4(enc_asimd_mod_imm(rd, q_op, 0b1111, imm));
2875            }
2876            &Inst::VecDupImm {
2877                rd,
2878                imm,
2879                invert,
2880                size,
2881            } => {
2882                let (imm, shift, shift_ones) = imm.value();
2883                let (op, cmode) = match size.lane_size() {
2884                    ScalarSize::Size8 => {
2885                        assert!(!invert);
2886                        assert_eq!(shift, 0);
2887
2888                        (0, 0b1110)
2889                    }
2890                    ScalarSize::Size16 => {
2891                        let s = shift & 8;
2892
2893                        assert!(!shift_ones);
2894                        assert_eq!(s, shift);
2895
2896                        (invert as u32, 0b1000 | (s >> 2))
2897                    }
2898                    ScalarSize::Size32 => {
2899                        if shift_ones {
2900                            assert!(shift == 8 || shift == 16);
2901
2902                            (invert as u32, 0b1100 | (shift >> 4))
2903                        } else {
2904                            let s = shift & 24;
2905
2906                            assert_eq!(s, shift);
2907
2908                            (invert as u32, 0b0000 | (s >> 2))
2909                        }
2910                    }
2911                    ScalarSize::Size64 => {
2912                        assert!(!invert);
2913                        assert_eq!(shift, 0);
2914
2915                        (1, 0b1110)
2916                    }
2917                    _ => unreachable!(),
2918                };
2919                let q_op = op | ((size.is_128bits() as u32) << 1);
2920
2921                sink.put4(enc_asimd_mod_imm(rd, q_op, cmode, imm));
2922            }
2923            &Inst::VecExtend {
2924                t,
2925                rd,
2926                rn,
2927                high_half,
2928                lane_size,
2929            } => {
2930                let immh = match lane_size {
2931                    ScalarSize::Size16 => 0b001,
2932                    ScalarSize::Size32 => 0b010,
2933                    ScalarSize::Size64 => 0b100,
2934                    _ => panic!("Unexpected VecExtend to lane size of {lane_size:?}"),
2935                };
2936                let u = match t {
2937                    VecExtendOp::Sxtl => 0b0,
2938                    VecExtendOp::Uxtl => 0b1,
2939                };
2940                sink.put4(
2941                    0b000_011110_0000_000_101001_00000_00000
2942                        | ((high_half as u32) << 30)
2943                        | (u << 29)
2944                        | (immh << 19)
2945                        | (machreg_to_vec(rn) << 5)
2946                        | machreg_to_vec(rd.to_reg()),
2947                );
2948            }
2949            &Inst::VecRRLong {
2950                op,
2951                rd,
2952                rn,
2953                high_half,
2954            } => {
2955                let (u, size, bits_12_16) = match op {
2956                    VecRRLongOp::Fcvtl16 => (0b0, 0b00, 0b10111),
2957                    VecRRLongOp::Fcvtl32 => (0b0, 0b01, 0b10111),
2958                    VecRRLongOp::Shll8 => (0b1, 0b00, 0b10011),
2959                    VecRRLongOp::Shll16 => (0b1, 0b01, 0b10011),
2960                    VecRRLongOp::Shll32 => (0b1, 0b10, 0b10011),
2961                };
2962
2963                sink.put4(enc_vec_rr_misc(
2964                    ((high_half as u32) << 1) | u,
2965                    size,
2966                    bits_12_16,
2967                    rd,
2968                    rn,
2969                ));
2970            }
2971            &Inst::VecRRNarrowLow {
2972                op,
2973                rd,
2974                rn,
2975                lane_size,
2976            }
2977            | &Inst::VecRRNarrowHigh {
2978                op,
2979                rd,
2980                rn,
2981                lane_size,
2982                ..
2983            } => {
2984                let high_half = match self {
2985                    &Inst::VecRRNarrowLow { .. } => false,
2986                    &Inst::VecRRNarrowHigh { .. } => true,
2987                    _ => unreachable!(),
2988                };
2989
2990                let size = match lane_size {
2991                    ScalarSize::Size8 => 0b00,
2992                    ScalarSize::Size16 => 0b01,
2993                    ScalarSize::Size32 => 0b10,
2994                    _ => panic!("unsupported size: {lane_size:?}"),
2995                };
2996
2997                // Floats use a single bit, to encode either half or single.
2998                let size = match op {
2999                    VecRRNarrowOp::Fcvtn => size >> 1,
3000                    _ => size,
3001                };
3002
3003                let (u, bits_12_16) = match op {
3004                    VecRRNarrowOp::Xtn => (0b0, 0b10010),
3005                    VecRRNarrowOp::Sqxtn => (0b0, 0b10100),
3006                    VecRRNarrowOp::Sqxtun => (0b1, 0b10010),
3007                    VecRRNarrowOp::Uqxtn => (0b1, 0b10100),
3008                    VecRRNarrowOp::Fcvtn => (0b0, 0b10110),
3009                };
3010
3011                sink.put4(enc_vec_rr_misc(
3012                    ((high_half as u32) << 1) | u,
3013                    size,
3014                    bits_12_16,
3015                    rd,
3016                    rn,
3017                ));
3018            }
3019            &Inst::VecMovElement {
3020                rd,
3021                ri,
3022                rn,
3023                dest_idx,
3024                src_idx,
3025                size,
3026            } => {
3027                debug_assert_eq!(rd.to_reg(), ri);
3028                let (imm5, shift) = match size.lane_size() {
3029                    ScalarSize::Size8 => (0b00001, 1),
3030                    ScalarSize::Size16 => (0b00010, 2),
3031                    ScalarSize::Size32 => (0b00100, 3),
3032                    ScalarSize::Size64 => (0b01000, 4),
3033                    _ => unreachable!(),
3034                };
3035                let mask = 0b11111 >> shift;
3036                debug_assert_eq!(dest_idx & mask, dest_idx);
3037                debug_assert_eq!(src_idx & mask, src_idx);
3038                let imm4 = (src_idx as u32) << (shift - 1);
3039                let imm5 = imm5 | ((dest_idx as u32) << shift);
3040                sink.put4(
3041                    0b011_01110000_00000_0_0000_1_00000_00000
3042                        | (imm5 << 16)
3043                        | (imm4 << 11)
3044                        | (machreg_to_vec(rn) << 5)
3045                        | machreg_to_vec(rd.to_reg()),
3046                );
3047            }
3048            &Inst::VecRRPair { op, rd, rn } => {
3049                let bits_12_16 = match op {
3050                    VecPairOp::Addp => 0b11011,
3051                };
3052
3053                sink.put4(enc_vec_rr_pair(bits_12_16, rd, rn));
3054            }
3055            &Inst::VecRRRLong {
3056                rd,
3057                rn,
3058                rm,
3059                alu_op,
3060                high_half,
3061            } => {
3062                let (u, size, bit14) = match alu_op {
3063                    VecRRRLongOp::Smull8 => (0b0, 0b00, 0b1),
3064                    VecRRRLongOp::Smull16 => (0b0, 0b01, 0b1),
3065                    VecRRRLongOp::Smull32 => (0b0, 0b10, 0b1),
3066                    VecRRRLongOp::Umull8 => (0b1, 0b00, 0b1),
3067                    VecRRRLongOp::Umull16 => (0b1, 0b01, 0b1),
3068                    VecRRRLongOp::Umull32 => (0b1, 0b10, 0b1),
3069                };
3070                sink.put4(enc_vec_rrr_long(
3071                    high_half as u32,
3072                    u,
3073                    size,
3074                    bit14,
3075                    rm,
3076                    rn,
3077                    rd,
3078                ));
3079            }
3080            &Inst::VecRRRLongMod {
3081                rd,
3082                ri,
3083                rn,
3084                rm,
3085                alu_op,
3086                high_half,
3087            } => {
3088                debug_assert_eq!(rd.to_reg(), ri);
3089                let (u, size, bit14) = match alu_op {
3090                    VecRRRLongModOp::Umlal8 => (0b1, 0b00, 0b0),
3091                    VecRRRLongModOp::Umlal16 => (0b1, 0b01, 0b0),
3092                    VecRRRLongModOp::Umlal32 => (0b1, 0b10, 0b0),
3093                };
3094                sink.put4(enc_vec_rrr_long(
3095                    high_half as u32,
3096                    u,
3097                    size,
3098                    bit14,
3099                    rm,
3100                    rn,
3101                    rd,
3102                ));
3103            }
3104            &Inst::VecRRPairLong { op, rd, rn } => {
3105                let (u, size) = match op {
3106                    VecRRPairLongOp::Saddlp8 => (0b0, 0b0),
3107                    VecRRPairLongOp::Uaddlp8 => (0b1, 0b0),
3108                    VecRRPairLongOp::Saddlp16 => (0b0, 0b1),
3109                    VecRRPairLongOp::Uaddlp16 => (0b1, 0b1),
3110                };
3111
3112                sink.put4(enc_vec_rr_pair_long(u, size, rd, rn));
3113            }
3114            &Inst::VecRRR {
3115                rd,
3116                rn,
3117                rm,
3118                alu_op,
3119                size,
3120            } => {
3121                let (q, enc_size) = size.enc_size();
3122                let is_float = match alu_op {
3123                    VecALUOp::Fcmeq
3124                    | VecALUOp::Fcmgt
3125                    | VecALUOp::Fcmge
3126                    | VecALUOp::Fadd
3127                    | VecALUOp::Fsub
3128                    | VecALUOp::Fdiv
3129                    | VecALUOp::Fmax
3130                    | VecALUOp::Fmin
3131                    | VecALUOp::Fmul => true,
3132                    _ => false,
3133                };
3134
3135                let (top11, bit15_10) = match alu_op {
3136                    VecALUOp::Sqadd => (0b000_01110_00_1 | enc_size << 1, 0b000011),
3137                    VecALUOp::Sqsub => (0b000_01110_00_1 | enc_size << 1, 0b001011),
3138                    VecALUOp::Uqadd => (0b001_01110_00_1 | enc_size << 1, 0b000011),
3139                    VecALUOp::Uqsub => (0b001_01110_00_1 | enc_size << 1, 0b001011),
3140                    VecALUOp::Cmeq => (0b001_01110_00_1 | enc_size << 1, 0b100011),
3141                    VecALUOp::Cmge => (0b000_01110_00_1 | enc_size << 1, 0b001111),
3142                    VecALUOp::Cmgt => (0b000_01110_00_1 | enc_size << 1, 0b001101),
3143                    VecALUOp::Cmhi => (0b001_01110_00_1 | enc_size << 1, 0b001101),
3144                    VecALUOp::Cmhs => (0b001_01110_00_1 | enc_size << 1, 0b001111),
3145                    VecALUOp::Fcmeq => (0b000_01110_00_1, 0b111001),
3146                    VecALUOp::Fcmgt => (0b001_01110_10_1, 0b111001),
3147                    VecALUOp::Fcmge => (0b001_01110_00_1, 0b111001),
3148                    // The following logical instructions operate on bytes, so are not encoded differently
3149                    // for the different vector types.
3150                    VecALUOp::And => (0b000_01110_00_1, 0b000111),
3151                    VecALUOp::Bic => (0b000_01110_01_1, 0b000111),
3152                    VecALUOp::Orr => (0b000_01110_10_1, 0b000111),
3153                    VecALUOp::Orn => (0b000_01110_11_1, 0b000111),
3154                    VecALUOp::Eor => (0b001_01110_00_1, 0b000111),
3155                    VecALUOp::Umaxp => {
3156                        debug_assert_ne!(size, VectorSize::Size64x2);
3157
3158                        (0b001_01110_00_1 | enc_size << 1, 0b101001)
3159                    }
3160                    VecALUOp::Add => (0b000_01110_00_1 | enc_size << 1, 0b100001),
3161                    VecALUOp::Sub => (0b001_01110_00_1 | enc_size << 1, 0b100001),
3162                    VecALUOp::Mul => {
3163                        debug_assert_ne!(size, VectorSize::Size64x2);
3164                        (0b000_01110_00_1 | enc_size << 1, 0b100111)
3165                    }
3166                    VecALUOp::Sshl => (0b000_01110_00_1 | enc_size << 1, 0b010001),
3167                    VecALUOp::Ushl => (0b001_01110_00_1 | enc_size << 1, 0b010001),
3168                    VecALUOp::Umin => {
3169                        debug_assert_ne!(size, VectorSize::Size64x2);
3170
3171                        (0b001_01110_00_1 | enc_size << 1, 0b011011)
3172                    }
3173                    VecALUOp::Smin => {
3174                        debug_assert_ne!(size, VectorSize::Size64x2);
3175
3176                        (0b000_01110_00_1 | enc_size << 1, 0b011011)
3177                    }
3178                    VecALUOp::Umax => {
3179                        debug_assert_ne!(size, VectorSize::Size64x2);
3180
3181                        (0b001_01110_00_1 | enc_size << 1, 0b011001)
3182                    }
3183                    VecALUOp::Smax => {
3184                        debug_assert_ne!(size, VectorSize::Size64x2);
3185
3186                        (0b000_01110_00_1 | enc_size << 1, 0b011001)
3187                    }
3188                    VecALUOp::Urhadd => {
3189                        debug_assert_ne!(size, VectorSize::Size64x2);
3190
3191                        (0b001_01110_00_1 | enc_size << 1, 0b000101)
3192                    }
3193                    VecALUOp::Fadd => (0b000_01110_00_1, 0b110101),
3194                    VecALUOp::Fsub => (0b000_01110_10_1, 0b110101),
3195                    VecALUOp::Fdiv => (0b001_01110_00_1, 0b111111),
3196                    VecALUOp::Fmax => (0b000_01110_00_1, 0b111101),
3197                    VecALUOp::Fmin => (0b000_01110_10_1, 0b111101),
3198                    VecALUOp::Fmul => (0b001_01110_00_1, 0b110111),
3199                    VecALUOp::Addp => (0b000_01110_00_1 | enc_size << 1, 0b101111),
3200                    VecALUOp::Zip1 => (0b01001110_00_0 | enc_size << 1, 0b001110),
3201                    VecALUOp::Zip2 => (0b01001110_00_0 | enc_size << 1, 0b011110),
3202                    VecALUOp::Sqrdmulh => {
3203                        debug_assert!(
3204                            size.lane_size() == ScalarSize::Size16
3205                                || size.lane_size() == ScalarSize::Size32
3206                        );
3207
3208                        (0b001_01110_00_1 | enc_size << 1, 0b101101)
3209                    }
3210                    VecALUOp::Uzp1 => (0b01001110_00_0 | enc_size << 1, 0b000110),
3211                    VecALUOp::Uzp2 => (0b01001110_00_0 | enc_size << 1, 0b010110),
3212                    VecALUOp::Trn1 => (0b01001110_00_0 | enc_size << 1, 0b001010),
3213                    VecALUOp::Trn2 => (0b01001110_00_0 | enc_size << 1, 0b011010),
3214                };
3215                let top11 = if is_float {
3216                    top11 | size.enc_float_size() << 1
3217                } else {
3218                    top11
3219                };
3220                sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
3221            }
3222            &Inst::VecRRRMod {
3223                rd,
3224                ri,
3225                rn,
3226                rm,
3227                alu_op,
3228                size,
3229            } => {
3230                debug_assert_eq!(rd.to_reg(), ri);
3231                let (q, _enc_size) = size.enc_size();
3232
3233                let (top11, bit15_10) = match alu_op {
3234                    VecALUModOp::Bsl => (0b001_01110_01_1, 0b000111),
3235                    VecALUModOp::Fmla => {
3236                        (0b000_01110_00_1 | (size.enc_float_size() << 1), 0b110011)
3237                    }
3238                    VecALUModOp::Fmls => {
3239                        (0b000_01110_10_1 | (size.enc_float_size() << 1), 0b110011)
3240                    }
3241                    // SDOT Vd.4S, Vn.16B, Vm.16B (FEAT_DotProd). The size/element
3242                    // field (bits 23:22 = 0b10) is part of the dot-product opcode,
3243                    // so it is baked into top11; only Q (from `size`) is variable.
3244                    // top11 (Q=0) | q<<9 with bit15_10 yields 0x4E809400 for .4S/.16B.
3245                    VecALUModOp::Sdot => (0b000_01110_10_0, 0b100101),
3246                    // USDOT Vd.4S, Vn.16B, Vm.16B (FEAT_I8MM). Same shape as
3247                    // SDOT; only the opcode field differs.
3248                    VecALUModOp::Usdot => (0b000_01110_10_0, 0b100111),
3249                };
3250                sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
3251            }
3252            &Inst::VecFmlaElem {
3253                rd,
3254                ri,
3255                rn,
3256                rm,
3257                alu_op,
3258                size,
3259                idx,
3260            } => {
3261                debug_assert_eq!(rd.to_reg(), ri);
3262                let idx = u32::from(idx);
3263
3264                let (q, _size) = size.enc_size();
3265                let o2 = match alu_op {
3266                    VecALUModOp::Fmla => 0b0,
3267                    VecALUModOp::Fmls => 0b1,
3268                    _ => unreachable!(),
3269                };
3270
3271                let (h, l) = match size {
3272                    VectorSize::Size32x4 => {
3273                        assert!(idx < 4);
3274                        (idx >> 1, idx & 1)
3275                    }
3276                    VectorSize::Size64x2 => {
3277                        assert!(idx < 2);
3278                        (idx, 0)
3279                    }
3280                    _ => unreachable!(),
3281                };
3282
3283                let top11 = 0b000_011111_00 | (q << 9) | (size.enc_float_size() << 1) | l;
3284                let bit15_10 = 0b000100 | (o2 << 4) | (h << 1);
3285                sink.put4(enc_vec_rrr(top11, rm, bit15_10, rn, rd));
3286            }
3287            &Inst::VecLoadReplicate {
3288                rd,
3289                rn,
3290                size,
3291                flags,
3292            } => {
3293                let (q, size) = size.enc_size();
3294
3295                if let Some(trap_code) = flags.trap_code() {
3296                    // Register the offset at which the actual load instruction starts.
3297                    sink.add_trap(trap_code);
3298                }
3299
3300                sink.put4(enc_ldst_vec(q, size, rn, rd));
3301            }
3302            &Inst::VecCSel { rd, rn, rm, cond } => {
3303                /* Emit this:
3304                      b.cond  else
3305                      mov     rd, rm
3306                      b       out
3307                     else:
3308                      mov     rd, rn
3309                     out:
3310
3311                   Note, we could do better in the cases where rd == rn or rd == rm.
3312                */
3313                let else_label = sink.get_label();
3314                let out_label = sink.get_label();
3315
3316                // b.cond else
3317                let br_else_offset = sink.cur_offset();
3318                sink.put4(enc_conditional_br(
3319                    BranchTarget::Label(else_label),
3320                    CondBrKind::Cond(cond),
3321                ));
3322                sink.use_label_at_offset(br_else_offset, else_label, LabelUse::Branch19);
3323
3324                // mov rd, rm
3325                sink.put4(enc_vecmov(/* 16b = */ true, rd, rm));
3326
3327                // b out
3328                let b_out_offset = sink.cur_offset();
3329                sink.use_label_at_offset(b_out_offset, out_label, LabelUse::Branch26);
3330                sink.add_uncond_branch(b_out_offset, b_out_offset + 4, out_label);
3331                sink.put4(enc_jump26(0b000101, 0 /* will be fixed up later */));
3332
3333                // else:
3334                sink.bind_label(else_label, &mut state.ctrl_plane);
3335
3336                // mov rd, rn
3337                sink.put4(enc_vecmov(/* 16b = */ true, rd, rn));
3338
3339                // out:
3340                sink.bind_label(out_label, &mut state.ctrl_plane);
3341            }
3342            &Inst::MovToNZCV { rn } => {
3343                sink.put4(0xd51b4200 | machreg_to_gpr(rn));
3344            }
3345            &Inst::MovFromNZCV { rd } => {
3346                sink.put4(0xd53b4200 | machreg_to_gpr(rd.to_reg()));
3347            }
3348            &Inst::BitfieldMove {
3349                size,
3350                bfm_op,
3351                rd,
3352                rn,
3353                immr,
3354                imms,
3355            } => {
3356                let opc = bfm_op.opc();
3357                sink.put4(enc_bfm(opc, size, rd, rn, immr.value(), imms.value()));
3358            }
3359            &Inst::BitfieldMoveMod {
3360                size,
3361                rd,
3362                ri,
3363                rn,
3364                immr,
3365                imms,
3366            } => {
3367                debug_assert_eq!(rd.to_reg(), ri);
3368                sink.put4(enc_bfm(0b01, size, rd, rn, immr.value(), imms.value()));
3369            }
3370            &Inst::Jump { ref dest } => {
3371                let off = sink.cur_offset();
3372                // Indicate that the jump uses a label, if so, so that a fixup can occur later.
3373                if let Some(l) = dest.as_label() {
3374                    sink.use_label_at_offset(off, l, LabelUse::Branch26);
3375                    sink.add_uncond_branch(off, off + 4, l);
3376                }
3377                // Emit the jump itself.
3378                sink.put4(enc_jump26(0b000101, dest.as_offset26_or_zero()));
3379            }
3380            &Inst::Args { .. } | &Inst::Rets { .. } => {
3381                // Nothing: this is a pseudoinstruction that serves
3382                // only to constrain registers at a certain point.
3383            }
3384            &Inst::Ret {} => {
3385                sink.put4(0xd65f03c0);
3386            }
3387            &Inst::AuthenticatedRet { key, is_hint } => {
3388                let (op2, is_hint) = match key {
3389                    APIKey::AZ => (0b100, true),
3390                    APIKey::ASP => (0b101, is_hint),
3391                    APIKey::BZ => (0b110, true),
3392                    APIKey::BSP => (0b111, is_hint),
3393                };
3394
3395                if is_hint {
3396                    sink.put4(key.enc_auti_hint());
3397                    Inst::Ret {}.emit(sink, emit_info, state);
3398                } else {
3399                    sink.put4(0xd65f0bff | (op2 << 9)); // reta{key}
3400                }
3401            }
3402            &Inst::Call { ref info } => {
3403                let start = sink.cur_offset();
3404                sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
3405                sink.put4(enc_jump26(0b100101, 0));
3406                if let Some(s) = state.stack_maps.take(sink) {
3407                    let offset = sink.cur_offset();
3408                    state.stack_maps.push(sink, &state.frame_layout, offset, s);
3409                }
3410
3411                if let Some(try_call) = info.try_call_info.as_ref() {
3412                    sink.add_try_call_site(
3413                        Some(state.frame_layout.sp_to_fp()),
3414                        try_call.exception_handlers(&state.frame_layout),
3415                    );
3416                } else {
3417                    sink.add_call_site();
3418                }
3419
3420                if info.callee_pop_size > 0 {
3421                    let callee_pop_size =
3422                        i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB");
3423                    for inst in AArch64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) {
3424                        inst.emit(sink, emit_info, state);
3425                    }
3426                }
3427
3428                if info.patchable {
3429                    sink.add_patchable_call_site(sink.cur_offset() - start);
3430                } else {
3431                    // Load any stack-carried return values.
3432                    info.emit_retval_loads::<AArch64MachineDeps, _, _>(
3433                        state.frame_layout().stackslots_size,
3434                        |inst| inst.emit(sink, emit_info, state),
3435                        |needed_space| Some(Inst::EmitIsland { needed_space }),
3436                    );
3437                }
3438
3439                // If this is a try-call, jump to the continuation
3440                // (normal-return) block.
3441                if let Some(try_call) = info.try_call_info.as_ref() {
3442                    let jmp = Inst::Jump {
3443                        dest: BranchTarget::Label(try_call.continuation),
3444                    };
3445                    jmp.emit(sink, emit_info, state);
3446                }
3447
3448                // We produce an island above if needed, so disable
3449                // the worst-case-size check in this case.
3450                start_off = sink.cur_offset();
3451            }
3452            &Inst::CallInd { ref info } => {
3453                sink.put4(
3454                    0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(info.dest) << 5),
3455                );
3456                if let Some(s) = state.stack_maps.take(sink) {
3457                    let offset = sink.cur_offset();
3458                    state.stack_maps.push(sink, &state.frame_layout, offset, s);
3459                }
3460
3461                if let Some(try_call) = info.try_call_info.as_ref() {
3462                    sink.add_try_call_site(
3463                        Some(state.frame_layout.sp_to_fp()),
3464                        try_call.exception_handlers(&state.frame_layout),
3465                    );
3466                } else {
3467                    sink.add_call_site();
3468                }
3469
3470                if info.callee_pop_size > 0 {
3471                    let callee_pop_size =
3472                        i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB");
3473                    for inst in AArch64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) {
3474                        inst.emit(sink, emit_info, state);
3475                    }
3476                }
3477
3478                // Load any stack-carried return values.
3479                info.emit_retval_loads::<AArch64MachineDeps, _, _>(
3480                    state.frame_layout().stackslots_size,
3481                    |inst| inst.emit(sink, emit_info, state),
3482                    |needed_space| Some(Inst::EmitIsland { needed_space }),
3483                );
3484
3485                // If this is a try-call, jump to the continuation
3486                // (normal-return) block.
3487                if let Some(try_call) = info.try_call_info.as_ref() {
3488                    let jmp = Inst::Jump {
3489                        dest: BranchTarget::Label(try_call.continuation),
3490                    };
3491                    jmp.emit(sink, emit_info, state);
3492                }
3493
3494                // We produce an island above if needed, so disable
3495                // the worst-case-size check in this case.
3496                start_off = sink.cur_offset();
3497            }
3498            &Inst::ReturnCall { ref info } => {
3499                emit_return_call_common_sequence(sink, emit_info, state, info);
3500
3501                // Note: this is not `Inst::Jump { .. }.emit(..)` because we
3502                // have different metadata in this case: we don't have a label
3503                // for the target, but rather a function relocation.
3504                sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
3505                sink.put4(enc_jump26(0b000101, 0));
3506                sink.add_call_site();
3507
3508                // `emit_return_call_common_sequence` emits an island if
3509                // necessary, so we can safely disable the worst-case-size check
3510                // in this case.
3511                start_off = sink.cur_offset();
3512            }
3513            &Inst::ReturnCallInd { ref info } => {
3514                emit_return_call_common_sequence(sink, emit_info, state, info);
3515
3516                // Indirect tail calls must use x16 or x17 when BTI is enabled
3517                // so that they are compatible with the callee's `BTI C`
3518                // landing pad. Other registers are only accepted by `BTI J`,
3519                // which is not a valid landing pad for a function entry.
3520                //
3521                // x16 is the spill-temp register, which by Cranelift
3522                // convention we are allowed to use as a scratch register
3523                // within one VCode inst, so this is safe.
3524                let dest = if emit_info.isa_flags.use_bti() {
3525                    let dest = xreg(16);
3526                    if info.dest != dest {
3527                        Inst::Mov {
3528                            size: OperandSize::Size64,
3529                            rd: writable_xreg(16),
3530                            rm: info.dest,
3531                        }
3532                        .emit(sink, emit_info, state);
3533                    }
3534                    dest
3535                } else {
3536                    info.dest
3537                };
3538
3539                Inst::IndirectBr {
3540                    rn: dest,
3541                    targets: vec![],
3542                }
3543                .emit(sink, emit_info, state);
3544                sink.add_call_site();
3545
3546                // `emit_return_call_common_sequence` emits an island if
3547                // necessary, so we can safely disable the worst-case-size check
3548                // in this case.
3549                start_off = sink.cur_offset();
3550            }
3551            &Inst::CondBr {
3552                taken,
3553                not_taken,
3554                kind,
3555            } => {
3556                // Conditional part first.
3557                let cond_off = sink.cur_offset();
3558                if let Some(l) = taken.as_label() {
3559                    sink.use_label_at_offset(cond_off, l, LabelUse::Branch19);
3560                    let inverted = enc_conditional_br(taken, kind.invert()).to_le_bytes();
3561                    sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
3562                }
3563                sink.put4(enc_conditional_br(taken, kind));
3564
3565                // Unconditional part next.
3566                let uncond_off = sink.cur_offset();
3567                if let Some(l) = not_taken.as_label() {
3568                    sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
3569                    sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
3570                }
3571                sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
3572            }
3573            &Inst::TestBitAndBranch {
3574                taken,
3575                not_taken,
3576                kind,
3577                rn,
3578                bit,
3579            } => {
3580                // Emit the conditional branch first
3581                let cond_off = sink.cur_offset();
3582                if let Some(l) = taken.as_label() {
3583                    sink.use_label_at_offset(cond_off, l, LabelUse::Branch14);
3584                    let inverted =
3585                        enc_test_bit_and_branch(kind.complement(), taken, rn, bit).to_le_bytes();
3586                    sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
3587                }
3588                sink.put4(enc_test_bit_and_branch(kind, taken, rn, bit));
3589
3590                // Unconditional part next.
3591                let uncond_off = sink.cur_offset();
3592                if let Some(l) = not_taken.as_label() {
3593                    sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
3594                    sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
3595                }
3596                sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
3597            }
3598            &Inst::TrapIf { kind, trap_code } => {
3599                let label = sink.defer_trap(trap_code);
3600                // condbr KIND, LABEL
3601                let off = sink.cur_offset();
3602                sink.put4(enc_conditional_br(BranchTarget::Label(label), kind));
3603                sink.use_label_at_offset(off, label, LabelUse::Branch19);
3604            }
3605            &Inst::IndirectBr { rn, .. } => {
3606                sink.put4(enc_br(rn));
3607            }
3608            &Inst::Nop0 => {}
3609            &Inst::Nop4 => {
3610                sink.put4(0xd503201f);
3611            }
3612            &Inst::Brk => {
3613                sink.put4(0xd43e0000);
3614            }
3615            &Inst::Udf { trap_code } => {
3616                sink.add_trap(trap_code);
3617                sink.put_data(Inst::TRAP_OPCODE);
3618            }
3619            &Inst::Adr { rd, off } => {
3620                assert!(off > -(1 << 20));
3621                assert!(off < (1 << 20));
3622                sink.put4(enc_adr(off, rd));
3623            }
3624            &Inst::Adrp { rd, off } => {
3625                assert!(off > -(1 << 20));
3626                assert!(off < (1 << 20));
3627                sink.put4(enc_adrp(off, rd));
3628            }
3629            &Inst::Word4 { data } => {
3630                sink.put4(data);
3631            }
3632            &Inst::Word8 { data } => {
3633                sink.put8(data);
3634            }
3635            &Inst::JTSequence {
3636                ridx,
3637                rtmp1,
3638                rtmp2,
3639                default,
3640                ref targets,
3641                ..
3642            } => {
3643                // This sequence is *one* instruction in the vcode, and is expanded only here at
3644                // emission time, because we cannot allow the regalloc to insert spills/reloads in
3645                // the middle; we depend on hardcoded PC-rel addressing below.
3646
3647                // The default target is stored as the last entry in the
3648                // jump table (at index `targets.len()`), and we clamp the
3649                // index to that entry when it is out of bounds. This avoids
3650                // any conditional branch to the default target: such a
3651                // branch would have limited range, and the inline jump
3652                // table that follows it can be arbitrarily large.
3653                //
3654                // The flags are set by a prior comparison of `ridx` against
3655                // `targets.len()`; "lo" (unsigned less-than) means in-bounds.
3656                let default_index = u64::try_from(targets.len()).unwrap();
3657                for inst in Inst::load_constant(rtmp2, default_index) {
3658                    inst.emit(sink, emit_info, state);
3659                }
3660                // Clamp the index: in-bounds indices are kept, and
3661                // out-of-bounds indices select the default entry. This is
3662                // also the Spectre mitigation, since there is no
3663                // conditional branch to mispredict.
3664                let inst = Inst::CSel {
3665                    rd: rtmp2,
3666                    cond: Cond::Lo,
3667                    rn: ridx,
3668                    rm: rtmp2.to_reg(),
3669                };
3670                inst.emit(sink, emit_info, state);
3671                // Prevent any data value speculation if spectre mitigations are
3672                // enabled.
3673                if emit_info.flags.enable_table_access_spectre_mitigation()
3674                    && emit_info.isa_flags.use_csdb()
3675                {
3676                    Inst::Csdb.emit(sink, emit_info, state);
3677                }
3678
3679                // Load address of jump table
3680                let inst = Inst::Adr { rd: rtmp1, off: 16 };
3681                inst.emit(sink, emit_info, state);
3682                // Load value out of jump table
3683                let inst = Inst::SLoad32 {
3684                    rd: rtmp2,
3685                    mem: AMode::reg_plus_reg_scaled_extended(
3686                        rtmp1.to_reg(),
3687                        rtmp2.to_reg(),
3688                        ExtendOp::UXTW,
3689                    ),
3690                    flags: MemFlagsData::trusted(),
3691                };
3692                inst.emit(sink, emit_info, state);
3693                // Add base of jump table to jump-table-sourced block offset
3694                let inst = Inst::AluRRR {
3695                    alu_op: ALUOp::Add,
3696                    size: OperandSize::Size64,
3697                    rd: rtmp1,
3698                    rn: rtmp1.to_reg(),
3699                    rm: rtmp2.to_reg(),
3700                };
3701                inst.emit(sink, emit_info, state);
3702                // Branch to computed address. (`targets` here is only used for successor queries
3703                // and is not needed for emission.)
3704                let inst = Inst::IndirectBr {
3705                    rn: rtmp1.to_reg(),
3706                    targets: vec![],
3707                };
3708                inst.emit(sink, emit_info, state);
3709                // Emit jump table (table of 32-bit offsets), with the
3710                // default target as the final entry.
3711                let jt_off = sink.cur_offset();
3712                for &target in targets.iter().chain(core::iter::once(&default)) {
3713                    let word_off = sink.cur_offset();
3714                    // off_into_table is an addend here embedded in the label to be later patched
3715                    // at the end of codegen. The offset is initially relative to this jump table
3716                    // entry; with the extra addend, it'll be relative to the jump table's start,
3717                    // after patching.
3718                    let off_into_table = word_off - jt_off;
3719                    sink.use_label_at_offset(word_off, target, LabelUse::PCRel32);
3720                    sink.put4(off_into_table);
3721                }
3722
3723                // Lowering produces an EmitIsland before using a JTSequence, so we can safely
3724                // disable the worst-case-size check in this case.
3725                start_off = sink.cur_offset();
3726            }
3727            &Inst::LoadExtNameGot { rd, ref name } => {
3728                // See this CE Example for the variations of this with and without BTI & PAUTH
3729                // https://godbolt.org/z/ncqjbbvvn
3730                //
3731                // Emit the following code:
3732                //   adrp    rd, :got:X
3733                //   ldr     rd, [rd, :got_lo12:X]
3734
3735                // adrp rd, symbol
3736                sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);
3737                let inst = Inst::Adrp { rd, off: 0 };
3738                inst.emit(sink, emit_info, state);
3739
3740                // ldr rd, [rd, :got_lo12:X]
3741                sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);
3742                let inst = Inst::ULoad64 {
3743                    rd,
3744                    mem: AMode::reg(rd.to_reg()),
3745                    flags: MemFlagsData::trusted(),
3746                };
3747                inst.emit(sink, emit_info, state);
3748            }
3749            &Inst::LoadExtNameNear {
3750                rd,
3751                ref name,
3752                offset,
3753            } => {
3754                // Emit the following code:
3755                //   adrp    rd, X
3756                //   add     rd, rd, :lo12:X
3757                //
3758                // See https://godbolt.org/z/855KEvM5r for an example.
3759
3760                // adrp rd, symbol
3761                sink.add_reloc(Reloc::Aarch64AdrPrelPgHi21, &**name, offset);
3762                let inst = Inst::Adrp { rd, off: 0 };
3763                inst.emit(sink, emit_info, state);
3764
3765                // add rd, rd, :lo12:X
3766                sink.add_reloc(Reloc::Aarch64AddAbsLo12Nc, &**name, offset);
3767                let inst = Inst::AluRRImm12 {
3768                    alu_op: ALUOp::Add,
3769                    size: OperandSize::Size64,
3770                    rd,
3771                    rn: rd.to_reg(),
3772                    imm12: Imm12::ZERO,
3773                };
3774                inst.emit(sink, emit_info, state);
3775            }
3776            &Inst::LoadExtNameFar {
3777                rd,
3778                ref name,
3779                offset,
3780            } => {
3781                // With absolute offsets we set up a load from a preallocated space, and then jump
3782                // over it.
3783                //
3784                // Emit the following code:
3785                //   ldr     rd, #8
3786                //   b       #0x10
3787                //   <8 byte space>
3788
3789                let inst = Inst::ULoad64 {
3790                    rd,
3791                    mem: AMode::Label {
3792                        label: MemLabel::PCRel(8),
3793                    },
3794                    flags: MemFlagsData::trusted(),
3795                };
3796                inst.emit(sink, emit_info, state);
3797                let inst = Inst::Jump {
3798                    dest: BranchTarget::ResolvedOffset(12),
3799                };
3800                inst.emit(sink, emit_info, state);
3801                sink.add_reloc(Reloc::Abs8, &**name, offset);
3802                sink.put8(0);
3803            }
3804            &Inst::LoadAddr { rd, ref mem } => {
3805                let mem = mem.clone();
3806                let (mem_insts, mem) = mem_finalize(Some(sink), &mem, I8, state);
3807                for inst in mem_insts.into_iter() {
3808                    inst.emit(sink, emit_info, state);
3809                }
3810
3811                let (reg, index_reg, offset) = match mem {
3812                    AMode::RegExtended { rn, rm, extendop } => {
3813                        let r = rn;
3814                        (r, Some((rm, extendop)), 0)
3815                    }
3816                    AMode::Unscaled { rn, simm9 } => {
3817                        let r = rn;
3818                        (r, None, simm9.value())
3819                    }
3820                    AMode::UnsignedOffset { rn, uimm12 } => {
3821                        let r = rn;
3822                        (r, None, uimm12.value() as i32)
3823                    }
3824                    _ => panic!("Unsupported case for LoadAddr: {mem:?}"),
3825                };
3826                let abs_offset = if offset < 0 {
3827                    -offset as u64
3828                } else {
3829                    offset as u64
3830                };
3831                let alu_op = if offset < 0 { ALUOp::Sub } else { ALUOp::Add };
3832
3833                if let Some((idx, extendop)) = index_reg {
3834                    let add = Inst::AluRRRExtend {
3835                        alu_op: ALUOp::Add,
3836                        size: OperandSize::Size64,
3837                        rd,
3838                        rn: reg,
3839                        rm: idx,
3840                        extendop,
3841                    };
3842
3843                    add.emit(sink, emit_info, state);
3844                } else if offset == 0 {
3845                    if reg != rd.to_reg() {
3846                        let mov = Inst::Mov {
3847                            size: OperandSize::Size64,
3848                            rd,
3849                            rm: reg,
3850                        };
3851
3852                        mov.emit(sink, emit_info, state);
3853                    }
3854                } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
3855                    let add = Inst::AluRRImm12 {
3856                        alu_op,
3857                        size: OperandSize::Size64,
3858                        rd,
3859                        rn: reg,
3860                        imm12,
3861                    };
3862                    add.emit(sink, emit_info, state);
3863                } else {
3864                    // Use `tmp2` here: `reg` may be `spilltmp` if the `AMode` on this instruction
3865                    // was initially an `SPOffset`. Assert that `tmp2` is truly free to use. Note
3866                    // that no other instructions will be inserted here (we're emitting directly),
3867                    // and a live range of `tmp2` should not span this instruction, so this use
3868                    // should otherwise be correct.
3869                    debug_assert!(rd.to_reg() != tmp2_reg());
3870                    debug_assert!(reg != tmp2_reg());
3871                    let tmp = writable_tmp2_reg();
3872                    for insn in Inst::load_constant(tmp, abs_offset).into_iter() {
3873                        insn.emit(sink, emit_info, state);
3874                    }
3875                    let add = Inst::AluRRR {
3876                        alu_op,
3877                        size: OperandSize::Size64,
3878                        rd,
3879                        rn: reg,
3880                        rm: tmp.to_reg(),
3881                    };
3882                    add.emit(sink, emit_info, state);
3883                }
3884            }
3885            &Inst::Paci { key } => {
3886                let (crm, op2) = match key {
3887                    APIKey::AZ => (0b0011, 0b000),
3888                    APIKey::ASP => (0b0011, 0b001),
3889                    APIKey::BZ => (0b0011, 0b010),
3890                    APIKey::BSP => (0b0011, 0b011),
3891                };
3892
3893                sink.put4(0xd503211f | (crm << 8) | (op2 << 5));
3894            }
3895            &Inst::Xpaclri => sink.put4(0xd50320ff),
3896            &Inst::Bti { targets } => {
3897                let targets = match targets {
3898                    BranchTargetType::None => 0b00,
3899                    BranchTargetType::C => 0b01,
3900                    BranchTargetType::J => 0b10,
3901                    BranchTargetType::JC => 0b11,
3902                };
3903
3904                sink.put4(0xd503241f | targets << 6);
3905            }
3906            &Inst::EmitIsland { needed_space } => {
3907                if sink.island_needed(needed_space + 4) {
3908                    let jump_around_label = sink.get_label();
3909                    let jmp = Inst::Jump {
3910                        dest: BranchTarget::Label(jump_around_label),
3911                    };
3912                    jmp.emit(sink, emit_info, state);
3913                    sink.emit_island(needed_space + 4, &mut state.ctrl_plane);
3914                    sink.bind_label(jump_around_label, &mut state.ctrl_plane);
3915                }
3916            }
3917
3918            &Inst::ElfTlsGetAddr {
3919                ref symbol,
3920                rd,
3921                tmp,
3922            } => {
3923                assert_eq!(xreg(0), rd.to_reg());
3924
3925                // See the original proposal for TLSDESC.
3926                // http://www.fsfla.org/~lxoliva/writeups/TLS/paper-lk2006.pdf
3927                //
3928                // Implement the TLSDESC instruction sequence:
3929                //   adrp x0, :tlsdesc:tlsvar
3930                //   ldr  tmp, [x0, :tlsdesc_lo12:tlsvar]
3931                //   add  x0, x0, :tlsdesc_lo12:tlsvar
3932                //   blr  tmp
3933                //   mrs  tmp, tpidr_el0
3934                //   add  x0, x0, tmp
3935                //
3936                // This is the instruction sequence that GCC emits for ELF GD TLS Relocations in aarch64
3937                // See: https://gcc.godbolt.org/z/e4j7MdErh
3938
3939                // adrp x0, :tlsdesc:tlsvar
3940                sink.add_reloc(Reloc::Aarch64TlsDescAdrPage21, &**symbol, 0);
3941                Inst::Adrp { rd, off: 0 }.emit(sink, emit_info, state);
3942
3943                // ldr  tmp, [x0, :tlsdesc_lo12:tlsvar]
3944                sink.add_reloc(Reloc::Aarch64TlsDescLd64Lo12, &**symbol, 0);
3945                Inst::ULoad64 {
3946                    rd: tmp,
3947                    mem: AMode::reg(rd.to_reg()),
3948                    flags: MemFlagsData::trusted(),
3949                }
3950                .emit(sink, emit_info, state);
3951
3952                // add x0, x0, :tlsdesc_lo12:tlsvar
3953                sink.add_reloc(Reloc::Aarch64TlsDescAddLo12, &**symbol, 0);
3954                Inst::AluRRImm12 {
3955                    alu_op: ALUOp::Add,
3956                    size: OperandSize::Size64,
3957                    rd,
3958                    rn: rd.to_reg(),
3959                    imm12: Imm12::maybe_from_u64(0).unwrap(),
3960                }
3961                .emit(sink, emit_info, state);
3962
3963                // blr tmp
3964                sink.add_reloc(Reloc::Aarch64TlsDescCall, &**symbol, 0);
3965                Inst::CallInd {
3966                    info: crate::isa::Box::new(CallInfo::empty(tmp.to_reg(), CallConv::SystemV)),
3967                }
3968                .emit(sink, emit_info, state);
3969
3970                // mrs tmp, tpidr_el0
3971                sink.put4(0xd53bd040 | machreg_to_gpr(tmp.to_reg()));
3972
3973                // add x0, x0, tmp
3974                Inst::AluRRR {
3975                    alu_op: ALUOp::Add,
3976                    size: OperandSize::Size64,
3977                    rd,
3978                    rn: rd.to_reg(),
3979                    rm: tmp.to_reg(),
3980                }
3981                .emit(sink, emit_info, state);
3982            }
3983
3984            &Inst::MachOTlsGetAddr { ref symbol, rd } => {
3985                // Each thread local variable gets a descriptor, where the first xword of the descriptor is a pointer
3986                // to a function that takes the descriptor address in x0, and after the function returns x0
3987                // contains the address for the thread local variable
3988                //
3989                // what we want to emit is basically:
3990                //
3991                // adrp x0, <label>@TLVPPAGE  ; Load the address of the page of the thread local variable pointer (TLVP)
3992                // ldr x0, [x0, <label>@TLVPPAGEOFF] ; Load the descriptor's address into x0
3993                // ldr x1, [x0] ; Load the function pointer (the first part of the descriptor)
3994                // blr x1 ; Call the function pointer with the descriptor address in x0
3995                // ; x0 now contains the TLV address
3996
3997                assert_eq!(xreg(0), rd.to_reg());
3998                let rtmp = writable_xreg(1);
3999
4000                // adrp x0, <label>@TLVPPAGE
4001                sink.add_reloc(Reloc::MachOAarch64TlsAdrPage21, symbol, 0);
4002                sink.put4(0x90000000);
4003
4004                // ldr x0, [x0, <label>@TLVPPAGEOFF]
4005                sink.add_reloc(Reloc::MachOAarch64TlsAdrPageOff12, symbol, 0);
4006                sink.put4(0xf9400000);
4007
4008                // load [x0] into temp register
4009                Inst::ULoad64 {
4010                    rd: rtmp,
4011                    mem: AMode::reg(rd.to_reg()),
4012                    flags: MemFlagsData::trusted(),
4013                }
4014                .emit(sink, emit_info, state);
4015
4016                // call function pointer in temp register
4017                Inst::CallInd {
4018                    info: crate::isa::Box::new(CallInfo::empty(
4019                        rtmp.to_reg(),
4020                        CallConv::AppleAarch64,
4021                    )),
4022                }
4023                .emit(sink, emit_info, state);
4024            }
4025
4026            &Inst::Unwind { ref inst } => {
4027                sink.add_unwind(inst.clone());
4028            }
4029
4030            &Inst::DummyUse { .. } => {}
4031
4032            &Inst::LabelAddress { dst, label } => {
4033                // We emit an ADR only, which is +/- 2MiB range. This
4034                // should be sufficient for the typical use-case of
4035                // this instruction, which is insmall trampolines to
4036                // get exception-handler addresses.
4037                let inst = Inst::Adr { rd: dst, off: 0 };
4038                let offset = sink.cur_offset();
4039                inst.emit(sink, emit_info, state);
4040                sink.use_label_at_offset(offset, label, LabelUse::Adr21);
4041            }
4042
4043            &Inst::SequencePoint { .. } => {
4044                // Nothing.
4045            }
4046
4047            &Inst::StackProbeLoop { start, end, step } => {
4048                assert!(emit_info.flags.enable_probestack());
4049
4050                // The loop generated here uses `start` as a counter register to
4051                // count backwards until negating it exceeds `end`. In other
4052                // words `start` is an offset from `sp` we're testing where
4053                // `end` is the max size we need to test. The loop looks like:
4054                //
4055                //      loop_start:
4056                //          sub start, start, #step
4057                //          stur xzr, [sp, start]
4058                //          cmn start, end
4059                //          br.gt loop_start
4060                //      loop_end:
4061                //
4062                // Note that this loop cannot use the spilltmp and tmp2
4063                // registers as those are currently used as the input to this
4064                // loop when generating the instruction. This means that some
4065                // more flavorful address modes and lowerings need to be
4066                // avoided.
4067                //
4068                // Perhaps someone more clever than I can figure out how to use
4069                // `subs` or the like and skip the `cmn`, but I can't figure it
4070                // out at this time.
4071
4072                let loop_start = sink.get_label();
4073                sink.bind_label(loop_start, &mut state.ctrl_plane);
4074
4075                Inst::AluRRImm12 {
4076                    alu_op: ALUOp::Sub,
4077                    size: OperandSize::Size64,
4078                    rd: start,
4079                    rn: start.to_reg(),
4080                    imm12: step,
4081                }
4082                .emit(sink, emit_info, state);
4083                Inst::Store32 {
4084                    rd: regs::zero_reg(),
4085                    mem: AMode::RegReg {
4086                        rn: regs::stack_reg(),
4087                        rm: start.to_reg(),
4088                    },
4089                    flags: MemFlagsData::trusted(),
4090                }
4091                .emit(sink, emit_info, state);
4092                Inst::AluRRR {
4093                    alu_op: ALUOp::AddS,
4094                    size: OperandSize::Size64,
4095                    rd: regs::writable_zero_reg(),
4096                    rn: start.to_reg(),
4097                    rm: end,
4098                }
4099                .emit(sink, emit_info, state);
4100
4101                let loop_end = sink.get_label();
4102                Inst::CondBr {
4103                    taken: BranchTarget::Label(loop_start),
4104                    not_taken: BranchTarget::Label(loop_end),
4105                    kind: CondBrKind::Cond(Cond::Gt),
4106                }
4107                .emit(sink, emit_info, state);
4108                sink.bind_label(loop_end, &mut state.ctrl_plane);
4109            }
4110
4111            &Inst::InterruptPoll {
4112                dst,
4113                load_ptr,
4114                trap_code,
4115                ..
4116            } => {
4117                // Note: this may emit a NOP, so it must come before recording
4118                // the trap below.
4119                let stack_map = state.stack_maps.take(sink);
4120
4121                // Record the address of the load in the trap table so a signal
4122                // handler can later distinguish whether a segfault is its
4123                // fault.
4124                sink.add_trap(trap_code);
4125
4126                // SIGSEGV reports the PC of the faulting load, so the stack map
4127                // is associated with the load itself.
4128                if let Some(stack_map) = stack_map {
4129                    let offset = sink.cur_offset();
4130                    state
4131                        .stack_maps
4132                        .push(sink, &state.frame_layout, offset, stack_map);
4133                }
4134
4135                // Emit `ldr dst, [load_ptr]`. Reuse the `dst` address as the
4136                // destination of the dead load, since we are clobbering it
4137                // anyway.
4138                Inst::ULoad64 {
4139                    rd: dst,
4140                    mem: AMode::UnsignedOffset {
4141                        rn: load_ptr,
4142                        uimm12: UImm12Scaled::zero(I64),
4143                    },
4144                    flags: MemFlagsData::trusted(),
4145                }
4146                .emit(sink, emit_info, state);
4147            }
4148        }
4149
4150        let end_off = sink.cur_offset();
4151        debug_assert!(
4152            (end_off - start_off) <= Inst::worst_case_size()
4153                || matches!(self, Inst::EmitIsland { .. }),
4154            "Worst case size exceed for {:?}: {}",
4155            self,
4156            end_off - start_off
4157        );
4158    }
4159
4160    fn pretty_print_inst(&self, state: &mut Self::State) -> String {
4161        self.print_with_state(state)
4162    }
4163}
4164
4165fn emit_return_call_common_sequence<T>(
4166    sink: &mut MachBuffer<Inst>,
4167    emit_info: &EmitInfo,
4168    state: &mut EmitState,
4169    info: &ReturnCallInfo<T>,
4170) {
4171    for inst in AArch64MachineDeps::gen_clobber_restore(
4172        CallConv::Tail,
4173        &emit_info.flags,
4174        state.frame_layout(),
4175    ) {
4176        inst.emit(sink, emit_info, state);
4177    }
4178
4179    let setup_area_size = state.frame_layout().setup_area_size;
4180    if setup_area_size > 0 {
4181        // N.B.: sp is already adjusted to the appropriate place by the
4182        // clobber-restore code (which also frees the fixed frame). Hence, there
4183        // is no need for the usual `mov sp, fp` here.
4184
4185        // `ldp fp, lr, [sp], #16`
4186        Inst::LoadP64 {
4187            rt: writable_fp_reg(),
4188            rt2: writable_link_reg(),
4189            mem: PairAMode::SPPostIndexed {
4190                // TODO: we could fold the increment for incoming_args_diff here, as long as that
4191                // value is less than 502*8, by adding it to `setup_area_size`.
4192                // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/LDP--Load-Pair-of-Registers-
4193                simm7: SImm7Scaled::maybe_from_i64(i64::from(setup_area_size), types::I64).unwrap(),
4194            },
4195            flags: MemFlagsData::trusted(),
4196        }
4197        .emit(sink, emit_info, state);
4198    }
4199
4200    // Adjust SP to account for the possible over-allocation in the prologue.
4201    let incoming_args_diff = state.frame_layout().tail_args_size - info.new_stack_arg_size;
4202    if incoming_args_diff > 0 {
4203        for inst in
4204            AArch64MachineDeps::gen_sp_reg_adjust(i32::try_from(incoming_args_diff).unwrap())
4205        {
4206            inst.emit(sink, emit_info, state);
4207        }
4208    }
4209
4210    if (setup_area_size > 0 || info.sign_return_address_all)
4211        && let Some(key) = info.key
4212    {
4213        sink.put4(key.enc_auti_hint());
4214    }
4215}