1use cranelift_control::ControlPlane;
4
5use crate::ir::types::*;
6use crate::isa::aarch64;
7use crate::isa::aarch64::inst::*;
8use crate::trace;
9
10pub fn mem_finalize(
15 sink: Option<&mut MachBuffer<Inst>>,
16 mem: &AMode,
17 access_ty: Type,
18 state: &EmitState,
19) -> (SmallVec<[Inst; 4]>, AMode) {
20 match mem {
21 &AMode::RegOffset { off, .. }
22 | &AMode::SPOffset { off }
23 | &AMode::FPOffset { off }
24 | &AMode::IncomingArg { off }
25 | &AMode::SlotOffset { off } => {
26 let basereg = match mem {
27 &AMode::RegOffset { rn, .. } => rn,
28 &AMode::SPOffset { .. }
29 | &AMode::SlotOffset { .. }
30 | &AMode::IncomingArg { .. } => stack_reg(),
31 &AMode::FPOffset { .. } => fp_reg(),
32 _ => unreachable!(),
33 };
34 let off = match mem {
35 &AMode::IncomingArg { .. } => {
36 let frame_layout = state.frame_layout();
37 i64::from(
38 frame_layout.setup_area_size
39 + frame_layout.tail_args_size
40 + frame_layout.clobber_size
41 + frame_layout.fixed_frame_storage_size
42 + frame_layout.outgoing_args_size,
43 ) - off
44 }
45 &AMode::SlotOffset { .. } => {
46 let adj = i64::from(state.frame_layout().outgoing_args_size);
47 trace!(
48 "mem_finalize: slot offset {} + adj {} -> {}",
49 off,
50 adj,
51 off + adj
52 );
53 off + adj
54 }
55 _ => off,
56 };
57
58 if let Some(simm9) = SImm9::maybe_from_i64(off) {
59 let mem = AMode::Unscaled { rn: basereg, simm9 };
60 (smallvec![], mem)
61 } else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, access_ty) {
62 let mem = AMode::UnsignedOffset {
63 rn: basereg,
64 uimm12,
65 };
66 (smallvec![], mem)
67 } else {
68 let tmp = if basereg == spilltmp_reg() {
69 writable_tmp2_reg()
70 } else {
71 writable_spilltmp_reg()
72 };
73 (
74 Inst::load_constant(tmp, off as u64),
75 AMode::RegExtended {
76 rn: basereg,
77 rm: tmp.to_reg(),
78 extendop: ExtendOp::SXTX,
79 },
80 )
81 }
82 }
83
84 AMode::Const { addr } => {
85 let sink = match sink {
86 Some(sink) => sink,
87 None => return (smallvec![], mem.clone()),
88 };
89 let label = sink.get_label_for_constant(*addr);
90 let label = MemLabel::Mach(label);
91 (smallvec![], AMode::Label { label })
92 }
93
94 _ => (smallvec![], mem.clone()),
95 }
96}
97
98pub(crate) fn machreg_to_gpr(m: Reg) -> u32 {
102 assert_eq!(m.class(), RegClass::Int);
103 u32::from(m.to_real_reg().unwrap().hw_enc() & 31)
104}
105
106pub(crate) fn machreg_to_vec(m: Reg) -> u32 {
107 assert_eq!(m.class(), RegClass::Float);
108 u32::from(m.to_real_reg().unwrap().hw_enc())
109}
110
111fn machreg_to_gpr_or_vec(m: Reg) -> u32 {
112 u32::from(m.to_real_reg().unwrap().hw_enc() & 31)
113}
114
115pub fn enc_arith_rrr(bits_31_21: u32, bits_15_10: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
117 (bits_31_21 << 21)
118 | (bits_15_10 << 10)
119 | machreg_to_gpr(rd.to_reg())
120 | (machreg_to_gpr(rn) << 5)
121 | (machreg_to_gpr(rm) << 16)
122}
123
124fn enc_arith_rr_imm12(
125 bits_31_24: u32,
126 immshift: u32,
127 imm12: u32,
128 rn: Reg,
129 rd: Writable<Reg>,
130) -> u32 {
131 (bits_31_24 << 24)
132 | (immshift << 22)
133 | (imm12 << 10)
134 | (machreg_to_gpr(rn) << 5)
135 | machreg_to_gpr(rd.to_reg())
136}
137
138fn enc_arith_rr_imml(bits_31_23: u32, imm_bits: u32, rn: Reg, rd: Writable<Reg>) -> u32 {
139 (bits_31_23 << 23) | (imm_bits << 10) | (machreg_to_gpr(rn) << 5) | machreg_to_gpr(rd.to_reg())
140}
141
142fn enc_arith_rrrr(top11: u32, rm: Reg, bit15: u32, ra: Reg, rn: Reg, rd: Writable<Reg>) -> u32 {
143 (top11 << 21)
144 | (machreg_to_gpr(rm) << 16)
145 | (bit15 << 15)
146 | (machreg_to_gpr(ra) << 10)
147 | (machreg_to_gpr(rn) << 5)
148 | machreg_to_gpr(rd.to_reg())
149}
150
151fn enc_jump26(op_31_26: u32, off_26_0: u32) -> u32 {
152 assert!(off_26_0 < (1 << 26));
153 (op_31_26 << 26) | off_26_0
154}
155
156fn enc_cmpbr(op_31_24: u32, off_18_0: u32, reg: Reg) -> u32 {
157 assert!(off_18_0 < (1 << 19));
158 (op_31_24 << 24) | (off_18_0 << 5) | machreg_to_gpr(reg)
159}
160
161fn enc_cbr(op_31_24: u32, off_18_0: u32, op_4: u32, cond: u32) -> u32 {
162 assert!(off_18_0 < (1 << 19));
163 assert!(cond < (1 << 4));
164 (op_31_24 << 24) | (off_18_0 << 5) | (op_4 << 4) | cond
165}
166
167fn enc_op_size(op: u32, size: OperandSize) -> u32 {
169 (op & !(1 << 31)) | (size.sf_bit() << 31)
170}
171
172fn enc_conditional_br(taken: BranchTarget, kind: CondBrKind) -> u32 {
173 match kind {
174 CondBrKind::Zero(reg, size) => enc_op_size(
175 enc_cmpbr(0b0_011010_0, taken.as_offset19_or_zero(), reg),
176 size,
177 ),
178 CondBrKind::NotZero(reg, size) => enc_op_size(
179 enc_cmpbr(0b0_011010_1, taken.as_offset19_or_zero(), reg),
180 size,
181 ),
182 CondBrKind::Cond(c) => enc_cbr(0b01010100, taken.as_offset19_or_zero(), 0b0, c.bits()),
183 }
184}
185
186fn enc_test_bit_and_branch(
187 kind: TestBitAndBranchKind,
188 taken: BranchTarget,
189 reg: Reg,
190 bit: u8,
191) -> u32 {
192 assert!(bit < 64);
193 let op_31 = u32::from(bit >> 5);
194 let op_23_19 = u32::from(bit & 0b11111);
195 let op_30_24 = 0b0110110
196 | match kind {
197 TestBitAndBranchKind::Z => 0,
198 TestBitAndBranchKind::NZ => 1,
199 };
200 (op_31 << 31)
201 | (op_30_24 << 24)
202 | (op_23_19 << 19)
203 | (taken.as_offset14_or_zero() << 5)
204 | machreg_to_gpr(reg)
205}
206
207pub fn enc_move_wide(
209 op: MoveWideOp,
210 rd: Writable<Reg>,
211 imm: MoveWideConst,
212 size: OperandSize,
213) -> u32 {
214 assert!(imm.shift <= 0b11);
215 let op = match op {
216 MoveWideOp::MovN => 0b00,
217 MoveWideOp::MovZ => 0b10,
218 };
219 0x12800000
220 | size.sf_bit() << 31
221 | op << 29
222 | u32::from(imm.shift) << 21
223 | u32::from(imm.bits) << 5
224 | machreg_to_gpr(rd.to_reg())
225}
226
227pub fn enc_movk(rd: Writable<Reg>, imm: MoveWideConst, size: OperandSize) -> u32 {
229 assert!(imm.shift <= 0b11);
230 0x72800000
231 | size.sf_bit() << 31
232 | u32::from(imm.shift) << 21
233 | u32::from(imm.bits) << 5
234 | machreg_to_gpr(rd.to_reg())
235}
236
237fn enc_ldst_pair(op_31_22: u32, simm7: SImm7Scaled, rn: Reg, rt: Reg, rt2: Reg) -> u32 {
238 (op_31_22 << 22)
239 | (simm7.bits() << 15)
240 | (machreg_to_gpr(rt2) << 10)
241 | (machreg_to_gpr(rn) << 5)
242 | machreg_to_gpr(rt)
243}
244
245fn enc_ldst_simm9(op_31_22: u32, simm9: SImm9, op_11_10: u32, rn: Reg, rd: Reg) -> u32 {
246 (op_31_22 << 22)
247 | (simm9.bits() << 12)
248 | (op_11_10 << 10)
249 | (machreg_to_gpr(rn) << 5)
250 | machreg_to_gpr_or_vec(rd)
251}
252
253fn enc_ldst_uimm12(op_31_22: u32, uimm12: UImm12Scaled, rn: Reg, rd: Reg) -> u32 {
254 (op_31_22 << 22)
255 | (0b1 << 24)
256 | (uimm12.bits() << 10)
257 | (machreg_to_gpr(rn) << 5)
258 | machreg_to_gpr_or_vec(rd)
259}
260
261fn enc_ldst_reg(
262 op_31_22: u32,
263 rn: Reg,
264 rm: Reg,
265 s_bit: bool,
266 extendop: Option<ExtendOp>,
267 rd: Reg,
268) -> u32 {
269 let s_bit = if s_bit { 1 } else { 0 };
270 let extend_bits = match extendop {
271 Some(ExtendOp::UXTW) => 0b010,
272 Some(ExtendOp::SXTW) => 0b110,
273 Some(ExtendOp::SXTX) => 0b111,
274 None => 0b011, _ => panic!("bad extend mode for ld/st AMode"),
276 };
277 (op_31_22 << 22)
278 | (1 << 21)
279 | (machreg_to_gpr(rm) << 16)
280 | (extend_bits << 13)
281 | (s_bit << 12)
282 | (0b10 << 10)
283 | (machreg_to_gpr(rn) << 5)
284 | machreg_to_gpr_or_vec(rd)
285}
286
287pub(crate) fn enc_ldst_imm19(op_31_24: u32, imm19: u32, rd: Reg) -> u32 {
288 (op_31_24 << 24) | (imm19 << 5) | machreg_to_gpr_or_vec(rd)
289}
290
291fn enc_ldst_vec(q: u32, size: u32, rn: Reg, rt: Writable<Reg>) -> u32 {
292 debug_assert_eq!(q & 0b1, q);
293 debug_assert_eq!(size & 0b11, size);
294 0b0_0_0011010_10_00000_110_0_00_00000_00000
295 | q << 30
296 | size << 10
297 | machreg_to_gpr(rn) << 5
298 | machreg_to_vec(rt.to_reg())
299}
300
301fn enc_ldst_vec_pair(
302 opc: u32,
303 amode: u32,
304 is_load: bool,
305 simm7: SImm7Scaled,
306 rn: Reg,
307 rt: Reg,
308 rt2: Reg,
309) -> u32 {
310 debug_assert_eq!(opc & 0b11, opc);
311 debug_assert_eq!(amode & 0b11, amode);
312
313 0b00_10110_00_0_0000000_00000_00000_00000
314 | opc << 30
315 | amode << 23
316 | (is_load as u32) << 22
317 | simm7.bits() << 15
318 | machreg_to_vec(rt2) << 10
319 | machreg_to_gpr(rn) << 5
320 | machreg_to_vec(rt)
321}
322
323fn enc_vec_rrr(top11: u32, rm: Reg, bit15_10: u32, rn: Reg, rd: Writable<Reg>) -> u32 {
324 (top11 << 21)
325 | (machreg_to_vec(rm) << 16)
326 | (bit15_10 << 10)
327 | (machreg_to_vec(rn) << 5)
328 | machreg_to_vec(rd.to_reg())
329}
330
331fn enc_vec_rrr_long(
332 q: u32,
333 u: u32,
334 size: u32,
335 bit14: u32,
336 rm: Reg,
337 rn: Reg,
338 rd: Writable<Reg>,
339) -> u32 {
340 debug_assert_eq!(q & 0b1, q);
341 debug_assert_eq!(u & 0b1, u);
342 debug_assert_eq!(size & 0b11, size);
343 debug_assert_eq!(bit14 & 0b1, bit14);
344
345 0b0_0_0_01110_00_1_00000_100000_00000_00000
346 | q << 30
347 | u << 29
348 | size << 22
349 | bit14 << 14
350 | (machreg_to_vec(rm) << 16)
351 | (machreg_to_vec(rn) << 5)
352 | machreg_to_vec(rd.to_reg())
353}
354
355fn enc_bit_rr(size: u32, opcode2: u32, opcode1: u32, rn: Reg, rd: Writable<Reg>) -> u32 {
356 (0b01011010110 << 21)
357 | size << 31
358 | opcode2 << 16
359 | opcode1 << 10
360 | machreg_to_gpr(rn) << 5
361 | machreg_to_gpr(rd.to_reg())
362}
363
364pub(crate) fn enc_br(rn: Reg) -> u32 {
365 0b1101011_0000_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5)
366}
367
368pub(crate) fn enc_adr_inst(opcode: u32, off: i32, rd: Writable<Reg>) -> u32 {
369 let off = u32::try_from(off).unwrap();
370 let immlo = off & 3;
371 let immhi = (off >> 2) & ((1 << 19) - 1);
372 opcode | (immlo << 29) | (immhi << 5) | machreg_to_gpr(rd.to_reg())
373}
374
375pub(crate) fn enc_adr(off: i32, rd: Writable<Reg>) -> u32 {
376 let opcode = 0b00010000 << 24;
377 enc_adr_inst(opcode, off, rd)
378}
379
380pub(crate) fn enc_adrp(off: i32, rd: Writable<Reg>) -> u32 {
381 let opcode = 0b10010000 << 24;
382 enc_adr_inst(opcode, off, rd)
383}
384
385fn enc_csel(rd: Writable<Reg>, rn: Reg, rm: Reg, cond: Cond, op: u32, o2: u32) -> u32 {
386 debug_assert_eq!(op & 0b1, op);
387 debug_assert_eq!(o2 & 0b1, o2);
388 0b100_11010100_00000_0000_00_00000_00000
389 | (op << 30)
390 | (machreg_to_gpr(rm) << 16)
391 | (cond.bits() << 12)
392 | (o2 << 10)
393 | (machreg_to_gpr(rn) << 5)
394 | machreg_to_gpr(rd.to_reg())
395}
396
397fn enc_fcsel(rd: Writable<Reg>, rn: Reg, rm: Reg, cond: Cond, size: ScalarSize) -> u32 {
398 0b000_11110_00_1_00000_0000_11_00000_00000
399 | (size.ftype() << 22)
400 | (machreg_to_vec(rm) << 16)
401 | (machreg_to_vec(rn) << 5)
402 | machreg_to_vec(rd.to_reg())
403 | (cond.bits() << 12)
404}
405
406fn enc_ccmp(size: OperandSize, rn: Reg, rm: Reg, nzcv: NZCV, cond: Cond) -> u32 {
407 0b0_1_1_11010010_00000_0000_00_00000_0_0000
408 | size.sf_bit() << 31
409 | machreg_to_gpr(rm) << 16
410 | cond.bits() << 12
411 | machreg_to_gpr(rn) << 5
412 | nzcv.bits()
413}
414
415fn enc_ccmp_imm(size: OperandSize, rn: Reg, imm: UImm5, nzcv: NZCV, cond: Cond) -> u32 {
416 0b0_1_1_11010010_00000_0000_10_00000_0_0000
417 | size.sf_bit() << 31
418 | imm.bits() << 16
419 | cond.bits() << 12
420 | machreg_to_gpr(rn) << 5
421 | nzcv.bits()
422}
423
424impl BfmOp {
425 fn opc(self) -> u8 {
426 match self {
427 BfmOp::UBfm => 0b10,
428 BfmOp::SBfm => 0b00,
429 }
430 }
431}
432
433fn enc_bfm(opc: u8, size: OperandSize, rd: Writable<Reg>, rn: Reg, immr: u8, imms: u8) -> u32 {
434 match size {
435 OperandSize::Size64 => {
436 debug_assert!(immr <= 63);
437 debug_assert!(imms <= 63);
438 }
439 OperandSize::Size32 => {
440 debug_assert!(immr <= 31);
441 debug_assert!(imms <= 31);
442 }
443 }
444 debug_assert_eq!(opc & 0b11, opc);
445 let n_bit = size.sf_bit();
446 0b0_00_100110_0_000000_000000_00000_00000
447 | size.sf_bit() << 31
448 | u32::from(opc) << 29
449 | n_bit << 22
450 | u32::from(immr) << 16
451 | u32::from(imms) << 10
452 | machreg_to_gpr(rn) << 5
453 | machreg_to_gpr(rd.to_reg())
454}
455
456fn enc_vecmov(is_16b: bool, rd: Writable<Reg>, rn: Reg) -> u32 {
457 0b00001110_101_00000_00011_1_00000_00000
458 | ((is_16b as u32) << 30)
459 | machreg_to_vec(rd.to_reg())
460 | (machreg_to_vec(rn) << 16)
461 | (machreg_to_vec(rn) << 5)
462}
463
464fn enc_fpurr(top22: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
465 (top22 << 10) | (machreg_to_vec(rn) << 5) | machreg_to_vec(rd.to_reg())
466}
467
468fn enc_fpurrr(top22: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
469 (top22 << 10)
470 | (machreg_to_vec(rm) << 16)
471 | (machreg_to_vec(rn) << 5)
472 | machreg_to_vec(rd.to_reg())
473}
474
475fn enc_fpurrrr(top17: u32, rd: Writable<Reg>, rn: Reg, rm: Reg, ra: Reg) -> u32 {
476 (top17 << 15)
477 | (machreg_to_vec(rm) << 16)
478 | (machreg_to_vec(ra) << 10)
479 | (machreg_to_vec(rn) << 5)
480 | machreg_to_vec(rd.to_reg())
481}
482
483fn enc_fcmp(size: ScalarSize, rn: Reg, rm: Reg) -> u32 {
484 0b000_11110_00_1_00000_00_1000_00000_00000
485 | (size.ftype() << 22)
486 | (machreg_to_vec(rm) << 16)
487 | (machreg_to_vec(rn) << 5)
488}
489
490fn enc_fputoint(top16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
491 (top16 << 16) | (machreg_to_vec(rn) << 5) | machreg_to_gpr(rd.to_reg())
492}
493
494fn enc_inttofpu(top16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
495 (top16 << 16) | (machreg_to_gpr(rn) << 5) | machreg_to_vec(rd.to_reg())
496}
497
498fn enc_fround(top22: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
499 (top22 << 10) | (machreg_to_vec(rn) << 5) | machreg_to_vec(rd.to_reg())
500}
501
502fn enc_vec_rr_misc(qu: u32, size: u32, bits_12_16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
503 debug_assert_eq!(qu & 0b11, qu);
504 debug_assert_eq!(size & 0b11, size);
505 debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16);
506 let bits = 0b0_00_01110_00_10000_00000_10_00000_00000;
507 bits | qu << 29
508 | size << 22
509 | bits_12_16 << 12
510 | machreg_to_vec(rn) << 5
511 | machreg_to_vec(rd.to_reg())
512}
513
514fn enc_vec_rr_pair(bits_12_16: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
515 debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16);
516
517 0b010_11110_11_11000_11011_10_00000_00000
518 | bits_12_16 << 12
519 | machreg_to_vec(rn) << 5
520 | machreg_to_vec(rd.to_reg())
521}
522
523fn enc_vec_rr_pair_long(u: u32, enc_size: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
524 debug_assert_eq!(u & 0b1, u);
525 debug_assert_eq!(enc_size & 0b1, enc_size);
526
527 0b0_1_0_01110_00_10000_00_0_10_10_00000_00000
528 | u << 29
529 | enc_size << 22
530 | machreg_to_vec(rn) << 5
531 | machreg_to_vec(rd.to_reg())
532}
533
534fn enc_vec_lanes(q: u32, u: u32, size: u32, opcode: u32, rd: Writable<Reg>, rn: Reg) -> u32 {
535 debug_assert_eq!(q & 0b1, q);
536 debug_assert_eq!(u & 0b1, u);
537 debug_assert_eq!(size & 0b11, size);
538 debug_assert_eq!(opcode & 0b11111, opcode);
539 0b0_0_0_01110_00_11000_0_0000_10_00000_00000
540 | q << 30
541 | u << 29
542 | size << 22
543 | opcode << 12
544 | machreg_to_vec(rn) << 5
545 | machreg_to_vec(rd.to_reg())
546}
547
548fn enc_tbl(is_extension: bool, len: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
549 debug_assert_eq!(len & 0b11, len);
550 0b0_1_001110_000_00000_0_00_0_00_00000_00000
551 | (machreg_to_vec(rm) << 16)
552 | len << 13
553 | (is_extension as u32) << 12
554 | (machreg_to_vec(rn) << 5)
555 | machreg_to_vec(rd.to_reg())
556}
557
558fn enc_dmb_ish() -> u32 {
559 0xD5033BBF
560}
561
562fn enc_acq_rel(ty: Type, op: AtomicRMWOp, rs: Reg, rt: Writable<Reg>, rn: Reg) -> u32 {
563 assert!(machreg_to_gpr(rt.to_reg()) != 31);
564 let sz = match ty {
565 I64 => 0b11,
566 I32 => 0b10,
567 I16 => 0b01,
568 I8 => 0b00,
569 _ => unreachable!(),
570 };
571 let bit15 = match op {
572 AtomicRMWOp::Swp => 0b1,
573 _ => 0b0,
574 };
575 let op = match op {
576 AtomicRMWOp::Add => 0b000,
577 AtomicRMWOp::Clr => 0b001,
578 AtomicRMWOp::Eor => 0b010,
579 AtomicRMWOp::Set => 0b011,
580 AtomicRMWOp::Smax => 0b100,
581 AtomicRMWOp::Smin => 0b101,
582 AtomicRMWOp::Umax => 0b110,
583 AtomicRMWOp::Umin => 0b111,
584 AtomicRMWOp::Swp => 0b000,
585 };
586 0b00_111_000_111_00000_0_000_00_00000_00000
587 | (sz << 30)
588 | (machreg_to_gpr(rs) << 16)
589 | bit15 << 15
590 | (op << 12)
591 | (machreg_to_gpr(rn) << 5)
592 | machreg_to_gpr(rt.to_reg())
593}
594
595fn enc_ldar(ty: Type, rt: Writable<Reg>, rn: Reg) -> u32 {
596 let sz = match ty {
597 I64 => 0b11,
598 I32 => 0b10,
599 I16 => 0b01,
600 I8 => 0b00,
601 _ => unreachable!(),
602 };
603 0b00_001000_1_1_0_11111_1_11111_00000_00000
604 | (sz << 30)
605 | (machreg_to_gpr(rn) << 5)
606 | machreg_to_gpr(rt.to_reg())
607}
608
609fn enc_stlr(ty: Type, rt: Reg, rn: Reg) -> u32 {
610 let sz = match ty {
611 I64 => 0b11,
612 I32 => 0b10,
613 I16 => 0b01,
614 I8 => 0b00,
615 _ => unreachable!(),
616 };
617 0b00_001000_100_11111_1_11111_00000_00000
618 | (sz << 30)
619 | (machreg_to_gpr(rn) << 5)
620 | machreg_to_gpr(rt)
621}
622
623fn enc_ldaxr(ty: Type, rt: Writable<Reg>, rn: Reg) -> u32 {
624 let sz = match ty {
625 I64 => 0b11,
626 I32 => 0b10,
627 I16 => 0b01,
628 I8 => 0b00,
629 _ => unreachable!(),
630 };
631 0b00_001000_0_1_0_11111_1_11111_00000_00000
632 | (sz << 30)
633 | (machreg_to_gpr(rn) << 5)
634 | machreg_to_gpr(rt.to_reg())
635}
636
637fn enc_ldaxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {
638 let sz = match ty {
639 I64 => 0b1,
640 I32 => 0b0,
641 _ => unreachable!(),
642 };
643 0b10_0010000_1_1_11111_1_00000_00000_00000
644 | (sz << 30)
645 | (machreg_to_gpr(rt2.to_reg()) << 10)
646 | (machreg_to_gpr(rn) << 5)
647 | machreg_to_gpr(rt.to_reg())
648}
649
650fn enc_ldxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {
651 let sz = match ty {
652 I64 => 0b1,
653 I32 => 0b0,
654 _ => unreachable!(),
655 };
656 0b10_0010000_1_1_11111_0_00000_00000_00000
657 | (sz << 30)
658 | (machreg_to_gpr(rt2.to_reg()) << 10)
659 | (machreg_to_gpr(rn) << 5)
660 | machreg_to_gpr(rt.to_reg())
661}
662
663fn enc_stlxr(ty: Type, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
664 let sz = match ty {
665 I64 => 0b11,
666 I32 => 0b10,
667 I16 => 0b01,
668 I8 => 0b00,
669 _ => unreachable!(),
670 };
671 0b00_001000_000_00000_1_11111_00000_00000
672 | (sz << 30)
673 | (machreg_to_gpr(rs.to_reg()) << 16)
674 | (machreg_to_gpr(rn) << 5)
675 | machreg_to_gpr(rt)
676}
677
678fn enc_stlxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {
679 let sz = match ty {
680 I64 => 0b1,
681 I32 => 0b0,
682 _ => unreachable!(),
683 };
684 0b10_0010000_0_1_00000_1_00000_00000_00000
685 | (sz << 30)
686 | (machreg_to_gpr(rs.to_reg()) << 16)
687 | (machreg_to_gpr(rt2) << 10)
688 | (machreg_to_gpr(rn) << 5)
689 | machreg_to_gpr(rt)
690}
691
692fn enc_stxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {
693 let sz = match ty {
694 I64 => 0b1,
695 I32 => 0b0,
696 _ => unreachable!(),
697 };
698 0b10_0010000_0_1_00000_0_00000_00000_00000
699 | (sz << 30)
700 | (machreg_to_gpr(rs.to_reg()) << 16)
701 | (machreg_to_gpr(rt2) << 10)
702 | (machreg_to_gpr(rn) << 5)
703 | machreg_to_gpr(rt)
704}
705
706fn enc_cas(size: u32, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
707 debug_assert_eq!(size & 0b11, size);
708
709 0b00_0010001_1_1_00000_1_11111_00000_00000
710 | size << 30
711 | machreg_to_gpr(rs.to_reg()) << 16
712 | machreg_to_gpr(rn) << 5
713 | machreg_to_gpr(rt)
714}
715
716fn enc_casp(rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
717 debug_assert_eq!(machreg_to_gpr(rs.to_reg()) & 1, 0);
718 debug_assert_eq!(machreg_to_gpr(rt) & 1, 0);
719
720 0b0_1_0010000_1_1_00000_1_11111_00000_00000
721 | machreg_to_gpr(rs.to_reg()) << 16
722 | machreg_to_gpr(rn) << 5
723 | machreg_to_gpr(rt)
724}
725
726fn enc_asimd_mod_imm(rd: Writable<Reg>, q_op: u32, cmode: u32, imm: u8) -> u32 {
727 let abc = (imm >> 5) as u32;
728 let defgh = (imm & 0b11111) as u32;
729
730 debug_assert_eq!(cmode & 0b1111, cmode);
731 debug_assert_eq!(q_op & 0b11, q_op);
732
733 0b0_0_0_0111100000_000_0000_01_00000_00000
734 | (q_op << 29)
735 | (abc << 16)
736 | (cmode << 12)
737 | (defgh << 5)
738 | machreg_to_vec(rd.to_reg())
739}
740
741#[derive(Default, Clone, Debug)]
743pub struct EmitState {
744 stack_maps: StackMapEmitter<Inst>,
746
747 ctrl_plane: ControlPlane,
750
751 frame_layout: FrameLayout,
752}
753
754impl MachInstEmitState<Inst> for EmitState {
755 fn new(abi: &Callee<AArch64MachineDeps>, ctrl_plane: ControlPlane) -> Self {
756 EmitState {
757 stack_maps: StackMapEmitter::default(),
758 ctrl_plane,
759 frame_layout: abi.frame_layout().clone(),
760 }
761 }
762
763 fn stack_map_emitter(&mut self) -> &mut StackMapEmitter<Inst> {
764 &mut self.stack_maps
765 }
766
767 fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
768 &mut self.ctrl_plane
769 }
770
771 fn take_ctrl_plane(self) -> ControlPlane {
772 self.ctrl_plane
773 }
774
775 fn frame_layout(&self) -> &FrameLayout {
776 &self.frame_layout
777 }
778}
779
780pub struct EmitInfo {
782 flags: settings::Flags,
783 isa_flags: aarch64::settings::Flags,
784}
785
786impl EmitInfo {
787 pub fn new(flags: settings::Flags, isa_flags: aarch64::settings::Flags) -> Self {
789 Self { flags, isa_flags }
790 }
791}
792
793impl MachInstEmit for Inst {
794 type State = EmitState;
795 type Info = EmitInfo;
796
797 fn emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState) {
798 let mut start_off = sink.cur_offset();
804
805 match self {
806 &Inst::AluRRR {
807 alu_op,
808 size,
809 rd,
810 rn,
811 rm,
812 } => {
813 debug_assert!(match alu_op {
814 ALUOp::SMulH | ALUOp::UMulH => size == OperandSize::Size64,
815 _ => true,
816 });
817 let top11 = match alu_op {
818 ALUOp::Add => 0b00001011_000,
819 ALUOp::Adc => 0b00011010_000,
820 ALUOp::AdcS => 0b00111010_000,
821 ALUOp::Sub => 0b01001011_000,
822 ALUOp::Sbc => 0b01011010_000,
823 ALUOp::SbcS => 0b01111010_000,
824 ALUOp::Orr => 0b00101010_000,
825 ALUOp::And => 0b00001010_000,
826 ALUOp::AndS => 0b01101010_000,
827 ALUOp::Eor => 0b01001010_000,
828 ALUOp::OrrNot => 0b00101010_001,
829 ALUOp::AndNot => 0b00001010_001,
830 ALUOp::EorNot => 0b01001010_001,
831 ALUOp::AddS => 0b00101011_000,
832 ALUOp::SubS => 0b01101011_000,
833 ALUOp::SDiv | ALUOp::UDiv => 0b00011010_110,
834 ALUOp::Extr | ALUOp::Lsr | ALUOp::Asr | ALUOp::Lsl => 0b00011010_110,
835 ALUOp::SMulH => 0b10011011_010,
836 ALUOp::UMulH => 0b10011011_110,
837 };
838
839 let top11 = top11 | size.sf_bit() << 10;
840 let bit15_10 = match alu_op {
841 ALUOp::SDiv => 0b000011,
842 ALUOp::UDiv => 0b000010,
843 ALUOp::Extr => 0b001011,
844 ALUOp::Lsr => 0b001001,
845 ALUOp::Asr => 0b001010,
846 ALUOp::Lsl => 0b001000,
847 ALUOp::SMulH | ALUOp::UMulH => 0b011111,
848 _ => 0b000000,
849 };
850 debug_assert_ne!(writable_stack_reg(), rd);
851 debug_assert_ne!(stack_reg(), rn);
854 debug_assert_ne!(stack_reg(), rm);
855 sink.put4(enc_arith_rrr(top11, bit15_10, rd, rn, rm));
856 }
857 &Inst::AluRRRR {
858 alu_op,
859 size,
860 rd,
861 rm,
862 rn,
863 ra,
864 } => {
865 let (top11, bit15) = match alu_op {
866 ALUOp3::MAdd => (0b0_00_11011_000, 0),
867 ALUOp3::MSub => (0b0_00_11011_000, 1),
868 ALUOp3::UMAddL => {
869 debug_assert!(size == OperandSize::Size32);
870 (0b1_00_11011_1_01, 0)
871 }
872 ALUOp3::SMAddL => {
873 debug_assert!(size == OperandSize::Size32);
874 (0b1_00_11011_0_01, 0)
875 }
876 };
877 let top11 = top11 | size.sf_bit() << 10;
878 sink.put4(enc_arith_rrrr(top11, rm, bit15, ra, rn, rd));
879 }
880 &Inst::AluRRImm12 {
881 alu_op,
882 size,
883 rd,
884 rn,
885 ref imm12,
886 } => {
887 let top8 = match alu_op {
888 ALUOp::Add => 0b000_10001,
889 ALUOp::Sub => 0b010_10001,
890 ALUOp::AddS => 0b001_10001,
891 ALUOp::SubS => 0b011_10001,
892 _ => unimplemented!("{:?}", alu_op),
893 };
894 let top8 = top8 | size.sf_bit() << 7;
895 sink.put4(enc_arith_rr_imm12(
896 top8,
897 imm12.shift_bits(),
898 imm12.imm_bits(),
899 rn,
900 rd,
901 ));
902 }
903 &Inst::AluRRImmLogic {
904 alu_op,
905 size,
906 rd,
907 rn,
908 ref imml,
909 } => {
910 let (top9, inv) = match alu_op {
911 ALUOp::Orr => (0b001_100100, false),
912 ALUOp::And => (0b000_100100, false),
913 ALUOp::AndS => (0b011_100100, false),
914 ALUOp::Eor => (0b010_100100, false),
915 ALUOp::OrrNot => (0b001_100100, true),
916 ALUOp::AndNot => (0b000_100100, true),
917 ALUOp::EorNot => (0b010_100100, true),
918 _ => unimplemented!("{:?}", alu_op),
919 };
920 let top9 = top9 | size.sf_bit() << 8;
921 let imml = if inv { imml.invert() } else { *imml };
922 sink.put4(enc_arith_rr_imml(top9, imml.enc_bits(), rn, rd));
923 }
924
925 &Inst::AluRRImmShift {
926 alu_op,
927 size,
928 rd,
929 rn,
930 ref immshift,
931 } => {
932 let amt = immshift.value();
933 let (top10, immr, imms) = match alu_op {
934 ALUOp::Extr => (0b0001001110, machreg_to_gpr(rn), u32::from(amt)),
935 ALUOp::Lsr => (0b0101001100, u32::from(amt), 0b011111),
936 ALUOp::Asr => (0b0001001100, u32::from(amt), 0b011111),
937 ALUOp::Lsl => {
938 let bits = if size.is64() { 64 } else { 32 };
939 (
940 0b0101001100,
941 u32::from((bits - amt) % bits),
942 u32::from(bits - 1 - amt),
943 )
944 }
945 _ => unimplemented!("{:?}", alu_op),
946 };
947 let top10 = top10 | size.sf_bit() << 9 | size.sf_bit();
948 let imms = match alu_op {
949 ALUOp::Lsr | ALUOp::Asr => imms | size.sf_bit() << 5,
950 _ => imms,
951 };
952 sink.put4(
953 (top10 << 22)
954 | (immr << 16)
955 | (imms << 10)
956 | (machreg_to_gpr(rn) << 5)
957 | machreg_to_gpr(rd.to_reg()),
958 );
959 }
960
961 &Inst::AluRRRShift {
962 alu_op,
963 size,
964 rd,
965 rn,
966 rm,
967 ref shiftop,
968 } => {
969 let top11: u32 = match alu_op {
970 ALUOp::Add => 0b000_01011000,
971 ALUOp::AddS => 0b001_01011000,
972 ALUOp::Sub => 0b010_01011000,
973 ALUOp::SubS => 0b011_01011000,
974 ALUOp::Orr => 0b001_01010000,
975 ALUOp::And => 0b000_01010000,
976 ALUOp::AndS => 0b011_01010000,
977 ALUOp::Eor => 0b010_01010000,
978 ALUOp::OrrNot => 0b001_01010001,
979 ALUOp::EorNot => 0b010_01010001,
980 ALUOp::AndNot => 0b000_01010001,
981 ALUOp::Extr => 0b000_10011100,
982 _ => unimplemented!("{:?}", alu_op),
983 };
984 let top11 = top11 | size.sf_bit() << 10;
985 let top11 = top11 | (u32::from(shiftop.op().bits()) << 1);
986 let bits_15_10 = u32::from(shiftop.amt().value());
987 sink.put4(enc_arith_rrr(top11, bits_15_10, rd, rn, rm));
988 }
989
990 &Inst::AluRRRExtend {
991 alu_op,
992 size,
993 rd,
994 rn,
995 rm,
996 extendop,
997 } => {
998 let top11: u32 = match alu_op {
999 ALUOp::Add => 0b00001011001,
1000 ALUOp::Sub => 0b01001011001,
1001 ALUOp::AddS => 0b00101011001,
1002 ALUOp::SubS => 0b01101011001,
1003 _ => unimplemented!("{:?}", alu_op),
1004 };
1005 let top11 = top11 | size.sf_bit() << 10;
1006 let bits_15_10 = u32::from(extendop.bits()) << 3;
1007 sink.put4(enc_arith_rrr(top11, bits_15_10, rd, rn, rm));
1008 }
1009
1010 &Inst::BitRR {
1011 op, size, rd, rn, ..
1012 } => {
1013 let (op1, op2) = match op {
1014 BitOp::RBit => (0b00000, 0b000000),
1015 BitOp::Clz => (0b00000, 0b000100),
1016 BitOp::Cls => (0b00000, 0b000101),
1017 BitOp::Rev16 => (0b00000, 0b000001),
1018 BitOp::Rev32 => (0b00000, 0b000010),
1019 BitOp::Rev64 => (0b00000, 0b000011),
1020 };
1021 sink.put4(enc_bit_rr(size.sf_bit(), op1, op2, rn, rd))
1022 }
1023
1024 &Inst::ULoad8 { rd, ref mem, flags }
1025 | &Inst::SLoad8 { rd, ref mem, flags }
1026 | &Inst::ULoad16 { rd, ref mem, flags }
1027 | &Inst::SLoad16 { rd, ref mem, flags }
1028 | &Inst::ULoad32 { rd, ref mem, flags }
1029 | &Inst::SLoad32 { rd, ref mem, flags }
1030 | &Inst::ULoad64 {
1031 rd, ref mem, flags, ..
1032 }
1033 | &Inst::FpuLoad16 { rd, ref mem, flags }
1034 | &Inst::FpuLoad32 { rd, ref mem, flags }
1035 | &Inst::FpuLoad64 { rd, ref mem, flags }
1036 | &Inst::FpuLoad128 { rd, ref mem, flags } => {
1037 let mem = mem.clone();
1038 let access_ty = self.mem_type().unwrap();
1039 let (mem_insts, mem) = mem_finalize(Some(sink), &mem, access_ty, state);
1040
1041 for inst in mem_insts.into_iter() {
1042 inst.emit(sink, emit_info, state);
1043 }
1044
1045 let rd = rd.to_reg();
1047
1048 let op = match self {
1052 Inst::ULoad8 { .. } => 0b0011100001,
1053 Inst::SLoad8 { .. } => 0b0011100010,
1054 Inst::ULoad16 { .. } => 0b0111100001,
1055 Inst::SLoad16 { .. } => 0b0111100010,
1056 Inst::ULoad32 { .. } => 0b1011100001,
1057 Inst::SLoad32 { .. } => 0b1011100010,
1058 Inst::ULoad64 { .. } => 0b1111100001,
1059 Inst::FpuLoad16 { .. } => 0b0111110001,
1060 Inst::FpuLoad32 { .. } => 0b1011110001,
1061 Inst::FpuLoad64 { .. } => 0b1111110001,
1062 Inst::FpuLoad128 { .. } => 0b0011110011,
1063 _ => unreachable!(),
1064 };
1065
1066 if let Some(trap_code) = flags.trap_code() {
1067 sink.add_trap(trap_code);
1069 }
1070
1071 match &mem {
1072 &AMode::Unscaled { rn, simm9 } => {
1073 let reg = rn;
1074 sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
1075 }
1076 &AMode::UnsignedOffset { rn, uimm12 } => {
1077 let reg = rn;
1078 sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
1079 }
1080 &AMode::RegReg { rn, rm } => {
1081 let r1 = rn;
1082 let r2 = rm;
1083 sink.put4(enc_ldst_reg(
1084 op, r1, r2, false, None, rd,
1085 ));
1086 }
1087 &AMode::RegScaled { rn, rm } | &AMode::RegScaledExtended { rn, rm, .. } => {
1088 let r1 = rn;
1089 let r2 = rm;
1090 let extendop = match &mem {
1091 &AMode::RegScaled { .. } => None,
1092 &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
1093 _ => unreachable!(),
1094 };
1095 sink.put4(enc_ldst_reg(
1096 op, r1, r2, true, extendop, rd,
1097 ));
1098 }
1099 &AMode::RegExtended { rn, rm, extendop } => {
1100 let r1 = rn;
1101 let r2 = rm;
1102 sink.put4(enc_ldst_reg(
1103 op,
1104 r1,
1105 r2,
1106 false,
1107 Some(extendop),
1108 rd,
1109 ));
1110 }
1111 &AMode::Label { ref label } => {
1112 let offset = match label {
1113 MemLabel::PCRel(off) => *off as u32,
1115 MemLabel::Mach(label) => {
1120 sink.use_label_at_offset(
1121 sink.cur_offset(),
1122 *label,
1123 LabelUse::Ldr19,
1124 );
1125 0
1126 }
1127 } / 4;
1128 assert!(offset < (1 << 19));
1129 match self {
1130 &Inst::ULoad32 { .. } => {
1131 sink.put4(enc_ldst_imm19(0b00011000, offset, rd));
1132 }
1133 &Inst::SLoad32 { .. } => {
1134 sink.put4(enc_ldst_imm19(0b10011000, offset, rd));
1135 }
1136 &Inst::FpuLoad32 { .. } => {
1137 sink.put4(enc_ldst_imm19(0b00011100, offset, rd));
1138 }
1139 &Inst::ULoad64 { .. } => {
1140 sink.put4(enc_ldst_imm19(0b01011000, offset, rd));
1141 }
1142 &Inst::FpuLoad64 { .. } => {
1143 sink.put4(enc_ldst_imm19(0b01011100, offset, rd));
1144 }
1145 &Inst::FpuLoad128 { .. } => {
1146 sink.put4(enc_ldst_imm19(0b10011100, offset, rd));
1147 }
1148 _ => panic!("Unsupported size for LDR from constant pool!"),
1149 }
1150 }
1151 &AMode::SPPreIndexed { simm9 } => {
1152 let reg = stack_reg();
1153 sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
1154 }
1155 &AMode::SPPostIndexed { simm9 } => {
1156 let reg = stack_reg();
1157 sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
1158 }
1159 &AMode::SPOffset { .. }
1161 | &AMode::FPOffset { .. }
1162 | &AMode::IncomingArg { .. }
1163 | &AMode::SlotOffset { .. }
1164 | &AMode::Const { .. }
1165 | &AMode::RegOffset { .. } => {
1166 panic!("Should not see {mem:?} here!")
1167 }
1168 }
1169 }
1170
1171 &Inst::Store8 { rd, ref mem, flags }
1172 | &Inst::Store16 { rd, ref mem, flags }
1173 | &Inst::Store32 { rd, ref mem, flags }
1174 | &Inst::Store64 { rd, ref mem, flags }
1175 | &Inst::FpuStore16 { rd, ref mem, flags }
1176 | &Inst::FpuStore32 { rd, ref mem, flags }
1177 | &Inst::FpuStore64 { rd, ref mem, flags }
1178 | &Inst::FpuStore128 { rd, ref mem, flags } => {
1179 let mem = mem.clone();
1180 let access_ty = self.mem_type().unwrap();
1181 let (mem_insts, mem) = mem_finalize(Some(sink), &mem, access_ty, state);
1182
1183 for inst in mem_insts.into_iter() {
1184 inst.emit(sink, emit_info, state);
1185 }
1186
1187 let op = match self {
1188 Inst::Store8 { .. } => 0b0011100000,
1189 Inst::Store16 { .. } => 0b0111100000,
1190 Inst::Store32 { .. } => 0b1011100000,
1191 Inst::Store64 { .. } => 0b1111100000,
1192 Inst::FpuStore16 { .. } => 0b0111110000,
1193 Inst::FpuStore32 { .. } => 0b1011110000,
1194 Inst::FpuStore64 { .. } => 0b1111110000,
1195 Inst::FpuStore128 { .. } => 0b0011110010,
1196 _ => unreachable!(),
1197 };
1198
1199 if let Some(trap_code) = flags.trap_code() {
1200 sink.add_trap(trap_code);
1202 }
1203
1204 match &mem {
1205 &AMode::Unscaled { rn, simm9 } => {
1206 let reg = rn;
1207 sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
1208 }
1209 &AMode::UnsignedOffset { rn, uimm12 } => {
1210 let reg = rn;
1211 sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
1212 }
1213 &AMode::RegReg { rn, rm } => {
1214 let r1 = rn;
1215 let r2 = rm;
1216 sink.put4(enc_ldst_reg(
1217 op, r1, r2, false, None, rd,
1218 ));
1219 }
1220 &AMode::RegScaled { rn, rm } | &AMode::RegScaledExtended { rn, rm, .. } => {
1221 let r1 = rn;
1222 let r2 = rm;
1223 let extendop = match &mem {
1224 &AMode::RegScaled { .. } => None,
1225 &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
1226 _ => unreachable!(),
1227 };
1228 sink.put4(enc_ldst_reg(
1229 op, r1, r2, true, extendop, rd,
1230 ));
1231 }
1232 &AMode::RegExtended { rn, rm, extendop } => {
1233 let r1 = rn;
1234 let r2 = rm;
1235 sink.put4(enc_ldst_reg(
1236 op,
1237 r1,
1238 r2,
1239 false,
1240 Some(extendop),
1241 rd,
1242 ));
1243 }
1244 &AMode::Label { .. } => {
1245 panic!("Store to a MemLabel not implemented!");
1246 }
1247 &AMode::SPPreIndexed { simm9 } => {
1248 let reg = stack_reg();
1249 sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
1250 }
1251 &AMode::SPPostIndexed { simm9 } => {
1252 let reg = stack_reg();
1253 sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
1254 }
1255 &AMode::SPOffset { .. }
1257 | &AMode::FPOffset { .. }
1258 | &AMode::IncomingArg { .. }
1259 | &AMode::SlotOffset { .. }
1260 | &AMode::Const { .. }
1261 | &AMode::RegOffset { .. } => {
1262 panic!("Should not see {mem:?} here!")
1263 }
1264 }
1265 }
1266
1267 &Inst::StoreP64 {
1268 rt,
1269 rt2,
1270 ref mem,
1271 flags,
1272 } => {
1273 let mem = mem.clone();
1274 if let Some(trap_code) = flags.trap_code() {
1275 sink.add_trap(trap_code);
1277 }
1278 match &mem {
1279 &PairAMode::SignedOffset { reg, simm7 } => {
1280 assert_eq!(simm7.scale_ty, I64);
1281 sink.put4(enc_ldst_pair(0b1010100100, simm7, reg, rt, rt2));
1282 }
1283 &PairAMode::SPPreIndexed { simm7 } => {
1284 assert_eq!(simm7.scale_ty, I64);
1285 let reg = stack_reg();
1286 sink.put4(enc_ldst_pair(0b1010100110, simm7, reg, rt, rt2));
1287 }
1288 &PairAMode::SPPostIndexed { simm7 } => {
1289 assert_eq!(simm7.scale_ty, I64);
1290 let reg = stack_reg();
1291 sink.put4(enc_ldst_pair(0b1010100010, simm7, reg, rt, rt2));
1292 }
1293 }
1294 }
1295 &Inst::LoadP64 {
1296 rt,
1297 rt2,
1298 ref mem,
1299 flags,
1300 } => {
1301 let rt = rt.to_reg();
1302 let rt2 = rt2.to_reg();
1303 let mem = mem.clone();
1304 if let Some(trap_code) = flags.trap_code() {
1305 sink.add_trap(trap_code);
1307 }
1308
1309 match &mem {
1310 &PairAMode::SignedOffset { reg, simm7 } => {
1311 assert_eq!(simm7.scale_ty, I64);
1312 sink.put4(enc_ldst_pair(0b1010100101, simm7, reg, rt, rt2));
1313 }
1314 &PairAMode::SPPreIndexed { simm7 } => {
1315 assert_eq!(simm7.scale_ty, I64);
1316 let reg = stack_reg();
1317 sink.put4(enc_ldst_pair(0b1010100111, simm7, reg, rt, rt2));
1318 }
1319 &PairAMode::SPPostIndexed { simm7 } => {
1320 assert_eq!(simm7.scale_ty, I64);
1321 let reg = stack_reg();
1322 sink.put4(enc_ldst_pair(0b1010100011, simm7, reg, rt, rt2));
1323 }
1324 }
1325 }
1326 &Inst::FpuLoadP64 {
1327 rt,
1328 rt2,
1329 ref mem,
1330 flags,
1331 }
1332 | &Inst::FpuLoadP128 {
1333 rt,
1334 rt2,
1335 ref mem,
1336 flags,
1337 } => {
1338 let rt = rt.to_reg();
1339 let rt2 = rt2.to_reg();
1340 let mem = mem.clone();
1341
1342 if let Some(trap_code) = flags.trap_code() {
1343 sink.add_trap(trap_code);
1345 }
1346
1347 let opc = match self {
1348 &Inst::FpuLoadP64 { .. } => 0b01,
1349 &Inst::FpuLoadP128 { .. } => 0b10,
1350 _ => unreachable!(),
1351 };
1352
1353 match &mem {
1354 &PairAMode::SignedOffset { reg, simm7 } => {
1355 assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1356 sink.put4(enc_ldst_vec_pair(opc, 0b10, true, simm7, reg, rt, rt2));
1357 }
1358 &PairAMode::SPPreIndexed { simm7 } => {
1359 assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1360 let reg = stack_reg();
1361 sink.put4(enc_ldst_vec_pair(opc, 0b11, true, simm7, reg, rt, rt2));
1362 }
1363 &PairAMode::SPPostIndexed { simm7 } => {
1364 assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1365 let reg = stack_reg();
1366 sink.put4(enc_ldst_vec_pair(opc, 0b01, true, simm7, reg, rt, rt2));
1367 }
1368 }
1369 }
1370 &Inst::FpuStoreP64 {
1371 rt,
1372 rt2,
1373 ref mem,
1374 flags,
1375 }
1376 | &Inst::FpuStoreP128 {
1377 rt,
1378 rt2,
1379 ref mem,
1380 flags,
1381 } => {
1382 let mem = mem.clone();
1383
1384 if let Some(trap_code) = flags.trap_code() {
1385 sink.add_trap(trap_code);
1387 }
1388
1389 let opc = match self {
1390 &Inst::FpuStoreP64 { .. } => 0b01,
1391 &Inst::FpuStoreP128 { .. } => 0b10,
1392 _ => unreachable!(),
1393 };
1394
1395 match &mem {
1396 &PairAMode::SignedOffset { reg, simm7 } => {
1397 assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1398 sink.put4(enc_ldst_vec_pair(opc, 0b10, false, simm7, reg, rt, rt2));
1399 }
1400 &PairAMode::SPPreIndexed { simm7 } => {
1401 assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1402 let reg = stack_reg();
1403 sink.put4(enc_ldst_vec_pair(opc, 0b11, false, simm7, reg, rt, rt2));
1404 }
1405 &PairAMode::SPPostIndexed { simm7 } => {
1406 assert!(simm7.scale_ty == F64 || simm7.scale_ty == I8X16);
1407 let reg = stack_reg();
1408 sink.put4(enc_ldst_vec_pair(opc, 0b01, false, simm7, reg, rt, rt2));
1409 }
1410 }
1411 }
1412 &Inst::Mov { size, rd, rm } => {
1413 assert!(rd.to_reg().class() == rm.class());
1414 assert!(rm.class() == RegClass::Int);
1415
1416 match size {
1417 OperandSize::Size64 => {
1418 assert!(rd.to_reg() != stack_reg());
1421
1422 if rm == stack_reg() {
1423 let imm12 = Imm12::maybe_from_u64(0).unwrap();
1425 sink.put4(enc_arith_rr_imm12(
1426 0b100_10001,
1427 imm12.shift_bits(),
1428 imm12.imm_bits(),
1429 rm,
1430 rd,
1431 ));
1432 } else {
1433 sink.put4(enc_arith_rrr(0b10101010_000, 0b000_000, rd, zero_reg(), rm));
1435 }
1436 }
1437 OperandSize::Size32 => {
1438 assert!(machreg_to_gpr(rd.to_reg()) != 31);
1441 sink.put4(enc_arith_rrr(0b00101010_000, 0b000_000, rd, zero_reg(), rm));
1443 }
1444 }
1445 }
1446 &Inst::MovFromPReg { rd, rm } => {
1447 let rm: Reg = rm.into();
1448 debug_assert!(
1449 [
1450 regs::fp_reg(),
1451 regs::stack_reg(),
1452 regs::link_reg(),
1453 regs::pinned_reg()
1454 ]
1455 .contains(&rm)
1456 );
1457 assert!(rm.class() == RegClass::Int);
1458 assert!(rd.to_reg().class() == rm.class());
1459 let size = OperandSize::Size64;
1460 Inst::Mov { size, rd, rm }.emit(sink, emit_info, state);
1461 }
1462 &Inst::MovToPReg { rd, rm } => {
1463 let rd: Writable<Reg> = Writable::from_reg(rd.into());
1464 debug_assert!(
1465 [
1466 regs::fp_reg(),
1467 regs::stack_reg(),
1468 regs::link_reg(),
1469 regs::pinned_reg()
1470 ]
1471 .contains(&rd.to_reg())
1472 );
1473 assert!(rd.to_reg().class() == RegClass::Int);
1474 assert!(rm.class() == rd.to_reg().class());
1475 let size = OperandSize::Size64;
1476 Inst::Mov { size, rd, rm }.emit(sink, emit_info, state);
1477 }
1478 &Inst::MovWide { op, rd, imm, size } => {
1479 sink.put4(enc_move_wide(op, rd, imm, size));
1480 }
1481 &Inst::MovK { rd, rn, imm, size } => {
1482 debug_assert_eq!(rn, rd.to_reg());
1483 sink.put4(enc_movk(rd, imm, size));
1484 }
1485 &Inst::CSel { rd, rn, rm, cond } => {
1486 sink.put4(enc_csel(rd, rn, rm, cond, 0, 0));
1487 }
1488 &Inst::CSNeg { rd, rn, rm, cond } => {
1489 sink.put4(enc_csel(rd, rn, rm, cond, 1, 1));
1490 }
1491 &Inst::CSet { rd, cond } => {
1492 sink.put4(enc_csel(rd, zero_reg(), zero_reg(), cond.invert(), 0, 1));
1493 }
1494 &Inst::CSetm { rd, cond } => {
1495 sink.put4(enc_csel(rd, zero_reg(), zero_reg(), cond.invert(), 1, 0));
1496 }
1497 &Inst::CCmp {
1498 size,
1499 rn,
1500 rm,
1501 nzcv,
1502 cond,
1503 } => {
1504 sink.put4(enc_ccmp(size, rn, rm, nzcv, cond));
1505 }
1506 &Inst::CCmpImm {
1507 size,
1508 rn,
1509 imm,
1510 nzcv,
1511 cond,
1512 } => {
1513 sink.put4(enc_ccmp_imm(size, rn, imm, nzcv, cond));
1514 }
1515 &Inst::AtomicRMW {
1516 ty,
1517 op,
1518 rs,
1519 rt,
1520 rn,
1521 flags,
1522 } => {
1523 if let Some(trap_code) = flags.trap_code() {
1524 sink.add_trap(trap_code);
1525 }
1526
1527 sink.put4(enc_acq_rel(ty, op, rs, rt, rn));
1528 }
1529 &Inst::AtomicRMWLoop { ty, op, flags, .. } => {
1530 let xzr = zero_reg();
1550 let x24 = xreg(24);
1551 let x25 = xreg(25);
1552 let x26 = xreg(26);
1553 let x27 = xreg(27);
1554 let x28 = xreg(28);
1555 let x24wr = writable_xreg(24);
1556 let x27wr = writable_xreg(27);
1557 let x28wr = writable_xreg(28);
1558 let again_label = sink.get_label();
1559
1560 sink.bind_label(again_label, &mut state.ctrl_plane);
1562
1563 if let Some(trap_code) = flags.trap_code() {
1564 sink.add_trap(trap_code);
1565 }
1566
1567 sink.put4(enc_ldaxr(ty, x27wr, x25)); let size = OperandSize::from_ty(ty);
1569 let sign_ext = match op {
1570 AtomicRMWLoopOp::Smin | AtomicRMWLoopOp::Smax => match ty {
1571 I16 => Some(ExtendOp::SXTH),
1572 I8 => Some(ExtendOp::SXTB),
1573 _ => None,
1574 },
1575 _ => None,
1576 };
1577 let zero_ext = match op {
1578 AtomicRMWLoopOp::Umin | AtomicRMWLoopOp::Umax => match ty {
1579 I16 => Some(ExtendOp::UXTH),
1580 I8 => Some(ExtendOp::UXTB),
1581 _ => None,
1582 },
1583 _ => None,
1584 };
1585 if let Some(op) = sign_ext {
1587 Inst::gen_extend(x27wr, x27, op, size).emit(sink, emit_info, state);
1588 }
1589
1590 match op {
1591 AtomicRMWLoopOp::Xchg => {} AtomicRMWLoopOp::Nand => {
1593 Inst::AluRRR {
1597 alu_op: ALUOp::And,
1598 size,
1599 rd: x28wr,
1600 rn: x27,
1601 rm: x26,
1602 }
1603 .emit(sink, emit_info, state);
1604
1605 Inst::AluRRR {
1606 alu_op: ALUOp::OrrNot,
1607 size,
1608 rd: x28wr,
1609 rn: xzr,
1610 rm: x28,
1611 }
1612 .emit(sink, emit_info, state);
1613 }
1614 AtomicRMWLoopOp::Umin
1615 | AtomicRMWLoopOp::Umax
1616 | AtomicRMWLoopOp::Smin
1617 | AtomicRMWLoopOp::Smax => {
1618 let cond = match op {
1622 AtomicRMWLoopOp::Umin => Cond::Lo,
1623 AtomicRMWLoopOp::Umax => Cond::Hi,
1624 AtomicRMWLoopOp::Smin => Cond::Lt,
1625 AtomicRMWLoopOp::Smax => Cond::Gt,
1626 _ => unreachable!(),
1627 };
1628
1629 if let Some(extendop) = sign_ext.or(zero_ext) {
1630 Inst::AluRRRExtend {
1631 alu_op: ALUOp::SubS,
1632 size,
1633 rd: writable_zero_reg(),
1634 rn: x27,
1635 rm: x26,
1636 extendop,
1637 }
1638 .emit(sink, emit_info, state);
1639 } else {
1640 Inst::AluRRR {
1641 alu_op: ALUOp::SubS,
1642 size,
1643 rd: writable_zero_reg(),
1644 rn: x27,
1645 rm: x26,
1646 }
1647 .emit(sink, emit_info, state);
1648 }
1649
1650 Inst::CSel {
1651 cond,
1652 rd: x28wr,
1653 rn: x27,
1654 rm: x26,
1655 }
1656 .emit(sink, emit_info, state);
1657 }
1658 _ => {
1659 let alu_op = match op {
1661 AtomicRMWLoopOp::Add => ALUOp::Add,
1662 AtomicRMWLoopOp::Sub => ALUOp::Sub,
1663 AtomicRMWLoopOp::And => ALUOp::And,
1664 AtomicRMWLoopOp::Orr => ALUOp::Orr,
1665 AtomicRMWLoopOp::Eor => ALUOp::Eor,
1666 AtomicRMWLoopOp::Nand
1667 | AtomicRMWLoopOp::Umin
1668 | AtomicRMWLoopOp::Umax
1669 | AtomicRMWLoopOp::Smin
1670 | AtomicRMWLoopOp::Smax
1671 | AtomicRMWLoopOp::Xchg => unreachable!(),
1672 };
1673
1674 Inst::AluRRR {
1675 alu_op,
1676 size,
1677 rd: x28wr,
1678 rn: x27,
1679 rm: x26,
1680 }
1681 .emit(sink, emit_info, state);
1682 }
1683 }
1684
1685 if let Some(trap_code) = flags.trap_code() {
1686 sink.add_trap(trap_code);
1687 }
1688 if op == AtomicRMWLoopOp::Xchg {
1689 sink.put4(enc_stlxr(ty, x24wr, x26, x25)); } else {
1691 sink.put4(enc_stlxr(ty, x24wr, x28, x25)); }
1693
1694 let br_offset = sink.cur_offset();
1698 sink.put4(enc_conditional_br(
1699 BranchTarget::Label(again_label),
1700 CondBrKind::NotZero(x24, OperandSize::Size64),
1701 ));
1702 sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
1703 }
1704 &Inst::AtomicRMW128Loop { op, flags, .. } => {
1705 let xzr = zero_reg();
1724 let x21 = xreg(21);
1725 let x22 = xreg(22);
1726 let x23 = xreg(23);
1727 let x24 = xreg(24);
1728 let x25 = xreg(25);
1729 let x26 = xreg(26);
1730 let x27 = xreg(27);
1731 let x28 = xreg(28);
1732 let x21wr = writable_xreg(21);
1733 let x23wr = writable_xreg(23);
1734 let x24wr = writable_xreg(24);
1735 let x27wr = writable_xreg(27);
1736 let x28wr = writable_xreg(28);
1737 let again_label = sink.get_label();
1738
1739 sink.bind_label(again_label, &mut state.ctrl_plane);
1741
1742 if let Some(trap_code) = flags.trap_code() {
1743 sink.add_trap(trap_code);
1744 }
1745
1746 sink.put4(enc_ldaxp(I64, x27wr, x23wr, x25)); match op {
1749 AtomicRMWLoopOp::Xchg => {} AtomicRMWLoopOp::Smax
1751 | AtomicRMWLoopOp::Umax
1752 | AtomicRMWLoopOp::Smin
1753 | AtomicRMWLoopOp::Umin => {
1754 let cond = match op {
1760 AtomicRMWLoopOp::Smax => Cond::Lt,
1761 AtomicRMWLoopOp::Umax => Cond::Lo,
1762 AtomicRMWLoopOp::Smin => Cond::Ge,
1763 AtomicRMWLoopOp::Umin => Cond::Hs,
1764 _ => unreachable!(),
1765 };
1766
1767 Inst::AluRRR {
1768 alu_op: ALUOp::SubS,
1769 size: OperandSize::Size64,
1770 rd: writable_zero_reg(),
1771 rn: x27,
1772 rm: x26,
1773 }
1774 .emit(sink, emit_info, state);
1775
1776 Inst::AluRRR {
1777 alu_op: ALUOp::SbcS,
1778 size: OperandSize::Size64,
1779 rd: writable_zero_reg(),
1780 rn: x23,
1781 rm: x22,
1782 }
1783 .emit(sink, emit_info, state);
1784
1785 Inst::CSel {
1786 cond,
1787 rd: x21wr,
1788 rn: x22,
1789 rm: x23,
1790 }
1791 .emit(sink, emit_info, state);
1792
1793 Inst::CSel {
1794 cond,
1795 rd: x28wr,
1796 rn: x26,
1797 rm: x27,
1798 }
1799 .emit(sink, emit_info, state);
1800 }
1801 _ => {
1802 let (op_lo, op_hi) = match op {
1803 AtomicRMWLoopOp::Add => (ALUOp::AddS, ALUOp::Adc),
1806 AtomicRMWLoopOp::Sub => (ALUOp::SubS, ALUOp::Sbc),
1809 AtomicRMWLoopOp::And | AtomicRMWLoopOp::Nand => {
1812 (ALUOp::And, ALUOp::And)
1813 }
1814 AtomicRMWLoopOp::Orr => (ALUOp::Orr, ALUOp::Orr),
1817 AtomicRMWLoopOp::Eor => (ALUOp::Eor, ALUOp::Eor),
1820 _ => unreachable!(),
1821 };
1822
1823 Inst::AluRRR {
1824 alu_op: op_lo,
1825 size: OperandSize::Size64,
1826 rd: x28wr,
1827 rn: x27,
1828 rm: x26,
1829 }
1830 .emit(sink, emit_info, state);
1831
1832 Inst::AluRRR {
1833 alu_op: op_hi,
1834 size: OperandSize::Size64,
1835 rd: x21wr,
1836 rn: x23,
1837 rm: x22,
1838 }
1839 .emit(sink, emit_info, state);
1840
1841 if op == AtomicRMWLoopOp::Nand {
1842 Inst::AluRRR {
1846 alu_op: ALUOp::OrrNot,
1847 size: OperandSize::Size64,
1848 rd: x28wr,
1849 rn: xzr,
1850 rm: x28,
1851 }
1852 .emit(sink, emit_info, state);
1853
1854 Inst::AluRRR {
1855 alu_op: ALUOp::OrrNot,
1856 size: OperandSize::Size64,
1857 rd: x21wr,
1858 rn: xzr,
1859 rm: x21,
1860 }
1861 .emit(sink, emit_info, state);
1862 }
1863 }
1864 }
1865
1866 if let Some(trap_code) = flags.trap_code() {
1867 sink.add_trap(trap_code);
1868 }
1869
1870 if op == AtomicRMWLoopOp::Xchg {
1871 sink.put4(enc_stlxp(I64, x24wr, x26, x22, x25)); } else {
1873 sink.put4(enc_stlxp(I64, x24wr, x28, x21, x25)); }
1875
1876 let br_offset = sink.cur_offset();
1880 sink.put4(enc_conditional_br(
1881 BranchTarget::Label(again_label),
1882 CondBrKind::NotZero(x24, OperandSize::Size64),
1883 ));
1884 sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
1885 }
1886 &Inst::AtomicCAS {
1887 rd,
1888 rs,
1889 rt,
1890 rn,
1891 ty,
1892 flags,
1893 } => {
1894 debug_assert_eq!(rd.to_reg(), rs);
1895 let size = match ty {
1896 I8 => 0b00,
1897 I16 => 0b01,
1898 I32 => 0b10,
1899 I64 => 0b11,
1900 _ => panic!("Unsupported type: {ty}"),
1901 };
1902
1903 if let Some(trap_code) = flags.trap_code() {
1904 sink.add_trap(trap_code);
1905 }
1906
1907 sink.put4(enc_cas(size, rd, rt, rn));
1908 }
1909 Inst::AtomicCAS128 { args } => {
1910 let &AtomicCAS128Args {
1911 rd_lo,
1912 rd_hi,
1913 rs_lo,
1914 rs_hi,
1915 rt_lo,
1916 rt_hi,
1917 rn,
1918 flags,
1919 } = &**args;
1920 debug_assert_eq!(rd_lo.to_reg(), rs_lo);
1921 debug_assert_eq!(rd_hi.to_reg(), rs_hi);
1922
1923 debug_assert_eq!(rs_hi, xreg(machreg_to_gpr(rs_lo) as u8 + 1));
1925 debug_assert_eq!(rt_hi, xreg(machreg_to_gpr(rt_lo) as u8 + 1));
1926
1927 if let Some(trap_code) = flags.trap_code() {
1928 sink.add_trap(trap_code);
1929 }
1930
1931 sink.put4(enc_casp(rd_lo, rt_lo, rn));
1932 }
1933 &Inst::AtomicCASLoop { ty, flags, .. } => {
1934 let x24 = xreg(24);
1948 let x25 = xreg(25);
1949 let x26 = xreg(26);
1950 let x27 = xreg(27);
1951 let x28 = xreg(28);
1952 let xzrwr = writable_zero_reg();
1953 let x24wr = writable_xreg(24);
1954 let x27wr = writable_xreg(27);
1955 let again_label = sink.get_label();
1956 let out_label = sink.get_label();
1957
1958 sink.bind_label(again_label, &mut state.ctrl_plane);
1960
1961 if let Some(trap_code) = flags.trap_code() {
1962 sink.add_trap(trap_code);
1963 }
1964
1965 sink.put4(enc_ldaxr(ty, x27wr, x25));
1967
1968 let (sf, bit21, extend_op) = match ty {
1976 I8 => (0b1, 0b1, 0b000000),
1977 I16 => (0b1, 0b1, 0b001000),
1978 I32 => (0b0, 0b0, 0b000000),
1979 _ => (0b1, 0b0, 0b000000),
1980 };
1981 let bits_31_21 = (sf << 10) | 0b011_01011_000 | bit21;
1982 sink.put4(enc_arith_rrr(bits_31_21, extend_op, xzrwr, x27, x26));
1984
1985 let br_out_offset = sink.cur_offset();
1987 sink.put4(enc_conditional_br(
1988 BranchTarget::Label(out_label),
1989 CondBrKind::Cond(Cond::Ne),
1990 ));
1991 sink.use_label_at_offset(br_out_offset, out_label, LabelUse::Branch19);
1992
1993 if let Some(trap_code) = flags.trap_code() {
1994 sink.add_trap(trap_code);
1995 }
1996
1997 sink.put4(enc_stlxr(ty, x24wr, x28, x25)); let br_again_offset = sink.cur_offset();
2003 sink.put4(enc_conditional_br(
2004 BranchTarget::Label(again_label),
2005 CondBrKind::NotZero(x24, OperandSize::Size64),
2006 ));
2007 sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2008
2009 sink.bind_label(out_label, &mut state.ctrl_plane);
2011 }
2012 &Inst::AtomicCAS128Loop { flags, .. } => {
2013 let x21 = xreg(21);
2033 let x22 = xreg(22);
2034 let x23 = xreg(23);
2035 let x24 = xreg(24);
2036 let x25 = xreg(25);
2037 let x26 = xreg(26);
2038 let x27 = xreg(27);
2039 let x28 = xreg(28);
2040 let xzrwr = writable_zero_reg();
2041 let x21wr = writable_xreg(21);
2042 let x24wr = writable_xreg(24);
2043 let x27wr = writable_xreg(27);
2044 let again_label = sink.get_label();
2045 let keep_label = sink.get_label();
2046 let out_label = sink.get_label();
2047
2048 sink.bind_label(again_label, &mut state.ctrl_plane);
2050
2051 if let Some(trap_code) = flags.trap_code() {
2052 sink.add_trap(trap_code);
2053 }
2054
2055 sink.put4(enc_ldaxp(I64, x27wr, x21wr, x25));
2057
2058 Inst::AluRRR {
2060 alu_op: ALUOp::SubS,
2061 size: OperandSize::Size64,
2062 rd: xzrwr,
2063 rn: x27,
2064 rm: x26,
2065 }
2066 .emit(sink, emit_info, state);
2067
2068 let br_keep_offset = sink.cur_offset();
2070 sink.put4(enc_conditional_br(
2071 BranchTarget::Label(keep_label),
2072 CondBrKind::Cond(Cond::Ne),
2073 ));
2074 sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);
2075
2076 Inst::AluRRR {
2078 alu_op: ALUOp::SubS,
2079 size: OperandSize::Size64,
2080 rd: xzrwr,
2081 rn: x21,
2082 rm: x23,
2083 }
2084 .emit(sink, emit_info, state);
2085
2086 let br_keep_offset = sink.cur_offset();
2088 sink.put4(enc_conditional_br(
2089 BranchTarget::Label(keep_label),
2090 CondBrKind::Cond(Cond::Ne),
2091 ));
2092 sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);
2093
2094 if let Some(trap_code) = flags.trap_code() {
2095 sink.add_trap(trap_code);
2096 }
2097
2098 sink.put4(enc_stlxp(I64, x24wr, x28, x22, x25));
2100
2101 let br_again_offset = sink.cur_offset();
2103 sink.put4(enc_conditional_br(
2104 BranchTarget::Label(again_label),
2105 CondBrKind::NotZero(x24, OperandSize::Size64),
2106 ));
2107 sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2108
2109 let b_out_offset = sink.cur_offset();
2111 sink.put4(enc_jump26(
2112 0b000101,
2113 BranchTarget::Label(out_label).as_offset26_or_zero(),
2114 ));
2115 sink.use_label_at_offset(b_out_offset, out_label, LabelUse::Branch26);
2116
2117 sink.bind_label(keep_label, &mut state.ctrl_plane);
2119
2120 if let Some(trap_code) = flags.trap_code() {
2121 sink.add_trap(trap_code);
2122 }
2123
2124 sink.put4(enc_stlxp(I64, x24wr, x27, x21, x25));
2126
2127 let br_again_offset = sink.cur_offset();
2129 sink.put4(enc_conditional_br(
2130 BranchTarget::Label(again_label),
2131 CondBrKind::NotZero(x24, OperandSize::Size64),
2132 ));
2133 sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2134
2135 sink.bind_label(out_label, &mut state.ctrl_plane);
2137 }
2138 &Inst::LoadAcquire {
2139 access_ty,
2140 rt,
2141 rn,
2142 flags,
2143 } => {
2144 if let Some(trap_code) = flags.trap_code() {
2145 sink.add_trap(trap_code);
2146 }
2147
2148 sink.put4(enc_ldar(access_ty, rt, rn));
2149 }
2150 &Inst::LoadAcquire128 {
2151 rt1,
2152 rt2,
2153 rn,
2154 scratch,
2155 flags,
2156 } => {
2157 let again_label = sink.get_label();
2158
2159 sink.bind_label(again_label, &mut state.ctrl_plane);
2161
2162 if let Some(trap_code) = flags.trap_code() {
2163 sink.add_trap(trap_code);
2164 }
2165
2166 sink.put4(enc_ldaxp(I64, rt1, rt2, rn));
2168
2169 if let Some(trap_code) = flags.trap_code() {
2170 sink.add_trap(trap_code);
2171 }
2172
2173 sink.put4(enc_stxp(I64, scratch, rt1.to_reg(), rt2.to_reg(), rn));
2175
2176 let br_again_offset = sink.cur_offset();
2178 sink.put4(enc_conditional_br(
2179 BranchTarget::Label(again_label),
2180 CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),
2181 ));
2182 sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2183 }
2184 &Inst::StoreRelease {
2185 access_ty,
2186 rt,
2187 rn,
2188 flags,
2189 } => {
2190 if let Some(trap_code) = flags.trap_code() {
2191 sink.add_trap(trap_code);
2192 }
2193
2194 sink.put4(enc_stlr(access_ty, rt, rn));
2195 }
2196 &Inst::StoreRelease128 {
2197 rt1,
2198 rt2,
2199 rn,
2200 scratch,
2201 flags,
2202 } => {
2203 let again_label = sink.get_label();
2204
2205 sink.bind_label(again_label, &mut state.ctrl_plane);
2207
2208 if let Some(trap_code) = flags.trap_code() {
2209 sink.add_trap(trap_code);
2210 }
2211
2212 sink.put4(enc_ldxp(I64, writable_zero_reg(), scratch, rn));
2214
2215 if let Some(trap_code) = flags.trap_code() {
2216 sink.add_trap(trap_code);
2217 }
2218
2219 sink.put4(enc_stlxp(I64, scratch, rt1, rt2, rn));
2221
2222 let br_again_offset = sink.cur_offset();
2224 sink.put4(enc_conditional_br(
2225 BranchTarget::Label(again_label),
2226 CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),
2227 ));
2228 sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
2229 }
2230 &Inst::Fence {} => {
2231 sink.put4(enc_dmb_ish()); }
2233 &Inst::Csdb {} => {
2234 sink.put4(0xd503229f);
2235 }
2236 &Inst::FpuMove32 { rd, rn } => {
2237 sink.put4(enc_fpurr(0b000_11110_00_1_000000_10000, rd, rn));
2238 }
2239 &Inst::FpuMove64 { rd, rn } => {
2240 sink.put4(enc_fpurr(0b000_11110_01_1_000000_10000, rd, rn));
2241 }
2242 &Inst::FpuMove128 { rd, rn } => {
2243 sink.put4(enc_vecmov(true, rd, rn));
2244 }
2245 &Inst::FpuMoveFromVec { rd, rn, idx, size } => {
2246 let (imm5, shift, mask) = match size.lane_size() {
2247 ScalarSize::Size32 => (0b00100, 3, 0b011),
2248 ScalarSize::Size64 => (0b01000, 4, 0b001),
2249 _ => unimplemented!(),
2250 };
2251 debug_assert_eq!(idx & mask, idx);
2252 let imm5 = imm5 | ((idx as u32) << shift);
2253 sink.put4(
2254 0b010_11110000_00000_000001_00000_00000
2255 | (imm5 << 16)
2256 | (machreg_to_vec(rn) << 5)
2257 | machreg_to_vec(rd.to_reg()),
2258 );
2259 }
2260 &Inst::FpuExtend { rd, rn, size } => {
2261 sink.put4(enc_fpurr(
2262 0b000_11110_00_1_000000_10000 | (size.ftype() << 12),
2263 rd,
2264 rn,
2265 ));
2266 }
2267 &Inst::FpuRR {
2268 fpu_op,
2269 size,
2270 rd,
2271 rn,
2272 } => {
2273 let top22 = match fpu_op {
2274 FPUOp1::Abs => 0b000_11110_00_1_000001_10000,
2275 FPUOp1::Neg => 0b000_11110_00_1_000010_10000,
2276 FPUOp1::Sqrt => 0b000_11110_00_1_000011_10000,
2277 FPUOp1::Cvt16To32 => {
2278 debug_assert_eq!(size, ScalarSize::Size16);
2279 0b000_11110_11_1_000100_10000
2280 }
2281 FPUOp1::Cvt32To64 => {
2282 debug_assert_eq!(size, ScalarSize::Size32);
2283 0b000_11110_00_1_000101_10000
2284 }
2285 FPUOp1::Cvt64To32 => {
2286 debug_assert_eq!(size, ScalarSize::Size64);
2287 0b000_11110_01_1_000100_10000
2288 }
2289 };
2290 let top22 = top22 | size.ftype() << 12;
2291 sink.put4(enc_fpurr(top22, rd, rn));
2292 }
2293 &Inst::FpuRRR {
2294 fpu_op,
2295 size,
2296 rd,
2297 rn,
2298 rm,
2299 } => {
2300 let top22 = match fpu_op {
2301 FPUOp2::Add => 0b000_11110_00_1_00000_001010,
2302 FPUOp2::Sub => 0b000_11110_00_1_00000_001110,
2303 FPUOp2::Mul => 0b000_11110_00_1_00000_000010,
2304 FPUOp2::Div => 0b000_11110_00_1_00000_000110,
2305 FPUOp2::Max => 0b000_11110_00_1_00000_010010,
2306 FPUOp2::Min => 0b000_11110_00_1_00000_010110,
2307 };
2308 let top22 = top22 | size.ftype() << 12;
2309 sink.put4(enc_fpurrr(top22, rd, rn, rm));
2310 }
2311 &Inst::FpuRRI { fpu_op, rd, rn } => match fpu_op {
2312 FPUOpRI::UShr32(imm) => {
2313 debug_assert_eq!(32, imm.lane_size_in_bits);
2314 sink.put4(
2315 0b0_0_1_011110_0000000_00_0_0_0_1_00000_00000
2316 | imm.enc() << 16
2317 | machreg_to_vec(rn) << 5
2318 | machreg_to_vec(rd.to_reg()),
2319 )
2320 }
2321 FPUOpRI::UShr64(imm) => {
2322 debug_assert_eq!(64, imm.lane_size_in_bits);
2323 sink.put4(
2324 0b01_1_111110_0000000_00_0_0_0_1_00000_00000
2325 | imm.enc() << 16
2326 | machreg_to_vec(rn) << 5
2327 | machreg_to_vec(rd.to_reg()),
2328 )
2329 }
2330 },
2331 &Inst::FpuRRIMod { fpu_op, rd, ri, rn } => {
2332 debug_assert_eq!(rd.to_reg(), ri);
2333 match fpu_op {
2334 FPUOpRIMod::Sli64(imm) => {
2335 debug_assert_eq!(64, imm.lane_size_in_bits);
2336 sink.put4(
2337 0b01_1_111110_0000000_010101_00000_00000
2338 | imm.enc() << 16
2339 | machreg_to_vec(rn) << 5
2340 | machreg_to_vec(rd.to_reg()),
2341 )
2342 }
2343 FPUOpRIMod::Sli32(imm) => {
2344 debug_assert_eq!(32, imm.lane_size_in_bits);
2345 sink.put4(
2346 0b0_0_1_011110_0000000_010101_00000_00000
2347 | imm.enc() << 16
2348 | machreg_to_vec(rn) << 5
2349 | machreg_to_vec(rd.to_reg()),
2350 )
2351 }
2352 }
2353 }
2354 &Inst::FpuRRRR {
2355 fpu_op,
2356 size,
2357 rd,
2358 rn,
2359 rm,
2360 ra,
2361 } => {
2362 let top17 = match fpu_op {
2363 FPUOp3::MAdd => 0b000_11111_00_0_00000_0,
2364 FPUOp3::MSub => 0b000_11111_00_0_00000_1,
2365 FPUOp3::NMAdd => 0b000_11111_00_1_00000_0,
2366 FPUOp3::NMSub => 0b000_11111_00_1_00000_1,
2367 };
2368 let top17 = top17 | size.ftype() << 7;
2369 sink.put4(enc_fpurrrr(top17, rd, rn, rm, ra));
2370 }
2371 &Inst::VecMisc { op, rd, rn, size } => {
2372 let (q, enc_size) = size.enc_size();
2373 let (u, bits_12_16, size) = match op {
2374 VecMisc2::Not => (0b1, 0b00101, 0b00),
2375 VecMisc2::Neg => (0b1, 0b01011, enc_size),
2376 VecMisc2::Abs => (0b0, 0b01011, enc_size),
2377 VecMisc2::Fabs => {
2378 debug_assert!(
2379 size == VectorSize::Size32x2
2380 || size == VectorSize::Size32x4
2381 || size == VectorSize::Size64x2
2382 );
2383 (0b0, 0b01111, enc_size)
2384 }
2385 VecMisc2::Fneg => {
2386 debug_assert!(
2387 size == VectorSize::Size32x2
2388 || size == VectorSize::Size32x4
2389 || size == VectorSize::Size64x2
2390 );
2391 (0b1, 0b01111, enc_size)
2392 }
2393 VecMisc2::Fsqrt => {
2394 debug_assert!(
2395 size == VectorSize::Size32x2
2396 || size == VectorSize::Size32x4
2397 || size == VectorSize::Size64x2
2398 );
2399 (0b1, 0b11111, enc_size)
2400 }
2401 VecMisc2::Rev16 => {
2402 debug_assert_eq!(size, VectorSize::Size8x16);
2403 (0b0, 0b00001, enc_size)
2404 }
2405 VecMisc2::Rev32 => {
2406 debug_assert!(size == VectorSize::Size8x16 || size == VectorSize::Size16x8);
2407 (0b1, 0b00000, enc_size)
2408 }
2409 VecMisc2::Rev64 => {
2410 debug_assert!(
2411 size == VectorSize::Size8x16
2412 || size == VectorSize::Size16x8
2413 || size == VectorSize::Size32x4
2414 );
2415 (0b0, 0b00000, enc_size)
2416 }
2417 VecMisc2::Fcvtzs => {
2418 debug_assert!(
2419 size == VectorSize::Size32x2
2420 || size == VectorSize::Size32x4
2421 || size == VectorSize::Size64x2
2422 );
2423 (0b0, 0b11011, enc_size)
2424 }
2425 VecMisc2::Fcvtzu => {
2426 debug_assert!(
2427 size == VectorSize::Size32x2
2428 || size == VectorSize::Size32x4
2429 || size == VectorSize::Size64x2
2430 );
2431 (0b1, 0b11011, enc_size)
2432 }
2433 VecMisc2::Scvtf => {
2434 debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2);
2435 (0b0, 0b11101, enc_size & 0b1)
2436 }
2437 VecMisc2::Ucvtf => {
2438 debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2);
2439 (0b1, 0b11101, enc_size & 0b1)
2440 }
2441 VecMisc2::Frintn => {
2442 debug_assert!(
2443 size == VectorSize::Size32x2
2444 || size == VectorSize::Size32x4
2445 || size == VectorSize::Size64x2
2446 );
2447 (0b0, 0b11000, enc_size & 0b01)
2448 }
2449 VecMisc2::Frintz => {
2450 debug_assert!(
2451 size == VectorSize::Size32x2
2452 || size == VectorSize::Size32x4
2453 || size == VectorSize::Size64x2
2454 );
2455 (0b0, 0b11001, enc_size)
2456 }
2457 VecMisc2::Frintm => {
2458 debug_assert!(
2459 size == VectorSize::Size32x2
2460 || size == VectorSize::Size32x4
2461 || size == VectorSize::Size64x2
2462 );
2463 (0b0, 0b11001, enc_size & 0b01)
2464 }
2465 VecMisc2::Frintp => {
2466 debug_assert!(
2467 size == VectorSize::Size32x2
2468 || size == VectorSize::Size32x4
2469 || size == VectorSize::Size64x2
2470 );
2471 (0b0, 0b11000, enc_size)
2472 }
2473 VecMisc2::Cnt => {
2474 debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16);
2475 (0b0, 0b00101, enc_size)
2476 }
2477 VecMisc2::Cmeq0 => (0b0, 0b01001, enc_size),
2478 VecMisc2::Cmge0 => (0b1, 0b01000, enc_size),
2479 VecMisc2::Cmgt0 => (0b0, 0b01000, enc_size),
2480 VecMisc2::Cmle0 => (0b1, 0b01001, enc_size),
2481 VecMisc2::Cmlt0 => (0b0, 0b01010, enc_size),
2482 VecMisc2::Fcmeq0 => {
2483 debug_assert!(
2484 size == VectorSize::Size32x2
2485 || size == VectorSize::Size32x4
2486 || size == VectorSize::Size64x2
2487 );
2488 (0b0, 0b01101, enc_size)
2489 }
2490 VecMisc2::Fcmge0 => {
2491 debug_assert!(
2492 size == VectorSize::Size32x2
2493 || size == VectorSize::Size32x4
2494 || size == VectorSize::Size64x2
2495 );
2496 (0b1, 0b01100, enc_size)
2497 }
2498 VecMisc2::Fcmgt0 => {
2499 debug_assert!(
2500 size == VectorSize::Size32x2
2501 || size == VectorSize::Size32x4
2502 || size == VectorSize::Size64x2
2503 );
2504 (0b0, 0b01100, enc_size)
2505 }
2506 VecMisc2::Fcmle0 => {
2507 debug_assert!(
2508 size == VectorSize::Size32x2
2509 || size == VectorSize::Size32x4
2510 || size == VectorSize::Size64x2
2511 );
2512 (0b1, 0b01101, enc_size)
2513 }
2514 VecMisc2::Fcmlt0 => {
2515 debug_assert!(
2516 size == VectorSize::Size32x2
2517 || size == VectorSize::Size32x4
2518 || size == VectorSize::Size64x2
2519 );
2520 (0b0, 0b01110, enc_size)
2521 }
2522 };
2523 sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn));
2524 }
2525 &Inst::VecLanes { op, rd, rn, size } => {
2526 let (q, size) = match size {
2527 VectorSize::Size8x8 => (0b0, 0b00),
2528 VectorSize::Size8x16 => (0b1, 0b00),
2529 VectorSize::Size16x4 => (0b0, 0b01),
2530 VectorSize::Size16x8 => (0b1, 0b01),
2531 VectorSize::Size32x4 => (0b1, 0b10),
2532 _ => unreachable!(),
2533 };
2534 let (u, opcode) = match op {
2535 VecLanesOp::Uminv => (0b1, 0b11010),
2536 VecLanesOp::Addv => (0b0, 0b11011),
2537 };
2538 sink.put4(enc_vec_lanes(q, u, size, opcode, rd, rn));
2539 }
2540 &Inst::VecShiftImm {
2541 op,
2542 rd,
2543 rn,
2544 size,
2545 imm,
2546 } => {
2547 let (is_shr, mut template) = match op {
2548 VecShiftImmOp::Ushr => (true, 0b_001_011110_0000_000_000001_00000_00000_u32),
2549 VecShiftImmOp::Sshr => (true, 0b_000_011110_0000_000_000001_00000_00000_u32),
2550 VecShiftImmOp::Shl => (false, 0b_000_011110_0000_000_010101_00000_00000_u32),
2551 };
2552 if size.is_128bits() {
2553 template |= 0b1 << 30;
2554 }
2555 let imm = imm as u32;
2556 let immh_immb = match (size.lane_size(), is_shr) {
2559 (ScalarSize::Size64, true) if imm >= 1 && imm <= 64 => {
2560 0b_1000_000_u32 | (64 - imm)
2561 }
2562 (ScalarSize::Size32, true) if imm >= 1 && imm <= 32 => {
2563 0b_0100_000_u32 | (32 - imm)
2564 }
2565 (ScalarSize::Size16, true) if imm >= 1 && imm <= 16 => {
2566 0b_0010_000_u32 | (16 - imm)
2567 }
2568 (ScalarSize::Size8, true) if imm >= 1 && imm <= 8 => {
2569 0b_0001_000_u32 | (8 - imm)
2570 }
2571 (ScalarSize::Size64, false) if imm <= 63 => 0b_1000_000_u32 | imm,
2572 (ScalarSize::Size32, false) if imm <= 31 => 0b_0100_000_u32 | imm,
2573 (ScalarSize::Size16, false) if imm <= 15 => 0b_0010_000_u32 | imm,
2574 (ScalarSize::Size8, false) if imm <= 7 => 0b_0001_000_u32 | imm,
2575 _ => panic!(
2576 "aarch64: Inst::VecShiftImm: emit: invalid op/size/imm {op:?}, {size:?}, {imm:?}"
2577 ),
2578 };
2579 let rn_enc = machreg_to_vec(rn);
2580 let rd_enc = machreg_to_vec(rd.to_reg());
2581 sink.put4(template | (immh_immb << 16) | (rn_enc << 5) | rd_enc);
2582 }
2583 &Inst::VecShiftImmMod {
2584 op,
2585 rd,
2586 ri,
2587 rn,
2588 size,
2589 imm,
2590 } => {
2591 debug_assert_eq!(rd.to_reg(), ri);
2592 let (is_shr, mut template) = match op {
2593 VecShiftImmModOp::Sli => (false, 0b_001_011110_0000_000_010101_00000_00000_u32),
2594 };
2595 if size.is_128bits() {
2596 template |= 0b1 << 30;
2597 }
2598 let imm = imm as u32;
2599 let immh_immb = match (size.lane_size(), is_shr) {
2602 (ScalarSize::Size64, true) if imm >= 1 && imm <= 64 => {
2603 0b_1000_000_u32 | (64 - imm)
2604 }
2605 (ScalarSize::Size32, true) if imm >= 1 && imm <= 32 => {
2606 0b_0100_000_u32 | (32 - imm)
2607 }
2608 (ScalarSize::Size16, true) if imm >= 1 && imm <= 16 => {
2609 0b_0010_000_u32 | (16 - imm)
2610 }
2611 (ScalarSize::Size8, true) if imm >= 1 && imm <= 8 => {
2612 0b_0001_000_u32 | (8 - imm)
2613 }
2614 (ScalarSize::Size64, false) if imm <= 63 => 0b_1000_000_u32 | imm,
2615 (ScalarSize::Size32, false) if imm <= 31 => 0b_0100_000_u32 | imm,
2616 (ScalarSize::Size16, false) if imm <= 15 => 0b_0010_000_u32 | imm,
2617 (ScalarSize::Size8, false) if imm <= 7 => 0b_0001_000_u32 | imm,
2618 _ => panic!(
2619 "aarch64: Inst::VecShiftImmMod: emit: invalid op/size/imm {op:?}, {size:?}, {imm:?}"
2620 ),
2621 };
2622 let rn_enc = machreg_to_vec(rn);
2623 let rd_enc = machreg_to_vec(rd.to_reg());
2624 sink.put4(template | (immh_immb << 16) | (rn_enc << 5) | rd_enc);
2625 }
2626 &Inst::VecExtract { rd, rn, rm, imm4 } => {
2627 if imm4 < 16 {
2628 let template = 0b_01_101110_000_00000_0_0000_0_00000_00000_u32;
2629 let rm_enc = machreg_to_vec(rm);
2630 let rn_enc = machreg_to_vec(rn);
2631 let rd_enc = machreg_to_vec(rd.to_reg());
2632 sink.put4(
2633 template | (rm_enc << 16) | ((imm4 as u32) << 11) | (rn_enc << 5) | rd_enc,
2634 );
2635 } else {
2636 panic!("aarch64: Inst::VecExtract: emit: invalid extract index {imm4}");
2637 }
2638 }
2639 &Inst::VecTbl { rd, rn, rm } => {
2640 sink.put4(enc_tbl(false, 0b00, rd, rn, rm));
2641 }
2642 &Inst::VecTblExt { rd, ri, rn, rm } => {
2643 debug_assert_eq!(rd.to_reg(), ri);
2644 sink.put4(enc_tbl(true, 0b00, rd, rn, rm));
2645 }
2646 &Inst::VecTbl2 { rd, rn, rn2, rm } => {
2647 assert_eq!(machreg_to_vec(rn2), (machreg_to_vec(rn) + 1) % 32);
2648 sink.put4(enc_tbl(false, 0b01, rd, rn, rm));
2649 }
2650 &Inst::VecTbl2Ext {
2651 rd,
2652 ri,
2653 rn,
2654 rn2,
2655 rm,
2656 } => {
2657 debug_assert_eq!(rd.to_reg(), ri);
2658 assert_eq!(machreg_to_vec(rn2), (machreg_to_vec(rn) + 1) % 32);
2659 sink.put4(enc_tbl(true, 0b01, rd, rn, rm));
2660 }
2661 &Inst::FpuCmp { size, rn, rm } => {
2662 sink.put4(enc_fcmp(size, rn, rm));
2663 }
2664 &Inst::FpuToInt { op, rd, rn } => {
2665 let top16 = match op {
2666 FpuToIntOp::F32ToI32 => 0b000_11110_00_1_11_000,
2668 FpuToIntOp::F32ToU32 => 0b000_11110_00_1_11_001,
2670 FpuToIntOp::F32ToI64 => 0b100_11110_00_1_11_000,
2672 FpuToIntOp::F32ToU64 => 0b100_11110_00_1_11_001,
2674 FpuToIntOp::F64ToI32 => 0b000_11110_01_1_11_000,
2676 FpuToIntOp::F64ToU32 => 0b000_11110_01_1_11_001,
2678 FpuToIntOp::F64ToI64 => 0b100_11110_01_1_11_000,
2680 FpuToIntOp::F64ToU64 => 0b100_11110_01_1_11_001,
2682 };
2683 sink.put4(enc_fputoint(top16, rd, rn));
2684 }
2685 &Inst::IntToFpu { op, rd, rn } => {
2686 let top16 = match op {
2687 IntToFpuOp::I32ToF32 => 0b000_11110_00_1_00_010,
2689 IntToFpuOp::U32ToF32 => 0b000_11110_00_1_00_011,
2691 IntToFpuOp::I64ToF32 => 0b100_11110_00_1_00_010,
2693 IntToFpuOp::U64ToF32 => 0b100_11110_00_1_00_011,
2695 IntToFpuOp::I32ToF64 => 0b000_11110_01_1_00_010,
2697 IntToFpuOp::U32ToF64 => 0b000_11110_01_1_00_011,
2699 IntToFpuOp::I64ToF64 => 0b100_11110_01_1_00_010,
2701 IntToFpuOp::U64ToF64 => 0b100_11110_01_1_00_011,
2703 };
2704 sink.put4(enc_inttofpu(top16, rd, rn));
2705 }
2706 &Inst::FpuCSel16 { rd, rn, rm, cond } => {
2707 sink.put4(enc_fcsel(rd, rn, rm, cond, ScalarSize::Size16));
2708 }
2709 &Inst::FpuCSel32 { rd, rn, rm, cond } => {
2710 sink.put4(enc_fcsel(rd, rn, rm, cond, ScalarSize::Size32));
2711 }
2712 &Inst::FpuCSel64 { rd, rn, rm, cond } => {
2713 sink.put4(enc_fcsel(rd, rn, rm, cond, ScalarSize::Size64));
2714 }
2715 &Inst::FpuRound { op, rd, rn } => {
2716 let top22 = match op {
2717 FpuRoundMode::Minus32 => 0b000_11110_00_1_001_010_10000,
2718 FpuRoundMode::Minus64 => 0b000_11110_01_1_001_010_10000,
2719 FpuRoundMode::Plus32 => 0b000_11110_00_1_001_001_10000,
2720 FpuRoundMode::Plus64 => 0b000_11110_01_1_001_001_10000,
2721 FpuRoundMode::Zero32 => 0b000_11110_00_1_001_011_10000,
2722 FpuRoundMode::Zero64 => 0b000_11110_01_1_001_011_10000,
2723 FpuRoundMode::Nearest32 => 0b000_11110_00_1_001_000_10000,
2724 FpuRoundMode::Nearest64 => 0b000_11110_01_1_001_000_10000,
2725 };
2726 sink.put4(enc_fround(top22, rd, rn));
2727 }
2728 &Inst::MovToFpu { rd, rn, size } => {
2729 let template = match size {
2730 ScalarSize::Size16 => 0b000_11110_11_1_00_111_000000_00000_00000,
2731 ScalarSize::Size32 => 0b000_11110_00_1_00_111_000000_00000_00000,
2732 ScalarSize::Size64 => 0b100_11110_01_1_00_111_000000_00000_00000,
2733 _ => unreachable!(),
2734 };
2735 sink.put4(template | (machreg_to_gpr(rn) << 5) | machreg_to_vec(rd.to_reg()));
2736 }
2737 &Inst::FpuMoveFPImm { rd, imm, size } => {
2738 sink.put4(
2739 0b000_11110_00_1_00_000_000100_00000_00000
2740 | size.ftype() << 22
2741 | ((imm.enc_bits() as u32) << 13)
2742 | machreg_to_vec(rd.to_reg()),
2743 );
2744 }
2745 &Inst::MovToVec {
2746 rd,
2747 ri,
2748 rn,
2749 idx,
2750 size,
2751 } => {
2752 debug_assert_eq!(rd.to_reg(), ri);
2753 let (imm5, shift) = match size.lane_size() {
2754 ScalarSize::Size8 => (0b00001, 1),
2755 ScalarSize::Size16 => (0b00010, 2),
2756 ScalarSize::Size32 => (0b00100, 3),
2757 ScalarSize::Size64 => (0b01000, 4),
2758 _ => unreachable!(),
2759 };
2760 debug_assert_eq!(idx & (0b11111 >> shift), idx);
2761 let imm5 = imm5 | ((idx as u32) << shift);
2762 sink.put4(
2763 0b010_01110000_00000_0_0011_1_00000_00000
2764 | (imm5 << 16)
2765 | (machreg_to_gpr(rn) << 5)
2766 | machreg_to_vec(rd.to_reg()),
2767 );
2768 }
2769 &Inst::MovFromVec { rd, rn, idx, size } => {
2770 let (q, imm5, shift, mask) = match size {
2771 ScalarSize::Size8 => (0b0, 0b00001, 1, 0b1111),
2772 ScalarSize::Size16 => (0b0, 0b00010, 2, 0b0111),
2773 ScalarSize::Size32 => (0b0, 0b00100, 3, 0b0011),
2774 ScalarSize::Size64 => (0b1, 0b01000, 4, 0b0001),
2775 _ => panic!("Unexpected scalar FP operand size: {size:?}"),
2776 };
2777 debug_assert_eq!(idx & mask, idx);
2778 let imm5 = imm5 | ((idx as u32) << shift);
2779 sink.put4(
2780 0b000_01110000_00000_0_0111_1_00000_00000
2781 | (q << 30)
2782 | (imm5 << 16)
2783 | (machreg_to_vec(rn) << 5)
2784 | machreg_to_gpr(rd.to_reg()),
2785 );
2786 }
2787 &Inst::MovFromVecSigned {
2788 rd,
2789 rn,
2790 idx,
2791 size,
2792 scalar_size,
2793 } => {
2794 let (imm5, shift, half) = match size {
2795 VectorSize::Size8x8 => (0b00001, 1, true),
2796 VectorSize::Size8x16 => (0b00001, 1, false),
2797 VectorSize::Size16x4 => (0b00010, 2, true),
2798 VectorSize::Size16x8 => (0b00010, 2, false),
2799 VectorSize::Size32x2 => {
2800 debug_assert_ne!(scalar_size, OperandSize::Size32);
2801 (0b00100, 3, true)
2802 }
2803 VectorSize::Size32x4 => {
2804 debug_assert_ne!(scalar_size, OperandSize::Size32);
2805 (0b00100, 3, false)
2806 }
2807 _ => panic!("Unexpected vector operand size"),
2808 };
2809 debug_assert_eq!(idx & (0b11111 >> (half as u32 + shift)), idx);
2810 let imm5 = imm5 | ((idx as u32) << shift);
2811 sink.put4(
2812 0b000_01110000_00000_0_0101_1_00000_00000
2813 | (scalar_size.is64() as u32) << 30
2814 | (imm5 << 16)
2815 | (machreg_to_vec(rn) << 5)
2816 | machreg_to_gpr(rd.to_reg()),
2817 );
2818 }
2819 &Inst::VecDup { rd, rn, size } => {
2820 let q = size.is_128bits() as u32;
2821 let imm5 = match size.lane_size() {
2822 ScalarSize::Size8 => 0b00001,
2823 ScalarSize::Size16 => 0b00010,
2824 ScalarSize::Size32 => 0b00100,
2825 ScalarSize::Size64 => 0b01000,
2826 _ => unreachable!(),
2827 };
2828 sink.put4(
2829 0b0_0_0_01110000_00000_000011_00000_00000
2830 | (q << 30)
2831 | (imm5 << 16)
2832 | (machreg_to_gpr(rn) << 5)
2833 | machreg_to_vec(rd.to_reg()),
2834 );
2835 }
2836 &Inst::VecDupFromFpu { rd, rn, size, lane } => {
2837 let q = size.is_128bits() as u32;
2838 let imm5 = match size.lane_size() {
2839 ScalarSize::Size8 => {
2840 assert!(lane < 16);
2841 0b00001 | (u32::from(lane) << 1)
2842 }
2843 ScalarSize::Size16 => {
2844 assert!(lane < 8);
2845 0b00010 | (u32::from(lane) << 2)
2846 }
2847 ScalarSize::Size32 => {
2848 assert!(lane < 4);
2849 0b00100 | (u32::from(lane) << 3)
2850 }
2851 ScalarSize::Size64 => {
2852 assert!(lane < 2);
2853 0b01000 | (u32::from(lane) << 4)
2854 }
2855 _ => unimplemented!(),
2856 };
2857 sink.put4(
2858 0b000_01110000_00000_000001_00000_00000
2859 | (q << 30)
2860 | (imm5 << 16)
2861 | (machreg_to_vec(rn) << 5)
2862 | machreg_to_vec(rd.to_reg()),
2863 );
2864 }
2865 &Inst::VecDupFPImm { rd, imm, size } => {
2866 let imm = imm.enc_bits();
2867 let op = match size.lane_size() {
2868 ScalarSize::Size32 => 0,
2869 ScalarSize::Size64 => 1,
2870 _ => unimplemented!(),
2871 };
2872 let q_op = op | ((size.is_128bits() as u32) << 1);
2873
2874 sink.put4(enc_asimd_mod_imm(rd, q_op, 0b1111, imm));
2875 }
2876 &Inst::VecDupImm {
2877 rd,
2878 imm,
2879 invert,
2880 size,
2881 } => {
2882 let (imm, shift, shift_ones) = imm.value();
2883 let (op, cmode) = match size.lane_size() {
2884 ScalarSize::Size8 => {
2885 assert!(!invert);
2886 assert_eq!(shift, 0);
2887
2888 (0, 0b1110)
2889 }
2890 ScalarSize::Size16 => {
2891 let s = shift & 8;
2892
2893 assert!(!shift_ones);
2894 assert_eq!(s, shift);
2895
2896 (invert as u32, 0b1000 | (s >> 2))
2897 }
2898 ScalarSize::Size32 => {
2899 if shift_ones {
2900 assert!(shift == 8 || shift == 16);
2901
2902 (invert as u32, 0b1100 | (shift >> 4))
2903 } else {
2904 let s = shift & 24;
2905
2906 assert_eq!(s, shift);
2907
2908 (invert as u32, 0b0000 | (s >> 2))
2909 }
2910 }
2911 ScalarSize::Size64 => {
2912 assert!(!invert);
2913 assert_eq!(shift, 0);
2914
2915 (1, 0b1110)
2916 }
2917 _ => unreachable!(),
2918 };
2919 let q_op = op | ((size.is_128bits() as u32) << 1);
2920
2921 sink.put4(enc_asimd_mod_imm(rd, q_op, cmode, imm));
2922 }
2923 &Inst::VecExtend {
2924 t,
2925 rd,
2926 rn,
2927 high_half,
2928 lane_size,
2929 } => {
2930 let immh = match lane_size {
2931 ScalarSize::Size16 => 0b001,
2932 ScalarSize::Size32 => 0b010,
2933 ScalarSize::Size64 => 0b100,
2934 _ => panic!("Unexpected VecExtend to lane size of {lane_size:?}"),
2935 };
2936 let u = match t {
2937 VecExtendOp::Sxtl => 0b0,
2938 VecExtendOp::Uxtl => 0b1,
2939 };
2940 sink.put4(
2941 0b000_011110_0000_000_101001_00000_00000
2942 | ((high_half as u32) << 30)
2943 | (u << 29)
2944 | (immh << 19)
2945 | (machreg_to_vec(rn) << 5)
2946 | machreg_to_vec(rd.to_reg()),
2947 );
2948 }
2949 &Inst::VecRRLong {
2950 op,
2951 rd,
2952 rn,
2953 high_half,
2954 } => {
2955 let (u, size, bits_12_16) = match op {
2956 VecRRLongOp::Fcvtl16 => (0b0, 0b00, 0b10111),
2957 VecRRLongOp::Fcvtl32 => (0b0, 0b01, 0b10111),
2958 VecRRLongOp::Shll8 => (0b1, 0b00, 0b10011),
2959 VecRRLongOp::Shll16 => (0b1, 0b01, 0b10011),
2960 VecRRLongOp::Shll32 => (0b1, 0b10, 0b10011),
2961 };
2962
2963 sink.put4(enc_vec_rr_misc(
2964 ((high_half as u32) << 1) | u,
2965 size,
2966 bits_12_16,
2967 rd,
2968 rn,
2969 ));
2970 }
2971 &Inst::VecRRNarrowLow {
2972 op,
2973 rd,
2974 rn,
2975 lane_size,
2976 }
2977 | &Inst::VecRRNarrowHigh {
2978 op,
2979 rd,
2980 rn,
2981 lane_size,
2982 ..
2983 } => {
2984 let high_half = match self {
2985 &Inst::VecRRNarrowLow { .. } => false,
2986 &Inst::VecRRNarrowHigh { .. } => true,
2987 _ => unreachable!(),
2988 };
2989
2990 let size = match lane_size {
2991 ScalarSize::Size8 => 0b00,
2992 ScalarSize::Size16 => 0b01,
2993 ScalarSize::Size32 => 0b10,
2994 _ => panic!("unsupported size: {lane_size:?}"),
2995 };
2996
2997 let size = match op {
2999 VecRRNarrowOp::Fcvtn => size >> 1,
3000 _ => size,
3001 };
3002
3003 let (u, bits_12_16) = match op {
3004 VecRRNarrowOp::Xtn => (0b0, 0b10010),
3005 VecRRNarrowOp::Sqxtn => (0b0, 0b10100),
3006 VecRRNarrowOp::Sqxtun => (0b1, 0b10010),
3007 VecRRNarrowOp::Uqxtn => (0b1, 0b10100),
3008 VecRRNarrowOp::Fcvtn => (0b0, 0b10110),
3009 };
3010
3011 sink.put4(enc_vec_rr_misc(
3012 ((high_half as u32) << 1) | u,
3013 size,
3014 bits_12_16,
3015 rd,
3016 rn,
3017 ));
3018 }
3019 &Inst::VecMovElement {
3020 rd,
3021 ri,
3022 rn,
3023 dest_idx,
3024 src_idx,
3025 size,
3026 } => {
3027 debug_assert_eq!(rd.to_reg(), ri);
3028 let (imm5, shift) = match size.lane_size() {
3029 ScalarSize::Size8 => (0b00001, 1),
3030 ScalarSize::Size16 => (0b00010, 2),
3031 ScalarSize::Size32 => (0b00100, 3),
3032 ScalarSize::Size64 => (0b01000, 4),
3033 _ => unreachable!(),
3034 };
3035 let mask = 0b11111 >> shift;
3036 debug_assert_eq!(dest_idx & mask, dest_idx);
3037 debug_assert_eq!(src_idx & mask, src_idx);
3038 let imm4 = (src_idx as u32) << (shift - 1);
3039 let imm5 = imm5 | ((dest_idx as u32) << shift);
3040 sink.put4(
3041 0b011_01110000_00000_0_0000_1_00000_00000
3042 | (imm5 << 16)
3043 | (imm4 << 11)
3044 | (machreg_to_vec(rn) << 5)
3045 | machreg_to_vec(rd.to_reg()),
3046 );
3047 }
3048 &Inst::VecRRPair { op, rd, rn } => {
3049 let bits_12_16 = match op {
3050 VecPairOp::Addp => 0b11011,
3051 };
3052
3053 sink.put4(enc_vec_rr_pair(bits_12_16, rd, rn));
3054 }
3055 &Inst::VecRRRLong {
3056 rd,
3057 rn,
3058 rm,
3059 alu_op,
3060 high_half,
3061 } => {
3062 let (u, size, bit14) = match alu_op {
3063 VecRRRLongOp::Smull8 => (0b0, 0b00, 0b1),
3064 VecRRRLongOp::Smull16 => (0b0, 0b01, 0b1),
3065 VecRRRLongOp::Smull32 => (0b0, 0b10, 0b1),
3066 VecRRRLongOp::Umull8 => (0b1, 0b00, 0b1),
3067 VecRRRLongOp::Umull16 => (0b1, 0b01, 0b1),
3068 VecRRRLongOp::Umull32 => (0b1, 0b10, 0b1),
3069 };
3070 sink.put4(enc_vec_rrr_long(
3071 high_half as u32,
3072 u,
3073 size,
3074 bit14,
3075 rm,
3076 rn,
3077 rd,
3078 ));
3079 }
3080 &Inst::VecRRRLongMod {
3081 rd,
3082 ri,
3083 rn,
3084 rm,
3085 alu_op,
3086 high_half,
3087 } => {
3088 debug_assert_eq!(rd.to_reg(), ri);
3089 let (u, size, bit14) = match alu_op {
3090 VecRRRLongModOp::Umlal8 => (0b1, 0b00, 0b0),
3091 VecRRRLongModOp::Umlal16 => (0b1, 0b01, 0b0),
3092 VecRRRLongModOp::Umlal32 => (0b1, 0b10, 0b0),
3093 };
3094 sink.put4(enc_vec_rrr_long(
3095 high_half as u32,
3096 u,
3097 size,
3098 bit14,
3099 rm,
3100 rn,
3101 rd,
3102 ));
3103 }
3104 &Inst::VecRRPairLong { op, rd, rn } => {
3105 let (u, size) = match op {
3106 VecRRPairLongOp::Saddlp8 => (0b0, 0b0),
3107 VecRRPairLongOp::Uaddlp8 => (0b1, 0b0),
3108 VecRRPairLongOp::Saddlp16 => (0b0, 0b1),
3109 VecRRPairLongOp::Uaddlp16 => (0b1, 0b1),
3110 };
3111
3112 sink.put4(enc_vec_rr_pair_long(u, size, rd, rn));
3113 }
3114 &Inst::VecRRR {
3115 rd,
3116 rn,
3117 rm,
3118 alu_op,
3119 size,
3120 } => {
3121 let (q, enc_size) = size.enc_size();
3122 let is_float = match alu_op {
3123 VecALUOp::Fcmeq
3124 | VecALUOp::Fcmgt
3125 | VecALUOp::Fcmge
3126 | VecALUOp::Fadd
3127 | VecALUOp::Fsub
3128 | VecALUOp::Fdiv
3129 | VecALUOp::Fmax
3130 | VecALUOp::Fmin
3131 | VecALUOp::Fmul => true,
3132 _ => false,
3133 };
3134
3135 let (top11, bit15_10) = match alu_op {
3136 VecALUOp::Sqadd => (0b000_01110_00_1 | enc_size << 1, 0b000011),
3137 VecALUOp::Sqsub => (0b000_01110_00_1 | enc_size << 1, 0b001011),
3138 VecALUOp::Uqadd => (0b001_01110_00_1 | enc_size << 1, 0b000011),
3139 VecALUOp::Uqsub => (0b001_01110_00_1 | enc_size << 1, 0b001011),
3140 VecALUOp::Cmeq => (0b001_01110_00_1 | enc_size << 1, 0b100011),
3141 VecALUOp::Cmge => (0b000_01110_00_1 | enc_size << 1, 0b001111),
3142 VecALUOp::Cmgt => (0b000_01110_00_1 | enc_size << 1, 0b001101),
3143 VecALUOp::Cmhi => (0b001_01110_00_1 | enc_size << 1, 0b001101),
3144 VecALUOp::Cmhs => (0b001_01110_00_1 | enc_size << 1, 0b001111),
3145 VecALUOp::Fcmeq => (0b000_01110_00_1, 0b111001),
3146 VecALUOp::Fcmgt => (0b001_01110_10_1, 0b111001),
3147 VecALUOp::Fcmge => (0b001_01110_00_1, 0b111001),
3148 VecALUOp::And => (0b000_01110_00_1, 0b000111),
3151 VecALUOp::Bic => (0b000_01110_01_1, 0b000111),
3152 VecALUOp::Orr => (0b000_01110_10_1, 0b000111),
3153 VecALUOp::Orn => (0b000_01110_11_1, 0b000111),
3154 VecALUOp::Eor => (0b001_01110_00_1, 0b000111),
3155 VecALUOp::Umaxp => {
3156 debug_assert_ne!(size, VectorSize::Size64x2);
3157
3158 (0b001_01110_00_1 | enc_size << 1, 0b101001)
3159 }
3160 VecALUOp::Add => (0b000_01110_00_1 | enc_size << 1, 0b100001),
3161 VecALUOp::Sub => (0b001_01110_00_1 | enc_size << 1, 0b100001),
3162 VecALUOp::Mul => {
3163 debug_assert_ne!(size, VectorSize::Size64x2);
3164 (0b000_01110_00_1 | enc_size << 1, 0b100111)
3165 }
3166 VecALUOp::Sshl => (0b000_01110_00_1 | enc_size << 1, 0b010001),
3167 VecALUOp::Ushl => (0b001_01110_00_1 | enc_size << 1, 0b010001),
3168 VecALUOp::Umin => {
3169 debug_assert_ne!(size, VectorSize::Size64x2);
3170
3171 (0b001_01110_00_1 | enc_size << 1, 0b011011)
3172 }
3173 VecALUOp::Smin => {
3174 debug_assert_ne!(size, VectorSize::Size64x2);
3175
3176 (0b000_01110_00_1 | enc_size << 1, 0b011011)
3177 }
3178 VecALUOp::Umax => {
3179 debug_assert_ne!(size, VectorSize::Size64x2);
3180
3181 (0b001_01110_00_1 | enc_size << 1, 0b011001)
3182 }
3183 VecALUOp::Smax => {
3184 debug_assert_ne!(size, VectorSize::Size64x2);
3185
3186 (0b000_01110_00_1 | enc_size << 1, 0b011001)
3187 }
3188 VecALUOp::Urhadd => {
3189 debug_assert_ne!(size, VectorSize::Size64x2);
3190
3191 (0b001_01110_00_1 | enc_size << 1, 0b000101)
3192 }
3193 VecALUOp::Fadd => (0b000_01110_00_1, 0b110101),
3194 VecALUOp::Fsub => (0b000_01110_10_1, 0b110101),
3195 VecALUOp::Fdiv => (0b001_01110_00_1, 0b111111),
3196 VecALUOp::Fmax => (0b000_01110_00_1, 0b111101),
3197 VecALUOp::Fmin => (0b000_01110_10_1, 0b111101),
3198 VecALUOp::Fmul => (0b001_01110_00_1, 0b110111),
3199 VecALUOp::Addp => (0b000_01110_00_1 | enc_size << 1, 0b101111),
3200 VecALUOp::Zip1 => (0b01001110_00_0 | enc_size << 1, 0b001110),
3201 VecALUOp::Zip2 => (0b01001110_00_0 | enc_size << 1, 0b011110),
3202 VecALUOp::Sqrdmulh => {
3203 debug_assert!(
3204 size.lane_size() == ScalarSize::Size16
3205 || size.lane_size() == ScalarSize::Size32
3206 );
3207
3208 (0b001_01110_00_1 | enc_size << 1, 0b101101)
3209 }
3210 VecALUOp::Uzp1 => (0b01001110_00_0 | enc_size << 1, 0b000110),
3211 VecALUOp::Uzp2 => (0b01001110_00_0 | enc_size << 1, 0b010110),
3212 VecALUOp::Trn1 => (0b01001110_00_0 | enc_size << 1, 0b001010),
3213 VecALUOp::Trn2 => (0b01001110_00_0 | enc_size << 1, 0b011010),
3214 };
3215 let top11 = if is_float {
3216 top11 | size.enc_float_size() << 1
3217 } else {
3218 top11
3219 };
3220 sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
3221 }
3222 &Inst::VecRRRMod {
3223 rd,
3224 ri,
3225 rn,
3226 rm,
3227 alu_op,
3228 size,
3229 } => {
3230 debug_assert_eq!(rd.to_reg(), ri);
3231 let (q, _enc_size) = size.enc_size();
3232
3233 let (top11, bit15_10) = match alu_op {
3234 VecALUModOp::Bsl => (0b001_01110_01_1, 0b000111),
3235 VecALUModOp::Fmla => {
3236 (0b000_01110_00_1 | (size.enc_float_size() << 1), 0b110011)
3237 }
3238 VecALUModOp::Fmls => {
3239 (0b000_01110_10_1 | (size.enc_float_size() << 1), 0b110011)
3240 }
3241 VecALUModOp::Sdot => (0b000_01110_10_0, 0b100101),
3246 VecALUModOp::Usdot => (0b000_01110_10_0, 0b100111),
3249 };
3250 sink.put4(enc_vec_rrr(top11 | q << 9, rm, bit15_10, rn, rd));
3251 }
3252 &Inst::VecFmlaElem {
3253 rd,
3254 ri,
3255 rn,
3256 rm,
3257 alu_op,
3258 size,
3259 idx,
3260 } => {
3261 debug_assert_eq!(rd.to_reg(), ri);
3262 let idx = u32::from(idx);
3263
3264 let (q, _size) = size.enc_size();
3265 let o2 = match alu_op {
3266 VecALUModOp::Fmla => 0b0,
3267 VecALUModOp::Fmls => 0b1,
3268 _ => unreachable!(),
3269 };
3270
3271 let (h, l) = match size {
3272 VectorSize::Size32x4 => {
3273 assert!(idx < 4);
3274 (idx >> 1, idx & 1)
3275 }
3276 VectorSize::Size64x2 => {
3277 assert!(idx < 2);
3278 (idx, 0)
3279 }
3280 _ => unreachable!(),
3281 };
3282
3283 let top11 = 0b000_011111_00 | (q << 9) | (size.enc_float_size() << 1) | l;
3284 let bit15_10 = 0b000100 | (o2 << 4) | (h << 1);
3285 sink.put4(enc_vec_rrr(top11, rm, bit15_10, rn, rd));
3286 }
3287 &Inst::VecLoadReplicate {
3288 rd,
3289 rn,
3290 size,
3291 flags,
3292 } => {
3293 let (q, size) = size.enc_size();
3294
3295 if let Some(trap_code) = flags.trap_code() {
3296 sink.add_trap(trap_code);
3298 }
3299
3300 sink.put4(enc_ldst_vec(q, size, rn, rd));
3301 }
3302 &Inst::VecCSel { rd, rn, rm, cond } => {
3303 let else_label = sink.get_label();
3314 let out_label = sink.get_label();
3315
3316 let br_else_offset = sink.cur_offset();
3318 sink.put4(enc_conditional_br(
3319 BranchTarget::Label(else_label),
3320 CondBrKind::Cond(cond),
3321 ));
3322 sink.use_label_at_offset(br_else_offset, else_label, LabelUse::Branch19);
3323
3324 sink.put4(enc_vecmov(true, rd, rm));
3326
3327 let b_out_offset = sink.cur_offset();
3329 sink.use_label_at_offset(b_out_offset, out_label, LabelUse::Branch26);
3330 sink.add_uncond_branch(b_out_offset, b_out_offset + 4, out_label);
3331 sink.put4(enc_jump26(0b000101, 0 ));
3332
3333 sink.bind_label(else_label, &mut state.ctrl_plane);
3335
3336 sink.put4(enc_vecmov(true, rd, rn));
3338
3339 sink.bind_label(out_label, &mut state.ctrl_plane);
3341 }
3342 &Inst::MovToNZCV { rn } => {
3343 sink.put4(0xd51b4200 | machreg_to_gpr(rn));
3344 }
3345 &Inst::MovFromNZCV { rd } => {
3346 sink.put4(0xd53b4200 | machreg_to_gpr(rd.to_reg()));
3347 }
3348 &Inst::BitfieldMove {
3349 size,
3350 bfm_op,
3351 rd,
3352 rn,
3353 immr,
3354 imms,
3355 } => {
3356 let opc = bfm_op.opc();
3357 sink.put4(enc_bfm(opc, size, rd, rn, immr.value(), imms.value()));
3358 }
3359 &Inst::BitfieldMoveMod {
3360 size,
3361 rd,
3362 ri,
3363 rn,
3364 immr,
3365 imms,
3366 } => {
3367 debug_assert_eq!(rd.to_reg(), ri);
3368 sink.put4(enc_bfm(0b01, size, rd, rn, immr.value(), imms.value()));
3369 }
3370 &Inst::Jump { ref dest } => {
3371 let off = sink.cur_offset();
3372 if let Some(l) = dest.as_label() {
3374 sink.use_label_at_offset(off, l, LabelUse::Branch26);
3375 sink.add_uncond_branch(off, off + 4, l);
3376 }
3377 sink.put4(enc_jump26(0b000101, dest.as_offset26_or_zero()));
3379 }
3380 &Inst::Args { .. } | &Inst::Rets { .. } => {
3381 }
3384 &Inst::Ret {} => {
3385 sink.put4(0xd65f03c0);
3386 }
3387 &Inst::AuthenticatedRet { key, is_hint } => {
3388 let (op2, is_hint) = match key {
3389 APIKey::AZ => (0b100, true),
3390 APIKey::ASP => (0b101, is_hint),
3391 APIKey::BZ => (0b110, true),
3392 APIKey::BSP => (0b111, is_hint),
3393 };
3394
3395 if is_hint {
3396 sink.put4(key.enc_auti_hint());
3397 Inst::Ret {}.emit(sink, emit_info, state);
3398 } else {
3399 sink.put4(0xd65f0bff | (op2 << 9)); }
3401 }
3402 &Inst::Call { ref info } => {
3403 let start = sink.cur_offset();
3404 sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
3405 sink.put4(enc_jump26(0b100101, 0));
3406 if let Some(s) = state.stack_maps.take(sink) {
3407 let offset = sink.cur_offset();
3408 state.stack_maps.push(sink, &state.frame_layout, offset, s);
3409 }
3410
3411 if let Some(try_call) = info.try_call_info.as_ref() {
3412 sink.add_try_call_site(
3413 Some(state.frame_layout.sp_to_fp()),
3414 try_call.exception_handlers(&state.frame_layout),
3415 );
3416 } else {
3417 sink.add_call_site();
3418 }
3419
3420 if info.callee_pop_size > 0 {
3421 let callee_pop_size =
3422 i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB");
3423 for inst in AArch64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) {
3424 inst.emit(sink, emit_info, state);
3425 }
3426 }
3427
3428 if info.patchable {
3429 sink.add_patchable_call_site(sink.cur_offset() - start);
3430 } else {
3431 info.emit_retval_loads::<AArch64MachineDeps, _, _>(
3433 state.frame_layout().stackslots_size,
3434 |inst| inst.emit(sink, emit_info, state),
3435 |needed_space| Some(Inst::EmitIsland { needed_space }),
3436 );
3437 }
3438
3439 if let Some(try_call) = info.try_call_info.as_ref() {
3442 let jmp = Inst::Jump {
3443 dest: BranchTarget::Label(try_call.continuation),
3444 };
3445 jmp.emit(sink, emit_info, state);
3446 }
3447
3448 start_off = sink.cur_offset();
3451 }
3452 &Inst::CallInd { ref info } => {
3453 sink.put4(
3454 0b1101011_0001_11111_000000_00000_00000 | (machreg_to_gpr(info.dest) << 5),
3455 );
3456 if let Some(s) = state.stack_maps.take(sink) {
3457 let offset = sink.cur_offset();
3458 state.stack_maps.push(sink, &state.frame_layout, offset, s);
3459 }
3460
3461 if let Some(try_call) = info.try_call_info.as_ref() {
3462 sink.add_try_call_site(
3463 Some(state.frame_layout.sp_to_fp()),
3464 try_call.exception_handlers(&state.frame_layout),
3465 );
3466 } else {
3467 sink.add_call_site();
3468 }
3469
3470 if info.callee_pop_size > 0 {
3471 let callee_pop_size =
3472 i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB");
3473 for inst in AArch64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) {
3474 inst.emit(sink, emit_info, state);
3475 }
3476 }
3477
3478 info.emit_retval_loads::<AArch64MachineDeps, _, _>(
3480 state.frame_layout().stackslots_size,
3481 |inst| inst.emit(sink, emit_info, state),
3482 |needed_space| Some(Inst::EmitIsland { needed_space }),
3483 );
3484
3485 if let Some(try_call) = info.try_call_info.as_ref() {
3488 let jmp = Inst::Jump {
3489 dest: BranchTarget::Label(try_call.continuation),
3490 };
3491 jmp.emit(sink, emit_info, state);
3492 }
3493
3494 start_off = sink.cur_offset();
3497 }
3498 &Inst::ReturnCall { ref info } => {
3499 emit_return_call_common_sequence(sink, emit_info, state, info);
3500
3501 sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
3505 sink.put4(enc_jump26(0b000101, 0));
3506 sink.add_call_site();
3507
3508 start_off = sink.cur_offset();
3512 }
3513 &Inst::ReturnCallInd { ref info } => {
3514 emit_return_call_common_sequence(sink, emit_info, state, info);
3515
3516 let dest = if emit_info.isa_flags.use_bti() {
3525 let dest = xreg(16);
3526 if info.dest != dest {
3527 Inst::Mov {
3528 size: OperandSize::Size64,
3529 rd: writable_xreg(16),
3530 rm: info.dest,
3531 }
3532 .emit(sink, emit_info, state);
3533 }
3534 dest
3535 } else {
3536 info.dest
3537 };
3538
3539 Inst::IndirectBr {
3540 rn: dest,
3541 targets: vec![],
3542 }
3543 .emit(sink, emit_info, state);
3544 sink.add_call_site();
3545
3546 start_off = sink.cur_offset();
3550 }
3551 &Inst::CondBr {
3552 taken,
3553 not_taken,
3554 kind,
3555 } => {
3556 let cond_off = sink.cur_offset();
3558 if let Some(l) = taken.as_label() {
3559 sink.use_label_at_offset(cond_off, l, LabelUse::Branch19);
3560 let inverted = enc_conditional_br(taken, kind.invert()).to_le_bytes();
3561 sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
3562 }
3563 sink.put4(enc_conditional_br(taken, kind));
3564
3565 let uncond_off = sink.cur_offset();
3567 if let Some(l) = not_taken.as_label() {
3568 sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
3569 sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
3570 }
3571 sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
3572 }
3573 &Inst::TestBitAndBranch {
3574 taken,
3575 not_taken,
3576 kind,
3577 rn,
3578 bit,
3579 } => {
3580 let cond_off = sink.cur_offset();
3582 if let Some(l) = taken.as_label() {
3583 sink.use_label_at_offset(cond_off, l, LabelUse::Branch14);
3584 let inverted =
3585 enc_test_bit_and_branch(kind.complement(), taken, rn, bit).to_le_bytes();
3586 sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
3587 }
3588 sink.put4(enc_test_bit_and_branch(kind, taken, rn, bit));
3589
3590 let uncond_off = sink.cur_offset();
3592 if let Some(l) = not_taken.as_label() {
3593 sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
3594 sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
3595 }
3596 sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
3597 }
3598 &Inst::TrapIf { kind, trap_code } => {
3599 let label = sink.defer_trap(trap_code);
3600 let off = sink.cur_offset();
3602 sink.put4(enc_conditional_br(BranchTarget::Label(label), kind));
3603 sink.use_label_at_offset(off, label, LabelUse::Branch19);
3604 }
3605 &Inst::IndirectBr { rn, .. } => {
3606 sink.put4(enc_br(rn));
3607 }
3608 &Inst::Nop0 => {}
3609 &Inst::Nop4 => {
3610 sink.put4(0xd503201f);
3611 }
3612 &Inst::Brk => {
3613 sink.put4(0xd43e0000);
3614 }
3615 &Inst::Udf { trap_code } => {
3616 sink.add_trap(trap_code);
3617 sink.put_data(Inst::TRAP_OPCODE);
3618 }
3619 &Inst::Adr { rd, off } => {
3620 assert!(off > -(1 << 20));
3621 assert!(off < (1 << 20));
3622 sink.put4(enc_adr(off, rd));
3623 }
3624 &Inst::Adrp { rd, off } => {
3625 assert!(off > -(1 << 20));
3626 assert!(off < (1 << 20));
3627 sink.put4(enc_adrp(off, rd));
3628 }
3629 &Inst::Word4 { data } => {
3630 sink.put4(data);
3631 }
3632 &Inst::Word8 { data } => {
3633 sink.put8(data);
3634 }
3635 &Inst::JTSequence {
3636 ridx,
3637 rtmp1,
3638 rtmp2,
3639 default,
3640 ref targets,
3641 ..
3642 } => {
3643 let default_index = u64::try_from(targets.len()).unwrap();
3657 for inst in Inst::load_constant(rtmp2, default_index) {
3658 inst.emit(sink, emit_info, state);
3659 }
3660 let inst = Inst::CSel {
3665 rd: rtmp2,
3666 cond: Cond::Lo,
3667 rn: ridx,
3668 rm: rtmp2.to_reg(),
3669 };
3670 inst.emit(sink, emit_info, state);
3671 if emit_info.flags.enable_table_access_spectre_mitigation()
3674 && emit_info.isa_flags.use_csdb()
3675 {
3676 Inst::Csdb.emit(sink, emit_info, state);
3677 }
3678
3679 let inst = Inst::Adr { rd: rtmp1, off: 16 };
3681 inst.emit(sink, emit_info, state);
3682 let inst = Inst::SLoad32 {
3684 rd: rtmp2,
3685 mem: AMode::reg_plus_reg_scaled_extended(
3686 rtmp1.to_reg(),
3687 rtmp2.to_reg(),
3688 ExtendOp::UXTW,
3689 ),
3690 flags: MemFlagsData::trusted(),
3691 };
3692 inst.emit(sink, emit_info, state);
3693 let inst = Inst::AluRRR {
3695 alu_op: ALUOp::Add,
3696 size: OperandSize::Size64,
3697 rd: rtmp1,
3698 rn: rtmp1.to_reg(),
3699 rm: rtmp2.to_reg(),
3700 };
3701 inst.emit(sink, emit_info, state);
3702 let inst = Inst::IndirectBr {
3705 rn: rtmp1.to_reg(),
3706 targets: vec![],
3707 };
3708 inst.emit(sink, emit_info, state);
3709 let jt_off = sink.cur_offset();
3712 for &target in targets.iter().chain(core::iter::once(&default)) {
3713 let word_off = sink.cur_offset();
3714 let off_into_table = word_off - jt_off;
3719 sink.use_label_at_offset(word_off, target, LabelUse::PCRel32);
3720 sink.put4(off_into_table);
3721 }
3722
3723 start_off = sink.cur_offset();
3726 }
3727 &Inst::LoadExtNameGot { rd, ref name } => {
3728 sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);
3737 let inst = Inst::Adrp { rd, off: 0 };
3738 inst.emit(sink, emit_info, state);
3739
3740 sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);
3742 let inst = Inst::ULoad64 {
3743 rd,
3744 mem: AMode::reg(rd.to_reg()),
3745 flags: MemFlagsData::trusted(),
3746 };
3747 inst.emit(sink, emit_info, state);
3748 }
3749 &Inst::LoadExtNameNear {
3750 rd,
3751 ref name,
3752 offset,
3753 } => {
3754 sink.add_reloc(Reloc::Aarch64AdrPrelPgHi21, &**name, offset);
3762 let inst = Inst::Adrp { rd, off: 0 };
3763 inst.emit(sink, emit_info, state);
3764
3765 sink.add_reloc(Reloc::Aarch64AddAbsLo12Nc, &**name, offset);
3767 let inst = Inst::AluRRImm12 {
3768 alu_op: ALUOp::Add,
3769 size: OperandSize::Size64,
3770 rd,
3771 rn: rd.to_reg(),
3772 imm12: Imm12::ZERO,
3773 };
3774 inst.emit(sink, emit_info, state);
3775 }
3776 &Inst::LoadExtNameFar {
3777 rd,
3778 ref name,
3779 offset,
3780 } => {
3781 let inst = Inst::ULoad64 {
3790 rd,
3791 mem: AMode::Label {
3792 label: MemLabel::PCRel(8),
3793 },
3794 flags: MemFlagsData::trusted(),
3795 };
3796 inst.emit(sink, emit_info, state);
3797 let inst = Inst::Jump {
3798 dest: BranchTarget::ResolvedOffset(12),
3799 };
3800 inst.emit(sink, emit_info, state);
3801 sink.add_reloc(Reloc::Abs8, &**name, offset);
3802 sink.put8(0);
3803 }
3804 &Inst::LoadAddr { rd, ref mem } => {
3805 let mem = mem.clone();
3806 let (mem_insts, mem) = mem_finalize(Some(sink), &mem, I8, state);
3807 for inst in mem_insts.into_iter() {
3808 inst.emit(sink, emit_info, state);
3809 }
3810
3811 let (reg, index_reg, offset) = match mem {
3812 AMode::RegExtended { rn, rm, extendop } => {
3813 let r = rn;
3814 (r, Some((rm, extendop)), 0)
3815 }
3816 AMode::Unscaled { rn, simm9 } => {
3817 let r = rn;
3818 (r, None, simm9.value())
3819 }
3820 AMode::UnsignedOffset { rn, uimm12 } => {
3821 let r = rn;
3822 (r, None, uimm12.value() as i32)
3823 }
3824 _ => panic!("Unsupported case for LoadAddr: {mem:?}"),
3825 };
3826 let abs_offset = if offset < 0 {
3827 -offset as u64
3828 } else {
3829 offset as u64
3830 };
3831 let alu_op = if offset < 0 { ALUOp::Sub } else { ALUOp::Add };
3832
3833 if let Some((idx, extendop)) = index_reg {
3834 let add = Inst::AluRRRExtend {
3835 alu_op: ALUOp::Add,
3836 size: OperandSize::Size64,
3837 rd,
3838 rn: reg,
3839 rm: idx,
3840 extendop,
3841 };
3842
3843 add.emit(sink, emit_info, state);
3844 } else if offset == 0 {
3845 if reg != rd.to_reg() {
3846 let mov = Inst::Mov {
3847 size: OperandSize::Size64,
3848 rd,
3849 rm: reg,
3850 };
3851
3852 mov.emit(sink, emit_info, state);
3853 }
3854 } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
3855 let add = Inst::AluRRImm12 {
3856 alu_op,
3857 size: OperandSize::Size64,
3858 rd,
3859 rn: reg,
3860 imm12,
3861 };
3862 add.emit(sink, emit_info, state);
3863 } else {
3864 debug_assert!(rd.to_reg() != tmp2_reg());
3870 debug_assert!(reg != tmp2_reg());
3871 let tmp = writable_tmp2_reg();
3872 for insn in Inst::load_constant(tmp, abs_offset).into_iter() {
3873 insn.emit(sink, emit_info, state);
3874 }
3875 let add = Inst::AluRRR {
3876 alu_op,
3877 size: OperandSize::Size64,
3878 rd,
3879 rn: reg,
3880 rm: tmp.to_reg(),
3881 };
3882 add.emit(sink, emit_info, state);
3883 }
3884 }
3885 &Inst::Paci { key } => {
3886 let (crm, op2) = match key {
3887 APIKey::AZ => (0b0011, 0b000),
3888 APIKey::ASP => (0b0011, 0b001),
3889 APIKey::BZ => (0b0011, 0b010),
3890 APIKey::BSP => (0b0011, 0b011),
3891 };
3892
3893 sink.put4(0xd503211f | (crm << 8) | (op2 << 5));
3894 }
3895 &Inst::Xpaclri => sink.put4(0xd50320ff),
3896 &Inst::Bti { targets } => {
3897 let targets = match targets {
3898 BranchTargetType::None => 0b00,
3899 BranchTargetType::C => 0b01,
3900 BranchTargetType::J => 0b10,
3901 BranchTargetType::JC => 0b11,
3902 };
3903
3904 sink.put4(0xd503241f | targets << 6);
3905 }
3906 &Inst::EmitIsland { needed_space } => {
3907 if sink.island_needed(needed_space + 4) {
3908 let jump_around_label = sink.get_label();
3909 let jmp = Inst::Jump {
3910 dest: BranchTarget::Label(jump_around_label),
3911 };
3912 jmp.emit(sink, emit_info, state);
3913 sink.emit_island(needed_space + 4, &mut state.ctrl_plane);
3914 sink.bind_label(jump_around_label, &mut state.ctrl_plane);
3915 }
3916 }
3917
3918 &Inst::ElfTlsGetAddr {
3919 ref symbol,
3920 rd,
3921 tmp,
3922 } => {
3923 assert_eq!(xreg(0), rd.to_reg());
3924
3925 sink.add_reloc(Reloc::Aarch64TlsDescAdrPage21, &**symbol, 0);
3941 Inst::Adrp { rd, off: 0 }.emit(sink, emit_info, state);
3942
3943 sink.add_reloc(Reloc::Aarch64TlsDescLd64Lo12, &**symbol, 0);
3945 Inst::ULoad64 {
3946 rd: tmp,
3947 mem: AMode::reg(rd.to_reg()),
3948 flags: MemFlagsData::trusted(),
3949 }
3950 .emit(sink, emit_info, state);
3951
3952 sink.add_reloc(Reloc::Aarch64TlsDescAddLo12, &**symbol, 0);
3954 Inst::AluRRImm12 {
3955 alu_op: ALUOp::Add,
3956 size: OperandSize::Size64,
3957 rd,
3958 rn: rd.to_reg(),
3959 imm12: Imm12::maybe_from_u64(0).unwrap(),
3960 }
3961 .emit(sink, emit_info, state);
3962
3963 sink.add_reloc(Reloc::Aarch64TlsDescCall, &**symbol, 0);
3965 Inst::CallInd {
3966 info: crate::isa::Box::new(CallInfo::empty(tmp.to_reg(), CallConv::SystemV)),
3967 }
3968 .emit(sink, emit_info, state);
3969
3970 sink.put4(0xd53bd040 | machreg_to_gpr(tmp.to_reg()));
3972
3973 Inst::AluRRR {
3975 alu_op: ALUOp::Add,
3976 size: OperandSize::Size64,
3977 rd,
3978 rn: rd.to_reg(),
3979 rm: tmp.to_reg(),
3980 }
3981 .emit(sink, emit_info, state);
3982 }
3983
3984 &Inst::MachOTlsGetAddr { ref symbol, rd } => {
3985 assert_eq!(xreg(0), rd.to_reg());
3998 let rtmp = writable_xreg(1);
3999
4000 sink.add_reloc(Reloc::MachOAarch64TlsAdrPage21, symbol, 0);
4002 sink.put4(0x90000000);
4003
4004 sink.add_reloc(Reloc::MachOAarch64TlsAdrPageOff12, symbol, 0);
4006 sink.put4(0xf9400000);
4007
4008 Inst::ULoad64 {
4010 rd: rtmp,
4011 mem: AMode::reg(rd.to_reg()),
4012 flags: MemFlagsData::trusted(),
4013 }
4014 .emit(sink, emit_info, state);
4015
4016 Inst::CallInd {
4018 info: crate::isa::Box::new(CallInfo::empty(
4019 rtmp.to_reg(),
4020 CallConv::AppleAarch64,
4021 )),
4022 }
4023 .emit(sink, emit_info, state);
4024 }
4025
4026 &Inst::Unwind { ref inst } => {
4027 sink.add_unwind(inst.clone());
4028 }
4029
4030 &Inst::DummyUse { .. } => {}
4031
4032 &Inst::LabelAddress { dst, label } => {
4033 let inst = Inst::Adr { rd: dst, off: 0 };
4038 let offset = sink.cur_offset();
4039 inst.emit(sink, emit_info, state);
4040 sink.use_label_at_offset(offset, label, LabelUse::Adr21);
4041 }
4042
4043 &Inst::SequencePoint { .. } => {
4044 }
4046
4047 &Inst::StackProbeLoop { start, end, step } => {
4048 assert!(emit_info.flags.enable_probestack());
4049
4050 let loop_start = sink.get_label();
4073 sink.bind_label(loop_start, &mut state.ctrl_plane);
4074
4075 Inst::AluRRImm12 {
4076 alu_op: ALUOp::Sub,
4077 size: OperandSize::Size64,
4078 rd: start,
4079 rn: start.to_reg(),
4080 imm12: step,
4081 }
4082 .emit(sink, emit_info, state);
4083 Inst::Store32 {
4084 rd: regs::zero_reg(),
4085 mem: AMode::RegReg {
4086 rn: regs::stack_reg(),
4087 rm: start.to_reg(),
4088 },
4089 flags: MemFlagsData::trusted(),
4090 }
4091 .emit(sink, emit_info, state);
4092 Inst::AluRRR {
4093 alu_op: ALUOp::AddS,
4094 size: OperandSize::Size64,
4095 rd: regs::writable_zero_reg(),
4096 rn: start.to_reg(),
4097 rm: end,
4098 }
4099 .emit(sink, emit_info, state);
4100
4101 let loop_end = sink.get_label();
4102 Inst::CondBr {
4103 taken: BranchTarget::Label(loop_start),
4104 not_taken: BranchTarget::Label(loop_end),
4105 kind: CondBrKind::Cond(Cond::Gt),
4106 }
4107 .emit(sink, emit_info, state);
4108 sink.bind_label(loop_end, &mut state.ctrl_plane);
4109 }
4110
4111 &Inst::InterruptPoll {
4112 dst,
4113 load_ptr,
4114 trap_code,
4115 ..
4116 } => {
4117 let stack_map = state.stack_maps.take(sink);
4120
4121 sink.add_trap(trap_code);
4125
4126 if let Some(stack_map) = stack_map {
4129 let offset = sink.cur_offset();
4130 state
4131 .stack_maps
4132 .push(sink, &state.frame_layout, offset, stack_map);
4133 }
4134
4135 Inst::ULoad64 {
4139 rd: dst,
4140 mem: AMode::UnsignedOffset {
4141 rn: load_ptr,
4142 uimm12: UImm12Scaled::zero(I64),
4143 },
4144 flags: MemFlagsData::trusted(),
4145 }
4146 .emit(sink, emit_info, state);
4147 }
4148 }
4149
4150 let end_off = sink.cur_offset();
4151 debug_assert!(
4152 (end_off - start_off) <= Inst::worst_case_size()
4153 || matches!(self, Inst::EmitIsland { .. }),
4154 "Worst case size exceed for {:?}: {}",
4155 self,
4156 end_off - start_off
4157 );
4158 }
4159
4160 fn pretty_print_inst(&self, state: &mut Self::State) -> String {
4161 self.print_with_state(state)
4162 }
4163}
4164
4165fn emit_return_call_common_sequence<T>(
4166 sink: &mut MachBuffer<Inst>,
4167 emit_info: &EmitInfo,
4168 state: &mut EmitState,
4169 info: &ReturnCallInfo<T>,
4170) {
4171 for inst in AArch64MachineDeps::gen_clobber_restore(
4172 CallConv::Tail,
4173 &emit_info.flags,
4174 state.frame_layout(),
4175 ) {
4176 inst.emit(sink, emit_info, state);
4177 }
4178
4179 let setup_area_size = state.frame_layout().setup_area_size;
4180 if setup_area_size > 0 {
4181 Inst::LoadP64 {
4187 rt: writable_fp_reg(),
4188 rt2: writable_link_reg(),
4189 mem: PairAMode::SPPostIndexed {
4190 simm7: SImm7Scaled::maybe_from_i64(i64::from(setup_area_size), types::I64).unwrap(),
4194 },
4195 flags: MemFlagsData::trusted(),
4196 }
4197 .emit(sink, emit_info, state);
4198 }
4199
4200 let incoming_args_diff = state.frame_layout().tail_args_size - info.new_stack_arg_size;
4202 if incoming_args_diff > 0 {
4203 for inst in
4204 AArch64MachineDeps::gen_sp_reg_adjust(i32::try_from(incoming_args_diff).unwrap())
4205 {
4206 inst.emit(sink, emit_info, state);
4207 }
4208 }
4209
4210 if (setup_area_size > 0 || info.sign_return_address_all)
4211 && let Some(key) = info.key
4212 {
4213 sink.put4(key.enc_auti_hint());
4214 }
4215}