1use crate::binemit::{Addend, CodeOffset, Reloc};
4use crate::ir::types::{F16, F32, F64, F128, I8, I8X16, I16, I32, I64, I128};
5use crate::ir::{MemFlags, Type, types};
6use crate::isa::{CallConv, FunctionAlignment};
7use crate::machinst::*;
8use crate::{CodegenError, CodegenResult, settings};
9
10use crate::machinst::{PrettyPrint, Reg, RegClass, Writable};
11
12use alloc::vec::Vec;
13use core::slice;
14use smallvec::{SmallVec, smallvec};
15use std::fmt::Write;
16use std::string::{String, ToString};
17
18pub(crate) mod regs;
19pub(crate) use self::regs::*;
20pub mod imms;
21pub use self::imms::*;
22pub mod args;
23pub use self::args::*;
24pub mod emit;
25pub(crate) use self::emit::*;
26use crate::isa::aarch64::abi::AArch64MachineDeps;
27
28pub(crate) mod unwind;
29
30#[cfg(test)]
31mod emit_tests;
32
33pub use crate::isa::aarch64::lower::isle::generated_code::{
37 ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, BranchTargetType, FPUOp1,
38 FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp,
39 VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp,
40 VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,
41};
42
43#[derive(Copy, Clone, Debug)]
45pub enum FPUOpRI {
46 UShr32(FPURightShiftImm),
48 UShr64(FPURightShiftImm),
50}
51
52#[derive(Copy, Clone, Debug)]
56pub enum FPUOpRIMod {
57 Sli32(FPULeftShiftImm),
59 Sli64(FPULeftShiftImm),
61}
62
63impl BitOp {
64 pub fn op_str(&self) -> &'static str {
66 match self {
67 BitOp::RBit => "rbit",
68 BitOp::Clz => "clz",
69 BitOp::Cls => "cls",
70 BitOp::Rev16 => "rev16",
71 BitOp::Rev32 => "rev32",
72 BitOp::Rev64 => "rev64",
73 }
74 }
75}
76
77#[derive(Clone, Debug)]
80pub struct ReturnCallInfo<T> {
81 pub dest: T,
83 pub uses: CallArgList,
85 pub new_stack_arg_size: u32,
89 pub key: Option<APIKey>,
91}
92
93fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
94 let mut count = 0;
95 for _ in 0..num_half_words {
96 if value & 0xffff == 0 {
97 count += 1;
98 }
99 value >>= 16;
100 }
101
102 count
103}
104
105impl Inst {
106 pub fn load_constant(rd: Writable<Reg>, value: u64) -> SmallVec<[Inst; 4]> {
109 if let Some(imm) = MoveWideConst::maybe_from_u64(value) {
114 smallvec![Inst::MovWide {
116 op: MoveWideOp::MovZ,
117 rd,
118 imm,
119 size: OperandSize::Size64
120 }]
121 } else if let Some(imm) = MoveWideConst::maybe_from_u64(!value) {
122 smallvec![Inst::MovWide {
124 op: MoveWideOp::MovN,
125 rd,
126 imm,
127 size: OperandSize::Size64
128 }]
129 } else if let Some(imml) = ImmLogic::maybe_from_u64(value, I64) {
130 smallvec![Inst::AluRRImmLogic {
132 alu_op: ALUOp::Orr,
133 size: OperandSize::Size64,
134 rd,
135 rn: zero_reg(),
136 imml,
137 }]
138 } else {
139 let mut insts = smallvec![];
140
141 let (num_half_words, size, negated) = if value >> 32 == 0 {
143 (2, OperandSize::Size32, (!value << 32) >> 32)
144 } else {
145 (4, OperandSize::Size64, !value)
146 };
147
148 let first_is_inverted = count_zero_half_words(negated, num_half_words)
151 > count_zero_half_words(value, num_half_words);
152
153 let ignored_halfword = if first_is_inverted { 0xffff } else { 0 };
156
157 let halfwords: SmallVec<[_; 4]> = (0..num_half_words)
158 .filter_map(|i| {
159 let imm16 = (value >> (16 * i)) & 0xffff;
160 if imm16 == ignored_halfword {
161 None
162 } else {
163 Some((i, imm16))
164 }
165 })
166 .collect();
167
168 let mut prev_result = None;
169 for (i, imm16) in halfwords {
170 let shift = i * 16;
171
172 if let Some(rn) = prev_result {
173 let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
174 insts.push(Inst::MovK { rd, rn, imm, size });
175 } else {
176 if first_is_inverted {
177 let imm =
178 MoveWideConst::maybe_with_shift(((!imm16) & 0xffff) as u16, shift)
179 .unwrap();
180 insts.push(Inst::MovWide {
181 op: MoveWideOp::MovN,
182 rd,
183 imm,
184 size,
185 });
186 } else {
187 let imm = MoveWideConst::maybe_with_shift(imm16 as u16, shift).unwrap();
188 insts.push(Inst::MovWide {
189 op: MoveWideOp::MovZ,
190 rd,
191 imm,
192 size,
193 });
194 }
195 }
196
197 prev_result = Some(rd.to_reg());
198 }
199
200 assert!(prev_result.is_some());
201
202 insts
203 }
204 }
205
206 pub fn gen_load(into_reg: Writable<Reg>, mem: AMode, ty: Type, flags: MemFlags) -> Inst {
208 match ty {
209 I8 => Inst::ULoad8 {
210 rd: into_reg,
211 mem,
212 flags,
213 },
214 I16 => Inst::ULoad16 {
215 rd: into_reg,
216 mem,
217 flags,
218 },
219 I32 => Inst::ULoad32 {
220 rd: into_reg,
221 mem,
222 flags,
223 },
224 I64 => Inst::ULoad64 {
225 rd: into_reg,
226 mem,
227 flags,
228 },
229 _ => {
230 if ty.is_vector() || ty.is_float() {
231 let bits = ty_bits(ty);
232 let rd = into_reg;
233
234 match bits {
235 128 => Inst::FpuLoad128 { rd, mem, flags },
236 64 => Inst::FpuLoad64 { rd, mem, flags },
237 32 => Inst::FpuLoad32 { rd, mem, flags },
238 16 => Inst::FpuLoad16 { rd, mem, flags },
239 _ => unimplemented!("gen_load({})", ty),
240 }
241 } else {
242 unimplemented!("gen_load({})", ty);
243 }
244 }
245 }
246 }
247
248 pub fn gen_store(mem: AMode, from_reg: Reg, ty: Type, flags: MemFlags) -> Inst {
250 match ty {
251 I8 => Inst::Store8 {
252 rd: from_reg,
253 mem,
254 flags,
255 },
256 I16 => Inst::Store16 {
257 rd: from_reg,
258 mem,
259 flags,
260 },
261 I32 => Inst::Store32 {
262 rd: from_reg,
263 mem,
264 flags,
265 },
266 I64 => Inst::Store64 {
267 rd: from_reg,
268 mem,
269 flags,
270 },
271 _ => {
272 if ty.is_vector() || ty.is_float() {
273 let bits = ty_bits(ty);
274 let rd = from_reg;
275
276 match bits {
277 128 => Inst::FpuStore128 { rd, mem, flags },
278 64 => Inst::FpuStore64 { rd, mem, flags },
279 32 => Inst::FpuStore32 { rd, mem, flags },
280 16 => Inst::FpuStore16 { rd, mem, flags },
281 _ => unimplemented!("gen_store({})", ty),
282 }
283 } else {
284 unimplemented!("gen_store({})", ty);
285 }
286 }
287 }
288 }
289
290 pub fn mem_type(&self) -> Option<Type> {
294 match self {
295 Inst::ULoad8 { .. } => Some(I8),
296 Inst::SLoad8 { .. } => Some(I8),
297 Inst::ULoad16 { .. } => Some(I16),
298 Inst::SLoad16 { .. } => Some(I16),
299 Inst::ULoad32 { .. } => Some(I32),
300 Inst::SLoad32 { .. } => Some(I32),
301 Inst::ULoad64 { .. } => Some(I64),
302 Inst::FpuLoad16 { .. } => Some(F16),
303 Inst::FpuLoad32 { .. } => Some(F32),
304 Inst::FpuLoad64 { .. } => Some(F64),
305 Inst::FpuLoad128 { .. } => Some(I8X16),
306 Inst::Store8 { .. } => Some(I8),
307 Inst::Store16 { .. } => Some(I16),
308 Inst::Store32 { .. } => Some(I32),
309 Inst::Store64 { .. } => Some(I64),
310 Inst::FpuStore16 { .. } => Some(F16),
311 Inst::FpuStore32 { .. } => Some(F32),
312 Inst::FpuStore64 { .. } => Some(F64),
313 Inst::FpuStore128 { .. } => Some(I8X16),
314 _ => None,
315 }
316 }
317}
318
319fn memarg_operands(memarg: &mut AMode, collector: &mut impl OperandVisitor) {
323 match memarg {
324 AMode::Unscaled { rn, .. } | AMode::UnsignedOffset { rn, .. } => {
325 collector.reg_use(rn);
326 }
327 AMode::RegReg { rn, rm, .. }
328 | AMode::RegScaled { rn, rm, .. }
329 | AMode::RegScaledExtended { rn, rm, .. }
330 | AMode::RegExtended { rn, rm, .. } => {
331 collector.reg_use(rn);
332 collector.reg_use(rm);
333 }
334 AMode::Label { .. } => {}
335 AMode::SPPreIndexed { .. } | AMode::SPPostIndexed { .. } => {}
336 AMode::FPOffset { .. } | AMode::IncomingArg { .. } => {}
337 AMode::SPOffset { .. } | AMode::SlotOffset { .. } => {}
338 AMode::RegOffset { rn, .. } => {
339 collector.reg_use(rn);
340 }
341 AMode::Const { .. } => {}
342 }
343}
344
345fn pairmemarg_operands(pairmemarg: &mut PairAMode, collector: &mut impl OperandVisitor) {
346 match pairmemarg {
347 PairAMode::SignedOffset { reg, .. } => {
348 collector.reg_use(reg);
349 }
350 PairAMode::SPPreIndexed { .. } | PairAMode::SPPostIndexed { .. } => {}
351 }
352}
353
354fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
355 match inst {
356 Inst::AluRRR { rd, rn, rm, .. } => {
357 collector.reg_def(rd);
358 collector.reg_use(rn);
359 collector.reg_use(rm);
360 }
361 Inst::AluRRRR { rd, rn, rm, ra, .. } => {
362 collector.reg_def(rd);
363 collector.reg_use(rn);
364 collector.reg_use(rm);
365 collector.reg_use(ra);
366 }
367 Inst::AluRRImm12 { rd, rn, .. } => {
368 collector.reg_def(rd);
369 collector.reg_use(rn);
370 }
371 Inst::AluRRImmLogic { rd, rn, .. } => {
372 collector.reg_def(rd);
373 collector.reg_use(rn);
374 }
375 Inst::AluRRImmShift { rd, rn, .. } => {
376 collector.reg_def(rd);
377 collector.reg_use(rn);
378 }
379 Inst::AluRRRShift { rd, rn, rm, .. } => {
380 collector.reg_def(rd);
381 collector.reg_use(rn);
382 collector.reg_use(rm);
383 }
384 Inst::AluRRRExtend { rd, rn, rm, .. } => {
385 collector.reg_def(rd);
386 collector.reg_use(rn);
387 collector.reg_use(rm);
388 }
389 Inst::BitRR { rd, rn, .. } => {
390 collector.reg_def(rd);
391 collector.reg_use(rn);
392 }
393 Inst::ULoad8 { rd, mem, .. }
394 | Inst::SLoad8 { rd, mem, .. }
395 | Inst::ULoad16 { rd, mem, .. }
396 | Inst::SLoad16 { rd, mem, .. }
397 | Inst::ULoad32 { rd, mem, .. }
398 | Inst::SLoad32 { rd, mem, .. }
399 | Inst::ULoad64 { rd, mem, .. } => {
400 collector.reg_def(rd);
401 memarg_operands(mem, collector);
402 }
403 Inst::Store8 { rd, mem, .. }
404 | Inst::Store16 { rd, mem, .. }
405 | Inst::Store32 { rd, mem, .. }
406 | Inst::Store64 { rd, mem, .. } => {
407 collector.reg_use(rd);
408 memarg_operands(mem, collector);
409 }
410 Inst::StoreP64 { rt, rt2, mem, .. } => {
411 collector.reg_use(rt);
412 collector.reg_use(rt2);
413 pairmemarg_operands(mem, collector);
414 }
415 Inst::LoadP64 { rt, rt2, mem, .. } => {
416 collector.reg_def(rt);
417 collector.reg_def(rt2);
418 pairmemarg_operands(mem, collector);
419 }
420 Inst::Mov { rd, rm, .. } => {
421 collector.reg_def(rd);
422 collector.reg_use(rm);
423 }
424 Inst::MovFromPReg { rd, rm } => {
425 debug_assert!(rd.to_reg().is_virtual());
426 collector.reg_def(rd);
427 collector.reg_fixed_nonallocatable(*rm);
428 }
429 Inst::MovToPReg { rd, rm } => {
430 debug_assert!(rm.is_virtual());
431 collector.reg_fixed_nonallocatable(*rd);
432 collector.reg_use(rm);
433 }
434 Inst::MovK { rd, rn, .. } => {
435 collector.reg_use(rn);
436 collector.reg_reuse_def(rd, 0); }
438 Inst::MovWide { rd, .. } => {
439 collector.reg_def(rd);
440 }
441 Inst::CSel { rd, rn, rm, .. } => {
442 collector.reg_def(rd);
443 collector.reg_use(rn);
444 collector.reg_use(rm);
445 }
446 Inst::CSNeg { rd, rn, rm, .. } => {
447 collector.reg_def(rd);
448 collector.reg_use(rn);
449 collector.reg_use(rm);
450 }
451 Inst::CSet { rd, .. } | Inst::CSetm { rd, .. } => {
452 collector.reg_def(rd);
453 }
454 Inst::CCmp { rn, rm, .. } => {
455 collector.reg_use(rn);
456 collector.reg_use(rm);
457 }
458 Inst::CCmpImm { rn, .. } => {
459 collector.reg_use(rn);
460 }
461 Inst::AtomicRMWLoop {
462 op,
463 addr,
464 operand,
465 oldval,
466 scratch1,
467 scratch2,
468 ..
469 } => {
470 collector.reg_fixed_use(addr, xreg(25));
471 collector.reg_fixed_use(operand, xreg(26));
472 collector.reg_fixed_def(oldval, xreg(27));
473 collector.reg_fixed_def(scratch1, xreg(24));
474 if *op != AtomicRMWLoopOp::Xchg {
475 collector.reg_fixed_def(scratch2, xreg(28));
476 }
477 }
478 Inst::AtomicRMW { rs, rt, rn, .. } => {
479 collector.reg_use(rs);
480 collector.reg_def(rt);
481 collector.reg_use(rn);
482 }
483 Inst::AtomicCAS { rd, rs, rt, rn, .. } => {
484 collector.reg_reuse_def(rd, 1); collector.reg_use(rs);
486 collector.reg_use(rt);
487 collector.reg_use(rn);
488 }
489 Inst::AtomicCASLoop {
490 addr,
491 expected,
492 replacement,
493 oldval,
494 scratch,
495 ..
496 } => {
497 collector.reg_fixed_use(addr, xreg(25));
498 collector.reg_fixed_use(expected, xreg(26));
499 collector.reg_fixed_use(replacement, xreg(28));
500 collector.reg_fixed_def(oldval, xreg(27));
501 collector.reg_fixed_def(scratch, xreg(24));
502 }
503 Inst::LoadAcquire { rt, rn, .. } => {
504 collector.reg_use(rn);
505 collector.reg_def(rt);
506 }
507 Inst::StoreRelease { rt, rn, .. } => {
508 collector.reg_use(rn);
509 collector.reg_use(rt);
510 }
511 Inst::Fence {} | Inst::Csdb {} => {}
512 Inst::FpuMove32 { rd, rn } => {
513 collector.reg_def(rd);
514 collector.reg_use(rn);
515 }
516 Inst::FpuMove64 { rd, rn } => {
517 collector.reg_def(rd);
518 collector.reg_use(rn);
519 }
520 Inst::FpuMove128 { rd, rn } => {
521 collector.reg_def(rd);
522 collector.reg_use(rn);
523 }
524 Inst::FpuMoveFromVec { rd, rn, .. } => {
525 collector.reg_def(rd);
526 collector.reg_use(rn);
527 }
528 Inst::FpuExtend { rd, rn, .. } => {
529 collector.reg_def(rd);
530 collector.reg_use(rn);
531 }
532 Inst::FpuRR { rd, rn, .. } => {
533 collector.reg_def(rd);
534 collector.reg_use(rn);
535 }
536 Inst::FpuRRR { rd, rn, rm, .. } => {
537 collector.reg_def(rd);
538 collector.reg_use(rn);
539 collector.reg_use(rm);
540 }
541 Inst::FpuRRI { rd, rn, .. } => {
542 collector.reg_def(rd);
543 collector.reg_use(rn);
544 }
545 Inst::FpuRRIMod { rd, ri, rn, .. } => {
546 collector.reg_reuse_def(rd, 1); collector.reg_use(ri);
548 collector.reg_use(rn);
549 }
550 Inst::FpuRRRR { rd, rn, rm, ra, .. } => {
551 collector.reg_def(rd);
552 collector.reg_use(rn);
553 collector.reg_use(rm);
554 collector.reg_use(ra);
555 }
556 Inst::VecMisc { rd, rn, .. } => {
557 collector.reg_def(rd);
558 collector.reg_use(rn);
559 }
560
561 Inst::VecLanes { rd, rn, .. } => {
562 collector.reg_def(rd);
563 collector.reg_use(rn);
564 }
565 Inst::VecShiftImm { rd, rn, .. } => {
566 collector.reg_def(rd);
567 collector.reg_use(rn);
568 }
569 Inst::VecShiftImmMod { rd, ri, rn, .. } => {
570 collector.reg_reuse_def(rd, 1); collector.reg_use(ri);
572 collector.reg_use(rn);
573 }
574 Inst::VecExtract { rd, rn, rm, .. } => {
575 collector.reg_def(rd);
576 collector.reg_use(rn);
577 collector.reg_use(rm);
578 }
579 Inst::VecTbl { rd, rn, rm } => {
580 collector.reg_use(rn);
581 collector.reg_use(rm);
582 collector.reg_def(rd);
583 }
584 Inst::VecTblExt { rd, ri, rn, rm } => {
585 collector.reg_use(rn);
586 collector.reg_use(rm);
587 collector.reg_reuse_def(rd, 3); collector.reg_use(ri);
589 }
590
591 Inst::VecTbl2 { rd, rn, rn2, rm } => {
592 collector.reg_fixed_use(rn, vreg(30));
596 collector.reg_fixed_use(rn2, vreg(31));
597 collector.reg_use(rm);
598 collector.reg_def(rd);
599 }
600 Inst::VecTbl2Ext {
601 rd,
602 ri,
603 rn,
604 rn2,
605 rm,
606 } => {
607 collector.reg_fixed_use(rn, vreg(30));
611 collector.reg_fixed_use(rn2, vreg(31));
612 collector.reg_use(rm);
613 collector.reg_reuse_def(rd, 4); collector.reg_use(ri);
615 }
616 Inst::VecLoadReplicate { rd, rn, .. } => {
617 collector.reg_def(rd);
618 collector.reg_use(rn);
619 }
620 Inst::VecCSel { rd, rn, rm, .. } => {
621 collector.reg_def(rd);
622 collector.reg_use(rn);
623 collector.reg_use(rm);
624 }
625 Inst::FpuCmp { rn, rm, .. } => {
626 collector.reg_use(rn);
627 collector.reg_use(rm);
628 }
629 Inst::FpuLoad16 { rd, mem, .. } => {
630 collector.reg_def(rd);
631 memarg_operands(mem, collector);
632 }
633 Inst::FpuLoad32 { rd, mem, .. } => {
634 collector.reg_def(rd);
635 memarg_operands(mem, collector);
636 }
637 Inst::FpuLoad64 { rd, mem, .. } => {
638 collector.reg_def(rd);
639 memarg_operands(mem, collector);
640 }
641 Inst::FpuLoad128 { rd, mem, .. } => {
642 collector.reg_def(rd);
643 memarg_operands(mem, collector);
644 }
645 Inst::FpuStore16 { rd, mem, .. } => {
646 collector.reg_use(rd);
647 memarg_operands(mem, collector);
648 }
649 Inst::FpuStore32 { rd, mem, .. } => {
650 collector.reg_use(rd);
651 memarg_operands(mem, collector);
652 }
653 Inst::FpuStore64 { rd, mem, .. } => {
654 collector.reg_use(rd);
655 memarg_operands(mem, collector);
656 }
657 Inst::FpuStore128 { rd, mem, .. } => {
658 collector.reg_use(rd);
659 memarg_operands(mem, collector);
660 }
661 Inst::FpuLoadP64 { rt, rt2, mem, .. } => {
662 collector.reg_def(rt);
663 collector.reg_def(rt2);
664 pairmemarg_operands(mem, collector);
665 }
666 Inst::FpuStoreP64 { rt, rt2, mem, .. } => {
667 collector.reg_use(rt);
668 collector.reg_use(rt2);
669 pairmemarg_operands(mem, collector);
670 }
671 Inst::FpuLoadP128 { rt, rt2, mem, .. } => {
672 collector.reg_def(rt);
673 collector.reg_def(rt2);
674 pairmemarg_operands(mem, collector);
675 }
676 Inst::FpuStoreP128 { rt, rt2, mem, .. } => {
677 collector.reg_use(rt);
678 collector.reg_use(rt2);
679 pairmemarg_operands(mem, collector);
680 }
681 Inst::FpuToInt { rd, rn, .. } => {
682 collector.reg_def(rd);
683 collector.reg_use(rn);
684 }
685 Inst::IntToFpu { rd, rn, .. } => {
686 collector.reg_def(rd);
687 collector.reg_use(rn);
688 }
689 Inst::FpuCSel16 { rd, rn, rm, .. }
690 | Inst::FpuCSel32 { rd, rn, rm, .. }
691 | Inst::FpuCSel64 { rd, rn, rm, .. } => {
692 collector.reg_def(rd);
693 collector.reg_use(rn);
694 collector.reg_use(rm);
695 }
696 Inst::FpuRound { rd, rn, .. } => {
697 collector.reg_def(rd);
698 collector.reg_use(rn);
699 }
700 Inst::MovToFpu { rd, rn, .. } => {
701 collector.reg_def(rd);
702 collector.reg_use(rn);
703 }
704 Inst::FpuMoveFPImm { rd, .. } => {
705 collector.reg_def(rd);
706 }
707 Inst::MovToVec { rd, ri, rn, .. } => {
708 collector.reg_reuse_def(rd, 1); collector.reg_use(ri);
710 collector.reg_use(rn);
711 }
712 Inst::MovFromVec { rd, rn, .. } | Inst::MovFromVecSigned { rd, rn, .. } => {
713 collector.reg_def(rd);
714 collector.reg_use(rn);
715 }
716 Inst::VecDup { rd, rn, .. } => {
717 collector.reg_def(rd);
718 collector.reg_use(rn);
719 }
720 Inst::VecDupFromFpu { rd, rn, .. } => {
721 collector.reg_def(rd);
722 collector.reg_use(rn);
723 }
724 Inst::VecDupFPImm { rd, .. } => {
725 collector.reg_def(rd);
726 }
727 Inst::VecDupImm { rd, .. } => {
728 collector.reg_def(rd);
729 }
730 Inst::VecExtend { rd, rn, .. } => {
731 collector.reg_def(rd);
732 collector.reg_use(rn);
733 }
734 Inst::VecMovElement { rd, ri, rn, .. } => {
735 collector.reg_reuse_def(rd, 1); collector.reg_use(ri);
737 collector.reg_use(rn);
738 }
739 Inst::VecRRLong { rd, rn, .. } => {
740 collector.reg_def(rd);
741 collector.reg_use(rn);
742 }
743 Inst::VecRRNarrowLow { rd, rn, .. } => {
744 collector.reg_use(rn);
745 collector.reg_def(rd);
746 }
747 Inst::VecRRNarrowHigh { rd, ri, rn, .. } => {
748 collector.reg_use(rn);
749 collector.reg_reuse_def(rd, 2); collector.reg_use(ri);
751 }
752 Inst::VecRRPair { rd, rn, .. } => {
753 collector.reg_def(rd);
754 collector.reg_use(rn);
755 }
756 Inst::VecRRRLong { rd, rn, rm, .. } => {
757 collector.reg_def(rd);
758 collector.reg_use(rn);
759 collector.reg_use(rm);
760 }
761 Inst::VecRRRLongMod { rd, ri, rn, rm, .. } => {
762 collector.reg_reuse_def(rd, 1); collector.reg_use(ri);
764 collector.reg_use(rn);
765 collector.reg_use(rm);
766 }
767 Inst::VecRRPairLong { rd, rn, .. } => {
768 collector.reg_def(rd);
769 collector.reg_use(rn);
770 }
771 Inst::VecRRR { rd, rn, rm, .. } => {
772 collector.reg_def(rd);
773 collector.reg_use(rn);
774 collector.reg_use(rm);
775 }
776 Inst::VecRRRMod { rd, ri, rn, rm, .. } | Inst::VecFmlaElem { rd, ri, rn, rm, .. } => {
777 collector.reg_reuse_def(rd, 1); collector.reg_use(ri);
779 collector.reg_use(rn);
780 collector.reg_use(rm);
781 }
782 Inst::MovToNZCV { rn } => {
783 collector.reg_use(rn);
784 }
785 Inst::MovFromNZCV { rd } => {
786 collector.reg_def(rd);
787 }
788 Inst::Extend { rd, rn, .. } => {
789 collector.reg_def(rd);
790 collector.reg_use(rn);
791 }
792 Inst::Args { args } => {
793 for ArgPair { vreg, preg } in args {
794 collector.reg_fixed_def(vreg, *preg);
795 }
796 }
797 Inst::Rets { rets } => {
798 for RetPair { vreg, preg } in rets {
799 collector.reg_fixed_use(vreg, *preg);
800 }
801 }
802 Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}
803 Inst::Jump { .. } => {}
804 Inst::Call { info, .. } => {
805 let CallInfo { uses, defs, .. } = &mut **info;
806 for CallArgPair { vreg, preg } in uses {
807 collector.reg_fixed_use(vreg, *preg);
808 }
809 for CallRetPair { vreg, location } in defs {
810 match location {
811 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
812 RetLocation::Stack(..) => collector.any_def(vreg),
813 }
814 }
815 collector.reg_clobbers(info.clobbers);
816 if let Some(try_call_info) = &mut info.try_call_info {
817 try_call_info.collect_operands(collector);
818 }
819 }
820 Inst::CallInd { info, .. } => {
821 let CallInfo {
822 dest, uses, defs, ..
823 } = &mut **info;
824 collector.reg_use(dest);
825 for CallArgPair { vreg, preg } in uses {
826 collector.reg_fixed_use(vreg, *preg);
827 }
828 for CallRetPair { vreg, location } in defs {
829 match location {
830 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
831 RetLocation::Stack(..) => collector.any_def(vreg),
832 }
833 }
834 collector.reg_clobbers(info.clobbers);
835 if let Some(try_call_info) = &mut info.try_call_info {
836 try_call_info.collect_operands(collector);
837 }
838 }
839 Inst::ReturnCall { info } => {
840 for CallArgPair { vreg, preg } in &mut info.uses {
841 collector.reg_fixed_use(vreg, *preg);
842 }
843 }
844 Inst::ReturnCallInd { info } => {
845 collector.reg_fixed_use(&mut info.dest, xreg(1));
850 for CallArgPair { vreg, preg } in &mut info.uses {
851 collector.reg_fixed_use(vreg, *preg);
852 }
853 }
854 Inst::CondBr { kind, .. } => match kind {
855 CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
856 CondBrKind::Cond(_) => {}
857 },
858 Inst::TestBitAndBranch { rn, .. } => {
859 collector.reg_use(rn);
860 }
861 Inst::IndirectBr { rn, .. } => {
862 collector.reg_use(rn);
863 }
864 Inst::Nop0 | Inst::Nop4 => {}
865 Inst::Brk => {}
866 Inst::Udf { .. } => {}
867 Inst::TrapIf { kind, .. } => match kind {
868 CondBrKind::Zero(rt, _) | CondBrKind::NotZero(rt, _) => collector.reg_use(rt),
869 CondBrKind::Cond(_) => {}
870 },
871 Inst::Adr { rd, .. } | Inst::Adrp { rd, .. } => {
872 collector.reg_def(rd);
873 }
874 Inst::Word4 { .. } | Inst::Word8 { .. } => {}
875 Inst::JTSequence {
876 ridx, rtmp1, rtmp2, ..
877 } => {
878 collector.reg_use(ridx);
879 collector.reg_early_def(rtmp1);
880 collector.reg_early_def(rtmp2);
881 }
882 Inst::LoadExtName { rd, .. } => {
883 collector.reg_def(rd);
884 }
885 Inst::LoadAddr { rd, mem } => {
886 collector.reg_def(rd);
887 memarg_operands(mem, collector);
888 }
889 Inst::Paci { .. } | Inst::Xpaclri => {
890 }
893 Inst::Bti { .. } => {}
894
895 Inst::ElfTlsGetAddr { rd, tmp, .. } => {
896 collector.reg_fixed_def(rd, regs::xreg(0));
903 collector.reg_early_def(tmp);
904 }
905 Inst::MachOTlsGetAddr { rd, .. } => {
906 collector.reg_fixed_def(rd, regs::xreg(0));
907 let mut clobbers =
908 AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64, false);
909 clobbers.remove(regs::xreg_preg(0));
910 collector.reg_clobbers(clobbers);
911 }
912 Inst::Unwind { .. } => {}
913 Inst::EmitIsland { .. } => {}
914 Inst::DummyUse { reg } => {
915 collector.reg_use(reg);
916 }
917 Inst::StackProbeLoop { start, end, .. } => {
918 collector.reg_early_def(start);
919 collector.reg_use(end);
920 }
921 }
922}
923
924impl MachInst for Inst {
928 type ABIMachineSpec = AArch64MachineDeps;
929 type LabelUse = LabelUse;
930
931 const TRAP_OPCODE: &'static [u8] = &0xc11f_u32.to_le_bytes();
934
935 fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
936 aarch64_get_operands(self, collector);
937 }
938
939 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
940 match self {
941 &Inst::Mov {
942 size: OperandSize::Size64,
943 rd,
944 rm,
945 } => Some((rd, rm)),
946 &Inst::FpuMove64 { rd, rn } => Some((rd, rn)),
947 &Inst::FpuMove128 { rd, rn } => Some((rd, rn)),
948 _ => None,
949 }
950 }
951
952 fn is_included_in_clobbers(&self) -> bool {
953 let (caller, callee, is_exception) = match self {
954 Inst::Args { .. } => return false,
955 Inst::Call { info } => (
956 info.caller_conv,
957 info.callee_conv,
958 info.try_call_info.is_some(),
959 ),
960 Inst::CallInd { info } => (
961 info.caller_conv,
962 info.callee_conv,
963 info.try_call_info.is_some(),
964 ),
965 _ => return true,
966 };
967
968 let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller, is_exception);
980 let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee, is_exception);
981
982 let mut all_clobbers = caller_clobbers;
983 all_clobbers.union_from(callee_clobbers);
984 all_clobbers != caller_clobbers
985 }
986
987 fn is_trap(&self) -> bool {
988 match self {
989 Self::Udf { .. } => true,
990 _ => false,
991 }
992 }
993
994 fn is_args(&self) -> bool {
995 match self {
996 Self::Args { .. } => true,
997 _ => false,
998 }
999 }
1000
1001 fn is_term(&self) -> MachTerminator {
1002 match self {
1003 &Inst::Rets { .. } => MachTerminator::Ret,
1004 &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
1005 &Inst::Jump { .. } => MachTerminator::Branch,
1006 &Inst::CondBr { .. } => MachTerminator::Branch,
1007 &Inst::TestBitAndBranch { .. } => MachTerminator::Branch,
1008 &Inst::IndirectBr { .. } => MachTerminator::Branch,
1009 &Inst::JTSequence { .. } => MachTerminator::Branch,
1010 &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1011 &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1012 _ => MachTerminator::None,
1013 }
1014 }
1015
1016 fn is_mem_access(&self) -> bool {
1017 match self {
1018 &Inst::ULoad8 { .. }
1019 | &Inst::SLoad8 { .. }
1020 | &Inst::ULoad16 { .. }
1021 | &Inst::SLoad16 { .. }
1022 | &Inst::ULoad32 { .. }
1023 | &Inst::SLoad32 { .. }
1024 | &Inst::ULoad64 { .. }
1025 | &Inst::LoadP64 { .. }
1026 | &Inst::FpuLoad16 { .. }
1027 | &Inst::FpuLoad32 { .. }
1028 | &Inst::FpuLoad64 { .. }
1029 | &Inst::FpuLoad128 { .. }
1030 | &Inst::FpuLoadP64 { .. }
1031 | &Inst::FpuLoadP128 { .. }
1032 | &Inst::Store8 { .. }
1033 | &Inst::Store16 { .. }
1034 | &Inst::Store32 { .. }
1035 | &Inst::Store64 { .. }
1036 | &Inst::StoreP64 { .. }
1037 | &Inst::FpuStore16 { .. }
1038 | &Inst::FpuStore32 { .. }
1039 | &Inst::FpuStore64 { .. }
1040 | &Inst::FpuStore128 { .. } => true,
1041 _ => false,
1043 }
1044 }
1045
1046 fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
1047 let bits = ty.bits();
1048
1049 assert!(bits <= 128);
1050 assert!(to_reg.to_reg().class() == from_reg.class());
1051 match from_reg.class() {
1052 RegClass::Int => Inst::Mov {
1053 size: OperandSize::Size64,
1054 rd: to_reg,
1055 rm: from_reg,
1056 },
1057 RegClass::Float => {
1058 if bits > 64 {
1059 Inst::FpuMove128 {
1060 rd: to_reg,
1061 rn: from_reg,
1062 }
1063 } else {
1064 Inst::FpuMove64 {
1065 rd: to_reg,
1066 rn: from_reg,
1067 }
1068 }
1069 }
1070 RegClass::Vector => unreachable!(),
1071 }
1072 }
1073
1074 fn is_safepoint(&self) -> bool {
1075 match self {
1076 Inst::Call { .. } | Inst::CallInd { .. } => true,
1077 _ => false,
1078 }
1079 }
1080
1081 fn gen_dummy_use(reg: Reg) -> Inst {
1082 Inst::DummyUse { reg }
1083 }
1084
1085 fn gen_nop(preferred_size: usize) -> Inst {
1086 if preferred_size == 0 {
1087 return Inst::Nop0;
1088 }
1089 assert!(preferred_size >= 4);
1091 Inst::Nop4
1092 }
1093
1094 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
1095 match ty {
1096 I8 => Ok((&[RegClass::Int], &[I8])),
1097 I16 => Ok((&[RegClass::Int], &[I16])),
1098 I32 => Ok((&[RegClass::Int], &[I32])),
1099 I64 => Ok((&[RegClass::Int], &[I64])),
1100 F16 => Ok((&[RegClass::Float], &[F16])),
1101 F32 => Ok((&[RegClass::Float], &[F32])),
1102 F64 => Ok((&[RegClass::Float], &[F64])),
1103 F128 => Ok((&[RegClass::Float], &[F128])),
1104 I128 => Ok((&[RegClass::Int, RegClass::Int], &[I64, I64])),
1105 _ if ty.is_vector() && ty.bits() <= 128 => {
1106 let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1107 Ok((
1108 &[RegClass::Float],
1109 slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1110 ))
1111 }
1112 _ if ty.is_dynamic_vector() => Ok((&[RegClass::Float], &[I8X16])),
1113 _ => Err(CodegenError::Unsupported(format!(
1114 "Unexpected SSA-value type: {ty}"
1115 ))),
1116 }
1117 }
1118
1119 fn canonical_type_for_rc(rc: RegClass) -> Type {
1120 match rc {
1121 RegClass::Float => types::I8X16,
1122 RegClass::Int => types::I64,
1123 RegClass::Vector => unreachable!(),
1124 }
1125 }
1126
1127 fn gen_jump(target: MachLabel) -> Inst {
1128 Inst::Jump {
1129 dest: BranchTarget::Label(target),
1130 }
1131 }
1132
1133 fn worst_case_size() -> CodeOffset {
1134 44
1142 }
1143
1144 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
1145 RegClass::Int
1146 }
1147
1148 fn gen_block_start(
1149 is_indirect_branch_target: bool,
1150 is_forward_edge_cfi_enabled: bool,
1151 ) -> Option<Self> {
1152 if is_indirect_branch_target && is_forward_edge_cfi_enabled {
1153 Some(Inst::Bti {
1154 targets: BranchTargetType::J,
1155 })
1156 } else {
1157 None
1158 }
1159 }
1160
1161 fn function_alignment() -> FunctionAlignment {
1162 FunctionAlignment {
1165 minimum: 4,
1166 preferred: 32,
1167 }
1168 }
1169}
1170
1171fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (String, String) {
1175 let (mem_insts, mem) = mem_finalize(None, mem, access_ty, state);
1176 let mut mem_str = mem_insts
1177 .into_iter()
1178 .map(|inst| inst.print_with_state(&mut EmitState::default()))
1179 .collect::<Vec<_>>()
1180 .join(" ; ");
1181 if !mem_str.is_empty() {
1182 mem_str += " ; ";
1183 }
1184
1185 let mem = mem.pretty_print(access_ty.bytes() as u8);
1186 (mem_str, mem)
1187}
1188
1189fn pretty_print_try_call(info: &TryCallInfo) -> String {
1190 format!(
1191 "; b {:?}; catch [{}]",
1192 info.continuation,
1193 info.pretty_print_dests()
1194 )
1195}
1196
1197impl Inst {
1198 fn print_with_state(&self, state: &mut EmitState) -> String {
1199 fn op_name(alu_op: ALUOp) -> &'static str {
1200 match alu_op {
1201 ALUOp::Add => "add",
1202 ALUOp::Sub => "sub",
1203 ALUOp::Orr => "orr",
1204 ALUOp::And => "and",
1205 ALUOp::AndS => "ands",
1206 ALUOp::Eor => "eor",
1207 ALUOp::AddS => "adds",
1208 ALUOp::SubS => "subs",
1209 ALUOp::SMulH => "smulh",
1210 ALUOp::UMulH => "umulh",
1211 ALUOp::SDiv => "sdiv",
1212 ALUOp::UDiv => "udiv",
1213 ALUOp::AndNot => "bic",
1214 ALUOp::OrrNot => "orn",
1215 ALUOp::EorNot => "eon",
1216 ALUOp::Extr => "extr",
1217 ALUOp::Lsr => "lsr",
1218 ALUOp::Asr => "asr",
1219 ALUOp::Lsl => "lsl",
1220 ALUOp::Adc => "adc",
1221 ALUOp::AdcS => "adcs",
1222 ALUOp::Sbc => "sbc",
1223 ALUOp::SbcS => "sbcs",
1224 }
1225 }
1226
1227 match self {
1228 &Inst::Nop0 => "nop-zero-len".to_string(),
1229 &Inst::Nop4 => "nop".to_string(),
1230 &Inst::AluRRR {
1231 alu_op,
1232 size,
1233 rd,
1234 rn,
1235 rm,
1236 } => {
1237 let op = op_name(alu_op);
1238 let rd = pretty_print_ireg(rd.to_reg(), size);
1239 let rn = pretty_print_ireg(rn, size);
1240 let rm = pretty_print_ireg(rm, size);
1241 format!("{op} {rd}, {rn}, {rm}")
1242 }
1243 &Inst::AluRRRR {
1244 alu_op,
1245 size,
1246 rd,
1247 rn,
1248 rm,
1249 ra,
1250 } => {
1251 let (op, da_size) = match alu_op {
1252 ALUOp3::MAdd => ("madd", size),
1253 ALUOp3::MSub => ("msub", size),
1254 ALUOp3::UMAddL => ("umaddl", OperandSize::Size64),
1255 ALUOp3::SMAddL => ("smaddl", OperandSize::Size64),
1256 };
1257 let rd = pretty_print_ireg(rd.to_reg(), da_size);
1258 let rn = pretty_print_ireg(rn, size);
1259 let rm = pretty_print_ireg(rm, size);
1260 let ra = pretty_print_ireg(ra, da_size);
1261
1262 format!("{op} {rd}, {rn}, {rm}, {ra}")
1263 }
1264 &Inst::AluRRImm12 {
1265 alu_op,
1266 size,
1267 rd,
1268 rn,
1269 ref imm12,
1270 } => {
1271 let op = op_name(alu_op);
1272 let rd = pretty_print_ireg(rd.to_reg(), size);
1273 let rn = pretty_print_ireg(rn, size);
1274
1275 if imm12.bits == 0 && alu_op == ALUOp::Add && size.is64() {
1276 format!("mov {rd}, {rn}")
1278 } else {
1279 let imm12 = imm12.pretty_print(0);
1280 format!("{op} {rd}, {rn}, {imm12}")
1281 }
1282 }
1283 &Inst::AluRRImmLogic {
1284 alu_op,
1285 size,
1286 rd,
1287 rn,
1288 ref imml,
1289 } => {
1290 let op = op_name(alu_op);
1291 let rd = pretty_print_ireg(rd.to_reg(), size);
1292 let rn = pretty_print_ireg(rn, size);
1293 let imml = imml.pretty_print(0);
1294 format!("{op} {rd}, {rn}, {imml}")
1295 }
1296 &Inst::AluRRImmShift {
1297 alu_op,
1298 size,
1299 rd,
1300 rn,
1301 ref immshift,
1302 } => {
1303 let op = op_name(alu_op);
1304 let rd = pretty_print_ireg(rd.to_reg(), size);
1305 let rn = pretty_print_ireg(rn, size);
1306 let immshift = immshift.pretty_print(0);
1307 format!("{op} {rd}, {rn}, {immshift}")
1308 }
1309 &Inst::AluRRRShift {
1310 alu_op,
1311 size,
1312 rd,
1313 rn,
1314 rm,
1315 ref shiftop,
1316 } => {
1317 let op = op_name(alu_op);
1318 let rd = pretty_print_ireg(rd.to_reg(), size);
1319 let rn = pretty_print_ireg(rn, size);
1320 let rm = pretty_print_ireg(rm, size);
1321 let shiftop = shiftop.pretty_print(0);
1322 format!("{op} {rd}, {rn}, {rm}, {shiftop}")
1323 }
1324 &Inst::AluRRRExtend {
1325 alu_op,
1326 size,
1327 rd,
1328 rn,
1329 rm,
1330 ref extendop,
1331 } => {
1332 let op = op_name(alu_op);
1333 let rd = pretty_print_ireg(rd.to_reg(), size);
1334 let rn = pretty_print_ireg(rn, size);
1335 let rm = pretty_print_ireg(rm, size);
1336 let extendop = extendop.pretty_print(0);
1337 format!("{op} {rd}, {rn}, {rm}, {extendop}")
1338 }
1339 &Inst::BitRR { op, size, rd, rn } => {
1340 let op = op.op_str();
1341 let rd = pretty_print_ireg(rd.to_reg(), size);
1342 let rn = pretty_print_ireg(rn, size);
1343 format!("{op} {rd}, {rn}")
1344 }
1345 &Inst::ULoad8 { rd, ref mem, .. }
1346 | &Inst::SLoad8 { rd, ref mem, .. }
1347 | &Inst::ULoad16 { rd, ref mem, .. }
1348 | &Inst::SLoad16 { rd, ref mem, .. }
1349 | &Inst::ULoad32 { rd, ref mem, .. }
1350 | &Inst::SLoad32 { rd, ref mem, .. }
1351 | &Inst::ULoad64 { rd, ref mem, .. } => {
1352 let is_unscaled = match &mem {
1353 &AMode::Unscaled { .. } => true,
1354 _ => false,
1355 };
1356 let (op, size) = match (self, is_unscaled) {
1357 (&Inst::ULoad8 { .. }, false) => ("ldrb", OperandSize::Size32),
1358 (&Inst::ULoad8 { .. }, true) => ("ldurb", OperandSize::Size32),
1359 (&Inst::SLoad8 { .. }, false) => ("ldrsb", OperandSize::Size64),
1360 (&Inst::SLoad8 { .. }, true) => ("ldursb", OperandSize::Size64),
1361 (&Inst::ULoad16 { .. }, false) => ("ldrh", OperandSize::Size32),
1362 (&Inst::ULoad16 { .. }, true) => ("ldurh", OperandSize::Size32),
1363 (&Inst::SLoad16 { .. }, false) => ("ldrsh", OperandSize::Size64),
1364 (&Inst::SLoad16 { .. }, true) => ("ldursh", OperandSize::Size64),
1365 (&Inst::ULoad32 { .. }, false) => ("ldr", OperandSize::Size32),
1366 (&Inst::ULoad32 { .. }, true) => ("ldur", OperandSize::Size32),
1367 (&Inst::SLoad32 { .. }, false) => ("ldrsw", OperandSize::Size64),
1368 (&Inst::SLoad32 { .. }, true) => ("ldursw", OperandSize::Size64),
1369 (&Inst::ULoad64 { .. }, false) => ("ldr", OperandSize::Size64),
1370 (&Inst::ULoad64 { .. }, true) => ("ldur", OperandSize::Size64),
1371 _ => unreachable!(),
1372 };
1373
1374 let rd = pretty_print_ireg(rd.to_reg(), size);
1375 let mem = mem.clone();
1376 let access_ty = self.mem_type().unwrap();
1377 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1378
1379 format!("{mem_str}{op} {rd}, {mem}")
1380 }
1381 &Inst::Store8 { rd, ref mem, .. }
1382 | &Inst::Store16 { rd, ref mem, .. }
1383 | &Inst::Store32 { rd, ref mem, .. }
1384 | &Inst::Store64 { rd, ref mem, .. } => {
1385 let is_unscaled = match &mem {
1386 &AMode::Unscaled { .. } => true,
1387 _ => false,
1388 };
1389 let (op, size) = match (self, is_unscaled) {
1390 (&Inst::Store8 { .. }, false) => ("strb", OperandSize::Size32),
1391 (&Inst::Store8 { .. }, true) => ("sturb", OperandSize::Size32),
1392 (&Inst::Store16 { .. }, false) => ("strh", OperandSize::Size32),
1393 (&Inst::Store16 { .. }, true) => ("sturh", OperandSize::Size32),
1394 (&Inst::Store32 { .. }, false) => ("str", OperandSize::Size32),
1395 (&Inst::Store32 { .. }, true) => ("stur", OperandSize::Size32),
1396 (&Inst::Store64 { .. }, false) => ("str", OperandSize::Size64),
1397 (&Inst::Store64 { .. }, true) => ("stur", OperandSize::Size64),
1398 _ => unreachable!(),
1399 };
1400
1401 let rd = pretty_print_ireg(rd, size);
1402 let mem = mem.clone();
1403 let access_ty = self.mem_type().unwrap();
1404 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1405
1406 format!("{mem_str}{op} {rd}, {mem}")
1407 }
1408 &Inst::StoreP64 {
1409 rt, rt2, ref mem, ..
1410 } => {
1411 let rt = pretty_print_ireg(rt, OperandSize::Size64);
1412 let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);
1413 let mem = mem.clone();
1414 let mem = mem.pretty_print_default();
1415 format!("stp {rt}, {rt2}, {mem}")
1416 }
1417 &Inst::LoadP64 {
1418 rt, rt2, ref mem, ..
1419 } => {
1420 let rt = pretty_print_ireg(rt.to_reg(), OperandSize::Size64);
1421 let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);
1422 let mem = mem.clone();
1423 let mem = mem.pretty_print_default();
1424 format!("ldp {rt}, {rt2}, {mem}")
1425 }
1426 &Inst::Mov { size, rd, rm } => {
1427 let rd = pretty_print_ireg(rd.to_reg(), size);
1428 let rm = pretty_print_ireg(rm, size);
1429 format!("mov {rd}, {rm}")
1430 }
1431 &Inst::MovFromPReg { rd, rm } => {
1432 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1433 let rm = show_ireg_sized(rm.into(), OperandSize::Size64);
1434 format!("mov {rd}, {rm}")
1435 }
1436 &Inst::MovToPReg { rd, rm } => {
1437 let rd = show_ireg_sized(rd.into(), OperandSize::Size64);
1438 let rm = pretty_print_ireg(rm, OperandSize::Size64);
1439 format!("mov {rd}, {rm}")
1440 }
1441 &Inst::MovWide {
1442 op,
1443 rd,
1444 ref imm,
1445 size,
1446 } => {
1447 let op_str = match op {
1448 MoveWideOp::MovZ => "movz",
1449 MoveWideOp::MovN => "movn",
1450 };
1451 let rd = pretty_print_ireg(rd.to_reg(), size);
1452 let imm = imm.pretty_print(0);
1453 format!("{op_str} {rd}, {imm}")
1454 }
1455 &Inst::MovK {
1456 rd,
1457 rn,
1458 ref imm,
1459 size,
1460 } => {
1461 let rn = pretty_print_ireg(rn, size);
1462 let rd = pretty_print_ireg(rd.to_reg(), size);
1463 let imm = imm.pretty_print(0);
1464 format!("movk {rd}, {rn}, {imm}")
1465 }
1466 &Inst::CSel { rd, rn, rm, cond } => {
1467 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1468 let rn = pretty_print_ireg(rn, OperandSize::Size64);
1469 let rm = pretty_print_ireg(rm, OperandSize::Size64);
1470 let cond = cond.pretty_print(0);
1471 format!("csel {rd}, {rn}, {rm}, {cond}")
1472 }
1473 &Inst::CSNeg { rd, rn, rm, cond } => {
1474 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1475 let rn = pretty_print_ireg(rn, OperandSize::Size64);
1476 let rm = pretty_print_ireg(rm, OperandSize::Size64);
1477 let cond = cond.pretty_print(0);
1478 format!("csneg {rd}, {rn}, {rm}, {cond}")
1479 }
1480 &Inst::CSet { rd, cond } => {
1481 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1482 let cond = cond.pretty_print(0);
1483 format!("cset {rd}, {cond}")
1484 }
1485 &Inst::CSetm { rd, cond } => {
1486 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size64);
1487 let cond = cond.pretty_print(0);
1488 format!("csetm {rd}, {cond}")
1489 }
1490 &Inst::CCmp {
1491 size,
1492 rn,
1493 rm,
1494 nzcv,
1495 cond,
1496 } => {
1497 let rn = pretty_print_ireg(rn, size);
1498 let rm = pretty_print_ireg(rm, size);
1499 let nzcv = nzcv.pretty_print(0);
1500 let cond = cond.pretty_print(0);
1501 format!("ccmp {rn}, {rm}, {nzcv}, {cond}")
1502 }
1503 &Inst::CCmpImm {
1504 size,
1505 rn,
1506 imm,
1507 nzcv,
1508 cond,
1509 } => {
1510 let rn = pretty_print_ireg(rn, size);
1511 let imm = imm.pretty_print(0);
1512 let nzcv = nzcv.pretty_print(0);
1513 let cond = cond.pretty_print(0);
1514 format!("ccmp {rn}, {imm}, {nzcv}, {cond}")
1515 }
1516 &Inst::AtomicRMW {
1517 rs, rt, rn, ty, op, ..
1518 } => {
1519 let op = match op {
1520 AtomicRMWOp::Add => "ldaddal",
1521 AtomicRMWOp::Clr => "ldclral",
1522 AtomicRMWOp::Eor => "ldeoral",
1523 AtomicRMWOp::Set => "ldsetal",
1524 AtomicRMWOp::Smax => "ldsmaxal",
1525 AtomicRMWOp::Umax => "ldumaxal",
1526 AtomicRMWOp::Smin => "ldsminal",
1527 AtomicRMWOp::Umin => "lduminal",
1528 AtomicRMWOp::Swp => "swpal",
1529 };
1530
1531 let size = OperandSize::from_ty(ty);
1532 let rs = pretty_print_ireg(rs, size);
1533 let rt = pretty_print_ireg(rt.to_reg(), size);
1534 let rn = pretty_print_ireg(rn, OperandSize::Size64);
1535
1536 let ty_suffix = match ty {
1537 I8 => "b",
1538 I16 => "h",
1539 _ => "",
1540 };
1541 format!("{op}{ty_suffix} {rs}, {rt}, [{rn}]")
1542 }
1543 &Inst::AtomicRMWLoop {
1544 ty,
1545 op,
1546 addr,
1547 operand,
1548 oldval,
1549 scratch1,
1550 scratch2,
1551 ..
1552 } => {
1553 let op = match op {
1554 AtomicRMWLoopOp::Add => "add",
1555 AtomicRMWLoopOp::Sub => "sub",
1556 AtomicRMWLoopOp::Eor => "eor",
1557 AtomicRMWLoopOp::Orr => "orr",
1558 AtomicRMWLoopOp::And => "and",
1559 AtomicRMWLoopOp::Nand => "nand",
1560 AtomicRMWLoopOp::Smin => "smin",
1561 AtomicRMWLoopOp::Smax => "smax",
1562 AtomicRMWLoopOp::Umin => "umin",
1563 AtomicRMWLoopOp::Umax => "umax",
1564 AtomicRMWLoopOp::Xchg => "xchg",
1565 };
1566 let addr = pretty_print_ireg(addr, OperandSize::Size64);
1567 let operand = pretty_print_ireg(operand, OperandSize::Size64);
1568 let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1569 let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);
1570 let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);
1571 format!(
1572 "atomic_rmw_loop_{}_{} addr={} operand={} oldval={} scratch1={} scratch2={}",
1573 op,
1574 ty.bits(),
1575 addr,
1576 operand,
1577 oldval,
1578 scratch1,
1579 scratch2,
1580 )
1581 }
1582 &Inst::AtomicCAS {
1583 rd, rs, rt, rn, ty, ..
1584 } => {
1585 let op = match ty {
1586 I8 => "casalb",
1587 I16 => "casalh",
1588 I32 | I64 => "casal",
1589 _ => panic!("Unsupported type: {ty}"),
1590 };
1591 let size = OperandSize::from_ty(ty);
1592 let rd = pretty_print_ireg(rd.to_reg(), size);
1593 let rs = pretty_print_ireg(rs, size);
1594 let rt = pretty_print_ireg(rt, size);
1595 let rn = pretty_print_ireg(rn, OperandSize::Size64);
1596
1597 format!("{op} {rd}, {rs}, {rt}, [{rn}]")
1598 }
1599 &Inst::AtomicCASLoop {
1600 ty,
1601 addr,
1602 expected,
1603 replacement,
1604 oldval,
1605 scratch,
1606 ..
1607 } => {
1608 let addr = pretty_print_ireg(addr, OperandSize::Size64);
1609 let expected = pretty_print_ireg(expected, OperandSize::Size64);
1610 let replacement = pretty_print_ireg(replacement, OperandSize::Size64);
1611 let oldval = pretty_print_ireg(oldval.to_reg(), OperandSize::Size64);
1612 let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
1613 format!(
1614 "atomic_cas_loop_{} addr={}, expect={}, replacement={}, oldval={}, scratch={}",
1615 ty.bits(),
1616 addr,
1617 expected,
1618 replacement,
1619 oldval,
1620 scratch,
1621 )
1622 }
1623 &Inst::LoadAcquire {
1624 access_ty, rt, rn, ..
1625 } => {
1626 let (op, ty) = match access_ty {
1627 I8 => ("ldarb", I32),
1628 I16 => ("ldarh", I32),
1629 I32 => ("ldar", I32),
1630 I64 => ("ldar", I64),
1631 _ => panic!("Unsupported type: {access_ty}"),
1632 };
1633 let size = OperandSize::from_ty(ty);
1634 let rn = pretty_print_ireg(rn, OperandSize::Size64);
1635 let rt = pretty_print_ireg(rt.to_reg(), size);
1636 format!("{op} {rt}, [{rn}]")
1637 }
1638 &Inst::StoreRelease {
1639 access_ty, rt, rn, ..
1640 } => {
1641 let (op, ty) = match access_ty {
1642 I8 => ("stlrb", I32),
1643 I16 => ("stlrh", I32),
1644 I32 => ("stlr", I32),
1645 I64 => ("stlr", I64),
1646 _ => panic!("Unsupported type: {access_ty}"),
1647 };
1648 let size = OperandSize::from_ty(ty);
1649 let rn = pretty_print_ireg(rn, OperandSize::Size64);
1650 let rt = pretty_print_ireg(rt, size);
1651 format!("{op} {rt}, [{rn}]")
1652 }
1653 &Inst::Fence {} => {
1654 format!("dmb ish")
1655 }
1656 &Inst::Csdb {} => {
1657 format!("csdb")
1658 }
1659 &Inst::FpuMove32 { rd, rn } => {
1660 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1661 let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
1662 format!("fmov {rd}, {rn}")
1663 }
1664 &Inst::FpuMove64 { rd, rn } => {
1665 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1666 let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
1667 format!("fmov {rd}, {rn}")
1668 }
1669 &Inst::FpuMove128 { rd, rn } => {
1670 let rd = pretty_print_reg(rd.to_reg());
1671 let rn = pretty_print_reg(rn);
1672 format!("mov {rd}.16b, {rn}.16b")
1673 }
1674 &Inst::FpuMoveFromVec { rd, rn, idx, size } => {
1675 let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
1676 let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
1677 format!("mov {rd}, {rn}")
1678 }
1679 &Inst::FpuExtend { rd, rn, size } => {
1680 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1681 let rn = pretty_print_vreg_scalar(rn, size);
1682 format!("fmov {rd}, {rn}")
1683 }
1684 &Inst::FpuRR {
1685 fpu_op,
1686 size,
1687 rd,
1688 rn,
1689 } => {
1690 let op = match fpu_op {
1691 FPUOp1::Abs => "fabs",
1692 FPUOp1::Neg => "fneg",
1693 FPUOp1::Sqrt => "fsqrt",
1694 FPUOp1::Cvt32To64 | FPUOp1::Cvt64To32 => "fcvt",
1695 };
1696 let dst_size = match fpu_op {
1697 FPUOp1::Cvt32To64 => ScalarSize::Size64,
1698 FPUOp1::Cvt64To32 => ScalarSize::Size32,
1699 _ => size,
1700 };
1701 let rd = pretty_print_vreg_scalar(rd.to_reg(), dst_size);
1702 let rn = pretty_print_vreg_scalar(rn, size);
1703 format!("{op} {rd}, {rn}")
1704 }
1705 &Inst::FpuRRR {
1706 fpu_op,
1707 size,
1708 rd,
1709 rn,
1710 rm,
1711 } => {
1712 let op = match fpu_op {
1713 FPUOp2::Add => "fadd",
1714 FPUOp2::Sub => "fsub",
1715 FPUOp2::Mul => "fmul",
1716 FPUOp2::Div => "fdiv",
1717 FPUOp2::Max => "fmax",
1718 FPUOp2::Min => "fmin",
1719 };
1720 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1721 let rn = pretty_print_vreg_scalar(rn, size);
1722 let rm = pretty_print_vreg_scalar(rm, size);
1723 format!("{op} {rd}, {rn}, {rm}")
1724 }
1725 &Inst::FpuRRI { fpu_op, rd, rn } => {
1726 let (op, imm, vector) = match fpu_op {
1727 FPUOpRI::UShr32(imm) => ("ushr", imm.pretty_print(0), true),
1728 FPUOpRI::UShr64(imm) => ("ushr", imm.pretty_print(0), false),
1729 };
1730
1731 let (rd, rn) = if vector {
1732 (
1733 pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
1734 pretty_print_vreg_vector(rn, VectorSize::Size32x2),
1735 )
1736 } else {
1737 (
1738 pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
1739 pretty_print_vreg_scalar(rn, ScalarSize::Size64),
1740 )
1741 };
1742 format!("{op} {rd}, {rn}, {imm}")
1743 }
1744 &Inst::FpuRRIMod { fpu_op, rd, ri, rn } => {
1745 let (op, imm, vector) = match fpu_op {
1746 FPUOpRIMod::Sli32(imm) => ("sli", imm.pretty_print(0), true),
1747 FPUOpRIMod::Sli64(imm) => ("sli", imm.pretty_print(0), false),
1748 };
1749
1750 let (rd, ri, rn) = if vector {
1751 (
1752 pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size32x2),
1753 pretty_print_vreg_vector(ri, VectorSize::Size32x2),
1754 pretty_print_vreg_vector(rn, VectorSize::Size32x2),
1755 )
1756 } else {
1757 (
1758 pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64),
1759 pretty_print_vreg_scalar(ri, ScalarSize::Size64),
1760 pretty_print_vreg_scalar(rn, ScalarSize::Size64),
1761 )
1762 };
1763 format!("{op} {rd}, {ri}, {rn}, {imm}")
1764 }
1765 &Inst::FpuRRRR {
1766 fpu_op,
1767 size,
1768 rd,
1769 rn,
1770 rm,
1771 ra,
1772 } => {
1773 let op = match fpu_op {
1774 FPUOp3::MAdd => "fmadd",
1775 FPUOp3::MSub => "fmsub",
1776 FPUOp3::NMAdd => "fnmadd",
1777 FPUOp3::NMSub => "fnmsub",
1778 };
1779 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1780 let rn = pretty_print_vreg_scalar(rn, size);
1781 let rm = pretty_print_vreg_scalar(rm, size);
1782 let ra = pretty_print_vreg_scalar(ra, size);
1783 format!("{op} {rd}, {rn}, {rm}, {ra}")
1784 }
1785 &Inst::FpuCmp { size, rn, rm } => {
1786 let rn = pretty_print_vreg_scalar(rn, size);
1787 let rm = pretty_print_vreg_scalar(rm, size);
1788 format!("fcmp {rn}, {rm}")
1789 }
1790 &Inst::FpuLoad16 { rd, ref mem, .. } => {
1791 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
1792 let mem = mem.clone();
1793 let access_ty = self.mem_type().unwrap();
1794 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1795 format!("{mem_str}ldr {rd}, {mem}")
1796 }
1797 &Inst::FpuLoad32 { rd, ref mem, .. } => {
1798 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1799 let mem = mem.clone();
1800 let access_ty = self.mem_type().unwrap();
1801 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1802 format!("{mem_str}ldr {rd}, {mem}")
1803 }
1804 &Inst::FpuLoad64 { rd, ref mem, .. } => {
1805 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1806 let mem = mem.clone();
1807 let access_ty = self.mem_type().unwrap();
1808 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1809 format!("{mem_str}ldr {rd}, {mem}")
1810 }
1811 &Inst::FpuLoad128 { rd, ref mem, .. } => {
1812 let rd = pretty_print_reg(rd.to_reg());
1813 let rd = "q".to_string() + &rd[1..];
1814 let mem = mem.clone();
1815 let access_ty = self.mem_type().unwrap();
1816 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1817 format!("{mem_str}ldr {rd}, {mem}")
1818 }
1819 &Inst::FpuStore16 { rd, ref mem, .. } => {
1820 let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size16);
1821 let mem = mem.clone();
1822 let access_ty = self.mem_type().unwrap();
1823 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1824 format!("{mem_str}str {rd}, {mem}")
1825 }
1826 &Inst::FpuStore32 { rd, ref mem, .. } => {
1827 let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size32);
1828 let mem = mem.clone();
1829 let access_ty = self.mem_type().unwrap();
1830 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1831 format!("{mem_str}str {rd}, {mem}")
1832 }
1833 &Inst::FpuStore64 { rd, ref mem, .. } => {
1834 let rd = pretty_print_vreg_scalar(rd, ScalarSize::Size64);
1835 let mem = mem.clone();
1836 let access_ty = self.mem_type().unwrap();
1837 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1838 format!("{mem_str}str {rd}, {mem}")
1839 }
1840 &Inst::FpuStore128 { rd, ref mem, .. } => {
1841 let rd = pretty_print_reg(rd);
1842 let rd = "q".to_string() + &rd[1..];
1843 let mem = mem.clone();
1844 let access_ty = self.mem_type().unwrap();
1845 let (mem_str, mem) = mem_finalize_for_show(&mem, access_ty, state);
1846 format!("{mem_str}str {rd}, {mem}")
1847 }
1848 &Inst::FpuLoadP64 {
1849 rt, rt2, ref mem, ..
1850 } => {
1851 let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size64);
1852 let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size64);
1853 let mem = mem.clone();
1854 let mem = mem.pretty_print_default();
1855
1856 format!("ldp {rt}, {rt2}, {mem}")
1857 }
1858 &Inst::FpuStoreP64 {
1859 rt, rt2, ref mem, ..
1860 } => {
1861 let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size64);
1862 let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size64);
1863 let mem = mem.clone();
1864 let mem = mem.pretty_print_default();
1865
1866 format!("stp {rt}, {rt2}, {mem}")
1867 }
1868 &Inst::FpuLoadP128 {
1869 rt, rt2, ref mem, ..
1870 } => {
1871 let rt = pretty_print_vreg_scalar(rt.to_reg(), ScalarSize::Size128);
1872 let rt2 = pretty_print_vreg_scalar(rt2.to_reg(), ScalarSize::Size128);
1873 let mem = mem.clone();
1874 let mem = mem.pretty_print_default();
1875
1876 format!("ldp {rt}, {rt2}, {mem}")
1877 }
1878 &Inst::FpuStoreP128 {
1879 rt, rt2, ref mem, ..
1880 } => {
1881 let rt = pretty_print_vreg_scalar(rt, ScalarSize::Size128);
1882 let rt2 = pretty_print_vreg_scalar(rt2, ScalarSize::Size128);
1883 let mem = mem.clone();
1884 let mem = mem.pretty_print_default();
1885
1886 format!("stp {rt}, {rt2}, {mem}")
1887 }
1888 &Inst::FpuToInt { op, rd, rn } => {
1889 let (op, sizesrc, sizedest) = match op {
1890 FpuToIntOp::F32ToI32 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size32),
1891 FpuToIntOp::F32ToU32 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size32),
1892 FpuToIntOp::F32ToI64 => ("fcvtzs", ScalarSize::Size32, OperandSize::Size64),
1893 FpuToIntOp::F32ToU64 => ("fcvtzu", ScalarSize::Size32, OperandSize::Size64),
1894 FpuToIntOp::F64ToI32 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size32),
1895 FpuToIntOp::F64ToU32 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size32),
1896 FpuToIntOp::F64ToI64 => ("fcvtzs", ScalarSize::Size64, OperandSize::Size64),
1897 FpuToIntOp::F64ToU64 => ("fcvtzu", ScalarSize::Size64, OperandSize::Size64),
1898 };
1899 let rd = pretty_print_ireg(rd.to_reg(), sizedest);
1900 let rn = pretty_print_vreg_scalar(rn, sizesrc);
1901 format!("{op} {rd}, {rn}")
1902 }
1903 &Inst::IntToFpu { op, rd, rn } => {
1904 let (op, sizesrc, sizedest) = match op {
1905 IntToFpuOp::I32ToF32 => ("scvtf", OperandSize::Size32, ScalarSize::Size32),
1906 IntToFpuOp::U32ToF32 => ("ucvtf", OperandSize::Size32, ScalarSize::Size32),
1907 IntToFpuOp::I64ToF32 => ("scvtf", OperandSize::Size64, ScalarSize::Size32),
1908 IntToFpuOp::U64ToF32 => ("ucvtf", OperandSize::Size64, ScalarSize::Size32),
1909 IntToFpuOp::I32ToF64 => ("scvtf", OperandSize::Size32, ScalarSize::Size64),
1910 IntToFpuOp::U32ToF64 => ("ucvtf", OperandSize::Size32, ScalarSize::Size64),
1911 IntToFpuOp::I64ToF64 => ("scvtf", OperandSize::Size64, ScalarSize::Size64),
1912 IntToFpuOp::U64ToF64 => ("ucvtf", OperandSize::Size64, ScalarSize::Size64),
1913 };
1914 let rd = pretty_print_vreg_scalar(rd.to_reg(), sizedest);
1915 let rn = pretty_print_ireg(rn, sizesrc);
1916 format!("{op} {rd}, {rn}")
1917 }
1918 &Inst::FpuCSel16 { rd, rn, rm, cond } => {
1919 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size16);
1920 let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size16);
1921 let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size16);
1922 let cond = cond.pretty_print(0);
1923 format!("fcsel {rd}, {rn}, {rm}, {cond}")
1924 }
1925 &Inst::FpuCSel32 { rd, rn, rm, cond } => {
1926 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size32);
1927 let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size32);
1928 let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size32);
1929 let cond = cond.pretty_print(0);
1930 format!("fcsel {rd}, {rn}, {rm}, {cond}")
1931 }
1932 &Inst::FpuCSel64 { rd, rn, rm, cond } => {
1933 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
1934 let rn = pretty_print_vreg_scalar(rn, ScalarSize::Size64);
1935 let rm = pretty_print_vreg_scalar(rm, ScalarSize::Size64);
1936 let cond = cond.pretty_print(0);
1937 format!("fcsel {rd}, {rn}, {rm}, {cond}")
1938 }
1939 &Inst::FpuRound { op, rd, rn } => {
1940 let (inst, size) = match op {
1941 FpuRoundMode::Minus32 => ("frintm", ScalarSize::Size32),
1942 FpuRoundMode::Minus64 => ("frintm", ScalarSize::Size64),
1943 FpuRoundMode::Plus32 => ("frintp", ScalarSize::Size32),
1944 FpuRoundMode::Plus64 => ("frintp", ScalarSize::Size64),
1945 FpuRoundMode::Zero32 => ("frintz", ScalarSize::Size32),
1946 FpuRoundMode::Zero64 => ("frintz", ScalarSize::Size64),
1947 FpuRoundMode::Nearest32 => ("frintn", ScalarSize::Size32),
1948 FpuRoundMode::Nearest64 => ("frintn", ScalarSize::Size64),
1949 };
1950 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1951 let rn = pretty_print_vreg_scalar(rn, size);
1952 format!("{inst} {rd}, {rn}")
1953 }
1954 &Inst::MovToFpu { rd, rn, size } => {
1955 let operand_size = size.operand_size();
1956 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1957 let rn = pretty_print_ireg(rn, operand_size);
1958 format!("fmov {rd}, {rn}")
1959 }
1960 &Inst::FpuMoveFPImm { rd, imm, size } => {
1961 let imm = imm.pretty_print(0);
1962 let rd = pretty_print_vreg_scalar(rd.to_reg(), size);
1963
1964 format!("fmov {rd}, {imm}")
1965 }
1966 &Inst::MovToVec {
1967 rd,
1968 ri,
1969 rn,
1970 idx,
1971 size,
1972 } => {
1973 let rd = pretty_print_vreg_element(rd.to_reg(), idx as usize, size.lane_size());
1974 let ri = pretty_print_vreg_element(ri, idx as usize, size.lane_size());
1975 let rn = pretty_print_ireg(rn, size.operand_size());
1976 format!("mov {rd}, {ri}, {rn}")
1977 }
1978 &Inst::MovFromVec { rd, rn, idx, size } => {
1979 let op = match size {
1980 ScalarSize::Size8 => "umov",
1981 ScalarSize::Size16 => "umov",
1982 ScalarSize::Size32 => "mov",
1983 ScalarSize::Size64 => "mov",
1984 _ => unimplemented!(),
1985 };
1986 let rd = pretty_print_ireg(rd.to_reg(), size.operand_size());
1987 let rn = pretty_print_vreg_element(rn, idx as usize, size);
1988 format!("{op} {rd}, {rn}")
1989 }
1990 &Inst::MovFromVecSigned {
1991 rd,
1992 rn,
1993 idx,
1994 size,
1995 scalar_size,
1996 } => {
1997 let rd = pretty_print_ireg(rd.to_reg(), scalar_size);
1998 let rn = pretty_print_vreg_element(rn, idx as usize, size.lane_size());
1999 format!("smov {rd}, {rn}")
2000 }
2001 &Inst::VecDup { rd, rn, size } => {
2002 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2003 let rn = pretty_print_ireg(rn, size.operand_size());
2004 format!("dup {rd}, {rn}")
2005 }
2006 &Inst::VecDupFromFpu { rd, rn, size, lane } => {
2007 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2008 let rn = pretty_print_vreg_element(rn, lane.into(), size.lane_size());
2009 format!("dup {rd}, {rn}")
2010 }
2011 &Inst::VecDupFPImm { rd, imm, size } => {
2012 let imm = imm.pretty_print(0);
2013 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2014
2015 format!("fmov {rd}, {imm}")
2016 }
2017 &Inst::VecDupImm {
2018 rd,
2019 imm,
2020 invert,
2021 size,
2022 } => {
2023 let imm = imm.pretty_print(0);
2024 let op = if invert { "mvni" } else { "movi" };
2025 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2026
2027 format!("{op} {rd}, {imm}")
2028 }
2029 &Inst::VecExtend {
2030 t,
2031 rd,
2032 rn,
2033 high_half,
2034 lane_size,
2035 } => {
2036 let vec64 = VectorSize::from_lane_size(lane_size.narrow(), false);
2037 let vec128 = VectorSize::from_lane_size(lane_size.narrow(), true);
2038 let rd_size = VectorSize::from_lane_size(lane_size, true);
2039 let (op, rn_size) = match (t, high_half) {
2040 (VecExtendOp::Sxtl, false) => ("sxtl", vec64),
2041 (VecExtendOp::Sxtl, true) => ("sxtl2", vec128),
2042 (VecExtendOp::Uxtl, false) => ("uxtl", vec64),
2043 (VecExtendOp::Uxtl, true) => ("uxtl2", vec128),
2044 };
2045 let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2046 let rn = pretty_print_vreg_vector(rn, rn_size);
2047 format!("{op} {rd}, {rn}")
2048 }
2049 &Inst::VecMovElement {
2050 rd,
2051 ri,
2052 rn,
2053 dest_idx,
2054 src_idx,
2055 size,
2056 } => {
2057 let rd =
2058 pretty_print_vreg_element(rd.to_reg(), dest_idx as usize, size.lane_size());
2059 let ri = pretty_print_vreg_element(ri, dest_idx as usize, size.lane_size());
2060 let rn = pretty_print_vreg_element(rn, src_idx as usize, size.lane_size());
2061 format!("mov {rd}, {ri}, {rn}")
2062 }
2063 &Inst::VecRRLong {
2064 op,
2065 rd,
2066 rn,
2067 high_half,
2068 } => {
2069 let (op, rd_size, size, suffix) = match (op, high_half) {
2070 (VecRRLongOp::Fcvtl16, false) => {
2071 ("fcvtl", VectorSize::Size32x4, VectorSize::Size16x4, "")
2072 }
2073 (VecRRLongOp::Fcvtl16, true) => {
2074 ("fcvtl2", VectorSize::Size32x4, VectorSize::Size16x8, "")
2075 }
2076 (VecRRLongOp::Fcvtl32, false) => {
2077 ("fcvtl", VectorSize::Size64x2, VectorSize::Size32x2, "")
2078 }
2079 (VecRRLongOp::Fcvtl32, true) => {
2080 ("fcvtl2", VectorSize::Size64x2, VectorSize::Size32x4, "")
2081 }
2082 (VecRRLongOp::Shll8, false) => {
2083 ("shll", VectorSize::Size16x8, VectorSize::Size8x8, ", #8")
2084 }
2085 (VecRRLongOp::Shll8, true) => {
2086 ("shll2", VectorSize::Size16x8, VectorSize::Size8x16, ", #8")
2087 }
2088 (VecRRLongOp::Shll16, false) => {
2089 ("shll", VectorSize::Size32x4, VectorSize::Size16x4, ", #16")
2090 }
2091 (VecRRLongOp::Shll16, true) => {
2092 ("shll2", VectorSize::Size32x4, VectorSize::Size16x8, ", #16")
2093 }
2094 (VecRRLongOp::Shll32, false) => {
2095 ("shll", VectorSize::Size64x2, VectorSize::Size32x2, ", #32")
2096 }
2097 (VecRRLongOp::Shll32, true) => {
2098 ("shll2", VectorSize::Size64x2, VectorSize::Size32x4, ", #32")
2099 }
2100 };
2101 let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2102 let rn = pretty_print_vreg_vector(rn, size);
2103
2104 format!("{op} {rd}, {rn}{suffix}")
2105 }
2106 &Inst::VecRRNarrowLow {
2107 op,
2108 rd,
2109 rn,
2110 lane_size,
2111 ..
2112 }
2113 | &Inst::VecRRNarrowHigh {
2114 op,
2115 rd,
2116 rn,
2117 lane_size,
2118 ..
2119 } => {
2120 let vec64 = VectorSize::from_lane_size(lane_size, false);
2121 let vec128 = VectorSize::from_lane_size(lane_size, true);
2122 let rn_size = VectorSize::from_lane_size(lane_size.widen(), true);
2123 let high_half = match self {
2124 &Inst::VecRRNarrowLow { .. } => false,
2125 &Inst::VecRRNarrowHigh { .. } => true,
2126 _ => unreachable!(),
2127 };
2128 let (op, rd_size) = match (op, high_half) {
2129 (VecRRNarrowOp::Xtn, false) => ("xtn", vec64),
2130 (VecRRNarrowOp::Xtn, true) => ("xtn2", vec128),
2131 (VecRRNarrowOp::Sqxtn, false) => ("sqxtn", vec64),
2132 (VecRRNarrowOp::Sqxtn, true) => ("sqxtn2", vec128),
2133 (VecRRNarrowOp::Sqxtun, false) => ("sqxtun", vec64),
2134 (VecRRNarrowOp::Sqxtun, true) => ("sqxtun2", vec128),
2135 (VecRRNarrowOp::Uqxtn, false) => ("uqxtn", vec64),
2136 (VecRRNarrowOp::Uqxtn, true) => ("uqxtn2", vec128),
2137 (VecRRNarrowOp::Fcvtn, false) => ("fcvtn", vec64),
2138 (VecRRNarrowOp::Fcvtn, true) => ("fcvtn2", vec128),
2139 };
2140 let rn = pretty_print_vreg_vector(rn, rn_size);
2141 let rd = pretty_print_vreg_vector(rd.to_reg(), rd_size);
2142 let ri = match self {
2143 &Inst::VecRRNarrowLow { .. } => "".to_string(),
2144 &Inst::VecRRNarrowHigh { ri, .. } => {
2145 format!("{}, ", pretty_print_vreg_vector(ri, rd_size))
2146 }
2147 _ => unreachable!(),
2148 };
2149
2150 format!("{op} {rd}, {ri}{rn}")
2151 }
2152 &Inst::VecRRPair { op, rd, rn } => {
2153 let op = match op {
2154 VecPairOp::Addp => "addp",
2155 };
2156 let rd = pretty_print_vreg_scalar(rd.to_reg(), ScalarSize::Size64);
2157 let rn = pretty_print_vreg_vector(rn, VectorSize::Size64x2);
2158
2159 format!("{op} {rd}, {rn}")
2160 }
2161 &Inst::VecRRPairLong { op, rd, rn } => {
2162 let (op, dest, src) = match op {
2163 VecRRPairLongOp::Saddlp8 => {
2164 ("saddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2165 }
2166 VecRRPairLongOp::Saddlp16 => {
2167 ("saddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2168 }
2169 VecRRPairLongOp::Uaddlp8 => {
2170 ("uaddlp", VectorSize::Size16x8, VectorSize::Size8x16)
2171 }
2172 VecRRPairLongOp::Uaddlp16 => {
2173 ("uaddlp", VectorSize::Size32x4, VectorSize::Size16x8)
2174 }
2175 };
2176 let rd = pretty_print_vreg_vector(rd.to_reg(), dest);
2177 let rn = pretty_print_vreg_vector(rn, src);
2178
2179 format!("{op} {rd}, {rn}")
2180 }
2181 &Inst::VecRRR {
2182 rd,
2183 rn,
2184 rm,
2185 alu_op,
2186 size,
2187 } => {
2188 let (op, size) = match alu_op {
2189 VecALUOp::Sqadd => ("sqadd", size),
2190 VecALUOp::Uqadd => ("uqadd", size),
2191 VecALUOp::Sqsub => ("sqsub", size),
2192 VecALUOp::Uqsub => ("uqsub", size),
2193 VecALUOp::Cmeq => ("cmeq", size),
2194 VecALUOp::Cmge => ("cmge", size),
2195 VecALUOp::Cmgt => ("cmgt", size),
2196 VecALUOp::Cmhs => ("cmhs", size),
2197 VecALUOp::Cmhi => ("cmhi", size),
2198 VecALUOp::Fcmeq => ("fcmeq", size),
2199 VecALUOp::Fcmgt => ("fcmgt", size),
2200 VecALUOp::Fcmge => ("fcmge", size),
2201 VecALUOp::And => ("and", VectorSize::Size8x16),
2202 VecALUOp::Bic => ("bic", VectorSize::Size8x16),
2203 VecALUOp::Orr => ("orr", VectorSize::Size8x16),
2204 VecALUOp::Eor => ("eor", VectorSize::Size8x16),
2205 VecALUOp::Umaxp => ("umaxp", size),
2206 VecALUOp::Add => ("add", size),
2207 VecALUOp::Sub => ("sub", size),
2208 VecALUOp::Mul => ("mul", size),
2209 VecALUOp::Sshl => ("sshl", size),
2210 VecALUOp::Ushl => ("ushl", size),
2211 VecALUOp::Umin => ("umin", size),
2212 VecALUOp::Smin => ("smin", size),
2213 VecALUOp::Umax => ("umax", size),
2214 VecALUOp::Smax => ("smax", size),
2215 VecALUOp::Urhadd => ("urhadd", size),
2216 VecALUOp::Fadd => ("fadd", size),
2217 VecALUOp::Fsub => ("fsub", size),
2218 VecALUOp::Fdiv => ("fdiv", size),
2219 VecALUOp::Fmax => ("fmax", size),
2220 VecALUOp::Fmin => ("fmin", size),
2221 VecALUOp::Fmul => ("fmul", size),
2222 VecALUOp::Addp => ("addp", size),
2223 VecALUOp::Zip1 => ("zip1", size),
2224 VecALUOp::Zip2 => ("zip2", size),
2225 VecALUOp::Sqrdmulh => ("sqrdmulh", size),
2226 VecALUOp::Uzp1 => ("uzp1", size),
2227 VecALUOp::Uzp2 => ("uzp2", size),
2228 VecALUOp::Trn1 => ("trn1", size),
2229 VecALUOp::Trn2 => ("trn2", size),
2230 };
2231 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2232 let rn = pretty_print_vreg_vector(rn, size);
2233 let rm = pretty_print_vreg_vector(rm, size);
2234 format!("{op} {rd}, {rn}, {rm}")
2235 }
2236 &Inst::VecRRRMod {
2237 rd,
2238 ri,
2239 rn,
2240 rm,
2241 alu_op,
2242 size,
2243 } => {
2244 let (op, size) = match alu_op {
2245 VecALUModOp::Bsl => ("bsl", VectorSize::Size8x16),
2246 VecALUModOp::Fmla => ("fmla", size),
2247 VecALUModOp::Fmls => ("fmls", size),
2248 };
2249 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2250 let ri = pretty_print_vreg_vector(ri, size);
2251 let rn = pretty_print_vreg_vector(rn, size);
2252 let rm = pretty_print_vreg_vector(rm, size);
2253 format!("{op} {rd}, {ri}, {rn}, {rm}")
2254 }
2255 &Inst::VecFmlaElem {
2256 rd,
2257 ri,
2258 rn,
2259 rm,
2260 alu_op,
2261 size,
2262 idx,
2263 } => {
2264 let (op, size) = match alu_op {
2265 VecALUModOp::Fmla => ("fmla", size),
2266 VecALUModOp::Fmls => ("fmls", size),
2267 _ => unreachable!(),
2268 };
2269 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2270 let ri = pretty_print_vreg_vector(ri, size);
2271 let rn = pretty_print_vreg_vector(rn, size);
2272 let rm = pretty_print_vreg_element(rm, idx.into(), size.lane_size());
2273 format!("{op} {rd}, {ri}, {rn}, {rm}")
2274 }
2275 &Inst::VecRRRLong {
2276 rd,
2277 rn,
2278 rm,
2279 alu_op,
2280 high_half,
2281 } => {
2282 let (op, dest_size, src_size) = match (alu_op, high_half) {
2283 (VecRRRLongOp::Smull8, false) => {
2284 ("smull", VectorSize::Size16x8, VectorSize::Size8x8)
2285 }
2286 (VecRRRLongOp::Smull8, true) => {
2287 ("smull2", VectorSize::Size16x8, VectorSize::Size8x16)
2288 }
2289 (VecRRRLongOp::Smull16, false) => {
2290 ("smull", VectorSize::Size32x4, VectorSize::Size16x4)
2291 }
2292 (VecRRRLongOp::Smull16, true) => {
2293 ("smull2", VectorSize::Size32x4, VectorSize::Size16x8)
2294 }
2295 (VecRRRLongOp::Smull32, false) => {
2296 ("smull", VectorSize::Size64x2, VectorSize::Size32x2)
2297 }
2298 (VecRRRLongOp::Smull32, true) => {
2299 ("smull2", VectorSize::Size64x2, VectorSize::Size32x4)
2300 }
2301 (VecRRRLongOp::Umull8, false) => {
2302 ("umull", VectorSize::Size16x8, VectorSize::Size8x8)
2303 }
2304 (VecRRRLongOp::Umull8, true) => {
2305 ("umull2", VectorSize::Size16x8, VectorSize::Size8x16)
2306 }
2307 (VecRRRLongOp::Umull16, false) => {
2308 ("umull", VectorSize::Size32x4, VectorSize::Size16x4)
2309 }
2310 (VecRRRLongOp::Umull16, true) => {
2311 ("umull2", VectorSize::Size32x4, VectorSize::Size16x8)
2312 }
2313 (VecRRRLongOp::Umull32, false) => {
2314 ("umull", VectorSize::Size64x2, VectorSize::Size32x2)
2315 }
2316 (VecRRRLongOp::Umull32, true) => {
2317 ("umull2", VectorSize::Size64x2, VectorSize::Size32x4)
2318 }
2319 };
2320 let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2321 let rn = pretty_print_vreg_vector(rn, src_size);
2322 let rm = pretty_print_vreg_vector(rm, src_size);
2323 format!("{op} {rd}, {rn}, {rm}")
2324 }
2325 &Inst::VecRRRLongMod {
2326 rd,
2327 ri,
2328 rn,
2329 rm,
2330 alu_op,
2331 high_half,
2332 } => {
2333 let (op, dest_size, src_size) = match (alu_op, high_half) {
2334 (VecRRRLongModOp::Umlal8, false) => {
2335 ("umlal", VectorSize::Size16x8, VectorSize::Size8x8)
2336 }
2337 (VecRRRLongModOp::Umlal8, true) => {
2338 ("umlal2", VectorSize::Size16x8, VectorSize::Size8x16)
2339 }
2340 (VecRRRLongModOp::Umlal16, false) => {
2341 ("umlal", VectorSize::Size32x4, VectorSize::Size16x4)
2342 }
2343 (VecRRRLongModOp::Umlal16, true) => {
2344 ("umlal2", VectorSize::Size32x4, VectorSize::Size16x8)
2345 }
2346 (VecRRRLongModOp::Umlal32, false) => {
2347 ("umlal", VectorSize::Size64x2, VectorSize::Size32x2)
2348 }
2349 (VecRRRLongModOp::Umlal32, true) => {
2350 ("umlal2", VectorSize::Size64x2, VectorSize::Size32x4)
2351 }
2352 };
2353 let rd = pretty_print_vreg_vector(rd.to_reg(), dest_size);
2354 let ri = pretty_print_vreg_vector(ri, dest_size);
2355 let rn = pretty_print_vreg_vector(rn, src_size);
2356 let rm = pretty_print_vreg_vector(rm, src_size);
2357 format!("{op} {rd}, {ri}, {rn}, {rm}")
2358 }
2359 &Inst::VecMisc { op, rd, rn, size } => {
2360 let (op, size, suffix) = match op {
2361 VecMisc2::Not => (
2362 "mvn",
2363 if size.is_128bits() {
2364 VectorSize::Size8x16
2365 } else {
2366 VectorSize::Size8x8
2367 },
2368 "",
2369 ),
2370 VecMisc2::Neg => ("neg", size, ""),
2371 VecMisc2::Abs => ("abs", size, ""),
2372 VecMisc2::Fabs => ("fabs", size, ""),
2373 VecMisc2::Fneg => ("fneg", size, ""),
2374 VecMisc2::Fsqrt => ("fsqrt", size, ""),
2375 VecMisc2::Rev16 => ("rev16", size, ""),
2376 VecMisc2::Rev32 => ("rev32", size, ""),
2377 VecMisc2::Rev64 => ("rev64", size, ""),
2378 VecMisc2::Fcvtzs => ("fcvtzs", size, ""),
2379 VecMisc2::Fcvtzu => ("fcvtzu", size, ""),
2380 VecMisc2::Scvtf => ("scvtf", size, ""),
2381 VecMisc2::Ucvtf => ("ucvtf", size, ""),
2382 VecMisc2::Frintn => ("frintn", size, ""),
2383 VecMisc2::Frintz => ("frintz", size, ""),
2384 VecMisc2::Frintm => ("frintm", size, ""),
2385 VecMisc2::Frintp => ("frintp", size, ""),
2386 VecMisc2::Cnt => ("cnt", size, ""),
2387 VecMisc2::Cmeq0 => ("cmeq", size, ", #0"),
2388 VecMisc2::Cmge0 => ("cmge", size, ", #0"),
2389 VecMisc2::Cmgt0 => ("cmgt", size, ", #0"),
2390 VecMisc2::Cmle0 => ("cmle", size, ", #0"),
2391 VecMisc2::Cmlt0 => ("cmlt", size, ", #0"),
2392 VecMisc2::Fcmeq0 => ("fcmeq", size, ", #0.0"),
2393 VecMisc2::Fcmge0 => ("fcmge", size, ", #0.0"),
2394 VecMisc2::Fcmgt0 => ("fcmgt", size, ", #0.0"),
2395 VecMisc2::Fcmle0 => ("fcmle", size, ", #0.0"),
2396 VecMisc2::Fcmlt0 => ("fcmlt", size, ", #0.0"),
2397 };
2398 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2399 let rn = pretty_print_vreg_vector(rn, size);
2400 format!("{op} {rd}, {rn}{suffix}")
2401 }
2402 &Inst::VecLanes { op, rd, rn, size } => {
2403 let op = match op {
2404 VecLanesOp::Uminv => "uminv",
2405 VecLanesOp::Addv => "addv",
2406 };
2407 let rd = pretty_print_vreg_scalar(rd.to_reg(), size.lane_size());
2408 let rn = pretty_print_vreg_vector(rn, size);
2409 format!("{op} {rd}, {rn}")
2410 }
2411 &Inst::VecShiftImm {
2412 op,
2413 rd,
2414 rn,
2415 size,
2416 imm,
2417 } => {
2418 let op = match op {
2419 VecShiftImmOp::Shl => "shl",
2420 VecShiftImmOp::Ushr => "ushr",
2421 VecShiftImmOp::Sshr => "sshr",
2422 };
2423 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2424 let rn = pretty_print_vreg_vector(rn, size);
2425 format!("{op} {rd}, {rn}, #{imm}")
2426 }
2427 &Inst::VecShiftImmMod {
2428 op,
2429 rd,
2430 ri,
2431 rn,
2432 size,
2433 imm,
2434 } => {
2435 let op = match op {
2436 VecShiftImmModOp::Sli => "sli",
2437 };
2438 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2439 let ri = pretty_print_vreg_vector(ri, size);
2440 let rn = pretty_print_vreg_vector(rn, size);
2441 format!("{op} {rd}, {ri}, {rn}, #{imm}")
2442 }
2443 &Inst::VecExtract { rd, rn, rm, imm4 } => {
2444 let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2445 let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2446 let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2447 format!("ext {rd}, {rn}, {rm}, #{imm4}")
2448 }
2449 &Inst::VecTbl { rd, rn, rm } => {
2450 let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2451 let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2452 let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2453 format!("tbl {rd}, {{ {rn} }}, {rm}")
2454 }
2455 &Inst::VecTblExt { rd, ri, rn, rm } => {
2456 let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2457 let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2458 let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2459 let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2460 format!("tbx {rd}, {ri}, {{ {rn} }}, {rm}")
2461 }
2462 &Inst::VecTbl2 { rd, rn, rn2, rm } => {
2463 let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2464 let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2465 let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2466 let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2467 format!("tbl {rd}, {{ {rn}, {rn2} }}, {rm}")
2468 }
2469 &Inst::VecTbl2Ext {
2470 rd,
2471 ri,
2472 rn,
2473 rn2,
2474 rm,
2475 } => {
2476 let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2477 let rn2 = pretty_print_vreg_vector(rn2, VectorSize::Size8x16);
2478 let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2479 let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2480 let ri = pretty_print_vreg_vector(ri, VectorSize::Size8x16);
2481 format!("tbx {rd}, {ri}, {{ {rn}, {rn2} }}, {rm}")
2482 }
2483 &Inst::VecLoadReplicate { rd, rn, size, .. } => {
2484 let rd = pretty_print_vreg_vector(rd.to_reg(), size);
2485 let rn = pretty_print_reg(rn);
2486
2487 format!("ld1r {{ {rd} }}, [{rn}]")
2488 }
2489 &Inst::VecCSel { rd, rn, rm, cond } => {
2490 let rd = pretty_print_vreg_vector(rd.to_reg(), VectorSize::Size8x16);
2491 let rn = pretty_print_vreg_vector(rn, VectorSize::Size8x16);
2492 let rm = pretty_print_vreg_vector(rm, VectorSize::Size8x16);
2493 let cond = cond.pretty_print(0);
2494 format!("vcsel {rd}, {rn}, {rm}, {cond} (if-then-else diamond)")
2495 }
2496 &Inst::MovToNZCV { rn } => {
2497 let rn = pretty_print_reg(rn);
2498 format!("msr nzcv, {rn}")
2499 }
2500 &Inst::MovFromNZCV { rd } => {
2501 let rd = pretty_print_reg(rd.to_reg());
2502 format!("mrs {rd}, nzcv")
2503 }
2504 &Inst::Extend {
2505 rd,
2506 rn,
2507 signed: false,
2508 from_bits: 1,
2509 ..
2510 } => {
2511 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size32);
2512 let rn = pretty_print_ireg(rn, OperandSize::Size32);
2513 format!("and {rd}, {rn}, #1")
2514 }
2515 &Inst::Extend {
2516 rd,
2517 rn,
2518 signed: false,
2519 from_bits: 32,
2520 to_bits: 64,
2521 } => {
2522 let rd = pretty_print_ireg(rd.to_reg(), OperandSize::Size32);
2526 let rn = pretty_print_ireg(rn, OperandSize::Size32);
2527 format!("mov {rd}, {rn}")
2528 }
2529 &Inst::Extend {
2530 rd,
2531 rn,
2532 signed,
2533 from_bits,
2534 to_bits,
2535 } => {
2536 assert!(from_bits <= to_bits);
2537 let op = match (signed, from_bits) {
2538 (false, 8) => "uxtb",
2539 (true, 8) => "sxtb",
2540 (false, 16) => "uxth",
2541 (true, 16) => "sxth",
2542 (true, 32) => "sxtw",
2543 (true, _) => "sbfx",
2544 (false, _) => "ubfx",
2545 };
2546 if op == "sbfx" || op == "ubfx" {
2547 let dest_size = OperandSize::from_bits(to_bits);
2548 let rd = pretty_print_ireg(rd.to_reg(), dest_size);
2549 let rn = pretty_print_ireg(rn, dest_size);
2550 format!("{op} {rd}, {rn}, #0, #{from_bits}")
2551 } else {
2552 let dest_size = if signed {
2553 OperandSize::from_bits(to_bits)
2554 } else {
2555 OperandSize::Size32
2556 };
2557 let rd = pretty_print_ireg(rd.to_reg(), dest_size);
2558 let rn = pretty_print_ireg(rn, OperandSize::from_bits(from_bits));
2559 format!("{op} {rd}, {rn}")
2560 }
2561 }
2562 &Inst::Call { ref info } => {
2563 let try_call = info
2564 .try_call_info
2565 .as_ref()
2566 .map(|tci| pretty_print_try_call(tci))
2567 .unwrap_or_default();
2568 format!("bl 0{try_call}")
2569 }
2570 &Inst::CallInd { ref info } => {
2571 let rn = pretty_print_reg(info.dest);
2572 let try_call = info
2573 .try_call_info
2574 .as_ref()
2575 .map(|tci| pretty_print_try_call(tci))
2576 .unwrap_or_default();
2577 format!("blr {rn}{try_call}")
2578 }
2579 &Inst::ReturnCall { ref info } => {
2580 let mut s = format!(
2581 "return_call {:?} new_stack_arg_size:{}",
2582 info.dest, info.new_stack_arg_size
2583 );
2584 for ret in &info.uses {
2585 let preg = pretty_print_reg(ret.preg);
2586 let vreg = pretty_print_reg(ret.vreg);
2587 write!(&mut s, " {vreg}={preg}").unwrap();
2588 }
2589 s
2590 }
2591 &Inst::ReturnCallInd { ref info } => {
2592 let callee = pretty_print_reg(info.dest);
2593 let mut s = format!(
2594 "return_call_ind {callee} new_stack_arg_size:{}",
2595 info.new_stack_arg_size
2596 );
2597 for ret in &info.uses {
2598 let preg = pretty_print_reg(ret.preg);
2599 let vreg = pretty_print_reg(ret.vreg);
2600 write!(&mut s, " {vreg}={preg}").unwrap();
2601 }
2602 s
2603 }
2604 &Inst::Args { ref args } => {
2605 let mut s = "args".to_string();
2606 for arg in args {
2607 let preg = pretty_print_reg(arg.preg);
2608 let def = pretty_print_reg(arg.vreg.to_reg());
2609 write!(&mut s, " {def}={preg}").unwrap();
2610 }
2611 s
2612 }
2613 &Inst::Rets { ref rets } => {
2614 let mut s = "rets".to_string();
2615 for ret in rets {
2616 let preg = pretty_print_reg(ret.preg);
2617 let vreg = pretty_print_reg(ret.vreg);
2618 write!(&mut s, " {vreg}={preg}").unwrap();
2619 }
2620 s
2621 }
2622 &Inst::Ret {} => "ret".to_string(),
2623 &Inst::AuthenticatedRet { key, is_hint } => {
2624 let key = match key {
2625 APIKey::AZ => "az",
2626 APIKey::BZ => "bz",
2627 APIKey::ASP => "asp",
2628 APIKey::BSP => "bsp",
2629 };
2630 match is_hint {
2631 false => format!("reta{key}"),
2632 true => format!("auti{key} ; ret"),
2633 }
2634 }
2635 &Inst::Jump { ref dest } => {
2636 let dest = dest.pretty_print(0);
2637 format!("b {dest}")
2638 }
2639 &Inst::CondBr {
2640 ref taken,
2641 ref not_taken,
2642 ref kind,
2643 } => {
2644 let taken = taken.pretty_print(0);
2645 let not_taken = not_taken.pretty_print(0);
2646 match kind {
2647 &CondBrKind::Zero(reg, size) => {
2648 let reg = pretty_print_reg_sized(reg, size);
2649 format!("cbz {reg}, {taken} ; b {not_taken}")
2650 }
2651 &CondBrKind::NotZero(reg, size) => {
2652 let reg = pretty_print_reg_sized(reg, size);
2653 format!("cbnz {reg}, {taken} ; b {not_taken}")
2654 }
2655 &CondBrKind::Cond(c) => {
2656 let c = c.pretty_print(0);
2657 format!("b.{c} {taken} ; b {not_taken}")
2658 }
2659 }
2660 }
2661 &Inst::TestBitAndBranch {
2662 kind,
2663 ref taken,
2664 ref not_taken,
2665 rn,
2666 bit,
2667 } => {
2668 let cond = match kind {
2669 TestBitAndBranchKind::Z => "z",
2670 TestBitAndBranchKind::NZ => "nz",
2671 };
2672 let taken = taken.pretty_print(0);
2673 let not_taken = not_taken.pretty_print(0);
2674 let rn = pretty_print_reg(rn);
2675 format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
2676 }
2677 &Inst::IndirectBr { rn, .. } => {
2678 let rn = pretty_print_reg(rn);
2679 format!("br {rn}")
2680 }
2681 &Inst::Brk => "brk #0xf000".to_string(),
2682 &Inst::Udf { .. } => "udf #0xc11f".to_string(),
2683 &Inst::TrapIf {
2684 ref kind,
2685 trap_code,
2686 } => match kind {
2687 &CondBrKind::Zero(reg, size) => {
2688 let reg = pretty_print_reg_sized(reg, size);
2689 format!("cbz {reg}, #trap={trap_code}")
2690 }
2691 &CondBrKind::NotZero(reg, size) => {
2692 let reg = pretty_print_reg_sized(reg, size);
2693 format!("cbnz {reg}, #trap={trap_code}")
2694 }
2695 &CondBrKind::Cond(c) => {
2696 let c = c.pretty_print(0);
2697 format!("b.{c} #trap={trap_code}")
2698 }
2699 },
2700 &Inst::Adr { rd, off } => {
2701 let rd = pretty_print_reg(rd.to_reg());
2702 format!("adr {rd}, pc+{off}")
2703 }
2704 &Inst::Adrp { rd, off } => {
2705 let rd = pretty_print_reg(rd.to_reg());
2706 let byte_offset = off * 4096;
2708 format!("adrp {rd}, pc+{byte_offset}")
2709 }
2710 &Inst::Word4 { data } => format!("data.i32 {data}"),
2711 &Inst::Word8 { data } => format!("data.i64 {data}"),
2712 &Inst::JTSequence {
2713 default,
2714 ref targets,
2715 ridx,
2716 rtmp1,
2717 rtmp2,
2718 ..
2719 } => {
2720 let ridx = pretty_print_reg(ridx);
2721 let rtmp1 = pretty_print_reg(rtmp1.to_reg());
2722 let rtmp2 = pretty_print_reg(rtmp2.to_reg());
2723 let default_target = BranchTarget::Label(default).pretty_print(0);
2724 format!(
2725 concat!(
2726 "b.hs {} ; ",
2727 "csel {}, xzr, {}, hs ; ",
2728 "csdb ; ",
2729 "adr {}, pc+16 ; ",
2730 "ldrsw {}, [{}, {}, uxtw #2] ; ",
2731 "add {}, {}, {} ; ",
2732 "br {} ; ",
2733 "jt_entries {:?}"
2734 ),
2735 default_target,
2736 rtmp2,
2737 ridx,
2738 rtmp1,
2739 rtmp2,
2740 rtmp1,
2741 rtmp2,
2742 rtmp1,
2743 rtmp1,
2744 rtmp2,
2745 rtmp1,
2746 targets
2747 )
2748 }
2749 &Inst::LoadExtName {
2750 rd,
2751 ref name,
2752 offset,
2753 } => {
2754 let rd = pretty_print_reg(rd.to_reg());
2755 format!("load_ext_name {rd}, {name:?}+{offset}")
2756 }
2757 &Inst::LoadAddr { rd, ref mem } => {
2758 let mem = mem.clone();
2763 let (mem_insts, mem) = mem_finalize(None, &mem, I8, state);
2764 let mut ret = String::new();
2765 for inst in mem_insts.into_iter() {
2766 ret.push_str(&inst.print_with_state(&mut EmitState::default()));
2767 }
2768 let (reg, index_reg, offset) = match mem {
2769 AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0),
2770 AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()),
2771 AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32),
2772 _ => panic!("Unsupported case for LoadAddr: {mem:?}"),
2773 };
2774 let abs_offset = if offset < 0 {
2775 -offset as u64
2776 } else {
2777 offset as u64
2778 };
2779 let alu_op = if offset < 0 { ALUOp::Sub } else { ALUOp::Add };
2780
2781 if let Some((idx, extendop)) = index_reg {
2782 let add = Inst::AluRRRExtend {
2783 alu_op: ALUOp::Add,
2784 size: OperandSize::Size64,
2785 rd,
2786 rn: reg,
2787 rm: idx,
2788 extendop,
2789 };
2790
2791 ret.push_str(&add.print_with_state(&mut EmitState::default()));
2792 } else if offset == 0 {
2793 let mov = Inst::gen_move(rd, reg, I64);
2794 ret.push_str(&mov.print_with_state(&mut EmitState::default()));
2795 } else if let Some(imm12) = Imm12::maybe_from_u64(abs_offset) {
2796 let add = Inst::AluRRImm12 {
2797 alu_op,
2798 size: OperandSize::Size64,
2799 rd,
2800 rn: reg,
2801 imm12,
2802 };
2803 ret.push_str(&add.print_with_state(&mut EmitState::default()));
2804 } else {
2805 let tmp = writable_spilltmp_reg();
2806 for inst in Inst::load_constant(tmp, abs_offset).into_iter() {
2807 ret.push_str(&inst.print_with_state(&mut EmitState::default()));
2808 }
2809 let add = Inst::AluRRR {
2810 alu_op,
2811 size: OperandSize::Size64,
2812 rd,
2813 rn: reg,
2814 rm: tmp.to_reg(),
2815 };
2816 ret.push_str(&add.print_with_state(&mut EmitState::default()));
2817 }
2818 ret
2819 }
2820 &Inst::Paci { key } => {
2821 let key = match key {
2822 APIKey::AZ => "az",
2823 APIKey::BZ => "bz",
2824 APIKey::ASP => "asp",
2825 APIKey::BSP => "bsp",
2826 };
2827
2828 "paci".to_string() + key
2829 }
2830 &Inst::Xpaclri => "xpaclri".to_string(),
2831 &Inst::Bti { targets } => {
2832 let targets = match targets {
2833 BranchTargetType::None => "",
2834 BranchTargetType::C => " c",
2835 BranchTargetType::J => " j",
2836 BranchTargetType::JC => " jc",
2837 };
2838
2839 "bti".to_string() + targets
2840 }
2841 &Inst::EmitIsland { needed_space } => format!("emit_island {needed_space}"),
2842
2843 &Inst::ElfTlsGetAddr {
2844 ref symbol,
2845 rd,
2846 tmp,
2847 } => {
2848 let rd = pretty_print_reg(rd.to_reg());
2849 let tmp = pretty_print_reg(tmp.to_reg());
2850 format!("elf_tls_get_addr {}, {}, {}", rd, tmp, symbol.display(None))
2851 }
2852 &Inst::MachOTlsGetAddr { ref symbol, rd } => {
2853 let rd = pretty_print_reg(rd.to_reg());
2854 format!("macho_tls_get_addr {}, {}", rd, symbol.display(None))
2855 }
2856 &Inst::Unwind { ref inst } => {
2857 format!("unwind {inst:?}")
2858 }
2859 &Inst::DummyUse { reg } => {
2860 let reg = pretty_print_reg(reg);
2861 format!("dummy_use {reg}")
2862 }
2863 &Inst::StackProbeLoop { start, end, step } => {
2864 let start = pretty_print_reg(start.to_reg());
2865 let end = pretty_print_reg(end);
2866 let step = step.pretty_print(0);
2867 format!("stack_probe_loop {start}, {end}, {step}")
2868 }
2869 }
2870 }
2871}
2872
2873#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2878pub enum LabelUse {
2879 Branch14,
2882 Branch19,
2885 Branch26,
2888 Ldr19,
2891 Adr21,
2894 PCRel32,
2897}
2898
2899impl MachInstLabelUse for LabelUse {
2900 const ALIGN: CodeOffset = 4;
2902
2903 fn max_pos_range(self) -> CodeOffset {
2905 match self {
2906 LabelUse::Branch14 => (1 << 15) - 1,
2910 LabelUse::Branch19 => (1 << 20) - 1,
2911 LabelUse::Branch26 => (1 << 27) - 1,
2912 LabelUse::Ldr19 => (1 << 20) - 1,
2913 LabelUse::Adr21 => (1 << 20) - 1,
2916 LabelUse::PCRel32 => 0x7fffffff,
2917 }
2918 }
2919
2920 fn max_neg_range(self) -> CodeOffset {
2922 self.max_pos_range() + 1
2925 }
2926
2927 fn patch_size(self) -> CodeOffset {
2929 4
2931 }
2932
2933 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2935 let pc_rel = (label_offset as i64) - (use_offset as i64);
2936 debug_assert!(pc_rel <= self.max_pos_range() as i64);
2937 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2938 let pc_rel = pc_rel as u32;
2939 let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2940 let mask = match self {
2941 LabelUse::Branch14 => 0x0007ffe0, LabelUse::Branch19 => 0x00ffffe0, LabelUse::Branch26 => 0x03ffffff, LabelUse::Ldr19 => 0x00ffffe0, LabelUse::Adr21 => 0x60ffffe0, LabelUse::PCRel32 => 0xffffffff,
2947 };
2948 let pc_rel_shifted = match self {
2949 LabelUse::Adr21 | LabelUse::PCRel32 => pc_rel,
2950 _ => {
2951 debug_assert!(pc_rel & 3 == 0);
2952 pc_rel >> 2
2953 }
2954 };
2955 let pc_rel_inserted = match self {
2956 LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
2957 LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
2958 LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
2959 LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
2960 LabelUse::PCRel32 => pc_rel_shifted,
2961 };
2962 let is_add = match self {
2963 LabelUse::PCRel32 => true,
2964 _ => false,
2965 };
2966 let insn_word = if is_add {
2967 insn_word.wrapping_add(pc_rel_inserted)
2968 } else {
2969 (insn_word & !mask) | pc_rel_inserted
2970 };
2971 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
2972 }
2973
2974 fn supports_veneer(self) -> bool {
2976 match self {
2977 LabelUse::Branch14 | LabelUse::Branch19 => true, LabelUse::Branch26 => true, _ => false,
2980 }
2981 }
2982
2983 fn veneer_size(self) -> CodeOffset {
2985 match self {
2986 LabelUse::Branch14 | LabelUse::Branch19 => 4,
2987 LabelUse::Branch26 => 20,
2988 _ => unreachable!(),
2989 }
2990 }
2991
2992 fn worst_case_veneer_size() -> CodeOffset {
2993 20
2994 }
2995
2996 fn generate_veneer(
2999 self,
3000 buffer: &mut [u8],
3001 veneer_offset: CodeOffset,
3002 ) -> (CodeOffset, LabelUse) {
3003 match self {
3004 LabelUse::Branch14 | LabelUse::Branch19 => {
3005 let insn_word = 0b000101 << 26;
3008 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
3009 (veneer_offset, LabelUse::Branch26)
3010 }
3011
3012 LabelUse::Branch26 => {
3023 let tmp1 = regs::spilltmp_reg();
3024 let tmp1_w = regs::writable_spilltmp_reg();
3025 let tmp2 = regs::tmp2_reg();
3026 let tmp2_w = regs::writable_tmp2_reg();
3027 let ldr = emit::enc_ldst_imm19(0b1001_1000, 16 / 4, tmp1);
3029 let adr = emit::enc_adr(12, tmp2_w);
3031 let add = emit::enc_arith_rrr(0b10001011_000, 0, tmp1_w, tmp1, tmp2);
3033 let br = emit::enc_br(tmp1);
3035 buffer[0..4].clone_from_slice(&u32::to_le_bytes(ldr));
3036 buffer[4..8].clone_from_slice(&u32::to_le_bytes(adr));
3037 buffer[8..12].clone_from_slice(&u32::to_le_bytes(add));
3038 buffer[12..16].clone_from_slice(&u32::to_le_bytes(br));
3039 (veneer_offset + 16, LabelUse::PCRel32)
3042 }
3043
3044 _ => panic!("Unsupported label-reference type for veneer generation!"),
3045 }
3046 }
3047
3048 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {
3049 match (reloc, addend) {
3050 (Reloc::Arm64Call, 0) => Some(LabelUse::Branch26),
3051 _ => None,
3052 }
3053 }
3054}
3055
3056#[cfg(test)]
3057mod tests {
3058 use super::*;
3059
3060 #[test]
3061 fn inst_size_test() {
3062 let expected = if cfg!(target_pointer_width = "32") && !cfg!(target_arch = "arm") {
3065 28
3066 } else {
3067 32
3068 };
3069 assert_eq!(expected, std::mem::size_of::<Inst>());
3070 }
3071}