1pub use emit_state::EmitState;
4
5use crate::binemit::{Addend, CodeOffset, Reloc};
6use crate::ir::{types, ExternalName, LibCall, TrapCode, Type};
7use crate::isa::x64::abi::X64ABIMachineSpec;
8use crate::isa::x64::inst::regs::{pretty_print_reg, show_ireg_sized};
9use crate::isa::x64::settings as x64_settings;
10use crate::isa::{CallConv, FunctionAlignment};
11use crate::{machinst::*, trace};
12use crate::{settings, CodegenError, CodegenResult};
13use alloc::boxed::Box;
14use alloc::vec::Vec;
15use core::slice;
16use cranelift_assembler_x64 as asm;
17use smallvec::{smallvec, SmallVec};
18use std::fmt::{self, Write};
19use std::string::{String, ToString};
20
21pub mod args;
22mod emit;
23mod emit_state;
24#[cfg(test)]
25mod emit_tests;
26pub mod external;
27pub mod regs;
28mod stack_switch;
29pub mod unwind;
30
31use args::*;
32
33pub use super::lower::isle::generated_code::AtomicRmwSeqOp;
38pub use super::lower::isle::generated_code::MInst as Inst;
39
40#[derive(Clone, Debug)]
42pub struct ReturnCallInfo<T> {
43 pub dest: T,
45
46 pub new_stack_arg_size: u32,
49
50 pub uses: CallArgList,
52
53 pub tmp: WritableGpr,
55}
56
57#[test]
58#[cfg(target_pointer_width = "64")]
59fn inst_size_test() {
60 assert_eq!(48, std::mem::size_of::<Inst>());
63}
64
65pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
66 let xs = x as i64;
67 xs == ((xs << 32) >> 32)
68}
69
70impl Inst {
71 fn available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]> {
76 match self {
77 Inst::AtomicRmwSeq { .. }
80 | Inst::Bswap { .. }
81 | Inst::CallKnown { .. }
82 | Inst::CallUnknown { .. }
83 | Inst::ReturnCallKnown { .. }
84 | Inst::ReturnCallUnknown { .. }
85 | Inst::CheckedSRemSeq { .. }
86 | Inst::CheckedSRemSeq8 { .. }
87 | Inst::Cmove { .. }
88 | Inst::CmpRmiR { .. }
89 | Inst::CvtFloatToSintSeq { .. }
90 | Inst::CvtFloatToUintSeq { .. }
91 | Inst::CvtUint64ToFloatSeq { .. }
92 | Inst::Div { .. }
93 | Inst::Div8 { .. }
94 | Inst::Fence { .. }
95 | Inst::Hlt
96 | Inst::Imm { .. }
97 | Inst::JmpCond { .. }
98 | Inst::JmpCondOr { .. }
99 | Inst::WinchJmpIf { .. }
100 | Inst::JmpKnown { .. }
101 | Inst::JmpTableSeq { .. }
102 | Inst::JmpUnknown { .. }
103 | Inst::LoadEffectiveAddress { .. }
104 | Inst::LoadExtName { .. }
105 | Inst::LockCmpxchg { .. }
106 | Inst::LockXadd { .. }
107 | Inst::Xchg { .. }
108 | Inst::Mov64MR { .. }
109 | Inst::MovImmM { .. }
110 | Inst::MovRM { .. }
111 | Inst::MovRR { .. }
112 | Inst::MovFromPReg { .. }
113 | Inst::MovToPReg { .. }
114 | Inst::MovsxRmR { .. }
115 | Inst::MovzxRmR { .. }
116 | Inst::Mul { .. }
117 | Inst::Mul8 { .. }
118 | Inst::IMul { .. }
119 | Inst::IMulImm { .. }
120 | Inst::Nop { .. }
121 | Inst::Pop64 { .. }
122 | Inst::Push64 { .. }
123 | Inst::StackProbeLoop { .. }
124 | Inst::Args { .. }
125 | Inst::Rets { .. }
126 | Inst::Ret { .. }
127 | Inst::Setcc { .. }
128 | Inst::ShiftR { .. }
129 | Inst::SignExtendData { .. }
130 | Inst::StackSwitchBasic { .. }
131 | Inst::TrapIf { .. }
132 | Inst::TrapIfAnd { .. }
133 | Inst::TrapIfOr { .. }
134 | Inst::Ud2 { .. }
135 | Inst::XmmCmove { .. }
136 | Inst::XmmCmpRmR { .. }
137 | Inst::XmmMinMaxSeq { .. }
138 | Inst::XmmUninitializedValue { .. }
139 | Inst::GprUninitializedValue { .. }
140 | Inst::ElfTlsGetAddr { .. }
141 | Inst::MachOTlsGetAddr { .. }
142 | Inst::CoffTlsGetAddr { .. }
143 | Inst::Unwind { .. }
144 | Inst::DummyUse { .. } => smallvec![],
145
146 Inst::LockCmpxchg16b { .. }
147 | Inst::Atomic128RmwSeq { .. }
148 | Inst::Atomic128XchgSeq { .. } => smallvec![InstructionSet::CMPXCHG16b],
149
150 Inst::AluRmRVex { op, .. } => op.available_from(),
151 Inst::UnaryRmR { op, .. } => op.available_from(),
152 Inst::UnaryRmRVex { op, .. } => op.available_from(),
153 Inst::UnaryRmRImmVex { op, .. } => op.available_from(),
154
155 Inst::GprToXmm { op, .. }
157 | Inst::XmmMovRM { op, .. }
158 | Inst::XmmMovRMImm { op, .. }
159 | Inst::XmmRmiReg { opcode: op, .. }
160 | Inst::XmmRmR { op, .. }
161 | Inst::XmmRmRUnaligned { op, .. }
162 | Inst::XmmRmRBlend { op, .. }
163 | Inst::XmmRmRImm { op, .. }
164 | Inst::XmmToGpr { op, .. }
165 | Inst::XmmToGprImm { op, .. }
166 | Inst::XmmUnaryRmRImm { op, .. }
167 | Inst::XmmUnaryRmRUnaligned { op, .. }
168 | Inst::XmmUnaryRmR { op, .. }
169 | Inst::CvtIntToFloat { op, .. } => smallvec![op.available_from()],
170
171 Inst::XmmUnaryRmREvex { op, .. }
172 | Inst::XmmRmREvex { op, .. }
173 | Inst::XmmRmREvex3 { op, .. }
174 | Inst::XmmUnaryRmRImmEvex { op, .. } => op.available_from(),
175
176 Inst::XmmRmiRVex { op, .. }
177 | Inst::XmmRmRVex3 { op, .. }
178 | Inst::XmmRmRImmVex { op, .. }
179 | Inst::XmmRmRBlendVex { op, .. }
180 | Inst::XmmVexPinsr { op, .. }
181 | Inst::XmmUnaryRmRVex { op, .. }
182 | Inst::XmmUnaryRmRImmVex { op, .. }
183 | Inst::XmmMovRMVex { op, .. }
184 | Inst::XmmMovRMImmVex { op, .. }
185 | Inst::XmmToGprImmVex { op, .. }
186 | Inst::XmmToGprVex { op, .. }
187 | Inst::GprToXmmVex { op, .. }
188 | Inst::CvtIntToFloatVex { op, .. }
189 | Inst::XmmCmpRmRVex { op, .. } => op.available_from(),
190
191 Inst::MulX { .. } => smallvec![InstructionSet::BMI2],
192
193 Inst::External { inst } => {
194 use cranelift_assembler_x64::Feature::*;
195 let mut features = smallvec![];
196 for f in inst.features() {
197 match f {
198 _64b | compat => {}
199 sse => features.push(InstructionSet::SSE),
200 sse2 => features.push(InstructionSet::SSE2),
201 ssse3 => features.push(InstructionSet::SSSE3),
202 }
203 }
204 features
205 }
206 }
207 }
208}
209
210impl Inst {
213 pub(crate) fn nop(len: u8) -> Self {
214 debug_assert!(len <= 15);
215 Self::Nop { len }
216 }
217
218 pub(crate) fn addq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
219 let inst = if let Ok(simm8) = i8::try_from(simm32) {
220 asm::inst::addq_mi_sxb::new(dst, simm8).into()
221 } else {
222 asm::inst::addq_mi_sxl::new(dst, simm32).into()
223 };
224 Inst::External { inst }
225 }
226
227 pub(crate) fn subq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
228 let inst = if let Ok(simm8) = i8::try_from(simm32) {
229 asm::inst::subq_mi_sxb::new(dst, simm8).into()
230 } else {
231 asm::inst::subq_mi_sxl::new(dst, simm32).into()
232 };
233 Inst::External { inst }
234 }
235
236 #[allow(dead_code)]
237 pub(crate) fn unary_rm_r(
238 size: OperandSize,
239 op: UnaryRmROpcode,
240 src: RegMem,
241 dst: Writable<Reg>,
242 ) -> Self {
243 src.assert_regclass_is(RegClass::Int);
244 debug_assert!(dst.to_reg().class() == RegClass::Int);
245 debug_assert!(size.is_one_of(&[
246 OperandSize::Size16,
247 OperandSize::Size32,
248 OperandSize::Size64
249 ]));
250 Self::UnaryRmR {
251 size,
252 op,
253 src: GprMem::unwrap_new(src),
254 dst: WritableGpr::from_writable_reg(dst).unwrap(),
255 }
256 }
257
258 pub(crate) fn div(
259 size: OperandSize,
260 sign: DivSignedness,
261 trap: TrapCode,
262 divisor: RegMem,
263 dividend_lo: Gpr,
264 dividend_hi: Gpr,
265 dst_quotient: WritableGpr,
266 dst_remainder: WritableGpr,
267 ) -> Inst {
268 divisor.assert_regclass_is(RegClass::Int);
269 Inst::Div {
270 size,
271 sign,
272 trap,
273 divisor: GprMem::unwrap_new(divisor),
274 dividend_lo,
275 dividend_hi,
276 dst_quotient,
277 dst_remainder,
278 }
279 }
280
281 pub(crate) fn div8(
282 sign: DivSignedness,
283 trap: TrapCode,
284 divisor: RegMem,
285 dividend: Gpr,
286 dst: WritableGpr,
287 ) -> Inst {
288 divisor.assert_regclass_is(RegClass::Int);
289 Inst::Div8 {
290 sign,
291 trap,
292 divisor: GprMem::unwrap_new(divisor),
293 dividend,
294 dst,
295 }
296 }
297
298 pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
299 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
300 debug_assert!(dst.to_reg().class() == RegClass::Int);
301 let dst_size = match dst_size {
304 OperandSize::Size64 if simm64 > u32::max_value() as u64 => OperandSize::Size64,
305 _ => OperandSize::Size32,
306 };
307 Inst::Imm {
308 dst_size,
309 simm64,
310 dst: WritableGpr::from_writable_reg(dst).unwrap(),
311 }
312 }
313
314 pub(crate) fn mov_r_r(size: OperandSize, src: Reg, dst: Writable<Reg>) -> Inst {
315 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
316 debug_assert!(src.class() == RegClass::Int);
317 debug_assert!(dst.to_reg().class() == RegClass::Int);
318 let src = Gpr::unwrap_new(src);
319 let dst = WritableGpr::from_writable_reg(dst).unwrap();
320 Inst::MovRR { size, src, dst }
321 }
322
323 pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
325 src.assert_regclass_is(RegClass::Float);
326 debug_assert!(dst.to_reg().class() == RegClass::Float);
327 Inst::XmmUnaryRmR {
328 op,
329 src: XmmMemAligned::unwrap_new(src),
330 dst: WritableXmm::from_writable_reg(dst).unwrap(),
331 }
332 }
333
334 pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
335 src.assert_regclass_is(RegClass::Float);
336 debug_assert!(dst.to_reg().class() == RegClass::Float);
337 Inst::XmmRmR {
338 op,
339 src1: Xmm::unwrap_new(dst.to_reg()),
340 src2: XmmMemAligned::unwrap_new(src),
341 dst: WritableXmm::from_writable_reg(dst).unwrap(),
342 }
343 }
344
345 #[cfg(test)]
346 pub(crate) fn xmm_rmr_vex3(op: AvxOpcode, src3: RegMem, src2: Reg, dst: Writable<Reg>) -> Self {
347 src3.assert_regclass_is(RegClass::Float);
348 debug_assert!(src2.class() == RegClass::Float);
349 debug_assert!(dst.to_reg().class() == RegClass::Float);
350 Inst::XmmRmRVex3 {
351 op,
352 src3: XmmMem::unwrap_new(src3),
353 src2: Xmm::unwrap_new(src2),
354 src1: Xmm::unwrap_new(dst.to_reg()),
355 dst: WritableXmm::from_writable_reg(dst).unwrap(),
356 }
357 }
358
359 pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
360 debug_assert!(src.class() == RegClass::Float);
361 Inst::XmmMovRM {
362 op,
363 src: Xmm::unwrap_new(src),
364 dst: dst.into(),
365 }
366 }
367
368 pub(crate) fn xmm_to_gpr(
369 op: SseOpcode,
370 src: Reg,
371 dst: Writable<Reg>,
372 dst_size: OperandSize,
373 ) -> Inst {
374 debug_assert!(src.class() == RegClass::Float);
375 debug_assert!(dst.to_reg().class() == RegClass::Int);
376 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
377 Inst::XmmToGpr {
378 op,
379 src: Xmm::unwrap_new(src),
380 dst: WritableGpr::from_writable_reg(dst).unwrap(),
381 dst_size,
382 }
383 }
384
385 pub(crate) fn gpr_to_xmm(
386 op: SseOpcode,
387 src: RegMem,
388 src_size: OperandSize,
389 dst: Writable<Reg>,
390 ) -> Inst {
391 src.assert_regclass_is(RegClass::Int);
392 debug_assert!(src_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
393 debug_assert!(dst.to_reg().class() == RegClass::Float);
394 Inst::GprToXmm {
395 op,
396 src: GprMem::unwrap_new(src),
397 dst: WritableXmm::from_writable_reg(dst).unwrap(),
398 src_size,
399 }
400 }
401
402 pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src1: Reg, src2: RegMem) -> Inst {
403 src2.assert_regclass_is(RegClass::Float);
404 debug_assert!(src1.class() == RegClass::Float);
405 let src2 = XmmMemAligned::unwrap_new(src2);
406 let src1 = Xmm::unwrap_new(src1);
407 Inst::XmmCmpRmR { op, src1, src2 }
408 }
409
410 #[allow(dead_code)]
411 pub(crate) fn xmm_min_max_seq(
412 size: OperandSize,
413 is_min: bool,
414 lhs: Reg,
415 rhs: Reg,
416 dst: Writable<Reg>,
417 ) -> Inst {
418 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
419 debug_assert_eq!(lhs.class(), RegClass::Float);
420 debug_assert_eq!(rhs.class(), RegClass::Float);
421 debug_assert_eq!(dst.to_reg().class(), RegClass::Float);
422 Inst::XmmMinMaxSeq {
423 size,
424 is_min,
425 lhs: Xmm::unwrap_new(lhs),
426 rhs: Xmm::unwrap_new(rhs),
427 dst: WritableXmm::from_writable_reg(dst).unwrap(),
428 }
429 }
430
431 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
432 src.assert_regclass_is(RegClass::Int);
433 debug_assert!(dst.to_reg().class() == RegClass::Int);
434 let src = GprMem::unwrap_new(src);
435 let dst = WritableGpr::from_writable_reg(dst).unwrap();
436 Inst::MovzxRmR { ext_mode, src, dst }
437 }
438
439 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
440 src.assert_regclass_is(RegClass::Int);
441 debug_assert!(dst.to_reg().class() == RegClass::Int);
442 let src = GprMem::unwrap_new(src);
443 let dst = WritableGpr::from_writable_reg(dst).unwrap();
444 Inst::MovsxRmR { ext_mode, src, dst }
445 }
446
447 pub(crate) fn mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
448 debug_assert!(dst.to_reg().class() == RegClass::Int);
449 Inst::Mov64MR {
450 src: src.into(),
451 dst: WritableGpr::from_writable_reg(dst).unwrap(),
452 }
453 }
454
455 pub(crate) fn mov_r_m(size: OperandSize, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
456 debug_assert!(src.class() == RegClass::Int);
457 Inst::MovRM {
458 size,
459 src: Gpr::unwrap_new(src),
460 dst: dst.into(),
461 }
462 }
463
464 pub(crate) fn lea(addr: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
465 debug_assert!(dst.to_reg().class() == RegClass::Int);
466 Inst::LoadEffectiveAddress {
467 addr: addr.into(),
468 dst: WritableGpr::from_writable_reg(dst).unwrap(),
469 size: OperandSize::Size64,
470 }
471 }
472
473 pub(crate) fn shift_r(
474 size: OperandSize,
475 kind: ShiftKind,
476 num_bits: Imm8Gpr,
477 src: Reg,
478 dst: Writable<Reg>,
479 ) -> Inst {
480 if let &Imm8Reg::Imm8 { imm: num_bits } = num_bits.as_imm8_reg() {
481 debug_assert!(num_bits < size.to_bits());
482 }
483 debug_assert!(dst.to_reg().class() == RegClass::Int);
484 Inst::ShiftR {
485 size,
486 kind,
487 src: Gpr::unwrap_new(src),
488 num_bits,
489 dst: WritableGpr::from_writable_reg(dst).unwrap(),
490 }
491 }
492
493 pub(crate) fn cmp_rmi_r(size: OperandSize, src1: Reg, src2: RegMemImm) -> Inst {
496 src2.assert_regclass_is(RegClass::Int);
497 debug_assert_eq!(src1.class(), RegClass::Int);
498 Inst::CmpRmiR {
499 size,
500 src1: Gpr::unwrap_new(src1),
501 src2: GprMemImm::unwrap_new(src2),
502 opcode: CmpOpcode::Cmp,
503 }
504 }
505
506 pub(crate) fn trap(trap_code: TrapCode) -> Inst {
507 Inst::Ud2 { trap_code }
508 }
509
510 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
511 Inst::TrapIf { cc, trap_code }
512 }
513
514 pub(crate) fn cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
515 debug_assert!(size.is_one_of(&[
516 OperandSize::Size16,
517 OperandSize::Size32,
518 OperandSize::Size64
519 ]));
520 debug_assert!(dst.to_reg().class() == RegClass::Int);
521 Inst::Cmove {
522 size,
523 cc,
524 consequent: GprMem::unwrap_new(src),
525 alternative: Gpr::unwrap_new(dst.to_reg()),
526 dst: WritableGpr::from_writable_reg(dst).unwrap(),
527 }
528 }
529
530 pub(crate) fn push64(src: RegMemImm) -> Inst {
531 src.assert_regclass_is(RegClass::Int);
532 let src = GprMemImm::unwrap_new(src);
533 Inst::Push64 { src }
534 }
535
536 pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
537 debug_assert!(dst.to_reg().class() == RegClass::Int);
538 let dst = WritableGpr::from_writable_reg(dst).unwrap();
539 Inst::Pop64 { dst }
540 }
541
542 pub(crate) fn call_known(info: Box<CallInfo<ExternalName>>) -> Inst {
543 Inst::CallKnown { info }
544 }
545
546 pub(crate) fn call_unknown(info: Box<CallInfo<RegMem>>) -> Inst {
547 info.dest.assert_regclass_is(RegClass::Int);
548 Inst::CallUnknown { info }
549 }
550
551 pub(crate) fn ret(stack_bytes_to_pop: u32) -> Inst {
552 Inst::Ret { stack_bytes_to_pop }
553 }
554
555 pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
556 Inst::JmpKnown { dst }
557 }
558
559 pub(crate) fn jmp_unknown(target: RegMem) -> Inst {
560 target.assert_regclass_is(RegClass::Int);
561 Inst::JmpUnknown { target }
562 }
563
564 pub(crate) fn load(
568 ty: Type,
569 from_addr: impl Into<SyntheticAmode>,
570 to_reg: Writable<Reg>,
571 ext_kind: ExtKind,
572 ) -> Inst {
573 let rc = to_reg.to_reg().class();
574 match rc {
575 RegClass::Int => {
576 let ext_mode = match ty.bytes() {
577 1 => Some(ExtMode::BQ),
578 2 => Some(ExtMode::WQ),
579 4 => Some(ExtMode::LQ),
580 8 => None,
581 _ => unreachable!("the type should never use a scalar load: {}", ty),
582 };
583 if let Some(ext_mode) = ext_mode {
584 match ext_kind {
586 ExtKind::SignExtend => {
587 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
588 }
589 ExtKind::ZeroExtend => {
590 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
591 }
592 ExtKind::None => {
593 panic!("expected an extension kind for extension mode: {ext_mode:?}")
594 }
595 }
596 } else {
597 Inst::mov64_m_r(from_addr, to_reg)
599 }
600 }
601 RegClass::Float => {
602 let opcode = match ty {
603 types::F16 | types::I8X2 => {
604 panic!("loading a f16 or i8x2 requires multiple instructions")
605 }
606 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => SseOpcode::Movss,
607 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => SseOpcode::Movsd,
608 types::F32X4 => SseOpcode::Movups,
609 types::F64X2 => SseOpcode::Movupd,
610 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => SseOpcode::Movdqu,
611 _ => unimplemented!("unable to load type: {}", ty),
612 };
613 Inst::xmm_unary_rm_r(opcode, RegMem::mem(from_addr), to_reg)
614 }
615 RegClass::Vector => unreachable!(),
616 }
617 }
618
619 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
621 let rc = from_reg.class();
622 match rc {
623 RegClass::Int => Inst::mov_r_m(OperandSize::from_ty(ty), from_reg, to_addr),
624 RegClass::Float => {
625 let opcode = match ty {
626 types::F16 | types::I8X2 => {
627 panic!("storing a f16 or i8x2 requires multiple instructions")
628 }
629 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => SseOpcode::Movss,
630 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => SseOpcode::Movsd,
631 types::F32X4 => SseOpcode::Movups,
632 types::F64X2 => SseOpcode::Movupd,
633 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => SseOpcode::Movdqu,
634 _ => unimplemented!("unable to store type: {}", ty),
635 };
636 Inst::xmm_mov_r_m(opcode, from_reg, to_addr)
637 }
638 RegClass::Vector => unreachable!(),
639 }
640 }
641}
642
643impl PrettyPrint for Inst {
647 fn pretty_print(&self, _size: u8) -> String {
648 fn ljustify(s: String) -> String {
649 let w = 7;
650 if s.len() >= w {
651 s
652 } else {
653 let need = usize::min(w, w - s.len());
654 s + &format!("{nil: <width$}", nil = "", width = need)
655 }
656 }
657
658 fn ljustify2(s1: String, s2: String) -> String {
659 ljustify(s1 + &s2)
660 }
661
662 fn suffix_lq(size: OperandSize) -> String {
663 match size {
664 OperandSize::Size32 => "l",
665 OperandSize::Size64 => "q",
666 _ => unreachable!(),
667 }
668 .to_string()
669 }
670
671 #[allow(dead_code)]
672 fn suffix_lqb(size: OperandSize) -> String {
673 match size {
674 OperandSize::Size32 => "l",
675 OperandSize::Size64 => "q",
676 _ => unreachable!(),
677 }
678 .to_string()
679 }
680
681 fn suffix_bwlq(size: OperandSize) -> String {
682 match size {
683 OperandSize::Size8 => "b".to_string(),
684 OperandSize::Size16 => "w".to_string(),
685 OperandSize::Size32 => "l".to_string(),
686 OperandSize::Size64 => "q".to_string(),
687 }
688 }
689
690 match self {
691 Inst::Nop { len } => format!("{} len={}", ljustify("nop".to_string()), len),
692
693 Inst::AluRmRVex {
694 size,
695 op,
696 src1,
697 src2,
698 dst,
699 } => {
700 let size_bytes = size.to_bytes();
701 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
702 let src1 = pretty_print_reg(src1.to_reg(), size_bytes);
703 let src2 = src2.pretty_print(size_bytes);
704 let op = ljustify2(op.to_string(), String::new());
705 format!("{op} {src2}, {src1}, {dst}")
706 }
707 Inst::UnaryRmR { src, dst, op, size } => {
708 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
709 let src = src.pretty_print(size.to_bytes());
710 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
711 format!("{op} {src}, {dst}")
712 }
713
714 Inst::UnaryRmRVex { src, dst, op, size } => {
715 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
716 let src = src.pretty_print(size.to_bytes());
717 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
718 format!("{op} {src}, {dst}")
719 }
720
721 Inst::UnaryRmRImmVex {
722 src,
723 dst,
724 op,
725 size,
726 imm,
727 } => {
728 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
729 let src = src.pretty_print(size.to_bytes());
730 format!(
731 "{} ${imm}, {src}, {dst}",
732 ljustify2(op.to_string(), suffix_bwlq(*size))
733 )
734 }
735
736 Inst::Div {
737 size,
738 sign,
739 trap,
740 divisor,
741 dividend_lo,
742 dividend_hi,
743 dst_quotient,
744 dst_remainder,
745 } => {
746 let divisor = divisor.pretty_print(size.to_bytes());
747 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
748 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
749 let dst_quotient =
750 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
751 let dst_remainder =
752 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
753 let op = ljustify(match sign {
754 DivSignedness::Signed => "idiv".to_string(),
755 DivSignedness::Unsigned => "div".to_string(),
756 });
757 format!(
758 "{op} {dividend_lo}, {dividend_hi}, {divisor}, {dst_quotient}, {dst_remainder} ; trap={trap}"
759 )
760 }
761
762 Inst::Div8 {
763 sign,
764 trap,
765 divisor,
766 dividend,
767 dst,
768 } => {
769 let divisor = divisor.pretty_print(1);
770 let dividend = pretty_print_reg(dividend.to_reg(), 1);
771 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
772 let op = ljustify(match sign {
773 DivSignedness::Signed => "idiv".to_string(),
774 DivSignedness::Unsigned => "div".to_string(),
775 });
776 format!("{op} {dividend}, {divisor}, {dst} ; trap={trap}")
777 }
778
779 Inst::Mul {
780 size,
781 signed,
782 src1,
783 src2,
784 dst_lo,
785 dst_hi,
786 } => {
787 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
788 let dst_lo = pretty_print_reg(dst_lo.to_reg().to_reg(), size.to_bytes());
789 let dst_hi = pretty_print_reg(dst_hi.to_reg().to_reg(), size.to_bytes());
790 let src2 = src2.pretty_print(size.to_bytes());
791 let suffix = suffix_bwlq(*size);
792 let op = ljustify(if *signed {
793 format!("imul{suffix}")
794 } else {
795 format!("mul{suffix}")
796 });
797 format!("{op} {src1}, {src2}, {dst_lo}, {dst_hi}")
798 }
799
800 Inst::MulX {
801 size,
802 src1,
803 src2,
804 dst_lo,
805 dst_hi,
806 } => {
807 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
808 let dst_hi = pretty_print_reg(dst_hi.to_reg().to_reg(), size.to_bytes());
809 let dst_lo = if dst_lo.to_reg().is_invalid_sentinel() {
810 dst_hi.clone()
811 } else {
812 pretty_print_reg(dst_lo.to_reg().to_reg(), size.to_bytes())
813 };
814 let src2 = src2.pretty_print(size.to_bytes());
815 let suffix = suffix_bwlq(*size);
816 let op = ljustify(format!("mulx{suffix}"));
817 format!("{op} {src1}, {src2}, {dst_lo}, {dst_hi}")
818 }
819
820 Inst::Mul8 {
821 signed,
822 src1,
823 src2,
824 dst,
825 } => {
826 let src1 = pretty_print_reg(src1.to_reg(), 1);
827 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
828 let src2 = src2.pretty_print(1);
829 let op = ljustify(if *signed {
830 "imulb".to_string()
831 } else {
832 "mulb".to_string()
833 });
834 format!("{op} {src1}, {src2}, {dst}")
835 }
836
837 Inst::IMul {
838 size,
839 src1,
840 src2,
841 dst,
842 } => {
843 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
844 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
845 let src2 = src2.pretty_print(size.to_bytes());
846 let suffix = suffix_bwlq(*size);
847 let op = ljustify(format!("imul{suffix}"));
848 format!("{op} {src1}, {src2}, {dst}")
849 }
850
851 Inst::IMulImm {
852 size,
853 src1,
854 src2,
855 dst,
856 } => {
857 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
858 let src1 = src1.pretty_print(size.to_bytes());
859 let suffix = suffix_bwlq(*size);
860 let op = ljustify(format!("imul{suffix}"));
861 format!("{op} {src1}, {src2:#x}, {dst}")
862 }
863
864 Inst::CheckedSRemSeq {
865 size,
866 divisor,
867 dividend_lo,
868 dividend_hi,
869 dst_quotient,
870 dst_remainder,
871 } => {
872 let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes());
873 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
874 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
875 let dst_quotient =
876 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
877 let dst_remainder =
878 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
879 format!(
880 "checked_srem_seq {dividend_lo}, {dividend_hi}, \
881 {divisor}, {dst_quotient}, {dst_remainder}",
882 )
883 }
884
885 Inst::CheckedSRemSeq8 {
886 divisor,
887 dividend,
888 dst,
889 } => {
890 let divisor = pretty_print_reg(divisor.to_reg(), 1);
891 let dividend = pretty_print_reg(dividend.to_reg(), 1);
892 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
893 format!("checked_srem_seq {dividend}, {divisor}, {dst}")
894 }
895
896 Inst::SignExtendData { size, src, dst } => {
897 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
898 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
899 let op = match size {
900 OperandSize::Size8 => "cbw",
901 OperandSize::Size16 => "cwd",
902 OperandSize::Size32 => "cdq",
903 OperandSize::Size64 => "cqo",
904 };
905 format!("{op} {src}, {dst}")
906 }
907
908 Inst::XmmUnaryRmR { op, src, dst, .. } => {
909 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size());
910 let src = src.pretty_print(op.src_size());
911 let op = ljustify(op.to_string());
912 format!("{op} {src}, {dst}")
913 }
914
915 Inst::XmmUnaryRmRUnaligned { op, src, dst, .. } => {
916 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size());
917 let src = src.pretty_print(op.src_size());
918 let op = ljustify(op.to_string());
919 format!("{op} {src}, {dst}")
920 }
921
922 Inst::XmmUnaryRmRImm {
923 op, src, dst, imm, ..
924 } => {
925 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size());
926 let src = src.pretty_print(op.src_size());
927 let op = ljustify(op.to_string());
928 format!("{op} ${imm}, {src}, {dst}")
929 }
930
931 Inst::XmmUnaryRmRVex { op, src, dst, .. } => {
932 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
933 let src = src.pretty_print(8);
934 let op = ljustify(op.to_string());
935 format!("{op} {src}, {dst}")
936 }
937
938 Inst::XmmUnaryRmRImmVex {
939 op, src, dst, imm, ..
940 } => {
941 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
942 let src = src.pretty_print(8);
943 let op = ljustify(op.to_string());
944 format!("{op} ${imm}, {src}, {dst}")
945 }
946
947 Inst::XmmUnaryRmREvex { op, src, dst, .. } => {
948 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
949 let src = src.pretty_print(8);
950 let op = ljustify(op.to_string());
951 format!("{op} {src}, {dst}")
952 }
953
954 Inst::XmmUnaryRmRImmEvex {
955 op, src, dst, imm, ..
956 } => {
957 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
958 let src = src.pretty_print(8);
959 let op = ljustify(op.to_string());
960 format!("{op} ${imm}, {src}, {dst}")
961 }
962
963 Inst::XmmMovRM { op, src, dst, .. } => {
964 let src = pretty_print_reg(src.to_reg(), 8);
965 let dst = dst.pretty_print(8);
966 let op = ljustify(op.to_string());
967 format!("{op} {src}, {dst}")
968 }
969
970 Inst::XmmMovRMVex { op, src, dst, .. } => {
971 let src = pretty_print_reg(src.to_reg(), 8);
972 let dst = dst.pretty_print(8);
973 let op = ljustify(op.to_string());
974 format!("{op} {src}, {dst}")
975 }
976
977 Inst::XmmMovRMImm {
978 op, src, dst, imm, ..
979 } => {
980 let src = pretty_print_reg(src.to_reg(), 8);
981 let dst = dst.pretty_print(8);
982 let op = ljustify(op.to_string());
983 format!("{op} ${imm}, {src}, {dst}")
984 }
985
986 Inst::XmmMovRMImmVex {
987 op, src, dst, imm, ..
988 } => {
989 let src = pretty_print_reg(src.to_reg(), 8);
990 let dst = dst.pretty_print(8);
991 let op = ljustify(op.to_string());
992 format!("{op} ${imm}, {src}, {dst}")
993 }
994
995 Inst::XmmRmR {
996 op,
997 src1,
998 src2,
999 dst,
1000 ..
1001 } => {
1002 let src1 = pretty_print_reg(src1.to_reg(), 8);
1003 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1004 let src2 = src2.pretty_print(8);
1005 let op = ljustify(op.to_string());
1006 format!("{op} {src1}, {src2}, {dst}")
1007 }
1008
1009 Inst::XmmRmRUnaligned {
1010 op,
1011 src1,
1012 src2,
1013 dst,
1014 ..
1015 } => {
1016 let src1 = pretty_print_reg(src1.to_reg(), 8);
1017 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1018 let src2 = src2.pretty_print(8);
1019 let op = ljustify(op.to_string());
1020 format!("{op} {src1}, {src2}, {dst}")
1021 }
1022
1023 Inst::XmmRmRBlend {
1024 op,
1025 src1,
1026 src2,
1027 mask,
1028 dst,
1029 } => {
1030 let src1 = pretty_print_reg(src1.to_reg(), 8);
1031 let mask = mask.to_reg();
1032 let mask = if mask.is_virtual() {
1033 format!(" <{}>", show_ireg_sized(mask, 8))
1034 } else {
1035 debug_assert_eq!(mask, regs::xmm0());
1036 String::new()
1037 };
1038 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1039 let src2 = src2.pretty_print(8);
1040 let op = ljustify(op.to_string());
1041 format!("{op} {src1}, {src2}, {dst}{mask}")
1042 }
1043
1044 Inst::XmmRmiRVex {
1045 op,
1046 src1,
1047 src2,
1048 dst,
1049 ..
1050 } => {
1051 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1052 let src1 = pretty_print_reg(src1.to_reg(), 8);
1053 let src2 = src2.pretty_print(8);
1054 let op = ljustify(op.to_string());
1055 format!("{op} {src1}, {src2}, {dst}")
1056 }
1057
1058 Inst::XmmRmRImmVex {
1059 op,
1060 src1,
1061 src2,
1062 dst,
1063 imm,
1064 ..
1065 } => {
1066 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1067 let src1 = pretty_print_reg(src1.to_reg(), 8);
1068 let src2 = src2.pretty_print(8);
1069 let op = ljustify(op.to_string());
1070 format!("{op} ${imm}, {src1}, {src2}, {dst}")
1071 }
1072
1073 Inst::XmmVexPinsr {
1074 op,
1075 src1,
1076 src2,
1077 dst,
1078 imm,
1079 ..
1080 } => {
1081 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1082 let src1 = pretty_print_reg(src1.to_reg(), 8);
1083 let src2 = src2.pretty_print(8);
1084 let op = ljustify(op.to_string());
1085 format!("{op} ${imm}, {src1}, {src2}, {dst}")
1086 }
1087
1088 Inst::XmmRmRVex3 {
1089 op,
1090 src1,
1091 src2,
1092 src3,
1093 dst,
1094 ..
1095 } => {
1096 let src1 = pretty_print_reg(src1.to_reg(), 8);
1097 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1098 let src2 = pretty_print_reg(src2.to_reg(), 8);
1099 let src3 = src3.pretty_print(8);
1100 let op = ljustify(op.to_string());
1101 format!("{op} {src1}, {src2}, {src3}, {dst}")
1102 }
1103
1104 Inst::XmmRmRBlendVex {
1105 op,
1106 src1,
1107 src2,
1108 mask,
1109 dst,
1110 ..
1111 } => {
1112 let src1 = pretty_print_reg(src1.to_reg(), 8);
1113 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1114 let src2 = src2.pretty_print(8);
1115 let mask = pretty_print_reg(mask.to_reg(), 8);
1116 let op = ljustify(op.to_string());
1117 format!("{op} {src1}, {src2}, {mask}, {dst}")
1118 }
1119
1120 Inst::XmmRmREvex {
1121 op,
1122 src1,
1123 src2,
1124 dst,
1125 ..
1126 } => {
1127 let src1 = pretty_print_reg(src1.to_reg(), 8);
1128 let src2 = src2.pretty_print(8);
1129 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1130 let op = ljustify(op.to_string());
1131 format!("{op} {src2}, {src1}, {dst}")
1132 }
1133
1134 Inst::XmmRmREvex3 {
1135 op,
1136 src1,
1137 src2,
1138 src3,
1139 dst,
1140 ..
1141 } => {
1142 let src1 = pretty_print_reg(src1.to_reg(), 8);
1143 let src2 = pretty_print_reg(src2.to_reg(), 8);
1144 let src3 = src3.pretty_print(8);
1145 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1146 let op = ljustify(op.to_string());
1147 format!("{op} {src3}, {src2}, {src1}, {dst}")
1148 }
1149
1150 Inst::XmmMinMaxSeq {
1151 lhs,
1152 rhs,
1153 dst,
1154 is_min,
1155 size,
1156 } => {
1157 let rhs = pretty_print_reg(rhs.to_reg(), 8);
1158 let lhs = pretty_print_reg(lhs.to_reg(), 8);
1159 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1160 let op = ljustify2(
1161 if *is_min {
1162 "xmm min seq ".to_string()
1163 } else {
1164 "xmm max seq ".to_string()
1165 },
1166 format!("f{}", size.to_bits()),
1167 );
1168 format!("{op} {lhs}, {rhs}, {dst}")
1169 }
1170
1171 Inst::XmmRmRImm {
1172 op,
1173 src1,
1174 src2,
1175 dst,
1176 imm,
1177 size,
1178 ..
1179 } => {
1180 let src1 = pretty_print_reg(*src1, 8);
1181 let dst = pretty_print_reg(dst.to_reg(), 8);
1182 let src2 = src2.pretty_print(8);
1183 let op = ljustify(format!(
1184 "{}{}",
1185 op.to_string(),
1186 if *size == OperandSize::Size64 {
1187 ".w"
1188 } else {
1189 ""
1190 }
1191 ));
1192 format!("{op} ${imm}, {src1}, {src2}, {dst}")
1193 }
1194
1195 Inst::XmmUninitializedValue { dst } => {
1196 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1197 let op = ljustify("uninit".into());
1198 format!("{op} {dst}")
1199 }
1200
1201 Inst::GprUninitializedValue { dst } => {
1202 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1203 let op = ljustify("uninit".into());
1204 format!("{op} {dst}")
1205 }
1206
1207 Inst::XmmToGpr {
1208 op,
1209 src,
1210 dst,
1211 dst_size,
1212 } => {
1213 let dst_size = dst_size.to_bytes();
1214 let src = pretty_print_reg(src.to_reg(), 8);
1215 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
1216 let op = ljustify(op.to_string());
1217 format!("{op} {src}, {dst}")
1218 }
1219
1220 Inst::XmmToGprVex {
1221 op,
1222 src,
1223 dst,
1224 dst_size,
1225 } => {
1226 let dst_size = dst_size.to_bytes();
1227 let src = pretty_print_reg(src.to_reg(), 8);
1228 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
1229 let op = ljustify(op.to_string());
1230 format!("{op} {src}, {dst}")
1231 }
1232
1233 Inst::XmmToGprImm { op, src, dst, imm } => {
1234 let src = pretty_print_reg(src.to_reg(), 8);
1235 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1236 let op = ljustify(op.to_string());
1237 format!("{op} ${imm}, {src}, {dst}")
1238 }
1239
1240 Inst::XmmToGprImmVex { op, src, dst, imm } => {
1241 let src = pretty_print_reg(src.to_reg(), 8);
1242 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1243 let op = ljustify(op.to_string());
1244 format!("{op} ${imm}, {src}, {dst}")
1245 }
1246
1247 Inst::GprToXmm {
1248 op,
1249 src,
1250 src_size,
1251 dst,
1252 } => {
1253 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1254 let src = src.pretty_print(src_size.to_bytes());
1255 let op = ljustify(op.to_string());
1256 format!("{op} {src}, {dst}")
1257 }
1258
1259 Inst::GprToXmmVex {
1260 op,
1261 src,
1262 src_size,
1263 dst,
1264 } => {
1265 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1266 let src = src.pretty_print(src_size.to_bytes());
1267 let op = ljustify(op.to_string());
1268 format!("{op} {src}, {dst}")
1269 }
1270
1271 Inst::XmmCmpRmR { op, src1, src2 } => {
1272 let src1 = pretty_print_reg(src1.to_reg(), 8);
1273 let src2 = src2.pretty_print(8);
1274 let op = ljustify(op.to_string());
1275 format!("{op} {src2}, {src1}")
1276 }
1277
1278 Inst::CvtIntToFloat {
1279 op,
1280 src1,
1281 src2,
1282 dst,
1283 src2_size,
1284 } => {
1285 let src1 = pretty_print_reg(src1.to_reg(), 8);
1286 let dst = pretty_print_reg(*dst.to_reg(), 8);
1287 let src2 = src2.pretty_print(src2_size.to_bytes());
1288 let op = ljustify(op.to_string());
1289 format!("{op} {src1}, {src2}, {dst}")
1290 }
1291
1292 Inst::CvtIntToFloatVex {
1293 op,
1294 src1,
1295 src2,
1296 dst,
1297 src2_size,
1298 } => {
1299 let dst = pretty_print_reg(*dst.to_reg(), 8);
1300 let src1 = pretty_print_reg(src1.to_reg(), 8);
1301 let src2 = src2.pretty_print(src2_size.to_bytes());
1302 let op = ljustify(op.to_string());
1303 format!("{op} {src1}, {src2}, {dst}")
1304 }
1305
1306 Inst::XmmCmpRmRVex { op, src1, src2 } => {
1307 let src1 = pretty_print_reg(src1.to_reg(), 8);
1308 let src2 = src2.pretty_print(8);
1309 format!("{} {src2}, {src1}", ljustify(op.to_string()))
1310 }
1311
1312 Inst::CvtUint64ToFloatSeq {
1313 src,
1314 dst,
1315 dst_size,
1316 tmp_gpr1,
1317 tmp_gpr2,
1318 ..
1319 } => {
1320 let src = pretty_print_reg(src.to_reg(), 8);
1321 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1322 let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8);
1323 let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8);
1324 let op = ljustify(format!(
1325 "u64_to_{}_seq",
1326 if *dst_size == OperandSize::Size64 {
1327 "f64"
1328 } else {
1329 "f32"
1330 }
1331 ));
1332 format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}")
1333 }
1334
1335 Inst::CvtFloatToSintSeq {
1336 src,
1337 dst,
1338 src_size,
1339 dst_size,
1340 tmp_xmm,
1341 tmp_gpr,
1342 is_saturating,
1343 } => {
1344 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
1345 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1346 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
1347 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
1348 let op = ljustify(format!(
1349 "cvt_float{}_to_sint{}{}_seq",
1350 src_size.to_bits(),
1351 dst_size.to_bits(),
1352 if *is_saturating { "_sat" } else { "" },
1353 ));
1354 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}")
1355 }
1356
1357 Inst::CvtFloatToUintSeq {
1358 src,
1359 dst,
1360 src_size,
1361 dst_size,
1362 tmp_gpr,
1363 tmp_xmm,
1364 tmp_xmm2,
1365 is_saturating,
1366 } => {
1367 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
1368 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1369 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
1370 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
1371 let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8);
1372 let op = ljustify(format!(
1373 "cvt_float{}_to_uint{}{}_seq",
1374 src_size.to_bits(),
1375 dst_size.to_bits(),
1376 if *is_saturating { "_sat" } else { "" },
1377 ));
1378 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}, {tmp_xmm2}")
1379 }
1380
1381 Inst::Imm {
1382 dst_size,
1383 simm64,
1384 dst,
1385 } => {
1386 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1387 if *dst_size == OperandSize::Size64 {
1388 let op = ljustify("movabsq".to_string());
1389 let imm = *simm64 as i64;
1390 format!("{op} ${imm}, {dst}")
1391 } else {
1392 let op = ljustify("movl".to_string());
1393 let imm = (*simm64 as u32) as i32;
1394 format!("{op} ${imm}, {dst}")
1395 }
1396 }
1397
1398 Inst::MovImmM { size, simm32, dst } => {
1399 let dst = dst.pretty_print(size.to_bytes());
1400 let suffix = suffix_bwlq(*size);
1401 let imm = match *size {
1402 OperandSize::Size8 => ((*simm32 as u8) as i8).to_string(),
1403 OperandSize::Size16 => ((*simm32 as u16) as i16).to_string(),
1404 OperandSize::Size32 => simm32.to_string(),
1405 OperandSize::Size64 => (*simm32 as i64).to_string(),
1406 };
1407 let op = ljustify2("mov".to_string(), suffix);
1408 format!("{op} ${imm}, {dst}")
1409 }
1410
1411 Inst::MovRR { size, src, dst } => {
1412 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1413 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1414 let op = ljustify2("mov".to_string(), suffix_lq(*size));
1415 format!("{op} {src}, {dst}")
1416 }
1417
1418 Inst::MovFromPReg { src, dst } => {
1419 let src: Reg = (*src).into();
1420 let src = regs::show_ireg_sized(src, 8);
1421 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1422 let op = ljustify("movq".to_string());
1423 format!("{op} {src}, {dst}")
1424 }
1425
1426 Inst::MovToPReg { src, dst } => {
1427 let src = pretty_print_reg(src.to_reg(), 8);
1428 let dst: Reg = (*dst).into();
1429 let dst = regs::show_ireg_sized(dst, 8);
1430 let op = ljustify("movq".to_string());
1431 format!("{op} {src}, {dst}")
1432 }
1433
1434 Inst::MovzxRmR {
1435 ext_mode, src, dst, ..
1436 } => {
1437 let dst_size = if *ext_mode == ExtMode::LQ {
1438 4
1439 } else {
1440 ext_mode.dst_size()
1441 };
1442 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
1443 let src = src.pretty_print(ext_mode.src_size());
1444
1445 if *ext_mode == ExtMode::LQ {
1446 let op = ljustify("movl".to_string());
1447 format!("{op} {src}, {dst}")
1448 } else {
1449 let op = ljustify2("movz".to_string(), ext_mode.to_string());
1450 format!("{op} {src}, {dst}")
1451 }
1452 }
1453
1454 Inst::Mov64MR { src, dst, .. } => {
1455 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1456 let src = src.pretty_print(8);
1457 let op = ljustify("movq".to_string());
1458 format!("{op} {src}, {dst}")
1459 }
1460
1461 Inst::LoadEffectiveAddress { addr, dst, size } => {
1462 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1463 let addr = addr.pretty_print(8);
1464 let op = ljustify("lea".to_string());
1465 format!("{op} {addr}, {dst}")
1466 }
1467
1468 Inst::MovsxRmR {
1469 ext_mode, src, dst, ..
1470 } => {
1471 let dst = pretty_print_reg(dst.to_reg().to_reg(), ext_mode.dst_size());
1472 let src = src.pretty_print(ext_mode.src_size());
1473 let op = ljustify2("movs".to_string(), ext_mode.to_string());
1474 format!("{op} {src}, {dst}")
1475 }
1476
1477 Inst::MovRM { size, src, dst, .. } => {
1478 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1479 let dst = dst.pretty_print(size.to_bytes());
1480 let op = ljustify2("mov".to_string(), suffix_bwlq(*size));
1481 format!("{op} {src}, {dst}")
1482 }
1483
1484 Inst::ShiftR {
1485 size,
1486 kind,
1487 num_bits,
1488 src,
1489 dst,
1490 ..
1491 } => {
1492 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1493 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1494 match num_bits.as_imm8_reg() {
1495 &Imm8Reg::Reg { reg } => {
1496 let reg = pretty_print_reg(reg, 1);
1497 let op = ljustify2(kind.to_string(), suffix_bwlq(*size));
1498 format!("{op} {reg}, {src}, {dst}")
1499 }
1500
1501 &Imm8Reg::Imm8 { imm: num_bits } => {
1502 let op = ljustify2(kind.to_string(), suffix_bwlq(*size));
1503 format!("{op} ${num_bits}, {src}, {dst}")
1504 }
1505 }
1506 }
1507
1508 Inst::XmmRmiReg {
1509 opcode,
1510 src1,
1511 src2,
1512 dst,
1513 ..
1514 } => {
1515 let src1 = pretty_print_reg(src1.to_reg(), 8);
1516 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1517 let src2 = src2.pretty_print(8);
1518 let op = ljustify(opcode.to_string());
1519 format!("{op} {src1}, {src2}, {dst}")
1520 }
1521
1522 Inst::CmpRmiR {
1523 size,
1524 src1,
1525 src2,
1526 opcode,
1527 } => {
1528 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
1529 let src2 = src2.pretty_print(size.to_bytes());
1530 let op = match opcode {
1531 CmpOpcode::Cmp => "cmp",
1532 CmpOpcode::Test => "test",
1533 };
1534 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
1535 format!("{op} {src2}, {src1}")
1536 }
1537
1538 Inst::Setcc { cc, dst } => {
1539 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
1540 let op = ljustify2("set".to_string(), cc.to_string());
1541 format!("{op} {dst}")
1542 }
1543
1544 Inst::Bswap { size, src, dst } => {
1545 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1546 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1547 let op = ljustify2("bswap".to_string(), suffix_bwlq(*size));
1548 format!("{op} {src}, {dst}")
1549 }
1550
1551 Inst::Cmove {
1552 size,
1553 cc,
1554 consequent,
1555 alternative,
1556 dst,
1557 } => {
1558 let alternative = pretty_print_reg(alternative.to_reg(), size.to_bytes());
1559 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1560 let consequent = consequent.pretty_print(size.to_bytes());
1561 let op = ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size)));
1562 format!("{op} {consequent}, {alternative}, {dst}")
1563 }
1564
1565 Inst::XmmCmove {
1566 ty,
1567 cc,
1568 consequent,
1569 alternative,
1570 dst,
1571 ..
1572 } => {
1573 let size = u8::try_from(ty.bytes()).unwrap();
1574 let alternative = pretty_print_reg(alternative.to_reg(), size);
1575 let dst = pretty_print_reg(dst.to_reg().to_reg(), size);
1576 let consequent = pretty_print_reg(consequent.to_reg(), size);
1577 let suffix = match *ty {
1578 types::F64 => "sd",
1579 types::F32 => "ss",
1580 types::F16 => "ss",
1581 types::F32X4 => "aps",
1582 types::F64X2 => "apd",
1583 _ => "dqa",
1584 };
1585 let cc = cc.invert();
1586 format!(
1587 "mov{suffix} {alternative}, {dst}; \
1588 j{cc} $next; \
1589 mov{suffix} {consequent}, {dst}; \
1590 $next:"
1591 )
1592 }
1593
1594 Inst::Push64 { src } => {
1595 let src = src.pretty_print(8);
1596 let op = ljustify("pushq".to_string());
1597 format!("{op} {src}")
1598 }
1599
1600 Inst::StackProbeLoop {
1601 tmp,
1602 frame_size,
1603 guard_size,
1604 } => {
1605 let tmp = pretty_print_reg(tmp.to_reg(), 8);
1606 let op = ljustify("stack_probe_loop".to_string());
1607 format!("{op} {tmp}, frame_size={frame_size}, guard_size={guard_size}")
1608 }
1609
1610 Inst::Pop64 { dst } => {
1611 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1612 let op = ljustify("popq".to_string());
1613 format!("{op} {dst}")
1614 }
1615
1616 Inst::CallKnown { info } => {
1617 let op = ljustify("call".to_string());
1618 let try_call = info
1619 .try_call_info
1620 .as_ref()
1621 .map(|tci| pretty_print_try_call(tci))
1622 .unwrap_or_default();
1623 format!("{op} {:?}{try_call}", info.dest)
1624 }
1625
1626 Inst::CallUnknown { info } => {
1627 let dest = info.dest.pretty_print(8);
1628 let op = ljustify("call".to_string());
1629 let try_call = info
1630 .try_call_info
1631 .as_ref()
1632 .map(|tci| pretty_print_try_call(tci))
1633 .unwrap_or_default();
1634 format!("{op} *{dest}{try_call}")
1635 }
1636
1637 Inst::ReturnCallKnown { info } => {
1638 let ReturnCallInfo {
1639 uses,
1640 new_stack_arg_size,
1641 tmp,
1642 dest,
1643 } = &**info;
1644 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
1645 let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}");
1646 for ret in uses {
1647 let preg = regs::show_reg(ret.preg);
1648 let vreg = pretty_print_reg(ret.vreg, 8);
1649 write!(&mut s, " {vreg}={preg}").unwrap();
1650 }
1651 s
1652 }
1653
1654 Inst::ReturnCallUnknown { info } => {
1655 let ReturnCallInfo {
1656 uses,
1657 new_stack_arg_size,
1658 tmp,
1659 dest,
1660 } = &**info;
1661 let callee = pretty_print_reg(*dest, 8);
1662 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
1663 let mut s =
1664 format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}");
1665 for ret in uses {
1666 let preg = regs::show_reg(ret.preg);
1667 let vreg = pretty_print_reg(ret.vreg, 8);
1668 write!(&mut s, " {vreg}={preg}").unwrap();
1669 }
1670 s
1671 }
1672
1673 Inst::Args { args } => {
1674 let mut s = "args".to_string();
1675 for arg in args {
1676 let preg = regs::show_reg(arg.preg);
1677 let def = pretty_print_reg(arg.vreg.to_reg(), 8);
1678 write!(&mut s, " {def}={preg}").unwrap();
1679 }
1680 s
1681 }
1682
1683 Inst::Rets { rets } => {
1684 let mut s = "rets".to_string();
1685 for ret in rets {
1686 let preg = regs::show_reg(ret.preg);
1687 let vreg = pretty_print_reg(ret.vreg, 8);
1688 write!(&mut s, " {vreg}={preg}").unwrap();
1689 }
1690 s
1691 }
1692
1693 Inst::Ret { stack_bytes_to_pop } => {
1694 let mut s = "ret".to_string();
1695 if *stack_bytes_to_pop != 0 {
1696 write!(&mut s, " {stack_bytes_to_pop}").unwrap();
1697 }
1698 s
1699 }
1700
1701 Inst::StackSwitchBasic {
1702 store_context_ptr,
1703 load_context_ptr,
1704 in_payload0,
1705 out_payload0,
1706 } => {
1707 let store_context_ptr = pretty_print_reg(**store_context_ptr, 8);
1708 let load_context_ptr = pretty_print_reg(**load_context_ptr, 8);
1709 let in_payload0 = pretty_print_reg(**in_payload0, 8);
1710 let out_payload0 = pretty_print_reg(*out_payload0.to_reg(), 8);
1711 format!("{out_payload0} = stack_switch_basic {store_context_ptr}, {load_context_ptr}, {in_payload0}")
1712 }
1713
1714 Inst::JmpKnown { dst } => {
1715 let op = ljustify("jmp".to_string());
1716 let dst = dst.to_string();
1717 format!("{op} {dst}")
1718 }
1719
1720 Inst::WinchJmpIf { cc, taken } => {
1721 let taken = taken.to_string();
1722 let op = ljustify2("j".to_string(), cc.to_string());
1723 format!("{op} {taken}")
1724 }
1725
1726 Inst::JmpCondOr {
1727 cc1,
1728 cc2,
1729 taken,
1730 not_taken,
1731 } => {
1732 let taken = taken.to_string();
1733 let not_taken = not_taken.to_string();
1734 let op = ljustify(format!("j{cc1},{cc2}"));
1735 format!("{op} {taken}; j {not_taken}")
1736 }
1737
1738 Inst::JmpCond {
1739 cc,
1740 taken,
1741 not_taken,
1742 } => {
1743 let taken = taken.to_string();
1744 let not_taken = not_taken.to_string();
1745 let op = ljustify2("j".to_string(), cc.to_string());
1746 format!("{op} {taken}; j {not_taken}")
1747 }
1748
1749 Inst::JmpTableSeq {
1750 idx, tmp1, tmp2, ..
1751 } => {
1752 let idx = pretty_print_reg(*idx, 8);
1753 let tmp1 = pretty_print_reg(tmp1.to_reg(), 8);
1754 let tmp2 = pretty_print_reg(tmp2.to_reg(), 8);
1755 let op = ljustify("br_table".into());
1756 format!("{op} {idx}, {tmp1}, {tmp2}")
1757 }
1758
1759 Inst::JmpUnknown { target } => {
1760 let target = target.pretty_print(8);
1761 let op = ljustify("jmp".to_string());
1762 format!("{op} *{target}")
1763 }
1764
1765 Inst::TrapIf { cc, trap_code, .. } => {
1766 format!("j{cc} #trap={trap_code}")
1767 }
1768
1769 Inst::TrapIfAnd {
1770 cc1,
1771 cc2,
1772 trap_code,
1773 ..
1774 } => {
1775 let cc1 = cc1.invert();
1776 let cc2 = cc2.invert();
1777 format!("trap_if_and {cc1}, {cc2}, {trap_code}")
1778 }
1779
1780 Inst::TrapIfOr {
1781 cc1,
1782 cc2,
1783 trap_code,
1784 ..
1785 } => {
1786 let cc2 = cc2.invert();
1787 format!("trap_if_or {cc1}, {cc2}, {trap_code}")
1788 }
1789
1790 Inst::LoadExtName {
1791 dst, name, offset, ..
1792 } => {
1793 let dst = pretty_print_reg(dst.to_reg(), 8);
1794 let name = name.display(None);
1795 let op = ljustify("load_ext_name".into());
1796 format!("{op} {name}+{offset}, {dst}")
1797 }
1798
1799 Inst::LockCmpxchg {
1800 ty,
1801 replacement,
1802 expected,
1803 mem,
1804 dst_old,
1805 ..
1806 } => {
1807 let size = ty.bytes() as u8;
1808 let replacement = pretty_print_reg(*replacement, size);
1809 let expected = pretty_print_reg(*expected, size);
1810 let dst_old = pretty_print_reg(dst_old.to_reg(), size);
1811 let mem = mem.pretty_print(size);
1812 let suffix = suffix_bwlq(OperandSize::from_bytes(size as u32));
1813 format!(
1814 "lock cmpxchg{suffix} {replacement}, {mem}, expected={expected}, dst_old={dst_old}"
1815 )
1816 }
1817
1818 Inst::LockCmpxchg16b {
1819 replacement_low,
1820 replacement_high,
1821 expected_low,
1822 expected_high,
1823 mem,
1824 dst_old_low,
1825 dst_old_high,
1826 ..
1827 } => {
1828 let replacement_low = pretty_print_reg(*replacement_low, 8);
1829 let replacement_high = pretty_print_reg(*replacement_high, 8);
1830 let expected_low = pretty_print_reg(*expected_low, 8);
1831 let expected_high = pretty_print_reg(*expected_high, 8);
1832 let dst_old_low = pretty_print_reg(dst_old_low.to_reg(), 8);
1833 let dst_old_high = pretty_print_reg(dst_old_high.to_reg(), 8);
1834 let mem = mem.pretty_print(16);
1835 format!(
1836 "lock cmpxchg16b {mem}, replacement={replacement_high}:{replacement_low}, expected={expected_high}:{expected_low}, dst_old={dst_old_high}:{dst_old_low}"
1837 )
1838 }
1839
1840 Inst::LockXadd {
1841 size,
1842 operand,
1843 mem,
1844 dst_old,
1845 } => {
1846 let operand = pretty_print_reg(*operand, size.to_bytes());
1847 let dst_old = pretty_print_reg(dst_old.to_reg(), size.to_bytes());
1848 let mem = mem.pretty_print(size.to_bytes());
1849 let suffix = suffix_bwlq(*size);
1850 format!("lock xadd{suffix} {operand}, {mem}, dst_old={dst_old}")
1851 }
1852
1853 Inst::Xchg {
1854 size,
1855 operand,
1856 mem,
1857 dst_old,
1858 } => {
1859 let operand = pretty_print_reg(*operand, size.to_bytes());
1860 let dst_old = pretty_print_reg(dst_old.to_reg(), size.to_bytes());
1861 let mem = mem.pretty_print(size.to_bytes());
1862 let suffix = suffix_bwlq(*size);
1863 format!("xchg{suffix} {operand}, {mem}, dst_old={dst_old}")
1864 }
1865
1866 Inst::AtomicRmwSeq { ty, op, .. } => {
1867 let ty = ty.bits();
1868 format!(
1869 "atomically {{ {ty}_bits_at_[%r9] {op:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}"
1870 )
1871 }
1872
1873 Inst::Atomic128RmwSeq {
1874 op,
1875 mem,
1876 operand_low,
1877 operand_high,
1878 temp_low,
1879 temp_high,
1880 dst_old_low,
1881 dst_old_high,
1882 } => {
1883 let operand_low = pretty_print_reg(*operand_low, 8);
1884 let operand_high = pretty_print_reg(*operand_high, 8);
1885 let temp_low = pretty_print_reg(temp_low.to_reg(), 8);
1886 let temp_high = pretty_print_reg(temp_high.to_reg(), 8);
1887 let dst_old_low = pretty_print_reg(dst_old_low.to_reg(), 8);
1888 let dst_old_high = pretty_print_reg(dst_old_high.to_reg(), 8);
1889 let mem = mem.pretty_print(16);
1890 format!("atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem} = {temp_high}:{temp_low} }}")
1891 }
1892
1893 Inst::Atomic128XchgSeq {
1894 mem,
1895 operand_low,
1896 operand_high,
1897 dst_old_low,
1898 dst_old_high,
1899 } => {
1900 let operand_low = pretty_print_reg(*operand_low, 8);
1901 let operand_high = pretty_print_reg(*operand_high, 8);
1902 let dst_old_low = pretty_print_reg(dst_old_low.to_reg(), 8);
1903 let dst_old_high = pretty_print_reg(dst_old_high.to_reg(), 8);
1904 let mem = mem.pretty_print(16);
1905 format!("atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {mem} = {operand_high}:{operand_low} }}")
1906 }
1907
1908 Inst::Fence { kind } => match kind {
1909 FenceKind::MFence => "mfence".to_string(),
1910 FenceKind::LFence => "lfence".to_string(),
1911 FenceKind::SFence => "sfence".to_string(),
1912 },
1913
1914 Inst::Hlt => "hlt".into(),
1915
1916 Inst::Ud2 { trap_code } => format!("ud2 {trap_code}"),
1917
1918 Inst::ElfTlsGetAddr { symbol, dst } => {
1919 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1920 format!("{dst} = elf_tls_get_addr {symbol:?}")
1921 }
1922
1923 Inst::MachOTlsGetAddr { symbol, dst } => {
1924 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1925 format!("{dst} = macho_tls_get_addr {symbol:?}")
1926 }
1927
1928 Inst::CoffTlsGetAddr { symbol, dst, tmp } => {
1929 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1930 let tmp = tmp.to_reg().to_reg();
1931
1932 let mut s = format!("{dst} = coff_tls_get_addr {symbol:?}");
1933 if tmp.is_virtual() {
1934 let tmp = show_ireg_sized(tmp, 8);
1935 write!(&mut s, ", {tmp}").unwrap();
1936 };
1937
1938 s
1939 }
1940
1941 Inst::Unwind { inst } => format!("unwind {inst:?}"),
1942
1943 Inst::DummyUse { reg } => {
1944 let reg = pretty_print_reg(*reg, 8);
1945 format!("dummy_use {reg}")
1946 }
1947
1948 Inst::External { inst } => {
1949 format!("{inst}")
1950 }
1951 }
1952 }
1953}
1954
1955fn pretty_print_try_call(info: &TryCallInfo) -> String {
1956 let dests = info
1957 .exception_dests
1958 .iter()
1959 .map(|(tag, label)| format!("{tag:?}: {label:?}"))
1960 .collect::<Vec<_>>()
1961 .join(", ");
1962 format!("; jmp {:?}; catch [{dests}]", info.continuation)
1963}
1964
1965impl fmt::Debug for Inst {
1966 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1967 write!(fmt, "{}", self.pretty_print_inst(&mut Default::default()))
1968 }
1969}
1970
1971fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
1972 match inst {
1982 Inst::AluRmRVex {
1983 src1, src2, dst, ..
1984 } => {
1985 collector.reg_def(dst);
1986 collector.reg_use(src1);
1987 src2.get_operands(collector);
1988 }
1989 Inst::Div {
1990 divisor,
1991 dividend_lo,
1992 dividend_hi,
1993 dst_quotient,
1994 dst_remainder,
1995 ..
1996 } => {
1997 divisor.get_operands(collector);
1998 collector.reg_fixed_use(dividend_lo, regs::rax());
1999 collector.reg_fixed_use(dividend_hi, regs::rdx());
2000 collector.reg_fixed_def(dst_quotient, regs::rax());
2001 collector.reg_fixed_def(dst_remainder, regs::rdx());
2002 }
2003 Inst::CheckedSRemSeq {
2004 divisor,
2005 dividend_lo,
2006 dividend_hi,
2007 dst_quotient,
2008 dst_remainder,
2009 ..
2010 } => {
2011 collector.reg_use(divisor);
2012 collector.reg_fixed_use(dividend_lo, regs::rax());
2013 collector.reg_fixed_use(dividend_hi, regs::rdx());
2014 collector.reg_fixed_def(dst_quotient, regs::rax());
2015 collector.reg_fixed_def(dst_remainder, regs::rdx());
2016 }
2017 Inst::Div8 {
2018 divisor,
2019 dividend,
2020 dst,
2021 ..
2022 } => {
2023 divisor.get_operands(collector);
2024 collector.reg_fixed_use(dividend, regs::rax());
2025 collector.reg_fixed_def(dst, regs::rax());
2026 }
2027 Inst::CheckedSRemSeq8 {
2028 divisor,
2029 dividend,
2030 dst,
2031 ..
2032 } => {
2033 collector.reg_use(divisor);
2034 collector.reg_fixed_use(dividend, regs::rax());
2035 collector.reg_fixed_def(dst, regs::rax());
2036 }
2037 Inst::Mul {
2038 src1,
2039 src2,
2040 dst_lo,
2041 dst_hi,
2042 ..
2043 } => {
2044 collector.reg_fixed_use(src1, regs::rax());
2045 collector.reg_fixed_def(dst_lo, regs::rax());
2046 collector.reg_fixed_def(dst_hi, regs::rdx());
2047 src2.get_operands(collector);
2048 }
2049 Inst::Mul8 {
2050 src1, src2, dst, ..
2051 } => {
2052 collector.reg_fixed_use(src1, regs::rax());
2053 collector.reg_fixed_def(dst, regs::rax());
2054 src2.get_operands(collector);
2055 }
2056 Inst::IMul {
2057 src1, src2, dst, ..
2058 } => {
2059 collector.reg_use(src1);
2060 collector.reg_reuse_def(dst, 0);
2061 src2.get_operands(collector);
2062 }
2063 Inst::IMulImm { src1, dst, .. } => {
2064 collector.reg_def(dst);
2065 src1.get_operands(collector);
2066 }
2067 Inst::MulX {
2068 src1,
2069 src2,
2070 dst_lo,
2071 dst_hi,
2072 ..
2073 } => {
2074 if !dst_lo.to_reg().is_invalid_sentinel() {
2075 collector.reg_def(dst_lo);
2076 }
2077 collector.reg_def(dst_hi);
2078 collector.reg_fixed_use(src1, regs::rdx());
2079 src2.get_operands(collector);
2080 }
2081 Inst::SignExtendData { size, src, dst } => {
2082 match size {
2083 OperandSize::Size8 => {
2084 collector.reg_fixed_use(src, regs::rax());
2087 collector.reg_fixed_def(dst, regs::rax());
2088 }
2089 _ => {
2090 collector.reg_fixed_use(src, regs::rax());
2093 collector.reg_fixed_def(dst, regs::rdx());
2094 }
2095 }
2096 }
2097 Inst::UnaryRmR { src, dst, .. }
2098 | Inst::UnaryRmRVex { src, dst, .. }
2099 | Inst::UnaryRmRImmVex { src, dst, .. } => {
2100 collector.reg_def(dst);
2101 src.get_operands(collector);
2102 }
2103 Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmRImm { src, dst, .. } => {
2104 collector.reg_def(dst);
2105 src.get_operands(collector);
2106 }
2107 Inst::XmmUnaryRmREvex { src, dst, .. }
2108 | Inst::XmmUnaryRmRImmEvex { src, dst, .. }
2109 | Inst::XmmUnaryRmRUnaligned { src, dst, .. }
2110 | Inst::XmmUnaryRmRVex { src, dst, .. }
2111 | Inst::XmmUnaryRmRImmVex { src, dst, .. } => {
2112 collector.reg_def(dst);
2113 src.get_operands(collector);
2114 }
2115 Inst::XmmRmR {
2116 src1, src2, dst, ..
2117 } => {
2118 collector.reg_use(src1);
2119 collector.reg_reuse_def(dst, 0);
2120 src2.get_operands(collector);
2121 }
2122 Inst::XmmRmRUnaligned {
2123 src1, src2, dst, ..
2124 } => {
2125 collector.reg_use(src1);
2126 collector.reg_reuse_def(dst, 0);
2127 src2.get_operands(collector);
2128 }
2129 Inst::XmmRmRBlend {
2130 src1,
2131 src2,
2132 mask,
2133 dst,
2134 op,
2135 } => {
2136 assert!(matches!(
2137 op,
2138 SseOpcode::Blendvpd | SseOpcode::Blendvps | SseOpcode::Pblendvb
2139 ));
2140 collector.reg_use(src1);
2141 collector.reg_fixed_use(mask, regs::xmm0());
2142 collector.reg_reuse_def(dst, 0);
2143 src2.get_operands(collector);
2144 }
2145 Inst::XmmRmiRVex {
2146 src1, src2, dst, ..
2147 } => {
2148 collector.reg_def(dst);
2149 collector.reg_use(src1);
2150 src2.get_operands(collector);
2151 }
2152 Inst::XmmRmRImmVex {
2153 src1, src2, dst, ..
2154 } => {
2155 collector.reg_def(dst);
2156 collector.reg_use(src1);
2157 src2.get_operands(collector);
2158 }
2159 Inst::XmmVexPinsr {
2160 src1, src2, dst, ..
2161 } => {
2162 collector.reg_def(dst);
2163 collector.reg_use(src1);
2164 src2.get_operands(collector);
2165 }
2166 Inst::XmmRmRVex3 {
2167 src1,
2168 src2,
2169 src3,
2170 dst,
2171 ..
2172 } => {
2173 collector.reg_use(src1);
2174 collector.reg_reuse_def(dst, 0);
2175 collector.reg_use(src2);
2176 src3.get_operands(collector);
2177 }
2178 Inst::XmmRmRBlendVex {
2179 src1,
2180 src2,
2181 mask,
2182 dst,
2183 ..
2184 } => {
2185 collector.reg_def(dst);
2186 collector.reg_use(src1);
2187 src2.get_operands(collector);
2188 collector.reg_use(mask);
2189 }
2190 Inst::XmmRmREvex {
2191 op,
2192 src1,
2193 src2,
2194 dst,
2195 ..
2196 } => {
2197 assert_ne!(*op, Avx512Opcode::Vpermi2b);
2198 collector.reg_use(src1);
2199 src2.get_operands(collector);
2200 collector.reg_def(dst);
2201 }
2202 Inst::XmmRmREvex3 {
2203 op,
2204 src1,
2205 src2,
2206 src3,
2207 dst,
2208 ..
2209 } => {
2210 assert_eq!(*op, Avx512Opcode::Vpermi2b);
2211 collector.reg_use(src1);
2212 collector.reg_use(src2);
2213 src3.get_operands(collector);
2214 collector.reg_reuse_def(dst, 0); }
2216 Inst::XmmRmRImm {
2217 src1, src2, dst, ..
2218 } => {
2219 collector.reg_use(src1);
2220 collector.reg_reuse_def(dst, 0);
2221 src2.get_operands(collector);
2222 }
2223 Inst::XmmUninitializedValue { dst } => collector.reg_def(dst),
2224 Inst::GprUninitializedValue { dst } => collector.reg_def(dst),
2225 Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
2226 collector.reg_use(rhs);
2227 collector.reg_use(lhs);
2228 collector.reg_reuse_def(dst, 0); }
2230 Inst::XmmRmiReg {
2231 src1, src2, dst, ..
2232 } => {
2233 collector.reg_use(src1);
2234 collector.reg_reuse_def(dst, 0); src2.get_operands(collector);
2236 }
2237 Inst::XmmMovRM { src, dst, .. }
2238 | Inst::XmmMovRMVex { src, dst, .. }
2239 | Inst::XmmMovRMImm { src, dst, .. }
2240 | Inst::XmmMovRMImmVex { src, dst, .. } => {
2241 collector.reg_use(src);
2242 dst.get_operands(collector);
2243 }
2244 Inst::XmmCmpRmR { src1, src2, .. } => {
2245 collector.reg_use(src1);
2246 src2.get_operands(collector);
2247 }
2248 Inst::XmmCmpRmRVex { src1, src2, .. } => {
2249 collector.reg_use(src1);
2250 src2.get_operands(collector);
2251 }
2252 Inst::Imm { dst, .. } => {
2253 collector.reg_def(dst);
2254 }
2255 Inst::MovRR { src, dst, .. } => {
2256 collector.reg_use(src);
2257 collector.reg_def(dst);
2258 }
2259 Inst::MovFromPReg { dst, src } => {
2260 debug_assert!(dst.to_reg().to_reg().is_virtual());
2261 collector.reg_fixed_nonallocatable(*src);
2262 collector.reg_def(dst);
2263 }
2264 Inst::MovToPReg { dst, src } => {
2265 debug_assert!(src.to_reg().is_virtual());
2266 collector.reg_use(src);
2267 collector.reg_fixed_nonallocatable(*dst);
2268 }
2269 Inst::XmmToGpr { src, dst, .. }
2270 | Inst::XmmToGprVex { src, dst, .. }
2271 | Inst::XmmToGprImm { src, dst, .. }
2272 | Inst::XmmToGprImmVex { src, dst, .. } => {
2273 collector.reg_use(src);
2274 collector.reg_def(dst);
2275 }
2276 Inst::GprToXmm { src, dst, .. } | Inst::GprToXmmVex { src, dst, .. } => {
2277 collector.reg_def(dst);
2278 src.get_operands(collector);
2279 }
2280 Inst::CvtIntToFloat {
2281 src1, src2, dst, ..
2282 } => {
2283 collector.reg_use(src1);
2284 collector.reg_reuse_def(dst, 0);
2285 src2.get_operands(collector);
2286 }
2287 Inst::CvtIntToFloatVex {
2288 src1, src2, dst, ..
2289 } => {
2290 collector.reg_def(dst);
2291 collector.reg_use(src1);
2292 src2.get_operands(collector);
2293 }
2294 Inst::CvtUint64ToFloatSeq {
2295 src,
2296 dst,
2297 tmp_gpr1,
2298 tmp_gpr2,
2299 ..
2300 } => {
2301 collector.reg_use(src);
2302 collector.reg_early_def(dst);
2303 collector.reg_early_def(tmp_gpr1);
2304 collector.reg_early_def(tmp_gpr2);
2305 }
2306 Inst::CvtFloatToSintSeq {
2307 src,
2308 dst,
2309 tmp_xmm,
2310 tmp_gpr,
2311 ..
2312 } => {
2313 collector.reg_use(src);
2314 collector.reg_early_def(dst);
2315 collector.reg_early_def(tmp_gpr);
2316 collector.reg_early_def(tmp_xmm);
2317 }
2318 Inst::CvtFloatToUintSeq {
2319 src,
2320 dst,
2321 tmp_gpr,
2322 tmp_xmm,
2323 tmp_xmm2,
2324 ..
2325 } => {
2326 collector.reg_use(src);
2327 collector.reg_early_def(dst);
2328 collector.reg_early_def(tmp_gpr);
2329 collector.reg_early_def(tmp_xmm);
2330 collector.reg_early_def(tmp_xmm2);
2331 }
2332
2333 Inst::MovImmM { dst, .. } => {
2334 dst.get_operands(collector);
2335 }
2336
2337 Inst::MovzxRmR { src, dst, .. } => {
2338 collector.reg_def(dst);
2339 src.get_operands(collector);
2340 }
2341 Inst::Mov64MR { src, dst, .. } => {
2342 collector.reg_def(dst);
2343 src.get_operands(collector);
2344 }
2345 Inst::LoadEffectiveAddress { addr: src, dst, .. } => {
2346 collector.reg_def(dst);
2347 src.get_operands(collector);
2348 }
2349 Inst::MovsxRmR { src, dst, .. } => {
2350 collector.reg_def(dst);
2351 src.get_operands(collector);
2352 }
2353 Inst::MovRM { src, dst, .. } => {
2354 collector.reg_use(src);
2355 dst.get_operands(collector);
2356 }
2357 Inst::ShiftR {
2358 num_bits, src, dst, ..
2359 } => {
2360 collector.reg_use(src);
2361 collector.reg_reuse_def(dst, 0);
2362 if let Imm8Reg::Reg { reg } = num_bits.as_imm8_reg_mut() {
2363 collector.reg_fixed_use(reg, regs::rcx());
2364 }
2365 }
2366 Inst::CmpRmiR { src1, src2, .. } => {
2367 collector.reg_use(src1);
2368 src2.get_operands(collector);
2369 }
2370 Inst::Setcc { dst, .. } => {
2371 collector.reg_def(dst);
2372 }
2373 Inst::Bswap { src, dst, .. } => {
2374 collector.reg_use(src);
2375 collector.reg_reuse_def(dst, 0);
2376 }
2377 Inst::Cmove {
2378 consequent,
2379 alternative,
2380 dst,
2381 ..
2382 } => {
2383 collector.reg_use(alternative);
2384 collector.reg_reuse_def(dst, 0);
2385 consequent.get_operands(collector);
2386 }
2387 Inst::XmmCmove {
2388 consequent,
2389 alternative,
2390 dst,
2391 ..
2392 } => {
2393 collector.reg_use(alternative);
2394 collector.reg_reuse_def(dst, 0);
2395 collector.reg_use(consequent);
2396 }
2397 Inst::Push64 { src } => {
2398 src.get_operands(collector);
2399 }
2400 Inst::Pop64 { dst } => {
2401 collector.reg_def(dst);
2402 }
2403 Inst::StackProbeLoop { tmp, .. } => {
2404 collector.reg_early_def(tmp);
2405 }
2406
2407 Inst::CallKnown { info } => {
2408 let CallInfo {
2413 uses,
2414 defs,
2415 clobbers,
2416 dest,
2417 ..
2418 } = &mut **info;
2419 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
2420 for CallArgPair { vreg, preg } in uses {
2421 collector.reg_fixed_use(vreg, *preg);
2422 }
2423 for CallRetPair { vreg, location } in defs {
2424 match location {
2425 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
2426 RetLocation::Stack(..) => collector.any_def(vreg),
2427 }
2428 }
2429 collector.reg_clobbers(*clobbers);
2430 }
2431
2432 Inst::CallUnknown { info } => {
2433 let CallInfo {
2434 uses,
2435 defs,
2436 clobbers,
2437 callee_conv,
2438 dest,
2439 ..
2440 } = &mut **info;
2441 match dest {
2442 RegMem::Reg { reg } if *callee_conv == CallConv::Winch => {
2443 collector.reg_fixed_use(reg, regs::r10());
2447 }
2448 _ => dest.get_operands(collector),
2449 }
2450 for CallArgPair { vreg, preg } in uses {
2451 collector.reg_fixed_use(vreg, *preg);
2452 }
2453 for CallRetPair { vreg, location } in defs {
2454 match location {
2455 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
2456 RetLocation::Stack(..) => collector.any_def(vreg),
2457 }
2458 }
2459 collector.reg_clobbers(*clobbers);
2460 }
2461 Inst::StackSwitchBasic {
2462 store_context_ptr,
2463 load_context_ptr,
2464 in_payload0,
2465 out_payload0,
2466 } => {
2467 collector.reg_use(load_context_ptr);
2468 collector.reg_use(store_context_ptr);
2469 collector.reg_fixed_use(in_payload0, stack_switch::payload_register());
2470 collector.reg_fixed_def(out_payload0, stack_switch::payload_register());
2471
2472 let mut clobbers = crate::isa::x64::abi::ALL_CLOBBERS;
2473 clobbers.remove(
2475 stack_switch::payload_register()
2476 .to_real_reg()
2477 .unwrap()
2478 .into(),
2479 );
2480 collector.reg_clobbers(clobbers);
2481 }
2482
2483 Inst::ReturnCallKnown { info } => {
2484 let ReturnCallInfo {
2485 dest, uses, tmp, ..
2486 } = &mut **info;
2487 collector.reg_fixed_def(tmp, regs::r11());
2488 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
2490 for CallArgPair { vreg, preg } in uses {
2491 collector.reg_fixed_use(vreg, *preg);
2492 }
2493 }
2494
2495 Inst::ReturnCallUnknown { info } => {
2496 let ReturnCallInfo {
2497 dest, uses, tmp, ..
2498 } = &mut **info;
2499
2500 collector.reg_fixed_use(dest, regs::r10());
2506
2507 collector.reg_fixed_def(tmp, regs::r11());
2508 for CallArgPair { vreg, preg } in uses {
2509 collector.reg_fixed_use(vreg, *preg);
2510 }
2511 }
2512
2513 Inst::JmpTableSeq {
2514 idx, tmp1, tmp2, ..
2515 } => {
2516 collector.reg_use(idx);
2517 collector.reg_early_def(tmp1);
2518 collector.reg_def(tmp2);
2522 }
2523
2524 Inst::JmpUnknown { target } => {
2525 target.get_operands(collector);
2526 }
2527
2528 Inst::LoadExtName { dst, .. } => {
2529 collector.reg_def(dst);
2530 }
2531
2532 Inst::LockCmpxchg {
2533 replacement,
2534 expected,
2535 mem,
2536 dst_old,
2537 ..
2538 } => {
2539 collector.reg_use(replacement);
2540 collector.reg_fixed_use(expected, regs::rax());
2541 collector.reg_fixed_def(dst_old, regs::rax());
2542 mem.get_operands(collector);
2543 }
2544
2545 Inst::LockCmpxchg16b {
2546 replacement_low,
2547 replacement_high,
2548 expected_low,
2549 expected_high,
2550 mem,
2551 dst_old_low,
2552 dst_old_high,
2553 ..
2554 } => {
2555 collector.reg_fixed_use(replacement_low, regs::rbx());
2556 collector.reg_fixed_use(replacement_high, regs::rcx());
2557 collector.reg_fixed_use(expected_low, regs::rax());
2558 collector.reg_fixed_use(expected_high, regs::rdx());
2559 collector.reg_fixed_def(dst_old_low, regs::rax());
2560 collector.reg_fixed_def(dst_old_high, regs::rdx());
2561 mem.get_operands(collector);
2562 }
2563
2564 Inst::LockXadd {
2565 operand,
2566 mem,
2567 dst_old,
2568 ..
2569 } => {
2570 collector.reg_use(operand);
2571 collector.reg_reuse_def(dst_old, 0);
2572 mem.get_operands(collector);
2573 }
2574
2575 Inst::Xchg {
2576 operand,
2577 mem,
2578 dst_old,
2579 ..
2580 } => {
2581 collector.reg_use(operand);
2582 collector.reg_reuse_def(dst_old, 0);
2583 mem.get_operands(collector);
2584 }
2585
2586 Inst::AtomicRmwSeq {
2587 operand,
2588 temp,
2589 dst_old,
2590 mem,
2591 ..
2592 } => {
2593 collector.reg_late_use(operand);
2594 collector.reg_early_def(temp);
2595 collector.reg_fixed_def(dst_old, regs::rax());
2598 mem.get_operands_late(collector)
2599 }
2600
2601 Inst::Atomic128RmwSeq {
2602 operand_low,
2603 operand_high,
2604 temp_low,
2605 temp_high,
2606 dst_old_low,
2607 dst_old_high,
2608 mem,
2609 ..
2610 } => {
2611 collector.reg_late_use(operand_low);
2613 collector.reg_late_use(operand_high);
2614 collector.reg_fixed_def(temp_low, regs::rbx());
2615 collector.reg_fixed_def(temp_high, regs::rcx());
2616 collector.reg_fixed_def(dst_old_low, regs::rax());
2617 collector.reg_fixed_def(dst_old_high, regs::rdx());
2618 mem.get_operands_late(collector)
2619 }
2620
2621 Inst::Atomic128XchgSeq {
2622 operand_low,
2623 operand_high,
2624 dst_old_low,
2625 dst_old_high,
2626 mem,
2627 ..
2628 } => {
2629 collector.reg_fixed_late_use(operand_low, regs::rbx());
2631 collector.reg_fixed_late_use(operand_high, regs::rcx());
2632 collector.reg_fixed_def(dst_old_low, regs::rax());
2633 collector.reg_fixed_def(dst_old_high, regs::rdx());
2634 mem.get_operands_late(collector)
2635 }
2636
2637 Inst::Args { args } => {
2638 for ArgPair { vreg, preg } in args {
2639 collector.reg_fixed_def(vreg, *preg);
2640 }
2641 }
2642
2643 Inst::Rets { rets } => {
2644 for RetPair { vreg, preg } in rets {
2647 collector.reg_fixed_use(vreg, *preg);
2648 }
2649 }
2650
2651 Inst::JmpKnown { .. }
2652 | Inst::WinchJmpIf { .. }
2653 | Inst::JmpCond { .. }
2654 | Inst::JmpCondOr { .. }
2655 | Inst::Ret { .. }
2656 | Inst::Nop { .. }
2657 | Inst::TrapIf { .. }
2658 | Inst::TrapIfAnd { .. }
2659 | Inst::TrapIfOr { .. }
2660 | Inst::Hlt
2661 | Inst::Ud2 { .. }
2662 | Inst::Fence { .. } => {
2663 }
2665
2666 Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
2667 collector.reg_fixed_def(dst, regs::rax());
2668 let mut clobbers =
2675 X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false);
2676 clobbers.remove(regs::gpr_preg(regs::ENC_RAX));
2677 collector.reg_clobbers(clobbers);
2678 }
2679
2680 Inst::CoffTlsGetAddr { dst, tmp, .. } => {
2681 collector.reg_fixed_def(dst, regs::rax());
2686
2687 collector.reg_fixed_def(tmp, regs::rcx());
2689 }
2690
2691 Inst::Unwind { .. } => {}
2692
2693 Inst::DummyUse { reg } => {
2694 collector.reg_use(reg);
2695 }
2696
2697 Inst::External { inst } => {
2698 inst.visit(&mut external::RegallocVisitor { collector });
2699 }
2700 }
2701}
2702
2703impl MachInst for Inst {
2707 type ABIMachineSpec = X64ABIMachineSpec;
2708
2709 fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
2710 x64_get_operands(self, collector)
2711 }
2712
2713 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
2714 match self {
2715 Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
2720 Some((dst.to_writable_reg(), src.to_reg()))
2721 }
2722 Self::XmmUnaryRmR { op, src, dst, .. }
2727 if *op == SseOpcode::Movss
2728 || *op == SseOpcode::Movsd
2729 || *op == SseOpcode::Movaps
2730 || *op == SseOpcode::Movapd
2731 || *op == SseOpcode::Movups
2732 || *op == SseOpcode::Movupd
2733 || *op == SseOpcode::Movdqa
2734 || *op == SseOpcode::Movdqu =>
2735 {
2736 if let RegMem::Reg { reg } = src.clone().to_reg_mem() {
2737 Some((dst.to_writable_reg(), reg))
2738 } else {
2739 None
2740 }
2741 }
2742 _ => None,
2743 }
2744 }
2745
2746 fn is_included_in_clobbers(&self) -> bool {
2747 match self {
2748 &Inst::Args { .. } => false,
2749 _ => true,
2750 }
2751 }
2752
2753 fn is_trap(&self) -> bool {
2754 match self {
2755 Self::Ud2 { .. } => true,
2756 _ => false,
2757 }
2758 }
2759
2760 fn is_args(&self) -> bool {
2761 match self {
2762 Self::Args { .. } => true,
2763 _ => false,
2764 }
2765 }
2766
2767 fn is_term(&self) -> MachTerminator {
2768 match self {
2769 &Self::Rets { .. } => MachTerminator::Ret,
2771 &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => {
2772 MachTerminator::RetCall
2773 }
2774 &Self::JmpKnown { .. } => MachTerminator::Branch,
2775 &Self::JmpCond { .. } => MachTerminator::Branch,
2776 &Self::JmpCondOr { .. } => MachTerminator::Branch,
2777 &Self::JmpTableSeq { .. } => MachTerminator::Branch,
2778 &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
2779 &Self::CallUnknown { ref info } if info.try_call_info.is_some() => {
2780 MachTerminator::Branch
2781 }
2782 _ => MachTerminator::None,
2784 }
2785 }
2786
2787 fn is_low_level_branch(&self) -> bool {
2788 match self {
2789 &Self::WinchJmpIf { .. } => true,
2790 _ => false,
2791 }
2792 }
2793
2794 fn is_mem_access(&self) -> bool {
2795 panic!("TODO FILL ME OUT")
2796 }
2797
2798 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
2799 trace!(
2800 "Inst::gen_move {:?} -> {:?} (type: {:?})",
2801 src_reg,
2802 dst_reg.to_reg(),
2803 ty
2804 );
2805 let rc_dst = dst_reg.to_reg().class();
2806 let rc_src = src_reg.class();
2807 debug_assert!(rc_dst == rc_src);
2809 match rc_dst {
2810 RegClass::Int => Inst::mov_r_r(OperandSize::Size64, src_reg, dst_reg),
2811 RegClass::Float => {
2812 let opcode = match ty {
2817 types::F16 | types::F32 | types::F64 | types::F32X4 => SseOpcode::Movaps,
2818 types::F64X2 => SseOpcode::Movapd,
2819 _ if (ty.is_float() || ty.is_vector()) && ty.bits() <= 128 => SseOpcode::Movdqa,
2820 _ => unimplemented!("unable to move type: {}", ty),
2821 };
2822 Inst::xmm_unary_rm_r(opcode, RegMem::reg(src_reg), dst_reg)
2823 }
2824 RegClass::Vector => unreachable!(),
2825 }
2826 }
2827
2828 fn gen_nop(preferred_size: usize) -> Inst {
2829 Inst::nop(std::cmp::min(preferred_size, 15) as u8)
2830 }
2831
2832 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
2833 match ty {
2834 types::I8 => Ok((&[RegClass::Int], &[types::I8])),
2835 types::I16 => Ok((&[RegClass::Int], &[types::I16])),
2836 types::I32 => Ok((&[RegClass::Int], &[types::I32])),
2837 types::I64 => Ok((&[RegClass::Int], &[types::I64])),
2838 types::F16 => Ok((&[RegClass::Float], &[types::F16])),
2839 types::F32 => Ok((&[RegClass::Float], &[types::F32])),
2840 types::F64 => Ok((&[RegClass::Float], &[types::F64])),
2841 types::F128 => Ok((&[RegClass::Float], &[types::F128])),
2842 types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
2843 _ if ty.is_vector() && ty.bits() <= 128 => {
2844 let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
2845 Ok((
2846 &[RegClass::Float],
2847 slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
2848 ))
2849 }
2850 _ => Err(CodegenError::Unsupported(format!(
2851 "Unexpected SSA-value type: {ty}"
2852 ))),
2853 }
2854 }
2855
2856 fn canonical_type_for_rc(rc: RegClass) -> Type {
2857 match rc {
2858 RegClass::Float => types::I8X16,
2859 RegClass::Int => types::I64,
2860 RegClass::Vector => unreachable!(),
2861 }
2862 }
2863
2864 fn gen_jump(label: MachLabel) -> Inst {
2865 Inst::jmp_known(label)
2866 }
2867
2868 fn gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self> {
2869 Some(Inst::imm(OperandSize::Size64, value, dst))
2870 }
2871
2872 fn gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
2873 let imm_to_gpr = Inst::imm(OperandSize::Size64, value.to_bits(), tmp);
2874 let gpr_to_xmm = Self::gpr_to_xmm(
2875 SseOpcode::Movd,
2876 tmp.to_reg().into(),
2877 OperandSize::Size64,
2878 dst,
2879 );
2880 smallvec![imm_to_gpr, gpr_to_xmm]
2881 }
2882
2883 fn gen_dummy_use(reg: Reg) -> Self {
2884 Inst::DummyUse { reg }
2885 }
2886
2887 fn worst_case_size() -> CodeOffset {
2888 15
2889 }
2890
2891 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
2892 RegClass::Int
2893 }
2894
2895 fn is_safepoint(&self) -> bool {
2896 match self {
2897 Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,
2898 _ => false,
2899 }
2900 }
2901
2902 fn function_alignment() -> FunctionAlignment {
2903 FunctionAlignment {
2904 minimum: 1,
2905 preferred: 32,
2908 }
2909 }
2910
2911 type LabelUse = LabelUse;
2912
2913 const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
2914}
2915
2916pub struct EmitInfo {
2918 pub(super) flags: settings::Flags,
2919 isa_flags: x64_settings::Flags,
2920}
2921
2922impl EmitInfo {
2923 pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
2925 Self { flags, isa_flags }
2926 }
2927}
2928
2929impl MachInstEmit for Inst {
2930 type State = EmitState;
2931 type Info = EmitInfo;
2932
2933 fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) {
2934 emit::emit(self, sink, info, state);
2935 }
2936
2937 fn pretty_print_inst(&self, _: &mut Self::State) -> String {
2938 PrettyPrint::pretty_print(self, 0)
2939 }
2940}
2941
2942#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2944pub enum LabelUse {
2945 JmpRel32,
2949
2950 PCRel32,
2953}
2954
2955impl MachInstLabelUse for LabelUse {
2956 const ALIGN: CodeOffset = 1;
2957
2958 fn max_pos_range(self) -> CodeOffset {
2959 match self {
2960 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
2961 }
2962 }
2963
2964 fn max_neg_range(self) -> CodeOffset {
2965 match self {
2966 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
2967 }
2968 }
2969
2970 fn patch_size(self) -> CodeOffset {
2971 match self {
2972 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
2973 }
2974 }
2975
2976 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2977 let pc_rel = (label_offset as i64) - (use_offset as i64);
2978 debug_assert!(pc_rel <= self.max_pos_range() as i64);
2979 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2980 let pc_rel = pc_rel as u32;
2981 match self {
2982 LabelUse::JmpRel32 => {
2983 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2984 let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
2985 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2986 }
2987 LabelUse::PCRel32 => {
2988 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2989 let value = pc_rel.wrapping_add(addend);
2990 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2991 }
2992 }
2993 }
2994
2995 fn supports_veneer(self) -> bool {
2996 match self {
2997 LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
2998 }
2999 }
3000
3001 fn veneer_size(self) -> CodeOffset {
3002 match self {
3003 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
3004 }
3005 }
3006
3007 fn worst_case_veneer_size() -> CodeOffset {
3008 0
3009 }
3010
3011 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
3012 match self {
3013 LabelUse::JmpRel32 | LabelUse::PCRel32 => {
3014 panic!("Veneer not supported for JumpRel32 label-use.");
3015 }
3016 }
3017 }
3018
3019 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
3020 match (reloc, addend) {
3021 (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
3022 _ => None,
3023 }
3024 }
3025}