1pub use emit_state::EmitState;
4
5use crate::binemit::{Addend, CodeOffset, Reloc};
6use crate::ir::{ExternalName, LibCall, TrapCode, Type, types};
7use crate::isa::x64::abi::X64ABIMachineSpec;
8use crate::isa::x64::inst::regs::{pretty_print_reg, show_ireg_sized};
9use crate::isa::x64::settings as x64_settings;
10use crate::isa::{CallConv, FunctionAlignment};
11use crate::{CodegenError, CodegenResult, settings};
12use crate::{machinst::*, trace};
13use alloc::boxed::Box;
14use core::slice;
15use cranelift_assembler_x64 as asm;
16use cranelift_entity::{Signed, Unsigned};
17use smallvec::{SmallVec, smallvec};
18use std::fmt::{self, Write};
19use std::string::{String, ToString};
20
21pub mod args;
22mod emit;
23mod emit_state;
24#[cfg(test)]
25mod emit_tests;
26pub mod external;
27pub mod regs;
28mod stack_switch;
29pub mod unwind;
30
31use args::*;
32
33pub use super::lower::isle::generated_code::AtomicRmwSeqOp;
38pub use super::lower::isle::generated_code::MInst as Inst;
39
40#[derive(Clone, Debug)]
42pub struct ReturnCallInfo<T> {
43 pub dest: T,
45
46 pub new_stack_arg_size: u32,
49
50 pub uses: CallArgList,
52
53 pub tmp: WritableGpr,
55}
56
57#[test]
58#[cfg(target_pointer_width = "64")]
59fn inst_size_test() {
60 assert_eq!(48, std::mem::size_of::<Inst>());
63}
64
65impl Inst {
66 fn available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]> {
71 match self {
72 Inst::AtomicRmwSeq { .. }
75 | Inst::CallKnown { .. }
76 | Inst::CallUnknown { .. }
77 | Inst::ReturnCallKnown { .. }
78 | Inst::ReturnCallUnknown { .. }
79 | Inst::CheckedSRemSeq { .. }
80 | Inst::CheckedSRemSeq8 { .. }
81 | Inst::CvtFloatToSintSeq { .. }
82 | Inst::CvtFloatToUintSeq { .. }
83 | Inst::CvtUint64ToFloatSeq { .. }
84 | Inst::JmpCond { .. }
85 | Inst::JmpCondOr { .. }
86 | Inst::WinchJmpIf { .. }
87 | Inst::JmpKnown { .. }
88 | Inst::JmpTableSeq { .. }
89 | Inst::LoadExtName { .. }
90 | Inst::MovFromPReg { .. }
91 | Inst::MovToPReg { .. }
92 | Inst::StackProbeLoop { .. }
93 | Inst::Args { .. }
94 | Inst::Rets { .. }
95 | Inst::StackSwitchBasic { .. }
96 | Inst::TrapIf { .. }
97 | Inst::TrapIfAnd { .. }
98 | Inst::TrapIfOr { .. }
99 | Inst::XmmCmove { .. }
100 | Inst::XmmMinMaxSeq { .. }
101 | Inst::XmmUninitializedValue { .. }
102 | Inst::GprUninitializedValue { .. }
103 | Inst::ElfTlsGetAddr { .. }
104 | Inst::MachOTlsGetAddr { .. }
105 | Inst::CoffTlsGetAddr { .. }
106 | Inst::Unwind { .. }
107 | Inst::DummyUse { .. } => smallvec![],
108
109 Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => {
110 smallvec![InstructionSet::CMPXCHG16b]
111 }
112
113 Inst::External { inst } => {
114 use cranelift_assembler_x64::Feature::*;
115 let mut features = smallvec![];
116 for f in inst.features() {
117 match f {
118 _64b | compat => {}
119 sse => features.push(InstructionSet::SSE),
120 sse2 => features.push(InstructionSet::SSE2),
121 sse3 => features.push(InstructionSet::SSE3),
122 ssse3 => features.push(InstructionSet::SSSE3),
123 sse41 => features.push(InstructionSet::SSE41),
124 sse42 => features.push(InstructionSet::SSE42),
125 bmi1 => features.push(InstructionSet::BMI1),
126 bmi2 => features.push(InstructionSet::BMI2),
127 lzcnt => features.push(InstructionSet::Lzcnt),
128 popcnt => features.push(InstructionSet::Popcnt),
129 avx => features.push(InstructionSet::AVX),
130 avx2 => features.push(InstructionSet::AVX2),
131 avx512f => features.push(InstructionSet::AVX512F),
132 avx512vl => features.push(InstructionSet::AVX512VL),
133 avx512dq => features.push(InstructionSet::AVX512DQ),
134 avx512bitalg => features.push(InstructionSet::AVX512BITALG),
135 avx512vbmi => features.push(InstructionSet::AVX512VBMI),
136 cmpxchg16b => features.push(InstructionSet::CMPXCHG16b),
137 fma => features.push(InstructionSet::FMA),
138 }
139 }
140 features
141 }
142 }
143 }
144}
145
146impl Inst {
149 pub(crate) fn nop(len: u8) -> Self {
150 assert!(len > 0 && len <= 9);
151 let inst = match len {
152 1 => asm::inst::nop_1b::new().into(),
153 2 => asm::inst::nop_2b::new().into(),
154 3 => asm::inst::nop_3b::new().into(),
155 4 => asm::inst::nop_4b::new().into(),
156 5 => asm::inst::nop_5b::new().into(),
157 6 => asm::inst::nop_6b::new().into(),
158 7 => asm::inst::nop_7b::new().into(),
159 8 => asm::inst::nop_8b::new().into(),
160 9 => asm::inst::nop_9b::new().into(),
161 _ => unreachable!("nop length must be between 1 and 9"),
162 };
163 Self::External { inst }
164 }
165
166 pub(crate) fn addq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
167 let inst = if let Ok(simm8) = i8::try_from(simm32) {
168 asm::inst::addq_mi_sxb::new(dst, simm8).into()
169 } else {
170 asm::inst::addq_mi_sxl::new(dst, simm32).into()
171 };
172 Inst::External { inst }
173 }
174
175 pub(crate) fn subq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
176 let inst = if let Ok(simm8) = i8::try_from(simm32) {
177 asm::inst::subq_mi_sxb::new(dst, simm8).into()
178 } else {
179 asm::inst::subq_mi_sxl::new(dst, simm32).into()
180 };
181 Inst::External { inst }
182 }
183
184 pub fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
189 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
190 debug_assert!(dst.to_reg().class() == RegClass::Int);
191 let dst = WritableGpr::from_writable_reg(dst).unwrap();
192 let inst = match dst_size {
193 OperandSize::Size64 => match u32::try_from(simm64) {
194 Ok(imm32) => asm::inst::movl_oi::new(dst, imm32).into(),
197 _ => match i32::try_from(simm64.signed()) {
198 Ok(simm32) => asm::inst::movq_mi_sxl::new(dst, simm32).into(),
201 _ => asm::inst::movabsq_oi::new(dst, simm64).into(),
203 },
204 },
205 _ => asm::inst::movl_oi::new(dst, simm64 as u32).into(),
217 };
218 Inst::External { inst }
219 }
220
221 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
222 src.assert_regclass_is(RegClass::Int);
223 debug_assert!(dst.to_reg().class() == RegClass::Int);
224 let src = match src {
225 RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
226 RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
227 };
228 let inst = match ext_mode {
229 ExtMode::BL => asm::inst::movzbl_rm::new(dst, src).into(),
230 ExtMode::BQ => asm::inst::movzbq_rm::new(dst, src).into(),
231 ExtMode::WL => asm::inst::movzwl_rm::new(dst, src).into(),
232 ExtMode::WQ => asm::inst::movzwq_rm::new(dst, src).into(),
233 ExtMode::LQ => {
234 asm::inst::movl_rm::new(dst, src).into()
241 }
242 };
243 Inst::External { inst }
244 }
245
246 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
247 src.assert_regclass_is(RegClass::Int);
248 debug_assert!(dst.to_reg().class() == RegClass::Int);
249 let src = match src {
250 RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
251 RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
252 };
253 let inst = match ext_mode {
254 ExtMode::BL => asm::inst::movsbl_rm::new(dst, src).into(),
255 ExtMode::BQ => asm::inst::movsbq_rm::new(dst, src).into(),
256 ExtMode::WL => asm::inst::movswl_rm::new(dst, src).into(),
257 ExtMode::WQ => asm::inst::movswq_rm::new(dst, src).into(),
258 ExtMode::LQ => asm::inst::movslq_rm::new(dst, src).into(),
259 };
260 Inst::External { inst }
261 }
262
263 pub(crate) fn cmp_mi_sxb(size: OperandSize, src1: Gpr, src2: i8) -> Inst {
265 let inst = match size {
266 OperandSize::Size8 => asm::inst::cmpb_mi::new(src1, src2.unsigned()).into(),
267 OperandSize::Size16 => asm::inst::cmpw_mi_sxb::new(src1, src2).into(),
268 OperandSize::Size32 => asm::inst::cmpl_mi_sxb::new(src1, src2).into(),
269 OperandSize::Size64 => asm::inst::cmpq_mi_sxb::new(src1, src2).into(),
270 };
271 Inst::External { inst }
272 }
273
274 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
275 Inst::TrapIf { cc, trap_code }
276 }
277
278 pub(crate) fn call_known(info: Box<CallInfo<ExternalName>>) -> Inst {
279 Inst::CallKnown { info }
280 }
281
282 pub(crate) fn call_unknown(info: Box<CallInfo<RegMem>>) -> Inst {
283 info.dest.assert_regclass_is(RegClass::Int);
284 Inst::CallUnknown { info }
285 }
286
287 pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
288 Inst::JmpKnown { dst }
289 }
290
291 pub(crate) fn load(
295 ty: Type,
296 from_addr: impl Into<SyntheticAmode>,
297 to_reg: Writable<Reg>,
298 ext_kind: ExtKind,
299 ) -> Inst {
300 let rc = to_reg.to_reg().class();
301 match rc {
302 RegClass::Int => {
303 let ext_mode = match ty.bytes() {
304 1 => Some(ExtMode::BQ),
305 2 => Some(ExtMode::WQ),
306 4 => Some(ExtMode::LQ),
307 8 => None,
308 _ => unreachable!("the type should never use a scalar load: {}", ty),
309 };
310 if let Some(ext_mode) = ext_mode {
311 match ext_kind {
313 ExtKind::SignExtend => {
314 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
315 }
316 ExtKind::ZeroExtend => {
317 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
318 }
319 ExtKind::None => {
320 panic!("expected an extension kind for extension mode: {ext_mode:?}")
321 }
322 }
323 } else {
324 let from_addr = asm::GprMem::from(from_addr.into());
326 Inst::External {
327 inst: asm::inst::movq_rm::new(to_reg, from_addr).into(),
328 }
329 }
330 }
331 RegClass::Float => {
332 let to_reg = to_reg.map(|r| Xmm::new(r).unwrap());
333 let from_addr = from_addr.into();
334 let inst = match ty {
335 types::F16 | types::I8X2 => {
336 panic!("loading a f16 or i8x2 requires multiple instructions")
337 }
338 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => {
339 asm::inst::movss_a_m::new(to_reg, from_addr).into()
340 }
341 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => {
342 asm::inst::movsd_a_m::new(to_reg, from_addr).into()
343 }
344 types::F32X4 => asm::inst::movups_a::new(to_reg, from_addr).into(),
345 types::F64X2 => asm::inst::movupd_a::new(to_reg, from_addr).into(),
346 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => {
347 asm::inst::movdqu_a::new(to_reg, from_addr).into()
348 }
349 _ => unimplemented!("unable to load type: {}", ty),
350 };
351 Inst::External { inst }
352 }
353 RegClass::Vector => unreachable!(),
354 }
355 }
356
357 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
359 let rc = from_reg.class();
360 let to_addr = to_addr.into();
361 let inst = match rc {
362 RegClass::Int => {
363 let from_reg = Gpr::unwrap_new(from_reg);
364 match ty {
365 types::I8 => asm::inst::movb_mr::new(to_addr, from_reg).into(),
366 types::I16 => asm::inst::movw_mr::new(to_addr, from_reg).into(),
367 types::I32 => asm::inst::movl_mr::new(to_addr, from_reg).into(),
368 types::I64 => asm::inst::movq_mr::new(to_addr, from_reg).into(),
369 _ => unreachable!(),
370 }
371 }
372 RegClass::Float => {
373 let from_reg = Xmm::new(from_reg).unwrap();
374 match ty {
375 types::F16 | types::I8X2 => {
376 panic!("storing a f16 or i8x2 requires multiple instructions")
377 }
378 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => {
379 asm::inst::movss_c_m::new(to_addr, from_reg).into()
380 }
381 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => {
382 asm::inst::movsd_c_m::new(to_addr, from_reg).into()
383 }
384 types::F32X4 => asm::inst::movups_b::new(to_addr, from_reg).into(),
385 types::F64X2 => asm::inst::movupd_b::new(to_addr, from_reg).into(),
386 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => {
387 asm::inst::movdqu_b::new(to_addr, from_reg).into()
388 }
389 _ => unimplemented!("unable to store type: {}", ty),
390 }
391 }
392 RegClass::Vector => unreachable!(),
393 };
394 Inst::External { inst }
395 }
396}
397
398impl PrettyPrint for Inst {
402 fn pretty_print(&self, _size: u8) -> String {
403 fn ljustify(s: String) -> String {
404 let w = 7;
405 if s.len() >= w {
406 s
407 } else {
408 let need = usize::min(w, w - s.len());
409 s + &format!("{nil: <width$}", nil = "", width = need)
410 }
411 }
412
413 fn ljustify2(s1: String, s2: String) -> String {
414 ljustify(s1 + &s2)
415 }
416
417 match self {
418 Inst::CheckedSRemSeq {
419 size,
420 divisor,
421 dividend_lo,
422 dividend_hi,
423 dst_quotient,
424 dst_remainder,
425 } => {
426 let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes());
427 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
428 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
429 let dst_quotient =
430 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
431 let dst_remainder =
432 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
433 format!(
434 "checked_srem_seq {dividend_lo}, {dividend_hi}, \
435 {divisor}, {dst_quotient}, {dst_remainder}",
436 )
437 }
438
439 Inst::CheckedSRemSeq8 {
440 divisor,
441 dividend,
442 dst,
443 } => {
444 let divisor = pretty_print_reg(divisor.to_reg(), 1);
445 let dividend = pretty_print_reg(dividend.to_reg(), 1);
446 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
447 format!("checked_srem_seq {dividend}, {divisor}, {dst}")
448 }
449
450 Inst::XmmMinMaxSeq {
451 lhs,
452 rhs,
453 dst,
454 is_min,
455 size,
456 } => {
457 let rhs = pretty_print_reg(rhs.to_reg(), 8);
458 let lhs = pretty_print_reg(lhs.to_reg(), 8);
459 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
460 let op = ljustify2(
461 if *is_min {
462 "xmm min seq ".to_string()
463 } else {
464 "xmm max seq ".to_string()
465 },
466 format!("f{}", size.to_bits()),
467 );
468 format!("{op} {lhs}, {rhs}, {dst}")
469 }
470
471 Inst::XmmUninitializedValue { dst } => {
472 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
473 let op = ljustify("uninit".into());
474 format!("{op} {dst}")
475 }
476
477 Inst::GprUninitializedValue { dst } => {
478 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
479 let op = ljustify("uninit".into());
480 format!("{op} {dst}")
481 }
482
483 Inst::CvtUint64ToFloatSeq {
484 src,
485 dst,
486 dst_size,
487 tmp_gpr1,
488 tmp_gpr2,
489 ..
490 } => {
491 let src = pretty_print_reg(src.to_reg(), 8);
492 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
493 let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8);
494 let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8);
495 let op = ljustify(format!(
496 "u64_to_{}_seq",
497 if *dst_size == OperandSize::Size64 {
498 "f64"
499 } else {
500 "f32"
501 }
502 ));
503 format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}")
504 }
505
506 Inst::CvtFloatToSintSeq {
507 src,
508 dst,
509 src_size,
510 dst_size,
511 tmp_xmm,
512 tmp_gpr,
513 is_saturating,
514 } => {
515 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
516 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
517 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
518 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
519 let op = ljustify(format!(
520 "cvt_float{}_to_sint{}{}_seq",
521 src_size.to_bits(),
522 dst_size.to_bits(),
523 if *is_saturating { "_sat" } else { "" },
524 ));
525 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}")
526 }
527
528 Inst::CvtFloatToUintSeq {
529 src,
530 dst,
531 src_size,
532 dst_size,
533 tmp_gpr,
534 tmp_xmm,
535 tmp_xmm2,
536 is_saturating,
537 } => {
538 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
539 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
540 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
541 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
542 let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8);
543 let op = ljustify(format!(
544 "cvt_float{}_to_uint{}{}_seq",
545 src_size.to_bits(),
546 dst_size.to_bits(),
547 if *is_saturating { "_sat" } else { "" },
548 ));
549 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}, {tmp_xmm2}")
550 }
551
552 Inst::MovFromPReg { src, dst } => {
553 let src: Reg = (*src).into();
554 let src = regs::show_ireg_sized(src, 8);
555 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
556 let op = ljustify("movq".to_string());
557 format!("{op} {src}, {dst}")
558 }
559
560 Inst::MovToPReg { src, dst } => {
561 let src = pretty_print_reg(src.to_reg(), 8);
562 let dst: Reg = (*dst).into();
563 let dst = regs::show_ireg_sized(dst, 8);
564 let op = ljustify("movq".to_string());
565 format!("{op} {src}, {dst}")
566 }
567
568 Inst::XmmCmove {
569 ty,
570 cc,
571 consequent,
572 alternative,
573 dst,
574 ..
575 } => {
576 let size = u8::try_from(ty.bytes()).unwrap();
577 let alternative = pretty_print_reg(alternative.to_reg(), size);
578 let dst = pretty_print_reg(dst.to_reg().to_reg(), size);
579 let consequent = pretty_print_reg(consequent.to_reg(), size);
580 let suffix = match *ty {
581 types::F64 => "sd",
582 types::F32 => "ss",
583 types::F16 => "ss",
584 types::F32X4 => "aps",
585 types::F64X2 => "apd",
586 _ => "dqa",
587 };
588 let cc = cc.invert();
589 format!(
590 "mov{suffix} {alternative}, {dst}; \
591 j{cc} $next; \
592 mov{suffix} {consequent}, {dst}; \
593 $next:"
594 )
595 }
596
597 Inst::StackProbeLoop {
598 tmp,
599 frame_size,
600 guard_size,
601 } => {
602 let tmp = pretty_print_reg(tmp.to_reg(), 8);
603 let op = ljustify("stack_probe_loop".to_string());
604 format!("{op} {tmp}, frame_size={frame_size}, guard_size={guard_size}")
605 }
606
607 Inst::CallKnown { info } => {
608 let op = ljustify("call".to_string());
609 let try_call = info
610 .try_call_info
611 .as_ref()
612 .map(|tci| pretty_print_try_call(tci))
613 .unwrap_or_default();
614 format!("{op} {:?}{try_call}", info.dest)
615 }
616
617 Inst::CallUnknown { info } => {
618 let dest = info.dest.pretty_print(8);
619 let op = ljustify("call".to_string());
620 let try_call = info
621 .try_call_info
622 .as_ref()
623 .map(|tci| pretty_print_try_call(tci))
624 .unwrap_or_default();
625 format!("{op} *{dest}{try_call}")
626 }
627
628 Inst::ReturnCallKnown { info } => {
629 let ReturnCallInfo {
630 uses,
631 new_stack_arg_size,
632 tmp,
633 dest,
634 } = &**info;
635 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
636 let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}");
637 for ret in uses {
638 let preg = regs::show_reg(ret.preg);
639 let vreg = pretty_print_reg(ret.vreg, 8);
640 write!(&mut s, " {vreg}={preg}").unwrap();
641 }
642 s
643 }
644
645 Inst::ReturnCallUnknown { info } => {
646 let ReturnCallInfo {
647 uses,
648 new_stack_arg_size,
649 tmp,
650 dest,
651 } = &**info;
652 let callee = pretty_print_reg(*dest, 8);
653 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
654 let mut s =
655 format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}");
656 for ret in uses {
657 let preg = regs::show_reg(ret.preg);
658 let vreg = pretty_print_reg(ret.vreg, 8);
659 write!(&mut s, " {vreg}={preg}").unwrap();
660 }
661 s
662 }
663
664 Inst::Args { args } => {
665 let mut s = "args".to_string();
666 for arg in args {
667 let preg = regs::show_reg(arg.preg);
668 let def = pretty_print_reg(arg.vreg.to_reg(), 8);
669 write!(&mut s, " {def}={preg}").unwrap();
670 }
671 s
672 }
673
674 Inst::Rets { rets } => {
675 let mut s = "rets".to_string();
676 for ret in rets {
677 let preg = regs::show_reg(ret.preg);
678 let vreg = pretty_print_reg(ret.vreg, 8);
679 write!(&mut s, " {vreg}={preg}").unwrap();
680 }
681 s
682 }
683
684 Inst::StackSwitchBasic {
685 store_context_ptr,
686 load_context_ptr,
687 in_payload0,
688 out_payload0,
689 } => {
690 let store_context_ptr = pretty_print_reg(**store_context_ptr, 8);
691 let load_context_ptr = pretty_print_reg(**load_context_ptr, 8);
692 let in_payload0 = pretty_print_reg(**in_payload0, 8);
693 let out_payload0 = pretty_print_reg(*out_payload0.to_reg(), 8);
694 format!(
695 "{out_payload0} = stack_switch_basic {store_context_ptr}, {load_context_ptr}, {in_payload0}"
696 )
697 }
698
699 Inst::JmpKnown { dst } => {
700 let op = ljustify("jmp".to_string());
701 let dst = dst.to_string();
702 format!("{op} {dst}")
703 }
704
705 Inst::WinchJmpIf { cc, taken } => {
706 let taken = taken.to_string();
707 let op = ljustify2("j".to_string(), cc.to_string());
708 format!("{op} {taken}")
709 }
710
711 Inst::JmpCondOr {
712 cc1,
713 cc2,
714 taken,
715 not_taken,
716 } => {
717 let taken = taken.to_string();
718 let not_taken = not_taken.to_string();
719 let op = ljustify(format!("j{cc1},{cc2}"));
720 format!("{op} {taken}; j {not_taken}")
721 }
722
723 Inst::JmpCond {
724 cc,
725 taken,
726 not_taken,
727 } => {
728 let taken = taken.to_string();
729 let not_taken = not_taken.to_string();
730 let op = ljustify2("j".to_string(), cc.to_string());
731 format!("{op} {taken}; j {not_taken}")
732 }
733
734 Inst::JmpTableSeq {
735 idx, tmp1, tmp2, ..
736 } => {
737 let idx = pretty_print_reg(*idx, 8);
738 let tmp1 = pretty_print_reg(tmp1.to_reg(), 8);
739 let tmp2 = pretty_print_reg(tmp2.to_reg(), 8);
740 let op = ljustify("br_table".into());
741 format!("{op} {idx}, {tmp1}, {tmp2}")
742 }
743
744 Inst::TrapIf { cc, trap_code, .. } => {
745 format!("j{cc} #trap={trap_code}")
746 }
747
748 Inst::TrapIfAnd {
749 cc1,
750 cc2,
751 trap_code,
752 ..
753 } => {
754 let cc1 = cc1.invert();
755 let cc2 = cc2.invert();
756 format!("trap_if_and {cc1}, {cc2}, {trap_code}")
757 }
758
759 Inst::TrapIfOr {
760 cc1,
761 cc2,
762 trap_code,
763 ..
764 } => {
765 let cc2 = cc2.invert();
766 format!("trap_if_or {cc1}, {cc2}, {trap_code}")
767 }
768
769 Inst::LoadExtName {
770 dst, name, offset, ..
771 } => {
772 let dst = pretty_print_reg(*dst.to_reg(), 8);
773 let name = name.display(None);
774 let op = ljustify("load_ext_name".into());
775 format!("{op} {name}+{offset}, {dst}")
776 }
777
778 Inst::AtomicRmwSeq { ty, op, .. } => {
779 let ty = ty.bits();
780 format!(
781 "atomically {{ {ty}_bits_at_[%r9] {op:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}"
782 )
783 }
784
785 Inst::Atomic128RmwSeq {
786 op,
787 mem,
788 operand_low,
789 operand_high,
790 temp_low,
791 temp_high,
792 dst_old_low,
793 dst_old_high,
794 } => {
795 let operand_low = pretty_print_reg(**operand_low, 8);
796 let operand_high = pretty_print_reg(**operand_high, 8);
797 let temp_low = pretty_print_reg(*temp_low.to_reg(), 8);
798 let temp_high = pretty_print_reg(*temp_high.to_reg(), 8);
799 let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
800 let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
801 let mem = mem.pretty_print(16);
802 format!(
803 "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem} = {temp_high}:{temp_low} }}"
804 )
805 }
806
807 Inst::Atomic128XchgSeq {
808 mem,
809 operand_low,
810 operand_high,
811 dst_old_low,
812 dst_old_high,
813 } => {
814 let operand_low = pretty_print_reg(**operand_low, 8);
815 let operand_high = pretty_print_reg(**operand_high, 8);
816 let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
817 let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
818 let mem = mem.pretty_print(16);
819 format!(
820 "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {mem} = {operand_high}:{operand_low} }}"
821 )
822 }
823
824 Inst::ElfTlsGetAddr { symbol, dst } => {
825 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
826 format!("{dst} = elf_tls_get_addr {symbol:?}")
827 }
828
829 Inst::MachOTlsGetAddr { symbol, dst } => {
830 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
831 format!("{dst} = macho_tls_get_addr {symbol:?}")
832 }
833
834 Inst::CoffTlsGetAddr { symbol, dst, tmp } => {
835 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
836 let tmp = tmp.to_reg().to_reg();
837
838 let mut s = format!("{dst} = coff_tls_get_addr {symbol:?}");
839 if tmp.is_virtual() {
840 let tmp = show_ireg_sized(tmp, 8);
841 write!(&mut s, ", {tmp}").unwrap();
842 };
843
844 s
845 }
846
847 Inst::Unwind { inst } => format!("unwind {inst:?}"),
848
849 Inst::DummyUse { reg } => {
850 let reg = pretty_print_reg(*reg, 8);
851 format!("dummy_use {reg}")
852 }
853
854 Inst::External { inst } => {
855 format!("{inst}")
856 }
857 }
858 }
859}
860
861fn pretty_print_try_call(info: &TryCallInfo) -> String {
862 format!(
863 "; jmp {:?}; catch [{}]",
864 info.continuation,
865 info.pretty_print_dests()
866 )
867}
868
869impl fmt::Debug for Inst {
870 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
871 write!(fmt, "{}", self.pretty_print_inst(&mut Default::default()))
872 }
873}
874
875fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
876 match inst {
886 Inst::CheckedSRemSeq {
887 divisor,
888 dividend_lo,
889 dividend_hi,
890 dst_quotient,
891 dst_remainder,
892 ..
893 } => {
894 collector.reg_use(divisor);
895 collector.reg_fixed_use(dividend_lo, regs::rax());
896 collector.reg_fixed_use(dividend_hi, regs::rdx());
897 collector.reg_fixed_def(dst_quotient, regs::rax());
898 collector.reg_fixed_def(dst_remainder, regs::rdx());
899 }
900 Inst::CheckedSRemSeq8 {
901 divisor,
902 dividend,
903 dst,
904 ..
905 } => {
906 collector.reg_use(divisor);
907 collector.reg_fixed_use(dividend, regs::rax());
908 collector.reg_fixed_def(dst, regs::rax());
909 }
910 Inst::XmmUninitializedValue { dst } => collector.reg_def(dst),
911 Inst::GprUninitializedValue { dst } => collector.reg_def(dst),
912 Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
913 collector.reg_use(rhs);
914 collector.reg_use(lhs);
915 collector.reg_reuse_def(dst, 0); }
917 Inst::MovFromPReg { dst, src } => {
918 debug_assert!(dst.to_reg().to_reg().is_virtual());
919 collector.reg_fixed_nonallocatable(*src);
920 collector.reg_def(dst);
921 }
922 Inst::MovToPReg { dst, src } => {
923 debug_assert!(src.to_reg().is_virtual());
924 collector.reg_use(src);
925 collector.reg_fixed_nonallocatable(*dst);
926 }
927 Inst::CvtUint64ToFloatSeq {
928 src,
929 dst,
930 tmp_gpr1,
931 tmp_gpr2,
932 ..
933 } => {
934 collector.reg_use(src);
935 collector.reg_early_def(dst);
936 collector.reg_early_def(tmp_gpr1);
937 collector.reg_early_def(tmp_gpr2);
938 }
939 Inst::CvtFloatToSintSeq {
940 src,
941 dst,
942 tmp_xmm,
943 tmp_gpr,
944 ..
945 } => {
946 collector.reg_use(src);
947 collector.reg_early_def(dst);
948 collector.reg_early_def(tmp_gpr);
949 collector.reg_early_def(tmp_xmm);
950 }
951 Inst::CvtFloatToUintSeq {
952 src,
953 dst,
954 tmp_gpr,
955 tmp_xmm,
956 tmp_xmm2,
957 ..
958 } => {
959 collector.reg_use(src);
960 collector.reg_early_def(dst);
961 collector.reg_early_def(tmp_gpr);
962 collector.reg_early_def(tmp_xmm);
963 collector.reg_early_def(tmp_xmm2);
964 }
965
966 Inst::XmmCmove {
967 consequent,
968 alternative,
969 dst,
970 ..
971 } => {
972 collector.reg_use(alternative);
973 collector.reg_reuse_def(dst, 0);
974 collector.reg_use(consequent);
975 }
976 Inst::StackProbeLoop { tmp, .. } => {
977 collector.reg_early_def(tmp);
978 }
979
980 Inst::CallKnown { info } => {
981 let CallInfo {
986 uses,
987 defs,
988 clobbers,
989 dest,
990 try_call_info,
991 ..
992 } = &mut **info;
993 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
994 for CallArgPair { vreg, preg } in uses {
995 collector.reg_fixed_use(vreg, *preg);
996 }
997 for CallRetPair { vreg, location } in defs {
998 match location {
999 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
1000 RetLocation::Stack(..) => collector.any_def(vreg),
1001 }
1002 }
1003 collector.reg_clobbers(*clobbers);
1004 if let Some(try_call_info) = try_call_info {
1005 try_call_info.collect_operands(collector);
1006 }
1007 }
1008
1009 Inst::CallUnknown { info } => {
1010 let CallInfo {
1011 uses,
1012 defs,
1013 clobbers,
1014 callee_conv,
1015 dest,
1016 try_call_info,
1017 ..
1018 } = &mut **info;
1019 match dest {
1020 RegMem::Reg { reg } if *callee_conv == CallConv::Winch => {
1021 collector.reg_fixed_use(reg, regs::r10());
1025 }
1026 _ => dest.get_operands(collector),
1027 }
1028 for CallArgPair { vreg, preg } in uses {
1029 collector.reg_fixed_use(vreg, *preg);
1030 }
1031 for CallRetPair { vreg, location } in defs {
1032 match location {
1033 RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
1034 RetLocation::Stack(..) => collector.any_def(vreg),
1035 }
1036 }
1037 collector.reg_clobbers(*clobbers);
1038 if let Some(try_call_info) = try_call_info {
1039 try_call_info.collect_operands(collector);
1040 }
1041 }
1042 Inst::StackSwitchBasic {
1043 store_context_ptr,
1044 load_context_ptr,
1045 in_payload0,
1046 out_payload0,
1047 } => {
1048 collector.reg_use(load_context_ptr);
1049 collector.reg_use(store_context_ptr);
1050 collector.reg_fixed_use(in_payload0, stack_switch::payload_register());
1051 collector.reg_fixed_def(out_payload0, stack_switch::payload_register());
1052
1053 let mut clobbers = crate::isa::x64::abi::ALL_CLOBBERS;
1054 clobbers.remove(
1056 stack_switch::payload_register()
1057 .to_real_reg()
1058 .unwrap()
1059 .into(),
1060 );
1061 collector.reg_clobbers(clobbers);
1062 }
1063
1064 Inst::ReturnCallKnown { info } => {
1065 let ReturnCallInfo {
1066 dest, uses, tmp, ..
1067 } = &mut **info;
1068 collector.reg_fixed_def(tmp, regs::r11());
1069 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
1071 for CallArgPair { vreg, preg } in uses {
1072 collector.reg_fixed_use(vreg, *preg);
1073 }
1074 }
1075
1076 Inst::ReturnCallUnknown { info } => {
1077 let ReturnCallInfo {
1078 dest, uses, tmp, ..
1079 } = &mut **info;
1080
1081 collector.reg_fixed_use(dest, regs::r10());
1087
1088 collector.reg_fixed_def(tmp, regs::r11());
1089 for CallArgPair { vreg, preg } in uses {
1090 collector.reg_fixed_use(vreg, *preg);
1091 }
1092 }
1093
1094 Inst::JmpTableSeq {
1095 idx, tmp1, tmp2, ..
1096 } => {
1097 collector.reg_use(idx);
1098 collector.reg_early_def(tmp1);
1099 collector.reg_def(tmp2);
1103 }
1104
1105 Inst::LoadExtName { dst, .. } => {
1106 collector.reg_def(dst);
1107 }
1108
1109 Inst::AtomicRmwSeq {
1110 operand,
1111 temp,
1112 dst_old,
1113 mem,
1114 ..
1115 } => {
1116 collector.reg_late_use(operand);
1117 collector.reg_early_def(temp);
1118 collector.reg_fixed_def(dst_old, regs::rax());
1121 mem.get_operands_late(collector)
1122 }
1123
1124 Inst::Atomic128RmwSeq {
1125 operand_low,
1126 operand_high,
1127 temp_low,
1128 temp_high,
1129 dst_old_low,
1130 dst_old_high,
1131 mem,
1132 ..
1133 } => {
1134 collector.reg_late_use(operand_low);
1136 collector.reg_late_use(operand_high);
1137 collector.reg_fixed_def(temp_low, regs::rbx());
1138 collector.reg_fixed_def(temp_high, regs::rcx());
1139 collector.reg_fixed_def(dst_old_low, regs::rax());
1140 collector.reg_fixed_def(dst_old_high, regs::rdx());
1141 mem.get_operands_late(collector)
1142 }
1143
1144 Inst::Atomic128XchgSeq {
1145 operand_low,
1146 operand_high,
1147 dst_old_low,
1148 dst_old_high,
1149 mem,
1150 ..
1151 } => {
1152 collector.reg_fixed_late_use(operand_low, regs::rbx());
1154 collector.reg_fixed_late_use(operand_high, regs::rcx());
1155 collector.reg_fixed_def(dst_old_low, regs::rax());
1156 collector.reg_fixed_def(dst_old_high, regs::rdx());
1157 mem.get_operands_late(collector)
1158 }
1159
1160 Inst::Args { args } => {
1161 for ArgPair { vreg, preg } in args {
1162 collector.reg_fixed_def(vreg, *preg);
1163 }
1164 }
1165
1166 Inst::Rets { rets } => {
1167 for RetPair { vreg, preg } in rets {
1170 collector.reg_fixed_use(vreg, *preg);
1171 }
1172 }
1173
1174 Inst::JmpKnown { .. }
1175 | Inst::WinchJmpIf { .. }
1176 | Inst::JmpCond { .. }
1177 | Inst::JmpCondOr { .. }
1178 | Inst::TrapIf { .. }
1179 | Inst::TrapIfAnd { .. }
1180 | Inst::TrapIfOr { .. } => {
1181 }
1183
1184 Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
1185 collector.reg_fixed_def(dst, regs::rax());
1186 let mut clobbers =
1193 X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false);
1194 clobbers.remove(regs::gpr_preg(regs::ENC_RAX));
1195 collector.reg_clobbers(clobbers);
1196 }
1197
1198 Inst::CoffTlsGetAddr { dst, tmp, .. } => {
1199 collector.reg_fixed_def(dst, regs::rax());
1204
1205 collector.reg_fixed_def(tmp, regs::rcx());
1207 }
1208
1209 Inst::Unwind { .. } => {}
1210
1211 Inst::DummyUse { reg } => {
1212 collector.reg_use(reg);
1213 }
1214
1215 Inst::External { inst } => {
1216 inst.visit(&mut external::RegallocVisitor { collector });
1217 }
1218 }
1219}
1220
1221impl MachInst for Inst {
1225 type ABIMachineSpec = X64ABIMachineSpec;
1226
1227 fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
1228 x64_get_operands(self, collector)
1229 }
1230
1231 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
1232 use asm::inst::Inst as I;
1233 match self {
1234 Self::External {
1239 inst: I::movq_mr(asm::inst::movq_mr { rm64, r64 }),
1240 } => match rm64 {
1241 asm::GprMem::Gpr(reg) => Some((reg.map(|r| r.to_reg()), r64.as_ref().to_reg())),
1242 asm::GprMem::Mem(_) => None,
1243 },
1244 Self::External {
1245 inst: I::movq_rm(asm::inst::movq_rm { r64, rm64 }),
1246 } => match rm64 {
1247 asm::GprMem::Gpr(reg) => Some((r64.as_ref().map(|r| r.to_reg()), reg.to_reg())),
1248 asm::GprMem::Mem(_) => None,
1249 },
1250
1251 Self::External {
1260 inst:
1261 I::movaps_a(asm::inst::movaps_a { xmm1, xmm_m128 })
1262 | I::movups_a(asm::inst::movups_a { xmm1, xmm_m128 })
1263 | I::movapd_a(asm::inst::movapd_a { xmm1, xmm_m128 })
1264 | I::movupd_a(asm::inst::movupd_a { xmm1, xmm_m128 })
1265 | I::movdqa_a(asm::inst::movdqa_a { xmm1, xmm_m128 })
1266 | I::movdqu_a(asm::inst::movdqu_a { xmm1, xmm_m128 }),
1267 } => match xmm_m128 {
1268 asm::XmmMem::Xmm(xmm2) => Some((xmm1.as_ref().map(|r| r.to_reg()), xmm2.to_reg())),
1269 asm::XmmMem::Mem(_) => None,
1270 },
1271 Self::External {
1275 inst:
1276 I::movaps_b(asm::inst::movaps_b { xmm_m128, xmm1 })
1277 | I::movups_b(asm::inst::movups_b { xmm_m128, xmm1 })
1278 | I::movapd_b(asm::inst::movapd_b { xmm_m128, xmm1 })
1279 | I::movupd_b(asm::inst::movupd_b { xmm_m128, xmm1 })
1280 | I::movdqa_b(asm::inst::movdqa_b { xmm_m128, xmm1 })
1281 | I::movdqu_b(asm::inst::movdqu_b { xmm_m128, xmm1 }),
1282 } => match xmm_m128 {
1283 asm::XmmMem::Xmm(dst) => Some((dst.map(|r| r.to_reg()), xmm1.as_ref().to_reg())),
1284 asm::XmmMem::Mem(_) => None,
1285 },
1286 _ => None,
1287 }
1288 }
1289
1290 fn is_included_in_clobbers(&self) -> bool {
1291 match self {
1292 &Inst::Args { .. } => false,
1293 _ => true,
1294 }
1295 }
1296
1297 fn is_trap(&self) -> bool {
1298 match self {
1299 Self::External {
1300 inst: asm::inst::Inst::ud2_zo(..),
1301 } => true,
1302 _ => false,
1303 }
1304 }
1305
1306 fn is_args(&self) -> bool {
1307 match self {
1308 Self::Args { .. } => true,
1309 _ => false,
1310 }
1311 }
1312
1313 fn is_term(&self) -> MachTerminator {
1314 match self {
1315 &Self::Rets { .. } => MachTerminator::Ret,
1317 &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => {
1318 MachTerminator::RetCall
1319 }
1320 &Self::JmpKnown { .. } => MachTerminator::Branch,
1321 &Self::JmpCond { .. } => MachTerminator::Branch,
1322 &Self::JmpCondOr { .. } => MachTerminator::Branch,
1323 &Self::JmpTableSeq { .. } => MachTerminator::Branch,
1324 &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1325 &Self::CallUnknown { ref info } if info.try_call_info.is_some() => {
1326 MachTerminator::Branch
1327 }
1328 _ => MachTerminator::None,
1330 }
1331 }
1332
1333 fn is_low_level_branch(&self) -> bool {
1334 match self {
1335 &Self::WinchJmpIf { .. } => true,
1336 _ => false,
1337 }
1338 }
1339
1340 fn is_mem_access(&self) -> bool {
1341 panic!("TODO FILL ME OUT")
1342 }
1343
1344 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
1345 trace!(
1346 "Inst::gen_move {:?} -> {:?} (type: {:?})",
1347 src_reg,
1348 dst_reg.to_reg(),
1349 ty
1350 );
1351 let rc_dst = dst_reg.to_reg().class();
1352 let rc_src = src_reg.class();
1353 debug_assert!(rc_dst == rc_src);
1355 let inst = match rc_dst {
1356 RegClass::Int => {
1357 asm::inst::movq_mr::new(dst_reg.map(Gpr::unwrap_new), Gpr::unwrap_new(src_reg))
1358 .into()
1359 }
1360 RegClass::Float => {
1361 let dst_reg = dst_reg.map(|r| Xmm::new(r).unwrap());
1366 let src_reg = Xmm::new(src_reg).unwrap();
1367 match ty {
1368 types::F16 | types::F32 | types::F64 | types::F32X4 => {
1369 asm::inst::movaps_a::new(dst_reg, src_reg).into()
1370 }
1371 types::F64X2 => asm::inst::movapd_a::new(dst_reg, src_reg).into(),
1372 _ if (ty.is_float() || ty.is_vector()) && ty.bits() <= 128 => {
1373 asm::inst::movdqa_a::new(dst_reg, src_reg).into()
1374 }
1375 _ => unimplemented!("unable to move type: {}", ty),
1376 }
1377 }
1378 RegClass::Vector => unreachable!(),
1379 };
1380 Inst::External { inst }
1381 }
1382
1383 fn gen_nop(preferred_size: usize) -> Inst {
1384 Inst::nop(std::cmp::min(preferred_size, 9) as u8)
1385 }
1386
1387 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
1388 match ty {
1389 types::I8 => Ok((&[RegClass::Int], &[types::I8])),
1390 types::I16 => Ok((&[RegClass::Int], &[types::I16])),
1391 types::I32 => Ok((&[RegClass::Int], &[types::I32])),
1392 types::I64 => Ok((&[RegClass::Int], &[types::I64])),
1393 types::F16 => Ok((&[RegClass::Float], &[types::F16])),
1394 types::F32 => Ok((&[RegClass::Float], &[types::F32])),
1395 types::F64 => Ok((&[RegClass::Float], &[types::F64])),
1396 types::F128 => Ok((&[RegClass::Float], &[types::F128])),
1397 types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
1398 _ if ty.is_vector() && ty.bits() <= 128 => {
1399 let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1400 Ok((
1401 &[RegClass::Float],
1402 slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1403 ))
1404 }
1405 _ => Err(CodegenError::Unsupported(format!(
1406 "Unexpected SSA-value type: {ty}"
1407 ))),
1408 }
1409 }
1410
1411 fn canonical_type_for_rc(rc: RegClass) -> Type {
1412 match rc {
1413 RegClass::Float => types::I8X16,
1414 RegClass::Int => types::I64,
1415 RegClass::Vector => unreachable!(),
1416 }
1417 }
1418
1419 fn gen_jump(label: MachLabel) -> Inst {
1420 Inst::jmp_known(label)
1421 }
1422
1423 fn gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self> {
1424 Some(Inst::imm(OperandSize::Size64, value, dst))
1425 }
1426
1427 fn gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
1428 let imm_to_gpr = Inst::imm(OperandSize::Size64, value.to_bits(), tmp);
1429 let gpr_to_xmm = Inst::External {
1430 inst: asm::inst::movq_a::new(dst.map(|r| Xmm::new(r).unwrap()), tmp.to_reg()).into(),
1431 };
1432 smallvec![imm_to_gpr, gpr_to_xmm]
1433 }
1434
1435 fn gen_dummy_use(reg: Reg) -> Self {
1436 Inst::DummyUse { reg }
1437 }
1438
1439 fn worst_case_size() -> CodeOffset {
1440 15
1441 }
1442
1443 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
1444 RegClass::Int
1445 }
1446
1447 fn is_safepoint(&self) -> bool {
1448 match self {
1449 Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,
1450 _ => false,
1451 }
1452 }
1453
1454 fn function_alignment() -> FunctionAlignment {
1455 FunctionAlignment {
1456 minimum: 1,
1457 preferred: 32,
1460 }
1461 }
1462
1463 type LabelUse = LabelUse;
1464
1465 const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
1466}
1467
1468pub struct EmitInfo {
1470 pub(super) flags: settings::Flags,
1471 isa_flags: x64_settings::Flags,
1472}
1473
1474impl EmitInfo {
1475 pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
1477 Self { flags, isa_flags }
1478 }
1479}
1480
1481impl MachInstEmit for Inst {
1482 type State = EmitState;
1483 type Info = EmitInfo;
1484
1485 fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) {
1486 emit::emit(self, sink, info, state);
1487 }
1488
1489 fn pretty_print_inst(&self, _: &mut Self::State) -> String {
1490 PrettyPrint::pretty_print(self, 0)
1491 }
1492}
1493
1494#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1496pub enum LabelUse {
1497 JmpRel32,
1501
1502 PCRel32,
1505}
1506
1507impl MachInstLabelUse for LabelUse {
1508 const ALIGN: CodeOffset = 1;
1509
1510 fn max_pos_range(self) -> CodeOffset {
1511 match self {
1512 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
1513 }
1514 }
1515
1516 fn max_neg_range(self) -> CodeOffset {
1517 match self {
1518 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
1519 }
1520 }
1521
1522 fn patch_size(self) -> CodeOffset {
1523 match self {
1524 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
1525 }
1526 }
1527
1528 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
1529 let pc_rel = (label_offset as i64) - (use_offset as i64);
1530 debug_assert!(pc_rel <= self.max_pos_range() as i64);
1531 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
1532 let pc_rel = pc_rel as u32;
1533 match self {
1534 LabelUse::JmpRel32 => {
1535 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1536 let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
1537 buffer.copy_from_slice(&value.to_le_bytes()[..]);
1538 }
1539 LabelUse::PCRel32 => {
1540 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1541 let value = pc_rel.wrapping_add(addend);
1542 buffer.copy_from_slice(&value.to_le_bytes()[..]);
1543 }
1544 }
1545 }
1546
1547 fn supports_veneer(self) -> bool {
1548 match self {
1549 LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
1550 }
1551 }
1552
1553 fn veneer_size(self) -> CodeOffset {
1554 match self {
1555 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
1556 }
1557 }
1558
1559 fn worst_case_veneer_size() -> CodeOffset {
1560 0
1561 }
1562
1563 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
1564 match self {
1565 LabelUse::JmpRel32 | LabelUse::PCRel32 => {
1566 panic!("Veneer not supported for JumpRel32 label-use.");
1567 }
1568 }
1569 }
1570
1571 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
1572 match (reloc, addend) {
1573 (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
1574 _ => None,
1575 }
1576 }
1577}