cranelift_codegen/isa/x64/inst/
mod.rs

1//! This module defines x86_64-specific machine instruction types.
2
3pub use emit_state::EmitState;
4
5use crate::binemit::{Addend, CodeOffset, Reloc};
6use crate::ir::{ExternalName, LibCall, TrapCode, Type, types};
7use crate::isa::x64::abi::X64ABIMachineSpec;
8use crate::isa::x64::inst::regs::pretty_print_reg;
9use crate::isa::x64::settings as x64_settings;
10use crate::isa::{CallConv, FunctionAlignment};
11use crate::{CodegenError, CodegenResult, settings};
12use crate::{machinst::*, trace};
13use alloc::boxed::Box;
14use alloc::string::{String, ToString};
15use alloc::vec;
16use alloc::vec::Vec;
17use core::fmt::{self, Write};
18use core::slice;
19use cranelift_assembler_x64 as asm;
20use smallvec::{SmallVec, smallvec};
21
22pub mod args;
23mod emit;
24mod emit_state;
25#[cfg(test)]
26mod emit_tests;
27pub mod external;
28pub mod regs;
29mod stack_switch;
30pub mod unwind;
31
32use args::*;
33
34//=============================================================================
35// Instructions (top level): definition
36
37// `Inst` is defined inside ISLE as `MInst`. We publicly re-export it here.
38pub use super::lower::isle::generated_code::AtomicRmwSeqOp;
39pub use super::lower::isle::generated_code::MInst as Inst;
40
41/// Out-of-line data for return-calls, to keep the size of `Inst` down.
42#[derive(Clone, Debug)]
43pub struct ReturnCallInfo<T> {
44    /// Where this call is going.
45    pub dest: T,
46
47    /// The size of the argument area for this return-call, potentially smaller than that of the
48    /// caller, but never larger.
49    pub new_stack_arg_size: u32,
50
51    /// The in-register arguments and their constraints.
52    pub uses: CallArgList,
53
54    /// A temporary for use when moving the return address.
55    pub tmp: WritableGpr,
56}
57
58#[test]
59#[cfg(target_pointer_width = "64")]
60fn inst_size_test() {
61    // This test will help with unintentionally growing the size
62    // of the Inst enum.
63    assert_eq!(48, core::mem::size_of::<Inst>());
64}
65
66impl Inst {
67    /// Check if the instruction (or pseudo-instruction) can be emitted given
68    /// the current target architecture given by `emit_info`. For non-assembler
69    /// instructions, this assumes a baseline feature set (i.e., 64-bit AND SSE2
70    /// and below).
71    fn is_available(&self, emit_info: &EmitInfo) -> bool {
72        use asm::AvailableFeatures;
73
74        match self {
75            // These instructions are part of SSE2, which is a basic requirement
76            // in Cranelift, and don't have to be checked.
77            Inst::AtomicRmwSeq { .. }
78            | Inst::CallKnown { .. }
79            | Inst::CallUnknown { .. }
80            | Inst::ReturnCallKnown { .. }
81            | Inst::ReturnCallUnknown { .. }
82            | Inst::CheckedSRemSeq { .. }
83            | Inst::CheckedSRemSeq8 { .. }
84            | Inst::CvtFloatToSintSeq { .. }
85            | Inst::CvtFloatToUintSeq { .. }
86            | Inst::CvtUint64ToFloatSeq { .. }
87            | Inst::JmpCond { .. }
88            | Inst::JmpCondOr { .. }
89            | Inst::WinchJmpIf { .. }
90            | Inst::JmpKnown { .. }
91            | Inst::JmpTableSeq { .. }
92            | Inst::LoadExtName { .. }
93            | Inst::MovFromPReg { .. }
94            | Inst::MovToPReg { .. }
95            | Inst::StackProbeLoop { .. }
96            | Inst::Args { .. }
97            | Inst::Rets { .. }
98            | Inst::StackSwitchBasic { .. }
99            | Inst::InterruptPoll { .. }
100            | Inst::TrapIf { .. }
101            | Inst::TrapIfAnd { .. }
102            | Inst::TrapIfOr { .. }
103            | Inst::XmmCmove { .. }
104            | Inst::XmmMinMaxSeq { .. }
105            | Inst::XmmUninitializedValue { .. }
106            | Inst::GprUninitializedValue { .. }
107            | Inst::ElfTlsGetAddr { .. }
108            | Inst::MachOTlsGetAddr { .. }
109            | Inst::CoffTlsGetAddr { .. }
110            | Inst::Unwind { .. }
111            | Inst::DummyUse { .. }
112            | Inst::LabelAddress { .. }
113            | Inst::SequencePoint => true,
114
115            Inst::Atomic128RmwSeq { .. } | Inst::Atomic128XchgSeq { .. } => emit_info.cmpxchg16b(),
116
117            Inst::External { inst } => inst.is_available(&emit_info),
118        }
119    }
120}
121
122// Handy constructors for Insts.
123
124impl Inst {
125    pub(crate) fn nop(len: u8) -> Self {
126        assert!(len > 0 && len <= 9);
127        let inst = match len {
128            1 => asm::inst::nop_1b::new().into(),
129            2 => asm::inst::nop_2b::new().into(),
130            3 => asm::inst::nop_3b::new().into(),
131            4 => asm::inst::nop_4b::new().into(),
132            5 => asm::inst::nop_5b::new().into(),
133            6 => asm::inst::nop_6b::new().into(),
134            7 => asm::inst::nop_7b::new().into(),
135            8 => asm::inst::nop_8b::new().into(),
136            9 => asm::inst::nop_9b::new().into(),
137            _ => unreachable!("nop length must be between 1 and 9"),
138        };
139        Self::External { inst }
140    }
141
142    pub(crate) fn addq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
143        let inst = if let Ok(simm8) = i8::try_from(simm32) {
144            asm::inst::addq_mi_sxb::new(dst, simm8).into()
145        } else {
146            asm::inst::addq_mi_sxl::new(dst, simm32).into()
147        };
148        Inst::External { inst }
149    }
150
151    pub(crate) fn subq_mi(dst: Writable<Reg>, simm32: i32) -> Self {
152        let inst = if let Ok(simm8) = i8::try_from(simm32) {
153            asm::inst::subq_mi_sxb::new(dst, simm8).into()
154        } else {
155            asm::inst::subq_mi_sxl::new(dst, simm32).into()
156        };
157        Inst::External { inst }
158    }
159
160    /// Writes the `simm64` immediate into `dst`.
161    ///
162    /// Note that if `dst_size` is less than 64-bits then the upper bits of
163    /// `simm64` will be converted to zero.
164    pub fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
165        debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
166        debug_assert!(dst.to_reg().class() == RegClass::Int);
167        let dst = WritableGpr::from_writable_reg(dst).unwrap();
168        let inst = match dst_size {
169            OperandSize::Size64 => match u32::try_from(simm64) {
170                // If `simm64` is zero-extended use `movl` which zeros the
171                // upper bits.
172                Ok(imm32) => asm::inst::movl_oi::new(dst, imm32).into(),
173                _ => match i32::try_from(simm64.cast_signed()) {
174                    // If `simm64` is sign-extended use `movq` which sign the
175                    // upper bits.
176                    Ok(simm32) => asm::inst::movq_mi_sxl::new(dst, simm32).into(),
177                    // fall back to embedding the entire immediate.
178                    _ => asm::inst::movabsq_oi::new(dst, simm64).into(),
179                },
180            },
181            // FIXME: the input to this function is a logical `simm64` stored
182            // as `u64`. That means that ideally what we would do here is cast
183            // the `simm64` to an `i64`, perform a `i32::try_from()`, then cast
184            // that back to `u32`. That would ensure that the immediate loses
185            // no meaning and has the same logical value. Currently though
186            // Cranelift relies on discarding the upper bits because literals
187            // like `0x8000_0000_u64` fail to convert to an `i32`. In theory
188            // the input to this function should change to `i64`. In the
189            // meantime this is documented as discarding the upper bits,
190            // although this is an old function so that's unlikely to help
191            // much.
192            _ => asm::inst::movl_oi::new(dst, simm64 as u32).into(),
193        };
194        Inst::External { inst }
195    }
196
197    pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
198        src.assert_regclass_is(RegClass::Int);
199        debug_assert!(dst.to_reg().class() == RegClass::Int);
200        let src = match src {
201            RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
202            RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
203        };
204        let inst = match ext_mode {
205            ExtMode::BL => asm::inst::movzbl_rm::new(dst, src).into(),
206            ExtMode::BQ => asm::inst::movzbq_rm::new(dst, src).into(),
207            ExtMode::WL => asm::inst::movzwl_rm::new(dst, src).into(),
208            ExtMode::WQ => asm::inst::movzwq_rm::new(dst, src).into(),
209            ExtMode::LQ => {
210                // This instruction selection may seem strange but is correct in
211                // 64-bit mode: section 3.4.1.1 of the Intel manual says that
212                // "32-bit operands generate a 32-bit result, zero-extended to a
213                // 64-bit result in the destination general-purpose register."
214                // This is applicable beyond `mov` but we use this fact to
215                // zero-extend `src` into `dst`.
216                asm::inst::movl_rm::new(dst, src).into()
217            }
218        };
219        Inst::External { inst }
220    }
221
222    pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
223        src.assert_regclass_is(RegClass::Int);
224        debug_assert!(dst.to_reg().class() == RegClass::Int);
225        let src = match src {
226            RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::new(reg).unwrap()),
227            RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
228        };
229        let inst = match ext_mode {
230            ExtMode::BL => asm::inst::movsbl_rm::new(dst, src).into(),
231            ExtMode::BQ => asm::inst::movsbq_rm::new(dst, src).into(),
232            ExtMode::WL => asm::inst::movswl_rm::new(dst, src).into(),
233            ExtMode::WQ => asm::inst::movswq_rm::new(dst, src).into(),
234            ExtMode::LQ => asm::inst::movslq_rm::new(dst, src).into(),
235        };
236        Inst::External { inst }
237    }
238
239    /// Compares `src1` against `src2`
240    pub(crate) fn cmp_mi_sxb(size: OperandSize, src1: Gpr, src2: i8) -> Inst {
241        let inst = match size {
242            OperandSize::Size8 => asm::inst::cmpb_mi::new(src1, src2.cast_unsigned()).into(),
243            OperandSize::Size16 => asm::inst::cmpw_mi_sxb::new(src1, src2).into(),
244            OperandSize::Size32 => asm::inst::cmpl_mi_sxb::new(src1, src2).into(),
245            OperandSize::Size64 => asm::inst::cmpq_mi_sxb::new(src1, src2).into(),
246        };
247        Inst::External { inst }
248    }
249
250    pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
251        Inst::TrapIf { cc, trap_code }
252    }
253
254    pub(crate) fn call_known(info: Box<CallInfo<ExternalName>>) -> Inst {
255        Inst::CallKnown { info }
256    }
257
258    pub(crate) fn call_unknown(info: Box<CallInfo<RegMem>>) -> Inst {
259        info.dest.assert_regclass_is(RegClass::Int);
260        Inst::CallUnknown { info }
261    }
262
263    pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
264        Inst::JmpKnown { dst }
265    }
266
267    /// Choose which instruction to use for loading a register value from memory. For loads smaller
268    /// than 64 bits, this method expects a way to extend the value (i.e. [ExtKind::SignExtend],
269    /// [ExtKind::ZeroExtend]); loads with no extension necessary will ignore this.
270    pub(crate) fn load(
271        ty: Type,
272        from_addr: impl Into<SyntheticAmode>,
273        to_reg: Writable<Reg>,
274        ext_kind: ExtKind,
275    ) -> Inst {
276        let rc = to_reg.to_reg().class();
277        match rc {
278            RegClass::Int => {
279                let ext_mode = match ty.bytes() {
280                    1 => Some(ExtMode::BQ),
281                    2 => Some(ExtMode::WQ),
282                    4 => Some(ExtMode::LQ),
283                    8 => None,
284                    _ => unreachable!("the type should never use a scalar load: {}", ty),
285                };
286                if let Some(ext_mode) = ext_mode {
287                    // Values smaller than 64 bits must be extended in some way.
288                    match ext_kind {
289                        ExtKind::SignExtend => {
290                            Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
291                        }
292                        ExtKind::ZeroExtend => {
293                            Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
294                        }
295                        ExtKind::None => {
296                            panic!("expected an extension kind for extension mode: {ext_mode:?}")
297                        }
298                    }
299                } else {
300                    // 64-bit values can be moved directly.
301                    let from_addr = asm::GprMem::from(from_addr.into());
302                    Inst::External {
303                        inst: asm::inst::movq_rm::new(to_reg, from_addr).into(),
304                    }
305                }
306            }
307            RegClass::Float => {
308                let to_reg = to_reg.map(|r| Xmm::new(r).unwrap());
309                let from_addr = from_addr.into();
310                let inst = match ty {
311                    types::F16 | types::I8X2 => {
312                        panic!("loading a f16 or i8x2 requires multiple instructions")
313                    }
314                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => {
315                        asm::inst::movss_a_m::new(to_reg, from_addr).into()
316                    }
317                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => {
318                        asm::inst::movsd_a_m::new(to_reg, from_addr).into()
319                    }
320                    types::F32X4 => asm::inst::movups_a::new(to_reg, from_addr).into(),
321                    types::F64X2 => asm::inst::movupd_a::new(to_reg, from_addr).into(),
322                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => {
323                        asm::inst::movdqu_a::new(to_reg, from_addr).into()
324                    }
325                    _ => unimplemented!("unable to load type: {}", ty),
326                };
327                Inst::External { inst }
328            }
329            RegClass::Vector => unreachable!(),
330        }
331    }
332
333    /// Choose which instruction to use for storing a register value to memory.
334    pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
335        let rc = from_reg.class();
336        let to_addr = to_addr.into();
337        let inst = match rc {
338            RegClass::Int => {
339                let from_reg = Gpr::unwrap_new(from_reg);
340                match ty {
341                    types::I8 => asm::inst::movb_mr::new(to_addr, from_reg).into(),
342                    types::I16 => asm::inst::movw_mr::new(to_addr, from_reg).into(),
343                    types::I32 => asm::inst::movl_mr::new(to_addr, from_reg).into(),
344                    types::I64 => asm::inst::movq_mr::new(to_addr, from_reg).into(),
345                    _ => unreachable!(),
346                }
347            }
348            RegClass::Float => {
349                let from_reg = Xmm::new(from_reg).unwrap();
350                match ty {
351                    types::F16 | types::I8X2 => {
352                        panic!("storing a f16 or i8x2 requires multiple instructions")
353                    }
354                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 32 => {
355                        asm::inst::movss_c_m::new(to_addr, from_reg).into()
356                    }
357                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 64 => {
358                        asm::inst::movsd_c_m::new(to_addr, from_reg).into()
359                    }
360                    types::F32X4 => asm::inst::movups_b::new(to_addr, from_reg).into(),
361                    types::F64X2 => asm::inst::movupd_b::new(to_addr, from_reg).into(),
362                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => {
363                        asm::inst::movdqu_b::new(to_addr, from_reg).into()
364                    }
365                    _ => unimplemented!("unable to store type: {}", ty),
366                }
367            }
368            RegClass::Vector => unreachable!(),
369        };
370        Inst::External { inst }
371    }
372}
373
374//=============================================================================
375// Instructions: printing
376
377impl PrettyPrint for Inst {
378    fn pretty_print(&self, _size: u8) -> String {
379        fn ljustify(s: String) -> String {
380            let w = 7;
381            if s.len() >= w {
382                s
383            } else {
384                let need = usize::min(w, w - s.len());
385                s + &format!("{nil: <width$}", nil = "", width = need)
386            }
387        }
388
389        fn ljustify2(s1: String, s2: String) -> String {
390            ljustify(s1 + &s2)
391        }
392
393        match self {
394            Inst::CheckedSRemSeq {
395                size,
396                divisor,
397                dividend_lo,
398                dividend_hi,
399                dst_quotient,
400                dst_remainder,
401            } => {
402                let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes());
403                let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
404                let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
405                let dst_quotient =
406                    pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
407                let dst_remainder =
408                    pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
409                format!(
410                    "checked_srem_seq {dividend_lo}, {dividend_hi}, \
411                        {divisor}, {dst_quotient}, {dst_remainder}",
412                )
413            }
414
415            Inst::CheckedSRemSeq8 {
416                divisor,
417                dividend,
418                dst,
419            } => {
420                let divisor = pretty_print_reg(divisor.to_reg(), 1);
421                let dividend = pretty_print_reg(dividend.to_reg(), 1);
422                let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
423                format!("checked_srem_seq {dividend}, {divisor}, {dst}")
424            }
425
426            Inst::XmmMinMaxSeq {
427                lhs,
428                rhs,
429                dst,
430                is_min,
431                size,
432            } => {
433                let rhs = pretty_print_reg(rhs.to_reg(), 8);
434                let lhs = pretty_print_reg(lhs.to_reg(), 8);
435                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
436                let op = ljustify2(
437                    if *is_min {
438                        "xmm min seq ".to_string()
439                    } else {
440                        "xmm max seq ".to_string()
441                    },
442                    format!("f{}", size.to_bits()),
443                );
444                format!("{op} {lhs}, {rhs}, {dst}")
445            }
446
447            Inst::XmmUninitializedValue { dst } => {
448                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
449                let op = ljustify("uninit".into());
450                format!("{op} {dst}")
451            }
452
453            Inst::GprUninitializedValue { dst } => {
454                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
455                let op = ljustify("uninit".into());
456                format!("{op} {dst}")
457            }
458
459            Inst::CvtUint64ToFloatSeq {
460                src,
461                dst,
462                dst_size,
463                tmp_gpr1,
464                tmp_gpr2,
465                ..
466            } => {
467                let src = pretty_print_reg(src.to_reg(), 8);
468                let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
469                let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8);
470                let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8);
471                let op = ljustify(format!(
472                    "u64_to_{}_seq",
473                    if *dst_size == OperandSize::Size64 {
474                        "f64"
475                    } else {
476                        "f32"
477                    }
478                ));
479                format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}")
480            }
481
482            Inst::CvtFloatToSintSeq {
483                src,
484                dst,
485                src_size,
486                dst_size,
487                tmp_xmm,
488                tmp_gpr,
489                is_saturating,
490            } => {
491                let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
492                let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
493                let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
494                let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
495                let op = ljustify(format!(
496                    "cvt_float{}_to_sint{}{}_seq",
497                    src_size.to_bits(),
498                    dst_size.to_bits(),
499                    if *is_saturating { "_sat" } else { "" },
500                ));
501                format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}")
502            }
503
504            Inst::CvtFloatToUintSeq {
505                src,
506                dst,
507                src_size,
508                dst_size,
509                tmp_gpr,
510                tmp_xmm,
511                tmp_xmm2,
512                is_saturating,
513            } => {
514                let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
515                let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
516                let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
517                let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
518                let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8);
519                let op = ljustify(format!(
520                    "cvt_float{}_to_uint{}{}_seq",
521                    src_size.to_bits(),
522                    dst_size.to_bits(),
523                    if *is_saturating { "_sat" } else { "" },
524                ));
525                format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}, {tmp_xmm2}")
526            }
527
528            Inst::MovFromPReg { src, dst } => {
529                let src: Reg = (*src).into();
530                let src = pretty_print_reg(src, 8);
531                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
532                let op = ljustify("movq".to_string());
533                format!("{op} {src}, {dst}")
534            }
535
536            Inst::MovToPReg { src, dst } => {
537                let src = pretty_print_reg(src.to_reg(), 8);
538                let dst: Reg = (*dst).into();
539                let dst = pretty_print_reg(dst, 8);
540                let op = ljustify("movq".to_string());
541                format!("{op} {src}, {dst}")
542            }
543
544            Inst::XmmCmove {
545                ty,
546                cc,
547                consequent,
548                alternative,
549                dst,
550                ..
551            } => {
552                let size = u8::try_from(ty.bytes()).unwrap();
553                let alternative = pretty_print_reg(alternative.to_reg(), size);
554                let dst = pretty_print_reg(dst.to_reg().to_reg(), size);
555                let consequent = pretty_print_reg(consequent.to_reg(), size);
556                let suffix = match *ty {
557                    types::F64 => "sd",
558                    types::F32 => "ss",
559                    types::F16 => "ss",
560                    types::F32X4 => "aps",
561                    types::F64X2 => "apd",
562                    _ => "dqa",
563                };
564                let cc = cc.invert();
565                format!(
566                    "mov{suffix} {alternative}, {dst}; \
567                    j{cc} $next; \
568                    mov{suffix} {consequent}, {dst}; \
569                    $next:"
570                )
571            }
572
573            Inst::StackProbeLoop {
574                tmp,
575                frame_size,
576                guard_size,
577            } => {
578                let tmp = pretty_print_reg(tmp.to_reg(), 8);
579                let op = ljustify("stack_probe_loop".to_string());
580                format!("{op} {tmp}, frame_size={frame_size}, guard_size={guard_size}")
581            }
582
583            Inst::CallKnown { info } => {
584                let op = ljustify("call".to_string());
585                let try_call = info
586                    .try_call_info
587                    .as_ref()
588                    .map(|tci| pretty_print_try_call(tci))
589                    .unwrap_or_default();
590                format!("{op} {:?}{try_call}", info.dest)
591            }
592
593            Inst::CallUnknown { info } => {
594                let dest = info.dest.pretty_print(8);
595                let op = ljustify("call".to_string());
596                let try_call = info
597                    .try_call_info
598                    .as_ref()
599                    .map(|tci| pretty_print_try_call(tci))
600                    .unwrap_or_default();
601                format!("{op} *{dest}{try_call}")
602            }
603
604            Inst::ReturnCallKnown { info } => {
605                let ReturnCallInfo {
606                    uses,
607                    new_stack_arg_size,
608                    tmp,
609                    dest,
610                } = &**info;
611                let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
612                let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}");
613                for ret in uses {
614                    let preg = pretty_print_reg(ret.preg, 8);
615                    let vreg = pretty_print_reg(ret.vreg, 8);
616                    write!(&mut s, " {vreg}={preg}").unwrap();
617                }
618                s
619            }
620
621            Inst::ReturnCallUnknown { info } => {
622                let ReturnCallInfo {
623                    uses,
624                    new_stack_arg_size,
625                    tmp,
626                    dest,
627                } = &**info;
628                let callee = pretty_print_reg(*dest, 8);
629                let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
630                let mut s =
631                    format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}");
632                for ret in uses {
633                    let preg = pretty_print_reg(ret.preg, 8);
634                    let vreg = pretty_print_reg(ret.vreg, 8);
635                    write!(&mut s, " {vreg}={preg}").unwrap();
636                }
637                s
638            }
639
640            Inst::Args { args } => {
641                let mut s = "args".to_string();
642                for arg in args {
643                    let preg = pretty_print_reg(arg.preg, 8);
644                    let def = pretty_print_reg(arg.vreg.to_reg(), 8);
645                    write!(&mut s, " {def}={preg}").unwrap();
646                }
647                s
648            }
649
650            Inst::Rets { rets } => {
651                let mut s = "rets".to_string();
652                for ret in rets {
653                    let preg = pretty_print_reg(ret.preg, 8);
654                    let vreg = pretty_print_reg(ret.vreg, 8);
655                    write!(&mut s, " {vreg}={preg}").unwrap();
656                }
657                s
658            }
659
660            Inst::StackSwitchBasic {
661                store_context_ptr,
662                load_context_ptr,
663                in_payload0,
664                out_payload0,
665            } => {
666                let store_context_ptr = pretty_print_reg(**store_context_ptr, 8);
667                let load_context_ptr = pretty_print_reg(**load_context_ptr, 8);
668                let in_payload0 = pretty_print_reg(**in_payload0, 8);
669                let out_payload0 = pretty_print_reg(*out_payload0.to_reg(), 8);
670                format!(
671                    "{out_payload0} = stack_switch_basic {store_context_ptr}, {load_context_ptr}, {in_payload0}"
672                )
673            }
674
675            Inst::InterruptPoll {
676                dst,
677                load_ptr,
678                context,
679                trap_code,
680                next_load_ptr,
681            } => {
682                let dst = pretty_print_reg(*dst.to_reg(), 8);
683                let load_ptr = pretty_print_reg(**load_ptr, 8);
684                let context = pretty_print_reg(**context, 8);
685                let next_load_ptr = pretty_print_reg(*next_load_ptr.to_reg(), 8);
686                format!(
687                    "{next_load_ptr} = interrupt_poll load_ptr={load_ptr}, context={context}, scratch={dst} #trap={trap_code}"
688                )
689            }
690
691            Inst::JmpKnown { dst } => {
692                let op = ljustify("jmp".to_string());
693                let dst = dst.to_string();
694                format!("{op} {dst}")
695            }
696
697            Inst::WinchJmpIf { cc, taken } => {
698                let taken = taken.to_string();
699                let op = ljustify2("j".to_string(), cc.to_string());
700                format!("{op} {taken}")
701            }
702
703            Inst::JmpCondOr {
704                cc1,
705                cc2,
706                taken,
707                not_taken,
708            } => {
709                let taken = taken.to_string();
710                let not_taken = not_taken.to_string();
711                let op = ljustify(format!("j{cc1},{cc2}"));
712                format!("{op} {taken}; j {not_taken}")
713            }
714
715            Inst::JmpCond {
716                cc,
717                taken,
718                not_taken,
719            } => {
720                let taken = taken.to_string();
721                let not_taken = not_taken.to_string();
722                let op = ljustify2("j".to_string(), cc.to_string());
723                format!("{op} {taken}; j {not_taken}")
724            }
725
726            Inst::JmpTableSeq {
727                idx, tmp1, tmp2, ..
728            } => {
729                let idx = pretty_print_reg(*idx, 8);
730                let tmp1 = pretty_print_reg(tmp1.to_reg(), 8);
731                let tmp2 = pretty_print_reg(tmp2.to_reg(), 8);
732                let op = ljustify("br_table".into());
733                format!("{op} {idx}, {tmp1}, {tmp2}")
734            }
735
736            Inst::TrapIf { cc, trap_code, .. } => {
737                format!("j{cc} #trap={trap_code}")
738            }
739
740            Inst::TrapIfAnd {
741                cc1,
742                cc2,
743                trap_code,
744                ..
745            } => {
746                let cc1 = cc1.invert();
747                let cc2 = cc2.invert();
748                format!("trap_if_and {cc1}, {cc2}, {trap_code}")
749            }
750
751            Inst::TrapIfOr {
752                cc1,
753                cc2,
754                trap_code,
755                ..
756            } => {
757                let cc2 = cc2.invert();
758                format!("trap_if_or {cc1}, {cc2}, {trap_code}")
759            }
760
761            Inst::LoadExtName {
762                dst, name, offset, ..
763            } => {
764                let dst = pretty_print_reg(*dst.to_reg(), 8);
765                let name = name.display(None);
766                let op = ljustify("load_ext_name".into());
767                format!("{op} {name}+{offset}, {dst}")
768            }
769
770            Inst::AtomicRmwSeq { ty, op, .. } => {
771                let ty = ty.bits();
772                format!(
773                    "atomically {{ {ty}_bits_at_[%r9] {op:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}"
774                )
775            }
776
777            Inst::Atomic128RmwSeq { args } => {
778                let Atomic128RmwSeqArgs {
779                    op,
780                    mem_low,
781                    mem_high,
782                    operand_low,
783                    operand_high,
784                    temp_low,
785                    temp_high,
786                    dst_old_low,
787                    dst_old_high,
788                } = &**args;
789                let operand_low = pretty_print_reg(**operand_low, 8);
790                let operand_high = pretty_print_reg(**operand_high, 8);
791                let temp_low = pretty_print_reg(*temp_low.to_reg(), 8);
792                let temp_high = pretty_print_reg(*temp_high.to_reg(), 8);
793                let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
794                let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
795                let mem_low = mem_low.pretty_print(16);
796                let mem_high = mem_high.pretty_print(16);
797                format!(
798                    "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem_low}:{mem_high} = {temp_high}:{temp_low} }}"
799                )
800            }
801
802            Inst::Atomic128XchgSeq { args } => {
803                let Atomic128XchgSeqArgs {
804                    mem_low,
805                    mem_high,
806                    operand_low,
807                    operand_high,
808                    dst_old_low,
809                    dst_old_high,
810                } = &**args;
811                let operand_low = pretty_print_reg(**operand_low, 8);
812                let operand_high = pretty_print_reg(**operand_high, 8);
813                let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
814                let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
815                let mem_low = mem_low.pretty_print(16);
816                let mem_high = mem_high.pretty_print(16);
817                format!(
818                    "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {mem_low}:{mem_high} = {operand_high}:{operand_low} }}"
819                )
820            }
821
822            Inst::ElfTlsGetAddr { symbol, dst } => {
823                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
824                format!("{dst} = elf_tls_get_addr {symbol:?}")
825            }
826
827            Inst::MachOTlsGetAddr { symbol, dst } => {
828                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
829                format!("{dst} = macho_tls_get_addr {symbol:?}")
830            }
831
832            Inst::CoffTlsGetAddr { symbol, dst, tmp } => {
833                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
834                let tmp = tmp.to_reg().to_reg();
835
836                let mut s = format!("{dst} = coff_tls_get_addr {symbol:?}");
837                if tmp.is_virtual() {
838                    let tmp = pretty_print_reg(tmp, 8);
839                    write!(&mut s, ", {tmp}").unwrap();
840                };
841
842                s
843            }
844
845            Inst::Unwind { inst } => format!("unwind {inst:?}"),
846
847            Inst::DummyUse { reg } => {
848                let reg = pretty_print_reg(*reg, 8);
849                format!("dummy_use {reg}")
850            }
851
852            Inst::LabelAddress { dst, label } => {
853                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
854                format!("label_address {dst}, {label:?}")
855            }
856
857            Inst::SequencePoint {} => {
858                format!("sequence_point")
859            }
860
861            Inst::External { inst } => {
862                format!("{inst}")
863            }
864        }
865    }
866}
867
868fn pretty_print_try_call(info: &TryCallInfo) -> String {
869    format!(
870        "; jmp {:?}; catch [{}]",
871        info.continuation,
872        info.pretty_print_dests()
873    )
874}
875
876impl fmt::Debug for Inst {
877    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
878        write!(fmt, "{}", self.pretty_print_inst(&mut Default::default()))
879    }
880}
881
882fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
883    // Note: because we need to statically know the indices of each
884    // reg in the operands list in order to fetch its allocation
885    // later, we put the variable-operand-count bits (the RegMem,
886    // RegMemImm, etc args) last. regalloc2 doesn't care what order
887    // the operands come in; they can be freely reordered.
888
889    // N.B.: we MUST keep the below in careful sync with (i) emission,
890    // in `emit.rs`, and (ii) pretty-printing, in the `pretty_print`
891    // method above.
892    match inst {
893        Inst::CheckedSRemSeq {
894            divisor,
895            dividend_lo,
896            dividend_hi,
897            dst_quotient,
898            dst_remainder,
899            ..
900        } => {
901            collector.reg_use(divisor);
902            collector.reg_fixed_use(dividend_lo, regs::rax());
903            collector.reg_fixed_use(dividend_hi, regs::rdx());
904            collector.reg_fixed_def(dst_quotient, regs::rax());
905            collector.reg_fixed_def(dst_remainder, regs::rdx());
906        }
907        Inst::CheckedSRemSeq8 {
908            divisor,
909            dividend,
910            dst,
911            ..
912        } => {
913            collector.reg_use(divisor);
914            collector.reg_fixed_use(dividend, regs::rax());
915            collector.reg_fixed_def(dst, regs::rax());
916        }
917        Inst::XmmUninitializedValue { dst } => collector.reg_def(dst),
918        Inst::GprUninitializedValue { dst } => collector.reg_def(dst),
919        Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
920            collector.reg_use(rhs);
921            collector.reg_use(lhs);
922            collector.reg_reuse_def(dst, 0); // Reuse RHS.
923        }
924        Inst::MovFromPReg { dst, src } => {
925            debug_assert!(dst.to_reg().to_reg().is_virtual());
926            collector.reg_fixed_nonallocatable(*src);
927            collector.reg_def(dst);
928        }
929        Inst::MovToPReg { dst, src } => {
930            debug_assert!(src.to_reg().is_virtual());
931            collector.reg_use(src);
932            collector.reg_fixed_nonallocatable(*dst);
933        }
934        Inst::CvtUint64ToFloatSeq {
935            src,
936            dst,
937            tmp_gpr1,
938            tmp_gpr2,
939            ..
940        } => {
941            collector.reg_use(src);
942            collector.reg_early_def(dst);
943            collector.reg_early_def(tmp_gpr1);
944            collector.reg_early_def(tmp_gpr2);
945        }
946        Inst::CvtFloatToSintSeq {
947            src,
948            dst,
949            tmp_xmm,
950            tmp_gpr,
951            ..
952        } => {
953            collector.reg_use(src);
954            collector.reg_early_def(dst);
955            collector.reg_early_def(tmp_gpr);
956            collector.reg_early_def(tmp_xmm);
957        }
958        Inst::CvtFloatToUintSeq {
959            src,
960            dst,
961            tmp_gpr,
962            tmp_xmm,
963            tmp_xmm2,
964            ..
965        } => {
966            collector.reg_use(src);
967            collector.reg_early_def(dst);
968            collector.reg_early_def(tmp_gpr);
969            collector.reg_early_def(tmp_xmm);
970            collector.reg_early_def(tmp_xmm2);
971        }
972
973        Inst::XmmCmove {
974            consequent,
975            alternative,
976            dst,
977            ..
978        } => {
979            collector.reg_use(alternative);
980            collector.reg_reuse_def(dst, 0);
981            collector.reg_use(consequent);
982        }
983        Inst::StackProbeLoop { tmp, .. } => {
984            collector.reg_early_def(tmp);
985        }
986
987        Inst::CallKnown { info } => {
988            // Probestack is special and is only inserted after
989            // regalloc, so we do not need to represent its ABI to the
990            // register allocator. Assert that we don't alter that
991            // arrangement.
992            let CallInfo {
993                uses,
994                defs,
995                clobbers,
996                dest,
997                try_call_info,
998                ..
999            } = &mut **info;
1000            debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
1001            for CallArgPair { vreg, preg } in uses {
1002                collector.reg_fixed_use(vreg, *preg);
1003            }
1004            for CallRetPair { vreg, location } in defs {
1005                match location {
1006                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
1007                    RetLocation::Stack(..) => collector.any_def(vreg),
1008                }
1009            }
1010            collector.reg_clobbers(*clobbers);
1011            if let Some(try_call_info) = try_call_info {
1012                try_call_info.collect_operands(collector);
1013            }
1014        }
1015
1016        Inst::CallUnknown { info } => {
1017            let CallInfo {
1018                uses,
1019                defs,
1020                clobbers,
1021                callee_conv,
1022                dest,
1023                try_call_info,
1024                ..
1025            } = &mut **info;
1026            match dest {
1027                RegMem::Reg { reg } if *callee_conv == CallConv::Winch => {
1028                    // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
1029                    // This shouldn't be a fixed register constraint. r10 is caller-saved, so this
1030                    // should be safe to use.
1031                    collector.reg_fixed_use(reg, regs::r10());
1032                }
1033                _ => dest.get_operands(collector),
1034            }
1035            for CallArgPair { vreg, preg } in uses {
1036                collector.reg_fixed_use(vreg, *preg);
1037            }
1038            for CallRetPair { vreg, location } in defs {
1039                match location {
1040                    RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg),
1041                    RetLocation::Stack(..) => collector.any_def(vreg),
1042                }
1043            }
1044            collector.reg_clobbers(*clobbers);
1045            if let Some(try_call_info) = try_call_info {
1046                try_call_info.collect_operands(collector);
1047            }
1048        }
1049        Inst::StackSwitchBasic {
1050            store_context_ptr,
1051            load_context_ptr,
1052            in_payload0,
1053            out_payload0,
1054        } => {
1055            collector.reg_use(load_context_ptr);
1056            collector.reg_use(store_context_ptr);
1057            collector.reg_fixed_use(in_payload0, stack_switch::payload_register());
1058            collector.reg_fixed_def(out_payload0, stack_switch::payload_register());
1059
1060            let mut clobbers = crate::isa::x64::abi::ALL_CLOBBERS;
1061            // The return/payload reg must not be included in the clobber set
1062            clobbers.remove(
1063                stack_switch::payload_register()
1064                    .to_real_reg()
1065                    .unwrap()
1066                    .into(),
1067            );
1068            collector.reg_clobbers(clobbers);
1069        }
1070
1071        Inst::InterruptPoll {
1072            dst,
1073            load_ptr,
1074            context,
1075            trap_code: _,
1076            next_load_ptr,
1077        } => {
1078            // `load_ptr` is an input param. It is pinned to r11 so an update of
1079            // `next_load_ptr` updates this as well. r11 is chosen because it is
1080            // a caller-saved reg not used for arg-passing in Linux/x64.
1081            collector.reg_fixed_use(load_ptr, regs::r11());
1082            // Demand context (vmctx) go into RDI.
1083            collector.reg_fixed_use(context, regs::rdi());
1084            // Reserve r10 as scratch space.
1085            //
1086            // r10 is chosen because it is caller-saved and not used for
1087            // arg passing in Linux/x64. It is used as "a static chain pointer
1088            // in case of nested functions" according to SystemV, but Cranelift
1089            // does not emit those. (Do take care if you are interacting with
1090            // external ones.) It is also used to store the function stack limit
1091            // in Cranelift, but the stack-limit check is confined to the
1092            // function prologue and thus over by the time this instruction is
1093            // used.
1094            //
1095            // Also def it so we can use it as the destination of the dead load
1096            // rather than consuming another arbitrary reg.
1097            collector.reg_fixed_def(dst, regs::r10());
1098            // `next_load_ptr` is pinned so embedders know where to write to
1099            // fill it. It takes on the value of `load_ptr` unless a signal
1100            // handler invoked at the load overwrites it. We constrain
1101            // `next_load_ptr` and `load_ptr` to the same register so this
1102            // dataflow happens without a move.
1103            collector.reg_fixed_def(next_load_ptr, regs::r11());
1104        }
1105
1106        Inst::ReturnCallKnown { info } => {
1107            let ReturnCallInfo {
1108                dest, uses, tmp, ..
1109            } = &mut **info;
1110            collector.reg_fixed_def(tmp, regs::r11());
1111            // Same as in the `Inst::CallKnown` branch.
1112            debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
1113            for CallArgPair { vreg, preg } in uses {
1114                collector.reg_fixed_use(vreg, *preg);
1115            }
1116        }
1117
1118        Inst::ReturnCallUnknown { info } => {
1119            let ReturnCallInfo {
1120                dest, uses, tmp, ..
1121            } = &mut **info;
1122
1123            // TODO(https://github.com/bytecodealliance/regalloc2/issues/145):
1124            // This shouldn't be a fixed register constraint, but it's not clear how to
1125            // pick a register that won't be clobbered by the callee-save restore code
1126            // emitted with a return_call_indirect. r10 is caller-saved, so this should be
1127            // safe to use.
1128            collector.reg_fixed_use(dest, regs::r10());
1129
1130            collector.reg_fixed_def(tmp, regs::r11());
1131            for CallArgPair { vreg, preg } in uses {
1132                collector.reg_fixed_use(vreg, *preg);
1133            }
1134        }
1135
1136        Inst::JmpTableSeq {
1137            idx, tmp1, tmp2, ..
1138        } => {
1139            collector.reg_use(idx);
1140            collector.reg_early_def(tmp1);
1141            // In the sequence emitted for this pseudoinstruction in emit.rs,
1142            // tmp2 is only written after idx is read, so it doesn't need to be
1143            // an early def.
1144            collector.reg_def(tmp2);
1145        }
1146
1147        Inst::LoadExtName { dst, .. } => {
1148            collector.reg_def(dst);
1149        }
1150
1151        Inst::AtomicRmwSeq {
1152            operand,
1153            temp,
1154            dst_old,
1155            mem,
1156            ..
1157        } => {
1158            collector.reg_late_use(operand);
1159            collector.reg_early_def(temp);
1160            // This `fixed_def` is needed because `CMPXCHG` always uses this
1161            // register implicitly.
1162            collector.reg_fixed_def(dst_old, regs::rax());
1163            mem.get_operands_late(collector)
1164        }
1165
1166        Inst::Atomic128RmwSeq { args } => {
1167            let Atomic128RmwSeqArgs {
1168                mem_low,
1169                mem_high,
1170                operand_low,
1171                operand_high,
1172                temp_low,
1173                temp_high,
1174                dst_old_low,
1175                dst_old_high,
1176                op: _,
1177            } = &mut **args;
1178            // All registers are collected in the `Late` position so that they don't overlap.
1179            collector.reg_late_use(operand_low);
1180            collector.reg_late_use(operand_high);
1181            collector.reg_fixed_def(temp_low, regs::rbx());
1182            collector.reg_fixed_def(temp_high, regs::rcx());
1183            collector.reg_fixed_def(dst_old_low, regs::rax());
1184            collector.reg_fixed_def(dst_old_high, regs::rdx());
1185            mem_low.get_operands_late(collector);
1186            mem_high.get_operands_late(collector);
1187        }
1188
1189        Inst::Atomic128XchgSeq { args } => {
1190            let Atomic128XchgSeqArgs {
1191                mem_low,
1192                mem_high,
1193                operand_low,
1194                operand_high,
1195                dst_old_low,
1196                dst_old_high,
1197            } = &mut **args;
1198            // All registers are collected in the `Late` position so that they don't overlap.
1199            collector.reg_fixed_late_use(operand_low, regs::rbx());
1200            collector.reg_fixed_late_use(operand_high, regs::rcx());
1201            collector.reg_fixed_def(dst_old_low, regs::rax());
1202            collector.reg_fixed_def(dst_old_high, regs::rdx());
1203            mem_low.get_operands_late(collector);
1204            mem_high.get_operands_late(collector);
1205        }
1206
1207        Inst::Args { args } => {
1208            for ArgPair { vreg, preg } in args {
1209                collector.reg_fixed_def(vreg, *preg);
1210            }
1211        }
1212
1213        Inst::Rets { rets } => {
1214            // The return value(s) are live-out; we represent this
1215            // with register uses on the return instruction.
1216            for RetPair { vreg, preg } in rets {
1217                collector.reg_fixed_use(vreg, *preg);
1218            }
1219        }
1220
1221        Inst::JmpKnown { .. }
1222        | Inst::WinchJmpIf { .. }
1223        | Inst::JmpCond { .. }
1224        | Inst::JmpCondOr { .. }
1225        | Inst::TrapIf { .. }
1226        | Inst::TrapIfAnd { .. }
1227        | Inst::TrapIfOr { .. } => {
1228            // No registers are used.
1229        }
1230
1231        Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
1232            collector.reg_fixed_def(dst, regs::rax());
1233            // All caller-saves are clobbered.
1234            //
1235            // We use the SysV calling convention here because the
1236            // pseudoinstruction (and relocation that it emits) is specific to
1237            // ELF systems; other x86-64 targets with other conventions (i.e.,
1238            // Windows) use different TLS strategies.
1239            let mut clobbers =
1240                X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false);
1241            clobbers.remove(regs::gpr_preg(asm::gpr::enc::RAX));
1242            collector.reg_clobbers(clobbers);
1243        }
1244
1245        Inst::CoffTlsGetAddr { dst, tmp, .. } => {
1246            // We also use the gs register. But that register is not allocatable by the
1247            // register allocator, so we don't need to mark it as used here.
1248
1249            // We use %rax to set the address
1250            collector.reg_fixed_def(dst, regs::rax());
1251
1252            // We use %rcx as a temporary variable to load the _tls_index
1253            collector.reg_fixed_def(tmp, regs::rcx());
1254        }
1255
1256        Inst::Unwind { .. } => {}
1257
1258        Inst::DummyUse { reg } => {
1259            collector.reg_use(reg);
1260        }
1261
1262        Inst::LabelAddress { dst, .. } => {
1263            collector.reg_def(dst);
1264        }
1265
1266        Inst::SequencePoint { .. } => {}
1267
1268        Inst::External { inst } => {
1269            inst.visit(&mut external::RegallocVisitor { collector });
1270        }
1271    }
1272}
1273
1274//=============================================================================
1275// Instructions: misc functions and external interface
1276
1277impl MachInst for Inst {
1278    type ABIMachineSpec = X64ABIMachineSpec;
1279
1280    fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
1281        x64_get_operands(self, collector)
1282    }
1283
1284    fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
1285        use asm::inst::Inst as I;
1286        match self {
1287            // Note (carefully!) that a 32-bit mov *isn't* a no-op since it zeroes
1288            // out the upper 32 bits of the destination.  For example, we could
1289            // conceivably use `movl %reg, %reg` to zero out the top 32 bits of
1290            // %reg.
1291            Self::External {
1292                inst: I::movq_mr(asm::inst::movq_mr { rm64, r64 }),
1293            } => match rm64 {
1294                asm::GprMem::Gpr(reg) => Some((reg.map(|r| r.to_reg()), r64.as_ref().to_reg())),
1295                asm::GprMem::Mem(_) => None,
1296            },
1297            Self::External {
1298                inst: I::movq_rm(asm::inst::movq_rm { r64, rm64 }),
1299            } => match rm64 {
1300                asm::GprMem::Gpr(reg) => Some((r64.as_ref().map(|r| r.to_reg()), reg.to_reg())),
1301                asm::GprMem::Mem(_) => None,
1302            },
1303
1304            // Note that `movss_a_r` and `movsd_a_r` are specifically omitted
1305            // here because they only overwrite the low bits in the destination
1306            // register, otherwise preserving the upper bits. That can be used
1307            // for lane-insertion instructions, for example, meaning it's not
1308            // classified as a register move.
1309            //
1310            // Otherwise though all register-to-register movement instructions
1311            // which move 128-bits are registered as moves.
1312            Self::External {
1313                inst:
1314                    I::movaps_a(asm::inst::movaps_a { xmm1, xmm_m128 })
1315                    | I::movups_a(asm::inst::movups_a { xmm1, xmm_m128 })
1316                    | I::movapd_a(asm::inst::movapd_a { xmm1, xmm_m128 })
1317                    | I::movupd_a(asm::inst::movupd_a { xmm1, xmm_m128 })
1318                    | I::movdqa_a(asm::inst::movdqa_a { xmm1, xmm_m128 })
1319                    | I::movdqu_a(asm::inst::movdqu_a { xmm1, xmm_m128 }),
1320            } => match xmm_m128 {
1321                asm::XmmMem::Xmm(xmm2) => Some((xmm1.as_ref().map(|r| r.to_reg()), xmm2.to_reg())),
1322                asm::XmmMem::Mem(_) => None,
1323            },
1324            // In addition to the "A" format of instructions above also
1325            // recognize the "B" format which while it can be used for stores it
1326            // can also be used for register moves.
1327            Self::External {
1328                inst:
1329                    I::movaps_b(asm::inst::movaps_b { xmm_m128, xmm1 })
1330                    | I::movups_b(asm::inst::movups_b { xmm_m128, xmm1 })
1331                    | I::movapd_b(asm::inst::movapd_b { xmm_m128, xmm1 })
1332                    | I::movupd_b(asm::inst::movupd_b { xmm_m128, xmm1 })
1333                    | I::movdqa_b(asm::inst::movdqa_b { xmm_m128, xmm1 })
1334                    | I::movdqu_b(asm::inst::movdqu_b { xmm_m128, xmm1 }),
1335            } => match xmm_m128 {
1336                asm::XmmMem::Xmm(dst) => Some((dst.map(|r| r.to_reg()), xmm1.as_ref().to_reg())),
1337                asm::XmmMem::Mem(_) => None,
1338            },
1339            _ => None,
1340        }
1341    }
1342
1343    fn is_included_in_clobbers(&self) -> bool {
1344        match self {
1345            &Inst::Args { .. } => false,
1346            _ => true,
1347        }
1348    }
1349
1350    fn is_trap(&self) -> bool {
1351        match self {
1352            Self::External {
1353                inst: asm::inst::Inst::ud2_zo(..),
1354            } => true,
1355            _ => false,
1356        }
1357    }
1358
1359    fn is_args(&self) -> bool {
1360        match self {
1361            Self::Args { .. } => true,
1362            _ => false,
1363        }
1364    }
1365
1366    fn call_type(&self) -> CallType {
1367        match self {
1368            Inst::CallKnown { .. }
1369            | Inst::CallUnknown { .. }
1370            | Inst::ElfTlsGetAddr { .. }
1371            | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
1372
1373            Inst::ReturnCallKnown { .. } | Inst::ReturnCallUnknown { .. } => CallType::TailCall,
1374
1375            _ => CallType::None,
1376        }
1377    }
1378
1379    fn is_term(&self) -> MachTerminator {
1380        match self {
1381            // Interesting cases.
1382            &Self::Rets { .. } => MachTerminator::Ret,
1383            &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => {
1384                MachTerminator::RetCall
1385            }
1386            &Self::JmpKnown { .. } => MachTerminator::Branch,
1387            &Self::JmpCond { .. } => MachTerminator::Branch,
1388            &Self::JmpCondOr { .. } => MachTerminator::Branch,
1389            &Self::JmpTableSeq { .. } => MachTerminator::Branch,
1390            &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch,
1391            &Self::CallUnknown { ref info } if info.try_call_info.is_some() => {
1392                MachTerminator::Branch
1393            }
1394            // All other cases are boring.
1395            _ => MachTerminator::None,
1396        }
1397    }
1398
1399    fn is_low_level_branch(&self) -> bool {
1400        match self {
1401            &Self::WinchJmpIf { .. } => true,
1402            _ => false,
1403        }
1404    }
1405
1406    fn is_mem_access(&self) -> bool {
1407        panic!("TODO FILL ME OUT")
1408    }
1409
1410    fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
1411        trace!(
1412            "Inst::gen_move {:?} -> {:?} (type: {:?})",
1413            src_reg,
1414            dst_reg.to_reg(),
1415            ty
1416        );
1417        let rc_dst = dst_reg.to_reg().class();
1418        let rc_src = src_reg.class();
1419        // If this isn't true, we have gone way off the rails.
1420        debug_assert!(rc_dst == rc_src);
1421        let inst = match rc_dst {
1422            RegClass::Int => {
1423                asm::inst::movq_mr::new(dst_reg.map(Gpr::unwrap_new), Gpr::unwrap_new(src_reg))
1424                    .into()
1425            }
1426            RegClass::Float => {
1427                // The Intel optimization manual, in "3.5.1.13 Zero-Latency MOV Instructions",
1428                // doesn't include MOVSS/MOVSD as instructions with zero-latency. Use movaps for
1429                // those, which may write more lanes that we need, but are specified to have
1430                // zero-latency.
1431                let dst_reg = dst_reg.map(|r| Xmm::new(r).unwrap());
1432                let src_reg = Xmm::new(src_reg).unwrap();
1433                match ty {
1434                    types::F16 | types::F32 | types::F64 | types::F32X4 => {
1435                        asm::inst::movaps_a::new(dst_reg, src_reg).into()
1436                    }
1437                    types::F64X2 => asm::inst::movapd_a::new(dst_reg, src_reg).into(),
1438                    _ if (ty.is_float() || ty.is_vector()) && ty.bits() <= 128 => {
1439                        asm::inst::movdqa_a::new(dst_reg, src_reg).into()
1440                    }
1441                    _ => unimplemented!("unable to move type: {}", ty),
1442                }
1443            }
1444            RegClass::Vector => unreachable!(),
1445        };
1446        Inst::External { inst }
1447    }
1448
1449    fn gen_nop(preferred_size: usize) -> Inst {
1450        Inst::nop(core::cmp::min(preferred_size, 9) as u8)
1451    }
1452
1453    fn gen_nop_units() -> Vec<Vec<u8>> {
1454        vec![
1455            // Standard 1-byte NOP.
1456            vec![0x90],
1457            // 5-byte NOP useful for patching out patchable calls.
1458            vec![0x0f, 0x1f, 0x44, 0x00, 0x00],
1459        ]
1460    }
1461
1462    fn rc_for_type(ty: &Type) -> CodegenResult<(&[RegClass], &[Type])> {
1463        match *ty {
1464            types::I8 | types::I16 | types::I32 | types::I64 => {
1465                Ok((&[RegClass::Int], core::slice::from_ref(ty)))
1466            }
1467            types::F16 | types::F32 | types::F64 | types::F128 => {
1468                Ok((&[RegClass::Float], core::slice::from_ref(ty)))
1469            }
1470            types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
1471            _ if ty.is_vector() && ty.bits() <= 128 => {
1472                let types = &[types::I8X2, types::I8X4, types::I8X8, types::I8X16];
1473                Ok((
1474                    &[RegClass::Float],
1475                    slice::from_ref(&types[ty.bytes().ilog2() as usize - 1]),
1476                ))
1477            }
1478            _ => Err(CodegenError::Unsupported(format!(
1479                "Unexpected SSA-value type: {ty}"
1480            ))),
1481        }
1482    }
1483
1484    fn canonical_type_for_rc(rc: RegClass) -> Type {
1485        match rc {
1486            RegClass::Float => types::I8X16,
1487            RegClass::Int => types::I64,
1488            RegClass::Vector => unreachable!(),
1489        }
1490    }
1491
1492    fn gen_jump(label: MachLabel) -> Inst {
1493        Inst::jmp_known(label)
1494    }
1495
1496    fn gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self> {
1497        Some(Inst::imm(OperandSize::Size64, value, dst))
1498    }
1499
1500    fn gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
1501        let imm_to_gpr = Inst::imm(OperandSize::Size64, value.to_bits(), tmp);
1502        let gpr_to_xmm = Inst::External {
1503            inst: asm::inst::movq_a::new(dst.map(|r| Xmm::new(r).unwrap()), tmp.to_reg()).into(),
1504        };
1505        smallvec![imm_to_gpr, gpr_to_xmm]
1506    }
1507
1508    fn gen_dummy_use(reg: Reg) -> Self {
1509        Inst::DummyUse { reg }
1510    }
1511
1512    fn worst_case_size() -> CodeOffset {
1513        15
1514    }
1515
1516    fn worst_case_island_growth() -> CodeOffset {
1517        0
1518    }
1519
1520    fn is_safepoint(&self) -> bool {
1521        match self {
1522            Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => {
1523                true
1524            }
1525            _ => false,
1526        }
1527    }
1528
1529    fn function_alignment() -> FunctionAlignment {
1530        FunctionAlignment {
1531            minimum: 1,
1532            // Change the alignment from 16-bytes to 32-bytes for better performance.
1533            // fix-8573: https://github.com/bytecodealliance/wasmtime/issues/8573
1534            preferred: 32,
1535        }
1536    }
1537
1538    type LabelUse = LabelUse;
1539
1540    const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
1541}
1542
1543/// Constant state used during emissions of a sequence of instructions.
1544pub struct EmitInfo {
1545    pub(super) flags: settings::Flags,
1546    isa_flags: x64_settings::Flags,
1547}
1548
1549impl EmitInfo {
1550    /// Create a constant state for emission of instructions.
1551    pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
1552        Self { flags, isa_flags }
1553    }
1554}
1555
1556impl asm::AvailableFeatures for &EmitInfo {
1557    fn _64b(&self) -> bool {
1558        // Currently, this x64 backend always assumes 64-bit mode.
1559        true
1560    }
1561
1562    fn compat(&self) -> bool {
1563        // For 32-bit compatibility mode, see
1564        // https://github.com/bytecodealliance/wasmtime/issues/1980 (TODO).
1565        false
1566    }
1567
1568    fn sse(&self) -> bool {
1569        // Currently, this x64 backend always assumes SSE.
1570        true
1571    }
1572
1573    fn sse2(&self) -> bool {
1574        // Currently, this x64 backend always assumes SSE2.
1575        true
1576    }
1577
1578    fn sse3(&self) -> bool {
1579        self.isa_flags.has_sse3()
1580    }
1581
1582    fn ssse3(&self) -> bool {
1583        self.isa_flags.has_ssse3()
1584    }
1585
1586    fn sse41(&self) -> bool {
1587        self.isa_flags.has_sse41()
1588    }
1589
1590    fn sse42(&self) -> bool {
1591        self.isa_flags.has_sse42()
1592    }
1593
1594    fn bmi1(&self) -> bool {
1595        self.isa_flags.has_bmi1()
1596    }
1597
1598    fn bmi2(&self) -> bool {
1599        self.isa_flags.has_bmi2()
1600    }
1601
1602    fn lzcnt(&self) -> bool {
1603        self.isa_flags.has_lzcnt()
1604    }
1605
1606    fn popcnt(&self) -> bool {
1607        self.isa_flags.has_popcnt()
1608    }
1609
1610    fn avx(&self) -> bool {
1611        self.isa_flags.has_avx()
1612    }
1613
1614    fn avx2(&self) -> bool {
1615        self.isa_flags.has_avx2()
1616    }
1617
1618    fn avx512f(&self) -> bool {
1619        self.isa_flags.has_avx512f()
1620    }
1621
1622    fn avx512vl(&self) -> bool {
1623        self.isa_flags.has_avx512vl()
1624    }
1625
1626    fn cmpxchg16b(&self) -> bool {
1627        self.isa_flags.has_cmpxchg16b()
1628    }
1629
1630    fn fma(&self) -> bool {
1631        self.isa_flags.has_fma()
1632    }
1633    fn avx_vnni(&self) -> bool {
1634        self.isa_flags.has_avx_vnni()
1635    }
1636
1637    fn avx512dq(&self) -> bool {
1638        self.isa_flags.has_avx512dq()
1639    }
1640
1641    fn avx512bitalg(&self) -> bool {
1642        self.isa_flags.has_avx512bitalg()
1643    }
1644
1645    fn avx512vbmi(&self) -> bool {
1646        self.isa_flags.has_avx512vbmi()
1647    }
1648
1649    fn avx512vnni(&self) -> bool {
1650        self.isa_flags.has_avx512vnni()
1651    }
1652
1653    fn apx(&self) -> bool {
1654        self.isa_flags.has_apx()
1655    }
1656}
1657
1658impl MachInstEmit for Inst {
1659    type State = EmitState;
1660    type Info = EmitInfo;
1661
1662    fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) {
1663        emit::emit(self, sink, info, state);
1664    }
1665
1666    fn pretty_print_inst(&self, _: &mut Self::State) -> String {
1667        PrettyPrint::pretty_print(self, 0)
1668    }
1669}
1670
1671/// A label-use (internal relocation) in generated code.
1672#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1673pub enum LabelUse {
1674    /// A 32-bit offset from location of relocation itself, added to the existing value at that
1675    /// location. Used for control flow instructions which consider an offset from the start of the
1676    /// next instruction (so the size of the payload -- 4 bytes -- is subtracted from the payload).
1677    JmpRel32,
1678
1679    /// A 32-bit offset from location of relocation itself, added to the existing value at that
1680    /// location.
1681    PCRel32,
1682}
1683
1684impl MachInstLabelUse for LabelUse {
1685    const ALIGN: CodeOffset = 1;
1686
1687    fn max_pos_range(self) -> CodeOffset {
1688        match self {
1689            LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
1690        }
1691    }
1692
1693    fn max_neg_range(self) -> CodeOffset {
1694        match self {
1695            LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
1696        }
1697    }
1698
1699    fn patch_size(self) -> CodeOffset {
1700        match self {
1701            LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
1702        }
1703    }
1704
1705    fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
1706        let pc_rel = (label_offset as i64) - (use_offset as i64);
1707        debug_assert!(pc_rel <= self.max_pos_range() as i64);
1708        debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
1709        let pc_rel = pc_rel as u32;
1710        match self {
1711            LabelUse::JmpRel32 => {
1712                let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1713                let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
1714                buffer.copy_from_slice(&value.to_le_bytes()[..]);
1715            }
1716            LabelUse::PCRel32 => {
1717                let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
1718                let value = pc_rel.wrapping_add(addend);
1719                buffer.copy_from_slice(&value.to_le_bytes()[..]);
1720            }
1721        }
1722    }
1723
1724    fn supports_veneer(self) -> bool {
1725        match self {
1726            LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
1727        }
1728    }
1729
1730    fn veneer_size(self) -> CodeOffset {
1731        match self {
1732            LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
1733        }
1734    }
1735
1736    fn worst_case_veneer_size() -> CodeOffset {
1737        0
1738    }
1739
1740    fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
1741        match self {
1742            LabelUse::JmpRel32 | LabelUse::PCRel32 => {
1743                panic!("Veneer not supported for JumpRel32 label-use.");
1744            }
1745        }
1746    }
1747
1748    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
1749        match (reloc, addend) {
1750            (Reloc::X86PCRel4 | Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
1751            _ => None,
1752        }
1753    }
1754}
cranelift_codegen/isa/x64/inst/mod.rs

cranelift_codegen/isa/x64/inst/
mod.rs
