cranelift_codegen/isa/x64/inst/
args.rs

1//! Instruction operand sub-components (aka "parts"): definitions and printing.
2
3use super::regs::{self};
4use crate::ir::condcodes::{FloatCC, IntCC};
5use crate::ir::types::*;
6use crate::ir::MemFlags;
7use crate::isa::x64::inst::regs::pretty_print_reg;
8use crate::isa::x64::inst::Inst;
9use crate::machinst::*;
10use smallvec::{smallvec, SmallVec};
11use std::fmt;
12use std::string::String;
13
14pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;
15
16/// An extension trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
17pub trait ToWritableReg {
18    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
19    fn to_writable_reg(&self) -> Writable<Reg>;
20}
21
22/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.
23pub trait FromWritableReg: Sized {
24    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.
25    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
26}
27
28/// A macro for defining a newtype of `Reg` that enforces some invariant about
29/// the wrapped `Reg` (such as that it is of a particular register class).
30macro_rules! newtype_of_reg {
31    (
32        $newtype_reg:ident,
33        $newtype_writable_reg:ident,
34        $newtype_option_writable_reg:ident,
35        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
36        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
37        $newtype_imm8_reg:ident,
38        |$check_reg:ident| $check:expr
39    ) => {
40        /// A newtype wrapper around `Reg`.
41        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
42        pub struct $newtype_reg(Reg);
43
44        impl PartialEq<Reg> for $newtype_reg {
45            fn eq(&self, other: &Reg) -> bool {
46                self.0 == *other
47            }
48        }
49
50        impl From<$newtype_reg> for Reg {
51            fn from(r: $newtype_reg) -> Self {
52                r.0
53            }
54        }
55
56        impl $newtype_reg {
57            /// Create this newtype from the given register, or return `None` if the register
58            /// is not a valid instance of this newtype.
59            pub fn new($check_reg: Reg) -> Option<Self> {
60                if $check {
61                    Some(Self($check_reg))
62                } else {
63                    None
64                }
65            }
66
67            /// Like `Self::new(r).unwrap()` but with a better panic message on
68            /// failure.
69            pub fn unwrap_new($check_reg: Reg) -> Self {
70                if $check {
71                    Self($check_reg)
72                } else {
73                    panic!(
74                        "cannot construct {} from register {:?} with register class {:?}",
75                        stringify!($newtype_reg),
76                        $check_reg,
77                        $check_reg.class(),
78                    )
79                }
80            }
81
82            /// Get this newtype's underlying `Reg`.
83            pub fn to_reg(self) -> Reg {
84                self.0
85            }
86        }
87
88        // Convenience impl so that people working with this newtype can use it
89        // "just like" a plain `Reg`.
90        //
91        // NB: We cannot implement `DerefMut` because that would let people do
92        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the
93        // invariants that `Gpr` provides.
94        impl std::ops::Deref for $newtype_reg {
95            type Target = Reg;
96
97            fn deref(&self) -> &Reg {
98                &self.0
99            }
100        }
101
102        /// If you know what you're doing, you can explicitly mutably borrow the
103        /// underlying `Reg`. Don't make it point to the wrong type of register
104        /// please.
105        impl AsMut<Reg> for $newtype_reg {
106            fn as_mut(&mut self) -> &mut Reg {
107                &mut self.0
108            }
109        }
110
111        /// Writable Gpr.
112        pub type $newtype_writable_reg = Writable<$newtype_reg>;
113
114        #[allow(dead_code)] // Used by some newtypes and not others.
115        /// Optional writable Gpr.
116        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
117
118        impl ToWritableReg for $newtype_writable_reg {
119            fn to_writable_reg(&self) -> Writable<Reg> {
120                Writable::from_reg(self.to_reg().to_reg())
121            }
122        }
123
124        impl FromWritableReg for $newtype_writable_reg {
125            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {
126                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))
127            }
128        }
129
130        $(
131            /// A newtype wrapper around `RegMem` for general-purpose registers.
132            #[derive(Clone, Debug)]
133            pub struct $newtype_reg_mem(RegMem);
134
135            impl From<$newtype_reg_mem> for RegMem {
136                fn from(rm: $newtype_reg_mem) -> Self {
137                    rm.0
138                }
139            }
140            impl<'a> From<&'a $newtype_reg_mem> for &'a RegMem {
141                fn from(rm: &'a $newtype_reg_mem) -> &'a RegMem {
142                    &rm.0
143                }
144            }
145
146            impl From<$newtype_reg> for $newtype_reg_mem {
147                fn from(r: $newtype_reg) -> Self {
148                    $newtype_reg_mem(RegMem::reg(r.into()))
149                }
150            }
151
152            impl $newtype_reg_mem {
153                /// Construct a `RegMem` newtype from the given `RegMem`, or return
154                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
155                /// newtype.
156                pub fn new(rm: RegMem) -> Option<Self> {
157                    match rm {
158                        RegMem::Mem { addr } => {
159                            let mut _allow = true;
160                            $(
161                                if $aligned {
162                                    _allow = addr.aligned();
163                                }
164                            )?
165                            if _allow {
166                                Some(Self(RegMem::Mem { addr }))
167                            } else {
168                                None
169                            }
170                        }
171                        RegMem::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
172                    }
173                }
174
175                /// Like `Self::new(rm).unwrap()` but with better panic messages
176                /// in case of failure.
177                pub fn unwrap_new(rm: RegMem) -> Self {
178                    match rm {
179                        RegMem::Mem { addr } => {
180                            $(
181                                if $aligned && !addr.aligned() {
182                                    panic!(
183                                        "cannot create {} from an unaligned memory address: {addr:?}",
184                                        stringify!($newtype_reg_mem),
185                                    );
186                                }
187                            )?
188                            Self(RegMem::Mem { addr })
189                        }
190                        RegMem::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
191                    }
192                }
193
194                /// Convert this newtype into its underlying `RegMem`.
195                pub fn to_reg_mem(self) -> RegMem {
196                    self.0
197                }
198
199                #[allow(dead_code)] // Used by some newtypes and not others.
200                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
201                    self.0.get_operands(collector);
202                }
203            }
204            impl PrettyPrint for $newtype_reg_mem {
205                fn pretty_print(&self, size: u8) -> String {
206                    self.0.pretty_print(size)
207                }
208            }
209        )*
210
211        $(
212            /// A newtype wrapper around `RegMemImm`.
213            #[derive(Clone, Debug)]
214            pub struct $newtype_reg_mem_imm(RegMemImm);
215
216            impl From<$newtype_reg_mem_imm> for RegMemImm {
217                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
218                    rmi.0
219                }
220            }
221            impl<'a> From<&'a $newtype_reg_mem_imm> for &'a RegMemImm {
222                fn from(rmi: &'a $newtype_reg_mem_imm) -> &'a RegMemImm {
223                    &rmi.0
224                }
225            }
226
227            impl From<$newtype_reg> for $newtype_reg_mem_imm {
228                fn from(r: $newtype_reg) -> Self {
229                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
230                }
231            }
232
233            impl $newtype_reg_mem_imm {
234                /// Construct this newtype from the given `RegMemImm`, or return
235                /// `None` if the `RegMemImm` is not a valid instance of this
236                /// newtype.
237                pub fn new(rmi: RegMemImm) -> Option<Self> {
238                    match rmi {
239                        RegMemImm::Imm { .. } => Some(Self(rmi)),
240                        RegMemImm::Mem { addr } => {
241                            let mut _allow = true;
242                            $(
243                                if $aligned_imm {
244                                    _allow = addr.aligned();
245                                }
246                            )?
247                            if _allow {
248                                Some(Self(RegMemImm::Mem { addr }))
249                            } else {
250                                None
251                            }
252                        }
253                        RegMemImm::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
254                    }
255                }
256
257                /// Like `Self::new(rmi).unwrap()` but with better panic
258                /// messages in case of failure.
259                pub fn unwrap_new(rmi: RegMemImm) -> Self {
260                    match rmi {
261                        RegMemImm::Imm { .. } => Self(rmi),
262                        RegMemImm::Mem { addr } => {
263                            $(
264                                if $aligned_imm && !addr.aligned() {
265                                    panic!(
266                                        "cannot construct {} from unaligned memory address: {:?}",
267                                        stringify!($newtype_reg_mem_imm),
268                                        addr,
269                                    );
270                                }
271                            )?
272                            Self(RegMemImm::Mem { addr })
273
274                        }
275                        RegMemImm::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
276                    }
277                }
278
279                /// Convert this newtype into its underlying `RegMemImm`.
280                #[allow(dead_code)] // Used by some newtypes and not others.
281                pub fn to_reg_mem_imm(self) -> RegMemImm {
282                    self.0
283                }
284
285                #[allow(dead_code)] // Used by some newtypes and not others.
286                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
287                    self.0.get_operands(collector);
288                }
289            }
290
291            impl PrettyPrint for $newtype_reg_mem_imm {
292                fn pretty_print(&self, size: u8) -> String {
293                    self.0.pretty_print(size)
294                }
295            }
296        )*
297
298        /// A newtype wrapper around `Imm8Reg`.
299        #[derive(Clone, Debug)]
300        #[allow(dead_code)] // Used by some newtypes and not others.
301        pub struct $newtype_imm8_reg(Imm8Reg);
302
303        impl From<$newtype_reg> for $newtype_imm8_reg {
304            fn from(r: $newtype_reg) -> Self {
305                Self(Imm8Reg::Reg { reg: r.to_reg() })
306            }
307        }
308
309        impl $newtype_imm8_reg {
310            /// Construct this newtype from the given `Imm8Reg`, or return
311            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.
312            #[allow(dead_code)] // Used by some newtypes and not others.
313            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
314                match imm8_reg {
315                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
316                    Imm8Reg::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
317                }
318            }
319
320            /// Like `Self::new(imm8_reg).unwrap()` but with better panic
321            /// messages on failure.
322            pub fn unwrap_new(imm8_reg: Imm8Reg) -> Self {
323                match imm8_reg {
324                    Imm8Reg::Imm8 { .. } => Self(imm8_reg),
325                    Imm8Reg::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
326                }
327            }
328
329            /// Borrow this newtype as its underlying `Imm8Reg`.
330            #[allow(dead_code)] // Used by some newtypes and not others.
331            pub fn as_imm8_reg(&self) -> &Imm8Reg {
332                &self.0
333            }
334
335            /// Borrow this newtype as its underlying `Imm8Reg`.
336            #[allow(dead_code)] // Used by some newtypes and not others.
337            pub fn as_imm8_reg_mut(&mut self) -> &mut Imm8Reg {
338                &mut self.0
339            }
340        }
341    };
342}
343
344// Define a newtype of `Reg` for general-purpose registers.
345newtype_of_reg!(
346    Gpr,
347    WritableGpr,
348    OptionWritableGpr,
349    reg_mem: (GprMem),
350    reg_mem_imm: (GprMemImm),
351    Imm8Gpr,
352    |reg| reg.class() == RegClass::Int
353);
354
355// Define a newtype of `Reg` for XMM registers.
356newtype_of_reg!(
357    Xmm,
358    WritableXmm,
359    OptionWritableXmm,
360    reg_mem: (XmmMem, XmmMemAligned aligned:true),
361    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
362    Imm8Xmm,
363    |reg| reg.class() == RegClass::Float
364);
365
366// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
367// constructors here.
368
369// Re-export the type from the ISLE generated code.
370pub use crate::isa::x64::lower::isle::generated_code::Amode;
371
372impl Amode {
373    /// Create an immediate sign-extended and register addressing mode.
374    pub fn imm_reg(simm32: i32, base: Reg) -> Self {
375        debug_assert!(base.class() == RegClass::Int);
376        Self::ImmReg {
377            simm32,
378            base,
379            flags: MemFlags::trusted(),
380        }
381    }
382
383    /// Create a sign-extended-32-to-64 with register and shift addressing mode.
384    pub fn imm_reg_reg_shift(simm32: i32, base: Gpr, index: Gpr, shift: u8) -> Self {
385        debug_assert!(base.class() == RegClass::Int);
386        debug_assert!(index.class() == RegClass::Int);
387        debug_assert!(shift <= 3);
388        Self::ImmRegRegShift {
389            simm32,
390            base,
391            index,
392            shift,
393            flags: MemFlags::trusted(),
394        }
395    }
396
397    pub(crate) fn rip_relative(target: MachLabel) -> Self {
398        Self::RipRelative { target }
399    }
400
401    /// Set the specified [MemFlags] to the [Amode].
402    pub fn with_flags(&self, flags: MemFlags) -> Self {
403        match self {
404            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
405                simm32,
406                base,
407                flags,
408            },
409            &Self::ImmRegRegShift {
410                simm32,
411                base,
412                index,
413                shift,
414                ..
415            } => Self::ImmRegRegShift {
416                simm32,
417                base,
418                index,
419                shift,
420                flags,
421            },
422            _ => panic!("Amode {self:?} cannot take memflags"),
423        }
424    }
425
426    /// Add the registers mentioned by `self` to `collector`.
427    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
428        match self {
429            Amode::ImmReg { base, .. } => {
430                if *base != regs::rbp() && *base != regs::rsp() {
431                    collector.reg_use(base);
432                }
433            }
434            Amode::ImmRegRegShift { base, index, .. } => {
435                debug_assert_ne!(base.to_reg(), regs::rbp());
436                debug_assert_ne!(base.to_reg(), regs::rsp());
437                collector.reg_use(base);
438                debug_assert_ne!(index.to_reg(), regs::rbp());
439                debug_assert_ne!(index.to_reg(), regs::rsp());
440                collector.reg_use(index);
441            }
442            Amode::RipRelative { .. } => {
443                // RIP isn't involved in regalloc.
444            }
445        }
446    }
447
448    /// Same as `get_operands`, but add the registers in the "late" phase.
449    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
450        match self {
451            Amode::ImmReg { base, .. } => {
452                collector.reg_late_use(base);
453            }
454            Amode::ImmRegRegShift { base, index, .. } => {
455                collector.reg_late_use(base);
456                collector.reg_late_use(index);
457            }
458            Amode::RipRelative { .. } => {
459                // RIP isn't involved in regalloc.
460            }
461        }
462    }
463
464    pub(crate) fn get_flags(&self) -> MemFlags {
465        match self {
466            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,
467            Amode::RipRelative { .. } => MemFlags::trusted(),
468        }
469    }
470
471    /// Offset the amode by a fixed offset.
472    pub(crate) fn offset(&self, offset: i32) -> Self {
473        let mut ret = self.clone();
474        match &mut ret {
475            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
476            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
477            _ => panic!("Cannot offset amode: {self:?}"),
478        }
479        ret
480    }
481
482    pub(crate) fn aligned(&self) -> bool {
483        self.get_flags().aligned()
484    }
485}
486
487impl PrettyPrint for Amode {
488    fn pretty_print(&self, _size: u8) -> String {
489        match self {
490            Amode::ImmReg { simm32, base, .. } => {
491                // Note: size is always 8; the address is 64 bits,
492                // even if the addressed operand is smaller.
493                format!("{}({})", *simm32, pretty_print_reg(*base, 8))
494            }
495            Amode::ImmRegRegShift {
496                simm32,
497                base,
498                index,
499                shift,
500                ..
501            } => format!(
502                "{}({},{},{})",
503                *simm32,
504                pretty_print_reg(base.to_reg(), 8),
505                pretty_print_reg(index.to_reg(), 8),
506                1 << shift
507            ),
508            Amode::RipRelative { target } => format!("label{}(%rip)", target.as_u32()),
509        }
510    }
511}
512
513/// A Memory Address. These denote a 64-bit value only.
514/// Used for usual addressing modes as well as addressing modes used during compilation, when the
515/// moving SP offset is not known.
516#[derive(Clone, Debug)]
517pub enum SyntheticAmode {
518    /// A real amode.
519    Real(Amode),
520
521    /// A (virtual) offset into the incoming argument area.
522    IncomingArg {
523        /// The downward offset from the start of the incoming argument area.
524        offset: u32,
525    },
526
527    /// A (virtual) offset to the slot area of the function frame, which lies just above the
528    /// outgoing arguments.
529    SlotOffset {
530        /// The offset into the slot area.
531        simm32: i32,
532    },
533
534    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.
535    ConstantOffset(VCodeConstant),
536}
537
538impl SyntheticAmode {
539    /// Create a real addressing mode.
540    pub fn real(amode: Amode) -> Self {
541        Self::Real(amode)
542    }
543
544    pub(crate) fn slot_offset(simm32: i32) -> Self {
545        SyntheticAmode::SlotOffset { simm32 }
546    }
547
548    /// Add the registers mentioned by `self` to `collector`.
549    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
550        match self {
551            SyntheticAmode::Real(addr) => addr.get_operands(collector),
552            SyntheticAmode::IncomingArg { .. } => {
553                // Nothing to do; the base is known and isn't involved in regalloc.
554            }
555            SyntheticAmode::SlotOffset { .. } => {
556                // Nothing to do; the base is SP and isn't involved in regalloc.
557            }
558            SyntheticAmode::ConstantOffset(_) => {}
559        }
560    }
561
562    /// Same as `get_operands`, but add the register in the "late" phase.
563    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
564        match self {
565            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
566            SyntheticAmode::IncomingArg { .. } => {
567                // Nothing to do; the base is known and isn't involved in regalloc.
568            }
569            SyntheticAmode::SlotOffset { .. } => {
570                // Nothing to do; the base is SP and isn't involved in regalloc.
571            }
572            SyntheticAmode::ConstantOffset(_) => {}
573        }
574    }
575
576    pub(crate) fn finalize(&self, frame: &FrameLayout, buffer: &mut MachBuffer<Inst>) -> Amode {
577        match self {
578            SyntheticAmode::Real(addr) => addr.clone(),
579            SyntheticAmode::IncomingArg { offset } => {
580                // NOTE: this could be made relative to RSP by adding additional
581                // offsets from the frame_layout.
582                let args_max_fp_offset = frame.tail_args_size + frame.setup_area_size;
583                Amode::imm_reg(
584                    i32::try_from(args_max_fp_offset - offset).unwrap(),
585                    regs::rbp(),
586                )
587            }
588            SyntheticAmode::SlotOffset { simm32 } => {
589                let off = *simm32 as i64 + i64::from(frame.outgoing_args_size);
590                Amode::imm_reg(off.try_into().expect("invalid sp offset"), regs::rsp())
591            }
592            SyntheticAmode::ConstantOffset(c) => {
593                Amode::rip_relative(buffer.get_label_for_constant(*c))
594            }
595        }
596    }
597
598    pub(crate) fn aligned(&self) -> bool {
599        match self {
600            SyntheticAmode::Real(addr) => addr.aligned(),
601            &SyntheticAmode::IncomingArg { .. }
602            | SyntheticAmode::SlotOffset { .. }
603            | SyntheticAmode::ConstantOffset { .. } => true,
604        }
605    }
606}
607
608impl Into<SyntheticAmode> for Amode {
609    fn into(self) -> SyntheticAmode {
610        SyntheticAmode::Real(self)
611    }
612}
613
614impl Into<SyntheticAmode> for VCodeConstant {
615    fn into(self) -> SyntheticAmode {
616        SyntheticAmode::ConstantOffset(self)
617    }
618}
619
620impl PrettyPrint for SyntheticAmode {
621    fn pretty_print(&self, _size: u8) -> String {
622        match self {
623            // See note in `Amode` regarding constant size of `8`.
624            SyntheticAmode::Real(addr) => addr.pretty_print(8),
625            &SyntheticAmode::IncomingArg { offset } => {
626                format!("rbp(stack args max - {offset})")
627            }
628            SyntheticAmode::SlotOffset { simm32 } => {
629                format!("rsp({} + virtual offset)", *simm32)
630            }
631            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),
632        }
633    }
634}
635
636/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can
637/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only
638/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by
639/// `simm32` is its sign-extension out to 64 bits.
640#[derive(Clone, Debug)]
641pub enum RegMemImm {
642    /// A register operand.
643    Reg {
644        /// The underlying register.
645        reg: Reg,
646    },
647    /// A memory operand.
648    Mem {
649        /// The memory address.
650        addr: SyntheticAmode,
651    },
652    /// An immediate operand.
653    Imm {
654        /// The immediate value.
655        simm32: u32,
656    },
657}
658
659impl RegMemImm {
660    /// Create a register operand.
661    pub fn reg(reg: Reg) -> Self {
662        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
663        Self::Reg { reg }
664    }
665
666    /// Create a memory operand.
667    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
668        Self::Mem { addr: addr.into() }
669    }
670
671    /// Create an immediate operand.
672    pub fn imm(simm32: u32) -> Self {
673        Self::Imm { simm32 }
674    }
675
676    /// Asserts that in register mode, the reg class is the one that's expected.
677    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
678        if let Self::Reg { reg } = self {
679            debug_assert_eq!(reg.class(), expected_reg_class);
680        }
681    }
682
683    /// Add the regs mentioned by `self` to `collector`.
684    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
685        match self {
686            Self::Reg { reg } => collector.reg_use(reg),
687            Self::Mem { addr } => addr.get_operands(collector),
688            Self::Imm { .. } => {}
689        }
690    }
691}
692
693impl From<RegMem> for RegMemImm {
694    fn from(rm: RegMem) -> RegMemImm {
695        match rm {
696            RegMem::Reg { reg } => RegMemImm::Reg { reg },
697            RegMem::Mem { addr } => RegMemImm::Mem { addr },
698        }
699    }
700}
701
702impl From<Reg> for RegMemImm {
703    fn from(reg: Reg) -> Self {
704        RegMemImm::Reg { reg }
705    }
706}
707
708impl PrettyPrint for RegMemImm {
709    fn pretty_print(&self, size: u8) -> String {
710        match self {
711            Self::Reg { reg } => pretty_print_reg(*reg, size),
712            Self::Mem { addr } => addr.pretty_print(size),
713            Self::Imm { simm32 } => format!("${}", *simm32 as i32),
714        }
715    }
716}
717
718/// An operand which is either an 8-bit integer immediate or a register.
719#[derive(Clone, Debug)]
720pub enum Imm8Reg {
721    /// 8-bit immediate operand.
722    Imm8 {
723        /// The 8-bit immediate value.
724        imm: u8,
725    },
726    /// A register operand.
727    Reg {
728        /// The underlying register.
729        reg: Reg,
730    },
731}
732
733impl From<u8> for Imm8Reg {
734    fn from(imm: u8) -> Self {
735        Self::Imm8 { imm }
736    }
737}
738
739impl From<Reg> for Imm8Reg {
740    fn from(reg: Reg) -> Self {
741        Self::Reg { reg }
742    }
743}
744
745/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
746/// 32, 64, or 128 bit value.
747#[derive(Clone, Debug)]
748pub enum RegMem {
749    /// A register operand.
750    Reg {
751        /// The underlying register.
752        reg: Reg,
753    },
754    /// A memory operand.
755    Mem {
756        /// The memory address.
757        addr: SyntheticAmode,
758    },
759}
760
761impl RegMem {
762    /// Create a register operand.
763    pub fn reg(reg: Reg) -> Self {
764        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
765        Self::Reg { reg }
766    }
767
768    /// Create a memory operand.
769    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
770        Self::Mem { addr: addr.into() }
771    }
772    /// Asserts that in register mode, the reg class is the one that's expected.
773    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
774        if let Self::Reg { reg } = self {
775            debug_assert_eq!(reg.class(), expected_reg_class);
776        }
777    }
778    /// Add the regs mentioned by `self` to `collector`.
779    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
780        match self {
781            RegMem::Reg { reg } => collector.reg_use(reg),
782            RegMem::Mem { addr, .. } => addr.get_operands(collector),
783        }
784    }
785}
786
787impl From<Reg> for RegMem {
788    fn from(reg: Reg) -> RegMem {
789        RegMem::Reg { reg }
790    }
791}
792
793impl From<Writable<Reg>> for RegMem {
794    fn from(r: Writable<Reg>) -> Self {
795        RegMem::reg(r.to_reg())
796    }
797}
798
799impl PrettyPrint for RegMem {
800    fn pretty_print(&self, size: u8) -> String {
801        match self {
802            RegMem::Reg { reg } => pretty_print_reg(*reg, size),
803            RegMem::Mem { addr, .. } => addr.pretty_print(size),
804        }
805    }
806}
807
808pub use crate::isa::x64::lower::isle::generated_code::AluRmROpcode;
809
810impl AluRmROpcode {
811    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
812        match self {
813            AluRmROpcode::Andn => smallvec![InstructionSet::BMI1],
814            AluRmROpcode::Sarx | AluRmROpcode::Shrx | AluRmROpcode::Shlx | AluRmROpcode::Bzhi => {
815                smallvec![InstructionSet::BMI2]
816            }
817        }
818    }
819}
820
821impl fmt::Display for AluRmROpcode {
822    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
823        f.write_str(&format!("{self:?}").to_lowercase())
824    }
825}
826
827#[derive(Clone, PartialEq)]
828/// Unary operations requiring register or memory and register operands.
829pub enum UnaryRmROpcode {
830    /// Bit-scan reverse.
831    Bsr,
832    /// Bit-scan forward.
833    Bsf,
834    /// Counts leading zeroes (Leading Zero CouNT).
835    Lzcnt,
836    /// Counts trailing zeroes (Trailing Zero CouNT).
837    Tzcnt,
838    /// Counts the number of ones (POPulation CouNT).
839    Popcnt,
840}
841
842impl UnaryRmROpcode {
843    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
844        match self {
845            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],
846            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],
847            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],
848            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],
849        }
850    }
851}
852
853impl fmt::Debug for UnaryRmROpcode {
854    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
855        match self {
856            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
857            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
858            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
859            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
860            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
861        }
862    }
863}
864
865impl fmt::Display for UnaryRmROpcode {
866    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
867        fmt::Debug::fmt(self, f)
868    }
869}
870
871pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRVexOpcode;
872
873impl UnaryRmRVexOpcode {
874    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
875        match self {
876            UnaryRmRVexOpcode::Blsi | UnaryRmRVexOpcode::Blsmsk | UnaryRmRVexOpcode::Blsr => {
877                smallvec![InstructionSet::BMI1]
878            }
879        }
880    }
881}
882
883impl fmt::Display for UnaryRmRVexOpcode {
884    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
885        f.write_str(&format!("{self:?}").to_lowercase())
886    }
887}
888
889pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRImmVexOpcode;
890
891impl UnaryRmRImmVexOpcode {
892    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
893        match self {
894            UnaryRmRImmVexOpcode::Rorx => {
895                smallvec![InstructionSet::BMI2]
896            }
897        }
898    }
899}
900
901impl fmt::Display for UnaryRmRImmVexOpcode {
902    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
903        f.write_str(&format!("{self:?}").to_lowercase())
904    }
905}
906
907#[derive(Clone, Copy, PartialEq)]
908/// Comparison operations.
909pub enum CmpOpcode {
910    /// CMP instruction: compute `a - b` and set flags from result.
911    Cmp,
912    /// TEST instruction: compute `a & b` and set flags from result.
913    Test,
914}
915
916#[derive(Debug)]
917pub(crate) enum InstructionSet {
918    SSE,
919    SSE2,
920    CMPXCHG16b,
921    SSSE3,
922    SSE41,
923    SSE42,
924    Popcnt,
925    Lzcnt,
926    BMI1,
927    #[allow(dead_code)] // never constructed (yet).
928    BMI2,
929    FMA,
930    AVX,
931    AVX2,
932    AVX512BITALG,
933    AVX512DQ,
934    AVX512F,
935    AVX512VBMI,
936    AVX512VL,
937}
938
939/// Some SSE operations requiring 2 operands r/m and r.
940#[derive(Clone, Copy, PartialEq)]
941#[allow(dead_code)] // some variants here aren't used just yet
942#[allow(missing_docs)]
943pub enum SseOpcode {
944    Blendvpd,
945    Blendvps,
946    Comiss,
947    Comisd,
948    Cmpps,
949    Cmppd,
950    Cmpss,
951    Cmpsd,
952    Cvtdq2ps,
953    Cvtdq2pd,
954    Cvtpd2ps,
955    Cvtps2pd,
956    Cvtsd2ss,
957    Cvtsd2si,
958    Cvtsi2ss,
959    Cvtsi2sd,
960    Cvtss2si,
961    Cvtss2sd,
962    Cvttpd2dq,
963    Cvttps2dq,
964    Cvttss2si,
965    Cvttsd2si,
966    Divps,
967    Divpd,
968    Divss,
969    Divsd,
970    Insertps,
971    Maxps,
972    Maxpd,
973    Maxss,
974    Maxsd,
975    Minps,
976    Minpd,
977    Minss,
978    Minsd,
979    Movaps,
980    Movapd,
981    Movd,
982    Movdqa,
983    Movdqu,
984    Movlhps,
985    Movmskps,
986    Movmskpd,
987    Movq,
988    Movss,
989    Movsd,
990    Movups,
991    Movupd,
992    Mulps,
993    Mulpd,
994    Mulss,
995    Mulsd,
996    Pabsb,
997    Pabsw,
998    Pabsd,
999    Packssdw,
1000    Packsswb,
1001    Packusdw,
1002    Packuswb,
1003    Palignr,
1004    Pavgb,
1005    Pavgw,
1006    Pblendvb,
1007    Pcmpeqb,
1008    Pcmpeqw,
1009    Pcmpeqd,
1010    Pcmpeqq,
1011    Pcmpgtb,
1012    Pcmpgtw,
1013    Pcmpgtd,
1014    Pcmpgtq,
1015    Pextrb,
1016    Pextrw,
1017    Pextrd,
1018    Pextrq,
1019    Pinsrb,
1020    Pinsrw,
1021    Pinsrd,
1022    Pmaddubsw,
1023    Pmaddwd,
1024    Pmaxsb,
1025    Pmaxsw,
1026    Pmaxsd,
1027    Pmaxub,
1028    Pmaxuw,
1029    Pmaxud,
1030    Pminsb,
1031    Pminsw,
1032    Pminsd,
1033    Pminub,
1034    Pminuw,
1035    Pminud,
1036    Pmovmskb,
1037    Pmovsxbd,
1038    Pmovsxbw,
1039    Pmovsxbq,
1040    Pmovsxwd,
1041    Pmovsxwq,
1042    Pmovsxdq,
1043    Pmovzxbd,
1044    Pmovzxbw,
1045    Pmovzxbq,
1046    Pmovzxwd,
1047    Pmovzxwq,
1048    Pmovzxdq,
1049    Pmuldq,
1050    Pmulhw,
1051    Pmulhuw,
1052    Pmulhrsw,
1053    Pmulld,
1054    Pmullw,
1055    Pmuludq,
1056    Pshufb,
1057    Pshufd,
1058    Psllw,
1059    Pslld,
1060    Psllq,
1061    Psraw,
1062    Psrad,
1063    Psrlw,
1064    Psrld,
1065    Psrlq,
1066    Ptest,
1067    Punpckhbw,
1068    Punpckhwd,
1069    Punpcklbw,
1070    Punpcklwd,
1071    Rcpss,
1072    Roundps,
1073    Roundpd,
1074    Roundss,
1075    Roundsd,
1076    Rsqrtss,
1077    Shufps,
1078    Sqrtps,
1079    Sqrtpd,
1080    Sqrtss,
1081    Sqrtsd,
1082    Ucomiss,
1083    Ucomisd,
1084    Unpcklps,
1085    Unpcklpd,
1086    Unpckhps,
1087    Punpckhdq,
1088    Punpckldq,
1089    Punpckhqdq,
1090    Punpcklqdq,
1091    Pshuflw,
1092    Pshufhw,
1093    Pblendw,
1094    Movddup,
1095}
1096
1097impl SseOpcode {
1098    /// Which `InstructionSet` is the first supporting this opcode?
1099    pub(crate) fn available_from(&self) -> InstructionSet {
1100        use InstructionSet::*;
1101        match self {
1102            SseOpcode::Comiss
1103            | SseOpcode::Cmpps
1104            | SseOpcode::Cmpss
1105            | SseOpcode::Cvtsi2ss
1106            | SseOpcode::Cvtss2si
1107            | SseOpcode::Cvttss2si
1108            | SseOpcode::Divps
1109            | SseOpcode::Divss
1110            | SseOpcode::Maxps
1111            | SseOpcode::Maxss
1112            | SseOpcode::Minps
1113            | SseOpcode::Minss
1114            | SseOpcode::Movaps
1115            | SseOpcode::Movlhps
1116            | SseOpcode::Movmskps
1117            | SseOpcode::Movss
1118            | SseOpcode::Movups
1119            | SseOpcode::Mulps
1120            | SseOpcode::Mulss
1121            | SseOpcode::Rcpss
1122            | SseOpcode::Rsqrtss
1123            | SseOpcode::Shufps
1124            | SseOpcode::Sqrtps
1125            | SseOpcode::Sqrtss
1126            | SseOpcode::Ucomiss
1127            | SseOpcode::Unpcklps
1128            | SseOpcode::Unpckhps => SSE,
1129
1130            SseOpcode::Cmppd
1131            | SseOpcode::Cmpsd
1132            | SseOpcode::Comisd
1133            | SseOpcode::Cvtdq2ps
1134            | SseOpcode::Cvtdq2pd
1135            | SseOpcode::Cvtpd2ps
1136            | SseOpcode::Cvtps2pd
1137            | SseOpcode::Cvtsd2ss
1138            | SseOpcode::Cvtsd2si
1139            | SseOpcode::Cvtsi2sd
1140            | SseOpcode::Cvtss2sd
1141            | SseOpcode::Cvttpd2dq
1142            | SseOpcode::Cvttps2dq
1143            | SseOpcode::Cvttsd2si
1144            | SseOpcode::Divpd
1145            | SseOpcode::Divsd
1146            | SseOpcode::Maxpd
1147            | SseOpcode::Maxsd
1148            | SseOpcode::Minpd
1149            | SseOpcode::Minsd
1150            | SseOpcode::Movapd
1151            | SseOpcode::Movd
1152            | SseOpcode::Movmskpd
1153            | SseOpcode::Movq
1154            | SseOpcode::Movsd
1155            | SseOpcode::Movupd
1156            | SseOpcode::Movdqa
1157            | SseOpcode::Movdqu
1158            | SseOpcode::Mulpd
1159            | SseOpcode::Mulsd
1160            | SseOpcode::Packssdw
1161            | SseOpcode::Packsswb
1162            | SseOpcode::Packuswb
1163            | SseOpcode::Pavgb
1164            | SseOpcode::Pavgw
1165            | SseOpcode::Pcmpeqb
1166            | SseOpcode::Pcmpeqw
1167            | SseOpcode::Pcmpeqd
1168            | SseOpcode::Pcmpgtb
1169            | SseOpcode::Pcmpgtw
1170            | SseOpcode::Pcmpgtd
1171            | SseOpcode::Pextrw
1172            | SseOpcode::Pinsrw
1173            | SseOpcode::Pmaddwd
1174            | SseOpcode::Pmaxsw
1175            | SseOpcode::Pmaxub
1176            | SseOpcode::Pminsw
1177            | SseOpcode::Pminub
1178            | SseOpcode::Pmovmskb
1179            | SseOpcode::Pmulhw
1180            | SseOpcode::Pmulhuw
1181            | SseOpcode::Pmullw
1182            | SseOpcode::Pmuludq
1183            | SseOpcode::Pshufd
1184            | SseOpcode::Psllw
1185            | SseOpcode::Pslld
1186            | SseOpcode::Psllq
1187            | SseOpcode::Psraw
1188            | SseOpcode::Psrad
1189            | SseOpcode::Psrlw
1190            | SseOpcode::Psrld
1191            | SseOpcode::Psrlq
1192            | SseOpcode::Punpckhbw
1193            | SseOpcode::Punpckhwd
1194            | SseOpcode::Punpcklbw
1195            | SseOpcode::Punpcklwd
1196            | SseOpcode::Sqrtpd
1197            | SseOpcode::Sqrtsd
1198            | SseOpcode::Ucomisd
1199            | SseOpcode::Punpckldq
1200            | SseOpcode::Punpckhdq
1201            | SseOpcode::Punpcklqdq
1202            | SseOpcode::Punpckhqdq
1203            | SseOpcode::Pshuflw
1204            | SseOpcode::Pshufhw
1205            | SseOpcode::Unpcklpd => SSE2,
1206
1207            SseOpcode::Pabsb
1208            | SseOpcode::Pabsw
1209            | SseOpcode::Pabsd
1210            | SseOpcode::Palignr
1211            | SseOpcode::Pmulhrsw
1212            | SseOpcode::Pshufb
1213            | SseOpcode::Pmaddubsw
1214            | SseOpcode::Movddup => SSSE3,
1215
1216            SseOpcode::Blendvpd
1217            | SseOpcode::Blendvps
1218            | SseOpcode::Insertps
1219            | SseOpcode::Packusdw
1220            | SseOpcode::Pblendvb
1221            | SseOpcode::Pcmpeqq
1222            | SseOpcode::Pextrb
1223            | SseOpcode::Pextrd
1224            | SseOpcode::Pextrq
1225            | SseOpcode::Pinsrb
1226            | SseOpcode::Pinsrd
1227            | SseOpcode::Pmaxsb
1228            | SseOpcode::Pmaxsd
1229            | SseOpcode::Pmaxuw
1230            | SseOpcode::Pmaxud
1231            | SseOpcode::Pminsb
1232            | SseOpcode::Pminsd
1233            | SseOpcode::Pminuw
1234            | SseOpcode::Pminud
1235            | SseOpcode::Pmovsxbd
1236            | SseOpcode::Pmovsxbw
1237            | SseOpcode::Pmovsxbq
1238            | SseOpcode::Pmovsxwd
1239            | SseOpcode::Pmovsxwq
1240            | SseOpcode::Pmovsxdq
1241            | SseOpcode::Pmovzxbd
1242            | SseOpcode::Pmovzxbw
1243            | SseOpcode::Pmovzxbq
1244            | SseOpcode::Pmovzxwd
1245            | SseOpcode::Pmovzxwq
1246            | SseOpcode::Pmovzxdq
1247            | SseOpcode::Pmuldq
1248            | SseOpcode::Pmulld
1249            | SseOpcode::Ptest
1250            | SseOpcode::Roundps
1251            | SseOpcode::Roundpd
1252            | SseOpcode::Roundss
1253            | SseOpcode::Roundsd
1254            | SseOpcode::Pblendw => SSE41,
1255
1256            SseOpcode::Pcmpgtq => SSE42,
1257        }
1258    }
1259
1260    /// Returns the src operand size for an instruction.
1261    pub(crate) fn src_size(&self) -> u8 {
1262        match self {
1263            SseOpcode::Movd => 4,
1264            _ => 8,
1265        }
1266    }
1267
1268    /// Is `src2` with this opcode a scalar, as for lane insertions?
1269    pub(crate) fn has_scalar_src2(self) -> bool {
1270        match self {
1271            SseOpcode::Pinsrb | SseOpcode::Pinsrw | SseOpcode::Pinsrd => true,
1272            SseOpcode::Pmovsxbw
1273            | SseOpcode::Pmovsxbd
1274            | SseOpcode::Pmovsxbq
1275            | SseOpcode::Pmovsxwd
1276            | SseOpcode::Pmovsxwq
1277            | SseOpcode::Pmovsxdq => true,
1278            SseOpcode::Pmovzxbw
1279            | SseOpcode::Pmovzxbd
1280            | SseOpcode::Pmovzxbq
1281            | SseOpcode::Pmovzxwd
1282            | SseOpcode::Pmovzxwq
1283            | SseOpcode::Pmovzxdq => true,
1284            _ => false,
1285        }
1286    }
1287}
1288
1289impl fmt::Debug for SseOpcode {
1290    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1291        let name = match self {
1292            SseOpcode::Blendvpd => "blendvpd",
1293            SseOpcode::Blendvps => "blendvps",
1294            SseOpcode::Cmpps => "cmpps",
1295            SseOpcode::Cmppd => "cmppd",
1296            SseOpcode::Cmpss => "cmpss",
1297            SseOpcode::Cmpsd => "cmpsd",
1298            SseOpcode::Comiss => "comiss",
1299            SseOpcode::Comisd => "comisd",
1300            SseOpcode::Cvtdq2ps => "cvtdq2ps",
1301            SseOpcode::Cvtdq2pd => "cvtdq2pd",
1302            SseOpcode::Cvtpd2ps => "cvtpd2ps",
1303            SseOpcode::Cvtps2pd => "cvtps2pd",
1304            SseOpcode::Cvtsd2ss => "cvtsd2ss",
1305            SseOpcode::Cvtsd2si => "cvtsd2si",
1306            SseOpcode::Cvtsi2ss => "cvtsi2ss",
1307            SseOpcode::Cvtsi2sd => "cvtsi2sd",
1308            SseOpcode::Cvtss2si => "cvtss2si",
1309            SseOpcode::Cvtss2sd => "cvtss2sd",
1310            SseOpcode::Cvttpd2dq => "cvttpd2dq",
1311            SseOpcode::Cvttps2dq => "cvttps2dq",
1312            SseOpcode::Cvttss2si => "cvttss2si",
1313            SseOpcode::Cvttsd2si => "cvttsd2si",
1314            SseOpcode::Divps => "divps",
1315            SseOpcode::Divpd => "divpd",
1316            SseOpcode::Divss => "divss",
1317            SseOpcode::Divsd => "divsd",
1318            SseOpcode::Insertps => "insertps",
1319            SseOpcode::Maxps => "maxps",
1320            SseOpcode::Maxpd => "maxpd",
1321            SseOpcode::Maxss => "maxss",
1322            SseOpcode::Maxsd => "maxsd",
1323            SseOpcode::Minps => "minps",
1324            SseOpcode::Minpd => "minpd",
1325            SseOpcode::Minss => "minss",
1326            SseOpcode::Minsd => "minsd",
1327            SseOpcode::Movaps => "movaps",
1328            SseOpcode::Movapd => "movapd",
1329            SseOpcode::Movd => "movd",
1330            SseOpcode::Movdqa => "movdqa",
1331            SseOpcode::Movdqu => "movdqu",
1332            SseOpcode::Movlhps => "movlhps",
1333            SseOpcode::Movmskps => "movmskps",
1334            SseOpcode::Movmskpd => "movmskpd",
1335            SseOpcode::Movq => "movq",
1336            SseOpcode::Movss => "movss",
1337            SseOpcode::Movsd => "movsd",
1338            SseOpcode::Movups => "movups",
1339            SseOpcode::Movupd => "movupd",
1340            SseOpcode::Mulps => "mulps",
1341            SseOpcode::Mulpd => "mulpd",
1342            SseOpcode::Mulss => "mulss",
1343            SseOpcode::Mulsd => "mulsd",
1344            SseOpcode::Pabsb => "pabsb",
1345            SseOpcode::Pabsw => "pabsw",
1346            SseOpcode::Pabsd => "pabsd",
1347            SseOpcode::Packssdw => "packssdw",
1348            SseOpcode::Packsswb => "packsswb",
1349            SseOpcode::Packusdw => "packusdw",
1350            SseOpcode::Packuswb => "packuswb",
1351            SseOpcode::Palignr => "palignr",
1352            SseOpcode::Pavgb => "pavgb",
1353            SseOpcode::Pavgw => "pavgw",
1354            SseOpcode::Pblendvb => "pblendvb",
1355            SseOpcode::Pcmpeqb => "pcmpeqb",
1356            SseOpcode::Pcmpeqw => "pcmpeqw",
1357            SseOpcode::Pcmpeqd => "pcmpeqd",
1358            SseOpcode::Pcmpeqq => "pcmpeqq",
1359            SseOpcode::Pcmpgtb => "pcmpgtb",
1360            SseOpcode::Pcmpgtw => "pcmpgtw",
1361            SseOpcode::Pcmpgtd => "pcmpgtd",
1362            SseOpcode::Pcmpgtq => "pcmpgtq",
1363            SseOpcode::Pextrb => "pextrb",
1364            SseOpcode::Pextrw => "pextrw",
1365            SseOpcode::Pextrd => "pextrd",
1366            SseOpcode::Pextrq => "pextrq",
1367            SseOpcode::Pinsrb => "pinsrb",
1368            SseOpcode::Pinsrw => "pinsrw",
1369            SseOpcode::Pinsrd => "pinsrd",
1370            SseOpcode::Pmaddubsw => "pmaddubsw",
1371            SseOpcode::Pmaddwd => "pmaddwd",
1372            SseOpcode::Pmaxsb => "pmaxsb",
1373            SseOpcode::Pmaxsw => "pmaxsw",
1374            SseOpcode::Pmaxsd => "pmaxsd",
1375            SseOpcode::Pmaxub => "pmaxub",
1376            SseOpcode::Pmaxuw => "pmaxuw",
1377            SseOpcode::Pmaxud => "pmaxud",
1378            SseOpcode::Pminsb => "pminsb",
1379            SseOpcode::Pminsw => "pminsw",
1380            SseOpcode::Pminsd => "pminsd",
1381            SseOpcode::Pminub => "pminub",
1382            SseOpcode::Pminuw => "pminuw",
1383            SseOpcode::Pminud => "pminud",
1384            SseOpcode::Pmovmskb => "pmovmskb",
1385            SseOpcode::Pmovsxbd => "pmovsxbd",
1386            SseOpcode::Pmovsxbw => "pmovsxbw",
1387            SseOpcode::Pmovsxbq => "pmovsxbq",
1388            SseOpcode::Pmovsxwd => "pmovsxwd",
1389            SseOpcode::Pmovsxwq => "pmovsxwq",
1390            SseOpcode::Pmovsxdq => "pmovsxdq",
1391            SseOpcode::Pmovzxbd => "pmovzxbd",
1392            SseOpcode::Pmovzxbw => "pmovzxbw",
1393            SseOpcode::Pmovzxbq => "pmovzxbq",
1394            SseOpcode::Pmovzxwd => "pmovzxwd",
1395            SseOpcode::Pmovzxwq => "pmovzxwq",
1396            SseOpcode::Pmovzxdq => "pmovzxdq",
1397            SseOpcode::Pmuldq => "pmuldq",
1398            SseOpcode::Pmulhw => "pmulhw",
1399            SseOpcode::Pmulhuw => "pmulhuw",
1400            SseOpcode::Pmulhrsw => "pmulhrsw",
1401            SseOpcode::Pmulld => "pmulld",
1402            SseOpcode::Pmullw => "pmullw",
1403            SseOpcode::Pmuludq => "pmuludq",
1404            SseOpcode::Pshufb => "pshufb",
1405            SseOpcode::Pshufd => "pshufd",
1406            SseOpcode::Psllw => "psllw",
1407            SseOpcode::Pslld => "pslld",
1408            SseOpcode::Psllq => "psllq",
1409            SseOpcode::Psraw => "psraw",
1410            SseOpcode::Psrad => "psrad",
1411            SseOpcode::Psrlw => "psrlw",
1412            SseOpcode::Psrld => "psrld",
1413            SseOpcode::Psrlq => "psrlq",
1414            SseOpcode::Ptest => "ptest",
1415            SseOpcode::Punpckhbw => "punpckhbw",
1416            SseOpcode::Punpckhwd => "punpckhwd",
1417            SseOpcode::Punpcklbw => "punpcklbw",
1418            SseOpcode::Punpcklwd => "punpcklwd",
1419            SseOpcode::Rcpss => "rcpss",
1420            SseOpcode::Roundps => "roundps",
1421            SseOpcode::Roundpd => "roundpd",
1422            SseOpcode::Roundss => "roundss",
1423            SseOpcode::Roundsd => "roundsd",
1424            SseOpcode::Rsqrtss => "rsqrtss",
1425            SseOpcode::Shufps => "shufps",
1426            SseOpcode::Sqrtps => "sqrtps",
1427            SseOpcode::Sqrtpd => "sqrtpd",
1428            SseOpcode::Sqrtss => "sqrtss",
1429            SseOpcode::Sqrtsd => "sqrtsd",
1430            SseOpcode::Ucomiss => "ucomiss",
1431            SseOpcode::Ucomisd => "ucomisd",
1432            SseOpcode::Unpcklps => "unpcklps",
1433            SseOpcode::Unpckhps => "unpckhps",
1434            SseOpcode::Punpckldq => "punpckldq",
1435            SseOpcode::Punpckhdq => "punpckhdq",
1436            SseOpcode::Punpcklqdq => "punpcklqdq",
1437            SseOpcode::Punpckhqdq => "punpckhqdq",
1438            SseOpcode::Pshuflw => "pshuflw",
1439            SseOpcode::Pshufhw => "pshufhw",
1440            SseOpcode::Pblendw => "pblendw",
1441            SseOpcode::Movddup => "movddup",
1442            SseOpcode::Unpcklpd => "unpcklpd",
1443        };
1444        write!(fmt, "{name}")
1445    }
1446}
1447
1448impl fmt::Display for SseOpcode {
1449    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1450        fmt::Debug::fmt(self, f)
1451    }
1452}
1453
1454pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
1455
1456impl AvxOpcode {
1457    /// Which `InstructionSet`s support the opcode?
1458    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1459        match self {
1460            AvxOpcode::Vfmadd213ss
1461            | AvxOpcode::Vfmadd213sd
1462            | AvxOpcode::Vfmadd213ps
1463            | AvxOpcode::Vfmadd213pd
1464            | AvxOpcode::Vfmadd132ss
1465            | AvxOpcode::Vfmadd132sd
1466            | AvxOpcode::Vfmadd132ps
1467            | AvxOpcode::Vfmadd132pd
1468            | AvxOpcode::Vfnmadd213ss
1469            | AvxOpcode::Vfnmadd213sd
1470            | AvxOpcode::Vfnmadd213ps
1471            | AvxOpcode::Vfnmadd213pd
1472            | AvxOpcode::Vfnmadd132ss
1473            | AvxOpcode::Vfnmadd132sd
1474            | AvxOpcode::Vfnmadd132ps
1475            | AvxOpcode::Vfnmadd132pd
1476            | AvxOpcode::Vfmsub213ss
1477            | AvxOpcode::Vfmsub213sd
1478            | AvxOpcode::Vfmsub213ps
1479            | AvxOpcode::Vfmsub213pd
1480            | AvxOpcode::Vfmsub132ss
1481            | AvxOpcode::Vfmsub132sd
1482            | AvxOpcode::Vfmsub132ps
1483            | AvxOpcode::Vfmsub132pd
1484            | AvxOpcode::Vfnmsub213ss
1485            | AvxOpcode::Vfnmsub213sd
1486            | AvxOpcode::Vfnmsub213ps
1487            | AvxOpcode::Vfnmsub213pd
1488            | AvxOpcode::Vfnmsub132ss
1489            | AvxOpcode::Vfnmsub132sd
1490            | AvxOpcode::Vfnmsub132ps
1491            | AvxOpcode::Vfnmsub132pd => smallvec![InstructionSet::FMA],
1492            AvxOpcode::Vminps
1493            | AvxOpcode::Vminpd
1494            | AvxOpcode::Vmaxps
1495            | AvxOpcode::Vmaxpd
1496            | AvxOpcode::Vandnps
1497            | AvxOpcode::Vandnpd
1498            | AvxOpcode::Vpandn
1499            | AvxOpcode::Vcmpps
1500            | AvxOpcode::Vcmppd
1501            | AvxOpcode::Vpsrlw
1502            | AvxOpcode::Vpsrld
1503            | AvxOpcode::Vpsrlq
1504            | AvxOpcode::Vpaddb
1505            | AvxOpcode::Vpaddw
1506            | AvxOpcode::Vpaddd
1507            | AvxOpcode::Vpaddq
1508            | AvxOpcode::Vpaddsb
1509            | AvxOpcode::Vpaddsw
1510            | AvxOpcode::Vpaddusb
1511            | AvxOpcode::Vpaddusw
1512            | AvxOpcode::Vpsubb
1513            | AvxOpcode::Vpsubw
1514            | AvxOpcode::Vpsubd
1515            | AvxOpcode::Vpsubq
1516            | AvxOpcode::Vpsubsb
1517            | AvxOpcode::Vpsubsw
1518            | AvxOpcode::Vpsubusb
1519            | AvxOpcode::Vpsubusw
1520            | AvxOpcode::Vpavgb
1521            | AvxOpcode::Vpavgw
1522            | AvxOpcode::Vpand
1523            | AvxOpcode::Vandps
1524            | AvxOpcode::Vandpd
1525            | AvxOpcode::Vpor
1526            | AvxOpcode::Vorps
1527            | AvxOpcode::Vorpd
1528            | AvxOpcode::Vpxor
1529            | AvxOpcode::Vxorps
1530            | AvxOpcode::Vxorpd
1531            | AvxOpcode::Vpmullw
1532            | AvxOpcode::Vpmulld
1533            | AvxOpcode::Vpmulhw
1534            | AvxOpcode::Vpmulhd
1535            | AvxOpcode::Vpmulhrsw
1536            | AvxOpcode::Vpmulhuw
1537            | AvxOpcode::Vpmuldq
1538            | AvxOpcode::Vpmuludq
1539            | AvxOpcode::Vpunpckhwd
1540            | AvxOpcode::Vpunpcklwd
1541            | AvxOpcode::Vunpcklps
1542            | AvxOpcode::Vunpckhps
1543            | AvxOpcode::Vaddps
1544            | AvxOpcode::Vaddpd
1545            | AvxOpcode::Vsubps
1546            | AvxOpcode::Vsubpd
1547            | AvxOpcode::Vmulps
1548            | AvxOpcode::Vmulpd
1549            | AvxOpcode::Vdivps
1550            | AvxOpcode::Vdivpd
1551            | AvxOpcode::Vpcmpeqb
1552            | AvxOpcode::Vpcmpeqw
1553            | AvxOpcode::Vpcmpeqd
1554            | AvxOpcode::Vpcmpeqq
1555            | AvxOpcode::Vpcmpgtb
1556            | AvxOpcode::Vpcmpgtw
1557            | AvxOpcode::Vpcmpgtd
1558            | AvxOpcode::Vpcmpgtq
1559            | AvxOpcode::Vblendvps
1560            | AvxOpcode::Vblendvpd
1561            | AvxOpcode::Vpblendvb
1562            | AvxOpcode::Vmovlhps
1563            | AvxOpcode::Vpminsb
1564            | AvxOpcode::Vpminsw
1565            | AvxOpcode::Vpminsd
1566            | AvxOpcode::Vpminub
1567            | AvxOpcode::Vpminuw
1568            | AvxOpcode::Vpminud
1569            | AvxOpcode::Vpmaxsb
1570            | AvxOpcode::Vpmaxsw
1571            | AvxOpcode::Vpmaxsd
1572            | AvxOpcode::Vpmaxub
1573            | AvxOpcode::Vpmaxuw
1574            | AvxOpcode::Vpmaxud
1575            | AvxOpcode::Vpunpcklbw
1576            | AvxOpcode::Vpunpckhbw
1577            | AvxOpcode::Vpacksswb
1578            | AvxOpcode::Vpackssdw
1579            | AvxOpcode::Vpackuswb
1580            | AvxOpcode::Vpackusdw
1581            | AvxOpcode::Vpalignr
1582            | AvxOpcode::Vpinsrb
1583            | AvxOpcode::Vpinsrw
1584            | AvxOpcode::Vpinsrd
1585            | AvxOpcode::Vpinsrq
1586            | AvxOpcode::Vpmaddwd
1587            | AvxOpcode::Vpmaddubsw
1588            | AvxOpcode::Vinsertps
1589            | AvxOpcode::Vpshufb
1590            | AvxOpcode::Vshufps
1591            | AvxOpcode::Vpsllw
1592            | AvxOpcode::Vpslld
1593            | AvxOpcode::Vpsllq
1594            | AvxOpcode::Vpsraw
1595            | AvxOpcode::Vpsrad
1596            | AvxOpcode::Vpmovsxbw
1597            | AvxOpcode::Vpmovzxbw
1598            | AvxOpcode::Vpmovsxwd
1599            | AvxOpcode::Vpmovzxwd
1600            | AvxOpcode::Vpmovsxdq
1601            | AvxOpcode::Vpmovzxdq
1602            | AvxOpcode::Vaddss
1603            | AvxOpcode::Vaddsd
1604            | AvxOpcode::Vmulss
1605            | AvxOpcode::Vmulsd
1606            | AvxOpcode::Vsubss
1607            | AvxOpcode::Vsubsd
1608            | AvxOpcode::Vdivss
1609            | AvxOpcode::Vdivsd
1610            | AvxOpcode::Vpabsb
1611            | AvxOpcode::Vpabsw
1612            | AvxOpcode::Vpabsd
1613            | AvxOpcode::Vminss
1614            | AvxOpcode::Vminsd
1615            | AvxOpcode::Vmaxss
1616            | AvxOpcode::Vmaxsd
1617            | AvxOpcode::Vsqrtps
1618            | AvxOpcode::Vsqrtpd
1619            | AvxOpcode::Vroundpd
1620            | AvxOpcode::Vroundps
1621            | AvxOpcode::Vcvtdq2pd
1622            | AvxOpcode::Vcvtdq2ps
1623            | AvxOpcode::Vcvtpd2ps
1624            | AvxOpcode::Vcvtps2pd
1625            | AvxOpcode::Vcvttpd2dq
1626            | AvxOpcode::Vcvttps2dq
1627            | AvxOpcode::Vphaddw
1628            | AvxOpcode::Vphaddd
1629            | AvxOpcode::Vpunpckldq
1630            | AvxOpcode::Vpunpckhdq
1631            | AvxOpcode::Vpunpcklqdq
1632            | AvxOpcode::Vpunpckhqdq
1633            | AvxOpcode::Vpshuflw
1634            | AvxOpcode::Vpshufhw
1635            | AvxOpcode::Vpshufd
1636            | AvxOpcode::Vmovss
1637            | AvxOpcode::Vmovsd
1638            | AvxOpcode::Vmovups
1639            | AvxOpcode::Vmovupd
1640            | AvxOpcode::Vmovdqu
1641            | AvxOpcode::Vpextrb
1642            | AvxOpcode::Vpextrw
1643            | AvxOpcode::Vpextrd
1644            | AvxOpcode::Vpextrq
1645            | AvxOpcode::Vpblendw
1646            | AvxOpcode::Vmovddup
1647            | AvxOpcode::Vbroadcastss
1648            | AvxOpcode::Vmovd
1649            | AvxOpcode::Vmovq
1650            | AvxOpcode::Vmovmskps
1651            | AvxOpcode::Vmovmskpd
1652            | AvxOpcode::Vpmovmskb
1653            | AvxOpcode::Vcvtsi2ss
1654            | AvxOpcode::Vcvtsi2sd
1655            | AvxOpcode::Vcvtss2sd
1656            | AvxOpcode::Vcvtsd2ss
1657            | AvxOpcode::Vsqrtss
1658            | AvxOpcode::Vsqrtsd
1659            | AvxOpcode::Vroundss
1660            | AvxOpcode::Vroundsd
1661            | AvxOpcode::Vunpcklpd
1662            | AvxOpcode::Vptest
1663            | AvxOpcode::Vucomiss
1664            | AvxOpcode::Vucomisd => {
1665                smallvec![InstructionSet::AVX]
1666            }
1667
1668            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {
1669                smallvec![InstructionSet::AVX2]
1670            }
1671        }
1672    }
1673
1674    /// Is the opcode known to be commutative?
1675    ///
1676    /// Note that this method is not exhaustive, and there may be commutative
1677    /// opcodes that we don't recognize as commutative.
1678    pub(crate) fn is_commutative(&self) -> bool {
1679        match *self {
1680            AvxOpcode::Vpaddb
1681            | AvxOpcode::Vpaddw
1682            | AvxOpcode::Vpaddd
1683            | AvxOpcode::Vpaddq
1684            | AvxOpcode::Vpaddsb
1685            | AvxOpcode::Vpaddsw
1686            | AvxOpcode::Vpaddusb
1687            | AvxOpcode::Vpaddusw
1688            | AvxOpcode::Vpand
1689            | AvxOpcode::Vandps
1690            | AvxOpcode::Vandpd
1691            | AvxOpcode::Vpor
1692            | AvxOpcode::Vorps
1693            | AvxOpcode::Vorpd
1694            | AvxOpcode::Vpxor
1695            | AvxOpcode::Vxorps
1696            | AvxOpcode::Vxorpd
1697            | AvxOpcode::Vpmuldq
1698            | AvxOpcode::Vpmuludq
1699            | AvxOpcode::Vaddps
1700            | AvxOpcode::Vaddpd
1701            | AvxOpcode::Vmulps
1702            | AvxOpcode::Vmulpd
1703            | AvxOpcode::Vpcmpeqb
1704            | AvxOpcode::Vpcmpeqw
1705            | AvxOpcode::Vpcmpeqd
1706            | AvxOpcode::Vpcmpeqq
1707            | AvxOpcode::Vaddss
1708            | AvxOpcode::Vaddsd
1709            | AvxOpcode::Vmulss
1710            | AvxOpcode::Vmulsd => true,
1711            _ => false,
1712        }
1713    }
1714}
1715
1716impl fmt::Display for AvxOpcode {
1717    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1718        format!("{self:?}").to_lowercase().fmt(f)
1719    }
1720}
1721
1722#[derive(Copy, Clone, PartialEq)]
1723#[allow(missing_docs)]
1724pub enum Avx512TupleType {
1725    Full,
1726    FullMem,
1727    Mem128,
1728}
1729
1730pub use crate::isa::x64::lower::isle::generated_code::Avx512Opcode;
1731
1732impl Avx512Opcode {
1733    /// Which `InstructionSet`s support the opcode?
1734    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1735        match self {
1736            Avx512Opcode::Vcvtudq2ps
1737            | Avx512Opcode::Vpabsq
1738            | Avx512Opcode::Vpsraq
1739            | Avx512Opcode::VpsraqImm => {
1740                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]
1741            }
1742            Avx512Opcode::Vpermi2b => {
1743                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]
1744            }
1745            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],
1746            Avx512Opcode::Vpopcntb => {
1747                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]
1748            }
1749        }
1750    }
1751
1752    /// What is the "TupleType" of this opcode, which affects the scaling factor
1753    /// for 8-bit displacements when this instruction uses memory operands.
1754    ///
1755    /// This can be found in the encoding table for each instruction and is
1756    /// interpreted according to Table 2-34 and 2-35 in the Intel instruction
1757    /// manual.
1758    pub fn tuple_type(&self) -> Avx512TupleType {
1759        use Avx512Opcode::*;
1760        use Avx512TupleType::*;
1761
1762        match self {
1763            Vcvtudq2ps | Vpabsq | Vpmullq | VpsraqImm => Full,
1764            Vpermi2b | Vpopcntb => FullMem,
1765            Vpsraq => Mem128,
1766        }
1767    }
1768}
1769
1770impl fmt::Display for Avx512Opcode {
1771    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1772        let s = format!("{self:?}");
1773        f.write_str(&s.to_lowercase())
1774    }
1775}
1776
1777/// This defines the ways a value can be extended: either signed- or zero-extension, or none for
1778/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which
1779/// values can be extended.
1780#[allow(dead_code)]
1781#[derive(Clone, PartialEq)]
1782pub enum ExtKind {
1783    /// No extension.
1784    None,
1785    /// Sign-extend.
1786    SignExtend,
1787    /// Zero-extend.
1788    ZeroExtend,
1789}
1790
1791/// These indicate ways of extending (widening) a value, using the Intel
1792/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64
1793#[derive(Clone, PartialEq)]
1794pub enum ExtMode {
1795    /// Byte -> Longword.
1796    BL,
1797    /// Byte -> Quadword.
1798    BQ,
1799    /// Word -> Longword.
1800    WL,
1801    /// Word -> Quadword.
1802    WQ,
1803    /// Longword -> Quadword.
1804    LQ,
1805}
1806
1807impl ExtMode {
1808    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.
1809    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {
1810        match (from_bits, to_bits) {
1811            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
1812            (1, 64) | (8, 64) => Some(ExtMode::BQ),
1813            (16, 32) => Some(ExtMode::WL),
1814            (16, 64) => Some(ExtMode::WQ),
1815            (32, 64) => Some(ExtMode::LQ),
1816            _ => None,
1817        }
1818    }
1819
1820    /// Return the source register size in bytes.
1821    pub(crate) fn src_size(&self) -> u8 {
1822        match self {
1823            ExtMode::BL | ExtMode::BQ => 1,
1824            ExtMode::WL | ExtMode::WQ => 2,
1825            ExtMode::LQ => 4,
1826        }
1827    }
1828
1829    /// Return the destination register size in bytes.
1830    pub(crate) fn dst_size(&self) -> u8 {
1831        match self {
1832            ExtMode::BL | ExtMode::WL => 4,
1833            ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8,
1834        }
1835    }
1836
1837    /// Source size, as an integer type.
1838    pub(crate) fn src_type(&self) -> Type {
1839        match self {
1840            ExtMode::BL | ExtMode::BQ => I8,
1841            ExtMode::WL | ExtMode::WQ => I16,
1842            ExtMode::LQ => I32,
1843        }
1844    }
1845}
1846
1847impl fmt::Debug for ExtMode {
1848    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1849        let name = match self {
1850            ExtMode::BL => "bl",
1851            ExtMode::BQ => "bq",
1852            ExtMode::WL => "wl",
1853            ExtMode::WQ => "wq",
1854            ExtMode::LQ => "lq",
1855        };
1856        write!(fmt, "{name}")
1857    }
1858}
1859
1860impl fmt::Display for ExtMode {
1861    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1862        fmt::Debug::fmt(self, f)
1863    }
1864}
1865
1866/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
1867#[derive(Clone, Copy)]
1868pub enum ShiftKind {
1869    /// Left shift.
1870    ShiftLeft,
1871    /// Inserts zeros in the most significant bits.
1872    ShiftRightLogical,
1873    /// Replicates the sign bit in the most significant bits.
1874    ShiftRightArithmetic,
1875    /// Left rotation.
1876    RotateLeft,
1877    /// Right rotation.
1878    RotateRight,
1879}
1880
1881impl fmt::Debug for ShiftKind {
1882    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1883        let name = match self {
1884            ShiftKind::ShiftLeft => "shl",
1885            ShiftKind::ShiftRightLogical => "shr",
1886            ShiftKind::ShiftRightArithmetic => "sar",
1887            ShiftKind::RotateLeft => "rol",
1888            ShiftKind::RotateRight => "ror",
1889        };
1890        write!(fmt, "{name}")
1891    }
1892}
1893
1894impl fmt::Display for ShiftKind {
1895    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1896        fmt::Debug::fmt(self, f)
1897    }
1898}
1899
1900/// These indicate condition code tests.  Not all are represented since not all are useful in
1901/// compiler-generated code.
1902#[derive(Copy, Clone, PartialEq, Eq)]
1903#[repr(u8)]
1904pub enum CC {
1905    ///  overflow
1906    O = 0,
1907    /// no overflow
1908    NO = 1,
1909
1910    /// < unsigned
1911    B = 2,
1912    /// >= unsigned
1913    NB = 3,
1914
1915    /// zero
1916    Z = 4,
1917    /// not-zero
1918    NZ = 5,
1919
1920    /// <= unsigned
1921    BE = 6,
1922    /// > unsigned
1923    NBE = 7,
1924
1925    /// negative
1926    S = 8,
1927    /// not-negative
1928    NS = 9,
1929
1930    /// < signed
1931    L = 12,
1932    /// >= signed
1933    NL = 13,
1934
1935    /// <= signed
1936    LE = 14,
1937    /// > signed
1938    NLE = 15,
1939
1940    /// parity
1941    P = 10,
1942
1943    /// not parity
1944    NP = 11,
1945}
1946
1947impl CC {
1948    pub(crate) fn from_intcc(intcc: IntCC) -> Self {
1949        match intcc {
1950            IntCC::Equal => CC::Z,
1951            IntCC::NotEqual => CC::NZ,
1952            IntCC::SignedGreaterThanOrEqual => CC::NL,
1953            IntCC::SignedGreaterThan => CC::NLE,
1954            IntCC::SignedLessThanOrEqual => CC::LE,
1955            IntCC::SignedLessThan => CC::L,
1956            IntCC::UnsignedGreaterThanOrEqual => CC::NB,
1957            IntCC::UnsignedGreaterThan => CC::NBE,
1958            IntCC::UnsignedLessThanOrEqual => CC::BE,
1959            IntCC::UnsignedLessThan => CC::B,
1960        }
1961    }
1962
1963    pub(crate) fn invert(&self) -> Self {
1964        match self {
1965            CC::O => CC::NO,
1966            CC::NO => CC::O,
1967
1968            CC::B => CC::NB,
1969            CC::NB => CC::B,
1970
1971            CC::Z => CC::NZ,
1972            CC::NZ => CC::Z,
1973
1974            CC::BE => CC::NBE,
1975            CC::NBE => CC::BE,
1976
1977            CC::S => CC::NS,
1978            CC::NS => CC::S,
1979
1980            CC::L => CC::NL,
1981            CC::NL => CC::L,
1982
1983            CC::LE => CC::NLE,
1984            CC::NLE => CC::LE,
1985
1986            CC::P => CC::NP,
1987            CC::NP => CC::P,
1988        }
1989    }
1990
1991    pub(crate) fn get_enc(self) -> u8 {
1992        self as u8
1993    }
1994}
1995
1996impl fmt::Debug for CC {
1997    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1998        let name = match self {
1999            CC::O => "o",
2000            CC::NO => "no",
2001            CC::B => "b",
2002            CC::NB => "nb",
2003            CC::Z => "z",
2004            CC::NZ => "nz",
2005            CC::BE => "be",
2006            CC::NBE => "nbe",
2007            CC::S => "s",
2008            CC::NS => "ns",
2009            CC::L => "l",
2010            CC::NL => "nl",
2011            CC::LE => "le",
2012            CC::NLE => "nle",
2013            CC::P => "p",
2014            CC::NP => "np",
2015        };
2016        write!(fmt, "{name}")
2017    }
2018}
2019
2020impl fmt::Display for CC {
2021    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2022        fmt::Debug::fmt(self, f)
2023    }
2024}
2025
2026/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,
2027/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS
2028/// whereas [FcmpImm] is used as an immediate.
2029#[derive(Clone, Copy)]
2030pub enum FcmpImm {
2031    /// Equal comparison.
2032    Equal = 0x00,
2033    /// Less than comparison.
2034    LessThan = 0x01,
2035    /// Less than or equal comparison.
2036    LessThanOrEqual = 0x02,
2037    /// Unordered.
2038    Unordered = 0x03,
2039    /// Not equal comparison.
2040    NotEqual = 0x04,
2041    /// Unordered of greater than or equal comparison.
2042    UnorderedOrGreaterThanOrEqual = 0x05,
2043    /// Unordered or greater than comparison.
2044    UnorderedOrGreaterThan = 0x06,
2045    /// Ordered.
2046    Ordered = 0x07,
2047}
2048
2049impl FcmpImm {
2050    pub(crate) fn encode(self) -> u8 {
2051        self as u8
2052    }
2053}
2054
2055impl From<FloatCC> for FcmpImm {
2056    fn from(cond: FloatCC) -> Self {
2057        match cond {
2058            FloatCC::Equal => FcmpImm::Equal,
2059            FloatCC::LessThan => FcmpImm::LessThan,
2060            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,
2061            FloatCC::Unordered => FcmpImm::Unordered,
2062            FloatCC::NotEqual => FcmpImm::NotEqual,
2063            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,
2064            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,
2065            FloatCC::Ordered => FcmpImm::Ordered,
2066            _ => panic!("unable to create comparison predicate for {cond}"),
2067        }
2068    }
2069}
2070
2071/// Encode the rounding modes used as part of the Rounding Control field.
2072/// Note, these rounding immediates only consider the rounding control field
2073/// (i.e. the rounding mode) which only take up the first two bits when encoded.
2074/// However the rounding immediate which this field helps make up, also includes
2075/// bits 3 and 4 which define the rounding select and precision mask respectively.
2076/// These two bits are not defined here and are implicitly set to zero when encoded.
2077#[derive(Clone, Copy)]
2078pub enum RoundImm {
2079    /// Round to nearest mode.
2080    RoundNearest = 0x00,
2081    /// Round down mode.
2082    RoundDown = 0x01,
2083    /// Round up mode.
2084    RoundUp = 0x02,
2085    /// Round to zero mode.
2086    RoundZero = 0x03,
2087}
2088
2089impl RoundImm {
2090    pub(crate) fn encode(self) -> u8 {
2091        self as u8
2092    }
2093}
2094
2095/// An operand's size in bits.
2096#[derive(Clone, Copy, PartialEq)]
2097pub enum OperandSize {
2098    /// 8-bit.
2099    Size8,
2100    /// 16-bit.
2101    Size16,
2102    /// 32-bit.
2103    Size32,
2104    /// 64-bit.
2105    Size64,
2106}
2107
2108impl OperandSize {
2109    pub(crate) fn from_bytes(num_bytes: u32) -> Self {
2110        match num_bytes {
2111            1 => OperandSize::Size8,
2112            2 => OperandSize::Size16,
2113            4 => OperandSize::Size32,
2114            8 => OperandSize::Size64,
2115            _ => unreachable!("Invalid OperandSize: {}", num_bytes),
2116        }
2117    }
2118
2119    // Computes the OperandSize for a given type.
2120    // For vectors, the OperandSize of the lanes is returned.
2121    pub(crate) fn from_ty(ty: Type) -> Self {
2122        Self::from_bytes(ty.lane_type().bytes())
2123    }
2124
2125    // Check that the value of self is one of the allowed sizes.
2126    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {
2127        sizes.iter().any(|val| *self == *val)
2128    }
2129
2130    pub(crate) fn to_bytes(&self) -> u8 {
2131        match self {
2132            Self::Size8 => 1,
2133            Self::Size16 => 2,
2134            Self::Size32 => 4,
2135            Self::Size64 => 8,
2136        }
2137    }
2138
2139    pub(crate) fn to_bits(&self) -> u8 {
2140        self.to_bytes() * 8
2141    }
2142
2143    pub(crate) fn to_type(&self) -> Type {
2144        match self {
2145            Self::Size8 => I8,
2146            Self::Size16 => I16,
2147            Self::Size32 => I32,
2148            Self::Size64 => I64,
2149        }
2150    }
2151}
2152
2153/// An x64 memory fence kind.
2154#[derive(Clone)]
2155#[allow(dead_code)]
2156pub enum FenceKind {
2157    /// `mfence` instruction ("Memory Fence")
2158    MFence,
2159    /// `lfence` instruction ("Load Fence")
2160    LFence,
2161    /// `sfence` instruction ("Store Fence")
2162    SFence,
2163}
cranelift_codegen/isa/x64/inst/args.rs

cranelift_codegen/isa/x64/inst/
args.rs
