cranelift_codegen/isa/x64/inst/
external.rs

1//! Interface with the external assembler crate.
2
3use super::{
4    Amode, Gpr, Inst, LabelUse, MachBuffer, MachLabel, OperandVisitor, OperandVisitorImpl,
5    SyntheticAmode, VCodeConstant, WritableGpr, WritableXmm, Xmm, args::FromWritableReg,
6};
7use crate::{Reg, Writable, ir::TrapCode};
8use cranelift_assembler_x64 as asm;
9use regalloc2::{PReg, RegClass};
10use std::string::String;
11
12/// Define the types of registers Cranelift will use.
13#[derive(Clone, Debug)]
14pub struct CraneliftRegisters;
15impl asm::Registers for CraneliftRegisters {
16    type ReadGpr = Gpr;
17    type ReadWriteGpr = PairedGpr;
18    type WriteGpr = WritableGpr;
19    type ReadXmm = Xmm;
20    type ReadWriteXmm = PairedXmm;
21    type WriteXmm = WritableXmm;
22}
23
24/// Convenience type alias of `asm::inst::Inst` with `R = CraneliftRegisters`
25/// filled in.
26pub type AsmInst = asm::inst::Inst<CraneliftRegisters>;
27
28/// A pair of registers, one for reading and one for writing.
29///
30/// Due to how Cranelift's SSA form, we must track the read and write registers
31/// separately prior to register allocation. Once register allocation is
32/// complete, we expect the hardware encoding for both `read` and `write` to be
33/// the same.
34#[derive(Clone, Copy, Debug, PartialEq)]
35#[expect(missing_docs, reason = "self-describing variants")]
36pub struct PairedGpr {
37    pub read: Gpr,
38    pub write: WritableGpr,
39}
40
41impl From<WritableGpr> for PairedGpr {
42    fn from(wgpr: WritableGpr) -> Self {
43        let read = wgpr.to_reg();
44        let write = wgpr;
45        Self { read, write }
46    }
47}
48
49/// For ABI ergonomics.
50impl From<WritableGpr> for asm::Gpr<PairedGpr> {
51    fn from(wgpr: WritableGpr) -> Self {
52        asm::Gpr::new(wgpr.into())
53    }
54}
55
56// For ABI ergonomics.
57impl From<Writable<Reg>> for asm::GprMem<PairedGpr, Gpr> {
58    fn from(wgpr: Writable<Reg>) -> Self {
59        assert!(wgpr.to_reg().class() == RegClass::Int);
60        let wgpr = WritableGpr::from_writable_reg(wgpr).unwrap();
61        Self::Gpr(wgpr.into())
62    }
63}
64
65// For ABI ergonomics.
66impl From<Reg> for asm::GprMem<Gpr, Gpr> {
67    fn from(gpr: Reg) -> Self {
68        assert!(gpr.class() == RegClass::Int);
69        let gpr = Gpr::unwrap_new(gpr);
70        Self::Gpr(gpr)
71    }
72}
73
74// For ABI ergonomics.
75impl From<Writable<Reg>> for asm::GprMem<Gpr, Gpr> {
76    fn from(wgpr: Writable<Reg>) -> Self {
77        wgpr.to_reg().into()
78    }
79}
80
81// For ABI ergonomics.
82impl From<Writable<Reg>> for asm::Gpr<PairedGpr> {
83    fn from(wgpr: Writable<Reg>) -> Self {
84        assert!(wgpr.to_reg().class() == RegClass::Int);
85        let wgpr = WritableGpr::from_writable_reg(wgpr).unwrap();
86        Self::new(wgpr.into())
87    }
88}
89
90impl From<Writable<Reg>> for asm::Gpr<WritableGpr> {
91    fn from(wgpr: Writable<Reg>) -> Self {
92        assert!(wgpr.to_reg().class() == RegClass::Int);
93        let wgpr = WritableGpr::from_writable_reg(wgpr).unwrap();
94        Self::new(wgpr)
95    }
96}
97
98impl asm::AsReg for PairedGpr {
99    fn enc(&self) -> u8 {
100        let PairedGpr { read, write } = self;
101        let read = enc_gpr(read);
102        let write = enc_gpr(&write.to_reg());
103        assert_eq!(read, write);
104        write
105    }
106
107    fn to_string(&self, size: Option<asm::Size>) -> String {
108        if self.read.is_real() {
109            asm::gpr::enc::to_string(self.enc(), size.unwrap()).into()
110        } else {
111            let read = self.read.to_reg();
112            let write = self.write.to_reg().to_reg();
113            format!("(%{write:?} <- %{read:?})")
114        }
115    }
116
117    fn new(_: u8) -> Self {
118        panic!("disallow creation of new assembler registers")
119    }
120}
121
122/// A pair of XMM registers, one for reading and one for writing.
123#[derive(Clone, Copy, Debug, PartialEq)]
124#[expect(missing_docs, reason = "self-describing variants")]
125pub struct PairedXmm {
126    pub read: Xmm,
127    pub write: WritableXmm,
128}
129
130impl From<WritableXmm> for PairedXmm {
131    fn from(wxmm: WritableXmm) -> Self {
132        let read = wxmm.to_reg();
133        let write = wxmm;
134        Self { read, write }
135    }
136}
137
138/// For ABI ergonomics.
139impl From<WritableXmm> for asm::Xmm<PairedXmm> {
140    fn from(wgpr: WritableXmm) -> Self {
141        asm::Xmm::new(wgpr.into())
142    }
143}
144
145// For emission ergonomics.
146impl From<Writable<Reg>> for asm::Xmm<PairedXmm> {
147    fn from(wxmm: Writable<Reg>) -> Self {
148        assert!(wxmm.to_reg().class() == RegClass::Float);
149        let wxmm = WritableXmm::from_writable_reg(wxmm).unwrap();
150        Self::new(wxmm.into())
151    }
152}
153
154// For emission ergonomics.
155impl From<Reg> for asm::Xmm<Xmm> {
156    fn from(xmm: Reg) -> Self {
157        assert!(xmm.class() == RegClass::Float);
158        let xmm = Xmm::unwrap_new(xmm);
159        Self::new(xmm)
160    }
161}
162
163// For emission ergonomics.
164impl From<Reg> for asm::XmmMem<Xmm, Gpr> {
165    fn from(xmm: Reg) -> Self {
166        assert!(xmm.class() == RegClass::Float);
167        let xmm = Xmm::unwrap_new(xmm);
168        Self::Xmm(xmm)
169    }
170}
171
172impl asm::AsReg for PairedXmm {
173    fn enc(&self) -> u8 {
174        let PairedXmm { read, write } = self;
175        let read = enc_xmm(read);
176        let write = enc_xmm(&write.to_reg());
177        assert_eq!(read, write);
178        write
179    }
180
181    fn to_string(&self, size: Option<asm::Size>) -> String {
182        assert!(size.is_none(), "XMM registers do not have size variants");
183        if self.read.is_real() {
184            asm::xmm::enc::to_string(self.enc()).into()
185        } else {
186            let read = self.read.to_reg();
187            let write = self.write.to_reg().to_reg();
188            format!("(%{write:?} <- %{read:?})")
189        }
190    }
191
192    fn new(_: u8) -> Self {
193        panic!("disallow creation of new assembler registers")
194    }
195}
196
197/// This bridges the gap between codegen and assembler for general purpose register types.
198impl asm::AsReg for Gpr {
199    fn enc(&self) -> u8 {
200        enc_gpr(self)
201    }
202
203    fn to_string(&self, size: Option<asm::Size>) -> String {
204        if self.is_real() {
205            asm::gpr::enc::to_string(self.enc(), size.unwrap()).into()
206        } else {
207            format!("%{:?}", self.to_reg())
208        }
209    }
210
211    fn new(_: u8) -> Self {
212        panic!("disallow creation of new assembler registers")
213    }
214}
215
216/// This bridges the gap between codegen and assembler for xmm register types.
217impl asm::AsReg for Xmm {
218    fn enc(&self) -> u8 {
219        enc_xmm(self)
220    }
221
222    fn to_string(&self, size: Option<asm::Size>) -> String {
223        assert!(size.is_none(), "XMM registers do not have size variants");
224        if self.is_real() {
225            asm::xmm::enc::to_string(self.enc()).into()
226        } else {
227            format!("%{:?}", self.to_reg())
228        }
229    }
230
231    fn new(_: u8) -> Self {
232        panic!("disallow creation of new assembler registers")
233    }
234}
235
236/// A helper method for extracting the hardware encoding of a general purpose register.
237#[inline]
238fn enc_gpr(gpr: &Gpr) -> u8 {
239    if let Some(real) = gpr.to_reg().to_real_reg() {
240        real.hw_enc()
241    } else {
242        unreachable!()
243    }
244}
245
246/// A helper method for extracting the hardware encoding of an xmm register.
247#[inline]
248fn enc_xmm(xmm: &Xmm) -> u8 {
249    if let Some(real) = xmm.to_reg().to_real_reg() {
250        real.hw_enc()
251    } else {
252        unreachable!()
253    }
254}
255
256/// A wrapper to implement the `cranelift-assembler-x64` register allocation trait,
257/// `RegallocVisitor`, in terms of the trait used in Cranelift,
258/// `OperandVisitor`.
259pub(crate) struct RegallocVisitor<'a, T>
260where
261    T: OperandVisitorImpl,
262{
263    pub collector: &'a mut T,
264}
265
266impl<'a, T: OperandVisitor> asm::RegisterVisitor<CraneliftRegisters> for RegallocVisitor<'a, T> {
267    fn read_gpr(&mut self, reg: &mut Gpr) {
268        self.collector.reg_use(reg);
269    }
270
271    fn read_write_gpr(&mut self, reg: &mut PairedGpr) {
272        let PairedGpr { read, write } = reg;
273        self.collector.reg_use(read);
274        self.collector.reg_reuse_def(write, 0);
275    }
276
277    fn write_gpr(&mut self, reg: &mut WritableGpr) {
278        self.collector.reg_def(reg);
279    }
280
281    fn fixed_read_gpr(&mut self, reg: &mut Gpr, enc: u8) {
282        self.collector
283            .reg_fixed_use(reg, fixed_reg(enc, RegClass::Int));
284    }
285
286    fn fixed_read_write_gpr(&mut self, reg: &mut PairedGpr, enc: u8) {
287        let PairedGpr { read, write } = reg;
288        self.collector
289            .reg_fixed_use(read, fixed_reg(enc, RegClass::Int));
290        self.collector
291            .reg_fixed_def(write, fixed_reg(enc, RegClass::Int));
292    }
293
294    fn fixed_write_gpr(&mut self, reg: &mut WritableGpr, enc: u8) {
295        self.collector
296            .reg_fixed_def(reg, fixed_reg(enc, RegClass::Int));
297    }
298
299    fn read_xmm(&mut self, reg: &mut Xmm) {
300        self.collector.reg_use(reg);
301    }
302
303    fn read_write_xmm(&mut self, reg: &mut PairedXmm) {
304        let PairedXmm { read, write } = reg;
305        self.collector.reg_use(read);
306        self.collector.reg_reuse_def(write, 0);
307    }
308
309    fn write_xmm(&mut self, reg: &mut WritableXmm) {
310        self.collector.reg_def(reg);
311    }
312
313    fn fixed_read_xmm(&mut self, reg: &mut Xmm, enc: u8) {
314        self.collector
315            .reg_fixed_use(reg, fixed_reg(enc, RegClass::Float));
316    }
317
318    fn fixed_read_write_xmm(&mut self, reg: &mut PairedXmm, enc: u8) {
319        let PairedXmm { read, write } = reg;
320        self.collector
321            .reg_fixed_use(read, fixed_reg(enc, RegClass::Float));
322        self.collector
323            .reg_fixed_def(write, fixed_reg(enc, RegClass::Float));
324    }
325
326    fn fixed_write_xmm(&mut self, reg: &mut WritableXmm, enc: u8) {
327        self.collector
328            .reg_fixed_def(reg, fixed_reg(enc, RegClass::Float));
329    }
330}
331
332/// A helper for building a fixed register from its hardware encoding.
333fn fixed_reg(enc: u8, class: RegClass) -> Reg {
334    let preg = PReg::new(usize::from(enc), class);
335    Reg::from_real_reg(preg)
336}
337
338impl From<SyntheticAmode> for asm::Amode<Gpr> {
339    fn from(amode: SyntheticAmode) -> asm::Amode<Gpr> {
340        match amode {
341            SyntheticAmode::Real(amode) => amode.into(),
342            SyntheticAmode::IncomingArg { offset } => asm::Amode::ImmReg {
343                base: Gpr::RBP,
344                simm32: asm::AmodeOffsetPlusKnownOffset {
345                    simm32: (-i32::try_from(offset).unwrap()).into(),
346                    offset: Some(offsets::KEY_INCOMING_ARG),
347                },
348                trap: None,
349            },
350            SyntheticAmode::SlotOffset { simm32 } => asm::Amode::ImmReg {
351                base: Gpr::RSP,
352                simm32: asm::AmodeOffsetPlusKnownOffset {
353                    simm32: simm32.into(),
354                    offset: Some(offsets::KEY_SLOT_OFFSET),
355                },
356                trap: None,
357            },
358            SyntheticAmode::ConstantOffset(vcode_constant) => asm::Amode::RipRelative {
359                target: asm::DeferredTarget::Constant(asm::Constant(vcode_constant.as_u32())),
360            },
361        }
362    }
363}
364
365impl From<Amode> for asm::Amode<Gpr> {
366    fn from(amode: Amode) -> asm::Amode<Gpr> {
367        match amode {
368            Amode::ImmReg {
369                simm32,
370                base,
371                flags,
372            } => asm::Amode::ImmReg {
373                simm32: asm::AmodeOffsetPlusKnownOffset {
374                    simm32: simm32.into(),
375                    offset: None,
376                },
377                base: Gpr::unwrap_new(base),
378                trap: flags.trap_code().map(Into::into),
379            },
380            Amode::ImmRegRegShift {
381                simm32,
382                base,
383                index,
384                shift,
385                flags,
386            } => asm::Amode::ImmRegRegShift {
387                base,
388                index: asm::NonRspGpr::new(index),
389                scale: asm::Scale::new(shift),
390                simm32: simm32.into(),
391                trap: flags.trap_code().map(Into::into),
392            },
393            Amode::RipRelative { target } => asm::Amode::RipRelative {
394                target: asm::DeferredTarget::Label(asm::Label(target.as_u32())),
395            },
396        }
397    }
398}
399
400impl<R: asm::AsReg> From<SyntheticAmode> for asm::XmmMem<R, Gpr> {
401    fn from(amode: SyntheticAmode) -> Self {
402        asm::XmmMem::Mem(amode.into())
403    }
404}
405
406impl<R: asm::AsReg> From<SyntheticAmode> for asm::GprMem<R, Gpr> {
407    fn from(amode: SyntheticAmode) -> Self {
408        asm::GprMem::Mem(amode.into())
409    }
410}
411
412impl<R: asm::AsReg> From<Amode> for asm::XmmMem<R, Gpr> {
413    fn from(amode: Amode) -> Self {
414        asm::XmmMem::Mem(amode.into())
415    }
416}
417
418impl<R: asm::AsReg> From<Amode> for asm::GprMem<R, Gpr> {
419    fn from(amode: Amode) -> Self {
420        asm::GprMem::Mem(amode.into())
421    }
422}
423
424/// Keep track of the offset slots to fill in during emission; see
425/// `KnownOffsetTable`.
426#[expect(missing_docs, reason = "self-describing keys")]
427pub mod offsets {
428    pub const KEY_INCOMING_ARG: u8 = 0;
429    pub const KEY_SLOT_OFFSET: u8 = 1;
430}
431
432/// Implementor of the [`asm::CodeSink`] trait.
433pub struct AsmCodeSink<'a> {
434    /// The buffer this is emitting into.
435    pub sink: &'a mut MachBuffer<Inst>,
436    /// The value of `KEY_INCOMING_ARG`.
437    pub incoming_arg_offset: i32,
438    /// The value of `KEY_SLOT_OFFSET`.
439    pub slot_offset: i32,
440}
441
442impl asm::CodeSink for AsmCodeSink<'_> {
443    fn put1(&mut self, value: u8) {
444        self.sink.put1(value)
445    }
446
447    fn put2(&mut self, value: u16) {
448        self.sink.put2(value)
449    }
450
451    fn put4(&mut self, value: u32) {
452        self.sink.put4(value)
453    }
454
455    fn put8(&mut self, value: u64) {
456        self.sink.put8(value)
457    }
458
459    fn add_trap(&mut self, code: asm::TrapCode) {
460        self.sink.add_trap(code.into());
461    }
462
463    fn use_target(&mut self, target: asm::DeferredTarget) {
464        let offset = self.sink.cur_offset();
465        match target {
466            asm::DeferredTarget::Label(label) => {
467                self.sink
468                    .use_label_at_offset(offset, label.into(), LabelUse::JmpRel32);
469            }
470            asm::DeferredTarget::Constant(constant) => {
471                let label = self.sink.get_label_for_constant(constant.into());
472                self.sink
473                    .use_label_at_offset(offset, label, LabelUse::JmpRel32);
474            }
475            asm::DeferredTarget::None => {}
476        }
477    }
478
479    fn known_offset(&self, offset: asm::KnownOffset) -> i32 {
480        match offset {
481            offsets::KEY_INCOMING_ARG => self.incoming_arg_offset,
482            offsets::KEY_SLOT_OFFSET => self.slot_offset,
483            other => panic!("unknown \"known\" offset {other}"),
484        }
485    }
486}
487
488impl From<asm::TrapCode> for TrapCode {
489    fn from(value: asm::TrapCode) -> Self {
490        Self::from_raw(value.0)
491    }
492}
493
494impl From<TrapCode> for asm::TrapCode {
495    fn from(value: TrapCode) -> Self {
496        Self(value.as_raw())
497    }
498}
499
500impl From<asm::Label> for MachLabel {
501    fn from(value: asm::Label) -> Self {
502        Self::from_u32(value.0)
503    }
504}
505
506impl From<MachLabel> for asm::Label {
507    fn from(value: MachLabel) -> Self {
508        Self(value.as_u32())
509    }
510}
511
512impl From<asm::Constant> for VCodeConstant {
513    fn from(value: asm::Constant) -> Self {
514        Self::from_u32(value.0)
515    }
516}
517
518// Include code generated by `cranelift-codegen/meta/src/gen_asm.rs`. This file
519// contains a `isle_assembler_methods!` macro with Rust implementations of all
520// the assembler instructions exposed to ISLE.
521include!(concat!(env!("OUT_DIR"), "/assembler-isle-macro.rs"));
522pub(crate) use isle_assembler_methods;
523
524#[cfg(test)]
525mod tests {
526    use super::PairedGpr;
527    use super::asm::{AsReg, Size};
528    use crate::isa::x64::args::{FromWritableReg, Gpr, WritableGpr, WritableXmm, Xmm};
529    use crate::isa::x64::inst::external::PairedXmm;
530    use crate::{Reg, Writable};
531    use regalloc2::{RegClass, VReg};
532
533    #[test]
534    fn pretty_print_registers() {
535        // For logging, we need to be able to pretty-print the virtual registers
536        // that Cranelift uses before register allocation. This test ensures
537        // that these remain printable using the `AsReg::to_string` interface
538        // (see issue #10631).
539
540        let v200: Reg = VReg::new(200, RegClass::Int).into();
541        let gpr200 = Gpr::new(v200).unwrap();
542        assert_eq!(gpr200.to_string(Some(Size::Quadword)), "%v200");
543
544        let v300: Reg = VReg::new(300, RegClass::Int).into();
545        let wgpr300 = WritableGpr::from_writable_reg(Writable::from_reg(v300)).unwrap();
546        let pair = PairedGpr {
547            read: gpr200,
548            write: wgpr300,
549        };
550        assert_eq!(pair.to_string(Some(Size::Quadword)), "(%v300 <- %v200)");
551
552        let v400: Reg = VReg::new(400, RegClass::Float).into();
553        let xmm400 = Xmm::new(v400).unwrap();
554        assert_eq!(xmm400.to_string(None), "%v400");
555
556        let v500: Reg = VReg::new(500, RegClass::Float).into();
557        let wxmm500 = WritableXmm::from_writable_reg(Writable::from_reg(v500)).unwrap();
558        let pair = PairedXmm {
559            read: xmm400,
560            write: wxmm500,
561        };
562        assert_eq!(pair.to_string(None), "(%v500 <- %v400)");
563    }
564}
cranelift_codegen/isa/x64/inst/external.rs

cranelift_codegen/isa/x64/inst/
external.rs
