Skip to main content

cranelift_codegen/machinst/
mod.rs

1//! This module exposes the machine-specific backend definition pieces.
2//!
3//! The MachInst infrastructure is the compiler backend, from CLIF
4//! (ir::Function) to machine code. The purpose of this infrastructure is, at a
5//! high level, to do instruction selection/lowering (to machine instructions),
6//! register allocation, and then perform all the fixups to branches, constant
7//! data references, etc., needed to actually generate machine code.
8//!
9//! The container for machine instructions, at various stages of construction,
10//! is the `VCode` struct. We refer to a sequence of machine instructions organized
11//! into basic blocks as "vcode". This is short for "virtual-register code".
12//!
13//! The compilation pipeline, from an `ir::Function` (already optimized as much as
14//! you like by machine-independent optimization passes) onward, is as follows.
15//!
16//! ```plain
17//!
18//!     ir::Function                (SSA IR, machine-independent opcodes)
19//!         |
20//!         |  [lower]
21//!         |
22//!     VCode<arch_backend::Inst>   (machine instructions:
23//!         |                        - mostly virtual registers.
24//!         |                        - cond branches in two-target form.
25//!         |                        - branch targets are block indices.
26//!         |                        - in-memory constants held by insns,
27//!         |                          with unknown offsets.
28//!         |                        - critical edges (actually all edges)
29//!         |                          are split.)
30//!         |
31//!         | [regalloc --> `regalloc2::Output`; VCode is unchanged]
32//!         |
33//!         | [binary emission via MachBuffer]
34//!         |
35//!     Vec<u8>                     (machine code:
36//!         |                        - two-dest branches resolved via
37//!         |                          streaming branch resolution/simplification.
38//!         |                        - regalloc `Allocation` results used directly
39//!         |                          by instruction emission code.
40//!         |                        - prologue and epilogue(s) built and emitted
41//!         |                          directly during emission.
42//!         |                        - SP-relative offsets resolved by tracking
43//!         |                          EmitState.)
44//!
45//! ```
46
47use crate::binemit::{Addend, CodeInfo, CodeOffset, Reloc};
48use crate::ir::{
49    self, DynamicStackSlot, Endianness, RelSourceLoc, StackSlot, TrapCode, Type,
50    function::FunctionParameters,
51};
52use crate::isa::FunctionAlignment;
53use crate::result::CodegenResult;
54use crate::settings;
55use crate::value_label::ValueLabelsRanges;
56use alloc::string::String;
57use alloc::vec::Vec;
58use core::fmt;
59use core::fmt::Debug;
60use core::marker::PhantomData;
61use core::num::NonZeroU8;
62use cranelift_control::ControlPlane;
63use cranelift_entity::PrimaryMap;
64use regalloc2::VReg;
65use smallvec::{SmallVec, smallvec};
66
67#[cfg(feature = "enable-serde")]
68use serde_derive::{Deserialize, Serialize};
69
70/// Guaranteed to use "natural alignment" for the given type.
71const BIT_ALIGNED: u16 = 1 << 0;
72
73/// A load that reads data in memory that does not change for the
74/// duration of the function's execution.
75const BIT_READONLY: u16 = 1 << 1;
76
77/// Load multi-byte values from memory in a little-endian format.
78const BIT_LITTLE_ENDIAN: u16 = 1 << 2;
79
80/// Load multi-byte values from memory in a big-endian format.
81const BIT_BIG_ENDIAN: u16 = 1 << 3;
82
83/// A volatile access: it happens exactly as written (once, with its own width,
84/// in order with other volatile accesses), even when its result is unused.
85const BIT_VOLATILE: u16 = 1 << 4;
86
87/// Trap code, if any, for this memory operation.
88const MASK_TRAP_CODE: u16 = ((1 << TRAP_CODE_BITS) - 1) << TRAP_CODE_OFFSET;
89const TRAP_CODE_BITS: u16 = 8;
90const TRAP_CODE_OFFSET: u16 = 7;
91
92/// Whether this memory operation may be freely moved by the optimizer.
93const BIT_CAN_MOVE: u16 = 1 << 15;
94
95/// Backend memory-operation flags.
96///
97/// These are the bit-packed flags that backends operate on directly.
98///
99/// Unlike [`ir::MemFlagsData`], this does not carry alias-region metadata.
100#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
101#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
102pub struct MachMemFlags {
103    // Bit layout:
104    //
105    // - Bit 0: aligned
106    // - Bit 1: readonly
107    // - Bit 2: little-endian
108    // - Bit 3: big-endian
109    // - Bit 4: volatile
110    // - Bits 5..6: unused
111    // - Bits 7..14: trap code
112    // - Bit 15: can_move
113    bits: u16,
114}
115
116impl MachMemFlags {
117    /// Create a new empty set of flags.
118    pub const fn new() -> Self {
119        Self { bits: 0 }.with_trap_code(Some(TrapCode::HEAP_OUT_OF_BOUNDS))
120    }
121
122    /// Create a set of flags representing an access from a "trusted" address.
123    pub const fn trusted() -> Self {
124        Self::new().with_notrap().with_aligned(true)
125    }
126
127    const fn read_bit(self, bit: u16) -> bool {
128        self.bits & bit != 0
129    }
130
131    const fn with_bit(mut self, bit: u16, enabled: bool) -> Self {
132        if enabled {
133            self.bits |= bit;
134        } else {
135            self.bits &= !bit;
136        }
137        self
138    }
139
140    /// Return endianness of the memory access.
141    pub const fn endianness(self, native_endianness: Endianness) -> Endianness {
142        if self.read_bit(BIT_LITTLE_ENDIAN) {
143            Endianness::Little
144        } else if self.read_bit(BIT_BIG_ENDIAN) {
145            Endianness::Big
146        } else {
147            native_endianness
148        }
149    }
150
151    /// Return endianness of the memory access, if explicitly specified.
152    pub const fn explicit_endianness(self) -> Option<Endianness> {
153        if self.read_bit(BIT_LITTLE_ENDIAN) {
154            Some(Endianness::Little)
155        } else if self.read_bit(BIT_BIG_ENDIAN) {
156            Some(Endianness::Big)
157        } else {
158            None
159        }
160    }
161
162    /// Set endianness of the memory access, returning new flags.
163    pub const fn with_endianness(self, endianness: Endianness) -> Self {
164        let res = match endianness {
165            Endianness::Little => self.with_bit(BIT_LITTLE_ENDIAN, true),
166            Endianness::Big => self.with_bit(BIT_BIG_ENDIAN, true),
167        };
168        assert!(!(res.read_bit(BIT_LITTLE_ENDIAN) && res.read_bit(BIT_BIG_ENDIAN)));
169        res
170    }
171
172    /// Test if this memory access cannot trap.
173    pub const fn notrap(self) -> bool {
174        self.trap_code().is_none()
175    }
176
177    /// Set these flags to indicate this access does not trap.
178    pub const fn with_notrap(self) -> Self {
179        self.with_trap_code(None)
180    }
181
182    /// Test if the `can_move` flag is set.
183    pub const fn can_move(self) -> bool {
184        self.read_bit(BIT_CAN_MOVE)
185    }
186
187    /// Set or clear the `can_move` flag, returning new flags.
188    pub const fn with_can_move(self, enabled: bool) -> Self {
189        self.with_bit(BIT_CAN_MOVE, enabled)
190    }
191
192    /// Test if the `aligned` flag is set.
193    pub const fn aligned(self) -> bool {
194        self.read_bit(BIT_ALIGNED)
195    }
196
197    /// Set or clear the `aligned` flag, returning new flags.
198    pub const fn with_aligned(self, enabled: bool) -> Self {
199        self.with_bit(BIT_ALIGNED, enabled)
200    }
201
202    /// Test if the `readonly` flag is set.
203    pub const fn readonly(self) -> bool {
204        self.read_bit(BIT_READONLY)
205    }
206
207    /// Set or clear the `readonly` flag, returning new flags.
208    pub const fn with_readonly(self, enabled: bool) -> Self {
209        self.with_bit(BIT_READONLY, enabled)
210    }
211
212    /// Test if the `volatile` flag is set.
213    pub const fn volatile(self) -> bool {
214        self.read_bit(BIT_VOLATILE)
215    }
216
217    /// Set or clear the `volatile` flag, returning new flags.
218    pub const fn with_volatile(self, enabled: bool) -> Self {
219        self.with_bit(BIT_VOLATILE, enabled)
220    }
221
222    /// Get the trap code to report if this memory access traps.
223    pub const fn trap_code(self) -> Option<TrapCode> {
224        let byte = ((self.bits & MASK_TRAP_CODE) >> TRAP_CODE_OFFSET) as u8;
225        match NonZeroU8::new(byte) {
226            Some(code) => Some(TrapCode::from_raw(code)),
227            None => None,
228        }
229    }
230
231    /// Configures these flags with the specified trap code `code`.
232    pub const fn with_trap_code(mut self, code: Option<TrapCode>) -> Self {
233        let bits = match code {
234            Some(code) => code.as_raw().get() as u16,
235            None => 0,
236        };
237        self.bits &= !MASK_TRAP_CODE;
238        self.bits |= bits << TRAP_CODE_OFFSET;
239        self
240    }
241}
242
243impl fmt::Display for MachMemFlags {
244    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
245        match self.trap_code() {
246            None => write!(f, " notrap")?,
247            Some(TrapCode::HEAP_OUT_OF_BOUNDS) => {}
248            Some(t) => write!(f, " {t}")?,
249        }
250        if self.aligned() {
251            write!(f, " aligned")?;
252        }
253        if self.readonly() {
254            write!(f, " readonly")?;
255        }
256        if self.can_move() {
257            write!(f, " can_move")?;
258        }
259        if self.volatile() {
260            write!(f, " volatile")?;
261        }
262        if self.read_bit(BIT_BIG_ENDIAN) {
263            write!(f, " big")?;
264        }
265        if self.read_bit(BIT_LITTLE_ENDIAN) {
266            write!(f, " little")?;
267        }
268        Ok(())
269    }
270}
271
272#[macro_use]
273pub mod isle;
274
275pub mod lower;
276pub use lower::*;
277pub mod vcode;
278pub use vcode::*;
279pub mod compile;
280pub use compile::*;
281pub mod blockorder;
282pub use blockorder::*;
283pub mod abi;
284pub use abi::*;
285pub mod buffer;
286pub use buffer::*;
287pub mod helpers;
288pub use helpers::*;
289pub mod valueregs;
290pub use reg::*;
291pub use valueregs::*;
292pub mod reg;
293
294/// A machine instruction.
295pub trait MachInst: Clone + Debug {
296    /// The ABI machine spec for this `MachInst`.
297    type ABIMachineSpec: ABIMachineSpec<I = Self>;
298
299    /// Return the registers referenced by this machine instruction along with
300    /// the modes of reference (use, def, modify).
301    fn get_operands(&mut self, collector: &mut impl OperandVisitor);
302
303    /// If this is a simple move, return the (source, destination) tuple of registers.
304    fn is_move(&self) -> Option<(Writable<Reg>, Reg)>;
305
306    /// Is this a terminator (branch or ret)? If so, return its type
307    /// (ret/uncond/cond) and target if applicable.
308    fn is_term(&self) -> MachTerminator;
309
310    /// Is this an unconditional trap?
311    fn is_trap(&self) -> bool;
312
313    /// Is this an "args" pseudoinst?
314    fn is_args(&self) -> bool;
315
316    /// Classify the type of call instruction this is.
317    ///
318    /// This enables more granular function type analysis and optimization.
319    /// Returns `CallType::None` for non-call instructions, `CallType::Regular`
320    /// for normal calls that return to the caller, and `CallType::TailCall`
321    /// for tail calls that don't return to the caller.
322    fn call_type(&self) -> CallType;
323
324    /// Should this instruction's clobber-list be included in the
325    /// clobber-set?
326    fn is_included_in_clobbers(&self) -> bool;
327
328    /// Does this instruction access memory?
329    fn is_mem_access(&self) -> bool;
330
331    /// Generate a move.
332    fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
333
334    /// Generate a dummy instruction that will keep a value alive but
335    /// has no other purpose.
336    fn gen_dummy_use(reg: Reg) -> Self;
337
338    /// Determine register class(es) to store the given Cranelift type, and the
339    /// Cranelift type actually stored in the underlying register(s).  May return
340    /// an error if the type isn't supported by this backend.
341    ///
342    /// If the type requires multiple registers, then the list of registers is
343    /// returned in little-endian order.
344    ///
345    /// Note that the type actually stored in the register(s) may differ in the
346    /// case that a value is split across registers: for example, on a 32-bit
347    /// target, an I64 may be stored in two registers, each of which holds an
348    /// I32. The actually-stored types are used only to inform the backend when
349    /// generating spills and reloads for individual registers.
350    fn rc_for_type(ty: &Type) -> CodegenResult<(&[RegClass], &[Type])>;
351
352    /// Get an appropriate type that can fully hold a value in a given
353    /// register class. This may not be the only type that maps to
354    /// that class, but when used with `gen_move()` or the ABI trait's
355    /// load/spill constructors, it should produce instruction(s) that
356    /// move the entire register contents.
357    fn canonical_type_for_rc(rc: RegClass) -> Type;
358
359    /// Generate a jump to another target. Used during lowering of
360    /// control flow.
361    fn gen_jump(target: MachLabel) -> Self;
362
363    /// Generate a store of an immediate 64-bit integer to a register. Used by
364    /// the control plane to generate random instructions.
365    fn gen_imm_u64(_value: u64, _dst: Writable<Reg>) -> Option<Self> {
366        None
367    }
368
369    /// Generate a store of an immediate 64-bit integer to a register. Used by
370    /// the control plane to generate random instructions. The tmp register may
371    /// be used by architectures which don't support writing immediate values to
372    /// floating point registers directly.
373    fn gen_imm_f64(_value: f64, _tmp: Writable<Reg>, _dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
374        SmallVec::new()
375    }
376
377    /// Generate a NOP. The `preferred_size` parameter allows the caller to
378    /// request a NOP of that size, or as close to it as possible. The machine
379    /// backend may return a NOP whose binary encoding is smaller than the
380    /// preferred size, but must not return a NOP that is larger. However,
381    /// the instruction must have a nonzero size if preferred_size is nonzero.
382    fn gen_nop(preferred_size: usize) -> Self;
383
384    /// The various kinds of NOP, with size, sorted in ascending-size
385    /// order.
386    fn gen_nop_units() -> Vec<Vec<u8>>;
387
388    /// Align a basic block offset (from start of function).  By default, no
389    /// alignment occurs.
390    fn align_basic_block(offset: CodeOffset) -> CodeOffset {
391        offset
392    }
393
394    /// What is the worst-case instruction size emitted by this instruction type?
395    fn worst_case_size() -> CodeOffset;
396
397    /// Worst-case growth, in bytes, that emitting a single `MachInst`
398    /// instruction may add to the `MachBuffer`'s pending-island state
399    /// (constants, deferred traps, and worst-case veneers for new
400    /// fixups).
401    ///
402    /// `MachBuffer` treats one instruction emission as the atomic
403    /// "commit unit" and uses `worst_case_size() +
404    /// worst_case_island_growth()` as the per-instruction lookahead
405    /// bound when deciding whether to flush an island. Backends whose
406    /// label-use kinds always have wide enough range that islands are
407    /// never required may leave this at zero.
408    fn worst_case_island_growth() -> CodeOffset;
409
410    /// Is this a safepoint?
411    fn is_safepoint(&self) -> bool;
412
413    /// Generate an instruction that must appear at the beginning of a basic
414    /// block, if any. Note that the return value must not be subject to
415    /// register allocation.
416    fn gen_block_start(
417        _is_indirect_branch_target: bool,
418        _is_forward_edge_cfi_enabled: bool,
419    ) -> Option<Self> {
420        None
421    }
422
423    /// Returns a description of the alignment required for functions for this
424    /// architecture.
425    fn function_alignment() -> FunctionAlignment;
426
427    /// Is this a low-level, one-way branch, not meant for use in a
428    /// VCode body? These instructions are meant to be used only when
429    /// directly emitted, i.e. when `MachInst` is used as an assembler
430    /// library.
431    fn is_low_level_branch(&self) -> bool {
432        false
433    }
434
435    /// A label-use kind: a type that describes the types of label references that
436    /// can occur in an instruction.
437    type LabelUse: MachInstLabelUse;
438
439    /// Byte representation of a trap opcode which is inserted by `MachBuffer`
440    /// during its `defer_trap` method.
441    const TRAP_OPCODE: &'static [u8];
442}
443
444/// A descriptor of a label reference (use) in an instruction set.
445pub trait MachInstLabelUse: Clone + Copy + Debug + Eq {
446    /// Required alignment for any veneer. Usually the required instruction
447    /// alignment (e.g., 4 for a RISC with 32-bit instructions, or 1 for x86).
448    const ALIGN: CodeOffset;
449
450    /// What is the maximum PC-relative range (positive)? E.g., if `1024`, a
451    /// label-reference fixup at offset `x` is valid if the label resolves to `x
452    /// + 1024`.
453    fn max_pos_range(self) -> CodeOffset;
454    /// What is the maximum PC-relative range (negative)? This is the absolute
455    /// value; i.e., if `1024`, then a label-reference fixup at offset `x` is
456    /// valid if the label resolves to `x - 1024`.
457    fn max_neg_range(self) -> CodeOffset;
458    /// What is the size of code-buffer slice this label-use needs to patch in
459    /// the label's value?
460    fn patch_size(self) -> CodeOffset;
461    /// Perform a code-patch, given the offset into the buffer of this label use
462    /// and the offset into the buffer of the label's definition.
463    /// It is guaranteed that, given `delta = offset - label_offset`, we will
464    /// have `offset >= -self.max_neg_range()` and `offset <=
465    /// self.max_pos_range()`.
466    fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset);
467    /// Can the label-use be patched to a veneer that supports a longer range?
468    /// Usually valid for jumps (a short-range jump can jump to a longer-range
469    /// jump), but not for e.g. constant pool references, because the constant
470    /// load would require different code (one more level of indirection).
471    fn supports_veneer(self) -> bool;
472    /// How many bytes are needed for a veneer?
473    fn veneer_size(self) -> CodeOffset;
474    /// What's the largest possible veneer that may be generated?
475    fn worst_case_veneer_size() -> CodeOffset;
476    /// Generate a veneer. The given code-buffer slice is `self.veneer_size()`
477    /// bytes long at offset `veneer_offset` in the buffer. The original
478    /// label-use will be patched to refer to this veneer's offset.  A new
479    /// (offset, LabelUse) is returned that allows the veneer to use the actual
480    /// label. For veneers to work properly, it is expected that the new veneer
481    /// has a larger range; on most platforms this probably means either a
482    /// "long-range jump" (e.g., on ARM, the 26-bit form), or if already at that
483    /// stage, a jump that supports a full 32-bit range, for example.
484    fn generate_veneer(self, buffer: &mut [u8], veneer_offset: CodeOffset) -> (CodeOffset, Self);
485
486    /// Returns the corresponding label-use for the relocation specified.
487    ///
488    /// This returns `None` if the relocation doesn't have a corresponding
489    /// representation for the target architecture.
490    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self>;
491}
492
493/// Classification of call instruction types for granular analysis.
494#[derive(Clone, Copy, Debug, PartialEq, Eq)]
495pub enum CallType {
496    /// Not a call instruction.
497    None,
498    /// Regular call that returns to the caller.
499    Regular,
500    /// Tail call that doesn't return to the caller.
501    TailCall,
502}
503
504/// Function classification based on call patterns.
505///
506/// This enum classifies functions based on their calling behavior to enable
507/// targeted optimizations. Functions are categorized as:
508/// - `None`: No calls at all (can use simplified calling conventions)
509/// - `TailOnly`: Only tail calls (may skip frame setup in some cases)
510/// - `Regular`: Has regular calls (requires full calling convention support)
511#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
512pub enum FunctionCalls {
513    /// Function makes no calls at all.
514    #[default]
515    None,
516    /// Function only makes tail calls (no regular calls).
517    TailOnly,
518    /// Function makes at least one regular call (may also have tail calls).
519    Regular,
520}
521
522impl FunctionCalls {
523    /// Update the function classification based on a new call instruction.
524    ///
525    /// This method implements the merge logic for accumulating call patterns:
526    /// - Any regular call makes the function Regular
527    /// - Tail calls upgrade None to TailOnly
528    /// - Regular always stays Regular
529    pub fn update(&mut self, call_type: CallType) {
530        *self = match (*self, call_type) {
531            // No call instruction - state unchanged
532            (current, CallType::None) => current,
533            // Regular call always results in Regular classification
534            (_, CallType::Regular) => FunctionCalls::Regular,
535            // Tail call: None becomes TailOnly, others unchanged
536            (FunctionCalls::None, CallType::TailCall) => FunctionCalls::TailOnly,
537            (current, CallType::TailCall) => current,
538        };
539    }
540}
541
542/// Describes a block terminator (not call) in the VCode.
543///
544/// Actual targets are not included: the single-source-of-truth for
545/// those is the VCode itself, which holds, for each block, successors
546/// and outgoing branch args per successor.
547#[derive(Clone, Debug, PartialEq, Eq)]
548pub enum MachTerminator {
549    /// Not a terminator.
550    None,
551    /// A return instruction.
552    Ret,
553    /// A tail call.
554    RetCall,
555    /// A branch.
556    Branch,
557}
558
559/// A trait describing the ability to encode a MachInst into binary machine code.
560pub trait MachInstEmit: MachInst {
561    /// Persistent state carried across `emit` invocations.
562    type State: MachInstEmitState<Self>;
563
564    /// Constant information used in `emit` invocations.
565    type Info;
566
567    /// Emit the instruction.
568    fn emit(&self, code: &mut MachBuffer<Self>, info: &Self::Info, state: &mut Self::State);
569
570    /// Pretty-print the instruction.
571    fn pretty_print_inst(&self, state: &mut Self::State) -> String;
572}
573
574/// A trait describing the emission state carried between MachInsts when
575/// emitting a function body.
576pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {
577    /// Create a new emission state given the ABI object.
578    fn new(abi: &Callee<I::ABIMachineSpec>, ctrl_plane: ControlPlane) -> Self;
579
580    /// The helper that tracks user stack maps for safepoint instructions.
581    ///
582    /// The emission loop uses this to provide each safepoint instruction's
583    /// stack map before emitting it and to check afterwards that it was
584    /// consumed.
585    fn stack_map_emitter(&mut self) -> &mut StackMapEmitter<I>;
586
587    /// The emission state holds ownership of a control plane, so it doesn't
588    /// have to be passed around explicitly too much. `ctrl_plane_mut` may
589    /// be used if temporary access to the control plane is needed by some
590    /// other function that doesn't have access to the emission state.
591    fn ctrl_plane_mut(&mut self) -> &mut ControlPlane;
592
593    /// Used to continue using a control plane after the emission state is
594    /// not needed anymore.
595    fn take_ctrl_plane(self) -> ControlPlane;
596
597    /// A hook that triggers when first emitting a new block.
598    /// It is guaranteed to be called before any instructions are emitted.
599    fn on_new_block(&mut self) {}
600
601    /// The [`FrameLayout`] for the function currently being compiled.
602    fn frame_layout(&self) -> &FrameLayout;
603}
604
605/// Tracks the user stack maps of safepoint instructions as they are emitted.
606///
607/// Each backend's emit state holds one of these. The VCode emission loop
608/// provides each safepoint instruction's stack map (if any) via
609/// [`StackMapEmitter::pre_safepoint`], the backend's emission code for that
610/// instruction then [takes](StackMapEmitter::take) it and
611/// [records](StackMapEmitter::push) it at the appropriate code offset, and
612/// finally the emission loop checks that it was
613/// [consumed](StackMapEmitter::assert_consumed).
614///
615/// Stack maps are keyed by code offset, so no two may be recorded at the same
616/// offset. This helper remembers where the last stack map was recorded and
617/// inserts padding when a new safepoint would otherwise collide with it (e.g.,
618/// a potentially-faulting load placed directly at the return address of a
619/// preceding call).
620pub struct StackMapEmitter<I: VCodeInst> {
621    /// The user stack map for the upcoming safepoint instruction, as provided
622    /// to `pre_safepoint()`.
623    pending: Option<ir::UserStackMap>,
624
625    /// The code offset at which the most recent user stack map was recorded,
626    /// if any.
627    last_offset: Option<CodeOffset>,
628
629    _phantom: PhantomData<fn() -> I>,
630}
631
632impl<I: VCodeInst> Default for StackMapEmitter<I> {
633    fn default() -> Self {
634        Self {
635            pending: None,
636            last_offset: None,
637            _phantom: PhantomData,
638        }
639    }
640}
641
642impl<I: VCodeInst> Clone for StackMapEmitter<I> {
643    fn clone(&self) -> Self {
644        Self {
645            pending: self.pending.clone(),
646            last_offset: self.last_offset,
647            _phantom: PhantomData,
648        }
649    }
650}
651
652impl<I: VCodeInst> fmt::Debug for StackMapEmitter<I> {
653    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
654        f.debug_struct("StackMapEmitter")
655            .field("pending", &self.pending)
656            .field("last_offset", &self.last_offset)
657            .finish()
658    }
659}
660
661impl<I: VCodeInst> StackMapEmitter<I> {
662    /// Set the user stack map for the upcoming safepoint instruction.
663    ///
664    /// Called by the emission loop before emitting every safepoint
665    /// instruction.
666    pub fn pre_safepoint(&mut self, stack_map: Option<ir::UserStackMap>) {
667        self.pending = stack_map;
668    }
669
670    /// Take the user stack map for the safepoint instruction being emitted,
671    /// if any.
672    ///
673    /// If a stack map was already recorded at the current offset (e.g., at
674    /// the return address of an immediately-preceding call), a NOP is emitted
675    /// so that this safepoint's stack map lands at a distinct offset.
676    ///
677    /// For a safepoint whose stack map is recorded at the start of the
678    /// instruction (e.g., a potentially-faulting load), call this before
679    /// emitting any of the instruction's code. For a call, whose stack map is
680    /// recorded at the return address, call this after emitting the call
681    /// instruction itself, so that no padding is ever inserted.
682    pub fn take(&mut self, sink: &mut MachBuffer<I>) -> Option<ir::UserStackMap> {
683        let stack_map = self.pending.take()?;
684        if self.last_offset == Some(sink.cur_offset()) {
685            sink.put_data(&I::gen_nop_units()[0]);
686        }
687        Some(stack_map)
688    }
689
690    /// Discard the user stack map for the safepoint instruction being
691    /// emitted, if any, for safepoints that do not record one.
692    pub fn discard(&mut self) {
693        self.pending = None;
694    }
695
696    /// Record `stack_map` at code offset `offset` in `sink`.
697    pub fn push(
698        &mut self,
699        sink: &mut MachBuffer<I>,
700        frame_layout: &FrameLayout,
701        offset: CodeOffset,
702        stack_map: ir::UserStackMap,
703    ) {
704        sink.push_user_stack_map(frame_layout, offset, stack_map);
705        self.last_offset = Some(offset);
706    }
707
708    /// Assert (in debug builds) that the stack map provided to
709    /// `pre_safepoint()`, if any, has been consumed.
710    ///
711    /// Called by the emission loop after emitting every safepoint
712    /// instruction.
713    pub fn assert_consumed(&self) {
714        debug_assert!(
715            self.pending.is_none(),
716            "safepoint instruction did not consume its user stack map"
717        );
718    }
719}
720
721/// The result of a `MachBackend::compile_function()` call. Contains machine
722/// code (as bytes) and a disassembly, if requested.
723#[derive(PartialEq, Debug, Clone)]
724#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
725pub struct CompiledCode {
726    /// Machine code.
727    pub buffer: MachBufferFinalized,
728    /// Disassembly, if requested.
729    pub vcode: Option<String>,
730    /// Debug info: value labels to registers/stackslots at code offsets.
731    pub value_labels_ranges: ValueLabelsRanges,
732    /// Basic-block layout info: block start offsets.
733    ///
734    /// This info is generated only if the `machine_code_cfg_info`
735    /// flag is set.
736    pub bb_starts: Vec<CodeOffset>,
737    /// Basic-block layout info: block edges. Each edge is `(from,
738    /// to)`, where `from` and `to` are basic-block start offsets of
739    /// the respective blocks.
740    ///
741    /// This info is generated only if the `machine_code_cfg_info`
742    /// flag is set.
743    pub bb_edges: Vec<(CodeOffset, CodeOffset)>,
744}
745
746/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.
747///
748/// Only used internally, in a transient manner, for the incremental compilation cache.
749#[derive(PartialEq, Debug, Clone)]
750#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
751pub struct CompiledCodeStencil(pub(crate) CompiledCode);
752
753impl CompiledCodeStencil {
754    /// Apply function parameters to finalize a stencil into its final form.
755    pub fn apply_params(mut self, params: &FunctionParameters) -> CompiledCode {
756        self.0.buffer.apply_base_srcloc(params.base_srcloc());
757        self.0
758    }
759}
760
761impl CompiledCode {
762    /// Get a `CodeInfo` describing section sizes from this compilation result.
763    pub fn code_info(&self) -> CodeInfo {
764        CodeInfo {
765            total_size: self.buffer.total_size(),
766        }
767    }
768
769    /// Returns a reference to the machine code generated for this function compilation.
770    pub fn code_buffer(&self) -> &[u8] {
771        self.buffer.data()
772    }
773
774    /// Get the disassembly of the buffer, using the given capstone context.
775    #[cfg(feature = "disas")]
776    pub fn disassemble(
777        &self,
778        params: Option<&crate::ir::function::FunctionParameters>,
779        cs: &capstone::Capstone,
780    ) -> Result<String, anyhow::Error> {
781        use core::fmt::Write;
782
783        let mut buf = String::new();
784
785        let relocs = self.buffer.relocs();
786        let traps = self.buffer.traps();
787        let mut patchables = self.buffer.patchable_call_sites().peekable();
788
789        // Normalize the block starts to include an initial block of offset 0.
790        let mut block_starts = Vec::new();
791        if self.bb_starts.first().copied() != Some(0) {
792            block_starts.push(0);
793        }
794        block_starts.extend_from_slice(&self.bb_starts);
795        block_starts.push(self.buffer.data().len() as u32);
796
797        // Iterate over block regions, to ensure that we always produce block labels
798        for (n, (&start, &end)) in block_starts
799            .iter()
800            .zip(block_starts.iter().skip(1))
801            .enumerate()
802        {
803            writeln!(buf, "block{n}: ; offset 0x{start:x}")?;
804
805            let buffer = &self.buffer.data()[start as usize..end as usize];
806            let insns = cs.disasm_all(buffer, start as u64).map_err(map_caperr)?;
807            for i in insns.iter() {
808                write!(buf, "  ")?;
809
810                let op_str = i.op_str().unwrap_or("");
811                if let Some(s) = i.mnemonic() {
812                    write!(buf, "{s}")?;
813                    if !op_str.is_empty() {
814                        write!(buf, " ")?;
815                    }
816                }
817
818                write!(buf, "{op_str}")?;
819
820                let end = i.address() + i.bytes().len() as u64;
821                let contains = |off| i.address() <= off && off < end;
822
823                for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) {
824                    write!(
825                        buf,
826                        " ; reloc_external {} {} {}",
827                        reloc.kind,
828                        reloc.target.display(params),
829                        reloc.addend,
830                    )?;
831                }
832
833                if let Some(trap) = traps.iter().find(|trap| contains(trap.offset as u64)) {
834                    write!(buf, " ; trap: {}", trap.code)?;
835                }
836
837                if let Some(patchable) = patchables.peek()
838                    && patchable.ret_addr == end as u32
839                {
840                    write!(
841                        buf,
842                        " ; patchable call: NOP out last {} bytes",
843                        patchable.len
844                    )?;
845                    patchables.next();
846                }
847
848                writeln!(buf)?;
849            }
850
851            // `disasm_all` stops at the first instruction it cannot decode and
852            // reports success rather than an error, so without this the rest of
853            // the block would silently vanish from the listing. That matters
854            // most for `precise-output` filetests, whose expectations would
855            // then assert nothing at all about those bytes. Print them as
856            // `.byte` directives instead, the same form capstone itself
857            // produces for undecodable s390x instructions.
858            let decoded: usize = insns.iter().map(|i| i.bytes().len()).sum();
859            for (chunk_idx, chunk) in buffer[decoded..].chunks(8).enumerate() {
860                let addr = start as u64 + decoded as u64 + (chunk_idx * 8) as u64;
861                let chunk_end = addr + chunk.len() as u64;
862                let contains = |off| addr <= off && off < chunk_end;
863
864                write!(buf, "  .byte ")?;
865                for (i, byte) in chunk.iter().enumerate() {
866                    if i > 0 {
867                        write!(buf, ", ")?;
868                    }
869                    write!(buf, "{byte:#04x}")?;
870                }
871
872                for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) {
873                    write!(
874                        buf,
875                        " ; reloc_external {} {} {}",
876                        reloc.kind,
877                        reloc.target.display(params),
878                        reloc.addend,
879                    )?;
880                }
881
882                if let Some(trap) = traps.iter().find(|trap| contains(trap.offset as u64)) {
883                    write!(buf, " ; trap: {}", trap.code)?;
884                }
885
886                writeln!(buf)?;
887            }
888        }
889
890        return Ok(buf);
891
892        fn map_caperr(err: capstone::Error) -> anyhow::Error {
893            anyhow::format_err!("{err}")
894        }
895    }
896
897    /// If available, return information about the code layout in the
898    /// final machine code: the offsets (in bytes) of each basic-block
899    /// start, and all basic-block edges.
900    pub fn get_code_bb_layout(&self) -> (Vec<usize>, Vec<(usize, usize)>) {
901        (
902            self.bb_starts.iter().map(|&off| off as usize).collect(),
903            self.bb_edges
904                .iter()
905                .map(|&(from, to)| (from as usize, to as usize))
906                .collect(),
907        )
908    }
909
910    /// Creates unwind information for the function.
911    ///
912    /// Returns `None` if the function has no unwind information.
913    #[cfg(feature = "unwind")]
914    pub fn create_unwind_info(
915        &self,
916        isa: &dyn crate::isa::TargetIsa,
917    ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
918        use crate::isa::unwind::UnwindInfoKind;
919        let unwind_info_kind = match isa.triple().operating_system {
920            target_lexicon::OperatingSystem::Windows => UnwindInfoKind::Windows,
921            _ => UnwindInfoKind::SystemV,
922        };
923        self.create_unwind_info_of_kind(isa, unwind_info_kind)
924    }
925
926    /// Creates unwind information for the function using the supplied
927    /// "kind". Supports cross-OS (but not cross-arch) generation.
928    ///
929    /// Returns `None` if the function has no unwind information.
930    #[cfg(feature = "unwind")]
931    pub fn create_unwind_info_of_kind(
932        &self,
933        isa: &dyn crate::isa::TargetIsa,
934        unwind_info_kind: crate::isa::unwind::UnwindInfoKind,
935    ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
936        isa.emit_unwind_info(self, unwind_info_kind)
937    }
938}
939
940/// An object that can be used to create the text section of an executable.
941///
942/// This primarily handles resolving relative relocations at
943/// text-section-assembly time rather than at load/link time. This
944/// architecture-specific logic is sort of like a linker, but only for one
945/// object file at a time.
946pub trait TextSectionBuilder {
947    /// Appends `data` to the text section with the `align` specified.
948    ///
949    /// If `labeled` is `true` then this also binds the appended data to the
950    /// `n`th label for how many times this has been called with `labeled:
951    /// true`. The label target can be passed as the `target` argument to
952    /// `resolve_reloc`.
953    ///
954    /// This function returns the offset at which the data was placed in the
955    /// text section.
956    fn append(
957        &mut self,
958        labeled: bool,
959        data: &[u8],
960        align: u32,
961        ctrl_plane: &mut ControlPlane,
962    ) -> u64;
963
964    /// Attempts to resolve a relocation for this function.
965    ///
966    /// The `offset` is the offset of the relocation, within the text section.
967    /// The `reloc` is the kind of relocation.
968    /// The `addend` is the value to add to the relocation.
969    /// The `target` is the labeled function that is the target of this
970    /// relocation.
971    ///
972    /// Labeled functions are created with the `append` function above by
973    /// setting the `labeled` parameter to `true`.
974    ///
975    /// If this builder does not know how to handle `reloc` then this function
976    /// will return `false`. Otherwise this function will return `true` and this
977    /// relocation will be resolved in the final bytes returned by `finish`.
978    fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: usize) -> bool;
979
980    /// A debug-only option which is used to for
981    fn force_veneers(&mut self);
982
983    /// Write the `data` provided at `offset`, for example when resolving a
984    /// relocation.
985    fn write(&mut self, offset: u64, data: &[u8]);
986
987    /// Completes this text section, filling out any final details, and returns
988    /// the bytes of the text section.
989    fn finish(&mut self, ctrl_plane: &mut ControlPlane) -> Vec<u8>;
990}