cranelift_codegen/machinst/mod.rs
1//! This module exposes the machine-specific backend definition pieces.
2//!
3//! The MachInst infrastructure is the compiler backend, from CLIF
4//! (ir::Function) to machine code. The purpose of this infrastructure is, at a
5//! high level, to do instruction selection/lowering (to machine instructions),
6//! register allocation, and then perform all the fixups to branches, constant
7//! data references, etc., needed to actually generate machine code.
8//!
9//! The container for machine instructions, at various stages of construction,
10//! is the `VCode` struct. We refer to a sequence of machine instructions organized
11//! into basic blocks as "vcode". This is short for "virtual-register code".
12//!
13//! The compilation pipeline, from an `ir::Function` (already optimized as much as
14//! you like by machine-independent optimization passes) onward, is as follows.
15//!
16//! ```plain
17//!
18//! ir::Function (SSA IR, machine-independent opcodes)
19//! |
20//! | [lower]
21//! |
22//! VCode<arch_backend::Inst> (machine instructions:
23//! | - mostly virtual registers.
24//! | - cond branches in two-target form.
25//! | - branch targets are block indices.
26//! | - in-memory constants held by insns,
27//! | with unknown offsets.
28//! | - critical edges (actually all edges)
29//! | are split.)
30//! |
31//! | [regalloc --> `regalloc2::Output`; VCode is unchanged]
32//! |
33//! | [binary emission via MachBuffer]
34//! |
35//! Vec<u8> (machine code:
36//! | - two-dest branches resolved via
37//! | streaming branch resolution/simplification.
38//! | - regalloc `Allocation` results used directly
39//! | by instruction emission code.
40//! | - prologue and epilogue(s) built and emitted
41//! | directly during emission.
42//! | - SP-relative offsets resolved by tracking
43//! | EmitState.)
44//!
45//! ```
46
47use crate::binemit::{Addend, CodeInfo, CodeOffset, Reloc};
48use crate::ir::{
49 self, DynamicStackSlot, Endianness, RelSourceLoc, StackSlot, TrapCode, Type,
50 function::FunctionParameters,
51};
52use crate::isa::FunctionAlignment;
53use crate::result::CodegenResult;
54use crate::settings;
55use crate::value_label::ValueLabelsRanges;
56use alloc::string::String;
57use alloc::vec::Vec;
58use core::fmt;
59use core::fmt::Debug;
60use core::marker::PhantomData;
61use core::num::NonZeroU8;
62use cranelift_control::ControlPlane;
63use cranelift_entity::PrimaryMap;
64use regalloc2::VReg;
65use smallvec::{SmallVec, smallvec};
66
67#[cfg(feature = "enable-serde")]
68use serde_derive::{Deserialize, Serialize};
69
70/// Guaranteed to use "natural alignment" for the given type.
71const BIT_ALIGNED: u16 = 1 << 0;
72
73/// A load that reads data in memory that does not change for the
74/// duration of the function's execution.
75const BIT_READONLY: u16 = 1 << 1;
76
77/// Load multi-byte values from memory in a little-endian format.
78const BIT_LITTLE_ENDIAN: u16 = 1 << 2;
79
80/// Load multi-byte values from memory in a big-endian format.
81const BIT_BIG_ENDIAN: u16 = 1 << 3;
82
83/// A volatile access: it happens exactly as written (once, with its own width,
84/// in order with other volatile accesses), even when its result is unused.
85const BIT_VOLATILE: u16 = 1 << 4;
86
87/// Trap code, if any, for this memory operation.
88const MASK_TRAP_CODE: u16 = ((1 << TRAP_CODE_BITS) - 1) << TRAP_CODE_OFFSET;
89const TRAP_CODE_BITS: u16 = 8;
90const TRAP_CODE_OFFSET: u16 = 7;
91
92/// Whether this memory operation may be freely moved by the optimizer.
93const BIT_CAN_MOVE: u16 = 1 << 15;
94
95/// Backend memory-operation flags.
96///
97/// These are the bit-packed flags that backends operate on directly.
98///
99/// Unlike [`ir::MemFlagsData`], this does not carry alias-region metadata.
100#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
101#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
102pub struct MachMemFlags {
103 // Bit layout:
104 //
105 // - Bit 0: aligned
106 // - Bit 1: readonly
107 // - Bit 2: little-endian
108 // - Bit 3: big-endian
109 // - Bit 4: volatile
110 // - Bits 5..6: unused
111 // - Bits 7..14: trap code
112 // - Bit 15: can_move
113 bits: u16,
114}
115
116impl MachMemFlags {
117 /// Create a new empty set of flags.
118 pub const fn new() -> Self {
119 Self { bits: 0 }.with_trap_code(Some(TrapCode::HEAP_OUT_OF_BOUNDS))
120 }
121
122 /// Create a set of flags representing an access from a "trusted" address.
123 pub const fn trusted() -> Self {
124 Self::new().with_notrap().with_aligned(true)
125 }
126
127 const fn read_bit(self, bit: u16) -> bool {
128 self.bits & bit != 0
129 }
130
131 const fn with_bit(mut self, bit: u16, enabled: bool) -> Self {
132 if enabled {
133 self.bits |= bit;
134 } else {
135 self.bits &= !bit;
136 }
137 self
138 }
139
140 /// Return endianness of the memory access.
141 pub const fn endianness(self, native_endianness: Endianness) -> Endianness {
142 if self.read_bit(BIT_LITTLE_ENDIAN) {
143 Endianness::Little
144 } else if self.read_bit(BIT_BIG_ENDIAN) {
145 Endianness::Big
146 } else {
147 native_endianness
148 }
149 }
150
151 /// Return endianness of the memory access, if explicitly specified.
152 pub const fn explicit_endianness(self) -> Option<Endianness> {
153 if self.read_bit(BIT_LITTLE_ENDIAN) {
154 Some(Endianness::Little)
155 } else if self.read_bit(BIT_BIG_ENDIAN) {
156 Some(Endianness::Big)
157 } else {
158 None
159 }
160 }
161
162 /// Set endianness of the memory access, returning new flags.
163 pub const fn with_endianness(self, endianness: Endianness) -> Self {
164 let res = match endianness {
165 Endianness::Little => self.with_bit(BIT_LITTLE_ENDIAN, true),
166 Endianness::Big => self.with_bit(BIT_BIG_ENDIAN, true),
167 };
168 assert!(!(res.read_bit(BIT_LITTLE_ENDIAN) && res.read_bit(BIT_BIG_ENDIAN)));
169 res
170 }
171
172 /// Test if this memory access cannot trap.
173 pub const fn notrap(self) -> bool {
174 self.trap_code().is_none()
175 }
176
177 /// Set these flags to indicate this access does not trap.
178 pub const fn with_notrap(self) -> Self {
179 self.with_trap_code(None)
180 }
181
182 /// Test if the `can_move` flag is set.
183 pub const fn can_move(self) -> bool {
184 self.read_bit(BIT_CAN_MOVE)
185 }
186
187 /// Set or clear the `can_move` flag, returning new flags.
188 pub const fn with_can_move(self, enabled: bool) -> Self {
189 self.with_bit(BIT_CAN_MOVE, enabled)
190 }
191
192 /// Test if the `aligned` flag is set.
193 pub const fn aligned(self) -> bool {
194 self.read_bit(BIT_ALIGNED)
195 }
196
197 /// Set or clear the `aligned` flag, returning new flags.
198 pub const fn with_aligned(self, enabled: bool) -> Self {
199 self.with_bit(BIT_ALIGNED, enabled)
200 }
201
202 /// Test if the `readonly` flag is set.
203 pub const fn readonly(self) -> bool {
204 self.read_bit(BIT_READONLY)
205 }
206
207 /// Set or clear the `readonly` flag, returning new flags.
208 pub const fn with_readonly(self, enabled: bool) -> Self {
209 self.with_bit(BIT_READONLY, enabled)
210 }
211
212 /// Test if the `volatile` flag is set.
213 pub const fn volatile(self) -> bool {
214 self.read_bit(BIT_VOLATILE)
215 }
216
217 /// Set or clear the `volatile` flag, returning new flags.
218 pub const fn with_volatile(self, enabled: bool) -> Self {
219 self.with_bit(BIT_VOLATILE, enabled)
220 }
221
222 /// Get the trap code to report if this memory access traps.
223 pub const fn trap_code(self) -> Option<TrapCode> {
224 let byte = ((self.bits & MASK_TRAP_CODE) >> TRAP_CODE_OFFSET) as u8;
225 match NonZeroU8::new(byte) {
226 Some(code) => Some(TrapCode::from_raw(code)),
227 None => None,
228 }
229 }
230
231 /// Configures these flags with the specified trap code `code`.
232 pub const fn with_trap_code(mut self, code: Option<TrapCode>) -> Self {
233 let bits = match code {
234 Some(code) => code.as_raw().get() as u16,
235 None => 0,
236 };
237 self.bits &= !MASK_TRAP_CODE;
238 self.bits |= bits << TRAP_CODE_OFFSET;
239 self
240 }
241}
242
243impl fmt::Display for MachMemFlags {
244 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
245 match self.trap_code() {
246 None => write!(f, " notrap")?,
247 Some(TrapCode::HEAP_OUT_OF_BOUNDS) => {}
248 Some(t) => write!(f, " {t}")?,
249 }
250 if self.aligned() {
251 write!(f, " aligned")?;
252 }
253 if self.readonly() {
254 write!(f, " readonly")?;
255 }
256 if self.can_move() {
257 write!(f, " can_move")?;
258 }
259 if self.volatile() {
260 write!(f, " volatile")?;
261 }
262 if self.read_bit(BIT_BIG_ENDIAN) {
263 write!(f, " big")?;
264 }
265 if self.read_bit(BIT_LITTLE_ENDIAN) {
266 write!(f, " little")?;
267 }
268 Ok(())
269 }
270}
271
272#[macro_use]
273pub mod isle;
274
275pub mod lower;
276pub use lower::*;
277pub mod vcode;
278pub use vcode::*;
279pub mod compile;
280pub use compile::*;
281pub mod blockorder;
282pub use blockorder::*;
283pub mod abi;
284pub use abi::*;
285pub mod buffer;
286pub use buffer::*;
287pub mod helpers;
288pub use helpers::*;
289pub mod valueregs;
290pub use reg::*;
291pub use valueregs::*;
292pub mod reg;
293
294/// A machine instruction.
295pub trait MachInst: Clone + Debug {
296 /// The ABI machine spec for this `MachInst`.
297 type ABIMachineSpec: ABIMachineSpec<I = Self>;
298
299 /// Return the registers referenced by this machine instruction along with
300 /// the modes of reference (use, def, modify).
301 fn get_operands(&mut self, collector: &mut impl OperandVisitor);
302
303 /// If this is a simple move, return the (source, destination) tuple of registers.
304 fn is_move(&self) -> Option<(Writable<Reg>, Reg)>;
305
306 /// Is this a terminator (branch or ret)? If so, return its type
307 /// (ret/uncond/cond) and target if applicable.
308 fn is_term(&self) -> MachTerminator;
309
310 /// Is this an unconditional trap?
311 fn is_trap(&self) -> bool;
312
313 /// Is this an "args" pseudoinst?
314 fn is_args(&self) -> bool;
315
316 /// Classify the type of call instruction this is.
317 ///
318 /// This enables more granular function type analysis and optimization.
319 /// Returns `CallType::None` for non-call instructions, `CallType::Regular`
320 /// for normal calls that return to the caller, and `CallType::TailCall`
321 /// for tail calls that don't return to the caller.
322 fn call_type(&self) -> CallType;
323
324 /// Should this instruction's clobber-list be included in the
325 /// clobber-set?
326 fn is_included_in_clobbers(&self) -> bool;
327
328 /// Does this instruction access memory?
329 fn is_mem_access(&self) -> bool;
330
331 /// Generate a move.
332 fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
333
334 /// Generate a dummy instruction that will keep a value alive but
335 /// has no other purpose.
336 fn gen_dummy_use(reg: Reg) -> Self;
337
338 /// Determine register class(es) to store the given Cranelift type, and the
339 /// Cranelift type actually stored in the underlying register(s). May return
340 /// an error if the type isn't supported by this backend.
341 ///
342 /// If the type requires multiple registers, then the list of registers is
343 /// returned in little-endian order.
344 ///
345 /// Note that the type actually stored in the register(s) may differ in the
346 /// case that a value is split across registers: for example, on a 32-bit
347 /// target, an I64 may be stored in two registers, each of which holds an
348 /// I32. The actually-stored types are used only to inform the backend when
349 /// generating spills and reloads for individual registers.
350 fn rc_for_type(ty: &Type) -> CodegenResult<(&[RegClass], &[Type])>;
351
352 /// Get an appropriate type that can fully hold a value in a given
353 /// register class. This may not be the only type that maps to
354 /// that class, but when used with `gen_move()` or the ABI trait's
355 /// load/spill constructors, it should produce instruction(s) that
356 /// move the entire register contents.
357 fn canonical_type_for_rc(rc: RegClass) -> Type;
358
359 /// Generate a jump to another target. Used during lowering of
360 /// control flow.
361 fn gen_jump(target: MachLabel) -> Self;
362
363 /// Generate a store of an immediate 64-bit integer to a register. Used by
364 /// the control plane to generate random instructions.
365 fn gen_imm_u64(_value: u64, _dst: Writable<Reg>) -> Option<Self> {
366 None
367 }
368
369 /// Generate a store of an immediate 64-bit integer to a register. Used by
370 /// the control plane to generate random instructions. The tmp register may
371 /// be used by architectures which don't support writing immediate values to
372 /// floating point registers directly.
373 fn gen_imm_f64(_value: f64, _tmp: Writable<Reg>, _dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
374 SmallVec::new()
375 }
376
377 /// Generate a NOP. The `preferred_size` parameter allows the caller to
378 /// request a NOP of that size, or as close to it as possible. The machine
379 /// backend may return a NOP whose binary encoding is smaller than the
380 /// preferred size, but must not return a NOP that is larger. However,
381 /// the instruction must have a nonzero size if preferred_size is nonzero.
382 fn gen_nop(preferred_size: usize) -> Self;
383
384 /// The various kinds of NOP, with size, sorted in ascending-size
385 /// order.
386 fn gen_nop_units() -> Vec<Vec<u8>>;
387
388 /// Align a basic block offset (from start of function). By default, no
389 /// alignment occurs.
390 fn align_basic_block(offset: CodeOffset) -> CodeOffset {
391 offset
392 }
393
394 /// What is the worst-case instruction size emitted by this instruction type?
395 fn worst_case_size() -> CodeOffset;
396
397 /// Worst-case growth, in bytes, that emitting a single `MachInst`
398 /// instruction may add to the `MachBuffer`'s pending-island state
399 /// (constants, deferred traps, and worst-case veneers for new
400 /// fixups).
401 ///
402 /// `MachBuffer` treats one instruction emission as the atomic
403 /// "commit unit" and uses `worst_case_size() +
404 /// worst_case_island_growth()` as the per-instruction lookahead
405 /// bound when deciding whether to flush an island. Backends whose
406 /// label-use kinds always have wide enough range that islands are
407 /// never required may leave this at zero.
408 fn worst_case_island_growth() -> CodeOffset;
409
410 /// Is this a safepoint?
411 fn is_safepoint(&self) -> bool;
412
413 /// Generate an instruction that must appear at the beginning of a basic
414 /// block, if any. Note that the return value must not be subject to
415 /// register allocation.
416 fn gen_block_start(
417 _is_indirect_branch_target: bool,
418 _is_forward_edge_cfi_enabled: bool,
419 ) -> Option<Self> {
420 None
421 }
422
423 /// Returns a description of the alignment required for functions for this
424 /// architecture.
425 fn function_alignment() -> FunctionAlignment;
426
427 /// Is this a low-level, one-way branch, not meant for use in a
428 /// VCode body? These instructions are meant to be used only when
429 /// directly emitted, i.e. when `MachInst` is used as an assembler
430 /// library.
431 fn is_low_level_branch(&self) -> bool {
432 false
433 }
434
435 /// A label-use kind: a type that describes the types of label references that
436 /// can occur in an instruction.
437 type LabelUse: MachInstLabelUse;
438
439 /// Byte representation of a trap opcode which is inserted by `MachBuffer`
440 /// during its `defer_trap` method.
441 const TRAP_OPCODE: &'static [u8];
442}
443
444/// A descriptor of a label reference (use) in an instruction set.
445pub trait MachInstLabelUse: Clone + Copy + Debug + Eq {
446 /// Required alignment for any veneer. Usually the required instruction
447 /// alignment (e.g., 4 for a RISC with 32-bit instructions, or 1 for x86).
448 const ALIGN: CodeOffset;
449
450 /// What is the maximum PC-relative range (positive)? E.g., if `1024`, a
451 /// label-reference fixup at offset `x` is valid if the label resolves to `x
452 /// + 1024`.
453 fn max_pos_range(self) -> CodeOffset;
454 /// What is the maximum PC-relative range (negative)? This is the absolute
455 /// value; i.e., if `1024`, then a label-reference fixup at offset `x` is
456 /// valid if the label resolves to `x - 1024`.
457 fn max_neg_range(self) -> CodeOffset;
458 /// What is the size of code-buffer slice this label-use needs to patch in
459 /// the label's value?
460 fn patch_size(self) -> CodeOffset;
461 /// Perform a code-patch, given the offset into the buffer of this label use
462 /// and the offset into the buffer of the label's definition.
463 /// It is guaranteed that, given `delta = offset - label_offset`, we will
464 /// have `offset >= -self.max_neg_range()` and `offset <=
465 /// self.max_pos_range()`.
466 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset);
467 /// Can the label-use be patched to a veneer that supports a longer range?
468 /// Usually valid for jumps (a short-range jump can jump to a longer-range
469 /// jump), but not for e.g. constant pool references, because the constant
470 /// load would require different code (one more level of indirection).
471 fn supports_veneer(self) -> bool;
472 /// How many bytes are needed for a veneer?
473 fn veneer_size(self) -> CodeOffset;
474 /// What's the largest possible veneer that may be generated?
475 fn worst_case_veneer_size() -> CodeOffset;
476 /// Generate a veneer. The given code-buffer slice is `self.veneer_size()`
477 /// bytes long at offset `veneer_offset` in the buffer. The original
478 /// label-use will be patched to refer to this veneer's offset. A new
479 /// (offset, LabelUse) is returned that allows the veneer to use the actual
480 /// label. For veneers to work properly, it is expected that the new veneer
481 /// has a larger range; on most platforms this probably means either a
482 /// "long-range jump" (e.g., on ARM, the 26-bit form), or if already at that
483 /// stage, a jump that supports a full 32-bit range, for example.
484 fn generate_veneer(self, buffer: &mut [u8], veneer_offset: CodeOffset) -> (CodeOffset, Self);
485
486 /// Returns the corresponding label-use for the relocation specified.
487 ///
488 /// This returns `None` if the relocation doesn't have a corresponding
489 /// representation for the target architecture.
490 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self>;
491}
492
493/// Classification of call instruction types for granular analysis.
494#[derive(Clone, Copy, Debug, PartialEq, Eq)]
495pub enum CallType {
496 /// Not a call instruction.
497 None,
498 /// Regular call that returns to the caller.
499 Regular,
500 /// Tail call that doesn't return to the caller.
501 TailCall,
502}
503
504/// Function classification based on call patterns.
505///
506/// This enum classifies functions based on their calling behavior to enable
507/// targeted optimizations. Functions are categorized as:
508/// - `None`: No calls at all (can use simplified calling conventions)
509/// - `TailOnly`: Only tail calls (may skip frame setup in some cases)
510/// - `Regular`: Has regular calls (requires full calling convention support)
511#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
512pub enum FunctionCalls {
513 /// Function makes no calls at all.
514 #[default]
515 None,
516 /// Function only makes tail calls (no regular calls).
517 TailOnly,
518 /// Function makes at least one regular call (may also have tail calls).
519 Regular,
520}
521
522impl FunctionCalls {
523 /// Update the function classification based on a new call instruction.
524 ///
525 /// This method implements the merge logic for accumulating call patterns:
526 /// - Any regular call makes the function Regular
527 /// - Tail calls upgrade None to TailOnly
528 /// - Regular always stays Regular
529 pub fn update(&mut self, call_type: CallType) {
530 *self = match (*self, call_type) {
531 // No call instruction - state unchanged
532 (current, CallType::None) => current,
533 // Regular call always results in Regular classification
534 (_, CallType::Regular) => FunctionCalls::Regular,
535 // Tail call: None becomes TailOnly, others unchanged
536 (FunctionCalls::None, CallType::TailCall) => FunctionCalls::TailOnly,
537 (current, CallType::TailCall) => current,
538 };
539 }
540}
541
542/// Describes a block terminator (not call) in the VCode.
543///
544/// Actual targets are not included: the single-source-of-truth for
545/// those is the VCode itself, which holds, for each block, successors
546/// and outgoing branch args per successor.
547#[derive(Clone, Debug, PartialEq, Eq)]
548pub enum MachTerminator {
549 /// Not a terminator.
550 None,
551 /// A return instruction.
552 Ret,
553 /// A tail call.
554 RetCall,
555 /// A branch.
556 Branch,
557}
558
559/// A trait describing the ability to encode a MachInst into binary machine code.
560pub trait MachInstEmit: MachInst {
561 /// Persistent state carried across `emit` invocations.
562 type State: MachInstEmitState<Self>;
563
564 /// Constant information used in `emit` invocations.
565 type Info;
566
567 /// Emit the instruction.
568 fn emit(&self, code: &mut MachBuffer<Self>, info: &Self::Info, state: &mut Self::State);
569
570 /// Pretty-print the instruction.
571 fn pretty_print_inst(&self, state: &mut Self::State) -> String;
572}
573
574/// A trait describing the emission state carried between MachInsts when
575/// emitting a function body.
576pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {
577 /// Create a new emission state given the ABI object.
578 fn new(abi: &Callee<I::ABIMachineSpec>, ctrl_plane: ControlPlane) -> Self;
579
580 /// The helper that tracks user stack maps for safepoint instructions.
581 ///
582 /// The emission loop uses this to provide each safepoint instruction's
583 /// stack map before emitting it and to check afterwards that it was
584 /// consumed.
585 fn stack_map_emitter(&mut self) -> &mut StackMapEmitter<I>;
586
587 /// The emission state holds ownership of a control plane, so it doesn't
588 /// have to be passed around explicitly too much. `ctrl_plane_mut` may
589 /// be used if temporary access to the control plane is needed by some
590 /// other function that doesn't have access to the emission state.
591 fn ctrl_plane_mut(&mut self) -> &mut ControlPlane;
592
593 /// Used to continue using a control plane after the emission state is
594 /// not needed anymore.
595 fn take_ctrl_plane(self) -> ControlPlane;
596
597 /// A hook that triggers when first emitting a new block.
598 /// It is guaranteed to be called before any instructions are emitted.
599 fn on_new_block(&mut self) {}
600
601 /// The [`FrameLayout`] for the function currently being compiled.
602 fn frame_layout(&self) -> &FrameLayout;
603}
604
605/// Tracks the user stack maps of safepoint instructions as they are emitted.
606///
607/// Each backend's emit state holds one of these. The VCode emission loop
608/// provides each safepoint instruction's stack map (if any) via
609/// [`StackMapEmitter::pre_safepoint`], the backend's emission code for that
610/// instruction then [takes](StackMapEmitter::take) it and
611/// [records](StackMapEmitter::push) it at the appropriate code offset, and
612/// finally the emission loop checks that it was
613/// [consumed](StackMapEmitter::assert_consumed).
614///
615/// Stack maps are keyed by code offset, so no two may be recorded at the same
616/// offset. This helper remembers where the last stack map was recorded and
617/// inserts padding when a new safepoint would otherwise collide with it (e.g.,
618/// a potentially-faulting load placed directly at the return address of a
619/// preceding call).
620pub struct StackMapEmitter<I: VCodeInst> {
621 /// The user stack map for the upcoming safepoint instruction, as provided
622 /// to `pre_safepoint()`.
623 pending: Option<ir::UserStackMap>,
624
625 /// The code offset at which the most recent user stack map was recorded,
626 /// if any.
627 last_offset: Option<CodeOffset>,
628
629 _phantom: PhantomData<fn() -> I>,
630}
631
632impl<I: VCodeInst> Default for StackMapEmitter<I> {
633 fn default() -> Self {
634 Self {
635 pending: None,
636 last_offset: None,
637 _phantom: PhantomData,
638 }
639 }
640}
641
642impl<I: VCodeInst> Clone for StackMapEmitter<I> {
643 fn clone(&self) -> Self {
644 Self {
645 pending: self.pending.clone(),
646 last_offset: self.last_offset,
647 _phantom: PhantomData,
648 }
649 }
650}
651
652impl<I: VCodeInst> fmt::Debug for StackMapEmitter<I> {
653 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
654 f.debug_struct("StackMapEmitter")
655 .field("pending", &self.pending)
656 .field("last_offset", &self.last_offset)
657 .finish()
658 }
659}
660
661impl<I: VCodeInst> StackMapEmitter<I> {
662 /// Set the user stack map for the upcoming safepoint instruction.
663 ///
664 /// Called by the emission loop before emitting every safepoint
665 /// instruction.
666 pub fn pre_safepoint(&mut self, stack_map: Option<ir::UserStackMap>) {
667 self.pending = stack_map;
668 }
669
670 /// Take the user stack map for the safepoint instruction being emitted,
671 /// if any.
672 ///
673 /// If a stack map was already recorded at the current offset (e.g., at
674 /// the return address of an immediately-preceding call), a NOP is emitted
675 /// so that this safepoint's stack map lands at a distinct offset.
676 ///
677 /// For a safepoint whose stack map is recorded at the start of the
678 /// instruction (e.g., a potentially-faulting load), call this before
679 /// emitting any of the instruction's code. For a call, whose stack map is
680 /// recorded at the return address, call this after emitting the call
681 /// instruction itself, so that no padding is ever inserted.
682 pub fn take(&mut self, sink: &mut MachBuffer<I>) -> Option<ir::UserStackMap> {
683 let stack_map = self.pending.take()?;
684 if self.last_offset == Some(sink.cur_offset()) {
685 sink.put_data(&I::gen_nop_units()[0]);
686 }
687 Some(stack_map)
688 }
689
690 /// Discard the user stack map for the safepoint instruction being
691 /// emitted, if any, for safepoints that do not record one.
692 pub fn discard(&mut self) {
693 self.pending = None;
694 }
695
696 /// Record `stack_map` at code offset `offset` in `sink`.
697 pub fn push(
698 &mut self,
699 sink: &mut MachBuffer<I>,
700 frame_layout: &FrameLayout,
701 offset: CodeOffset,
702 stack_map: ir::UserStackMap,
703 ) {
704 sink.push_user_stack_map(frame_layout, offset, stack_map);
705 self.last_offset = Some(offset);
706 }
707
708 /// Assert (in debug builds) that the stack map provided to
709 /// `pre_safepoint()`, if any, has been consumed.
710 ///
711 /// Called by the emission loop after emitting every safepoint
712 /// instruction.
713 pub fn assert_consumed(&self) {
714 debug_assert!(
715 self.pending.is_none(),
716 "safepoint instruction did not consume its user stack map"
717 );
718 }
719}
720
721/// The result of a `MachBackend::compile_function()` call. Contains machine
722/// code (as bytes) and a disassembly, if requested.
723#[derive(PartialEq, Debug, Clone)]
724#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
725pub struct CompiledCode {
726 /// Machine code.
727 pub buffer: MachBufferFinalized,
728 /// Disassembly, if requested.
729 pub vcode: Option<String>,
730 /// Debug info: value labels to registers/stackslots at code offsets.
731 pub value_labels_ranges: ValueLabelsRanges,
732 /// Basic-block layout info: block start offsets.
733 ///
734 /// This info is generated only if the `machine_code_cfg_info`
735 /// flag is set.
736 pub bb_starts: Vec<CodeOffset>,
737 /// Basic-block layout info: block edges. Each edge is `(from,
738 /// to)`, where `from` and `to` are basic-block start offsets of
739 /// the respective blocks.
740 ///
741 /// This info is generated only if the `machine_code_cfg_info`
742 /// flag is set.
743 pub bb_edges: Vec<(CodeOffset, CodeOffset)>,
744}
745
746/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.
747///
748/// Only used internally, in a transient manner, for the incremental compilation cache.
749#[derive(PartialEq, Debug, Clone)]
750#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
751pub struct CompiledCodeStencil(pub(crate) CompiledCode);
752
753impl CompiledCodeStencil {
754 /// Apply function parameters to finalize a stencil into its final form.
755 pub fn apply_params(mut self, params: &FunctionParameters) -> CompiledCode {
756 self.0.buffer.apply_base_srcloc(params.base_srcloc());
757 self.0
758 }
759}
760
761impl CompiledCode {
762 /// Get a `CodeInfo` describing section sizes from this compilation result.
763 pub fn code_info(&self) -> CodeInfo {
764 CodeInfo {
765 total_size: self.buffer.total_size(),
766 }
767 }
768
769 /// Returns a reference to the machine code generated for this function compilation.
770 pub fn code_buffer(&self) -> &[u8] {
771 self.buffer.data()
772 }
773
774 /// Get the disassembly of the buffer, using the given capstone context.
775 #[cfg(feature = "disas")]
776 pub fn disassemble(
777 &self,
778 params: Option<&crate::ir::function::FunctionParameters>,
779 cs: &capstone::Capstone,
780 ) -> Result<String, anyhow::Error> {
781 use core::fmt::Write;
782
783 let mut buf = String::new();
784
785 let relocs = self.buffer.relocs();
786 let traps = self.buffer.traps();
787 let mut patchables = self.buffer.patchable_call_sites().peekable();
788
789 // Normalize the block starts to include an initial block of offset 0.
790 let mut block_starts = Vec::new();
791 if self.bb_starts.first().copied() != Some(0) {
792 block_starts.push(0);
793 }
794 block_starts.extend_from_slice(&self.bb_starts);
795 block_starts.push(self.buffer.data().len() as u32);
796
797 // Iterate over block regions, to ensure that we always produce block labels
798 for (n, (&start, &end)) in block_starts
799 .iter()
800 .zip(block_starts.iter().skip(1))
801 .enumerate()
802 {
803 writeln!(buf, "block{n}: ; offset 0x{start:x}")?;
804
805 let buffer = &self.buffer.data()[start as usize..end as usize];
806 let insns = cs.disasm_all(buffer, start as u64).map_err(map_caperr)?;
807 for i in insns.iter() {
808 write!(buf, " ")?;
809
810 let op_str = i.op_str().unwrap_or("");
811 if let Some(s) = i.mnemonic() {
812 write!(buf, "{s}")?;
813 if !op_str.is_empty() {
814 write!(buf, " ")?;
815 }
816 }
817
818 write!(buf, "{op_str}")?;
819
820 let end = i.address() + i.bytes().len() as u64;
821 let contains = |off| i.address() <= off && off < end;
822
823 for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) {
824 write!(
825 buf,
826 " ; reloc_external {} {} {}",
827 reloc.kind,
828 reloc.target.display(params),
829 reloc.addend,
830 )?;
831 }
832
833 if let Some(trap) = traps.iter().find(|trap| contains(trap.offset as u64)) {
834 write!(buf, " ; trap: {}", trap.code)?;
835 }
836
837 if let Some(patchable) = patchables.peek()
838 && patchable.ret_addr == end as u32
839 {
840 write!(
841 buf,
842 " ; patchable call: NOP out last {} bytes",
843 patchable.len
844 )?;
845 patchables.next();
846 }
847
848 writeln!(buf)?;
849 }
850
851 // `disasm_all` stops at the first instruction it cannot decode and
852 // reports success rather than an error, so without this the rest of
853 // the block would silently vanish from the listing. That matters
854 // most for `precise-output` filetests, whose expectations would
855 // then assert nothing at all about those bytes. Print them as
856 // `.byte` directives instead, the same form capstone itself
857 // produces for undecodable s390x instructions.
858 let decoded: usize = insns.iter().map(|i| i.bytes().len()).sum();
859 for (chunk_idx, chunk) in buffer[decoded..].chunks(8).enumerate() {
860 let addr = start as u64 + decoded as u64 + (chunk_idx * 8) as u64;
861 let chunk_end = addr + chunk.len() as u64;
862 let contains = |off| addr <= off && off < chunk_end;
863
864 write!(buf, " .byte ")?;
865 for (i, byte) in chunk.iter().enumerate() {
866 if i > 0 {
867 write!(buf, ", ")?;
868 }
869 write!(buf, "{byte:#04x}")?;
870 }
871
872 for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) {
873 write!(
874 buf,
875 " ; reloc_external {} {} {}",
876 reloc.kind,
877 reloc.target.display(params),
878 reloc.addend,
879 )?;
880 }
881
882 if let Some(trap) = traps.iter().find(|trap| contains(trap.offset as u64)) {
883 write!(buf, " ; trap: {}", trap.code)?;
884 }
885
886 writeln!(buf)?;
887 }
888 }
889
890 return Ok(buf);
891
892 fn map_caperr(err: capstone::Error) -> anyhow::Error {
893 anyhow::format_err!("{err}")
894 }
895 }
896
897 /// If available, return information about the code layout in the
898 /// final machine code: the offsets (in bytes) of each basic-block
899 /// start, and all basic-block edges.
900 pub fn get_code_bb_layout(&self) -> (Vec<usize>, Vec<(usize, usize)>) {
901 (
902 self.bb_starts.iter().map(|&off| off as usize).collect(),
903 self.bb_edges
904 .iter()
905 .map(|&(from, to)| (from as usize, to as usize))
906 .collect(),
907 )
908 }
909
910 /// Creates unwind information for the function.
911 ///
912 /// Returns `None` if the function has no unwind information.
913 #[cfg(feature = "unwind")]
914 pub fn create_unwind_info(
915 &self,
916 isa: &dyn crate::isa::TargetIsa,
917 ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
918 use crate::isa::unwind::UnwindInfoKind;
919 let unwind_info_kind = match isa.triple().operating_system {
920 target_lexicon::OperatingSystem::Windows => UnwindInfoKind::Windows,
921 _ => UnwindInfoKind::SystemV,
922 };
923 self.create_unwind_info_of_kind(isa, unwind_info_kind)
924 }
925
926 /// Creates unwind information for the function using the supplied
927 /// "kind". Supports cross-OS (but not cross-arch) generation.
928 ///
929 /// Returns `None` if the function has no unwind information.
930 #[cfg(feature = "unwind")]
931 pub fn create_unwind_info_of_kind(
932 &self,
933 isa: &dyn crate::isa::TargetIsa,
934 unwind_info_kind: crate::isa::unwind::UnwindInfoKind,
935 ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
936 isa.emit_unwind_info(self, unwind_info_kind)
937 }
938}
939
940/// An object that can be used to create the text section of an executable.
941///
942/// This primarily handles resolving relative relocations at
943/// text-section-assembly time rather than at load/link time. This
944/// architecture-specific logic is sort of like a linker, but only for one
945/// object file at a time.
946pub trait TextSectionBuilder {
947 /// Appends `data` to the text section with the `align` specified.
948 ///
949 /// If `labeled` is `true` then this also binds the appended data to the
950 /// `n`th label for how many times this has been called with `labeled:
951 /// true`. The label target can be passed as the `target` argument to
952 /// `resolve_reloc`.
953 ///
954 /// This function returns the offset at which the data was placed in the
955 /// text section.
956 fn append(
957 &mut self,
958 labeled: bool,
959 data: &[u8],
960 align: u32,
961 ctrl_plane: &mut ControlPlane,
962 ) -> u64;
963
964 /// Attempts to resolve a relocation for this function.
965 ///
966 /// The `offset` is the offset of the relocation, within the text section.
967 /// The `reloc` is the kind of relocation.
968 /// The `addend` is the value to add to the relocation.
969 /// The `target` is the labeled function that is the target of this
970 /// relocation.
971 ///
972 /// Labeled functions are created with the `append` function above by
973 /// setting the `labeled` parameter to `true`.
974 ///
975 /// If this builder does not know how to handle `reloc` then this function
976 /// will return `false`. Otherwise this function will return `true` and this
977 /// relocation will be resolved in the final bytes returned by `finish`.
978 fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: usize) -> bool;
979
980 /// A debug-only option which is used to for
981 fn force_veneers(&mut self);
982
983 /// Write the `data` provided at `offset`, for example when resolving a
984 /// relocation.
985 fn write(&mut self, offset: u64, data: &[u8]);
986
987 /// Completes this text section, filling out any final details, and returns
988 /// the bytes of the text section.
989 fn finish(&mut self, ctrl_plane: &mut ControlPlane) -> Vec<u8>;
990}