cranelift_codegen/machinst/mod.rs
1//! This module exposes the machine-specific backend definition pieces.
2//!
3//! The MachInst infrastructure is the compiler backend, from CLIF
4//! (ir::Function) to machine code. The purpose of this infrastructure is, at a
5//! high level, to do instruction selection/lowering (to machine instructions),
6//! register allocation, and then perform all the fixups to branches, constant
7//! data references, etc., needed to actually generate machine code.
8//!
9//! The container for machine instructions, at various stages of construction,
10//! is the `VCode` struct. We refer to a sequence of machine instructions organized
11//! into basic blocks as "vcode". This is short for "virtual-register code".
12//!
13//! The compilation pipeline, from an `ir::Function` (already optimized as much as
14//! you like by machine-independent optimization passes) onward, is as follows.
15//!
16//! ```plain
17//!
18//! ir::Function (SSA IR, machine-independent opcodes)
19//! |
20//! | [lower]
21//! |
22//! VCode<arch_backend::Inst> (machine instructions:
23//! | - mostly virtual registers.
24//! | - cond branches in two-target form.
25//! | - branch targets are block indices.
26//! | - in-memory constants held by insns,
27//! | with unknown offsets.
28//! | - critical edges (actually all edges)
29//! | are split.)
30//! |
31//! | [regalloc --> `regalloc2::Output`; VCode is unchanged]
32//! |
33//! | [binary emission via MachBuffer]
34//! |
35//! Vec<u8> (machine code:
36//! | - two-dest branches resolved via
37//! | streaming branch resolution/simplification.
38//! | - regalloc `Allocation` results used directly
39//! | by instruction emission code.
40//! | - prologue and epilogue(s) built and emitted
41//! | directly during emission.
42//! | - SP-relative offsets resolved by tracking
43//! | EmitState.)
44//!
45//! ```
46
47use crate::binemit::{Addend, CodeInfo, CodeOffset, Reloc};
48use crate::ir::{
49 self, DynamicStackSlot, RelSourceLoc, StackSlot, Type, function::FunctionParameters,
50};
51use crate::isa::FunctionAlignment;
52use crate::result::CodegenResult;
53use crate::settings;
54use crate::settings::Flags;
55use crate::value_label::ValueLabelsRanges;
56use alloc::vec::Vec;
57use core::fmt::Debug;
58use cranelift_control::ControlPlane;
59use cranelift_entity::PrimaryMap;
60use regalloc2::VReg;
61use smallvec::{SmallVec, smallvec};
62use std::string::String;
63
64#[cfg(feature = "enable-serde")]
65use serde_derive::{Deserialize, Serialize};
66
67#[macro_use]
68pub mod isle;
69
70pub mod lower;
71pub use lower::*;
72pub mod vcode;
73pub use vcode::*;
74pub mod compile;
75pub use compile::*;
76pub mod blockorder;
77pub use blockorder::*;
78pub mod abi;
79pub use abi::*;
80pub mod buffer;
81pub use buffer::*;
82pub mod helpers;
83pub use helpers::*;
84pub mod valueregs;
85pub use reg::*;
86pub use valueregs::*;
87pub mod pcc;
88pub mod reg;
89
90/// A machine instruction.
91pub trait MachInst: Clone + Debug {
92 /// The ABI machine spec for this `MachInst`.
93 type ABIMachineSpec: ABIMachineSpec<I = Self>;
94
95 /// Return the registers referenced by this machine instruction along with
96 /// the modes of reference (use, def, modify).
97 fn get_operands(&mut self, collector: &mut impl OperandVisitor);
98
99 /// If this is a simple move, return the (source, destination) tuple of registers.
100 fn is_move(&self) -> Option<(Writable<Reg>, Reg)>;
101
102 /// Is this a terminator (branch or ret)? If so, return its type
103 /// (ret/uncond/cond) and target if applicable.
104 fn is_term(&self) -> MachTerminator;
105
106 /// Is this an unconditional trap?
107 fn is_trap(&self) -> bool;
108
109 /// Is this an "args" pseudoinst?
110 fn is_args(&self) -> bool;
111
112 /// Classify the type of call instruction this is.
113 ///
114 /// This enables more granular function type analysis and optimization.
115 /// Returns `CallType::None` for non-call instructions, `CallType::Regular`
116 /// for normal calls that return to the caller, and `CallType::TailCall`
117 /// for tail calls that don't return to the caller.
118 fn call_type(&self) -> CallType;
119
120 /// Should this instruction's clobber-list be included in the
121 /// clobber-set?
122 fn is_included_in_clobbers(&self) -> bool;
123
124 /// Does this instruction access memory?
125 fn is_mem_access(&self) -> bool;
126
127 /// Generate a move.
128 fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
129
130 /// Generate a dummy instruction that will keep a value alive but
131 /// has no other purpose.
132 fn gen_dummy_use(reg: Reg) -> Self;
133
134 /// Determine register class(es) to store the given Cranelift type, and the
135 /// Cranelift type actually stored in the underlying register(s). May return
136 /// an error if the type isn't supported by this backend.
137 ///
138 /// If the type requires multiple registers, then the list of registers is
139 /// returned in little-endian order.
140 ///
141 /// Note that the type actually stored in the register(s) may differ in the
142 /// case that a value is split across registers: for example, on a 32-bit
143 /// target, an I64 may be stored in two registers, each of which holds an
144 /// I32. The actually-stored types are used only to inform the backend when
145 /// generating spills and reloads for individual registers.
146 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])>;
147
148 /// Get an appropriate type that can fully hold a value in a given
149 /// register class. This may not be the only type that maps to
150 /// that class, but when used with `gen_move()` or the ABI trait's
151 /// load/spill constructors, it should produce instruction(s) that
152 /// move the entire register contents.
153 fn canonical_type_for_rc(rc: RegClass) -> Type;
154
155 /// Generate a jump to another target. Used during lowering of
156 /// control flow.
157 fn gen_jump(target: MachLabel) -> Self;
158
159 /// Generate a store of an immediate 64-bit integer to a register. Used by
160 /// the control plane to generate random instructions.
161 fn gen_imm_u64(_value: u64, _dst: Writable<Reg>) -> Option<Self> {
162 None
163 }
164
165 /// Generate a store of an immediate 64-bit integer to a register. Used by
166 /// the control plane to generate random instructions. The tmp register may
167 /// be used by architectures which don't support writing immediate values to
168 /// floating point registers directly.
169 fn gen_imm_f64(_value: f64, _tmp: Writable<Reg>, _dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
170 SmallVec::new()
171 }
172
173 /// Generate a NOP. The `preferred_size` parameter allows the caller to
174 /// request a NOP of that size, or as close to it as possible. The machine
175 /// backend may return a NOP whose binary encoding is smaller than the
176 /// preferred size, but must not return a NOP that is larger. However,
177 /// the instruction must have a nonzero size if preferred_size is nonzero.
178 fn gen_nop(preferred_size: usize) -> Self;
179
180 /// Align a basic block offset (from start of function). By default, no
181 /// alignment occurs.
182 fn align_basic_block(offset: CodeOffset) -> CodeOffset {
183 offset
184 }
185
186 /// What is the worst-case instruction size emitted by this instruction type?
187 fn worst_case_size() -> CodeOffset;
188
189 /// What is the register class used for reference types (GC-observable pointers)? Can
190 /// be dependent on compilation flags.
191 fn ref_type_regclass(_flags: &Flags) -> RegClass;
192
193 /// Is this a safepoint?
194 fn is_safepoint(&self) -> bool;
195
196 /// Generate an instruction that must appear at the beginning of a basic
197 /// block, if any. Note that the return value must not be subject to
198 /// register allocation.
199 fn gen_block_start(
200 _is_indirect_branch_target: bool,
201 _is_forward_edge_cfi_enabled: bool,
202 ) -> Option<Self> {
203 None
204 }
205
206 /// Returns a description of the alignment required for functions for this
207 /// architecture.
208 fn function_alignment() -> FunctionAlignment;
209
210 /// Is this a low-level, one-way branch, not meant for use in a
211 /// VCode body? These instructions are meant to be used only when
212 /// directly emitted, i.e. when `MachInst` is used as an assembler
213 /// library.
214 fn is_low_level_branch(&self) -> bool {
215 false
216 }
217
218 /// A label-use kind: a type that describes the types of label references that
219 /// can occur in an instruction.
220 type LabelUse: MachInstLabelUse;
221
222 /// Byte representation of a trap opcode which is inserted by `MachBuffer`
223 /// during its `defer_trap` method.
224 const TRAP_OPCODE: &'static [u8];
225}
226
227/// A descriptor of a label reference (use) in an instruction set.
228pub trait MachInstLabelUse: Clone + Copy + Debug + Eq {
229 /// Required alignment for any veneer. Usually the required instruction
230 /// alignment (e.g., 4 for a RISC with 32-bit instructions, or 1 for x86).
231 const ALIGN: CodeOffset;
232
233 /// What is the maximum PC-relative range (positive)? E.g., if `1024`, a
234 /// label-reference fixup at offset `x` is valid if the label resolves to `x
235 /// + 1024`.
236 fn max_pos_range(self) -> CodeOffset;
237 /// What is the maximum PC-relative range (negative)? This is the absolute
238 /// value; i.e., if `1024`, then a label-reference fixup at offset `x` is
239 /// valid if the label resolves to `x - 1024`.
240 fn max_neg_range(self) -> CodeOffset;
241 /// What is the size of code-buffer slice this label-use needs to patch in
242 /// the label's value?
243 fn patch_size(self) -> CodeOffset;
244 /// Perform a code-patch, given the offset into the buffer of this label use
245 /// and the offset into the buffer of the label's definition.
246 /// It is guaranteed that, given `delta = offset - label_offset`, we will
247 /// have `offset >= -self.max_neg_range()` and `offset <=
248 /// self.max_pos_range()`.
249 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset);
250 /// Can the label-use be patched to a veneer that supports a longer range?
251 /// Usually valid for jumps (a short-range jump can jump to a longer-range
252 /// jump), but not for e.g. constant pool references, because the constant
253 /// load would require different code (one more level of indirection).
254 fn supports_veneer(self) -> bool;
255 /// How many bytes are needed for a veneer?
256 fn veneer_size(self) -> CodeOffset;
257 /// What's the largest possible veneer that may be generated?
258 fn worst_case_veneer_size() -> CodeOffset;
259 /// Generate a veneer. The given code-buffer slice is `self.veneer_size()`
260 /// bytes long at offset `veneer_offset` in the buffer. The original
261 /// label-use will be patched to refer to this veneer's offset. A new
262 /// (offset, LabelUse) is returned that allows the veneer to use the actual
263 /// label. For veneers to work properly, it is expected that the new veneer
264 /// has a larger range; on most platforms this probably means either a
265 /// "long-range jump" (e.g., on ARM, the 26-bit form), or if already at that
266 /// stage, a jump that supports a full 32-bit range, for example.
267 fn generate_veneer(self, buffer: &mut [u8], veneer_offset: CodeOffset) -> (CodeOffset, Self);
268
269 /// Returns the corresponding label-use for the relocation specified.
270 ///
271 /// This returns `None` if the relocation doesn't have a corresponding
272 /// representation for the target architecture.
273 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self>;
274}
275
276/// Classification of call instruction types for granular analysis.
277#[derive(Clone, Copy, Debug, PartialEq, Eq)]
278pub enum CallType {
279 /// Not a call instruction.
280 None,
281 /// Regular call that returns to the caller.
282 Regular,
283 /// Tail call that doesn't return to the caller.
284 TailCall,
285}
286
287/// Function classification based on call patterns.
288///
289/// This enum classifies functions based on their calling behavior to enable
290/// targeted optimizations. Functions are categorized as:
291/// - `None`: No calls at all (can use simplified calling conventions)
292/// - `TailOnly`: Only tail calls (may skip frame setup in some cases)
293/// - `Regular`: Has regular calls (requires full calling convention support)
294#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
295pub enum FunctionCalls {
296 /// Function makes no calls at all.
297 #[default]
298 None,
299 /// Function only makes tail calls (no regular calls).
300 TailOnly,
301 /// Function makes at least one regular call (may also have tail calls).
302 Regular,
303}
304
305impl FunctionCalls {
306 /// Update the function classification based on a new call instruction.
307 ///
308 /// This method implements the merge logic for accumulating call patterns:
309 /// - Any regular call makes the function Regular
310 /// - Tail calls upgrade None to TailOnly
311 /// - Regular always stays Regular
312 pub fn update(&mut self, call_type: CallType) {
313 *self = match (*self, call_type) {
314 // No call instruction - state unchanged
315 (current, CallType::None) => current,
316 // Regular call always results in Regular classification
317 (_, CallType::Regular) => FunctionCalls::Regular,
318 // Tail call: None becomes TailOnly, others unchanged
319 (FunctionCalls::None, CallType::TailCall) => FunctionCalls::TailOnly,
320 (current, CallType::TailCall) => current,
321 };
322 }
323}
324
325/// Describes a block terminator (not call) in the VCode.
326///
327/// Actual targets are not included: the single-source-of-truth for
328/// those is the VCode itself, which holds, for each block, successors
329/// and outgoing branch args per successor.
330#[derive(Clone, Debug, PartialEq, Eq)]
331pub enum MachTerminator {
332 /// Not a terminator.
333 None,
334 /// A return instruction.
335 Ret,
336 /// A tail call.
337 RetCall,
338 /// A branch.
339 Branch,
340}
341
342/// A trait describing the ability to encode a MachInst into binary machine code.
343pub trait MachInstEmit: MachInst {
344 /// Persistent state carried across `emit` invocations.
345 type State: MachInstEmitState<Self>;
346
347 /// Constant information used in `emit` invocations.
348 type Info;
349
350 /// Emit the instruction.
351 fn emit(&self, code: &mut MachBuffer<Self>, info: &Self::Info, state: &mut Self::State);
352
353 /// Pretty-print the instruction.
354 fn pretty_print_inst(&self, state: &mut Self::State) -> String;
355}
356
357/// A trait describing the emission state carried between MachInsts when
358/// emitting a function body.
359pub trait MachInstEmitState<I: VCodeInst>: Default + Clone + Debug {
360 /// Create a new emission state given the ABI object.
361 fn new(abi: &Callee<I::ABIMachineSpec>, ctrl_plane: ControlPlane) -> Self;
362
363 /// Update the emission state before emitting an instruction that is a
364 /// safepoint.
365 fn pre_safepoint(&mut self, user_stack_map: Option<ir::UserStackMap>);
366
367 /// The emission state holds ownership of a control plane, so it doesn't
368 /// have to be passed around explicitly too much. `ctrl_plane_mut` may
369 /// be used if temporary access to the control plane is needed by some
370 /// other function that doesn't have access to the emission state.
371 fn ctrl_plane_mut(&mut self) -> &mut ControlPlane;
372
373 /// Used to continue using a control plane after the emission state is
374 /// not needed anymore.
375 fn take_ctrl_plane(self) -> ControlPlane;
376
377 /// A hook that triggers when first emitting a new block.
378 /// It is guaranteed to be called before any instructions are emitted.
379 fn on_new_block(&mut self) {}
380
381 /// The [`FrameLayout`] for the function currently being compiled.
382 fn frame_layout(&self) -> &FrameLayout;
383}
384
385/// The result of a `MachBackend::compile_function()` call. Contains machine
386/// code (as bytes) and a disassembly, if requested.
387#[derive(PartialEq, Debug, Clone)]
388#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
389pub struct CompiledCodeBase<T: CompilePhase> {
390 /// Machine code.
391 pub buffer: MachBufferFinalized<T>,
392 /// Disassembly, if requested.
393 pub vcode: Option<String>,
394 /// Debug info: value labels to registers/stackslots at code offsets.
395 pub value_labels_ranges: ValueLabelsRanges,
396 /// Basic-block layout info: block start offsets.
397 ///
398 /// This info is generated only if the `machine_code_cfg_info`
399 /// flag is set.
400 pub bb_starts: Vec<CodeOffset>,
401 /// Basic-block layout info: block edges. Each edge is `(from,
402 /// to)`, where `from` and `to` are basic-block start offsets of
403 /// the respective blocks.
404 ///
405 /// This info is generated only if the `machine_code_cfg_info`
406 /// flag is set.
407 pub bb_edges: Vec<(CodeOffset, CodeOffset)>,
408}
409
410impl CompiledCodeStencil {
411 /// Apply function parameters to finalize a stencil into its final form.
412 pub fn apply_params(self, params: &FunctionParameters) -> CompiledCode {
413 CompiledCode {
414 buffer: self.buffer.apply_base_srcloc(params.base_srcloc()),
415 vcode: self.vcode,
416 value_labels_ranges: self.value_labels_ranges,
417 bb_starts: self.bb_starts,
418 bb_edges: self.bb_edges,
419 }
420 }
421}
422
423impl<T: CompilePhase> CompiledCodeBase<T> {
424 /// Get a `CodeInfo` describing section sizes from this compilation result.
425 pub fn code_info(&self) -> CodeInfo {
426 CodeInfo {
427 total_size: self.buffer.total_size(),
428 }
429 }
430
431 /// Returns a reference to the machine code generated for this function compilation.
432 pub fn code_buffer(&self) -> &[u8] {
433 self.buffer.data()
434 }
435
436 /// Get the disassembly of the buffer, using the given capstone context.
437 #[cfg(feature = "disas")]
438 pub fn disassemble(
439 &self,
440 params: Option<&crate::ir::function::FunctionParameters>,
441 cs: &capstone::Capstone,
442 ) -> Result<String, anyhow::Error> {
443 use std::fmt::Write;
444
445 let mut buf = String::new();
446
447 let relocs = self.buffer.relocs();
448 let traps = self.buffer.traps();
449
450 // Normalize the block starts to include an initial block of offset 0.
451 let mut block_starts = Vec::new();
452 if self.bb_starts.first().copied() != Some(0) {
453 block_starts.push(0);
454 }
455 block_starts.extend_from_slice(&self.bb_starts);
456 block_starts.push(self.buffer.data().len() as u32);
457
458 // Iterate over block regions, to ensure that we always produce block labels
459 for (n, (&start, &end)) in block_starts
460 .iter()
461 .zip(block_starts.iter().skip(1))
462 .enumerate()
463 {
464 writeln!(buf, "block{n}: ; offset 0x{start:x}")?;
465
466 let buffer = &self.buffer.data()[start as usize..end as usize];
467 let insns = cs.disasm_all(buffer, start as u64).map_err(map_caperr)?;
468 for i in insns.iter() {
469 write!(buf, " ")?;
470
471 let op_str = i.op_str().unwrap_or("");
472 if let Some(s) = i.mnemonic() {
473 write!(buf, "{s}")?;
474 if !op_str.is_empty() {
475 write!(buf, " ")?;
476 }
477 }
478
479 write!(buf, "{op_str}")?;
480
481 let end = i.address() + i.bytes().len() as u64;
482 let contains = |off| i.address() <= off && off < end;
483
484 for reloc in relocs.iter().filter(|reloc| contains(reloc.offset as u64)) {
485 write!(
486 buf,
487 " ; reloc_external {} {} {}",
488 reloc.kind,
489 reloc.target.display(params),
490 reloc.addend,
491 )?;
492 }
493
494 if let Some(trap) = traps.iter().find(|trap| contains(trap.offset as u64)) {
495 write!(buf, " ; trap: {}", trap.code)?;
496 }
497
498 writeln!(buf)?;
499 }
500 }
501
502 return Ok(buf);
503
504 fn map_caperr(err: capstone::Error) -> anyhow::Error {
505 anyhow::format_err!("{}", err)
506 }
507 }
508}
509
510/// Result of compiling a `FunctionStencil`, before applying `FunctionParameters` onto it.
511///
512/// Only used internally, in a transient manner, for the incremental compilation cache.
513pub type CompiledCodeStencil = CompiledCodeBase<Stencil>;
514
515/// `CompiledCode` in its final form (i.e. after `FunctionParameters` have been applied), ready for
516/// consumption.
517pub type CompiledCode = CompiledCodeBase<Final>;
518
519impl CompiledCode {
520 /// If available, return information about the code layout in the
521 /// final machine code: the offsets (in bytes) of each basic-block
522 /// start, and all basic-block edges.
523 pub fn get_code_bb_layout(&self) -> (Vec<usize>, Vec<(usize, usize)>) {
524 (
525 self.bb_starts.iter().map(|&off| off as usize).collect(),
526 self.bb_edges
527 .iter()
528 .map(|&(from, to)| (from as usize, to as usize))
529 .collect(),
530 )
531 }
532
533 /// Creates unwind information for the function.
534 ///
535 /// Returns `None` if the function has no unwind information.
536 #[cfg(feature = "unwind")]
537 pub fn create_unwind_info(
538 &self,
539 isa: &dyn crate::isa::TargetIsa,
540 ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
541 use crate::isa::unwind::UnwindInfoKind;
542 let unwind_info_kind = match isa.triple().operating_system {
543 target_lexicon::OperatingSystem::Windows => UnwindInfoKind::Windows,
544 _ => UnwindInfoKind::SystemV,
545 };
546 self.create_unwind_info_of_kind(isa, unwind_info_kind)
547 }
548
549 /// Creates unwind information for the function using the supplied
550 /// "kind". Supports cross-OS (but not cross-arch) generation.
551 ///
552 /// Returns `None` if the function has no unwind information.
553 #[cfg(feature = "unwind")]
554 pub fn create_unwind_info_of_kind(
555 &self,
556 isa: &dyn crate::isa::TargetIsa,
557 unwind_info_kind: crate::isa::unwind::UnwindInfoKind,
558 ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
559 isa.emit_unwind_info(self, unwind_info_kind)
560 }
561}
562
563/// An object that can be used to create the text section of an executable.
564///
565/// This primarily handles resolving relative relocations at
566/// text-section-assembly time rather than at load/link time. This
567/// architecture-specific logic is sort of like a linker, but only for one
568/// object file at a time.
569pub trait TextSectionBuilder {
570 /// Appends `data` to the text section with the `align` specified.
571 ///
572 /// If `labeled` is `true` then this also binds the appended data to the
573 /// `n`th label for how many times this has been called with `labeled:
574 /// true`. The label target can be passed as the `target` argument to
575 /// `resolve_reloc`.
576 ///
577 /// This function returns the offset at which the data was placed in the
578 /// text section.
579 fn append(
580 &mut self,
581 labeled: bool,
582 data: &[u8],
583 align: u32,
584 ctrl_plane: &mut ControlPlane,
585 ) -> u64;
586
587 /// Attempts to resolve a relocation for this function.
588 ///
589 /// The `offset` is the offset of the relocation, within the text section.
590 /// The `reloc` is the kind of relocation.
591 /// The `addend` is the value to add to the relocation.
592 /// The `target` is the labeled function that is the target of this
593 /// relocation.
594 ///
595 /// Labeled functions are created with the `append` function above by
596 /// setting the `labeled` parameter to `true`.
597 ///
598 /// If this builder does not know how to handle `reloc` then this function
599 /// will return `false`. Otherwise this function will return `true` and this
600 /// relocation will be resolved in the final bytes returned by `finish`.
601 fn resolve_reloc(&mut self, offset: u64, reloc: Reloc, addend: Addend, target: usize) -> bool;
602
603 /// A debug-only option which is used to for
604 fn force_veneers(&mut self);
605
606 /// Write the `data` provided at `offset`, for example when resolving a
607 /// relocation.
608 fn write(&mut self, offset: u64, data: &[u8]);
609
610 /// Completes this text section, filling out any final details, and returns
611 /// the bytes of the text section.
612 fn finish(&mut self, ctrl_plane: &mut ControlPlane) -> Vec<u8>;
613}