cranelift_codegen/isa/x64/
lower.rs

1//! Lowering rules for X64.
2
3// ISLE integration glue.
4pub(super) mod isle;
5
6use crate::ir::{
7    Endianness, ExternalName, Inst as IRInst, InstructionData, LibCall, Opcode, Type, types,
8};
9use crate::isa::x64::abi::*;
10use crate::isa::x64::inst::args::*;
11use crate::isa::x64::inst::*;
12use crate::isa::{CallConv, x64::X64Backend};
13use crate::machinst::*;
14use crate::result::CodegenResult;
15use crate::settings::Flags;
16use alloc::boxed::Box;
17use target_lexicon::Triple;
18
19/// Identifier for a particular input of an instruction.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21struct InsnInput {
22    insn: IRInst,
23    input: usize,
24}
25
26//=============================================================================
27// Helpers for instruction lowering.
28
29impl Lower<'_, Inst> {
30    #[inline]
31    pub fn temp_writable_gpr(&mut self) -> WritableGpr {
32        WritableGpr::from_writable_reg(self.alloc_tmp(types::I64).only_reg().unwrap()).unwrap()
33    }
34
35    #[inline]
36    pub fn temp_writable_xmm(&mut self) -> WritableXmm {
37        WritableXmm::from_writable_reg(self.alloc_tmp(types::F64).only_reg().unwrap()).unwrap()
38    }
39}
40
41fn is_int_or_ref_ty(ty: Type) -> bool {
42    match ty {
43        types::I8 | types::I16 | types::I32 | types::I64 => true,
44        _ => false,
45    }
46}
47
48/// Returns whether the given specified `input` is a result produced by an instruction with Opcode
49/// `op`.
50// TODO investigate failures with checking against the result index.
51fn matches_input(ctx: &mut Lower<Inst>, input: InsnInput, op: Opcode) -> Option<IRInst> {
52    let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
53    inputs.inst.as_inst().and_then(|(src_inst, _)| {
54        let data = ctx.data(src_inst);
55        if data.opcode() == op {
56            return Some(src_inst);
57        }
58        None
59    })
60}
61
62/// Put the given input into possibly multiple registers, and mark it as used (side-effect).
63fn put_input_in_regs(ctx: &mut Lower<Inst>, spec: InsnInput) -> ValueRegs<Reg> {
64    let ty = ctx.input_ty(spec.insn, spec.input);
65    let input = ctx.get_input_as_source_or_const(spec.insn, spec.input);
66
67    if let Some(c) = input.constant {
68        // Generate constants fresh at each use to minimize long-range register pressure.
69        let size = if ty_bits(ty) < 64 {
70            OperandSize::Size32
71        } else {
72            OperandSize::Size64
73        };
74        assert!(is_int_or_ref_ty(ty)); // Only used for addresses.
75        let cst_copy = ctx.alloc_tmp(ty);
76        ctx.emit(Inst::imm(size, c, cst_copy.only_reg().unwrap()));
77        non_writable_value_regs(cst_copy)
78    } else {
79        ctx.put_input_in_regs(spec.insn, spec.input)
80    }
81}
82
83/// Put the given input into a register, and mark it as used (side-effect).
84fn put_input_in_reg(ctx: &mut Lower<Inst>, spec: InsnInput) -> Reg {
85    put_input_in_regs(ctx, spec)
86        .only_reg()
87        .expect("Multi-register value not expected")
88}
89
90enum MergeableLoadSize {
91    /// The load size performed by a sinkable load merging operation is
92    /// precisely the size necessary for the type in question.
93    Exact,
94
95    /// Narrower-than-32-bit values are handled by ALU insts that are at least
96    /// 32 bits wide, which is normally OK as we ignore upper buts; but, if we
97    /// generate, e.g., a direct-from-memory 32-bit add for a byte value and
98    /// the byte is the last byte in a page, the extra data that we load is
99    /// incorrectly accessed. So we only allow loads to merge for
100    /// 32-bit-and-above widths.
101    Min32,
102}
103
104/// Determines whether a load operation (indicated by `src_insn`) can be merged
105/// into the current lowering point. If so, returns the address-base source (as
106/// an `InsnInput`) and an offset from that address from which to perform the
107/// load.
108fn is_mergeable_load(
109    ctx: &mut Lower<Inst>,
110    src_insn: IRInst,
111    size: MergeableLoadSize,
112) -> Option<(InsnInput, i32)> {
113    let insn_data = ctx.data(src_insn);
114    let inputs = ctx.num_inputs(src_insn);
115    if inputs != 1 {
116        return None;
117    }
118
119    // If this type is too small to get a merged load, don't merge the load.
120    let load_ty = ctx.output_ty(src_insn, 0);
121    if ty_bits(load_ty) < 32 {
122        match size {
123            MergeableLoadSize::Exact => {}
124            MergeableLoadSize::Min32 => return None,
125        }
126    }
127
128    // If the load's flags specify big-endian, we can't merge.
129    if let Some(flags) = ctx.memflags(src_insn) {
130        if flags.explicit_endianness() == Some(Endianness::Big) {
131            return None;
132        }
133    }
134
135    // Just testing the opcode is enough, because the width will always match if
136    // the type does (and the type should match if the CLIF is properly
137    // constructed).
138    if let &InstructionData::Load {
139        opcode: Opcode::Load,
140        offset,
141        ..
142    } = insn_data
143    {
144        Some((
145            InsnInput {
146                insn: src_insn,
147                input: 0,
148            },
149            offset.into(),
150        ))
151    } else {
152        None
153    }
154}
155
156fn input_to_imm(ctx: &mut Lower<Inst>, spec: InsnInput) -> Option<u64> {
157    ctx.get_input_as_source_or_const(spec.insn, spec.input)
158        .constant
159}
160
161fn emit_vm_call(
162    ctx: &mut Lower<Inst>,
163    flags: &Flags,
164    triple: &Triple,
165    libcall: LibCall,
166    inputs: &[ValueRegs<Reg>],
167) -> CodegenResult<InstOutput> {
168    let extname = ExternalName::LibCall(libcall);
169
170    // TODO avoid recreating signatures for every single Libcall function.
171    let call_conv = CallConv::for_libcall(flags, CallConv::triple_default(triple));
172    let sig = libcall.signature(call_conv, types::I64);
173    let outputs = ctx.gen_call_output(&sig);
174
175    if !ctx.sigs().have_abi_sig_for_signature(&sig) {
176        ctx.sigs_mut()
177            .make_abi_sig_from_ir_signature::<X64ABIMachineSpec>(sig.clone(), flags)?;
178    }
179    let sig = ctx.sigs().abi_sig_for_signature(&sig);
180
181    let uses = ctx.gen_call_args(sig, inputs);
182    let defs = ctx.gen_call_rets(sig, &outputs);
183
184    let stack_ret_space = ctx.sigs()[sig].sized_stack_ret_space();
185    let stack_arg_space = ctx.sigs()[sig].sized_stack_arg_space();
186    ctx.abi_mut()
187        .accumulate_outgoing_args_size(stack_ret_space + stack_arg_space);
188
189    if flags.use_colocated_libcalls() {
190        let call_info = ctx.gen_call_info(sig, extname, uses, defs, None, false);
191        ctx.emit(Inst::call_known(Box::new(call_info)));
192    } else {
193        let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
194        ctx.emit(Inst::LoadExtName {
195            dst: tmp.map(Gpr::unwrap_new),
196            name: Box::new(extname),
197            offset: 0,
198            distance: RelocDistance::Far,
199        });
200        let call_info = ctx.gen_call_info(sig, RegMem::reg(tmp.to_reg()), uses, defs, None, false);
201        ctx.emit(Inst::call_unknown(Box::new(call_info)));
202    }
203    Ok(outputs)
204}
205
206/// Returns whether the given input is a shift by a constant value less or equal than 3.
207/// The goal is to embed it within an address mode.
208fn matches_small_constant_shift(ctx: &mut Lower<Inst>, spec: InsnInput) -> Option<(InsnInput, u8)> {
209    matches_input(ctx, spec, Opcode::Ishl).and_then(|shift| {
210        match input_to_imm(
211            ctx,
212            InsnInput {
213                insn: shift,
214                input: 1,
215            },
216        ) {
217            Some(shift_amt) if shift_amt <= 3 => Some((
218                InsnInput {
219                    insn: shift,
220                    input: 0,
221                },
222                shift_amt as u8,
223            )),
224            _ => None,
225        }
226    })
227}
228
229/// Lowers an instruction to one of the x86 addressing modes.
230///
231/// Note: the 32-bit offset in Cranelift has to be sign-extended, which maps x86's behavior.
232fn lower_to_amode(ctx: &mut Lower<Inst>, spec: InsnInput, offset: i32) -> Amode {
233    let flags = ctx
234        .memflags(spec.insn)
235        .expect("Instruction with amode should have memflags");
236
237    // We now either have an add that we must materialize, or some other input; as well as the
238    // final offset.
239    if let Some(add) = matches_input(ctx, spec, Opcode::Iadd) {
240        let output_ty = ctx.output_ty(add, 0);
241        debug_assert_eq!(
242            output_ty,
243            types::I64,
244            "Address width of 64 expected, got {output_ty}"
245        );
246        let add_inputs = &[
247            InsnInput {
248                insn: add,
249                input: 0,
250            },
251            InsnInput {
252                insn: add,
253                input: 1,
254            },
255        ];
256
257        // TODO heap_addr legalization generates a uext64 *after* the shift, so these optimizations
258        // aren't happening in the wasm case. We could do better, given some range analysis.
259        let (base, index, shift) = if let Some((shift_input, shift_amt)) =
260            matches_small_constant_shift(ctx, add_inputs[0])
261        {
262            (
263                put_input_in_reg(ctx, add_inputs[1]),
264                put_input_in_reg(ctx, shift_input),
265                shift_amt,
266            )
267        } else if let Some((shift_input, shift_amt)) =
268            matches_small_constant_shift(ctx, add_inputs[1])
269        {
270            (
271                put_input_in_reg(ctx, add_inputs[0]),
272                put_input_in_reg(ctx, shift_input),
273                shift_amt,
274            )
275        } else {
276            for input in 0..=1 {
277                // Try to pierce through uextend.
278                let (inst, inst_input) = if let Some(uextend) =
279                    matches_input(ctx, InsnInput { insn: add, input }, Opcode::Uextend)
280                {
281                    (uextend, 0)
282                } else {
283                    (add, input)
284                };
285
286                // If it's a constant, add it directly!
287                if let Some(cst) = ctx.get_input_as_source_or_const(inst, inst_input).constant {
288                    let final_offset = (offset as i64).wrapping_add(cst as i64);
289                    if let Ok(final_offset) = i32::try_from(final_offset) {
290                        let base = put_input_in_reg(ctx, add_inputs[1 - input]);
291                        return Amode::imm_reg(final_offset, base).with_flags(flags.into());
292                    }
293                }
294            }
295
296            (
297                put_input_in_reg(ctx, add_inputs[0]),
298                put_input_in_reg(ctx, add_inputs[1]),
299                0,
300            )
301        };
302
303        return Amode::imm_reg_reg_shift(
304            offset,
305            Gpr::unwrap_new(base),
306            Gpr::unwrap_new(index),
307            shift,
308        )
309        .with_flags(flags.into());
310    }
311
312    let input = put_input_in_reg(ctx, spec);
313    Amode::imm_reg(offset, input).with_flags(flags.into())
314}
315
316//=============================================================================
317// Lowering-backend trait implementation.
318
319impl LowerBackend for X64Backend {
320    type MInst = Inst;
321
322    fn lower(&self, ctx: &mut Lower<Inst>, ir_inst: IRInst) -> Option<InstOutput> {
323        isle::lower(ctx, self, ir_inst)
324    }
325
326    fn lower_branch(
327        &self,
328        ctx: &mut Lower<Inst>,
329        ir_inst: IRInst,
330        targets: &[MachLabel],
331    ) -> Option<()> {
332        isle::lower_branch(ctx, self, ir_inst, targets)
333    }
334
335    fn maybe_pinned_reg(&self) -> Option<Reg> {
336        Some(regs::pinned_reg())
337    }
338}
cranelift_codegen/isa/x64/lower.rs

cranelift_codegen/isa/x64/
lower.rs
