1use crate::CodegenError;
21use crate::ir::pcc::*;
22use crate::ir::{self, Constant, ConstantData, ValueLabel, types};
23use crate::ranges::Ranges;
24use crate::timing;
25use crate::trace;
26use crate::{LabelValueLoc, ValueLocRange};
27use crate::{machinst::*, trace_log_enabled};
28use regalloc2::{
29 Edit, Function as RegallocFunction, InstOrEdit, InstPosition, InstRange, Operand,
30 OperandConstraint, OperandKind, PRegSet, ProgPoint, RegClass,
31};
32use rustc_hash::FxHashMap;
33
34use core::cmp::Ordering;
35use core::fmt::{self, Write};
36use core::mem::take;
37use cranelift_entity::{Keys, entity_impl};
38use std::collections::HashMap;
39use std::collections::hash_map::Entry;
40
41pub type InsnIndex = regalloc2::Inst;
43
44trait ToBackwardsInsnIndex {
47 fn to_backwards_insn_index(&self, num_insts: usize) -> BackwardsInsnIndex;
48}
49
50impl ToBackwardsInsnIndex for InsnIndex {
51 fn to_backwards_insn_index(&self, num_insts: usize) -> BackwardsInsnIndex {
52 BackwardsInsnIndex::new(num_insts - self.index() - 1)
53 }
54}
55
56#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
59#[cfg_attr(
60 feature = "enable-serde",
61 derive(::serde::Serialize, ::serde::Deserialize)
62)]
63pub struct BackwardsInsnIndex(InsnIndex);
64
65impl BackwardsInsnIndex {
66 pub fn new(i: usize) -> Self {
67 BackwardsInsnIndex(InsnIndex::new(i))
68 }
69}
70
71pub type BlockIndex = regalloc2::Block;
73
74pub trait VCodeInst: MachInst + MachInstEmit {}
77impl<I: MachInst + MachInstEmit> VCodeInst for I {}
78
79pub struct VCode<I: VCodeInst> {
92 vreg_types: Vec<Type>,
94
95 insts: Vec<I>,
97
98 user_stack_maps: FxHashMap<BackwardsInsnIndex, ir::UserStackMap>,
105
106 operands: Vec<Operand>,
111
112 operand_ranges: Ranges,
116
117 clobbers: FxHashMap<InsnIndex, PRegSet>,
119
120 srclocs: Vec<RelSourceLoc>,
123
124 entry: BlockIndex,
126
127 block_ranges: Ranges,
129
130 block_succ_range: Ranges,
132
133 block_succs: Vec<regalloc2::Block>,
138
139 block_pred_range: Ranges,
141
142 block_preds: Vec<regalloc2::Block>,
147
148 block_params_range: Ranges,
150
151 block_params: Vec<regalloc2::VReg>,
156
157 branch_block_args: Vec<regalloc2::VReg>,
167
168 branch_block_arg_range: Ranges,
174
175 branch_block_arg_succ_range: Ranges,
178
179 block_order: BlockLoweringOrder,
181
182 pub(crate) abi: Callee<I::ABIMachineSpec>,
184
185 emit_info: I::Info,
188
189 pub(crate) constants: VCodeConstants,
191
192 debug_value_labels: Vec<(VReg, InsnIndex, InsnIndex, u32)>,
194
195 pub(crate) sigs: SigSet,
196
197 facts: Vec<Option<Fact>>,
199
200 log2_min_function_alignment: u8,
201}
202
203pub struct EmitResult {
207 pub buffer: MachBufferFinalized<Stencil>,
209
210 pub bb_offsets: Vec<CodeOffset>,
213
214 pub bb_edges: Vec<(CodeOffset, CodeOffset)>,
217
218 pub disasm: Option<String>,
223
224 pub sized_stackslot_offsets: PrimaryMap<StackSlot, u32>,
226
227 pub dynamic_stackslot_offsets: PrimaryMap<DynamicStackSlot, u32>,
229
230 pub value_labels_ranges: ValueLabelsRanges,
232
233 pub frame_size: u32,
235}
236
237pub struct VCodeBuilder<I: VCodeInst> {
254 pub(crate) vcode: VCode<I>,
256
257 direction: VCodeBuildDirection,
259
260 debug_info: FxHashMap<ValueLabel, Vec<(InsnIndex, InsnIndex, VReg)>>,
264}
265
266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
268pub enum VCodeBuildDirection {
269 Backward,
273}
274
275impl<I: VCodeInst> VCodeBuilder<I> {
276 pub fn new(
278 sigs: SigSet,
279 abi: Callee<I::ABIMachineSpec>,
280 emit_info: I::Info,
281 block_order: BlockLoweringOrder,
282 constants: VCodeConstants,
283 direction: VCodeBuildDirection,
284 log2_min_function_alignment: u8,
285 ) -> Self {
286 let vcode = VCode::new(
287 sigs,
288 abi,
289 emit_info,
290 block_order,
291 constants,
292 log2_min_function_alignment,
293 );
294
295 VCodeBuilder {
296 vcode,
297 direction,
298 debug_info: FxHashMap::default(),
299 }
300 }
301
302 pub fn init_retval_area(&mut self, vregs: &mut VRegAllocator<I>) -> CodegenResult<()> {
303 self.vcode.abi.init_retval_area(&self.vcode.sigs, vregs)
304 }
305
306 pub fn abi(&self) -> &Callee<I::ABIMachineSpec> {
308 &self.vcode.abi
309 }
310
311 pub fn abi_mut(&mut self) -> &mut Callee<I::ABIMachineSpec> {
313 &mut self.vcode.abi
314 }
315
316 pub fn sigs(&self) -> &SigSet {
317 &self.vcode.sigs
318 }
319
320 pub fn sigs_mut(&mut self) -> &mut SigSet {
321 &mut self.vcode.sigs
322 }
323
324 pub fn block_order(&self) -> &BlockLoweringOrder {
326 &self.vcode.block_order
327 }
328
329 pub fn set_entry(&mut self, block: BlockIndex) {
331 self.vcode.entry = block;
332 }
333
334 pub fn end_bb(&mut self) {
337 let end_idx = self.vcode.insts.len();
338 self.vcode.block_ranges.push_end(end_idx);
340 let succ_end = self.vcode.block_succs.len();
342 self.vcode.block_succ_range.push_end(succ_end);
343 let block_params_end = self.vcode.block_params.len();
345 self.vcode.block_params_range.push_end(block_params_end);
346 let branch_block_arg_succ_end = self.vcode.branch_block_arg_range.len();
348 self.vcode
349 .branch_block_arg_succ_range
350 .push_end(branch_block_arg_succ_end);
351 }
352
353 pub fn add_block_param(&mut self, param: VirtualReg) {
354 self.vcode.block_params.push(param.into());
355 }
356
357 fn add_branch_args_for_succ(&mut self, args: &[Reg]) {
358 self.vcode
359 .branch_block_args
360 .extend(args.iter().map(|&arg| VReg::from(arg)));
361 let end = self.vcode.branch_block_args.len();
362 self.vcode.branch_block_arg_range.push_end(end);
363 }
364
365 pub fn push(&mut self, insn: I, loc: RelSourceLoc) {
368 assert!(!insn.is_low_level_branch()); self.vcode.insts.push(insn);
370 self.vcode.srclocs.push(loc);
371 }
372
373 pub fn add_succ(&mut self, block: BlockIndex, args: &[Reg]) {
375 self.vcode.block_succs.push(block);
376 self.add_branch_args_for_succ(args);
377 }
378
379 pub fn add_value_label(&mut self, reg: Reg, label: ValueLabel) {
381 let next_inst_index = self.vcode.insts.len();
401 if next_inst_index == 0 {
402 return;
404 }
405 let next_inst = InsnIndex::new(next_inst_index);
406 let labels = self.debug_info.entry(label).or_insert_with(|| vec![]);
407 let last = labels
408 .last()
409 .map(|(_start, end, _vreg)| *end)
410 .unwrap_or(InsnIndex::new(0));
411 labels.push((last, next_inst, reg.into()));
412 }
413
414 pub fn constants(&mut self) -> &mut VCodeConstants {
416 &mut self.vcode.constants
417 }
418
419 fn compute_preds_from_succs(&mut self) {
420 let mut starts = vec![0u32; self.vcode.num_blocks()];
423 for succ in &self.vcode.block_succs {
424 starts[succ.index()] += 1;
425 }
426
427 self.vcode.block_pred_range.reserve(starts.len());
431 let mut end = 0;
432 for count in starts.iter_mut() {
433 let start = end;
434 end += *count;
435 *count = start;
436 self.vcode.block_pred_range.push_end(end as usize);
437 }
438 let end = end as usize;
439 debug_assert_eq!(end, self.vcode.block_succs.len());
440
441 self.vcode.block_preds.resize(end, BlockIndex::invalid());
447 for (pred, range) in self.vcode.block_succ_range.iter() {
448 let pred = BlockIndex::new(pred);
449 for succ in &self.vcode.block_succs[range] {
450 let pos = &mut starts[succ.index()];
451 self.vcode.block_preds[*pos as usize] = pred;
452 *pos += 1;
453 }
454 }
455 debug_assert!(self.vcode.block_preds.iter().all(|pred| pred.is_valid()));
456 }
457
458 fn reverse_and_finalize(&mut self, vregs: &VRegAllocator<I>) {
462 let n_insts = self.vcode.insts.len();
463 if n_insts == 0 {
464 return;
465 }
466
467 self.vcode.block_ranges.reverse_index();
469 self.vcode.block_ranges.reverse_target(n_insts);
470 self.vcode.block_params_range.reverse_index();
476 self.vcode.block_succ_range.reverse_index();
479 self.vcode.insts.reverse();
480 self.vcode.srclocs.reverse();
481 self.vcode.branch_block_arg_succ_range.reverse_index();
484
485 let translate = |inst: InsnIndex| InsnIndex::new(n_insts - inst.index());
496
497 for (label, tuples) in &self.debug_info {
499 for &(start, end, vreg) in tuples {
500 let vreg = vregs.resolve_vreg_alias(vreg);
501 let fwd_start = translate(end);
502 let fwd_end = translate(start);
503 self.vcode
504 .debug_value_labels
505 .push((vreg, fwd_start, fwd_end, label.as_u32()));
506 }
507 }
508
509 self.vcode
512 .debug_value_labels
513 .sort_unstable_by_key(|(vreg, _, _, _)| *vreg);
514 }
515
516 fn collect_operands(&mut self, vregs: &VRegAllocator<I>) {
517 let allocatable = PRegSet::from(self.vcode.abi.machine_env());
518 for (i, insn) in self.vcode.insts.iter_mut().enumerate() {
519 let mut op_collector =
531 OperandCollector::new(&mut self.vcode.operands, allocatable, |vreg| {
532 vregs.resolve_vreg_alias(vreg)
533 });
534 insn.get_operands(&mut op_collector);
535 let (ops, clobbers) = op_collector.finish();
536 self.vcode.operand_ranges.push_end(ops);
537
538 if clobbers != PRegSet::default() {
539 self.vcode.clobbers.insert(InsnIndex::new(i), clobbers);
540 }
541
542 if let Some((dst, src)) = insn.is_move() {
543 assert!(
546 src.is_virtual(),
547 "the real register {src:?} was used as the source of a move instruction"
548 );
549 assert!(
550 dst.to_reg().is_virtual(),
551 "the real register {:?} was used as the destination of a move instruction",
552 dst.to_reg()
553 );
554 }
555 }
556
557 for arg in &mut self.vcode.branch_block_args {
559 let new_arg = vregs.resolve_vreg_alias(*arg);
560 trace!("operandcollector: block arg {:?} -> {:?}", arg, new_arg);
561 *arg = new_arg;
562 }
563 }
564
565 pub fn build(mut self, mut vregs: VRegAllocator<I>) -> VCode<I> {
567 self.vcode.vreg_types = take(&mut vregs.vreg_types);
568 self.vcode.facts = take(&mut vregs.facts);
569
570 if self.direction == VCodeBuildDirection::Backward {
571 self.reverse_and_finalize(&vregs);
572 }
573 self.collect_operands(&vregs);
574
575 self.compute_preds_from_succs();
576 self.vcode.debug_value_labels.sort_unstable();
577
578 vregs.debug_assert_no_vreg_aliases(self.vcode.operands.iter().map(|op| op.vreg()));
585 vregs.debug_assert_no_vreg_aliases(self.vcode.block_params.iter().copied());
587 vregs.debug_assert_no_vreg_aliases(self.vcode.branch_block_args.iter().copied());
589 vregs.debug_assert_no_vreg_aliases(
591 self.vcode.debug_value_labels.iter().map(|&(vreg, ..)| vreg),
592 );
593 vregs.debug_assert_no_vreg_aliases(
595 self.vcode
596 .facts
597 .iter()
598 .zip(&vregs.vreg_types)
599 .enumerate()
600 .filter(|(_, (fact, _))| fact.is_some())
601 .map(|(vreg, (_, &ty))| {
602 let (regclasses, _) = I::rc_for_type(ty).unwrap();
603 VReg::new(vreg, regclasses[0])
604 }),
605 );
606
607 self.vcode
608 }
609
610 pub fn add_user_stack_map(
612 &mut self,
613 inst: BackwardsInsnIndex,
614 entries: &[ir::UserStackMapEntry],
615 ) {
616 let stack_map = ir::UserStackMap::new(entries, self.vcode.abi.sized_stackslot_offsets());
617 let old_entry = self.vcode.user_stack_maps.insert(inst, stack_map);
618 debug_assert!(old_entry.is_none());
619 }
620}
621
622const NO_INST_OFFSET: CodeOffset = u32::MAX;
623
624impl<I: VCodeInst> VCode<I> {
625 fn new(
627 sigs: SigSet,
628 abi: Callee<I::ABIMachineSpec>,
629 emit_info: I::Info,
630 block_order: BlockLoweringOrder,
631 constants: VCodeConstants,
632 log2_min_function_alignment: u8,
633 ) -> Self {
634 let n_blocks = block_order.lowered_order().len();
635 VCode {
636 sigs,
637 vreg_types: vec![],
638 insts: Vec::with_capacity(10 * n_blocks),
639 user_stack_maps: FxHashMap::default(),
640 operands: Vec::with_capacity(30 * n_blocks),
641 operand_ranges: Ranges::with_capacity(10 * n_blocks),
642 clobbers: FxHashMap::default(),
643 srclocs: Vec::with_capacity(10 * n_blocks),
644 entry: BlockIndex::new(0),
645 block_ranges: Ranges::with_capacity(n_blocks),
646 block_succ_range: Ranges::with_capacity(n_blocks),
647 block_succs: Vec::with_capacity(n_blocks),
648 block_pred_range: Ranges::default(),
649 block_preds: Vec::new(),
650 block_params_range: Ranges::with_capacity(n_blocks),
651 block_params: Vec::with_capacity(5 * n_blocks),
652 branch_block_args: Vec::with_capacity(10 * n_blocks),
653 branch_block_arg_range: Ranges::with_capacity(2 * n_blocks),
654 branch_block_arg_succ_range: Ranges::with_capacity(n_blocks),
655 block_order,
656 abi,
657 emit_info,
658 constants,
659 debug_value_labels: vec![],
660 facts: vec![],
661 log2_min_function_alignment,
662 }
663 }
664
665 pub fn num_blocks(&self) -> usize {
668 self.block_ranges.len()
669 }
670
671 pub fn num_insts(&self) -> usize {
673 self.insts.len()
674 }
675
676 fn compute_clobbers(&self, regalloc: ®alloc2::Output) -> Vec<Writable<RealReg>> {
677 let mut clobbered = PRegSet::default();
678
679 for (_, Edit::Move { to, .. }) in ®alloc.edits {
681 if let Some(preg) = to.as_reg() {
682 clobbered.add(preg);
683 }
684 }
685
686 for (i, range) in self.operand_ranges.iter() {
687 let operands = &self.operands[range.clone()];
688 let allocs = ®alloc.allocs[range];
689 for (operand, alloc) in operands.iter().zip(allocs.iter()) {
690 if operand.kind() == OperandKind::Def {
691 if let Some(preg) = alloc.as_reg() {
692 clobbered.add(preg);
693 }
694 }
695 }
696
697 if self.insts[i].is_included_in_clobbers() {
717 if let Some(&inst_clobbered) = self.clobbers.get(&InsnIndex::new(i)) {
718 clobbered.union_from(inst_clobbered);
719 }
720 }
721 }
722
723 clobbered
724 .into_iter()
725 .map(|preg| Writable::from_reg(RealReg::from(preg)))
726 .collect()
727 }
728
729 pub fn emit(
737 mut self,
738 regalloc: ®alloc2::Output,
739 want_disasm: bool,
740 flags: &settings::Flags,
741 ctrl_plane: &mut ControlPlane,
742 ) -> EmitResult
743 where
744 I: VCodeInst,
745 {
746 let _tt = timing::vcode_emit();
747 let mut buffer = MachBuffer::new();
748 buffer.set_log2_min_function_alignment(self.log2_min_function_alignment);
749 let mut bb_starts: Vec<Option<CodeOffset>> = vec![];
750
751 buffer.reserve_labels_for_blocks(self.num_blocks());
753
754 buffer.register_constants(&self.constants);
758
759 let mut final_order: SmallVec<[BlockIndex; 16]> = smallvec![];
761 let mut cold_blocks: SmallVec<[BlockIndex; 16]> = smallvec![];
762 for block in 0..self.num_blocks() {
763 let block = BlockIndex::new(block);
764 if self.block_order.is_cold(block) {
765 cold_blocks.push(block);
766 } else {
767 final_order.push(block);
768 }
769 }
770 final_order.extend(cold_blocks.clone());
771
772 let clobbers = self.compute_clobbers(regalloc);
781 self.abi
782 .compute_frame_layout(&self.sigs, regalloc.num_spillslots, clobbers);
783
784 let mut cur_srcloc = None;
786 let mut last_offset = None;
787 let mut inst_offsets = vec![];
788 let mut state = I::State::new(&self.abi, std::mem::take(ctrl_plane));
789
790 let mut disasm = String::new();
791
792 if !self.debug_value_labels.is_empty() {
793 inst_offsets.resize(self.insts.len(), NO_INST_OFFSET);
794 }
795
796 let mut ra_edits_per_block: SmallVec<[u32; 64]> = smallvec![];
801 let mut edit_idx = 0;
802 for block in 0..self.num_blocks() {
803 let end_inst = InsnIndex::new(self.block_ranges.get(block).end);
804 let start_edit_idx = edit_idx;
805 while edit_idx < regalloc.edits.len() && regalloc.edits[edit_idx].0.inst() < end_inst {
806 edit_idx += 1;
807 }
808 let end_edit_idx = edit_idx;
809 ra_edits_per_block.push((end_edit_idx - start_edit_idx) as u32);
810 }
811
812 let is_forward_edge_cfi_enabled = self.abi.is_forward_edge_cfi_enabled();
813 let mut bb_padding = match flags.bb_padding_log2_minus_one() {
814 0 => Vec::new(),
815 n => vec![0; 1 << (n - 1)],
816 };
817 let mut total_bb_padding = 0;
818
819 for (block_order_idx, &block) in final_order.iter().enumerate() {
820 trace!("emitting block {:?}", block);
821
822 state.on_new_block();
824
825 let new_offset = I::align_basic_block(buffer.cur_offset());
827 while new_offset > buffer.cur_offset() {
828 let nop = I::gen_nop((new_offset - buffer.cur_offset()) as usize);
830 nop.emit(&mut buffer, &self.emit_info, &mut Default::default());
831 }
832 assert_eq!(buffer.cur_offset(), new_offset);
833
834 let do_emit = |inst: &I,
835 disasm: &mut String,
836 buffer: &mut MachBuffer<I>,
837 state: &mut I::State| {
838 if want_disasm && !inst.is_args() {
839 let mut s = state.clone();
840 writeln!(disasm, " {}", inst.pretty_print_inst(&mut s)).unwrap();
841 }
842 inst.emit(buffer, &self.emit_info, state);
843 };
844
845 if block == self.entry {
847 trace!(" -> entry block");
848 buffer.start_srcloc(Default::default());
849 for inst in &self.abi.gen_prologue() {
850 do_emit(&inst, &mut disasm, &mut buffer, &mut state);
851 }
852 buffer.end_srcloc();
853 }
854
855 buffer.bind_label(MachLabel::from_block(block), state.ctrl_plane_mut());
858
859 if want_disasm {
860 writeln!(&mut disasm, "block{}:", block.index()).unwrap();
861 }
862
863 if flags.machine_code_cfg_info() {
864 let cur_offset = buffer.cur_offset();
867 if last_offset.is_some() && cur_offset <= last_offset.unwrap() {
868 for i in (0..bb_starts.len()).rev() {
869 if bb_starts[i].is_some() && cur_offset > bb_starts[i].unwrap() {
870 break;
871 }
872 bb_starts[i] = None;
873 }
874 }
875 bb_starts.push(Some(cur_offset));
876 last_offset = Some(cur_offset);
877 }
878
879 if let Some(block_start) = I::gen_block_start(
880 self.block_order.is_indirect_branch_target(block),
881 is_forward_edge_cfi_enabled,
882 ) {
883 do_emit(&block_start, &mut disasm, &mut buffer, &mut state);
884 }
885
886 for inst_or_edit in regalloc.block_insts_and_edits(&self, block) {
887 match inst_or_edit {
888 InstOrEdit::Inst(iix) => {
889 if !self.debug_value_labels.is_empty() {
890 if !self.block_order.is_cold(block) {
903 inst_offsets[iix.index()] = buffer.cur_offset();
904 }
905 }
906
907 let srcloc = self.srclocs[iix.index()];
909 if cur_srcloc != Some(srcloc) {
910 if cur_srcloc.is_some() {
911 buffer.end_srcloc();
912 }
913 buffer.start_srcloc(srcloc);
914 cur_srcloc = Some(srcloc);
915 }
916
917 let stack_map_disasm = if self.insts[iix.index()].is_safepoint() {
920 let (user_stack_map, user_stack_map_disasm) = {
921 let index = iix.to_backwards_insn_index(self.num_insts());
928 let user_stack_map = self.user_stack_maps.remove(&index);
929 let user_stack_map_disasm =
930 user_stack_map.as_ref().map(|m| format!(" ; {m:?}"));
931 (user_stack_map, user_stack_map_disasm)
932 };
933
934 state.pre_safepoint(user_stack_map);
935
936 user_stack_map_disasm
937 } else {
938 None
939 };
940
941 if self.insts[iix.index()].is_term() == MachTerminator::Ret {
946 for inst in self.abi.gen_epilogue() {
947 do_emit(&inst, &mut disasm, &mut buffer, &mut state);
948 }
949 } else {
950 let mut allocs = regalloc.inst_allocs(iix).iter();
953 self.insts[iix.index()].get_operands(
954 &mut |reg: &mut Reg, constraint, _kind, _pos| {
955 let alloc =
956 allocs.next().expect("enough allocations for all operands");
957
958 if let Some(alloc) = alloc.as_reg() {
959 let alloc: Reg = alloc.into();
960 if let OperandConstraint::FixedReg(rreg) = constraint {
961 debug_assert_eq!(Reg::from(rreg), alloc);
962 }
963 *reg = alloc;
964 } else if let Some(alloc) = alloc.as_stack() {
965 let alloc: Reg = alloc.into();
966 *reg = alloc;
967 }
968 },
969 );
970 debug_assert!(allocs.next().is_none());
971
972 do_emit(
974 &self.insts[iix.index()],
975 &mut disasm,
976 &mut buffer,
977 &mut state,
978 );
979 if let Some(stack_map_disasm) = stack_map_disasm {
980 disasm.push_str(&stack_map_disasm);
981 disasm.push('\n');
982 }
983 }
984 }
985
986 InstOrEdit::Edit(Edit::Move { from, to }) => {
987 match (from.as_reg(), to.as_reg()) {
990 (Some(from), Some(to)) => {
991 let from_rreg = Reg::from(from);
993 let to_rreg = Writable::from_reg(Reg::from(to));
994 debug_assert_eq!(from.class(), to.class());
995 let ty = I::canonical_type_for_rc(from.class());
996 let mv = I::gen_move(to_rreg, from_rreg, ty);
997 do_emit(&mv, &mut disasm, &mut buffer, &mut state);
998 }
999 (Some(from), None) => {
1000 let to = to.as_stack().unwrap();
1002 let from_rreg = RealReg::from(from);
1003 let spill = self.abi.gen_spill(to, from_rreg);
1004 do_emit(&spill, &mut disasm, &mut buffer, &mut state);
1005 }
1006 (None, Some(to)) => {
1007 let from = from.as_stack().unwrap();
1009 let to_rreg = Writable::from_reg(RealReg::from(to));
1010 let reload = self.abi.gen_reload(to_rreg, from);
1011 do_emit(&reload, &mut disasm, &mut buffer, &mut state);
1012 }
1013 (None, None) => {
1014 panic!("regalloc2 should have eliminated stack-to-stack moves!");
1015 }
1016 }
1017 }
1018 }
1019 }
1020
1021 if cur_srcloc.is_some() {
1022 buffer.end_srcloc();
1023 cur_srcloc = None;
1024 }
1025
1026 let worst_case_next_bb = if block_order_idx < final_order.len() - 1 {
1030 let next_block = final_order[block_order_idx + 1];
1031 let next_block_range = self.block_ranges.get(next_block.index());
1032 let next_block_size = next_block_range.len() as u32;
1033 let next_block_ra_insertions = ra_edits_per_block[next_block.index()];
1034 I::worst_case_size() * (next_block_size + next_block_ra_insertions)
1035 } else {
1036 0
1037 };
1038 let padding = if bb_padding.is_empty() {
1039 0
1040 } else {
1041 bb_padding.len() as u32 + I::LabelUse::ALIGN - 1
1042 };
1043 if buffer.island_needed(padding + worst_case_next_bb) {
1044 buffer.emit_island(padding + worst_case_next_bb, ctrl_plane);
1045 }
1046
1047 if !bb_padding.is_empty() {
1056 buffer.put_data(&bb_padding);
1057 buffer.align_to(I::LabelUse::ALIGN);
1058 total_bb_padding += bb_padding.len();
1059 if total_bb_padding > (150 << 20) {
1060 bb_padding = Vec::new();
1061 }
1062 }
1063 }
1064
1065 debug_assert!(
1066 self.user_stack_maps.is_empty(),
1067 "any stack maps should have been consumed by instruction emission, still have: {:#?}",
1068 self.user_stack_maps,
1069 );
1070
1071 buffer.optimize_branches(ctrl_plane);
1074
1075 *ctrl_plane = state.take_ctrl_plane();
1077
1078 let func_body_len = buffer.cur_offset();
1079
1080 let mut bb_edges = vec![];
1082 let mut bb_offsets = vec![];
1083 if flags.machine_code_cfg_info() {
1084 for block in 0..self.num_blocks() {
1085 if bb_starts[block].is_none() {
1086 continue;
1088 }
1089 let from = bb_starts[block].unwrap();
1090
1091 bb_offsets.push(from);
1092 let succs = self.block_succs(BlockIndex::new(block));
1094 for &succ in succs.iter() {
1095 let to = buffer.resolve_label_offset(MachLabel::from_block(succ));
1096 bb_edges.push((from, to));
1097 }
1098 }
1099 }
1100
1101 self.monotonize_inst_offsets(&mut inst_offsets[..], func_body_len);
1102 let value_labels_ranges =
1103 self.compute_value_labels_ranges(regalloc, &inst_offsets[..], func_body_len);
1104 let frame_size = self.abi.frame_size();
1105
1106 EmitResult {
1107 buffer: buffer.finish(&self.constants, ctrl_plane),
1108 bb_offsets,
1109 bb_edges,
1110 disasm: if want_disasm { Some(disasm) } else { None },
1111 sized_stackslot_offsets: self.abi.sized_stackslot_offsets().clone(),
1112 dynamic_stackslot_offsets: self.abi.dynamic_stackslot_offsets().clone(),
1113 value_labels_ranges,
1114 frame_size,
1115 }
1116 }
1117
1118 fn monotonize_inst_offsets(&self, inst_offsets: &mut [CodeOffset], func_body_len: u32) {
1119 if self.debug_value_labels.is_empty() {
1120 return;
1121 }
1122
1123 let mut next_offset = func_body_len;
1135 for inst_index in (0..(inst_offsets.len() - 1)).rev() {
1136 let inst_offset = inst_offsets[inst_index];
1137
1138 if inst_offset == NO_INST_OFFSET {
1140 continue;
1141 }
1142
1143 if inst_offset > next_offset {
1144 trace!(
1145 "Fixing code offset of the removed Inst {}: {} -> {}",
1146 inst_index, inst_offset, next_offset
1147 );
1148 inst_offsets[inst_index] = next_offset;
1149 continue;
1150 }
1151
1152 next_offset = inst_offset;
1153 }
1154 }
1155
1156 fn compute_value_labels_ranges(
1157 &self,
1158 regalloc: ®alloc2::Output,
1159 inst_offsets: &[CodeOffset],
1160 func_body_len: u32,
1161 ) -> ValueLabelsRanges {
1162 if self.debug_value_labels.is_empty() {
1163 return ValueLabelsRanges::default();
1164 }
1165
1166 if trace_log_enabled!() {
1167 self.log_value_labels_ranges(regalloc, inst_offsets);
1168 }
1169
1170 let mut value_labels_ranges: ValueLabelsRanges = HashMap::new();
1171 for &(label, from, to, alloc) in ®alloc.debug_locations {
1172 let label = ValueLabel::from_u32(label);
1173 let ranges = value_labels_ranges.entry(label).or_insert_with(|| vec![]);
1174 let prog_point_to_inst = |prog_point: ProgPoint| {
1175 let mut inst = prog_point.inst();
1176 if prog_point.pos() == InstPosition::After {
1177 inst = inst.next();
1178 }
1179 inst.index()
1180 };
1181 let from_inst_index = prog_point_to_inst(from);
1182 let to_inst_index = prog_point_to_inst(to);
1183 let from_offset = inst_offsets[from_inst_index];
1184 let to_offset = if to_inst_index == inst_offsets.len() {
1185 func_body_len
1186 } else {
1187 inst_offsets[to_inst_index]
1188 };
1189
1190 if from_offset == NO_INST_OFFSET
1193 || to_offset == NO_INST_OFFSET
1194 || from_offset == to_offset
1195 {
1196 continue;
1197 }
1198
1199 let loc = if let Some(preg) = alloc.as_reg() {
1200 LabelValueLoc::Reg(Reg::from(preg))
1201 } else {
1202 let slot = alloc.as_stack().unwrap();
1203 let slot_offset = self.abi.get_spillslot_offset(slot);
1204 let slot_base_to_caller_sp_offset = self.abi.slot_base_to_caller_sp_offset();
1205 let caller_sp_to_cfa_offset =
1206 crate::isa::unwind::systemv::caller_sp_to_cfa_offset();
1207 let cfa_to_sp_offset =
1209 -((slot_base_to_caller_sp_offset + caller_sp_to_cfa_offset) as i64);
1210 LabelValueLoc::CFAOffset(cfa_to_sp_offset + slot_offset)
1211 };
1212
1213 if let Some(last_loc_range) = ranges.last_mut() {
1216 if last_loc_range.loc == loc && last_loc_range.end == from_offset {
1217 trace!(
1218 "Extending debug range for {:?} in {:?} to Inst {} ({})",
1219 label, loc, to_inst_index, to_offset
1220 );
1221 last_loc_range.end = to_offset;
1222 continue;
1223 }
1224 }
1225
1226 trace!(
1227 "Recording debug range for {:?} in {:?}: [Inst {}..Inst {}) [{}..{})",
1228 label, loc, from_inst_index, to_inst_index, from_offset, to_offset
1229 );
1230
1231 ranges.push(ValueLocRange {
1232 loc,
1233 start: from_offset,
1234 end: to_offset,
1235 });
1236 }
1237
1238 value_labels_ranges
1239 }
1240
1241 fn log_value_labels_ranges(&self, regalloc: ®alloc2::Output, inst_offsets: &[CodeOffset]) {
1242 debug_assert!(trace_log_enabled!());
1243
1244 let mut labels = vec![];
1247 for &(label, _, _, _) in ®alloc.debug_locations {
1248 if Some(&label) == labels.last() {
1249 continue;
1250 }
1251 labels.push(label);
1252 }
1253
1254 let mut vregs = vec![];
1258 for &(vreg, start, end, label) in &self.debug_value_labels {
1259 if matches!(labels.binary_search(&label), Ok(_)) {
1260 vregs.push((label, start, end, vreg));
1261 }
1262 }
1263 vregs.sort_unstable_by(
1264 |(l_label, l_start, _, _), (r_label, r_start, _, _)| match l_label.cmp(r_label) {
1265 Ordering::Equal => l_start.cmp(r_start),
1266 cmp => cmp,
1267 },
1268 );
1269
1270 #[derive(PartialEq)]
1271 enum Mode {
1272 Measure,
1273 Emit,
1274 }
1275 #[derive(PartialEq)]
1276 enum Row {
1277 Head,
1278 Line,
1279 Inst(usize, usize),
1280 }
1281
1282 let mut widths = vec![0; 3 + 2 * labels.len()];
1283 let mut row = String::new();
1284 let mut output_row = |row_kind: Row, mode: Mode| {
1285 let mut column_index = 0;
1286 row.clear();
1287
1288 macro_rules! output_cell_impl {
1289 ($fill:literal, $span:literal, $($cell_fmt:tt)*) => {
1290 let column_start = row.len();
1291 {
1292 row.push('|');
1293 write!(row, $($cell_fmt)*).unwrap();
1294 }
1295
1296 let next_column_index = column_index + $span;
1297 let expected_width: usize = widths[column_index..next_column_index].iter().sum();
1298 if mode == Mode::Measure {
1299 let actual_width = row.len() - column_start;
1300 if actual_width > expected_width {
1301 widths[next_column_index - 1] += actual_width - expected_width;
1302 }
1303 } else {
1304 let column_end = column_start + expected_width;
1305 while row.len() != column_end {
1306 row.push($fill);
1307 }
1308 }
1309 column_index = next_column_index;
1310 };
1311 }
1312 macro_rules! output_cell {
1313 ($($cell_fmt:tt)*) => {
1314 output_cell_impl!(' ', 1, $($cell_fmt)*);
1315 };
1316 }
1317
1318 match row_kind {
1319 Row::Head => {
1320 output_cell!("BB");
1321 output_cell!("Inst");
1322 output_cell!("IP");
1323 for label in &labels {
1324 output_cell_impl!(' ', 2, "{:?}", ValueLabel::from_u32(*label));
1325 }
1326 }
1327 Row::Line => {
1328 debug_assert!(mode == Mode::Emit);
1329 for _ in 0..3 {
1330 output_cell_impl!('-', 1, "");
1331 }
1332 for _ in &labels {
1333 output_cell_impl!('-', 2, "");
1334 }
1335 }
1336 Row::Inst(block_index, inst_index) => {
1337 debug_assert!(inst_index < self.num_insts());
1338 if self.block_ranges.get(block_index).start == inst_index {
1339 output_cell!("B{}", block_index);
1340 } else {
1341 output_cell!("");
1342 }
1343 output_cell!("Inst {inst_index} ");
1344 output_cell!("{} ", inst_offsets[inst_index]);
1345
1346 for label in &labels {
1347 use regalloc2::Inst;
1349 let vreg_cmp = |inst: usize,
1350 vreg_label: &u32,
1351 range_start: &Inst,
1352 range_end: &Inst| {
1353 match vreg_label.cmp(&label) {
1354 Ordering::Equal => {
1355 if range_end.index() <= inst {
1356 Ordering::Less
1357 } else if range_start.index() > inst {
1358 Ordering::Greater
1359 } else {
1360 Ordering::Equal
1361 }
1362 }
1363 cmp => cmp,
1364 }
1365 };
1366 let vreg_index =
1367 vregs.binary_search_by(|(l, s, e, _)| vreg_cmp(inst_index, l, s, e));
1368 if let Ok(vreg_index) = vreg_index {
1369 let mut prev_vreg = None;
1370 if inst_index > 0 {
1371 let prev_vreg_index = vregs.binary_search_by(|(l, s, e, _)| {
1372 vreg_cmp(inst_index - 1, l, s, e)
1373 });
1374 if let Ok(prev_vreg_index) = prev_vreg_index {
1375 prev_vreg = Some(vregs[prev_vreg_index].3);
1376 }
1377 }
1378
1379 let vreg = vregs[vreg_index].3;
1380 if Some(vreg) == prev_vreg {
1381 output_cell!("*");
1382 } else {
1383 output_cell!("{}", vreg);
1384 }
1385 } else {
1386 output_cell!("");
1387 }
1388
1389 let inst_prog_point = ProgPoint::before(Inst::new(inst_index));
1391 let range_index = regalloc.debug_locations.binary_search_by(
1392 |(range_label, range_start, range_end, _)| match range_label.cmp(label)
1393 {
1394 Ordering::Equal => {
1395 if *range_end <= inst_prog_point {
1396 Ordering::Less
1397 } else if *range_start > inst_prog_point {
1398 Ordering::Greater
1399 } else {
1400 Ordering::Equal
1401 }
1402 }
1403 cmp => cmp,
1404 },
1405 );
1406 if let Ok(range_index) = range_index {
1407 if let Some(reg) = regalloc.debug_locations[range_index].3.as_reg() {
1409 output_cell!("{:?}", Reg::from(reg));
1410 } else {
1411 output_cell!("Stk");
1412 }
1413 } else {
1414 output_cell!("");
1416 }
1417 }
1418 }
1419 }
1420 row.push('|');
1421
1422 if mode == Mode::Emit {
1423 trace!("{}", row.as_str());
1424 }
1425 };
1426
1427 for block_index in 0..self.num_blocks() {
1428 for inst_index in self.block_ranges.get(block_index) {
1429 output_row(Row::Inst(block_index, inst_index), Mode::Measure);
1430 }
1431 }
1432 output_row(Row::Head, Mode::Measure);
1433
1434 output_row(Row::Head, Mode::Emit);
1435 output_row(Row::Line, Mode::Emit);
1436 for block_index in 0..self.num_blocks() {
1437 for inst_index in self.block_ranges.get(block_index) {
1438 output_row(Row::Inst(block_index, inst_index), Mode::Emit);
1439 }
1440 }
1441 }
1442
1443 pub fn bindex_to_bb(&self, block: BlockIndex) -> Option<ir::Block> {
1445 self.block_order.lowered_order()[block.index()].orig_block()
1446 }
1447
1448 pub fn vreg_type(&self, vreg: VReg) -> Type {
1450 self.vreg_types[vreg.vreg()]
1451 }
1452
1453 pub fn vreg_fact(&self, vreg: VReg) -> Option<&Fact> {
1455 self.facts[vreg.vreg()].as_ref()
1456 }
1457
1458 pub fn set_vreg_fact(&mut self, vreg: VReg, fact: Fact) {
1460 trace!("set fact on {}: {:?}", vreg, fact);
1461 self.facts[vreg.vreg()] = Some(fact);
1462 }
1463
1464 pub fn inst_defines_facts(&self, inst: InsnIndex) -> bool {
1466 self.inst_operands(inst)
1467 .iter()
1468 .filter(|o| o.kind() == OperandKind::Def)
1469 .map(|o| o.vreg())
1470 .any(|vreg| self.facts[vreg.vreg()].is_some())
1471 }
1472
1473 pub fn get_user_stack_map(&self, inst: InsnIndex) -> Option<&ir::UserStackMap> {
1475 let index = inst.to_backwards_insn_index(self.num_insts());
1476 self.user_stack_maps.get(&index)
1477 }
1478}
1479
1480impl<I: VCodeInst> std::ops::Index<InsnIndex> for VCode<I> {
1481 type Output = I;
1482 fn index(&self, idx: InsnIndex) -> &Self::Output {
1483 &self.insts[idx.index()]
1484 }
1485}
1486
1487impl<I: VCodeInst> RegallocFunction for VCode<I> {
1488 fn num_insts(&self) -> usize {
1489 self.insts.len()
1490 }
1491
1492 fn num_blocks(&self) -> usize {
1493 self.block_ranges.len()
1494 }
1495
1496 fn entry_block(&self) -> BlockIndex {
1497 self.entry
1498 }
1499
1500 fn block_insns(&self, block: BlockIndex) -> InstRange {
1501 let range = self.block_ranges.get(block.index());
1502 InstRange::new(InsnIndex::new(range.start), InsnIndex::new(range.end))
1503 }
1504
1505 fn block_succs(&self, block: BlockIndex) -> &[BlockIndex] {
1506 let range = self.block_succ_range.get(block.index());
1507 &self.block_succs[range]
1508 }
1509
1510 fn block_preds(&self, block: BlockIndex) -> &[BlockIndex] {
1511 let range = self.block_pred_range.get(block.index());
1512 &self.block_preds[range]
1513 }
1514
1515 fn block_params(&self, block: BlockIndex) -> &[VReg] {
1516 if block == self.entry {
1519 return &[];
1520 }
1521
1522 let range = self.block_params_range.get(block.index());
1523 &self.block_params[range]
1524 }
1525
1526 fn branch_blockparams(&self, block: BlockIndex, _insn: InsnIndex, succ_idx: usize) -> &[VReg] {
1527 let succ_range = self.branch_block_arg_succ_range.get(block.index());
1528 debug_assert!(succ_idx < succ_range.len());
1529 let branch_block_args = self.branch_block_arg_range.get(succ_range.start + succ_idx);
1530 &self.branch_block_args[branch_block_args]
1531 }
1532
1533 fn is_ret(&self, insn: InsnIndex) -> bool {
1534 match self.insts[insn.index()].is_term() {
1535 MachTerminator::None => self.insts[insn.index()].is_trap(),
1537 MachTerminator::Ret | MachTerminator::RetCall => true,
1538 MachTerminator::Branch => false,
1539 }
1540 }
1541
1542 fn is_branch(&self, insn: InsnIndex) -> bool {
1543 match self.insts[insn.index()].is_term() {
1544 MachTerminator::Branch => true,
1545 _ => false,
1546 }
1547 }
1548
1549 fn inst_operands(&self, insn: InsnIndex) -> &[Operand] {
1550 let range = self.operand_ranges.get(insn.index());
1551 &self.operands[range]
1552 }
1553
1554 fn inst_clobbers(&self, insn: InsnIndex) -> PRegSet {
1555 self.clobbers.get(&insn).cloned().unwrap_or_default()
1556 }
1557
1558 fn num_vregs(&self) -> usize {
1559 self.vreg_types.len()
1560 }
1561
1562 fn debug_value_labels(&self) -> &[(VReg, InsnIndex, InsnIndex, u32)] {
1563 &self.debug_value_labels
1564 }
1565
1566 fn spillslot_size(&self, regclass: RegClass) -> usize {
1567 self.abi.get_spillslot_size(regclass) as usize
1568 }
1569
1570 fn allow_multiple_vreg_defs(&self) -> bool {
1571 true
1575 }
1576}
1577
1578impl<I: VCodeInst> Debug for VRegAllocator<I> {
1579 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1580 writeln!(f, "VRegAllocator {{")?;
1581
1582 let mut alias_keys = self.vreg_aliases.keys().cloned().collect::<Vec<_>>();
1583 alias_keys.sort_unstable();
1584 for key in alias_keys {
1585 let dest = self.vreg_aliases.get(&key).unwrap();
1586 writeln!(f, " {:?} := {:?}", Reg::from(key), Reg::from(*dest))?;
1587 }
1588
1589 for (vreg, fact) in self.facts.iter().enumerate() {
1590 if let Some(fact) = fact {
1591 writeln!(f, " v{vreg} ! {fact}")?;
1592 }
1593 }
1594
1595 writeln!(f, "}}")
1596 }
1597}
1598
1599impl<I: VCodeInst> fmt::Debug for VCode<I> {
1600 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1601 writeln!(f, "VCode {{")?;
1602 writeln!(f, " Entry block: {}", self.entry.index())?;
1603
1604 let mut state = Default::default();
1605
1606 for block in 0..self.num_blocks() {
1607 let block = BlockIndex::new(block);
1608 writeln!(
1609 f,
1610 "Block {}({:?}):",
1611 block.index(),
1612 self.block_params(block)
1613 )?;
1614 if let Some(bb) = self.bindex_to_bb(block) {
1615 writeln!(f, " (original IR block: {bb})")?;
1616 }
1617 for (succ_idx, succ) in self.block_succs(block).iter().enumerate() {
1618 writeln!(
1619 f,
1620 " (successor: Block {}({:?}))",
1621 succ.index(),
1622 self.branch_blockparams(block, InsnIndex::new(0) , succ_idx)
1623 )?;
1624 }
1625 for inst in self.block_ranges.get(block.index()) {
1626 writeln!(
1627 f,
1628 " Inst {}: {}",
1629 inst,
1630 self.insts[inst].pretty_print_inst(&mut state)
1631 )?;
1632 if !self.operands.is_empty() {
1633 for operand in self.inst_operands(InsnIndex::new(inst)) {
1634 if operand.kind() == OperandKind::Def {
1635 if let Some(fact) = &self.facts[operand.vreg().vreg()] {
1636 writeln!(f, " v{} ! {}", operand.vreg().vreg(), fact)?;
1637 }
1638 }
1639 }
1640 }
1641 if let Some(user_stack_map) = self.get_user_stack_map(InsnIndex::new(inst)) {
1642 writeln!(f, " {user_stack_map:?}")?;
1643 }
1644 }
1645 }
1646
1647 writeln!(f, "}}")?;
1648 Ok(())
1649 }
1650}
1651
1652pub struct VRegAllocator<I> {
1654 vreg_types: Vec<Type>,
1656
1657 vreg_aliases: FxHashMap<regalloc2::VReg, regalloc2::VReg>,
1664
1665 deferred_error: Option<CodegenError>,
1670
1671 facts: Vec<Option<Fact>>,
1673
1674 _inst: core::marker::PhantomData<I>,
1676}
1677
1678impl<I: VCodeInst> VRegAllocator<I> {
1679 pub fn with_capacity(capacity: usize) -> Self {
1681 let capacity = first_user_vreg_index() + capacity;
1682 let mut vreg_types = Vec::with_capacity(capacity);
1683 vreg_types.resize(first_user_vreg_index(), types::INVALID);
1684 Self {
1685 vreg_types,
1686 vreg_aliases: FxHashMap::with_capacity_and_hasher(capacity, Default::default()),
1687 deferred_error: None,
1688 facts: Vec::with_capacity(capacity),
1689 _inst: core::marker::PhantomData::default(),
1690 }
1691 }
1692
1693 pub fn alloc(&mut self, ty: Type) -> CodegenResult<ValueRegs<Reg>> {
1695 if self.deferred_error.is_some() {
1696 return Err(CodegenError::CodeTooLarge);
1697 }
1698 let v = self.vreg_types.len();
1699 let (regclasses, tys) = I::rc_for_type(ty)?;
1700 if v + regclasses.len() >= VReg::MAX {
1701 return Err(CodegenError::CodeTooLarge);
1702 }
1703
1704 let regs: ValueRegs<Reg> = match regclasses {
1705 &[rc0] => ValueRegs::one(VReg::new(v, rc0).into()),
1706 &[rc0, rc1] => ValueRegs::two(VReg::new(v, rc0).into(), VReg::new(v + 1, rc1).into()),
1707 _ => panic!("Value must reside in 1 or 2 registers"),
1711 };
1712 for (®_ty, ®) in tys.iter().zip(regs.regs().iter()) {
1713 let vreg = reg.to_virtual_reg().unwrap();
1714 debug_assert_eq!(self.vreg_types.len(), vreg.index());
1715 self.vreg_types.push(reg_ty);
1716 }
1717
1718 self.facts.resize(self.vreg_types.len(), None);
1720
1721 Ok(regs)
1722 }
1723
1724 pub fn alloc_with_deferred_error(&mut self, ty: Type) -> ValueRegs<Reg> {
1730 match self.alloc(ty) {
1731 Ok(x) => x,
1732 Err(e) => {
1733 self.deferred_error = Some(e);
1734 self.bogus_for_deferred_error(ty)
1735 }
1736 }
1737 }
1738
1739 pub fn take_deferred_error(&mut self) -> Option<CodegenError> {
1741 self.deferred_error.take()
1742 }
1743
1744 fn bogus_for_deferred_error(&self, ty: Type) -> ValueRegs<Reg> {
1748 let (regclasses, _tys) = I::rc_for_type(ty).expect("must have valid type");
1749 match regclasses {
1750 &[rc0] => ValueRegs::one(VReg::new(0, rc0).into()),
1751 &[rc0, rc1] => ValueRegs::two(VReg::new(0, rc0).into(), VReg::new(1, rc1).into()),
1752 _ => panic!("Value must reside in 1 or 2 registers"),
1753 }
1754 }
1755
1756 pub fn set_vreg_alias(&mut self, from: Reg, to: Reg) {
1758 let from = from.into();
1759 let resolved_to = self.resolve_vreg_alias(to.into());
1760 assert_ne!(resolved_to, from);
1762
1763 if let Some(fact) = self.facts[from.vreg()].take() {
1767 self.set_fact(resolved_to, fact);
1768 }
1769
1770 let old_alias = self.vreg_aliases.insert(from, resolved_to);
1771 debug_assert_eq!(old_alias, None);
1772 }
1773
1774 fn resolve_vreg_alias(&self, mut vreg: regalloc2::VReg) -> regalloc2::VReg {
1775 while let Some(to) = self.vreg_aliases.get(&vreg) {
1785 vreg = *to;
1786 }
1787 vreg
1788 }
1789
1790 #[inline]
1791 fn debug_assert_no_vreg_aliases(&self, mut list: impl Iterator<Item = VReg>) {
1792 debug_assert!(list.all(|vreg| !self.vreg_aliases.contains_key(&vreg)));
1793 }
1794
1795 fn set_fact(&mut self, vreg: regalloc2::VReg, fact: Fact) -> Option<Fact> {
1799 trace!("vreg {:?} has fact: {:?}", vreg, fact);
1800 debug_assert!(!self.vreg_aliases.contains_key(&vreg));
1801 self.facts[vreg.vreg()].replace(fact)
1802 }
1803
1804 pub fn set_fact_if_missing(&mut self, vreg: VirtualReg, fact: Fact) {
1806 let vreg = self.resolve_vreg_alias(vreg.into());
1807 if self.facts[vreg.vreg()].is_none() {
1808 self.set_fact(vreg, fact);
1809 }
1810 }
1811
1812 pub fn alloc_with_maybe_fact(
1815 &mut self,
1816 ty: Type,
1817 fact: Option<Fact>,
1818 ) -> CodegenResult<ValueRegs<Reg>> {
1819 let result = self.alloc(ty)?;
1820
1821 assert!(result.len() == 1 || fact.is_none());
1824 if let Some(fact) = fact {
1825 self.set_fact(result.regs()[0].into(), fact);
1826 }
1827
1828 Ok(result)
1829 }
1830}
1831
1832#[derive(Default)]
1844pub struct VCodeConstants {
1845 constants: PrimaryMap<VCodeConstant, VCodeConstantData>,
1846 pool_uses: HashMap<Constant, VCodeConstant>,
1847 well_known_uses: HashMap<*const [u8], VCodeConstant>,
1848 u64s: HashMap<[u8; 8], VCodeConstant>,
1849}
1850impl VCodeConstants {
1851 pub fn with_capacity(expected_num_constants: usize) -> Self {
1853 Self {
1854 constants: PrimaryMap::with_capacity(expected_num_constants),
1855 pool_uses: HashMap::with_capacity(expected_num_constants),
1856 well_known_uses: HashMap::new(),
1857 u64s: HashMap::new(),
1858 }
1859 }
1860
1861 pub fn insert(&mut self, data: VCodeConstantData) -> VCodeConstant {
1866 match data {
1867 VCodeConstantData::Generated(_) => self.constants.push(data),
1868 VCodeConstantData::Pool(constant, _) => match self.pool_uses.get(&constant) {
1869 None => {
1870 let vcode_constant = self.constants.push(data);
1871 self.pool_uses.insert(constant, vcode_constant);
1872 vcode_constant
1873 }
1874 Some(&vcode_constant) => vcode_constant,
1875 },
1876 VCodeConstantData::WellKnown(data_ref) => {
1877 match self.well_known_uses.entry(data_ref as *const [u8]) {
1878 Entry::Vacant(v) => {
1879 let vcode_constant = self.constants.push(data);
1880 v.insert(vcode_constant);
1881 vcode_constant
1882 }
1883 Entry::Occupied(o) => *o.get(),
1884 }
1885 }
1886 VCodeConstantData::U64(value) => match self.u64s.entry(value) {
1887 Entry::Vacant(v) => {
1888 let vcode_constant = self.constants.push(data);
1889 v.insert(vcode_constant);
1890 vcode_constant
1891 }
1892 Entry::Occupied(o) => *o.get(),
1893 },
1894 }
1895 }
1896
1897 pub fn len(&self) -> usize {
1899 self.constants.len()
1900 }
1901
1902 pub fn keys(&self) -> Keys<VCodeConstant> {
1904 self.constants.keys()
1905 }
1906
1907 pub fn iter(&self) -> impl Iterator<Item = (VCodeConstant, &VCodeConstantData)> {
1910 self.constants.iter()
1911 }
1912
1913 pub fn get(&self, c: VCodeConstant) -> &VCodeConstantData {
1915 &self.constants[c]
1916 }
1917
1918 pub fn pool_uses(&self, constant: &VCodeConstantData) -> bool {
1921 match constant {
1922 VCodeConstantData::Pool(c, _) => self.pool_uses.contains_key(c),
1923 _ => false,
1924 }
1925 }
1926}
1927
1928#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1930pub struct VCodeConstant(u32);
1931entity_impl!(VCodeConstant);
1932
1933pub enum VCodeConstantData {
1936 Pool(Constant, ConstantData),
1939 WellKnown(&'static [u8]),
1941 Generated(ConstantData),
1944 U64([u8; 8]),
1948}
1949impl VCodeConstantData {
1950 pub fn as_slice(&self) -> &[u8] {
1952 match self {
1953 VCodeConstantData::Pool(_, d) | VCodeConstantData::Generated(d) => d.as_slice(),
1954 VCodeConstantData::WellKnown(d) => d,
1955 VCodeConstantData::U64(value) => &value[..],
1956 }
1957 }
1958
1959 pub fn alignment(&self) -> u32 {
1961 if self.as_slice().len() <= 8 { 8 } else { 16 }
1962 }
1963}
1964
1965#[cfg(test)]
1966mod test {
1967 use super::*;
1968 use std::mem::size_of;
1969
1970 #[test]
1971 fn size_of_constant_structs() {
1972 assert_eq!(size_of::<Constant>(), 4);
1973 assert_eq!(size_of::<VCodeConstant>(), 4);
1974 assert_eq!(size_of::<ConstantData>(), 3 * size_of::<usize>());
1975 assert_eq!(size_of::<VCodeConstantData>(), 4 * size_of::<usize>());
1976 assert_eq!(
1977 size_of::<PrimaryMap<VCodeConstant, VCodeConstantData>>(),
1978 3 * size_of::<usize>()
1979 );
1980 }
1984}