cranelift_codegen/ir/
memflags.rs

1//! Memory operation flags.
2
3use super::TrapCode;
4use crate::HashMap;
5use crate::entity::{self, PrimaryMap};
6pub use crate::machinst::MachMemFlags;
7use alloc::borrow::Cow;
8use core::fmt;
9use core::hash::{Hash, Hasher};
10use core::ops::Index;
11use core::str::FromStr;
12use cranelift_entity::{entity_impl, packed_option::PackedOption};
13
14#[cfg(feature = "enable-serde")]
15use serde_derive::{Deserialize, Serialize};
16
17/// Endianness of a memory access.
18#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
19pub enum Endianness {
20    /// Little-endian
21    Little,
22    /// Big-endian
23    Big,
24}
25
26/// An opaque reference to an alias region.
27///
28/// Alias regions identify disjoint categories of memory for alias analysis.
29/// Two memory operations in different alias regions are known not to alias.
30#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
31#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
32pub struct AliasRegion(u32);
33entity_impl!(AliasRegion, "region");
34
35/// Data describing an alias region.
36#[derive(Clone, Debug, PartialEq, Eq, Hash)]
37#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
38pub struct AliasRegionData {
39    /// A unique, user-defined identifier for this alias region.
40    ///
41    /// Alias regions are deduplicated based on this identifier.
42    ///
43    /// This deduplication happens during inlining, for example, when a
44    /// callee's alias regions are merged with the caller's. Therefore, when
45    /// inlining is enabled this identifier should be globally unique across
46    /// the whole compilation. When inlining is disabled, it is sufficient
47    /// to be unique within the context of a single function.
48    pub user_id: u32,
49
50    /// Description of this alias region, e.g. "vmctx", "funcref table",
51    /// "global 42", or "gc struct `LinkedList` field `tail`".
52    ///
53    /// This only exists for printing in the CLIF text format.
54    pub description: Cow<'static, str>,
55}
56
57/// An opaque reference to memory operation flags stored in a
58/// [`MemFlagsSet`].
59///
60/// `MemFlags` is a u16 entity index that refers to a [`MemFlagsData`] entry in
61/// the [`MemFlagsSet`] stored in the
62/// [`DataFlowGraph`](super::dfg::DataFlowGraph).
63#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
64#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
65pub struct MemFlags(u16);
66
67impl MemFlags {
68    /// Create a new `MemFlags` from a `u32` index.
69    ///
70    /// Returns `None` if the index doesn't fit in a `u16`.
71    pub fn with_number(n: u32) -> Option<Self> {
72        let val = u16::try_from(n).ok()?;
73        if val == u16::MAX {
74            None
75        } else {
76            Some(Self(val))
77        }
78    }
79}
80
81impl entity::EntityRef for MemFlags {
82    #[inline]
83    fn new(index: usize) -> Self {
84        let val = u16::try_from(index).expect("MemFlags index overflow");
85        Self(val)
86    }
87
88    #[inline]
89    fn index(self) -> usize {
90        usize::from(self.0)
91    }
92}
93
94impl entity::packed_option::ReservedValue for MemFlags {
95    #[inline]
96    fn reserved_value() -> Self {
97        Self(u16::MAX)
98    }
99
100    #[inline]
101    fn is_reserved_value(&self) -> bool {
102        self.0 == u16::MAX
103    }
104}
105
106impl MemFlags {
107    /// Create a new instance from a `u32`.
108    #[inline]
109    pub fn from_u32(x: u32) -> Self {
110        Self(u16::try_from(x).unwrap())
111    }
112
113    /// Return the underlying index value as a `u32`.
114    #[inline]
115    pub fn as_u32(self) -> u32 {
116        u32::from(self.0)
117    }
118
119    /// Return the raw bit encoding for this instance.
120    #[inline]
121    pub fn as_bits(self) -> u32 {
122        u32::from(self.0)
123    }
124
125    /// Create a new instance from the raw bit encoding.
126    #[inline]
127    pub fn from_bits(x: u32) -> Self {
128        Self(u16::try_from(x).unwrap())
129    }
130}
131
132impl fmt::Display for MemFlags {
133    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
134        write!(f, "memflags{}", self.0)
135    }
136}
137
138impl fmt::Debug for MemFlags {
139    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
140        (self as &dyn fmt::Display).fmt(f)
141    }
142}
143
144/// Flags for memory operations like load/store.
145///
146/// Each of these flags introduce a limited form of undefined behavior. The flags each enable
147/// certain optimizations that need to make additional assumptions. Generally, the semantics of a
148/// program does not change when a flag is removed, but adding a flag will.
149///
150/// In addition, the flags determine the endianness of the memory access.  By default,
151/// any memory access uses the native endianness determined by the target ISA.  This can
152/// be overridden for individual accesses by explicitly specifying little- or big-endian
153/// semantics via the flags.
154#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
155#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
156pub struct MemFlagsData {
157    /// Backend-facing memory-operation flags.
158    flags: MachMemFlags,
159
160    /// The alias region for this memory operation, if any.
161    region: PackedOption<AliasRegion>,
162}
163
164const fn no_alias_region() -> PackedOption<AliasRegion> {
165    PackedOption::new(AliasRegion(u32::MAX))
166}
167
168impl MemFlagsData {
169    /// Create a new empty set of flags.
170    pub const fn new() -> Self {
171        Self {
172            flags: MachMemFlags::new(),
173            region: no_alias_region(),
174        }
175    }
176
177    /// Create a set of flags representing an access from a "trusted" address, meaning it's
178    /// known to be aligned and non-trapping.
179    pub const fn trusted() -> Self {
180        Self::new().with_notrap().with_aligned(true)
181    }
182
183    /// Reads the alias region that this memory operation works with.
184    pub fn alias_region(self) -> Option<AliasRegion> {
185        self.region.expand()
186    }
187
188    /// Sets the alias region that this works on to the specified `region`.
189    pub fn with_alias_region(mut self, region: Option<AliasRegion>) -> Self {
190        self.region = region.into();
191        self
192    }
193
194    /// Sets the alias region that this works on to the specified `region`.
195    pub fn set_alias_region(&mut self, region: Option<AliasRegion>) {
196        self.region = region.into();
197    }
198
199    /// Set a flag bit by name.
200    ///
201    /// Returns true if the flag was found and set, false for an unknown flag
202    /// name.
203    ///
204    /// # Errors
205    ///
206    /// Returns an error message if the `name` is known but couldn't be applied
207    /// due to it being a semantic error.
208    pub fn set_by_name(&mut self, name: &str) -> Result<bool, &'static str> {
209        *self = match name {
210            "notrap" => self.with_trap_code(None),
211            "aligned" => self.with_aligned(true),
212            "readonly" => self.with_readonly(true),
213            "little" => {
214                if self.flags.explicit_endianness() == Some(Endianness::Big) {
215                    return Err("cannot set both big and little endian bits");
216                }
217                self.with_endianness(Endianness::Little)
218            }
219            "big" => {
220                if self.flags.explicit_endianness() == Some(Endianness::Little) {
221                    return Err("cannot set both big and little endian bits");
222                }
223                self.with_endianness(Endianness::Big)
224            }
225            "can_move" => self.with_can_move(true),
226            "volatile" => self.with_volatile(true),
227
228            other => match TrapCode::from_str(other) {
229                Ok(code) => self.with_trap_code(Some(code)),
230                Err(()) => return Ok(false),
231            },
232        };
233        Ok(true)
234    }
235
236    /// Return endianness of the memory access.  This will return the endianness
237    /// explicitly specified by the flags if any, and will default to the native
238    /// endianness otherwise.  The native endianness has to be provided by the
239    /// caller since it is not explicitly encoded in CLIF IR -- this allows a
240    /// front end to create IR without having to know the target endianness.
241    pub const fn endianness(self, native_endianness: Endianness) -> Endianness {
242        self.flags.endianness(native_endianness)
243    }
244
245    /// Return endianness of the memory access, if explicitly specified.
246    ///
247    /// If the endianness is not explicitly specified, this will return `None`,
248    /// which means "native endianness".
249    pub const fn explicit_endianness(self) -> Option<Endianness> {
250        self.flags.explicit_endianness()
251    }
252
253    /// Set endianness of the memory access.
254    pub fn set_endianness(&mut self, endianness: Endianness) {
255        *self = self.with_endianness(endianness);
256    }
257
258    /// Set endianness of the memory access, returning new flags.
259    pub const fn with_endianness(mut self, endianness: Endianness) -> Self {
260        self.flags = self.flags.with_endianness(endianness);
261        self
262    }
263
264    /// Test if this memory operation cannot trap.
265    ///
266    /// By default `MemFlags` will assume that any load/store can trap and is
267    /// associated with a `TrapCode::HeapOutOfBounds` code. If the trap code is
268    /// configured to `None` though then this method will return `true` and
269    /// indicates that the memory operation will not trap.
270    ///
271    /// If this returns `true` then the memory is *accessible*, which means
272    /// that accesses will not trap. This makes it possible to delete an unused
273    /// load or a dead store instruction.
274    ///
275    /// This flag does *not* mean that the associated instruction can be
276    /// code-motioned to arbitrary places in the function so long as its data
277    /// dependencies are met. This only means that, given its current location
278    /// in the function, it will never trap. See the `can_move` method for more
279    /// details.
280    pub const fn notrap(self) -> bool {
281        self.trap_code().is_none()
282    }
283
284    /// Sets the trap code for this `MemFlagsData` to `None`.
285    pub fn set_notrap(&mut self) {
286        *self = self.with_notrap();
287    }
288
289    /// Sets the trap code for this `MemFlagsData` to `None`, returning the new
290    /// flags.
291    pub const fn with_notrap(self) -> Self {
292        self.with_trap_code(None)
293    }
294
295    /// Is this memory operation safe to move so long as its data dependencies
296    /// remain satisfied?
297    ///
298    /// If this is `true`, then it is okay to code motion this instruction to
299    /// arbitrary locations, in the function, including across blocks and
300    /// conditional branches, so long as data dependencies (and trap ordering,
301    /// if any) are upheld.
302    ///
303    /// If this is `false`, then this memory operation's safety potentially
304    /// relies upon invariants that are not reflected in its data dependencies,
305    /// and therefore it is not safe to code motion this operation. For example,
306    /// this operation could be in a block that is dominated by a control-flow
307    /// bounds check, which is not reflected in its operands, and it would be
308    /// unsafe to code motion it above the bounds check, even if its data
309    /// dependencies would still be satisfied.
310    pub const fn can_move(self) -> bool {
311        self.flags.can_move()
312    }
313
314    /// Set or clear the `can_move` flag.
315    pub const fn set_can_move(&mut self, enabled: bool) {
316        *self = self.with_can_move(enabled);
317    }
318
319    /// Set or clear the `can_move` flag, returning new flags.
320    pub const fn with_can_move(mut self, enabled: bool) -> Self {
321        self.flags = self.flags.with_can_move(enabled);
322        self
323    }
324
325    /// Test if the `aligned` flag is set.
326    ///
327    /// By default, Cranelift memory instructions work with any unaligned effective address. If the
328    /// `aligned` flag is set, the instruction is permitted to trap or return a wrong result if the
329    /// effective address is misaligned.
330    pub const fn aligned(self) -> bool {
331        self.flags.aligned()
332    }
333
334    /// Set or clear the `aligned` flag.
335    pub fn set_aligned(&mut self, enabled: bool) {
336        *self = self.with_aligned(enabled);
337    }
338
339    /// Set or clear the `aligned` flag, returning new flags.
340    pub const fn with_aligned(mut self, enabled: bool) -> Self {
341        self.flags = self.flags.with_aligned(enabled);
342        self
343    }
344
345    /// Test if the `readonly` flag is set.
346    ///
347    /// Loads with this flag may be eliminated if a previous `readonly` load
348    /// loaded from the same location. The previous load must dominate this
349    /// load, and otherwise it does not matter what happens in between.
350    ///
351    /// Note that if this load is `can_move` then the memory must be readonly
352    /// for the entire function that this load is within. If the load is NOT
353    /// `can_move`, then mutations may happen before this load but cannot
354    /// happen after this load.
355    pub const fn readonly(self) -> bool {
356        self.flags.readonly()
357    }
358
359    /// Set or clear the `readonly` flag.
360    pub fn set_readonly(&mut self, enabled: bool) {
361        *self = self.with_readonly(enabled);
362    }
363
364    /// Set or clear the `readonly` flag, returning new flags.
365    pub const fn with_readonly(mut self, enabled: bool) -> Self {
366        self.flags = self.flags.with_readonly(enabled);
367        self
368    }
369
370    /// Test if the `volatile` flag is set.
371    ///
372    /// A volatile memory operation must be performed exactly as written, as in
373    /// C's and Rust's volatile accesses (memory-mapped I/O, memory another
374    /// agent watches): it is never removed, even when its result is unused or
375    /// it stores a value already in memory; never merged with another access
376    /// (no redundant-load elimination, store-to-load forwarding or dead-store
377    /// elimination); and never reordered with another volatile access.
378    /// Accesses that are not volatile may still move around it.
379    ///
380    /// `volatile` is mutually exclusive with both `can_move` and `readonly`:
381    /// valid flags satisfy `!(volatile && (can_move || readonly))`, which the
382    /// verifier enforces.
383    pub const fn volatile(self) -> bool {
384        self.flags.volatile()
385    }
386
387    /// Set or clear the `volatile` flag.
388    pub fn set_volatile(&mut self, enabled: bool) {
389        *self = self.with_volatile(enabled);
390    }
391
392    /// Set or clear the `volatile` flag, returning new flags.
393    pub const fn with_volatile(mut self, enabled: bool) -> Self {
394        self.flags = self.flags.with_volatile(enabled);
395        self
396    }
397
398    /// Get the trap code to report if this memory access traps.
399    ///
400    /// A `None` trap code indicates that this memory access does not trap.
401    pub const fn trap_code(self) -> Option<TrapCode> {
402        self.flags.trap_code()
403    }
404
405    /// Configures these flags with the specified trap code `code`.
406    ///
407    /// A trap code indicates that this memory operation cannot be optimized
408    /// away and it must "stay where it is" in the programs. Traps are
409    /// considered side effects, for example, and have meaning through the trap
410    /// code that is communicated and which instruction trapped.
411    pub const fn with_trap_code(mut self, code: Option<TrapCode>) -> Self {
412        self.flags = self.flags.with_trap_code(code);
413        self
414    }
415}
416
417impl From<MemFlagsData> for MachMemFlags {
418    fn from(flags: MemFlagsData) -> Self {
419        flags.flags
420    }
421}
422
423impl From<MachMemFlags> for MemFlagsData {
424    fn from(flags: MachMemFlags) -> Self {
425        Self {
426            flags,
427            region: no_alias_region(),
428        }
429    }
430}
431
432impl fmt::Display for MemFlagsData {
433    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
434        write!(f, "{}", self.flags)?;
435        match self.alias_region() {
436            None => {}
437            Some(region) => write!(f, " {region}")?,
438        }
439        Ok(())
440    }
441}
442
443#[derive(Clone, Copy, Debug, PartialEq, Eq)]
444pub struct MemFlagsSetOverflow;
445
446/// A deduplicated set of mem flags.
447#[derive(Clone, Debug)]
448#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
449pub struct MemFlagsSet {
450    mem_flags: PrimaryMap<MemFlags, MemFlagsData>,
451    dedupe_map: HashMap<MemFlagsData, MemFlags>,
452}
453
454impl PartialEq for MemFlagsSet {
455    fn eq(&self, other: &Self) -> bool {
456        self.mem_flags == other.mem_flags
457    }
458}
459
460impl Eq for MemFlagsSet {}
461
462impl Hash for MemFlagsSet {
463    fn hash<H: Hasher>(&self, state: &mut H) {
464        self.mem_flags.hash(state);
465    }
466}
467
468impl MemFlagsSet {
469    /// Create a new empty set.
470    pub fn new() -> Self {
471        Self {
472            mem_flags: PrimaryMap::new(),
473            dedupe_map: HashMap::new(),
474        }
475    }
476
477    /// Insert new mem flags into this set.
478    ///
479    /// Returns an existing `MemFlags` if the data already exists.
480    pub fn insert(&mut self, data: MemFlagsData) -> Result<MemFlags, MemFlagsSetOverflow> {
481        if let Some(&existing) = self.dedupe_map.get(&data) {
482            return Ok(existing);
483        }
484        let next = u32::try_from(self.mem_flags.len())
485            .ok()
486            .and_then(MemFlags::with_number)
487            .ok_or(MemFlagsSetOverflow)?;
488        let key = self.mem_flags.push(data);
489        debug_assert_eq!(key, next);
490        self.dedupe_map.insert(data, key);
491        Ok(key)
492    }
493
494    /// Insert new mem flags into this set, panicking if the index does not fit.
495    pub fn insert_unchecked(&mut self, data: MemFlagsData) -> MemFlags {
496        match self.insert(data) {
497            Ok(flags) => flags,
498            Err(_) => panic!("MemFlags index overflow"),
499        }
500    }
501
502    /// Returns `true` if the given mem flags reference is valid.
503    pub fn is_valid(&self, mf: MemFlags) -> bool {
504        self.mem_flags.is_valid(mf)
505    }
506
507    /// Clear the set.
508    pub fn clear(&mut self) {
509        *self = Self::new();
510    }
511
512    /// Find the entity index for an existing [`MemFlagsData`] value.
513    pub fn get(&self, data: MemFlagsData) -> Option<MemFlags> {
514        self.dedupe_map.get(&data).copied()
515    }
516}
517
518// NB: Do not implement `IndexMut` because mem flags data is deduped and shared
519// by many instructions.
520impl Index<MemFlags> for MemFlagsSet {
521    type Output = MemFlagsData;
522
523    fn index(&self, mf: MemFlags) -> &MemFlagsData {
524        &self.mem_flags[mf]
525    }
526}
527
528/// A deduplicated set of alias regions.
529///
530/// Deduplication is based on `user_id`; the description string is not
531/// considered.
532#[derive(Clone, PartialEq)]
533#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
534pub struct AliasRegionSet {
535    alias_regions: PrimaryMap<AliasRegion, AliasRegionData>,
536    dedupe_map: HashMap<u32, AliasRegion>,
537}
538
539impl Hash for AliasRegionSet {
540    fn hash<H: Hasher>(&self, state: &mut H) {
541        self.alias_regions.hash(state);
542    }
543}
544
545impl AliasRegionSet {
546    /// Create a new empty set.
547    pub fn new() -> Self {
548        Self {
549            alias_regions: PrimaryMap::new(),
550            dedupe_map: HashMap::new(),
551        }
552    }
553
554    /// Insert a new alias region into this set.
555    ///
556    /// Returns an existing `AliasRegion` if one with the same `user_id`
557    /// already exists.
558    pub fn insert(&mut self, data: AliasRegionData) -> AliasRegion {
559        if let Some(&existing) = self.dedupe_map.get(&data.user_id) {
560            return existing;
561        }
562        let user_id = data.user_id;
563        let key = self.alias_regions.push(data);
564        self.dedupe_map.insert(user_id, key);
565        key
566    }
567
568    /// Push a new alias region, bypassing deduplication.
569    ///
570    /// This is used by the CLIF text parser to faithfully represent the
571    /// source text. The verifier will then check for duplicate `user_id`s.
572    pub fn push(&mut self, data: AliasRegionData) -> AliasRegion {
573        let user_id = data.user_id;
574        let key = self.alias_regions.push(data);
575        self.dedupe_map.insert(user_id, key);
576        key
577    }
578
579    /// Returns `true` if this set already contains a region with the given
580    /// `user_id`.
581    pub fn contains(&self, user_id: u32) -> bool {
582        self.dedupe_map.contains_key(&user_id)
583    }
584
585    /// Get the alias region with the given id, if any.
586    pub fn get(&self, user_id: u32) -> Option<AliasRegion> {
587        self.dedupe_map.get(&user_id).copied()
588    }
589
590    /// Returns `true` if the given alias region reference is valid.
591    pub fn is_valid(&self, ar: AliasRegion) -> bool {
592        self.alias_regions.is_valid(ar)
593    }
594
595    /// Return the number of alias regions in the set.
596    pub fn len(&self) -> usize {
597        self.alias_regions.len()
598    }
599
600    /// Iterate over all alias regions and their data.
601    pub fn iter(&self) -> impl Iterator<Item = (AliasRegion, &AliasRegionData)> {
602        self.alias_regions.iter()
603    }
604
605    /// Clear the set.
606    pub fn clear(&mut self) {
607        self.alias_regions.clear();
608        self.dedupe_map.clear();
609    }
610}
611
612// NB: Do not implement `IndexMut` because alias region data is deduped and
613// shared by many mem flags.
614impl Index<AliasRegion> for AliasRegionSet {
615    type Output = AliasRegionData;
616
617    fn index(&self, ar: AliasRegion) -> &AliasRegionData {
618        &self.alias_regions[ar]
619    }
620}
621
622#[cfg(test)]
623mod tests {
624    use super::*;
625    use cranelift_entity::EntityRef;
626
627    #[test]
628    fn roundtrip_traps() {
629        for trap in TrapCode::non_user_traps().iter().copied() {
630            let _flags = MemFlagsData::new().with_trap_code(Some(trap));
631        }
632        let _flags = MemFlagsData::new().with_trap_code(None);
633    }
634
635    #[test]
636    fn cannot_set_big_and_little() {
637        let _big = MemFlagsData::new().with_endianness(Endianness::Big);
638
639        let _little = MemFlagsData::new().with_endianness(Endianness::Little);
640    }
641
642    #[test]
643    fn only_one_region() {
644        let region0 = AliasRegion::new(0);
645        let region1 = AliasRegion::new(1);
646        let flags = MemFlagsData::new().with_alias_region(Some(region0));
647        assert_eq!(flags.alias_region(), Some(region0));
648
649        let flags = flags.with_alias_region(Some(region1));
650        assert_eq!(flags.alias_region(), Some(region1));
651
652        let flags = flags.with_alias_region(None);
653        assert_eq!(flags.alias_region(), None);
654    }
655
656    #[test]
657    fn clear_removes_entries() {
658        let mut set = MemFlagsSet::new();
659        let trusted = set.insert(MemFlagsData::trusted()).unwrap();
660        let custom = MemFlagsData::new()
661            .with_endianness(Endianness::Big)
662            .with_alias_region(Some(AliasRegion::new(0)));
663        let custom_key = set.insert(custom).unwrap();
664        assert!(set.is_valid(trusted));
665        assert!(set.is_valid(custom_key));
666
667        set.clear();
668
669        assert!(!set.is_valid(trusted));
670        assert!(!set.is_valid(custom_key));
671        let trusted = set.insert(MemFlagsData::trusted()).unwrap();
672        assert_eq!(set[trusted], MemFlagsData::trusted());
673    }
674}
cranelift_codegen/ir/memflags.rs

cranelift_codegen/ir/
memflags.rs
