cranelift_interpreter/
value.rs

1//! The [DataValueExt] trait is an extension trait for [DataValue]. It provides a lot of functions
2//! used by the rest of the interpreter.
3
4#![expect(trivial_numeric_casts, reason = "macro-generated code")]
5
6use core::fmt::{self, Display, Formatter};
7use core::ops::Neg;
8use cranelift_codegen::data_value::{DataValue, DataValueCastFailure};
9use cranelift_codegen::ir::immediates::{Ieee16, Ieee32, Ieee64, Ieee128};
10use cranelift_codegen::ir::{Type, types};
11use thiserror::Error;
12
13use crate::step::{SimdVec, extractlanes};
14
15pub type ValueResult<T> = Result<T, ValueError>;
16
17pub trait DataValueExt: Sized {
18    // Identity.
19    fn int(n: i128, ty: Type) -> ValueResult<Self>;
20    fn into_int_signed(self) -> ValueResult<i128>;
21    fn into_int_unsigned(self) -> ValueResult<u128>;
22    fn float(n: u64, ty: Type) -> ValueResult<Self>;
23    fn into_float(self) -> ValueResult<f64>;
24    fn is_float(&self) -> bool;
25    fn is_nan(&self) -> ValueResult<bool>;
26    fn bool(b: bool, vec_elem: bool, ty: Type) -> ValueResult<Self>;
27    fn into_bool(self) -> ValueResult<bool>;
28    fn vector(v: [u8; 16], ty: Type) -> ValueResult<Self>;
29    fn into_array(&self) -> ValueResult<[u8; 16]>;
30    fn convert(self, kind: ValueConversionKind) -> ValueResult<Self>;
31    fn concat(self, other: Self) -> ValueResult<Self>;
32
33    fn is_negative(&self) -> ValueResult<bool>;
34    fn is_zero(&self) -> ValueResult<bool>;
35
36    fn umax(self, other: Self) -> ValueResult<Self>;
37    fn smax(self, other: Self) -> ValueResult<Self>;
38    fn umin(self, other: Self) -> ValueResult<Self>;
39    fn smin(self, other: Self) -> ValueResult<Self>;
40
41    // Comparison.
42    fn uno(&self, other: &Self) -> ValueResult<bool>;
43
44    // Arithmetic.
45    fn add(self, other: Self) -> ValueResult<Self>;
46    fn sub(self, other: Self) -> ValueResult<Self>;
47    fn mul(self, other: Self) -> ValueResult<Self>;
48    fn udiv(self, other: Self) -> ValueResult<Self>;
49    fn sdiv(self, other: Self) -> ValueResult<Self>;
50    fn urem(self, other: Self) -> ValueResult<Self>;
51    fn srem(self, other: Self) -> ValueResult<Self>;
52    fn sqrt(self) -> ValueResult<Self>;
53    fn fma(self, a: Self, b: Self) -> ValueResult<Self>;
54    fn abs(self) -> ValueResult<Self>;
55    fn uadd_checked(self, other: Self) -> ValueResult<Option<Self>>;
56    fn sadd_checked(self, other: Self) -> ValueResult<Option<Self>>;
57    fn uadd_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
58    fn sadd_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
59    fn usub_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
60    fn ssub_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
61    fn umul_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
62    fn smul_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
63
64    // Float operations
65    fn neg(self) -> ValueResult<Self>;
66    fn copysign(self, sign: Self) -> ValueResult<Self>;
67    fn ceil(self) -> ValueResult<Self>;
68    fn floor(self) -> ValueResult<Self>;
69    fn trunc(self) -> ValueResult<Self>;
70    fn nearest(self) -> ValueResult<Self>;
71
72    // Saturating arithmetic.
73    fn uadd_sat(self, other: Self) -> ValueResult<Self>;
74    fn sadd_sat(self, other: Self) -> ValueResult<Self>;
75    fn usub_sat(self, other: Self) -> ValueResult<Self>;
76    fn ssub_sat(self, other: Self) -> ValueResult<Self>;
77
78    // Bitwise.
79    fn shl(self, other: Self) -> ValueResult<Self>;
80    fn ushr(self, other: Self) -> ValueResult<Self>;
81    fn sshr(self, other: Self) -> ValueResult<Self>;
82    fn rotl(self, other: Self) -> ValueResult<Self>;
83    fn rotr(self, other: Self) -> ValueResult<Self>;
84    fn and(self, other: Self) -> ValueResult<Self>;
85    fn or(self, other: Self) -> ValueResult<Self>;
86    fn xor(self, other: Self) -> ValueResult<Self>;
87    fn not(self) -> ValueResult<Self>;
88
89    // Bit counting.
90    fn count_ones(self) -> ValueResult<Self>;
91    fn leading_ones(self) -> ValueResult<Self>;
92    fn leading_zeros(self) -> ValueResult<Self>;
93    fn trailing_zeros(self) -> ValueResult<Self>;
94    fn reverse_bits(self) -> ValueResult<Self>;
95    fn swap_bytes(self) -> ValueResult<Self>;
96
97    // An iterator over the lanes of a SIMD type
98    fn iter_lanes(&self, ty: Type) -> ValueResult<DataValueIterator>;
99}
100
101#[derive(Error, Debug, PartialEq)]
102pub enum ValueError {
103    #[error("unable to convert type {1} into class {0}")]
104    InvalidType(ValueTypeClass, Type),
105    #[error("unable to convert value into type {0}")]
106    InvalidValue(Type),
107    #[error("unable to convert to primitive integer")]
108    InvalidInteger(#[from] std::num::TryFromIntError),
109    #[error("unable to cast data value")]
110    InvalidDataValueCast(#[from] DataValueCastFailure),
111    #[error("performed a division by zero")]
112    IntegerDivisionByZero,
113    #[error("performed a operation that overflowed this integer type")]
114    IntegerOverflow,
115}
116
117#[derive(Debug, PartialEq)]
118pub enum ValueTypeClass {
119    Integer,
120    Boolean,
121    Float,
122    Vector,
123}
124
125impl Display for ValueTypeClass {
126    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
127        match self {
128            ValueTypeClass::Integer => write!(f, "integer"),
129            ValueTypeClass::Boolean => write!(f, "boolean"),
130            ValueTypeClass::Float => write!(f, "float"),
131            ValueTypeClass::Vector => write!(f, "vector"),
132        }
133    }
134}
135
136#[derive(Debug, Clone)]
137pub enum ValueConversionKind {
138    /// Throw a [ValueError] if an exact conversion to [Type] is not possible; e.g. in `i32` to
139    /// `i16`, convert `0x00001234` to `0x1234`.
140    Exact(Type),
141    /// Truncate the value to fit into the specified [Type]; e.g. in `i16` to `i8`, `0x1234` becomes
142    /// `0x34`.
143    Truncate(Type),
144    ///  Similar to Truncate, but extracts from the top of the value; e.g. in a `i32` to `u8`,
145    /// `0x12345678` becomes `0x12`.
146    ExtractUpper(Type),
147    /// Convert to a larger integer type, extending the sign bit; e.g. in `i8` to `i16`, `0xff`
148    /// becomes `0xffff`.
149    SignExtend(Type),
150    /// Convert to a larger integer type, extending with zeroes; e.g. in `i8` to `i16`, `0xff`
151    /// becomes `0x00ff`.
152    ZeroExtend(Type),
153    /// Convert a floating point number by rounding to the nearest possible value with ties to even.
154    /// See `fdemote`, e.g.
155    RoundNearestEven(Type),
156    /// Converts an integer into a boolean, zero integers are converted into a
157    /// `false`, while other integers are converted into `true`. Booleans are passed through.
158    ToBoolean,
159    /// Converts an integer into either -1 or zero.
160    Mask(Type),
161}
162
163/// Helper for creating match expressions over [DataValue].
164macro_rules! unary_match {
165    ( $op:ident($arg1:expr); [ $( $data_value_ty:ident ),* ]; [ $( $return_value_ty:ident ),* ] ) => {
166        match $arg1 {
167            $( DataValue::$data_value_ty(a) => {
168                Ok(DataValue::$data_value_ty($return_value_ty::try_from(a.$op()).unwrap()))
169            } )*
170            _ => unimplemented!()
171        }
172    };
173    ( $op:ident($arg1:expr); [ $( $data_value_ty:ident ),* ] ) => {
174        match $arg1 {
175            $( DataValue::$data_value_ty(a) => { Ok(DataValue::$data_value_ty(a.$op())) } )*
176            _ => unimplemented!()
177        }
178    };
179}
180macro_rules! binary_match {
181    ( $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ] ) => {
182        match ($arg1, $arg2) {
183            $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty(a.$op(*b))) } )*
184            _ => unimplemented!()
185        }
186    };
187    ( $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
188        match ($arg1, $arg2) {
189            $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty((*a as $op_type).$op(*b as $op_type) as _)) } )*
190            _ => unimplemented!()
191        }
192    };
193    ( option $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
194        match ($arg1, $arg2) {
195            $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok((*a as $op_type).$op(*b as $op_type).map(|v| DataValue::$data_value_ty(v as _))) } )*
196            _ => unimplemented!()
197        }
198    };
199    ( pair $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
200        match ($arg1, $arg2) {
201            $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => {
202                let (f, s) = (*a as $op_type).$op(*b as $op_type);
203                Ok((DataValue::$data_value_ty(f as _), s))
204            } )*
205            _ => unimplemented!()
206        }
207    };
208    ( $op:tt($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ] ) => {
209        match ($arg1, $arg2) {
210            $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty(a $op b)) } )*
211            _ => unimplemented!()
212        }
213    };
214    ( $op:tt($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
215        match ($arg1, $arg2) {
216            $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty(((*a as $op_type) $op (*b as $op_type)) as _)) } )*
217            _ => unimplemented!()
218        }
219    };
220    ( $op:tt($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $a_type:ty ),* ]; rhs: $rhs:tt,$rhs_type:ty ) => {
221        match ($arg1, $arg2) {
222            $( (DataValue::$data_value_ty(a), DataValue::$rhs(b)) => { Ok(DataValue::$data_value_ty((*a as $a_type).$op(*b as $rhs_type) as _)) } )*
223            _ => unimplemented!()
224        }
225    };
226}
227
228macro_rules! bitop {
229    ( $op:tt($arg1:expr, $arg2:expr) ) => {
230        Ok(match ($arg1, $arg2) {
231            (DataValue::I8(a), DataValue::I8(b)) => DataValue::I8(a $op b),
232            (DataValue::I16(a), DataValue::I16(b)) => DataValue::I16(a $op b),
233            (DataValue::I32(a), DataValue::I32(b)) => DataValue::I32(a $op b),
234            (DataValue::I64(a), DataValue::I64(b)) => DataValue::I64(a $op b),
235            (DataValue::I128(a), DataValue::I128(b)) => DataValue::I128(a $op b),
236            (DataValue::F16(a), DataValue::F16(b)) => DataValue::F16(a $op b),
237            (DataValue::F32(a), DataValue::F32(b)) => DataValue::F32(a $op b),
238            (DataValue::F64(a), DataValue::F64(b)) => DataValue::F64(a $op b),
239            (DataValue::F128(a), DataValue::F128(b)) => DataValue::F128(a $op b),
240            (DataValue::V64(a), DataValue::V64(b)) => {
241                let mut a2 = a.clone();
242                for (a, b) in a2.iter_mut().zip(b.iter()) {
243                    *a = *a $op *b;
244                }
245                DataValue::V64(a2)
246            }
247            (DataValue::V128(a), DataValue::V128(b)) => {
248                let mut a2 = a.clone();
249                for (a, b) in a2.iter_mut().zip(b.iter()) {
250                    *a = *a $op *b;
251                }
252                DataValue::V128(a2)
253            }
254            _ => unimplemented!(),
255        })
256    };
257}
258
259/// f16 arithmetic runs in f32: `to_f32` is exact and `from_f32_rne` rounds once.
260fn f16_binop(a: Ieee16, b: Ieee16, f: impl Fn(f32, f32) -> f32) -> DataValue {
261    DataValue::F16(Ieee16::from_f32_rne(f(a.to_f32(), b.to_f32())))
262}
263
264fn f16_unop(a: Ieee16, f: impl Fn(f32) -> f32) -> DataValue {
265    DataValue::F16(Ieee16::from_f32_rne(f(a.to_f32())))
266}
267
268impl DataValueExt for DataValue {
269    fn int(n: i128, ty: Type) -> ValueResult<Self> {
270        if ty.is_vector() {
271            // match ensures graceful failure since read_from_slice_ne()
272            // panics on anything other than 8 and 16 bytes
273            match ty.bytes() {
274                8 | 16 => Ok(DataValue::read_from_slice_ne(&n.to_ne_bytes(), ty)),
275                _ => Err(ValueError::InvalidType(ValueTypeClass::Vector, ty)),
276            }
277        } else if ty.is_int() {
278            DataValue::from_integer(n, ty).map_err(|_| ValueError::InvalidValue(ty))
279        } else {
280            Err(ValueError::InvalidType(ValueTypeClass::Integer, ty))
281        }
282    }
283
284    fn into_int_signed(self) -> ValueResult<i128> {
285        match self {
286            DataValue::I8(n) => Ok(n as i128),
287            DataValue::I16(n) => Ok(n as i128),
288            DataValue::I32(n) => Ok(n as i128),
289            DataValue::I64(n) => Ok(n as i128),
290            DataValue::I128(n) => Ok(n),
291            _ => Err(ValueError::InvalidType(ValueTypeClass::Integer, self.ty())),
292        }
293    }
294
295    fn into_int_unsigned(self) -> ValueResult<u128> {
296        match self {
297            DataValue::I8(n) => Ok(n as u8 as u128),
298            DataValue::I16(n) => Ok(n as u16 as u128),
299            DataValue::I32(n) => Ok(n as u32 as u128),
300            DataValue::I64(n) => Ok(n as u64 as u128),
301            DataValue::I128(n) => Ok(n as u128),
302            _ => Err(ValueError::InvalidType(ValueTypeClass::Integer, self.ty())),
303        }
304    }
305
306    fn float(bits: u64, ty: Type) -> ValueResult<Self> {
307        match ty {
308            types::F16 => Ok(DataValue::F16(Ieee16::with_bits(u16::try_from(bits)?))),
309            types::F32 => Ok(DataValue::F32(Ieee32::with_bits(u32::try_from(bits)?))),
310            types::F64 => Ok(DataValue::F64(Ieee64::with_bits(bits))),
311            _ => Err(ValueError::InvalidType(ValueTypeClass::Float, ty)),
312        }
313    }
314
315    fn into_float(self) -> ValueResult<f64> {
316        match self {
317            DataValue::F16(n) => Ok(n.to_f32() as f64),
318            DataValue::F32(n) => Ok(n.as_f32() as f64),
319            DataValue::F64(n) => Ok(n.as_f64()),
320            _ => Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty())),
321        }
322    }
323
324    fn is_float(&self) -> bool {
325        match self {
326            DataValue::F16(_) | DataValue::F32(_) | DataValue::F64(_) | DataValue::F128(_) => true,
327            _ => false,
328        }
329    }
330
331    fn is_nan(&self) -> ValueResult<bool> {
332        match self {
333            DataValue::F16(f) => Ok(f.is_nan()),
334            DataValue::F32(f) => Ok(f.is_nan()),
335            DataValue::F64(f) => Ok(f.is_nan()),
336            _ => Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty())),
337        }
338    }
339
340    fn bool(b: bool, vec_elem: bool, ty: Type) -> ValueResult<Self> {
341        assert!(ty.is_int());
342        macro_rules! make_bool {
343            ($ty:ident) => {
344                Ok(DataValue::$ty(if b {
345                    if vec_elem { -1 } else { 1 }
346                } else {
347                    0
348                }))
349            };
350        }
351
352        match ty {
353            types::I8 => make_bool!(I8),
354            types::I16 => make_bool!(I16),
355            types::I32 => make_bool!(I32),
356            types::I64 => make_bool!(I64),
357            types::I128 => make_bool!(I128),
358            _ => Err(ValueError::InvalidType(ValueTypeClass::Integer, ty)),
359        }
360    }
361
362    fn into_bool(self) -> ValueResult<bool> {
363        match self {
364            DataValue::I8(b) => Ok(b != 0),
365            DataValue::I16(b) => Ok(b != 0),
366            DataValue::I32(b) => Ok(b != 0),
367            DataValue::I64(b) => Ok(b != 0),
368            DataValue::I128(b) => Ok(b != 0),
369            _ => Err(ValueError::InvalidType(ValueTypeClass::Boolean, self.ty())),
370        }
371    }
372
373    fn vector(v: [u8; 16], ty: Type) -> ValueResult<Self> {
374        assert!(ty.is_vector() && [2, 4, 8, 16].contains(&ty.bytes()));
375        match ty.bytes() {
376            16 => Ok(DataValue::V128(v)),
377            8 => Ok(DataValue::V64(v[..8].try_into().unwrap())),
378            4 => Ok(DataValue::V32(v[..4].try_into().unwrap())),
379            2 => Ok(DataValue::V16(v[..2].try_into().unwrap())),
380            _ => unreachable!(),
381        }
382    }
383
384    fn into_array(&self) -> ValueResult<[u8; 16]> {
385        match *self {
386            DataValue::V128(v) => Ok(v),
387            DataValue::V64(v) => {
388                let mut v128 = [0; 16];
389                v128[..8].clone_from_slice(&v);
390                Ok(v128)
391            }
392            DataValue::V32(v) => {
393                let mut v128 = [0; 16];
394                v128[..4].clone_from_slice(&v);
395                Ok(v128)
396            }
397            DataValue::V16(v) => {
398                let mut v128 = [0; 16];
399                v128[..2].clone_from_slice(&v);
400                Ok(v128)
401            }
402            _ => Err(ValueError::InvalidType(ValueTypeClass::Vector, self.ty())),
403        }
404    }
405
406    fn convert(self, kind: ValueConversionKind) -> ValueResult<Self> {
407        Ok(match kind {
408            ValueConversionKind::Exact(ty) => match (self, ty) {
409                // TODO a lot to do here: from bmask to ireduce to bitcast...
410                (val, ty) if val.ty().is_int() && ty.is_int() => {
411                    DataValue::from_integer(val.into_int_signed()?, ty)?
412                }
413                (DataValue::I16(n), types::F16) => DataValue::F16(Ieee16::with_bits(n as u16)),
414                (DataValue::I32(n), types::F32) => DataValue::F32(f32::from_bits(n as u32).into()),
415                (DataValue::I64(n), types::F64) => DataValue::F64(f64::from_bits(n as u64).into()),
416                (DataValue::I128(n), types::F128) => DataValue::F128(Ieee128::with_bits(n as u128)),
417                (DataValue::F16(n), types::I16) => DataValue::I16(n.bits() as i16),
418                (DataValue::F32(n), types::I32) => DataValue::I32(n.bits() as i32),
419                (DataValue::F64(n), types::I64) => DataValue::I64(n.bits() as i64),
420                (DataValue::F128(n), types::I128) => DataValue::I128(n.bits() as i128),
421                (DataValue::F16(n), types::F32) => DataValue::F32(n.to_f32().into()),
422                (DataValue::F16(n), types::F64) => DataValue::F64(f64::from(n.to_f32()).into()),
423                (DataValue::F32(n), types::F64) => DataValue::F64((n.as_f32() as f64).into()),
424                (dv, t) if (t.is_int() || t.is_float()) && dv.ty() == t => dv,
425                (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
426            },
427            ValueConversionKind::Truncate(ty) => {
428                assert!(
429                    ty.is_int(),
430                    "unimplemented conversion: {} -> {:?}",
431                    self.ty(),
432                    kind
433                );
434
435                let mask = (1 << (ty.bytes() * 8)) - 1i128;
436                let truncated = self.into_int_signed()? & mask;
437                Self::from_integer(truncated, ty)?
438            }
439            ValueConversionKind::ExtractUpper(ty) => {
440                assert!(
441                    ty.is_int(),
442                    "unimplemented conversion: {} -> {:?}",
443                    self.ty(),
444                    kind
445                );
446
447                let shift_amt = (self.ty().bytes() * 8) - (ty.bytes() * 8);
448                let mask = (1 << (ty.bytes() * 8)) - 1i128;
449                let shifted_mask = mask << shift_amt;
450
451                let extracted = (self.into_int_signed()? & shifted_mask) >> shift_amt;
452                Self::from_integer(extracted, ty)?
453            }
454            ValueConversionKind::SignExtend(ty) => match (self, ty) {
455                (DataValue::I8(n), types::I16) => DataValue::I16(n as i16),
456                (DataValue::I8(n), types::I32) => DataValue::I32(n as i32),
457                (DataValue::I8(n), types::I64) => DataValue::I64(n as i64),
458                (DataValue::I8(n), types::I128) => DataValue::I128(n as i128),
459                (DataValue::I16(n), types::I32) => DataValue::I32(n as i32),
460                (DataValue::I16(n), types::I64) => DataValue::I64(n as i64),
461                (DataValue::I16(n), types::I128) => DataValue::I128(n as i128),
462                (DataValue::I32(n), types::I64) => DataValue::I64(n as i64),
463                (DataValue::I32(n), types::I128) => DataValue::I128(n as i128),
464                (DataValue::I64(n), types::I128) => DataValue::I128(n as i128),
465                (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
466            },
467            ValueConversionKind::ZeroExtend(ty) => match (self, ty) {
468                (DataValue::I8(n), types::I16) => DataValue::I16(n as u8 as i16),
469                (DataValue::I8(n), types::I32) => DataValue::I32(n as u8 as i32),
470                (DataValue::I8(n), types::I64) => DataValue::I64(n as u8 as i64),
471                (DataValue::I8(n), types::I128) => DataValue::I128(n as u8 as i128),
472                (DataValue::I16(n), types::I32) => DataValue::I32(n as u16 as i32),
473                (DataValue::I16(n), types::I64) => DataValue::I64(n as u16 as i64),
474                (DataValue::I16(n), types::I128) => DataValue::I128(n as u16 as i128),
475                (DataValue::I32(n), types::I64) => DataValue::I64(n as u32 as i64),
476                (DataValue::I32(n), types::I128) => DataValue::I128(n as u32 as i128),
477                (DataValue::I64(n), types::I128) => DataValue::I128(n as u64 as i128),
478                (from, to) if from.ty() == to => from,
479                (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
480            },
481            ValueConversionKind::RoundNearestEven(ty) => match (self, ty) {
482                (DataValue::F64(n), types::F32) => DataValue::F32(Ieee32::from(n.as_f64() as f32)),
483                (DataValue::F32(n), types::F16) => DataValue::F16(Ieee16::from_f32_rne(n.as_f32())),
484                (DataValue::F64(n), types::F16) => DataValue::F16(Ieee16::from_f64_rne(n.as_f64())),
485                (s, _) => unimplemented!("conversion: {} -> {:?}", s.ty(), kind),
486            },
487            ValueConversionKind::ToBoolean => match self.ty() {
488                ty if ty.is_int() => {
489                    DataValue::I8(if self.into_int_signed()? != 0 { 1 } else { 0 })
490                }
491                ty => unimplemented!("conversion: {} -> {:?}", ty, kind),
492            },
493            ValueConversionKind::Mask(ty) => {
494                let b = self.into_bool()?;
495                Self::bool(b, true, ty).unwrap()
496            }
497        })
498    }
499
500    fn concat(self, other: Self) -> ValueResult<Self> {
501        match (self, other) {
502            (DataValue::I64(lhs), DataValue::I64(rhs)) => Ok(DataValue::I128(
503                (((lhs as u64) as u128) | (((rhs as u64) as u128) << 64)) as i128,
504            )),
505            (lhs, rhs) => unimplemented!("concat: {} -> {}", lhs.ty(), rhs.ty()),
506        }
507    }
508
509    fn is_negative(&self) -> ValueResult<bool> {
510        match self {
511            DataValue::F16(f) => Ok(f.is_negative()),
512            DataValue::F32(f) => Ok(f.is_negative()),
513            DataValue::F64(f) => Ok(f.is_negative()),
514            _ => Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty())),
515        }
516    }
517
518    fn is_zero(&self) -> ValueResult<bool> {
519        match self {
520            DataValue::I8(f) => Ok(*f == 0),
521            DataValue::I16(f) => Ok(*f == 0),
522            DataValue::I32(f) => Ok(*f == 0),
523            DataValue::I64(f) => Ok(*f == 0),
524            DataValue::I128(f) => Ok(*f == 0),
525            DataValue::F16(f) => Ok(f.is_zero()),
526            DataValue::F32(f) => Ok(f.is_zero()),
527            DataValue::F64(f) => Ok(f.is_zero()),
528            DataValue::F128(f) => Ok(f.is_zero()),
529            DataValue::V16(_) | DataValue::V32(_) | DataValue::V64(_) | DataValue::V128(_) => {
530                Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty()))
531            }
532        }
533    }
534
535    fn umax(self, other: Self) -> ValueResult<Self> {
536        let lhs = self.clone().into_int_unsigned()?;
537        let rhs = other.clone().into_int_unsigned()?;
538        if lhs > rhs { Ok(self) } else { Ok(other) }
539    }
540
541    fn smax(self, other: Self) -> ValueResult<Self> {
542        if self > other { Ok(self) } else { Ok(other) }
543    }
544
545    fn umin(self, other: Self) -> ValueResult<Self> {
546        let lhs = self.clone().into_int_unsigned()?;
547        let rhs = other.clone().into_int_unsigned()?;
548        if lhs < rhs { Ok(self) } else { Ok(other) }
549    }
550
551    fn smin(self, other: Self) -> ValueResult<Self> {
552        if self < other { Ok(self) } else { Ok(other) }
553    }
554
555    fn uno(&self, other: &Self) -> ValueResult<bool> {
556        Ok(self.is_nan()? || other.is_nan()?)
557    }
558
559    fn add(self, other: Self) -> ValueResult<Self> {
560        if self.is_float() {
561            if let (DataValue::F16(a), DataValue::F16(b)) = (&self, &other) {
562                return Ok(f16_binop(*a, *b, |x, y| x + y));
563            }
564            binary_match!(+(self, other); [F32, F64])
565        } else {
566            binary_match!(wrapping_add(&self, &other); [I8, I16, I32, I64, I128])
567        }
568    }
569
570    fn sub(self, other: Self) -> ValueResult<Self> {
571        if self.is_float() {
572            if let (DataValue::F16(a), DataValue::F16(b)) = (&self, &other) {
573                return Ok(f16_binop(*a, *b, |x, y| x - y));
574            }
575            binary_match!(-(self, other); [F32, F64])
576        } else {
577            binary_match!(wrapping_sub(&self, &other); [I8, I16, I32, I64, I128])
578        }
579    }
580
581    fn mul(self, other: Self) -> ValueResult<Self> {
582        if self.is_float() {
583            if let (DataValue::F16(a), DataValue::F16(b)) = (&self, &other) {
584                return Ok(f16_binop(*a, *b, |x, y| x * y));
585            }
586            binary_match!(*(self, other); [F32, F64])
587        } else {
588            binary_match!(wrapping_mul(&self, &other); [I8, I16, I32, I64, I128])
589        }
590    }
591
592    fn sdiv(self, other: Self) -> ValueResult<Self> {
593        if self.is_float() {
594            if let (DataValue::F16(a), DataValue::F16(b)) = (&self, &other) {
595                return Ok(f16_binop(*a, *b, |x, y| x / y));
596            }
597            return binary_match!(/(self, other); [F32, F64]);
598        }
599
600        let denominator = other.clone().into_int_signed()?;
601
602        // Check if we are dividing INT_MIN / -1. This causes an integer overflow trap.
603        let min = DataValueExt::int(1i128 << (self.ty().bits() - 1), self.ty())?;
604        if self == min && denominator == -1 {
605            return Err(ValueError::IntegerOverflow);
606        }
607
608        if denominator == 0 {
609            return Err(ValueError::IntegerDivisionByZero);
610        }
611
612        binary_match!(/(&self, &other); [I8, I16, I32, I64, I128])
613    }
614
615    fn udiv(self, other: Self) -> ValueResult<Self> {
616        if self.is_float() {
617            if let (DataValue::F16(a), DataValue::F16(b)) = (&self, &other) {
618                return Ok(f16_binop(*a, *b, |x, y| x / y));
619            }
620            return binary_match!(/(self, other); [F32, F64]);
621        }
622
623        let denominator = other.clone().into_int_unsigned()?;
624
625        if denominator == 0 {
626            return Err(ValueError::IntegerDivisionByZero);
627        }
628
629        binary_match!(/(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
630    }
631
632    fn srem(self, other: Self) -> ValueResult<Self> {
633        let denominator = other.clone().into_int_signed()?;
634
635        if denominator == 0 {
636            return Err(ValueError::IntegerDivisionByZero);
637        }
638
639        // The remainder of any value divided by -1 is 0. Unlike `sdiv`, this
640        // does not trap on INT_MIN / -1; return early to avoid Rust's
641        // overflow panic on `INT_MIN % -1`.
642        if denominator == -1 {
643            return DataValueExt::int(0, self.ty());
644        }
645
646        binary_match!(%(&self, &other); [I8, I16, I32, I64, I128])
647    }
648
649    fn urem(self, other: Self) -> ValueResult<Self> {
650        let denominator = other.clone().into_int_unsigned()?;
651
652        if denominator == 0 {
653            return Err(ValueError::IntegerDivisionByZero);
654        }
655
656        binary_match!(%(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
657    }
658
659    fn sqrt(self) -> ValueResult<Self> {
660        if let DataValue::F16(a) = &self {
661            return Ok(f16_unop(*a, f32::sqrt));
662        }
663        unary_match!(sqrt(&self); [F32, F64]; [Ieee32, Ieee64])
664    }
665
666    fn fma(self, b: Self, c: Self) -> ValueResult<Self> {
667        match (self, b, c) {
668            (DataValue::F16(a), DataValue::F16(b), DataValue::F16(c)) => {
669                // f64, not f32: a fused multiply-add must see the unrounded
670                // product and sum, which needs more than 24 bits.
671                let (x, y, z) = (a.to_f32() as f64, b.to_f32() as f64, c.to_f32() as f64);
672
673                #[cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))]
674                let res = libm::fma(x, y, z);
675
676                #[cfg(not(all(
677                    target_arch = "x86_64",
678                    target_os = "windows",
679                    target_env = "gnu"
680                )))]
681                let res = x.mul_add(y, z);
682
683                Ok(DataValue::F16(Ieee16::from_f64_rne(res)))
684            }
685            (DataValue::F32(a), DataValue::F32(b), DataValue::F32(c)) => {
686                // The `fma` function for `x86_64-pc-windows-gnu` is incorrect. Use `libm`'s instead.
687                // See: https://github.com/bytecodealliance/wasmtime/issues/4512
688                #[cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))]
689                let res = libm::fmaf(a.as_f32(), b.as_f32(), c.as_f32());
690
691                #[cfg(not(all(
692                    target_arch = "x86_64",
693                    target_os = "windows",
694                    target_env = "gnu"
695                )))]
696                let res = a.as_f32().mul_add(b.as_f32(), c.as_f32());
697
698                Ok(DataValue::F32(res.into()))
699            }
700            (DataValue::F64(a), DataValue::F64(b), DataValue::F64(c)) => {
701                #[cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))]
702                let res = libm::fma(a.as_f64(), b.as_f64(), c.as_f64());
703
704                #[cfg(not(all(
705                    target_arch = "x86_64",
706                    target_os = "windows",
707                    target_env = "gnu"
708                )))]
709                let res = a.as_f64().mul_add(b.as_f64(), c.as_f64());
710
711                Ok(DataValue::F64(res.into()))
712            }
713            (a, _b, _c) => Err(ValueError::InvalidType(ValueTypeClass::Float, a.ty())),
714        }
715    }
716
717    fn abs(self) -> ValueResult<Self> {
718        unary_match!(abs(&self); [F16, F32, F64])
719    }
720
721    fn sadd_checked(self, other: Self) -> ValueResult<Option<Self>> {
722        binary_match!(option checked_add(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
723    }
724
725    fn uadd_checked(self, other: Self) -> ValueResult<Option<Self>> {
726        binary_match!(option checked_add(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
727    }
728
729    fn sadd_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
730        binary_match!(pair overflowing_add(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
731    }
732
733    fn uadd_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
734        binary_match!(pair overflowing_add(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
735    }
736
737    fn ssub_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
738        binary_match!(pair overflowing_sub(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
739    }
740
741    fn usub_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
742        binary_match!(pair overflowing_sub(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
743    }
744
745    fn smul_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
746        binary_match!(pair overflowing_mul(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
747    }
748
749    fn umul_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
750        binary_match!(pair overflowing_mul(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
751    }
752
753    fn neg(self) -> ValueResult<Self> {
754        unary_match!(neg(&self); [F16, F32, F64])
755    }
756
757    fn copysign(self, sign: Self) -> ValueResult<Self> {
758        binary_match!(copysign(&self, &sign); [F16, F32, F64])
759    }
760
761    fn ceil(self) -> ValueResult<Self> {
762        if let DataValue::F16(a) = &self {
763            return Ok(f16_unop(*a, f32::ceil));
764        }
765        unary_match!(ceil(&self); [F32, F64])
766    }
767
768    fn floor(self) -> ValueResult<Self> {
769        if let DataValue::F16(a) = &self {
770            return Ok(f16_unop(*a, f32::floor));
771        }
772        unary_match!(floor(&self); [F32, F64])
773    }
774
775    fn trunc(self) -> ValueResult<Self> {
776        if let DataValue::F16(a) = &self {
777            return Ok(f16_unop(*a, f32::trunc));
778        }
779        unary_match!(trunc(&self); [F32, F64])
780    }
781
782    fn nearest(self) -> ValueResult<Self> {
783        if let DataValue::F16(a) = &self {
784            return Ok(f16_unop(*a, f32::round_ties_even));
785        }
786        unary_match!(round_ties_even(&self); [F32, F64])
787    }
788
789    fn sadd_sat(self, other: Self) -> ValueResult<Self> {
790        binary_match!(saturating_add(self, &other); [I8, I16, I32, I64, I128])
791    }
792
793    fn uadd_sat(self, other: Self) -> ValueResult<Self> {
794        binary_match!(saturating_add(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
795    }
796
797    fn ssub_sat(self, other: Self) -> ValueResult<Self> {
798        binary_match!(saturating_sub(self, &other); [I8, I16, I32, I64, I128])
799    }
800
801    fn usub_sat(self, other: Self) -> ValueResult<Self> {
802        binary_match!(saturating_sub(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
803    }
804
805    fn shl(self, other: Self) -> ValueResult<Self> {
806        let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
807        binary_match!(wrapping_shl(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
808    }
809
810    fn ushr(self, other: Self) -> ValueResult<Self> {
811        let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
812        binary_match!(wrapping_shr(&self, &amt); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128]; rhs: I32,u32)
813    }
814
815    fn sshr(self, other: Self) -> ValueResult<Self> {
816        let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
817        binary_match!(wrapping_shr(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
818    }
819
820    fn rotl(self, other: Self) -> ValueResult<Self> {
821        let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
822        binary_match!(rotate_left(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
823    }
824
825    fn rotr(self, other: Self) -> ValueResult<Self> {
826        let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
827        binary_match!(rotate_right(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
828    }
829
830    fn and(self, other: Self) -> ValueResult<Self> {
831        bitop!(&(self, other))
832    }
833
834    fn or(self, other: Self) -> ValueResult<Self> {
835        bitop!(|(self, other))
836    }
837
838    fn xor(self, other: Self) -> ValueResult<Self> {
839        bitop!(^(self, other))
840    }
841
842    fn not(self) -> ValueResult<Self> {
843        Ok(match self {
844            DataValue::I8(a) => DataValue::I8(!a),
845            DataValue::I16(a) => DataValue::I16(!a),
846            DataValue::I32(a) => DataValue::I32(!a),
847            DataValue::I64(a) => DataValue::I64(!a),
848            DataValue::I128(a) => DataValue::I128(!a),
849            DataValue::F16(a) => DataValue::F16(!a),
850            DataValue::F32(a) => DataValue::F32(!a),
851            DataValue::F64(a) => DataValue::F64(!a),
852            DataValue::F128(a) => DataValue::F128(!a),
853            DataValue::V64(mut a) => {
854                for byte in a.iter_mut() {
855                    *byte = !*byte;
856                }
857                DataValue::V64(a)
858            }
859            DataValue::V128(mut a) => {
860                for byte in a.iter_mut() {
861                    *byte = !*byte;
862                }
863                DataValue::V128(a)
864            }
865            _ => unimplemented!(),
866        })
867    }
868
869    fn count_ones(self) -> ValueResult<Self> {
870        unary_match!(count_ones(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
871    }
872
873    fn leading_ones(self) -> ValueResult<Self> {
874        unary_match!(leading_ones(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
875    }
876
877    fn leading_zeros(self) -> ValueResult<Self> {
878        unary_match!(leading_zeros(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
879    }
880
881    fn trailing_zeros(self) -> ValueResult<Self> {
882        unary_match!(trailing_zeros(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
883    }
884
885    fn reverse_bits(self) -> ValueResult<Self> {
886        unary_match!(reverse_bits(&self); [I8, I16, I32, I64, I128])
887    }
888
889    fn swap_bytes(self) -> ValueResult<Self> {
890        unary_match!(swap_bytes(&self); [I16, I32, I64, I128])
891    }
892
893    fn iter_lanes(&self, ty: Type) -> ValueResult<DataValueIterator> {
894        DataValueIterator::new(self, ty)
895    }
896}
897
898/// Iterator for DataValue's
899pub struct DataValueIterator {
900    ty: Type,
901    v: SimdVec<DataValue>,
902    idx: usize,
903}
904
905impl DataValueIterator {
906    fn new(dv: &DataValue, ty: Type) -> Result<Self, ValueError> {
907        match extractlanes(dv, ty) {
908            Ok(v) => return Ok(Self { ty, v, idx: 0 }),
909            Err(err) => return Err(err),
910        }
911    }
912}
913
914impl Iterator for DataValueIterator {
915    type Item = DataValue;
916
917    fn next(&mut self) -> Option<Self::Item> {
918        if self.idx >= self.ty.lane_count() as usize {
919            return None;
920        }
921
922        let dv = self.v[self.idx].clone();
923        self.idx += 1;
924        Some(dv)
925    }
926}
927
928#[cfg(test)]
929mod test {
930    use super::*;
931
932    #[test]
933    fn test_iterator_v128() {
934        let dv = DataValue::V128([99, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
935        assert_eq!(simd_sum(dv, types::I8X16), 219);
936    }
937
938    #[test]
939    fn test_iterator_v128_empty() {
940        let dv = DataValue::V128([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
941        assert_eq!(simd_sum(dv, types::I8X16), 0);
942    }
943
944    #[test]
945    fn test_iterator_v128_ones() {
946        let dv = DataValue::V128([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
947        assert_eq!(simd_sum(dv, types::I8X16), 16);
948    }
949
950    #[test]
951    fn test_iterator_v64_empty() {
952        let dv = DataValue::V64([0, 0, 0, 0, 0, 0, 0, 0]);
953        assert_eq!(simd_sum(dv, types::I8X8), 0);
954    }
955    #[test]
956    fn test_iterator_v64_ones() {
957        let dv = DataValue::V64([1, 1, 1, 1, 1, 1, 1, 1]);
958        assert_eq!(simd_sum(dv, types::I8X8), 8);
959    }
960    #[test]
961    fn test_iterator_v64() {
962        let dv = DataValue::V64([10, 20, 30, 40, 50, 60, 70, 80]);
963        assert_eq!(simd_sum(dv, types::I8X8), 360);
964    }
965
966    fn simd_sum(dv: DataValue, ty: types::Type) -> i128 {
967        let itr = dv.iter_lanes(ty).unwrap();
968
969        itr.map(|e| {
970            if let Some(v) = e.into_int_signed().ok() {
971                v
972            } else {
973                0
974            }
975        })
976        .sum()
977    }
978}
cranelift_interpreter/value.rs

cranelift_interpreter/
value.rs
