winch_codegen/isa/x64/
abi.rs

1use super::regs;
2use crate::{
3    abi::{align_to, ABIOperand, ABIParams, ABIResults, ABISig, ParamsOrReturns, ABI},
4    codegen::CodeGenError,
5    isa::{reg::Reg, CallingConvention},
6    RegIndexEnv,
7};
8use anyhow::{bail, Result};
9use wasmtime_environ::{WasmHeapType, WasmRefType, WasmValType};
10
11#[derive(Default)]
12pub(crate) struct X64ABI;
13
14impl ABI for X64ABI {
15    fn stack_align() -> u8 {
16        16
17    }
18
19    fn call_stack_align() -> u8 {
20        16
21    }
22
23    fn arg_base_offset() -> u8 {
24        // Two 8-byte slots, one for the return address and another
25        // one for the frame pointer.
26        // ┌──────────┬───────── Argument base
27        // │   Ret    │
28        // │   Addr   │
29        // ├──────────┼
30        // │          │
31        // │   FP     │
32        // └──────────┴
33        16
34    }
35
36    fn initial_frame_size() -> u8 {
37        // The initial frame size is equal to the space allocated to save the
38        // return address and the frame pointer.
39        Self::arg_base_offset()
40    }
41
42    fn word_bits() -> u8 {
43        64
44    }
45
46    fn sig_from(
47        params: &[WasmValType],
48        returns: &[WasmValType],
49        call_conv: &CallingConvention,
50    ) -> Result<ABISig> {
51        assert!(call_conv.is_fastcall() || call_conv.is_systemv() || call_conv.is_default());
52        let is_fastcall = call_conv.is_fastcall();
53        // In the fastcall calling convention, the callee gets a contiguous
54        // stack area of 32 bytes (4 register arguments) just before its frame.
55        // See
56        // https://learn.microsoft.com/en-us/cpp/build/stack-usage?view=msvc-170#stack-allocation
57        let (params_stack_offset, mut params_index_env) = if is_fastcall {
58            (32, RegIndexEnv::with_absolute_limit(4))
59        } else {
60            (0, RegIndexEnv::with_limits_per_class(6, 8))
61        };
62
63        let results = Self::abi_results(returns, call_conv)?;
64        let params = ABIParams::from::<_, Self>(
65            params,
66            params_stack_offset,
67            results.on_stack(),
68            |ty, stack_offset| {
69                Self::to_abi_operand(
70                    ty,
71                    stack_offset,
72                    &mut params_index_env,
73                    call_conv,
74                    ParamsOrReturns::Params,
75                )
76            },
77        )?;
78
79        Ok(ABISig::new(*call_conv, params, results))
80    }
81
82    fn abi_results(returns: &[WasmValType], call_conv: &CallingConvention) -> Result<ABIResults> {
83        assert!(call_conv.is_default() || call_conv.is_fastcall() || call_conv.is_systemv());
84        // Use absolute count for results given that for Winch's
85        // default CallingConvention only one register is used for results
86        // independent of the register class.
87        // In the case of 2+ results, the rest are passed in the stack,
88        // similar to how Wasmtime handles multi-value returns.
89        let mut results_index_env = RegIndexEnv::with_absolute_limit(1);
90        ABIResults::from(returns, call_conv, |ty, offset| {
91            Self::to_abi_operand(
92                ty,
93                offset,
94                &mut results_index_env,
95                call_conv,
96                ParamsOrReturns::Returns,
97            )
98        })
99    }
100
101    fn scratch_for(ty: &WasmValType) -> Reg {
102        match ty {
103            WasmValType::I32
104            | WasmValType::I64
105            | WasmValType::Ref(WasmRefType {
106                heap_type: WasmHeapType::Func,
107                ..
108            }) => regs::scratch(),
109            WasmValType::F32 | WasmValType::F64 | WasmValType::V128 => regs::scratch_xmm(),
110            _ => unimplemented!(),
111        }
112    }
113
114    fn vmctx_reg() -> Reg {
115        regs::vmctx()
116    }
117
118    fn stack_slot_size() -> u8 {
119        // Winch default calling convention follows SysV calling convention so
120        // we use one 8 byte slot for values that are smaller or equal to 8
121        // bytes in size and 2 8 byte slots for values that are 128 bits.
122        // See Section 3.2.3 in
123        // https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf for further
124        // details.
125        Self::word_bytes()
126    }
127
128    fn sizeof(ty: &WasmValType) -> u8 {
129        match ty {
130            WasmValType::Ref(rt) => match rt.heap_type {
131                WasmHeapType::Func | WasmHeapType::Extern => Self::word_bytes(),
132                ht => unimplemented!("Support for WasmHeapType: {ht}"),
133            },
134            WasmValType::F64 | WasmValType::I64 => Self::word_bytes(),
135            WasmValType::F32 | WasmValType::I32 => Self::word_bytes() / 2,
136            WasmValType::V128 => Self::word_bytes() * 2,
137        }
138    }
139}
140
141impl X64ABI {
142    fn to_abi_operand(
143        wasm_arg: &WasmValType,
144        stack_offset: u32,
145        index_env: &mut RegIndexEnv,
146        call_conv: &CallingConvention,
147        params_or_returns: ParamsOrReturns,
148    ) -> Result<(ABIOperand, u32)> {
149        let (reg, ty) = match wasm_arg {
150            ty @ WasmValType::Ref(rt) => match rt.heap_type {
151                WasmHeapType::Func | WasmHeapType::Extern => (
152                    Self::int_reg_for(index_env.next_gpr(), call_conv, params_or_returns),
153                    ty,
154                ),
155                _ => bail!(CodeGenError::unsupported_wasm_type()),
156            },
157
158            ty @ (WasmValType::I32 | WasmValType::I64) => (
159                Self::int_reg_for(index_env.next_gpr(), call_conv, params_or_returns),
160                ty,
161            ),
162
163            // v128 also uses an XMM register (that is, an fpr).
164            ty @ (WasmValType::F32 | WasmValType::F64 | WasmValType::V128) => (
165                Self::float_reg_for(index_env.next_fpr(), call_conv, params_or_returns),
166                ty,
167            ),
168        };
169
170        let ty_size = <Self as ABI>::sizeof(wasm_arg);
171        let default = || {
172            let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size as u32);
173            let slot_size = Self::stack_slot_size();
174            // Stack slots for parameters are aligned to a fixed slot size,
175            // in the case of x64, 8 bytes. Except if they are v128 values,
176            // in which case they use 16 bytes.
177            // Stack slots for returns are type-size aligned.
178            let next_stack = if params_or_returns == ParamsOrReturns::Params {
179                let alignment = if *ty == WasmValType::V128 {
180                    ty_size
181                } else {
182                    slot_size
183                };
184                align_to(stack_offset, alignment as u32) + (alignment as u32)
185            } else {
186                // For the default calling convention, we don't type-size align,
187                // given that results on the stack must match spills generated
188                // from within the compiler, which are not type-size aligned.
189                if call_conv.is_default() {
190                    stack_offset + (ty_size as u32)
191                } else {
192                    align_to(stack_offset, ty_size as u32) + (ty_size as u32)
193                }
194            };
195            (arg, next_stack)
196        };
197
198        Ok(reg.map_or_else(default, |reg| {
199            (ABIOperand::reg(reg, *ty, ty_size as u32), stack_offset)
200        }))
201    }
202
203    fn int_reg_for(
204        index: Option<u8>,
205        call_conv: &CallingConvention,
206        params_or_returns: ParamsOrReturns,
207    ) -> Option<Reg> {
208        use ParamsOrReturns::*;
209
210        let index = match index {
211            None => return None,
212            Some(index) => index,
213        };
214
215        if call_conv.is_fastcall() {
216            return match (index, params_or_returns) {
217                (0, Params) => Some(regs::rcx()),
218                (1, Params) => Some(regs::rdx()),
219                (2, Params) => Some(regs::r8()),
220                (3, Params) => Some(regs::r9()),
221                (0, Returns) => Some(regs::rax()),
222                _ => None,
223            };
224        }
225
226        if call_conv.is_systemv() || call_conv.is_default() {
227            return match (index, params_or_returns) {
228                (0, Params) => Some(regs::rdi()),
229                (1, Params) => Some(regs::rsi()),
230                (2, Params) => Some(regs::rdx()),
231                (3, Params) => Some(regs::rcx()),
232                (4, Params) => Some(regs::r8()),
233                (5, Params) => Some(regs::r9()),
234                (0, Returns) => Some(regs::rax()),
235                _ => None,
236            };
237        }
238
239        None
240    }
241
242    fn float_reg_for(
243        index: Option<u8>,
244        call_conv: &CallingConvention,
245        params_or_returns: ParamsOrReturns,
246    ) -> Option<Reg> {
247        use ParamsOrReturns::*;
248
249        let index = match index {
250            None => return None,
251            Some(index) => index,
252        };
253
254        if call_conv.is_fastcall() {
255            return match (index, params_or_returns) {
256                (0, Params) => Some(regs::xmm0()),
257                (1, Params) => Some(regs::xmm1()),
258                (2, Params) => Some(regs::xmm2()),
259                (3, Params) => Some(regs::xmm3()),
260                (0, Returns) => Some(regs::xmm0()),
261                _ => None,
262            };
263        }
264
265        if call_conv.is_systemv() || call_conv.is_default() {
266            return match (index, params_or_returns) {
267                (0, Params) => Some(regs::xmm0()),
268                (1, Params) => Some(regs::xmm1()),
269                (2, Params) => Some(regs::xmm2()),
270                (3, Params) => Some(regs::xmm3()),
271                (4, Params) => Some(regs::xmm4()),
272                (5, Params) => Some(regs::xmm5()),
273                (6, Params) => Some(regs::xmm6()),
274                (7, Params) => Some(regs::xmm7()),
275                (0, Returns) => Some(regs::xmm0()),
276                _ => None,
277            };
278        }
279
280        None
281    }
282}
283
284#[cfg(test)]
285mod tests {
286    use super::X64ABI;
287    use crate::{
288        abi::{ABIOperand, ABI},
289        isa::{reg::Reg, x64::regs, CallingConvention},
290    };
291
292    use anyhow::Result;
293
294    use wasmtime_environ::{
295        WasmFuncType,
296        WasmValType::{self, *},
297    };
298
299    #[test]
300    fn int_abi_sig() -> Result<()> {
301        let wasm_sig =
302            WasmFuncType::new([I32, I64, I32, I64, I32, I32, I64, I32].into(), [].into());
303
304        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?;
305        let params = sig.params;
306
307        match_reg_arg(params.get(0).unwrap(), I32, regs::rdi());
308        match_reg_arg(params.get(1).unwrap(), I64, regs::rsi());
309        match_reg_arg(params.get(2).unwrap(), I32, regs::rdx());
310        match_reg_arg(params.get(3).unwrap(), I64, regs::rcx());
311        match_reg_arg(params.get(4).unwrap(), I32, regs::r8());
312        match_reg_arg(params.get(5).unwrap(), I32, regs::r9());
313        match_stack_arg(params.get(6).unwrap(), I64, 0);
314        match_stack_arg(params.get(7).unwrap(), I32, 8);
315        Ok(())
316    }
317
318    #[test]
319    fn int_abi_sig_multi_returns() -> Result<()> {
320        let wasm_sig = WasmFuncType::new(
321            [I32, I64, I32, I64, I32, I32, I64, I32].into(),
322            [I32, I32, I32].into(),
323        );
324
325        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?;
326        let params = sig.params;
327        let results = sig.results;
328
329        match_reg_arg(params.get(0).unwrap(), I32, regs::rsi());
330        match_reg_arg(params.get(1).unwrap(), I64, regs::rdx());
331        match_reg_arg(params.get(2).unwrap(), I32, regs::rcx());
332        match_reg_arg(params.get(3).unwrap(), I64, regs::r8());
333        match_reg_arg(params.get(4).unwrap(), I32, regs::r9());
334        match_stack_arg(params.get(5).unwrap(), I32, 0);
335        match_stack_arg(params.get(6).unwrap(), I64, 8);
336        match_stack_arg(params.get(7).unwrap(), I32, 16);
337
338        match_stack_arg(results.get(0).unwrap(), I32, 4);
339        match_stack_arg(results.get(1).unwrap(), I32, 0);
340        match_reg_arg(results.get(2).unwrap(), I32, regs::rax());
341        Ok(())
342    }
343
344    #[test]
345    fn float_abi_sig() -> Result<()> {
346        let wasm_sig = WasmFuncType::new(
347            [F32, F64, F32, F64, F32, F32, F64, F32, F64].into(),
348            [].into(),
349        );
350
351        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?;
352        let params = sig.params;
353
354        match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0());
355        match_reg_arg(params.get(1).unwrap(), F64, regs::xmm1());
356        match_reg_arg(params.get(2).unwrap(), F32, regs::xmm2());
357        match_reg_arg(params.get(3).unwrap(), F64, regs::xmm3());
358        match_reg_arg(params.get(4).unwrap(), F32, regs::xmm4());
359        match_reg_arg(params.get(5).unwrap(), F32, regs::xmm5());
360        match_reg_arg(params.get(6).unwrap(), F64, regs::xmm6());
361        match_reg_arg(params.get(7).unwrap(), F32, regs::xmm7());
362        match_stack_arg(params.get(8).unwrap(), F64, 0);
363        Ok(())
364    }
365
366    #[test]
367    fn vector_abi_sig() -> Result<()> {
368        let wasm_sig = WasmFuncType::new(
369            [V128, V128, V128, V128, V128, V128, V128, V128, V128, V128].into(),
370            [].into(),
371        );
372
373        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?;
374        let params = sig.params;
375
376        match_reg_arg(params.get(0).unwrap(), V128, regs::xmm0());
377        match_reg_arg(params.get(1).unwrap(), V128, regs::xmm1());
378        match_reg_arg(params.get(2).unwrap(), V128, regs::xmm2());
379        match_reg_arg(params.get(3).unwrap(), V128, regs::xmm3());
380        match_reg_arg(params.get(4).unwrap(), V128, regs::xmm4());
381        match_reg_arg(params.get(5).unwrap(), V128, regs::xmm5());
382        match_reg_arg(params.get(6).unwrap(), V128, regs::xmm6());
383        match_reg_arg(params.get(7).unwrap(), V128, regs::xmm7());
384        match_stack_arg(params.get(8).unwrap(), V128, 0);
385        match_stack_arg(params.get(9).unwrap(), V128, 16);
386        Ok(())
387    }
388
389    #[test]
390    fn vector_abi_sig_multi_returns() -> Result<()> {
391        let wasm_sig = WasmFuncType::new([].into(), [V128, V128, V128].into());
392
393        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?;
394        let results = sig.results;
395
396        match_stack_arg(results.get(0).unwrap(), V128, 16);
397        match_stack_arg(results.get(1).unwrap(), V128, 0);
398        match_reg_arg(results.get(2).unwrap(), V128, regs::xmm0());
399        Ok(())
400    }
401
402    #[test]
403    fn mixed_abi_sig() -> Result<()> {
404        let wasm_sig = WasmFuncType::new(
405            [F32, I32, I64, F64, I32, F32, F64, F32, F64].into(),
406            [].into(),
407        );
408
409        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?;
410        let params = sig.params;
411
412        match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0());
413        match_reg_arg(params.get(1).unwrap(), I32, regs::rdi());
414        match_reg_arg(params.get(2).unwrap(), I64, regs::rsi());
415        match_reg_arg(params.get(3).unwrap(), F64, regs::xmm1());
416        match_reg_arg(params.get(4).unwrap(), I32, regs::rdx());
417        match_reg_arg(params.get(5).unwrap(), F32, regs::xmm2());
418        match_reg_arg(params.get(6).unwrap(), F64, regs::xmm3());
419        match_reg_arg(params.get(7).unwrap(), F32, regs::xmm4());
420        match_reg_arg(params.get(8).unwrap(), F64, regs::xmm5());
421        Ok(())
422    }
423
424    #[test]
425    fn system_v_call_conv() -> Result<()> {
426        let wasm_sig = WasmFuncType::new(
427            [F32, I32, I64, F64, I32, F32, F64, F32, F64].into(),
428            [].into(),
429        );
430
431        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::SystemV)?;
432        let params = sig.params;
433
434        match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0());
435        match_reg_arg(params.get(1).unwrap(), I32, regs::rdi());
436        match_reg_arg(params.get(2).unwrap(), I64, regs::rsi());
437        match_reg_arg(params.get(3).unwrap(), F64, regs::xmm1());
438        match_reg_arg(params.get(4).unwrap(), I32, regs::rdx());
439        match_reg_arg(params.get(5).unwrap(), F32, regs::xmm2());
440        match_reg_arg(params.get(6).unwrap(), F64, regs::xmm3());
441        match_reg_arg(params.get(7).unwrap(), F32, regs::xmm4());
442        match_reg_arg(params.get(8).unwrap(), F64, regs::xmm5());
443        Ok(())
444    }
445
446    #[test]
447    fn fastcall_call_conv() -> Result<()> {
448        let wasm_sig = WasmFuncType::new(
449            [F32, I32, I64, F64, I32, F32, F64, F32, F64].into(),
450            [].into(),
451        );
452
453        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::WindowsFastcall)?;
454        let params = sig.params;
455
456        match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0());
457        match_reg_arg(params.get(1).unwrap(), I32, regs::rdx());
458        match_reg_arg(params.get(2).unwrap(), I64, regs::r8());
459        match_reg_arg(params.get(3).unwrap(), F64, regs::xmm3());
460        match_stack_arg(params.get(4).unwrap(), I32, 32);
461        match_stack_arg(params.get(5).unwrap(), F32, 40);
462        Ok(())
463    }
464
465    #[test]
466    fn fastcall_call_conv_multi_returns() -> Result<()> {
467        let wasm_sig = WasmFuncType::new(
468            [F32, I32, I64, F64, I32, F32, F64, F32, F64].into(),
469            [I32, F32, I32, F32, I64].into(),
470        );
471
472        let sig = X64ABI::sig(&wasm_sig, &CallingConvention::WindowsFastcall)?;
473        let params = sig.params;
474        let results = sig.results;
475
476        match_reg_arg(params.get(0).unwrap(), F32, regs::xmm1());
477        match_reg_arg(params.get(1).unwrap(), I32, regs::r8());
478        match_reg_arg(params.get(2).unwrap(), I64, regs::r9());
479        // Each argument stack slot is 8 bytes.
480        match_stack_arg(params.get(3).unwrap(), F64, 32);
481        match_stack_arg(params.get(4).unwrap(), I32, 40);
482        match_stack_arg(params.get(5).unwrap(), F32, 48);
483
484        match_reg_arg(results.get(0).unwrap(), I32, regs::rax());
485
486        match_stack_arg(results.get(1).unwrap(), F32, 0);
487        match_stack_arg(results.get(2).unwrap(), I32, 4);
488        match_stack_arg(results.get(3).unwrap(), F32, 8);
489        match_stack_arg(results.get(4).unwrap(), I64, 12);
490        Ok(())
491    }
492
493    #[track_caller]
494    fn match_reg_arg(abi_arg: &ABIOperand, expected_ty: WasmValType, expected_reg: Reg) {
495        match abi_arg {
496            &ABIOperand::Reg { reg, ty, .. } => {
497                assert_eq!(reg, expected_reg);
498                assert_eq!(ty, expected_ty);
499            }
500            stack => panic!("Expected reg argument, got {stack:?}"),
501        }
502    }
503
504    #[track_caller]
505    fn match_stack_arg(abi_arg: &ABIOperand, expected_ty: WasmValType, expected_offset: u32) {
506        match abi_arg {
507            &ABIOperand::Stack { offset, ty, .. } => {
508                assert_eq!(offset, expected_offset);
509                assert_eq!(ty, expected_ty);
510            }
511            reg => panic!("Expected stack argument, got {reg:?}"),
512        }
513    }
514}
winch_codegen/isa/x64/abi.rs

winch_codegen/isa/x64/
abi.rs
