winch_codegen/codegen/call.rs
1//! Function call emission. For more details around the ABI and
2//! calling convention, see [ABI].
3//!
4//! This module exposes a single function [`FnCall::emit`], which is responsible
5//! of orchestrating the emission of calls. In general such orchestration
6//! takes place in 6 steps:
7//!
8//! 1. [`Callee`] resolution.
9//! 2. Mapping of the [`Callee`] to the [`CalleeKind`].
10//! 3. Spilling the value stack.
11//! 4. Calculate the return area, for 1+ results.
12//! 5. Emission.
13//! 6. Stack space cleanup.
14//!
15//! The stack space consumed by the function call is the amount
16//! of space used by any memory entries in the value stack present
17//! at the callsite (after spilling the value stack), that will be
18//! used as arguments for the function call. Any memory values in the
19//! value stack that are needed as part of the function
20//! arguments will be consumed by the function call (either by
21//! assigning those values to a register or by storing those
22//! values in a memory location if the callee argument is on
23//! the stack).
24//! This could also be done when assigning arguments every time a
25//! memory entry needs to be assigned to a particular location,
26//! but doing so will emit more instructions (e.g. a pop per
27//! argument that needs to be assigned); it's more efficient to
28//! calculate the space used by those memory values and reclaim it
29//! at once when cleaning up the stack after the call has been
30//! emitted.
31//!
32//! The machine stack throughout the function call is as follows:
33//! ┌──────────────────────────────────────────────────┐
34//! │ │
35//! │ Stack space created by any previous spills │
36//! │ from the value stack; and which memory values │
37//! │ are used as function arguments. │
38//! │ │
39//! ├──────────────────────────────────────────────────┤ ---> The Wasm value stack at this point in time would look like:
40//! │ │
41//! │ Stack space created by spilling locals and |
42//! │ registers at the callsite. │
43//! │ │
44//! ├─────────────────────────────────────────────────┬┤
45//! │ │
46//! │ Return Area (Multi-value results) │
47//! │ │
48//! │ │
49//! ├─────────────────────────────────────────────────┬┤ ---> The Wasm value stack at this point in time would look like:
50//! │ │ [ Mem(offset) | Mem(offset) | Mem(offset) | Mem(offset) ]
51//! │ │ Assuming that the callee takes 4 arguments, we calculate
52//! │ │ 4 memory values; all of which will be used as arguments to
53//! │ Stack space allocated for │ the call via `assign_args`, thus the sum of the size of the
54//! │ the callee function arguments in the stack; │ memory they represent is considered to be consumed by the call.
55//! │ represented by `arg_stack_space` │
56//! │ │
57//! │ │
58//! │ │
59//! └──────────────────────────────────────────────────┘ ------> Stack pointer when emitting the call
60
61use crate::{
62 abi::{scratch, vmctx, ABIOperand, ABISig, RetArea},
63 codegen::{BuiltinFunction, BuiltinType, Callee, CodeGenContext, CodeGenError, Emission},
64 masm::{
65 CalleeKind, ContextArgs, MacroAssembler, MemMoveDirection, OperandSize, SPOffset,
66 VMContextLoc,
67 },
68 reg::writable,
69 reg::Reg,
70 stack::Val,
71 FuncEnv,
72};
73use anyhow::{ensure, Result};
74use wasmtime_environ::{FuncIndex, PtrSize, VMOffsets};
75
76/// All the information needed to emit a function call.
77#[derive(Copy, Clone)]
78pub(crate) struct FnCall {}
79
80impl FnCall {
81 /// Orchestrates the emission of a function call:
82 /// 1. Resolves the [`Callee`] through the given callback.
83 /// 2. Lowers the resolved [`Callee`] to a ([`CalleeKind`], [ContextArgs])
84 /// 3. Spills the value stack.
85 /// 4. Creates the stack space needed for the return area.
86 /// 5. Emits the call.
87 /// 6. Cleans up the stack space.
88 pub fn emit<M: MacroAssembler>(
89 env: &mut FuncEnv<M::Ptr>,
90 masm: &mut M,
91 context: &mut CodeGenContext<Emission>,
92 callee: Callee,
93 ) -> Result<()> {
94 let (kind, callee_context) = Self::lower(env, context.vmoffsets, &callee, context, masm)?;
95
96 let sig = env.callee_sig::<M::ABI>(&callee)?;
97 context.spill(masm)?;
98 let ret_area = Self::make_ret_area(&sig, masm)?;
99 let arg_stack_space = sig.params_stack_size();
100 let reserved_stack = masm.call(arg_stack_space, |masm| {
101 Self::assign(sig, &callee_context, ret_area.as_ref(), context, masm)?;
102 Ok((kind, sig.call_conv))
103 })?;
104
105 Self::cleanup(
106 sig,
107 &callee_context,
108 &kind,
109 reserved_stack,
110 ret_area,
111 masm,
112 context,
113 )
114 }
115
116 /// Calculates the return area for the callee, if any.
117 fn make_ret_area<M: MacroAssembler>(
118 callee_sig: &ABISig,
119 masm: &mut M,
120 ) -> Result<Option<RetArea>> {
121 if callee_sig.has_stack_results() {
122 let base = masm.sp_offset()?.as_u32();
123 let end = base + callee_sig.results_stack_size();
124 if end > base {
125 masm.reserve_stack(end - base)?;
126 }
127 Ok(Some(RetArea::sp(SPOffset::from_u32(end))))
128 } else {
129 Ok(None)
130 }
131 }
132
133 /// Lowers the high-level [`Callee`] to a [`CalleeKind`] and
134 /// [ContextArgs] pair which contains all the metadata needed for
135 /// emission.
136 fn lower<M: MacroAssembler>(
137 env: &mut FuncEnv<M::Ptr>,
138 vmoffsets: &VMOffsets<u8>,
139 callee: &Callee,
140 context: &mut CodeGenContext<Emission>,
141 masm: &mut M,
142 ) -> Result<(CalleeKind, ContextArgs)> {
143 let ptr = vmoffsets.ptr.size();
144 match callee {
145 Callee::Builtin(b) => Ok(Self::lower_builtin(env, b)),
146 Callee::FuncRef(_) => {
147 Self::lower_funcref(env.callee_sig::<M::ABI>(callee)?, ptr, context, masm)
148 }
149 Callee::Local(i) => Ok(Self::lower_local(env, *i)),
150 Callee::Import(i) => {
151 let sig = env.callee_sig::<M::ABI>(callee)?;
152 Self::lower_import(*i, sig, context, masm, vmoffsets)
153 }
154 }
155 }
156
157 /// Lowers a builtin function by loading its address to the next available
158 /// register.
159 fn lower_builtin<P: PtrSize>(
160 env: &mut FuncEnv<P>,
161 builtin: &BuiltinFunction,
162 ) -> (CalleeKind, ContextArgs) {
163 match builtin.ty() {
164 BuiltinType::Builtin(idx) => (
165 CalleeKind::direct(env.name_builtin(idx)),
166 ContextArgs::pinned_vmctx(),
167 ),
168 BuiltinType::LibCall(c) => (CalleeKind::libcall(c), ContextArgs::none()),
169 }
170 }
171
172 /// Lower a local function to a [`CalleeKind`] and [ContextArgs] pair.
173 fn lower_local<P: PtrSize>(
174 env: &mut FuncEnv<P>,
175 index: FuncIndex,
176 ) -> (CalleeKind, ContextArgs) {
177 (
178 CalleeKind::direct(env.name_wasm(index)),
179 ContextArgs::pinned_callee_and_caller_vmctx(),
180 )
181 }
182
183 /// Lowers a function import by loading its address to the next available
184 /// register.
185 fn lower_import<M: MacroAssembler, P: PtrSize>(
186 index: FuncIndex,
187 sig: &ABISig,
188 context: &mut CodeGenContext<Emission>,
189 masm: &mut M,
190 vmoffsets: &VMOffsets<P>,
191 ) -> Result<(CalleeKind, ContextArgs)> {
192 let (callee, callee_vmctx) =
193 context.without::<Result<(Reg, Reg)>, M, _>(&sig.regs, masm, |context, masm| {
194 Ok((context.any_gpr(masm)?, context.any_gpr(masm)?))
195 })??;
196 let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(index);
197 let callee_vmctx_addr = masm.address_at_vmctx(callee_vmctx_offset)?;
198 masm.load_ptr(callee_vmctx_addr, writable!(callee_vmctx))?;
199
200 let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(index);
201 let callee_addr = masm.address_at_vmctx(callee_body_offset)?;
202 masm.load_ptr(callee_addr, writable!(callee))?;
203
204 Ok((
205 CalleeKind::indirect(callee),
206 ContextArgs::with_callee_and_pinned_caller(callee_vmctx),
207 ))
208 }
209
210 /// Lowers a function reference by loading its address into the next
211 /// available register.
212 fn lower_funcref<M: MacroAssembler>(
213 sig: &ABISig,
214 ptr: impl PtrSize,
215 context: &mut CodeGenContext<Emission>,
216 masm: &mut M,
217 ) -> Result<(CalleeKind, ContextArgs)> {
218 // Pop the funcref pointer to a register and allocate a register to hold the
219 // address of the funcref. Since the callee is not addressed from a global non
220 // allocatable register (like the vmctx in the case of an import), we load the
221 // funcref to a register ensuring that it doesn't get assigned to a register
222 // used in the callee's signature.
223 let (funcref_ptr, funcref, callee_vmctx) = context
224 .without::<Result<(Reg, Reg, Reg)>, M, _>(&sig.regs, masm, |cx, masm| {
225 Ok((
226 cx.pop_to_reg(masm, None)?.into(),
227 cx.any_gpr(masm)?,
228 cx.any_gpr(masm)?,
229 ))
230 })??;
231
232 // Load the callee VMContext, that will be passed as first argument to
233 // the function call.
234 masm.load_ptr(
235 masm.address_at_reg(funcref_ptr, ptr.vm_func_ref_vmctx().into())?,
236 writable!(callee_vmctx),
237 )?;
238
239 // Load the function pointer to be called.
240 masm.load_ptr(
241 masm.address_at_reg(funcref_ptr, ptr.vm_func_ref_wasm_call().into())?,
242 writable!(funcref),
243 )?;
244 context.free_reg(funcref_ptr);
245
246 Ok((
247 CalleeKind::indirect(funcref),
248 ContextArgs::with_callee_and_pinned_caller(callee_vmctx),
249 ))
250 }
251
252 /// Materializes any [ContextArgs] as a function argument.
253 fn assign_context_args<M: MacroAssembler>(
254 sig: &ABISig,
255 context: &ContextArgs,
256 masm: &mut M,
257 ) -> Result<()> {
258 ensure!(
259 sig.params().len() >= context.len(),
260 CodeGenError::vmcontext_arg_expected(),
261 );
262 for (context_arg, operand) in context
263 .as_slice()
264 .iter()
265 .zip(sig.params_without_retptr().iter().take(context.len()))
266 {
267 match (context_arg, operand) {
268 (VMContextLoc::Pinned, ABIOperand::Reg { ty, reg, .. }) => {
269 masm.mov(writable!(*reg), vmctx!(M).into(), (*ty).try_into()?)?;
270 }
271 (VMContextLoc::Pinned, ABIOperand::Stack { ty, offset, .. }) => {
272 let addr = masm.address_at_sp(SPOffset::from_u32(*offset))?;
273 masm.store(vmctx!(M).into(), addr, (*ty).try_into()?)?;
274 }
275
276 (VMContextLoc::Reg(src), ABIOperand::Reg { ty, reg, .. }) => {
277 masm.mov(writable!(*reg), (*src).into(), (*ty).try_into()?)?;
278 }
279
280 (VMContextLoc::Reg(src), ABIOperand::Stack { ty, offset, .. }) => {
281 let addr = masm.address_at_sp(SPOffset::from_u32(*offset))?;
282 masm.store((*src).into(), addr, (*ty).try_into()?)?;
283 }
284 }
285 }
286 Ok(())
287 }
288
289 /// Assign arguments for the function call.
290 fn assign<M: MacroAssembler>(
291 sig: &ABISig,
292 callee_context: &ContextArgs,
293 ret_area: Option<&RetArea>,
294 context: &mut CodeGenContext<Emission>,
295 masm: &mut M,
296 ) -> Result<()> {
297 let arg_count = sig.params.len_without_retptr();
298 debug_assert!(arg_count >= callee_context.len());
299 let stack = &context.stack;
300 let stack_values = stack.peekn(arg_count - callee_context.len());
301
302 if callee_context.len() > 0 {
303 Self::assign_context_args(&sig, &callee_context, masm)?;
304 }
305
306 for (arg, val) in sig
307 .params_without_retptr()
308 .iter()
309 .skip(callee_context.len())
310 .zip(stack_values)
311 {
312 match arg {
313 &ABIOperand::Reg { reg, .. } => {
314 context.move_val_to_reg(&val, reg, masm)?;
315 }
316 &ABIOperand::Stack { ty, offset, .. } => {
317 let addr = masm.address_at_sp(SPOffset::from_u32(offset))?;
318 let size: OperandSize = ty.try_into()?;
319 let scratch = scratch!(M, &ty);
320 context.move_val_to_reg(val, scratch, masm)?;
321 masm.store(scratch.into(), addr, size)?;
322 }
323 }
324 }
325
326 if sig.has_stack_results() {
327 let operand = sig.params.unwrap_results_area_operand();
328 let base = ret_area.unwrap().unwrap_sp();
329 let addr = masm.address_from_sp(base)?;
330
331 match operand {
332 &ABIOperand::Reg { ty, reg, .. } => {
333 masm.compute_addr(addr, writable!(reg), ty.try_into()?)?;
334 }
335 &ABIOperand::Stack { ty, offset, .. } => {
336 let slot = masm.address_at_sp(SPOffset::from_u32(offset))?;
337 // Don't rely on `ABI::scratch_for` as we always use
338 // an int register as the return pointer.
339 let scratch = scratch!(M);
340 masm.compute_addr(addr, writable!(scratch), ty.try_into()?)?;
341 masm.store(scratch.into(), slot, ty.try_into()?)?;
342 }
343 }
344 }
345 Ok(())
346 }
347
348 /// Cleanup stack space, handle multiple results, and free registers after
349 /// emitting the call.
350 fn cleanup<M: MacroAssembler>(
351 sig: &ABISig,
352 callee_context: &ContextArgs,
353 callee_kind: &CalleeKind,
354 reserved_space: u32,
355 ret_area: Option<RetArea>,
356 masm: &mut M,
357 context: &mut CodeGenContext<Emission>,
358 ) -> Result<()> {
359 // Free any registers holding any function references.
360 match callee_kind {
361 CalleeKind::Indirect(r) => context.free_reg(*r),
362 _ => {}
363 }
364
365 // Free any registers used as part of the [ContextArgs].
366 for loc in callee_context.as_slice() {
367 match loc {
368 VMContextLoc::Reg(r) => context.free_reg(*r),
369 _ => {}
370 }
371 }
372 // Deallocate the reserved space for stack arguments and for alignment,
373 // which was allocated last.
374 masm.free_stack(reserved_space)?;
375
376 ensure!(
377 sig.params.len_without_retptr() >= callee_context.len(),
378 CodeGenError::vmcontext_arg_expected()
379 );
380
381 // Drop params from value stack and calculate amount of machine stack
382 // space they consumed.
383 let mut stack_consumed = 0;
384 context.drop_last(
385 sig.params.len_without_retptr() - callee_context.len(),
386 |_regalloc, v| {
387 ensure!(
388 v.is_mem() || v.is_const(),
389 CodeGenError::unexpected_value_in_value_stack()
390 );
391 if let Val::Memory(mem) = v {
392 stack_consumed += mem.slot.size;
393 }
394 Ok(())
395 },
396 )?;
397
398 if let Some(ret_area) = ret_area {
399 if stack_consumed > 0 {
400 // Perform a memory move, by shuffling the result area to
401 // higher addresses. This is needed because the result area
402 // is located after any memory addresses located on the stack,
403 // and after spilled values consumed by the call.
404 let sp = ret_area.unwrap_sp();
405 let result_bytes = sig.results_stack_size();
406 ensure!(
407 sp.as_u32() >= stack_consumed + result_bytes,
408 CodeGenError::invalid_sp_offset(),
409 );
410 let dst = SPOffset::from_u32(sp.as_u32() - stack_consumed);
411 masm.memmove(sp, dst, result_bytes, MemMoveDirection::LowToHigh)?;
412 }
413 };
414
415 // Free the bytes consumed by the call.
416 masm.free_stack(stack_consumed)?;
417
418 let mut calculated_ret_area = None;
419
420 if let Some(area) = ret_area {
421 if stack_consumed > 0 {
422 // If there's a return area and stack space was consumed by the
423 // call, adjust the return area to be to the current stack
424 // pointer offset.
425 calculated_ret_area = Some(RetArea::sp(masm.sp_offset()?));
426 } else {
427 // Else if no stack space was consumed by the call, simply use
428 // the previously calculated area.
429 ensure!(
430 area.unwrap_sp() == masm.sp_offset()?,
431 CodeGenError::invalid_sp_offset()
432 );
433 calculated_ret_area = Some(area);
434 }
435 }
436
437 // In the case of [Callee], there's no need to set the [RetArea] of the
438 // signature, as it's only used here to push abi results.
439 context.push_abi_results(&sig.results, masm, |_, _, _| calculated_ret_area)?;
440 // Reload the [VMContext] pointer into the corresponding pinned
441 // register. Winch currently doesn't have any callee-saved registers in
442 // the default ABI. So the callee might clobber the designated pinned
443 // register.
444 context.load_vmctx(masm)
445 }
446}