wasmtime/runtime/vm/instance/allocator/pooling/
table_pool.rs

1use super::{
2    index_allocator::{SimpleIndexAllocator, SlotId},
3    TableAllocationIndex,
4};
5use crate::runtime::vm::sys::vm::commit_pages;
6use crate::runtime::vm::{
7    mmap::AlignedLength, InstanceAllocationRequest, Mmap, PoolingInstanceAllocatorConfig,
8    SendSyncPtr, Table,
9};
10use crate::{prelude::*, vm::HostAlignedByteCount};
11use std::mem;
12use std::ptr::NonNull;
13use wasmtime_environ::{Module, Tunables};
14
15/// Represents a pool of WebAssembly tables.
16///
17/// Each instance index into the pool returns an iterator over the base addresses
18/// of the instance's tables.
19#[derive(Debug)]
20pub struct TablePool {
21    index_allocator: SimpleIndexAllocator,
22    mapping: Mmap<AlignedLength>,
23    table_size: HostAlignedByteCount,
24    max_total_tables: usize,
25    tables_per_instance: usize,
26    keep_resident: HostAlignedByteCount,
27    table_elements: usize,
28}
29
30impl TablePool {
31    /// Create a new `TablePool`.
32    pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
33        let table_size = HostAlignedByteCount::new_rounded_up(
34            mem::size_of::<*mut u8>()
35                .checked_mul(config.limits.table_elements)
36                .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?,
37        )?;
38
39        let max_total_tables = usize::try_from(config.limits.total_tables).unwrap();
40        let tables_per_instance = usize::try_from(config.limits.max_tables_per_module).unwrap();
41
42        let allocation_size = table_size
43            .checked_mul(max_total_tables)
44            .context("total size of tables exceeds addressable memory")?;
45
46        let mapping = Mmap::accessible_reserved(allocation_size, allocation_size)
47            .context("failed to create table pool mapping")?;
48
49        Ok(Self {
50            index_allocator: SimpleIndexAllocator::new(config.limits.total_tables),
51            mapping,
52            table_size,
53            max_total_tables,
54            tables_per_instance,
55            keep_resident: HostAlignedByteCount::new_rounded_up(config.table_keep_resident)?,
56            table_elements: usize::try_from(config.limits.table_elements).unwrap(),
57        })
58    }
59
60    /// Validate whether this module's tables are allocatable by this pool.
61    pub fn validate(&self, module: &Module) -> Result<()> {
62        let tables = module.num_defined_tables();
63
64        if tables > usize::try_from(self.tables_per_instance).unwrap() {
65            bail!(
66                "defined tables count of {} exceeds the per-instance limit of {}",
67                tables,
68                self.tables_per_instance,
69            );
70        }
71
72        if tables > self.max_total_tables {
73            bail!(
74                "defined tables count of {} exceeds the total tables limit of {}",
75                tables,
76                self.max_total_tables,
77            );
78        }
79
80        for (i, table) in module.tables.iter().skip(module.num_imported_tables) {
81            if table.limits.min > u64::try_from(self.table_elements)? {
82                bail!(
83                    "table index {} has a minimum element size of {} which exceeds the limit of {}",
84                    i.as_u32(),
85                    table.limits.min,
86                    self.table_elements,
87                );
88            }
89        }
90        Ok(())
91    }
92
93    /// Are there zero slots in use right now?
94    #[allow(unused)] // some cfgs don't use this
95    pub fn is_empty(&self) -> bool {
96        self.index_allocator.is_empty()
97    }
98
99    /// Get the base pointer of the given table allocation.
100    fn get(&self, table_index: TableAllocationIndex) -> *mut u8 {
101        assert!(table_index.index() < self.max_total_tables);
102
103        unsafe {
104            self.mapping
105                .as_ptr()
106                .add(
107                    self.table_size
108                        .checked_mul(table_index.index())
109                        .expect(
110                            "checked in constructor that table_size * table_index doesn't overflow",
111                        )
112                        .byte_count(),
113                )
114                .cast_mut()
115        }
116    }
117
118    /// Allocate a single table for the given instance allocation request.
119    pub fn allocate(
120        &self,
121        request: &mut InstanceAllocationRequest,
122        ty: &wasmtime_environ::Table,
123        tunables: &Tunables,
124    ) -> Result<(TableAllocationIndex, Table)> {
125        let allocation_index = self
126            .index_allocator
127            .alloc()
128            .map(|slot| TableAllocationIndex(slot.0))
129            .ok_or_else(|| {
130                super::PoolConcurrencyLimitError::new(self.max_total_tables, "tables")
131            })?;
132
133        match (|| {
134            let base = self.get(allocation_index);
135
136            unsafe {
137                commit_pages(base, self.table_elements * mem::size_of::<*mut u8>())?;
138            }
139
140            let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut(
141                base.cast(),
142                self.table_elements * mem::size_of::<*mut u8>(),
143            ))
144            .unwrap();
145            unsafe {
146                Table::new_static(
147                    ty,
148                    tunables,
149                    SendSyncPtr::new(ptr),
150                    &mut *request.store.get().unwrap(),
151                )
152            }
153        })() {
154            Ok(table) => Ok((allocation_index, table)),
155            Err(e) => {
156                self.index_allocator.free(SlotId(allocation_index.0));
157                Err(e)
158            }
159        }
160    }
161
162    /// Deallocate a previously-allocated table.
163    ///
164    /// # Safety
165    ///
166    /// The table must have been previously-allocated by this pool and assigned
167    /// the given allocation index, it must currently be allocated, and it must
168    /// never be used again.
169    ///
170    /// The caller must have already called `reset_table_pages_to_zero` on the
171    /// memory and flushed any enqueued decommits for this table's memory.
172    pub unsafe fn deallocate(&self, allocation_index: TableAllocationIndex, table: Table) {
173        assert!(table.is_static());
174        drop(table);
175        self.index_allocator.free(SlotId(allocation_index.0));
176    }
177
178    /// Reset the given table's memory to zero.
179    ///
180    /// Invokes the given `decommit` function for each region of memory that
181    /// needs to be decommitted. It is the caller's responsibility to actually
182    /// perform that decommit before this table is reused.
183    ///
184    /// # Safety
185    ///
186    /// This table must not be in active use, and ready for returning to the
187    /// table pool once it is zeroed and decommitted.
188    pub unsafe fn reset_table_pages_to_zero(
189        &self,
190        allocation_index: TableAllocationIndex,
191        table: &mut Table,
192        mut decommit: impl FnMut(*mut u8, usize),
193    ) {
194        assert!(table.is_static());
195        let base = self.get(allocation_index);
196
197        // XXX Should we check that table.size() * mem::size_of::<*mut u8>()
198        // doesn't overflow? The only check that exists is for the boundary
199        // condition that table.size() * mem::size_of::<*mut u8>() is less than
200        // a host page smaller than usize::MAX.
201        let size = HostAlignedByteCount::new_rounded_up(table.size() * mem::size_of::<*mut u8>())
202            .expect("table entry size doesn't overflow");
203
204        // `memset` the first `keep_resident` bytes.
205        let size_to_memset = size.min(self.keep_resident);
206        std::ptr::write_bytes(base, 0, size_to_memset.byte_count());
207
208        // And decommit the rest of it.
209        decommit(
210            base.add(size_to_memset.byte_count()),
211            size.checked_sub(size_to_memset)
212                .expect("size_to_memset <= size")
213                .byte_count(),
214        );
215    }
216}
217
218#[cfg(test)]
219mod tests {
220    use super::*;
221    use crate::runtime::vm::InstanceLimits;
222
223    #[test]
224    fn test_table_pool() -> Result<()> {
225        let pool = TablePool::new(&PoolingInstanceAllocatorConfig {
226            limits: InstanceLimits {
227                total_tables: 7,
228                table_elements: 100,
229                max_memory_size: 0,
230                max_memories_per_module: 0,
231                ..Default::default()
232            },
233            ..Default::default()
234        })?;
235
236        let host_page_size = HostAlignedByteCount::host_page_size();
237
238        assert_eq!(pool.table_size, host_page_size);
239        assert_eq!(pool.max_total_tables, 7);
240        assert_eq!(pool.table_elements, 100);
241
242        let base = pool.mapping.as_ptr() as usize;
243
244        for i in 0..7 {
245            let index = TableAllocationIndex(i);
246            let ptr = pool.get(index);
247            assert_eq!(
248                ptr as usize - base,
249                pool.table_size.checked_mul(i as usize).unwrap()
250            );
251        }
252
253        Ok(())
254    }
255}
wasmtime/runtime/vm/instance/allocator/pooling/table_pool.rs

wasmtime/runtime/vm/instance/allocator/pooling/
table_pool.rs
