wasmtime/runtime/vm/
cow.rs

1//! Copy-on-write initialization support: creation of backing images for
2//! modules, and logic to support mapping these backing images into memory.
3
4use super::sys::DecommitBehavior;
5use crate::Engine;
6use crate::prelude::*;
7use crate::runtime::vm::mpk::ProtectionKey;
8use crate::runtime::vm::sys::vm::{self, MemoryImageSource, PageMap, reset_with_pagemap};
9use crate::runtime::vm::{
10    HostAlignedByteCount, MmapOffset, ModuleMemoryImageSource, host_page_size,
11};
12use alloc::sync::Arc;
13use core::fmt;
14use core::ops::Range;
15use wasmtime_environ::prelude::TryPrimaryMap;
16use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, MemoryTunables, Module};
17
18/// Backing images for memories in a module.
19///
20/// This is meant to be built once, when a module is first loaded/constructed,
21/// and then used many times for instantiation.
22pub struct ModuleMemoryImages {
23    memories: TryPrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryImage>>>,
24}
25
26impl ModuleMemoryImages {
27    /// Get the MemoryImage for a given memory.
28    pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
29        self.memories[defined_index].as_ref()
30    }
31}
32
33/// One backing image for one memory.
34pub struct MemoryImage {
35    /// The platform-specific source of this image.
36    ///
37    /// This might be a mapped `*.cwasm` file or on Unix it could also be a
38    /// `Memfd` as an anonymous file in memory on Linux. In either case this is
39    /// used as the backing-source for the CoW image.
40    source: MemoryImageSource,
41
42    /// Length of image, in bytes.
43    ///
44    /// Note that initial memory size may be larger; leading and trailing zeroes
45    /// are truncated (handled by backing fd).
46    ///
47    /// Must be a multiple of the system page size.
48    len: HostAlignedByteCount,
49
50    /// Image starts this many bytes into `source`.
51    ///
52    /// This is 0 for anonymous-backed memfd files and is the offset of the
53    /// data section in a `*.cwasm` file for `*.cwasm`-backed images.
54    ///
55    /// Must be a multiple of the system page size.
56    ///
57    /// ## Notes
58    ///
59    /// This currently isn't a `HostAlignedByteCount` because that's a usize and
60    /// this, being a file offset, is a u64.
61    source_offset: u64,
62
63    /// Image starts this many bytes into heap space.
64    ///
65    /// Must be a multiple of the system page size.
66    linear_memory_offset: HostAlignedByteCount,
67
68    /// The original source of data that this image is derived from.
69    module_source: Arc<dyn ModuleMemoryImageSource>,
70
71    /// The offset, within `module_source.wasm_data()`, that this image starts
72    /// at.
73    module_source_offset: usize,
74}
75
76impl MemoryImage {
77    fn new(
78        engine: &Engine,
79        page_size: u32,
80        linear_memory_offset: HostAlignedByteCount,
81        module_source: &Arc<impl ModuleMemoryImageSource>,
82        data_range: Range<usize>,
83    ) -> Result<Option<MemoryImage>> {
84        let assert_page_aligned = |val: usize| {
85            assert_eq!(val % (page_size as usize), 0);
86        };
87        // Sanity-check that various parameters are page-aligned.
88        let len =
89            HostAlignedByteCount::new(data_range.len()).expect("memory image data is page-aligned");
90
91        // If a backing `mmap` is present then `data` should be a sub-slice of
92        // the `mmap`. The sanity-checks here double-check that. Additionally
93        // compilation should have ensured that the `data` section is
94        // page-aligned within `mmap`, so that's also all double-checked here.
95        //
96        // Finally if the `mmap` itself comes from a backing file on disk, such
97        // as a `*.cwasm` file, then that's a valid source of data for the
98        // memory image so we simply return referencing that.
99        //
100        // Note that this path is platform-agnostic in the sense of all
101        // platforms we support support memory mapping copy-on-write data from
102        // files, but for now this is still a Linux-specific region of Wasmtime.
103        // Some work will be needed to get this file compiling for macOS and
104        // Windows.
105        let data = &module_source.wasm_data()[data_range.clone()];
106        if !engine.config().force_memory_init_memfd {
107            if let Some(mmap) = module_source.mmap() {
108                let start = mmap.as_ptr() as usize;
109                let end = start + mmap.len();
110                let data_start = data.as_ptr() as usize;
111                let data_end = data_start + data.len();
112                assert!(start <= data_start && data_end <= end);
113                assert_page_aligned(start);
114                assert_page_aligned(data_start);
115                assert_page_aligned(data_end);
116
117                #[cfg(feature = "std")]
118                if let Some(file) = mmap.original_file() {
119                    if let Some(source) = MemoryImageSource::from_file(file) {
120                        return Ok(Some(MemoryImage {
121                            source,
122                            source_offset: u64::try_from(data_start - start).unwrap(),
123                            linear_memory_offset,
124                            len,
125                            module_source: module_source.clone(),
126                            module_source_offset: data_range.start,
127                        }));
128                    }
129                }
130            }
131        }
132
133        // If `mmap` doesn't come from a file then platform-specific mechanisms
134        // may be used to place the data in a form that's amenable to an mmap.
135        if let Some(source) = MemoryImageSource::from_data(data)? {
136            return Ok(Some(MemoryImage {
137                source,
138                source_offset: 0,
139                linear_memory_offset,
140                len,
141                module_source: module_source.clone(),
142                module_source_offset: data_range.start,
143            }));
144        }
145
146        Ok(None)
147    }
148
149    /// Offset, in bytes, within linear memory at which this image starts.
150    pub(crate) fn linear_memory_offset(&self) -> usize {
151        self.linear_memory_offset.byte_count()
152    }
153
154    /// Length, in bytes, of this image.
155    pub(crate) fn len(&self) -> usize {
156        self.len.byte_count()
157    }
158
159    /// The bytes of this image, as stored in the module's data.
160    /// The length is exactly [`self.len`](Self::len).
161    pub(crate) fn data(&self) -> &[u8] {
162        &self.module_source.wasm_data()[self.module_source_offset..][..self.len.byte_count()]
163    }
164
165    /// Maps this image at `base + self.linear_memory_offset`.
166    ///
167    /// # Safety
168    ///
169    /// See [`crate::MemoryImage::map_at`].
170    pub(crate) unsafe fn map_at_ptr(&self, base: *mut u8) -> Result<()> {
171        if base.addr() % host_page_size() != 0 {
172            bail!("cannot map a memory image at {base:p}: not host-page-aligned");
173        }
174        if self.len.is_zero() {
175            return Ok(());
176        }
177        unsafe {
178            self.source.map_at(
179                base.add(self.linear_memory_offset.byte_count()),
180                self.source_offset,
181                self.len.byte_count(),
182            )?;
183        }
184        Ok(())
185    }
186
187    /// Maps this image into `mmap_base + self.linear_memory_offset`.
188    ///
189    /// # Safety
190    ///
191    /// The caller must ensure that no one else has a reference to the range
192    /// of `mmap_base` being mapped over.
193    pub(crate) unsafe fn map_at(&self, mmap_base: &MmapOffset) -> Result<()> {
194        unsafe {
195            mmap_base.map_image_at(
196                &self.source,
197                self.source_offset,
198                self.linear_memory_offset,
199                self.len,
200            )
201        }
202    }
203
204    unsafe fn remap_as_zeros_at(&self, base: *mut u8) -> Result<()> {
205        unsafe {
206            self.source.remap_as_zeros_at(
207                base.add(self.linear_memory_offset.byte_count()),
208                self.len.byte_count(),
209            )?;
210        }
211        Ok(())
212    }
213}
214
215impl ModuleMemoryImages {
216    /// Create a new `ModuleMemoryImages` for the given module. This can be
217    /// passed in as part of a `InstanceAllocationRequest` to speed up
218    /// instantiation and execution by using copy-on-write-backed memories.
219    pub fn new(
220        engine: &Engine,
221        module: &Module,
222        source: &Arc<impl ModuleMemoryImageSource>,
223    ) -> Result<Option<ModuleMemoryImages>> {
224        let map = match &module.memory_initialization {
225            MemoryInitialization::Static { map } => map,
226            _ => return Ok(None),
227        };
228        let mut memories = TryPrimaryMap::with_capacity(map.len())?;
229        let page_size = crate::runtime::vm::host_page_size();
230        let page_size = u32::try_from(page_size).unwrap();
231        for (memory_index, init) in map {
232            // mmap-based-initialization only works for defined memories with a
233            // known starting point of all zeros, so bail out if the mmeory is
234            // imported.
235            let defined_memory = match module.defined_memory_index(memory_index) {
236                Some(idx) => idx,
237                None => return Ok(None),
238            };
239
240            // If there's no initialization for this memory known then we don't
241            // need an image for the memory so push `None` and move on.
242            let (offset, runtime_index) = match init {
243                Some(init) => init,
244                None => {
245                    memories.push(None)?;
246                    continue;
247                }
248            };
249
250            let data_range = &module.runtime_data[*runtime_index];
251            let data_range = usize::try_from(data_range.start).unwrap()
252                ..usize::try_from(data_range.end).unwrap();
253
254            if module.memories[memory_index]
255                .minimum_byte_size()
256                .map_or(false, |mem_initial_len| {
257                    *offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len
258                })
259            {
260                // The image is rounded up to multiples of the host OS page
261                // size. But if Wasm is using a custom page size, the Wasm page
262                // size might be smaller than the host OS page size, and that
263                // rounding might have made the image larger than the Wasm
264                // memory's initial length. This is *probably* okay, since the
265                // rounding would have just introduced new runs of zeroes in the
266                // image, but out of an abundance of caution we don't generate
267                // CoW images in this scenario.
268                return Ok(None);
269            }
270
271            let offset_usize = match usize::try_from(*offset) {
272                Ok(offset) => offset,
273                Err(_) => return Ok(None),
274            };
275            let offset = HostAlignedByteCount::new(offset_usize)
276                .expect("memory init offset is a multiple of the host page size");
277
278            // If this creation fails then we fail creating
279            // `ModuleMemoryImages` since this memory couldn't be represented.
280            let image = match MemoryImage::new(engine, page_size, offset, source, data_range)? {
281                Some(image) => image,
282                None => return Ok(None),
283            };
284
285            let idx = memories.push(Some(try_new::<Arc<_>>(image)?))?;
286            assert_eq!(idx, defined_memory);
287        }
288
289        Ok(Some(ModuleMemoryImages { memories }))
290    }
291}
292
293/// Slot management of a copy-on-write image which can be reused for the pooling
294/// allocator.
295///
296/// This data structure manages a slot of linear memory, primarily in the
297/// pooling allocator, which optionally has a contiguous memory image in the
298/// middle of it. Pictorially this data structure manages a virtual memory
299/// region that looks like:
300///
301/// ```text
302///   +--------------------+-------------------+--------------+--------------+
303///   |   anonymous        |      optional     |   anonymous  |    PROT_NONE |
304///   |     zero           |       memory      |     zero     |     memory   |
305///   |    memory          |       image       |    memory    |              |
306///   +--------------------+-------------------+--------------+--------------+
307///   |                     <------+---------->
308///   |<-----+------------>         \
309///   |      \                   image.len
310///   |       \
311///   |  image.linear_memory_offset
312///   |
313///   \
314///  self.base is this virtual address
315///
316///    <------------------+------------------------------------------------>
317///                        \
318///                      static_size
319///
320///    <------------------+---------------------------------->
321///                        \
322///                      accessible
323/// ```
324///
325/// When a `MemoryImageSlot` is created it's told what the `static_size` and
326/// `accessible` limits are. Initially there is assumed to be no image in linear
327/// memory.
328///
329/// When `MemoryImageSlot::instantiate` is called then the method will perform
330/// a "synchronization" to take the image from its prior state to the new state
331/// for the image specified. The first instantiation for example will mmap the
332/// heap image into place. Upon reuse of a slot nothing happens except possibly
333/// shrinking `self.accessible`. When a new image is used then the old image is
334/// mapped to anonymous zero memory and then the new image is mapped in place.
335///
336/// A `MemoryImageSlot` is either `dirty` or it isn't. When a `MemoryImageSlot`
337/// is dirty then it is assumed that any memory beneath `self.accessible` could
338/// have any value. Instantiation cannot happen into a `dirty` slot, however, so
339/// the `MemoryImageSlot::clear_and_remain_ready` returns this memory back to
340/// its original state to mark `dirty = false`. This is done by resetting all
341/// anonymous memory back to zero and the image itself back to its initial
342/// contents.
343///
344/// On Linux this is achieved with the `madvise(MADV_DONTNEED)` syscall. This
345/// syscall will release the physical pages back to the OS but retain the
346/// original mappings, effectively resetting everything back to its initial
347/// state. Non-linux platforms will replace all memory below `self.accessible`
348/// with a fresh zero'd mmap, meaning that reuse is effectively not supported.
349#[cfg_attr(
350    not(feature = "pooling-allocator"),
351    expect(dead_code, reason = "only used by the pooling allocator")
352)]
353pub struct MemoryImageSlot {
354    /// The mmap and offset within it that contains the linear memory for this
355    /// slot.
356    base: MmapOffset,
357
358    /// The maximum static memory size which `self.accessible` can grow to.
359    static_size: usize,
360
361    /// An optional image that is currently being used in this linear memory.
362    ///
363    /// This can be `None` in which case memory is originally all zeros. When
364    /// `Some` the image describes where it's located within the image.
365    image: Option<Arc<MemoryImage>>,
366
367    /// The size of the heap that is readable and writable.
368    ///
369    /// Note that this may extend beyond the actual linear memory heap size in
370    /// the case of dynamic memories in use. Memory accesses to memory below
371    /// `self.accessible` may still page fault as pages are lazily brought in
372    /// but the faults will always be resolved by the kernel.
373    ///
374    /// Also note that this is always page-aligned.
375    accessible: HostAlignedByteCount,
376
377    /// Whether this slot may have "dirty" pages (pages written by an
378    /// instantiation). Set by `instantiate()` and cleared by
379    /// `clear_and_remain_ready()`, and used in assertions to ensure
380    /// those methods are called properly.
381    ///
382    /// Invariant: if !dirty, then this memory slot contains a clean
383    /// CoW mapping of `image`, if `Some(..)`, and anonymous-zero
384    /// memory beyond the image up to `static_size`. The addresses
385    /// from offset 0 to `self.accessible` are R+W and set to zero or the
386    /// initial image content, as appropriate. Everything between
387    /// `self.accessible` and `self.static_size` is inaccessible.
388    dirty: bool,
389
390    /// The MPK protection key that this slot's stripe was colored with, if the
391    /// pooling allocator is striping memory with protection keys.
392    ///
393    /// This must be re-applied after every `mmap` performed on this slot, since
394    /// `mmap` resets the affected pages back to the default key 0 which is
395    /// accessible from every stripe.
396    pkey: Option<ProtectionKey>,
397}
398
399impl fmt::Debug for MemoryImageSlot {
400    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
401        f.debug_struct("MemoryImageSlot")
402            .field("base", &self.base)
403            .field("static_size", &self.static_size)
404            .field("accessible", &self.accessible)
405            .field("dirty", &self.dirty)
406            .finish_non_exhaustive()
407    }
408}
409
410#[cfg_attr(
411    not(feature = "pooling-allocator"),
412    expect(dead_code, reason = "only used by the pooling allocator")
413)]
414impl MemoryImageSlot {
415    /// Create a new MemoryImageSlot. Assumes that there is an anonymous
416    /// mmap backing in the given range to start.
417    ///
418    /// The `accessible` parameter describes how much of linear memory is
419    /// already mapped as R/W with all zero-bytes. The `static_size` value is
420    /// the maximum size of this image which `accessible` cannot grow beyond,
421    /// and all memory from `accessible` from `static_size` should be mapped as
422    /// `PROT_NONE` backed by zero-bytes.
423    pub(crate) fn create(
424        base: MmapOffset,
425        accessible: HostAlignedByteCount,
426        static_size: usize,
427        pkey: Option<ProtectionKey>,
428    ) -> Self {
429        MemoryImageSlot {
430            base,
431            static_size,
432            accessible,
433            image: None,
434            dirty: false,
435            pkey,
436        }
437    }
438
439    pub(crate) fn set_heap_limit(&mut self, size_bytes: usize) -> Result<()> {
440        let size_bytes_aligned = HostAlignedByteCount::new_rounded_up(size_bytes)?;
441        assert!(size_bytes <= self.static_size);
442        assert!(size_bytes_aligned.byte_count() <= self.static_size);
443
444        // If the heap limit already addresses accessible bytes then no syscalls
445        // are necessary since the data is already mapped into the process and
446        // waiting to go.
447        //
448        // This is used for "dynamic" memories where memory is not always
449        // decommitted during recycling (but it's still always reset).
450        if size_bytes_aligned <= self.accessible {
451            return Ok(());
452        }
453
454        // Otherwise use `mprotect` to make the new pages read/write.
455        self.set_protection(self.accessible..size_bytes_aligned, true)?;
456        self.accessible = size_bytes_aligned;
457
458        Ok(())
459    }
460
461    /// Prepares this slot for the instantiation of a new instance with the
462    /// provided linear memory image.
463    ///
464    /// The `initial_size_bytes` parameter indicates the required initial size
465    /// of the heap for the instance. The `maybe_image` is an optional initial
466    /// image for linear memory to contains. The `style` is the way compiled
467    /// code will be accessing this memory.
468    ///
469    /// The purpose of this method is to take a previously pristine slot
470    /// (`!self.dirty`) and transform its prior state into state necessary for
471    /// the given parameters. This could include, for example:
472    ///
473    /// * More memory may be made read/write if `initial_size_bytes` is larger
474    ///   than `self.accessible`.
475    /// * For `MemoryStyle::Static` linear memory may be made `PROT_NONE` if
476    ///   `self.accessible` is larger than `initial_size_bytes`.
477    /// * If no image was previously in place or if the wrong image was
478    ///   previously in place then `mmap` may be used to setup the initial
479    ///   image.
480    pub(crate) fn instantiate(
481        &mut self,
482        initial_size_bytes: usize,
483        maybe_image: Option<&Arc<MemoryImage>>,
484        ty: &wasmtime_environ::Memory,
485        memory_tunables: &MemoryTunables<'_>,
486    ) -> Result<()> {
487        assert!(!self.dirty);
488        assert!(
489            initial_size_bytes <= self.static_size,
490            "initial_size_bytes <= self.static_size failed: \
491             initial_size_bytes={initial_size_bytes}, self.static_size={}",
492            self.static_size
493        );
494        let initial_size_bytes_page_aligned =
495            HostAlignedByteCount::new_rounded_up(initial_size_bytes)?;
496
497        // First order of business is to blow away the previous linear memory
498        // image if it doesn't match the image specified here. If one is
499        // detected then it's reset with anonymous memory which means that all
500        // of memory up to `self.accessible` will now be read/write and zero.
501        //
502        // Note that this intentionally a "small mmap" which only covers the
503        // extent of the prior initialization image in order to preserve
504        // resident memory that might come before or after the image.
505        let images_equal = match (self.image.as_ref(), maybe_image) {
506            (Some(a), Some(b)) if Arc::ptr_eq(a, b) => true,
507            (None, None) => true,
508            _ => false,
509        };
510        if !images_equal {
511            self.remove_image()?;
512        }
513
514        // The next order of business is to ensure that `self.accessible` is
515        // appropriate. First up is to grow the read/write portion of memory if
516        // it's not large enough to accommodate `initial_size_bytes`.
517        if self.accessible < initial_size_bytes_page_aligned {
518            self.set_protection(self.accessible..initial_size_bytes_page_aligned, true)?;
519            self.accessible = initial_size_bytes_page_aligned;
520        }
521
522        // If (1) the accessible region is not in its initial state, and (2) the
523        // memory relies on virtual memory at all (i.e. has offset guard
524        // pages), then we need to reset memory protections. Put another way,
525        // the only time it is safe to not reset protections is when we are
526        // using dynamic memory without any guard pages.
527        let host_page_size_log2 = u8::try_from(host_page_size().ilog2()).unwrap();
528        if initial_size_bytes_page_aligned < self.accessible
529            && (memory_tunables.guard_size() > 0
530                || ty.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2))
531        {
532            self.set_protection(initial_size_bytes_page_aligned..self.accessible, false)?;
533            self.accessible = initial_size_bytes_page_aligned;
534        }
535
536        // Now that memory is sized appropriately the final operation is to
537        // place the new image into linear memory. Note that this operation is
538        // skipped if `self.image` matches `maybe_image`.
539        assert!(initial_size_bytes <= self.accessible.byte_count());
540        assert!(initial_size_bytes_page_aligned <= self.accessible);
541        if !images_equal {
542            if let Some(image) = maybe_image.as_ref() {
543                assert!(
544                    image
545                        .linear_memory_offset
546                        .checked_add(image.len)
547                        .unwrap()
548                        .byte_count()
549                        <= initial_size_bytes
550                );
551                if !image.len.is_zero() {
552                    unsafe {
553                        image.map_at(&self.base)?;
554                    }
555                    // `map_at` above `mmap`'d over part of this slot, which
556                    // reset those pages to the default protection key. Restore
557                    // this slot's key so the image is not left accessible to
558                    // every other stripe.
559                    self.reapply_pkey(image.linear_memory_offset, image.len, true)?;
560                }
561            }
562            self.image = maybe_image.cloned();
563        }
564
565        // Flag ourselves as `dirty` which means that the next operation on this
566        // slot is required to be `clear_and_remain_ready`.
567        self.dirty = true;
568
569        Ok(())
570    }
571
572    pub(crate) fn remove_image(&mut self) -> Result<()> {
573        if let Some(image) = &self.image {
574            unsafe {
575                image.remap_as_zeros_at(self.base.as_mut_ptr())?;
576            }
577            // As in `instantiate`, the `mmap` above dropped this slot's
578            // protection key over the image's range, so restore it.
579            self.reapply_pkey(image.linear_memory_offset, image.len, true)?;
580            self.image = None;
581        }
582        Ok(())
583    }
584
585    /// Resets this linear memory slot back to a "pristine state".
586    ///
587    /// This will reset the memory back to its original contents on Linux or
588    /// reset the contents back to zero on other platforms. The `keep_resident`
589    /// argument is the maximum amount of memory to keep resident in this
590    /// process's memory on Linux. Up to that much memory will be `memset` to
591    /// zero where the rest of it will be reset or released with `madvise`.
592    ///
593    /// Returns the number of bytes still resident in memory after this function
594    /// has returned.
595
596    #[cfg_attr(
597        not(feature = "pooling-allocator"),
598        expect(dead_code, reason = "only used by the pooling allocator")
599    )]
600    pub(crate) fn clear_and_remain_ready(
601        &mut self,
602        pagemap: Option<&PageMap>,
603        keep_resident: HostAlignedByteCount,
604        decommit: impl FnMut(*mut u8, usize),
605    ) -> Result<usize> {
606        assert!(self.dirty);
607
608        let bytes_resident =
609            unsafe { self.reset_all_memory_contents(pagemap, keep_resident, decommit)? };
610
611        self.dirty = false;
612        Ok(bytes_resident)
613    }
614
615    #[cfg_attr(
616        not(feature = "pooling-allocator"),
617        expect(dead_code, reason = "only used by the pooling allocator")
618    )]
619    unsafe fn reset_all_memory_contents(
620        &mut self,
621        pagemap: Option<&PageMap>,
622        keep_resident: HostAlignedByteCount,
623        decommit: impl FnMut(*mut u8, usize),
624    ) -> Result<usize> {
625        match vm::decommit_behavior() {
626            DecommitBehavior::Zero => {
627                // If we're not on Linux then there's no generic platform way to
628                // reset memory back to its original state, so instead reset memory
629                // back to entirely zeros with an anonymous backing.
630                //
631                // Additionally the previous image, if any, is dropped here
632                // since it's no longer applicable to this mapping.
633                self.reset_with_anon_memory()?;
634                Ok(0)
635            }
636            DecommitBehavior::RestoreOriginalMapping => {
637                let bytes_resident =
638                    unsafe { self.reset_with_original_mapping(pagemap, keep_resident, decommit) };
639                Ok(bytes_resident)
640            }
641        }
642    }
643
644    #[cfg_attr(
645        not(feature = "pooling-allocator"),
646        expect(dead_code, reason = "only used by the pooling allocator")
647    )]
648    unsafe fn reset_with_original_mapping(
649        &mut self,
650        pagemap: Option<&PageMap>,
651        keep_resident: HostAlignedByteCount,
652        decommit: impl FnMut(*mut u8, usize),
653    ) -> usize {
654        assert_eq!(
655            vm::decommit_behavior(),
656            DecommitBehavior::RestoreOriginalMapping
657        );
658
659        unsafe {
660            return match &self.image {
661                // If there's a backing image then manually resetting a region
662                // is a bit trickier than without an image, so delegate to the
663                // helper function below.
664                Some(image) => reset_with_pagemap(
665                    pagemap,
666                    self.base.as_mut_ptr(),
667                    self.accessible,
668                    keep_resident,
669                    |region| manually_reset_region(self.base.as_mut_ptr().addr(), image, region),
670                    decommit,
671                ),
672
673                // If there's no memory image for this slot then pages are always
674                // manually reset back to zero or given to `decommit`.
675                None => reset_with_pagemap(
676                    pagemap,
677                    self.base.as_mut_ptr(),
678                    self.accessible,
679                    keep_resident,
680                    |region| region.fill(0),
681                    decommit,
682                ),
683            };
684        }
685
686        /// Manually resets `region` back to its original contents as specified
687        /// in `image`.
688        ///
689        /// This assumes that the original mmap starts at `base_addr` and
690        /// `region` is a subslice within the original mmap.
691        ///
692        /// # Panics
693        ///
694        /// Panics if `base_addr` is not the right index due to the various
695        /// indexing calculations below.
696        fn manually_reset_region(base_addr: usize, image: &MemoryImage, mut region: &mut [u8]) {
697            let image_start = image.linear_memory_offset.byte_count();
698            let image_end = image_start + image.len.byte_count();
699            let mut region_start = region.as_ptr().addr() - base_addr;
700            let region_end = region_start + region.len();
701            let image_bytes = image.module_source.wasm_data();
702            let image_bytes = &image_bytes[image.module_source_offset..][..image.len.byte_count()];
703
704            // 1. Zero out the part before the image (if any).
705            if let Some(len_before_image) = image_start.checked_sub(region_start) {
706                let len = len_before_image.min(region.len());
707                let (a, b) = region.split_at_mut(len);
708                a.fill(0);
709                region = b;
710                region_start += len;
711
712                if region.is_empty() {
713                    return;
714                }
715            }
716
717            debug_assert_eq!(region_end - region_start, region.len());
718            debug_assert!(region_start >= image_start);
719
720            // 2. Copy the original bytes from the image for the part that
721            //    overlaps with the image.
722            if let Some(len_in_image) = image_end.checked_sub(region_start) {
723                let len = len_in_image.min(region.len());
724                let (a, b) = region.split_at_mut(len);
725                a.copy_from_slice(&image_bytes[region_start - image_start..][..len]);
726                region = b;
727                region_start += len;
728
729                if region.is_empty() {
730                    return;
731                }
732            }
733
734            debug_assert_eq!(region_end - region_start, region.len());
735            debug_assert!(region_start >= image_end);
736
737            // 3. Zero out the part after the image.
738            region.fill(0);
739        }
740    }
741
742    fn set_protection(&self, range: Range<HostAlignedByteCount>, readwrite: bool) -> Result<()> {
743        let len = range
744            .end
745            .checked_sub(range.start)
746            .expect("range.start <= range.end");
747        assert!(range.end.byte_count() <= self.static_size);
748        if len.is_zero() {
749            return Ok(());
750        }
751
752        // TODO: use Mmap to change memory permissions instead of these free
753        // functions.
754        unsafe {
755            let start = self.base.as_mut_ptr().add(range.start.byte_count());
756            if readwrite {
757                vm::expose_existing_mapping(start, len.byte_count())?;
758            } else {
759                vm::hide_existing_mapping(start, len.byte_count())?;
760            }
761        }
762
763        Ok(())
764    }
765
766    /// Re-color `offset..offset + len` within this slot with this slot's MPK
767    /// protection key, if any.
768    ///
769    /// This is a no-op unless the pooling allocator is striping memory with
770    /// protection keys. It must be called after every `mmap` that lands inside
771    /// this slot: `mmap` associates the pages it replaces with the default key
772    /// 0, which is accessible regardless of which stripe is currently active,
773    /// so skipping this would let one instance read and write another
774    /// instance's memory.
775    ///
776    /// Note that `mprotect` preserves the existing key, so `set_protection`
777    /// does not need this treatment.
778    fn reapply_pkey(
779        &self,
780        offset: HostAlignedByteCount,
781        len: HostAlignedByteCount,
782        readwrite: bool,
783    ) -> Result<()> {
784        let Some(pkey) = self.pkey else {
785            return Ok(());
786        };
787        if len.is_zero() {
788            return Ok(());
789        }
790        // `mmap` rounds lengths up to a page boundary, so the restored range is
791        // allowed to extend to the end of the slot's final page.
792        debug_assert!(
793            offset.byte_count() + len.byte_count()
794                <= self.static_size.next_multiple_of(host_page_size())
795        );
796        unsafe {
797            let start = self.base.as_mut_ptr().add(offset.byte_count());
798            pkey.reprotect(start.addr(), len.byte_count(), readwrite)?;
799        }
800        Ok(())
801    }
802
803    pub(crate) fn has_image(&self) -> bool {
804        self.image.is_some()
805    }
806
807    #[cfg_attr(
808        not(feature = "pooling-allocator"),
809        expect(dead_code, reason = "only used by the pooling allocator")
810    )]
811    pub(crate) fn is_dirty(&self) -> bool {
812        self.dirty
813    }
814
815    /// Map anonymous zeroed memory across the whole slot,
816    /// inaccessible. Used both during instantiate and during drop.
817    pub(crate) fn reset_with_anon_memory(&mut self) -> Result<()> {
818        if self.static_size == 0 {
819            assert!(self.image.is_none());
820            assert_eq!(self.accessible, 0);
821            return Ok(());
822        }
823
824        unsafe {
825            vm::erase_existing_mapping(self.base.as_mut_ptr(), self.static_size)?;
826        }
827
828        // The `mmap` above covers the whole slot and left it inaccessible, so
829        // restore this slot's protection key across the same range.
830        let static_size = HostAlignedByteCount::new_rounded_up(self.static_size)?;
831        self.reapply_pkey(HostAlignedByteCount::ZERO, static_size, false)?;
832
833        self.image = None;
834        self.accessible = HostAlignedByteCount::ZERO;
835
836        Ok(())
837    }
838}
839
840#[cfg(all(test, target_os = "linux", not(miri), feature = "pooling-allocator"))]
841mod test {
842    use super::*;
843    use crate::runtime::vm::mmap::{AlignedLength, Mmap};
844    use crate::runtime::vm::sys::vm::{decommit_pages, iovec};
845    use crate::runtime::vm::{HostAlignedByteCount, MmapVec, host_page_size};
846    use std::sync::Arc;
847    use wasmtime_environ::{IndexType, Limits, Memory, MemoryKind, Tunables};
848
849    fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {
850        // offset must be a multiple of the page size.
851        let linear_memory_offset =
852            HostAlignedByteCount::new(offset).expect("offset is page-aligned");
853        // The image length is rounded up to the nearest page size
854        let image_len = HostAlignedByteCount::new_rounded_up(data.len()).unwrap();
855
856        let mut source = TestDataSource {
857            data: vec![0; image_len.byte_count()],
858        };
859        source.data[..data.len()].copy_from_slice(data);
860
861        return Ok(MemoryImage {
862            source: MemoryImageSource::from_data(data)?.unwrap(),
863            len: image_len,
864            source_offset: 0,
865            linear_memory_offset,
866            module_source: Arc::new(source),
867            module_source_offset: 0,
868        });
869
870        struct TestDataSource {
871            data: Vec<u8>,
872        }
873
874        impl ModuleMemoryImageSource for TestDataSource {
875            fn wasm_data(&self) -> &[u8] {
876                &self.data
877            }
878            fn mmap(&self) -> Option<&MmapVec> {
879                None
880            }
881        }
882    }
883
884    fn decommit(base: *mut u8, len: usize) {
885        unsafe {
886            decommit_pages(&[iovec {
887                iov_base: base.cast(),
888                iov_len: len,
889            }])
890            .unwrap();
891        }
892    }
893
894    fn dummy_memory() -> Memory {
895        Memory {
896            idx_type: IndexType::I32,
897            limits: Limits { min: 0, max: None },
898            shared: false,
899            page_size_log2: Memory::DEFAULT_PAGE_SIZE_LOG2,
900        }
901    }
902
903    fn mmap_4mib_inaccessible() -> Arc<Mmap<AlignedLength>> {
904        let four_mib = HostAlignedByteCount::new(4 << 20).expect("4 MiB is page aligned");
905        Arc::new(Mmap::accessible_reserved(HostAlignedByteCount::ZERO, four_mib).unwrap())
906    }
907
908    /// Presents a part of an mmap as a mutable slice within a callback.
909    ///
910    /// The callback ensures that the reference no longer lives after the
911    /// function is done.
912    ///
913    /// # Safety
914    ///
915    /// The caller must ensure that during this function call, the only way this
916    /// region of memory is not accessed by (read from or written to) is via the
917    /// reference. Making the callback `'static` goes some way towards ensuring
918    /// that, but it's still possible to squirrel away a reference into global
919    /// state. So don't do that.
920    unsafe fn with_slice_mut(
921        mmap: &Arc<Mmap<AlignedLength>>,
922        range: Range<usize>,
923        f: impl FnOnce(&mut [u8]) + 'static,
924    ) {
925        let ptr = mmap.as_ptr().cast_mut();
926        let slice = unsafe {
927            core::slice::from_raw_parts_mut(ptr.add(range.start), range.end - range.start)
928        };
929        f(slice);
930    }
931
932    #[test]
933    fn instantiate_no_image() {
934        let ty = dummy_memory();
935        let tunables = Tunables {
936            memory_reservation: 4 << 30,
937            ..Tunables::default_miri()
938        };
939        // 4 MiB mmap'd area, not accessible
940        let mmap = mmap_4mib_inaccessible();
941        // Create a MemoryImageSlot on top of it
942        let mut memfd = MemoryImageSlot::create(
943            mmap.zero_offset(),
944            HostAlignedByteCount::ZERO,
945            4 << 20,
946            None,
947        );
948        assert!(!memfd.is_dirty());
949        // instantiate with 64 KiB initial size
950        memfd
951            .instantiate(
952                64 << 10,
953                None,
954                &ty,
955                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
956            )
957            .unwrap();
958        assert!(memfd.is_dirty());
959
960        // We should be able to access this 64 KiB (try both ends) and
961        // it should consist of zeroes.
962        unsafe {
963            with_slice_mut(&mmap, 0..65536, |slice| {
964                assert_eq!(0, slice[0]);
965                assert_eq!(0, slice[65535]);
966                slice[1024] = 42;
967                assert_eq!(42, slice[1024]);
968            });
969        }
970
971        // grow the heap
972        memfd.set_heap_limit(128 << 10).unwrap();
973        let slice = unsafe { mmap.slice(0..1 << 20) };
974        assert_eq!(42, slice[1024]);
975        assert_eq!(0, slice[131071]);
976        // instantiate again; we should see zeroes, even as the
977        // reuse-anon-mmap-opt kicks in
978        memfd
979            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
980            .unwrap();
981        assert!(!memfd.is_dirty());
982        memfd
983            .instantiate(
984                64 << 10,
985                None,
986                &ty,
987                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
988            )
989            .unwrap();
990        let slice = unsafe { mmap.slice(0..65536) };
991        assert_eq!(0, slice[1024]);
992    }
993
994    #[test]
995    fn instantiate_image() {
996        let page_size = host_page_size();
997        let ty = dummy_memory();
998        let tunables = Tunables {
999            memory_reservation: 4 << 30,
1000            ..Tunables::default_miri()
1001        };
1002        // 4 MiB mmap'd area, not accessible
1003        let mmap = mmap_4mib_inaccessible();
1004        // Create a MemoryImageSlot on top of it
1005        let mut memfd = MemoryImageSlot::create(
1006            mmap.zero_offset(),
1007            HostAlignedByteCount::ZERO,
1008            4 << 20,
1009            None,
1010        );
1011        // Create an image with some data.
1012        let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap());
1013        // Instantiate with this image
1014        memfd
1015            .instantiate(
1016                64 << 10,
1017                Some(&image),
1018                &ty,
1019                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1020            )
1021            .unwrap();
1022        assert!(memfd.has_image());
1023
1024        unsafe {
1025            with_slice_mut(&mmap, 0..65536, move |slice| {
1026                assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1027                slice[page_size] = 5;
1028            });
1029        }
1030
1031        // Clear and re-instantiate same image
1032        memfd
1033            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1034            .unwrap();
1035        memfd
1036            .instantiate(
1037                64 << 10,
1038                Some(&image),
1039                &ty,
1040                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1041            )
1042            .unwrap();
1043        let slice = unsafe { mmap.slice(0..65536) };
1044        assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1045
1046        // Clear and re-instantiate no image
1047        memfd
1048            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1049            .unwrap();
1050        memfd
1051            .instantiate(
1052                64 << 10,
1053                None,
1054                &ty,
1055                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1056            )
1057            .unwrap();
1058        assert!(!memfd.has_image());
1059        let slice = unsafe { mmap.slice(0..65536) };
1060        assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);
1061
1062        // Clear and re-instantiate image again
1063        memfd
1064            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1065            .unwrap();
1066        memfd
1067            .instantiate(
1068                64 << 10,
1069                Some(&image),
1070                &ty,
1071                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1072            )
1073            .unwrap();
1074        let slice = unsafe { mmap.slice(0..65536) };
1075        assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1076
1077        // Create another image with different data.
1078        let image2 = Arc::new(create_memfd_with_data(page_size, &[10, 11, 12, 13]).unwrap());
1079        memfd
1080            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1081            .unwrap();
1082        memfd
1083            .instantiate(
1084                128 << 10,
1085                Some(&image2),
1086                &ty,
1087                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1088            )
1089            .unwrap();
1090        let slice = unsafe { mmap.slice(0..65536) };
1091        assert_eq!(&[10, 11, 12, 13], &slice[page_size..][..4]);
1092
1093        // Instantiate the original image again; we should notice it's
1094        // a different image and not reuse the mappings.
1095        memfd
1096            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1097            .unwrap();
1098        memfd
1099            .instantiate(
1100                64 << 10,
1101                Some(&image),
1102                &ty,
1103                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1104            )
1105            .unwrap();
1106        let slice = unsafe { mmap.slice(0..65536) };
1107        assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1108    }
1109
1110    #[test]
1111    #[cfg(target_os = "linux")]
1112    fn memset_instead_of_madvise() {
1113        let page_size = host_page_size();
1114        let ty = dummy_memory();
1115        let tunables = Tunables {
1116            memory_reservation: 100 << 16,
1117            ..Tunables::default_miri()
1118        };
1119        let mmap = mmap_4mib_inaccessible();
1120        let mut memfd = MemoryImageSlot::create(
1121            mmap.zero_offset(),
1122            HostAlignedByteCount::ZERO,
1123            4 << 20,
1124            None,
1125        );
1126
1127        // Test basics with the image
1128        for image_off in [0, page_size, page_size * 2] {
1129            let image = Arc::new(create_memfd_with_data(image_off, &[1, 2, 3, 4]).unwrap());
1130            for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {
1131                let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();
1132                memfd
1133                    .instantiate(
1134                        64 << 10,
1135                        Some(&image),
1136                        &ty,
1137                        &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1138                    )
1139                    .unwrap();
1140                assert!(memfd.has_image());
1141
1142                unsafe {
1143                    with_slice_mut(&mmap, 0..64 << 10, move |slice| {
1144                        if image_off > 0 {
1145                            assert_eq!(slice[image_off - 1], 0);
1146                        }
1147                        assert_eq!(slice[image_off + 5], 0);
1148                        assert_eq!(&[1, 2, 3, 4], &slice[image_off..][..4]);
1149                        slice[image_off] = 5;
1150                        assert_eq!(&[5, 2, 3, 4], &slice[image_off..][..4]);
1151                    })
1152                };
1153
1154                memfd
1155                    .clear_and_remain_ready(None, amt_to_memset, decommit)
1156                    .unwrap();
1157            }
1158        }
1159
1160        // Test without an image
1161        for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {
1162            let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();
1163            memfd
1164                .instantiate(
1165                    64 << 10,
1166                    None,
1167                    &ty,
1168                    &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1169                )
1170                .unwrap();
1171
1172            unsafe {
1173                with_slice_mut(&mmap, 0..64 << 10, |slice| {
1174                    for chunk in slice.chunks_mut(1024) {
1175                        assert_eq!(chunk[0], 0);
1176                        chunk[0] = 5;
1177                    }
1178                });
1179            }
1180            memfd
1181                .clear_and_remain_ready(None, amt_to_memset, decommit)
1182                .unwrap();
1183        }
1184    }
1185
1186    #[test]
1187    #[cfg(target_os = "linux")]
1188    fn dynamic() {
1189        let page_size = host_page_size();
1190        let ty = dummy_memory();
1191        let tunables = Tunables {
1192            memory_reservation: 0,
1193            memory_reservation_for_growth: 200,
1194            ..Tunables::default_miri()
1195        };
1196
1197        let mmap = mmap_4mib_inaccessible();
1198        let mut memfd = MemoryImageSlot::create(
1199            mmap.zero_offset(),
1200            HostAlignedByteCount::ZERO,
1201            4 << 20,
1202            None,
1203        );
1204        let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap());
1205        let initial = 64 << 10;
1206
1207        // Instantiate the image and test that memory remains accessible after
1208        // it's cleared.
1209        memfd
1210            .instantiate(
1211                initial,
1212                Some(&image),
1213                &ty,
1214                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1215            )
1216            .unwrap();
1217        assert!(memfd.has_image());
1218
1219        unsafe {
1220            with_slice_mut(&mmap, 0..(64 << 10) + page_size, move |slice| {
1221                assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1222                slice[page_size] = 5;
1223                assert_eq!(&[5, 2, 3, 4], &slice[page_size..][..4]);
1224            });
1225        }
1226
1227        memfd
1228            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1229            .unwrap();
1230        let slice = unsafe { mmap.slice(0..(64 << 10) + page_size) };
1231        assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1232
1233        // Re-instantiate make sure it preserves memory. Grow a bit and set data
1234        // beyond the initial size.
1235        memfd
1236            .instantiate(
1237                initial,
1238                Some(&image),
1239                &ty,
1240                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1241            )
1242            .unwrap();
1243        assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1244
1245        memfd.set_heap_limit(initial * 2).unwrap();
1246
1247        unsafe {
1248            with_slice_mut(&mmap, 0..(64 << 10) + page_size, move |slice| {
1249                assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1250                slice[initial] = 100;
1251                assert_eq!(&[100, 0], &slice[initial..initial + 2]);
1252            });
1253        }
1254
1255        memfd
1256            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1257            .unwrap();
1258
1259        // Test that memory is still accessible, but it's been reset
1260        assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1261
1262        // Instantiate again, and again memory beyond the initial size should
1263        // still be accessible. Grow into it again and make sure it works.
1264        memfd
1265            .instantiate(
1266                initial,
1267                Some(&image),
1268                &ty,
1269                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1270            )
1271            .unwrap();
1272        assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1273        memfd.set_heap_limit(initial * 2).unwrap();
1274
1275        unsafe {
1276            with_slice_mut(&mmap, 0..(64 << 10) + page_size, move |slice| {
1277                assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1278                slice[initial] = 100;
1279                assert_eq!(&[100, 0], &slice[initial..initial + 2]);
1280            });
1281        }
1282
1283        memfd
1284            .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1285            .unwrap();
1286
1287        // Reset the image to none and double-check everything is back to zero
1288        memfd
1289            .instantiate(
1290                64 << 10,
1291                None,
1292                &ty,
1293                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1294            )
1295            .unwrap();
1296        assert!(!memfd.has_image());
1297        assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);
1298        assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1299    }
1300
1301    #[test]
1302    fn reset_with_pagemap() {
1303        let page_size = host_page_size();
1304        let ty = dummy_memory();
1305        let tunables = Tunables {
1306            memory_reservation: 100 << 16,
1307            ..Tunables::default_miri()
1308        };
1309        let mmap = mmap_4mib_inaccessible();
1310        let mmap_len = page_size * 9;
1311        let mut memfd = MemoryImageSlot::create(
1312            mmap.zero_offset(),
1313            HostAlignedByteCount::ZERO,
1314            mmap_len,
1315            None,
1316        );
1317        let pagemap = PageMap::new();
1318        let pagemap = pagemap.as_ref();
1319
1320        let mut data = vec![0; 3 * page_size];
1321        for (i, chunk) in data.chunks_mut(page_size).enumerate() {
1322            for slot in chunk {
1323                *slot = u8::try_from(i + 1).unwrap();
1324            }
1325        }
1326        let image = Arc::new(create_memfd_with_data(3 * page_size, &data).unwrap());
1327
1328        memfd
1329            .instantiate(
1330                mmap_len,
1331                Some(&image),
1332                &ty,
1333                &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1334            )
1335            .unwrap();
1336
1337        let keep_resident = HostAlignedByteCount::new(mmap_len).unwrap();
1338        let assert_pristine_after_reset = |memfd: &mut MemoryImageSlot| unsafe {
1339            // Wipe the image, keeping some bytes resident.
1340            memfd
1341                .clear_and_remain_ready(pagemap, keep_resident, decommit)
1342                .unwrap();
1343
1344            // Double check that the contents of memory are as expected after
1345            // reset.
1346            with_slice_mut(&mmap, 0..mmap_len, move |slice| {
1347                for (i, chunk) in slice.chunks(page_size).enumerate() {
1348                    let expected = match i {
1349                        0..3 => 0,
1350                        3..6 => u8::try_from(i).unwrap() - 2,
1351                        6..9 => 0,
1352                        _ => unreachable!(),
1353                    };
1354                    for slot in chunk {
1355                        assert_eq!(*slot, expected);
1356                    }
1357                }
1358            });
1359
1360            // Re-instantiate, but then wipe the image entirely by keeping
1361            // nothing resident.
1362            memfd
1363                .instantiate(
1364                    mmap_len,
1365                    Some(&image),
1366                    &ty,
1367                    &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1368                )
1369                .unwrap();
1370            memfd
1371                .clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, decommit)
1372                .unwrap();
1373
1374            // Next re-instantiate a final time to get used for the next test.
1375            memfd
1376                .instantiate(
1377                    mmap_len,
1378                    Some(&image),
1379                    &ty,
1380                    &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1381                )
1382                .unwrap();
1383        };
1384
1385        let write_page = |_memfd: &mut MemoryImageSlot, page: usize| unsafe {
1386            with_slice_mut(
1387                &mmap,
1388                page * page_size..(page + 1) * page_size,
1389                move |slice| slice.fill(0xff),
1390            );
1391        };
1392
1393        // Test various combinations of dirty pages and regions. For example
1394        // test a dirty region of memory entirely in the zero-initialized zone
1395        // before/after the image and also test when the dirty region straddles
1396        // just the start of the image, just the end of the image, both ends,
1397        // and is entirely contained in just the image.
1398        assert_pristine_after_reset(&mut memfd);
1399
1400        for i in 0..9 {
1401            write_page(&mut memfd, i);
1402            assert_pristine_after_reset(&mut memfd);
1403        }
1404        write_page(&mut memfd, 0);
1405        write_page(&mut memfd, 1);
1406        assert_pristine_after_reset(&mut memfd);
1407        write_page(&mut memfd, 1);
1408        assert_pristine_after_reset(&mut memfd);
1409        write_page(&mut memfd, 2);
1410        write_page(&mut memfd, 3);
1411        assert_pristine_after_reset(&mut memfd);
1412        write_page(&mut memfd, 3);
1413        write_page(&mut memfd, 4);
1414        write_page(&mut memfd, 5);
1415        assert_pristine_after_reset(&mut memfd);
1416        write_page(&mut memfd, 0);
1417        write_page(&mut memfd, 1);
1418        write_page(&mut memfd, 2);
1419        assert_pristine_after_reset(&mut memfd);
1420        write_page(&mut memfd, 0);
1421        write_page(&mut memfd, 3);
1422        write_page(&mut memfd, 6);
1423        assert_pristine_after_reset(&mut memfd);
1424        write_page(&mut memfd, 2);
1425        write_page(&mut memfd, 3);
1426        write_page(&mut memfd, 4);
1427        write_page(&mut memfd, 5);
1428        write_page(&mut memfd, 6);
1429        assert_pristine_after_reset(&mut memfd);
1430        write_page(&mut memfd, 4);
1431        write_page(&mut memfd, 5);
1432        write_page(&mut memfd, 6);
1433        write_page(&mut memfd, 7);
1434        assert_pristine_after_reset(&mut memfd);
1435        write_page(&mut memfd, 4);
1436        write_page(&mut memfd, 5);
1437        write_page(&mut memfd, 8);
1438        assert_pristine_after_reset(&mut memfd);
1439    }
1440}
wasmtime/runtime/vm/cow.rs

wasmtime/runtime/vm/
cow.rs
