wasmtime/runtime/vm/cow.rs
1//! Copy-on-write initialization support: creation of backing images for
2//! modules, and logic to support mapping these backing images into memory.
3
4use super::sys::DecommitBehavior;
5use crate::Engine;
6use crate::prelude::*;
7use crate::runtime::vm::mpk::ProtectionKey;
8use crate::runtime::vm::sys::vm::{self, MemoryImageSource, PageMap, reset_with_pagemap};
9use crate::runtime::vm::{
10 HostAlignedByteCount, MmapOffset, ModuleMemoryImageSource, host_page_size,
11};
12use alloc::sync::Arc;
13use core::fmt;
14use core::ops::Range;
15use wasmtime_environ::prelude::TryPrimaryMap;
16use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, MemoryTunables, Module};
17
18/// Backing images for memories in a module.
19///
20/// This is meant to be built once, when a module is first loaded/constructed,
21/// and then used many times for instantiation.
22pub struct ModuleMemoryImages {
23 memories: TryPrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryImage>>>,
24}
25
26impl ModuleMemoryImages {
27 /// Get the MemoryImage for a given memory.
28 pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
29 self.memories[defined_index].as_ref()
30 }
31}
32
33/// One backing image for one memory.
34pub struct MemoryImage {
35 /// The platform-specific source of this image.
36 ///
37 /// This might be a mapped `*.cwasm` file or on Unix it could also be a
38 /// `Memfd` as an anonymous file in memory on Linux. In either case this is
39 /// used as the backing-source for the CoW image.
40 source: MemoryImageSource,
41
42 /// Length of image, in bytes.
43 ///
44 /// Note that initial memory size may be larger; leading and trailing zeroes
45 /// are truncated (handled by backing fd).
46 ///
47 /// Must be a multiple of the system page size.
48 len: HostAlignedByteCount,
49
50 /// Image starts this many bytes into `source`.
51 ///
52 /// This is 0 for anonymous-backed memfd files and is the offset of the
53 /// data section in a `*.cwasm` file for `*.cwasm`-backed images.
54 ///
55 /// Must be a multiple of the system page size.
56 ///
57 /// ## Notes
58 ///
59 /// This currently isn't a `HostAlignedByteCount` because that's a usize and
60 /// this, being a file offset, is a u64.
61 source_offset: u64,
62
63 /// Image starts this many bytes into heap space.
64 ///
65 /// Must be a multiple of the system page size.
66 linear_memory_offset: HostAlignedByteCount,
67
68 /// The original source of data that this image is derived from.
69 module_source: Arc<dyn ModuleMemoryImageSource>,
70
71 /// The offset, within `module_source.wasm_data()`, that this image starts
72 /// at.
73 module_source_offset: usize,
74}
75
76impl MemoryImage {
77 fn new(
78 engine: &Engine,
79 page_size: u32,
80 linear_memory_offset: HostAlignedByteCount,
81 module_source: &Arc<impl ModuleMemoryImageSource>,
82 data_range: Range<usize>,
83 ) -> Result<Option<MemoryImage>> {
84 let assert_page_aligned = |val: usize| {
85 assert_eq!(val % (page_size as usize), 0);
86 };
87 // Sanity-check that various parameters are page-aligned.
88 let len =
89 HostAlignedByteCount::new(data_range.len()).expect("memory image data is page-aligned");
90
91 // If a backing `mmap` is present then `data` should be a sub-slice of
92 // the `mmap`. The sanity-checks here double-check that. Additionally
93 // compilation should have ensured that the `data` section is
94 // page-aligned within `mmap`, so that's also all double-checked here.
95 //
96 // Finally if the `mmap` itself comes from a backing file on disk, such
97 // as a `*.cwasm` file, then that's a valid source of data for the
98 // memory image so we simply return referencing that.
99 //
100 // Note that this path is platform-agnostic in the sense of all
101 // platforms we support support memory mapping copy-on-write data from
102 // files, but for now this is still a Linux-specific region of Wasmtime.
103 // Some work will be needed to get this file compiling for macOS and
104 // Windows.
105 let data = &module_source.wasm_data()[data_range.clone()];
106 if !engine.config().force_memory_init_memfd {
107 if let Some(mmap) = module_source.mmap() {
108 let start = mmap.as_ptr() as usize;
109 let end = start + mmap.len();
110 let data_start = data.as_ptr() as usize;
111 let data_end = data_start + data.len();
112 assert!(start <= data_start && data_end <= end);
113 assert_page_aligned(start);
114 assert_page_aligned(data_start);
115 assert_page_aligned(data_end);
116
117 #[cfg(feature = "std")]
118 if let Some(file) = mmap.original_file() {
119 if let Some(source) = MemoryImageSource::from_file(file) {
120 return Ok(Some(MemoryImage {
121 source,
122 source_offset: u64::try_from(data_start - start).unwrap(),
123 linear_memory_offset,
124 len,
125 module_source: module_source.clone(),
126 module_source_offset: data_range.start,
127 }));
128 }
129 }
130 }
131 }
132
133 // If `mmap` doesn't come from a file then platform-specific mechanisms
134 // may be used to place the data in a form that's amenable to an mmap.
135 if let Some(source) = MemoryImageSource::from_data(data)? {
136 return Ok(Some(MemoryImage {
137 source,
138 source_offset: 0,
139 linear_memory_offset,
140 len,
141 module_source: module_source.clone(),
142 module_source_offset: data_range.start,
143 }));
144 }
145
146 Ok(None)
147 }
148
149 unsafe fn map_at(&self, mmap_base: &MmapOffset) -> Result<()> {
150 unsafe {
151 mmap_base.map_image_at(
152 &self.source,
153 self.source_offset,
154 self.linear_memory_offset,
155 self.len,
156 )
157 }
158 }
159
160 unsafe fn remap_as_zeros_at(&self, base: *mut u8) -> Result<()> {
161 unsafe {
162 self.source.remap_as_zeros_at(
163 base.add(self.linear_memory_offset.byte_count()),
164 self.len.byte_count(),
165 )?;
166 }
167 Ok(())
168 }
169}
170
171impl ModuleMemoryImages {
172 /// Create a new `ModuleMemoryImages` for the given module. This can be
173 /// passed in as part of a `InstanceAllocationRequest` to speed up
174 /// instantiation and execution by using copy-on-write-backed memories.
175 pub fn new(
176 engine: &Engine,
177 module: &Module,
178 source: &Arc<impl ModuleMemoryImageSource>,
179 ) -> Result<Option<ModuleMemoryImages>> {
180 let map = match &module.memory_initialization {
181 MemoryInitialization::Static { map } => map,
182 _ => return Ok(None),
183 };
184 let mut memories = TryPrimaryMap::with_capacity(map.len())?;
185 let page_size = crate::runtime::vm::host_page_size();
186 let page_size = u32::try_from(page_size).unwrap();
187 for (memory_index, init) in map {
188 // mmap-based-initialization only works for defined memories with a
189 // known starting point of all zeros, so bail out if the mmeory is
190 // imported.
191 let defined_memory = match module.defined_memory_index(memory_index) {
192 Some(idx) => idx,
193 None => return Ok(None),
194 };
195
196 // If there's no initialization for this memory known then we don't
197 // need an image for the memory so push `None` and move on.
198 let (offset, runtime_index) = match init {
199 Some(init) => init,
200 None => {
201 memories.push(None)?;
202 continue;
203 }
204 };
205
206 let data_range = &module.runtime_data[*runtime_index];
207 let data_range = usize::try_from(data_range.start).unwrap()
208 ..usize::try_from(data_range.end).unwrap();
209
210 if module.memories[memory_index]
211 .minimum_byte_size()
212 .map_or(false, |mem_initial_len| {
213 *offset + u64::try_from(data_range.len()).unwrap() > mem_initial_len
214 })
215 {
216 // The image is rounded up to multiples of the host OS page
217 // size. But if Wasm is using a custom page size, the Wasm page
218 // size might be smaller than the host OS page size, and that
219 // rounding might have made the image larger than the Wasm
220 // memory's initial length. This is *probably* okay, since the
221 // rounding would have just introduced new runs of zeroes in the
222 // image, but out of an abundance of caution we don't generate
223 // CoW images in this scenario.
224 return Ok(None);
225 }
226
227 let offset_usize = match usize::try_from(*offset) {
228 Ok(offset) => offset,
229 Err(_) => return Ok(None),
230 };
231 let offset = HostAlignedByteCount::new(offset_usize)
232 .expect("memory init offset is a multiple of the host page size");
233
234 // If this creation fails then we fail creating
235 // `ModuleMemoryImages` since this memory couldn't be represented.
236 let image = match MemoryImage::new(engine, page_size, offset, source, data_range)? {
237 Some(image) => image,
238 None => return Ok(None),
239 };
240
241 let idx = memories.push(Some(try_new::<Arc<_>>(image)?))?;
242 assert_eq!(idx, defined_memory);
243 }
244
245 Ok(Some(ModuleMemoryImages { memories }))
246 }
247}
248
249/// Slot management of a copy-on-write image which can be reused for the pooling
250/// allocator.
251///
252/// This data structure manages a slot of linear memory, primarily in the
253/// pooling allocator, which optionally has a contiguous memory image in the
254/// middle of it. Pictorially this data structure manages a virtual memory
255/// region that looks like:
256///
257/// ```text
258/// +--------------------+-------------------+--------------+--------------+
259/// | anonymous | optional | anonymous | PROT_NONE |
260/// | zero | memory | zero | memory |
261/// | memory | image | memory | |
262/// +--------------------+-------------------+--------------+--------------+
263/// | <------+---------->
264/// |<-----+------------> \
265/// | \ image.len
266/// | \
267/// | image.linear_memory_offset
268/// |
269/// \
270/// self.base is this virtual address
271///
272/// <------------------+------------------------------------------------>
273/// \
274/// static_size
275///
276/// <------------------+---------------------------------->
277/// \
278/// accessible
279/// ```
280///
281/// When a `MemoryImageSlot` is created it's told what the `static_size` and
282/// `accessible` limits are. Initially there is assumed to be no image in linear
283/// memory.
284///
285/// When `MemoryImageSlot::instantiate` is called then the method will perform
286/// a "synchronization" to take the image from its prior state to the new state
287/// for the image specified. The first instantiation for example will mmap the
288/// heap image into place. Upon reuse of a slot nothing happens except possibly
289/// shrinking `self.accessible`. When a new image is used then the old image is
290/// mapped to anonymous zero memory and then the new image is mapped in place.
291///
292/// A `MemoryImageSlot` is either `dirty` or it isn't. When a `MemoryImageSlot`
293/// is dirty then it is assumed that any memory beneath `self.accessible` could
294/// have any value. Instantiation cannot happen into a `dirty` slot, however, so
295/// the `MemoryImageSlot::clear_and_remain_ready` returns this memory back to
296/// its original state to mark `dirty = false`. This is done by resetting all
297/// anonymous memory back to zero and the image itself back to its initial
298/// contents.
299///
300/// On Linux this is achieved with the `madvise(MADV_DONTNEED)` syscall. This
301/// syscall will release the physical pages back to the OS but retain the
302/// original mappings, effectively resetting everything back to its initial
303/// state. Non-linux platforms will replace all memory below `self.accessible`
304/// with a fresh zero'd mmap, meaning that reuse is effectively not supported.
305pub struct MemoryImageSlot {
306 /// The mmap and offset within it that contains the linear memory for this
307 /// slot.
308 base: MmapOffset,
309
310 /// The maximum static memory size which `self.accessible` can grow to.
311 static_size: usize,
312
313 /// An optional image that is currently being used in this linear memory.
314 ///
315 /// This can be `None` in which case memory is originally all zeros. When
316 /// `Some` the image describes where it's located within the image.
317 image: Option<Arc<MemoryImage>>,
318
319 /// The size of the heap that is readable and writable.
320 ///
321 /// Note that this may extend beyond the actual linear memory heap size in
322 /// the case of dynamic memories in use. Memory accesses to memory below
323 /// `self.accessible` may still page fault as pages are lazily brought in
324 /// but the faults will always be resolved by the kernel.
325 ///
326 /// Also note that this is always page-aligned.
327 accessible: HostAlignedByteCount,
328
329 /// Whether this slot may have "dirty" pages (pages written by an
330 /// instantiation). Set by `instantiate()` and cleared by
331 /// `clear_and_remain_ready()`, and used in assertions to ensure
332 /// those methods are called properly.
333 ///
334 /// Invariant: if !dirty, then this memory slot contains a clean
335 /// CoW mapping of `image`, if `Some(..)`, and anonymous-zero
336 /// memory beyond the image up to `static_size`. The addresses
337 /// from offset 0 to `self.accessible` are R+W and set to zero or the
338 /// initial image content, as appropriate. Everything between
339 /// `self.accessible` and `self.static_size` is inaccessible.
340 dirty: bool,
341
342 /// The MPK protection key that this slot's stripe was colored with, if the
343 /// pooling allocator is striping memory with protection keys.
344 ///
345 /// This must be re-applied after every `mmap` performed on this slot, since
346 /// `mmap` resets the affected pages back to the default key 0 which is
347 /// accessible from every stripe.
348 pkey: Option<ProtectionKey>,
349}
350
351impl fmt::Debug for MemoryImageSlot {
352 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
353 f.debug_struct("MemoryImageSlot")
354 .field("base", &self.base)
355 .field("static_size", &self.static_size)
356 .field("accessible", &self.accessible)
357 .field("dirty", &self.dirty)
358 .finish_non_exhaustive()
359 }
360}
361
362impl MemoryImageSlot {
363 /// Create a new MemoryImageSlot. Assumes that there is an anonymous
364 /// mmap backing in the given range to start.
365 ///
366 /// The `accessible` parameter describes how much of linear memory is
367 /// already mapped as R/W with all zero-bytes. The `static_size` value is
368 /// the maximum size of this image which `accessible` cannot grow beyond,
369 /// and all memory from `accessible` from `static_size` should be mapped as
370 /// `PROT_NONE` backed by zero-bytes.
371 pub(crate) fn create(
372 base: MmapOffset,
373 accessible: HostAlignedByteCount,
374 static_size: usize,
375 pkey: Option<ProtectionKey>,
376 ) -> Self {
377 MemoryImageSlot {
378 base,
379 static_size,
380 accessible,
381 image: None,
382 dirty: false,
383 pkey,
384 }
385 }
386
387 pub(crate) fn set_heap_limit(&mut self, size_bytes: usize) -> Result<()> {
388 let size_bytes_aligned = HostAlignedByteCount::new_rounded_up(size_bytes)?;
389 assert!(size_bytes <= self.static_size);
390 assert!(size_bytes_aligned.byte_count() <= self.static_size);
391
392 // If the heap limit already addresses accessible bytes then no syscalls
393 // are necessary since the data is already mapped into the process and
394 // waiting to go.
395 //
396 // This is used for "dynamic" memories where memory is not always
397 // decommitted during recycling (but it's still always reset).
398 if size_bytes_aligned <= self.accessible {
399 return Ok(());
400 }
401
402 // Otherwise use `mprotect` to make the new pages read/write.
403 self.set_protection(self.accessible..size_bytes_aligned, true)?;
404 self.accessible = size_bytes_aligned;
405
406 Ok(())
407 }
408
409 /// Prepares this slot for the instantiation of a new instance with the
410 /// provided linear memory image.
411 ///
412 /// The `initial_size_bytes` parameter indicates the required initial size
413 /// of the heap for the instance. The `maybe_image` is an optional initial
414 /// image for linear memory to contains. The `style` is the way compiled
415 /// code will be accessing this memory.
416 ///
417 /// The purpose of this method is to take a previously pristine slot
418 /// (`!self.dirty`) and transform its prior state into state necessary for
419 /// the given parameters. This could include, for example:
420 ///
421 /// * More memory may be made read/write if `initial_size_bytes` is larger
422 /// than `self.accessible`.
423 /// * For `MemoryStyle::Static` linear memory may be made `PROT_NONE` if
424 /// `self.accessible` is larger than `initial_size_bytes`.
425 /// * If no image was previously in place or if the wrong image was
426 /// previously in place then `mmap` may be used to setup the initial
427 /// image.
428 pub(crate) fn instantiate(
429 &mut self,
430 initial_size_bytes: usize,
431 maybe_image: Option<&Arc<MemoryImage>>,
432 ty: &wasmtime_environ::Memory,
433 memory_tunables: &MemoryTunables<'_>,
434 ) -> Result<()> {
435 assert!(!self.dirty);
436 assert!(
437 initial_size_bytes <= self.static_size,
438 "initial_size_bytes <= self.static_size failed: \
439 initial_size_bytes={initial_size_bytes}, self.static_size={}",
440 self.static_size
441 );
442 let initial_size_bytes_page_aligned =
443 HostAlignedByteCount::new_rounded_up(initial_size_bytes)?;
444
445 // First order of business is to blow away the previous linear memory
446 // image if it doesn't match the image specified here. If one is
447 // detected then it's reset with anonymous memory which means that all
448 // of memory up to `self.accessible` will now be read/write and zero.
449 //
450 // Note that this intentionally a "small mmap" which only covers the
451 // extent of the prior initialization image in order to preserve
452 // resident memory that might come before or after the image.
453 let images_equal = match (self.image.as_ref(), maybe_image) {
454 (Some(a), Some(b)) if Arc::ptr_eq(a, b) => true,
455 (None, None) => true,
456 _ => false,
457 };
458 if !images_equal {
459 self.remove_image()?;
460 }
461
462 // The next order of business is to ensure that `self.accessible` is
463 // appropriate. First up is to grow the read/write portion of memory if
464 // it's not large enough to accommodate `initial_size_bytes`.
465 if self.accessible < initial_size_bytes_page_aligned {
466 self.set_protection(self.accessible..initial_size_bytes_page_aligned, true)?;
467 self.accessible = initial_size_bytes_page_aligned;
468 }
469
470 // If (1) the accessible region is not in its initial state, and (2) the
471 // memory relies on virtual memory at all (i.e. has offset guard
472 // pages), then we need to reset memory protections. Put another way,
473 // the only time it is safe to not reset protections is when we are
474 // using dynamic memory without any guard pages.
475 let host_page_size_log2 = u8::try_from(host_page_size().ilog2()).unwrap();
476 if initial_size_bytes_page_aligned < self.accessible
477 && (memory_tunables.guard_size() > 0
478 || ty.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2))
479 {
480 self.set_protection(initial_size_bytes_page_aligned..self.accessible, false)?;
481 self.accessible = initial_size_bytes_page_aligned;
482 }
483
484 // Now that memory is sized appropriately the final operation is to
485 // place the new image into linear memory. Note that this operation is
486 // skipped if `self.image` matches `maybe_image`.
487 assert!(initial_size_bytes <= self.accessible.byte_count());
488 assert!(initial_size_bytes_page_aligned <= self.accessible);
489 if !images_equal {
490 if let Some(image) = maybe_image.as_ref() {
491 assert!(
492 image
493 .linear_memory_offset
494 .checked_add(image.len)
495 .unwrap()
496 .byte_count()
497 <= initial_size_bytes
498 );
499 if !image.len.is_zero() {
500 unsafe {
501 image.map_at(&self.base)?;
502 }
503 // `map_at` above `mmap`'d over part of this slot, which
504 // reset those pages to the default protection key. Restore
505 // this slot's key so the image is not left accessible to
506 // every other stripe.
507 self.reapply_pkey(image.linear_memory_offset, image.len, true)?;
508 }
509 }
510 self.image = maybe_image.cloned();
511 }
512
513 // Flag ourselves as `dirty` which means that the next operation on this
514 // slot is required to be `clear_and_remain_ready`.
515 self.dirty = true;
516
517 Ok(())
518 }
519
520 pub(crate) fn remove_image(&mut self) -> Result<()> {
521 if let Some(image) = &self.image {
522 unsafe {
523 image.remap_as_zeros_at(self.base.as_mut_ptr())?;
524 }
525 // As in `instantiate`, the `mmap` above dropped this slot's
526 // protection key over the image's range, so restore it.
527 self.reapply_pkey(image.linear_memory_offset, image.len, true)?;
528 self.image = None;
529 }
530 Ok(())
531 }
532
533 /// Resets this linear memory slot back to a "pristine state".
534 ///
535 /// This will reset the memory back to its original contents on Linux or
536 /// reset the contents back to zero on other platforms. The `keep_resident`
537 /// argument is the maximum amount of memory to keep resident in this
538 /// process's memory on Linux. Up to that much memory will be `memset` to
539 /// zero where the rest of it will be reset or released with `madvise`.
540 ///
541 /// Returns the number of bytes still resident in memory after this function
542 /// has returned.
543 #[allow(dead_code, reason = "only used in some cfgs")]
544 pub(crate) fn clear_and_remain_ready(
545 &mut self,
546 pagemap: Option<&PageMap>,
547 keep_resident: HostAlignedByteCount,
548 decommit: impl FnMut(*mut u8, usize),
549 ) -> Result<usize> {
550 assert!(self.dirty);
551
552 let bytes_resident =
553 unsafe { self.reset_all_memory_contents(pagemap, keep_resident, decommit)? };
554
555 self.dirty = false;
556 Ok(bytes_resident)
557 }
558
559 #[allow(dead_code, reason = "only used in some cfgs")]
560 unsafe fn reset_all_memory_contents(
561 &mut self,
562 pagemap: Option<&PageMap>,
563 keep_resident: HostAlignedByteCount,
564 decommit: impl FnMut(*mut u8, usize),
565 ) -> Result<usize> {
566 match vm::decommit_behavior() {
567 DecommitBehavior::Zero => {
568 // If we're not on Linux then there's no generic platform way to
569 // reset memory back to its original state, so instead reset memory
570 // back to entirely zeros with an anonymous backing.
571 //
572 // Additionally the previous image, if any, is dropped here
573 // since it's no longer applicable to this mapping.
574 self.reset_with_anon_memory()?;
575 Ok(0)
576 }
577 DecommitBehavior::RestoreOriginalMapping => {
578 let bytes_resident =
579 unsafe { self.reset_with_original_mapping(pagemap, keep_resident, decommit) };
580 Ok(bytes_resident)
581 }
582 }
583 }
584
585 #[allow(dead_code, reason = "only used in some cfgs")]
586 unsafe fn reset_with_original_mapping(
587 &mut self,
588 pagemap: Option<&PageMap>,
589 keep_resident: HostAlignedByteCount,
590 decommit: impl FnMut(*mut u8, usize),
591 ) -> usize {
592 assert_eq!(
593 vm::decommit_behavior(),
594 DecommitBehavior::RestoreOriginalMapping
595 );
596
597 unsafe {
598 return match &self.image {
599 // If there's a backing image then manually resetting a region
600 // is a bit trickier than without an image, so delegate to the
601 // helper function below.
602 Some(image) => reset_with_pagemap(
603 pagemap,
604 self.base.as_mut_ptr(),
605 self.accessible,
606 keep_resident,
607 |region| manually_reset_region(self.base.as_mut_ptr().addr(), image, region),
608 decommit,
609 ),
610
611 // If there's no memory image for this slot then pages are always
612 // manually reset back to zero or given to `decommit`.
613 None => reset_with_pagemap(
614 pagemap,
615 self.base.as_mut_ptr(),
616 self.accessible,
617 keep_resident,
618 |region| region.fill(0),
619 decommit,
620 ),
621 };
622 }
623
624 /// Manually resets `region` back to its original contents as specified
625 /// in `image`.
626 ///
627 /// This assumes that the original mmap starts at `base_addr` and
628 /// `region` is a subslice within the original mmap.
629 ///
630 /// # Panics
631 ///
632 /// Panics if `base_addr` is not the right index due to the various
633 /// indexing calculations below.
634 fn manually_reset_region(base_addr: usize, image: &MemoryImage, mut region: &mut [u8]) {
635 let image_start = image.linear_memory_offset.byte_count();
636 let image_end = image_start + image.len.byte_count();
637 let mut region_start = region.as_ptr().addr() - base_addr;
638 let region_end = region_start + region.len();
639 let image_bytes = image.module_source.wasm_data();
640 let image_bytes = &image_bytes[image.module_source_offset..][..image.len.byte_count()];
641
642 // 1. Zero out the part before the image (if any).
643 if let Some(len_before_image) = image_start.checked_sub(region_start) {
644 let len = len_before_image.min(region.len());
645 let (a, b) = region.split_at_mut(len);
646 a.fill(0);
647 region = b;
648 region_start += len;
649
650 if region.is_empty() {
651 return;
652 }
653 }
654
655 debug_assert_eq!(region_end - region_start, region.len());
656 debug_assert!(region_start >= image_start);
657
658 // 2. Copy the original bytes from the image for the part that
659 // overlaps with the image.
660 if let Some(len_in_image) = image_end.checked_sub(region_start) {
661 let len = len_in_image.min(region.len());
662 let (a, b) = region.split_at_mut(len);
663 a.copy_from_slice(&image_bytes[region_start - image_start..][..len]);
664 region = b;
665 region_start += len;
666
667 if region.is_empty() {
668 return;
669 }
670 }
671
672 debug_assert_eq!(region_end - region_start, region.len());
673 debug_assert!(region_start >= image_end);
674
675 // 3. Zero out the part after the image.
676 region.fill(0);
677 }
678 }
679
680 fn set_protection(&self, range: Range<HostAlignedByteCount>, readwrite: bool) -> Result<()> {
681 let len = range
682 .end
683 .checked_sub(range.start)
684 .expect("range.start <= range.end");
685 assert!(range.end.byte_count() <= self.static_size);
686 if len.is_zero() {
687 return Ok(());
688 }
689
690 // TODO: use Mmap to change memory permissions instead of these free
691 // functions.
692 unsafe {
693 let start = self.base.as_mut_ptr().add(range.start.byte_count());
694 if readwrite {
695 vm::expose_existing_mapping(start, len.byte_count())?;
696 } else {
697 vm::hide_existing_mapping(start, len.byte_count())?;
698 }
699 }
700
701 Ok(())
702 }
703
704 /// Re-color `offset..offset + len` within this slot with this slot's MPK
705 /// protection key, if any.
706 ///
707 /// This is a no-op unless the pooling allocator is striping memory with
708 /// protection keys. It must be called after every `mmap` that lands inside
709 /// this slot: `mmap` associates the pages it replaces with the default key
710 /// 0, which is accessible regardless of which stripe is currently active,
711 /// so skipping this would let one instance read and write another
712 /// instance's memory.
713 ///
714 /// Note that `mprotect` preserves the existing key, so `set_protection`
715 /// does not need this treatment.
716 fn reapply_pkey(
717 &self,
718 offset: HostAlignedByteCount,
719 len: HostAlignedByteCount,
720 readwrite: bool,
721 ) -> Result<()> {
722 let Some(pkey) = self.pkey else {
723 return Ok(());
724 };
725 if len.is_zero() {
726 return Ok(());
727 }
728 // `mmap` rounds lengths up to a page boundary, so the restored range is
729 // allowed to extend to the end of the slot's final page.
730 debug_assert!(
731 offset.byte_count() + len.byte_count()
732 <= self.static_size.next_multiple_of(host_page_size())
733 );
734 unsafe {
735 let start = self.base.as_mut_ptr().add(offset.byte_count());
736 pkey.reprotect(start.addr(), len.byte_count(), readwrite)?;
737 }
738 Ok(())
739 }
740
741 pub(crate) fn has_image(&self) -> bool {
742 self.image.is_some()
743 }
744
745 #[allow(dead_code, reason = "only used in some cfgs")]
746 pub(crate) fn is_dirty(&self) -> bool {
747 self.dirty
748 }
749
750 /// Map anonymous zeroed memory across the whole slot,
751 /// inaccessible. Used both during instantiate and during drop.
752 pub(crate) fn reset_with_anon_memory(&mut self) -> Result<()> {
753 if self.static_size == 0 {
754 assert!(self.image.is_none());
755 assert_eq!(self.accessible, 0);
756 return Ok(());
757 }
758
759 unsafe {
760 vm::erase_existing_mapping(self.base.as_mut_ptr(), self.static_size)?;
761 }
762
763 // The `mmap` above covers the whole slot and left it inaccessible, so
764 // restore this slot's protection key across the same range.
765 let static_size = HostAlignedByteCount::new_rounded_up(self.static_size)?;
766 self.reapply_pkey(HostAlignedByteCount::ZERO, static_size, false)?;
767
768 self.image = None;
769 self.accessible = HostAlignedByteCount::ZERO;
770
771 Ok(())
772 }
773}
774
775#[cfg(all(test, target_os = "linux", not(miri)))]
776mod test {
777 use super::*;
778 use crate::runtime::vm::mmap::{AlignedLength, Mmap};
779 use crate::runtime::vm::sys::vm::{decommit_pages, iovec};
780 use crate::runtime::vm::{HostAlignedByteCount, MmapVec, host_page_size};
781 use std::sync::Arc;
782 use wasmtime_environ::{IndexType, Limits, Memory, MemoryKind, Tunables};
783
784 fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {
785 // offset must be a multiple of the page size.
786 let linear_memory_offset =
787 HostAlignedByteCount::new(offset).expect("offset is page-aligned");
788 // The image length is rounded up to the nearest page size
789 let image_len = HostAlignedByteCount::new_rounded_up(data.len()).unwrap();
790
791 let mut source = TestDataSource {
792 data: vec![0; image_len.byte_count()],
793 };
794 source.data[..data.len()].copy_from_slice(data);
795
796 return Ok(MemoryImage {
797 source: MemoryImageSource::from_data(data)?.unwrap(),
798 len: image_len,
799 source_offset: 0,
800 linear_memory_offset,
801 module_source: Arc::new(source),
802 module_source_offset: 0,
803 });
804
805 struct TestDataSource {
806 data: Vec<u8>,
807 }
808
809 impl ModuleMemoryImageSource for TestDataSource {
810 fn wasm_data(&self) -> &[u8] {
811 &self.data
812 }
813 fn mmap(&self) -> Option<&MmapVec> {
814 None
815 }
816 }
817 }
818
819 fn decommit(base: *mut u8, len: usize) {
820 unsafe {
821 decommit_pages(&[iovec {
822 iov_base: base.cast(),
823 iov_len: len,
824 }])
825 .unwrap();
826 }
827 }
828
829 fn dummy_memory() -> Memory {
830 Memory {
831 idx_type: IndexType::I32,
832 limits: Limits { min: 0, max: None },
833 shared: false,
834 page_size_log2: Memory::DEFAULT_PAGE_SIZE_LOG2,
835 }
836 }
837
838 fn mmap_4mib_inaccessible() -> Arc<Mmap<AlignedLength>> {
839 let four_mib = HostAlignedByteCount::new(4 << 20).expect("4 MiB is page aligned");
840 Arc::new(Mmap::accessible_reserved(HostAlignedByteCount::ZERO, four_mib).unwrap())
841 }
842
843 /// Presents a part of an mmap as a mutable slice within a callback.
844 ///
845 /// The callback ensures that the reference no longer lives after the
846 /// function is done.
847 ///
848 /// # Safety
849 ///
850 /// The caller must ensure that during this function call, the only way this
851 /// region of memory is not accessed by (read from or written to) is via the
852 /// reference. Making the callback `'static` goes some way towards ensuring
853 /// that, but it's still possible to squirrel away a reference into global
854 /// state. So don't do that.
855 unsafe fn with_slice_mut(
856 mmap: &Arc<Mmap<AlignedLength>>,
857 range: Range<usize>,
858 f: impl FnOnce(&mut [u8]) + 'static,
859 ) {
860 let ptr = mmap.as_ptr().cast_mut();
861 let slice = unsafe {
862 core::slice::from_raw_parts_mut(ptr.add(range.start), range.end - range.start)
863 };
864 f(slice);
865 }
866
867 #[test]
868 fn instantiate_no_image() {
869 let ty = dummy_memory();
870 let tunables = Tunables {
871 memory_reservation: 4 << 30,
872 ..Tunables::default_miri()
873 };
874 // 4 MiB mmap'd area, not accessible
875 let mmap = mmap_4mib_inaccessible();
876 // Create a MemoryImageSlot on top of it
877 let mut memfd = MemoryImageSlot::create(
878 mmap.zero_offset(),
879 HostAlignedByteCount::ZERO,
880 4 << 20,
881 None,
882 );
883 assert!(!memfd.is_dirty());
884 // instantiate with 64 KiB initial size
885 memfd
886 .instantiate(
887 64 << 10,
888 None,
889 &ty,
890 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
891 )
892 .unwrap();
893 assert!(memfd.is_dirty());
894
895 // We should be able to access this 64 KiB (try both ends) and
896 // it should consist of zeroes.
897 unsafe {
898 with_slice_mut(&mmap, 0..65536, |slice| {
899 assert_eq!(0, slice[0]);
900 assert_eq!(0, slice[65535]);
901 slice[1024] = 42;
902 assert_eq!(42, slice[1024]);
903 });
904 }
905
906 // grow the heap
907 memfd.set_heap_limit(128 << 10).unwrap();
908 let slice = unsafe { mmap.slice(0..1 << 20) };
909 assert_eq!(42, slice[1024]);
910 assert_eq!(0, slice[131071]);
911 // instantiate again; we should see zeroes, even as the
912 // reuse-anon-mmap-opt kicks in
913 memfd
914 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
915 .unwrap();
916 assert!(!memfd.is_dirty());
917 memfd
918 .instantiate(
919 64 << 10,
920 None,
921 &ty,
922 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
923 )
924 .unwrap();
925 let slice = unsafe { mmap.slice(0..65536) };
926 assert_eq!(0, slice[1024]);
927 }
928
929 #[test]
930 fn instantiate_image() {
931 let page_size = host_page_size();
932 let ty = dummy_memory();
933 let tunables = Tunables {
934 memory_reservation: 4 << 30,
935 ..Tunables::default_miri()
936 };
937 // 4 MiB mmap'd area, not accessible
938 let mmap = mmap_4mib_inaccessible();
939 // Create a MemoryImageSlot on top of it
940 let mut memfd = MemoryImageSlot::create(
941 mmap.zero_offset(),
942 HostAlignedByteCount::ZERO,
943 4 << 20,
944 None,
945 );
946 // Create an image with some data.
947 let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap());
948 // Instantiate with this image
949 memfd
950 .instantiate(
951 64 << 10,
952 Some(&image),
953 &ty,
954 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
955 )
956 .unwrap();
957 assert!(memfd.has_image());
958
959 unsafe {
960 with_slice_mut(&mmap, 0..65536, move |slice| {
961 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
962 slice[page_size] = 5;
963 });
964 }
965
966 // Clear and re-instantiate same image
967 memfd
968 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
969 .unwrap();
970 memfd
971 .instantiate(
972 64 << 10,
973 Some(&image),
974 &ty,
975 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
976 )
977 .unwrap();
978 let slice = unsafe { mmap.slice(0..65536) };
979 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
980
981 // Clear and re-instantiate no image
982 memfd
983 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
984 .unwrap();
985 memfd
986 .instantiate(
987 64 << 10,
988 None,
989 &ty,
990 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
991 )
992 .unwrap();
993 assert!(!memfd.has_image());
994 let slice = unsafe { mmap.slice(0..65536) };
995 assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);
996
997 // Clear and re-instantiate image again
998 memfd
999 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1000 .unwrap();
1001 memfd
1002 .instantiate(
1003 64 << 10,
1004 Some(&image),
1005 &ty,
1006 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1007 )
1008 .unwrap();
1009 let slice = unsafe { mmap.slice(0..65536) };
1010 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1011
1012 // Create another image with different data.
1013 let image2 = Arc::new(create_memfd_with_data(page_size, &[10, 11, 12, 13]).unwrap());
1014 memfd
1015 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1016 .unwrap();
1017 memfd
1018 .instantiate(
1019 128 << 10,
1020 Some(&image2),
1021 &ty,
1022 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1023 )
1024 .unwrap();
1025 let slice = unsafe { mmap.slice(0..65536) };
1026 assert_eq!(&[10, 11, 12, 13], &slice[page_size..][..4]);
1027
1028 // Instantiate the original image again; we should notice it's
1029 // a different image and not reuse the mappings.
1030 memfd
1031 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1032 .unwrap();
1033 memfd
1034 .instantiate(
1035 64 << 10,
1036 Some(&image),
1037 &ty,
1038 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1039 )
1040 .unwrap();
1041 let slice = unsafe { mmap.slice(0..65536) };
1042 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1043 }
1044
1045 #[test]
1046 #[cfg(target_os = "linux")]
1047 fn memset_instead_of_madvise() {
1048 let page_size = host_page_size();
1049 let ty = dummy_memory();
1050 let tunables = Tunables {
1051 memory_reservation: 100 << 16,
1052 ..Tunables::default_miri()
1053 };
1054 let mmap = mmap_4mib_inaccessible();
1055 let mut memfd = MemoryImageSlot::create(
1056 mmap.zero_offset(),
1057 HostAlignedByteCount::ZERO,
1058 4 << 20,
1059 None,
1060 );
1061
1062 // Test basics with the image
1063 for image_off in [0, page_size, page_size * 2] {
1064 let image = Arc::new(create_memfd_with_data(image_off, &[1, 2, 3, 4]).unwrap());
1065 for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {
1066 let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();
1067 memfd
1068 .instantiate(
1069 64 << 10,
1070 Some(&image),
1071 &ty,
1072 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1073 )
1074 .unwrap();
1075 assert!(memfd.has_image());
1076
1077 unsafe {
1078 with_slice_mut(&mmap, 0..64 << 10, move |slice| {
1079 if image_off > 0 {
1080 assert_eq!(slice[image_off - 1], 0);
1081 }
1082 assert_eq!(slice[image_off + 5], 0);
1083 assert_eq!(&[1, 2, 3, 4], &slice[image_off..][..4]);
1084 slice[image_off] = 5;
1085 assert_eq!(&[5, 2, 3, 4], &slice[image_off..][..4]);
1086 })
1087 };
1088
1089 memfd
1090 .clear_and_remain_ready(None, amt_to_memset, decommit)
1091 .unwrap();
1092 }
1093 }
1094
1095 // Test without an image
1096 for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {
1097 let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();
1098 memfd
1099 .instantiate(
1100 64 << 10,
1101 None,
1102 &ty,
1103 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1104 )
1105 .unwrap();
1106
1107 unsafe {
1108 with_slice_mut(&mmap, 0..64 << 10, |slice| {
1109 for chunk in slice.chunks_mut(1024) {
1110 assert_eq!(chunk[0], 0);
1111 chunk[0] = 5;
1112 }
1113 });
1114 }
1115 memfd
1116 .clear_and_remain_ready(None, amt_to_memset, decommit)
1117 .unwrap();
1118 }
1119 }
1120
1121 #[test]
1122 #[cfg(target_os = "linux")]
1123 fn dynamic() {
1124 let page_size = host_page_size();
1125 let ty = dummy_memory();
1126 let tunables = Tunables {
1127 memory_reservation: 0,
1128 memory_reservation_for_growth: 200,
1129 ..Tunables::default_miri()
1130 };
1131
1132 let mmap = mmap_4mib_inaccessible();
1133 let mut memfd = MemoryImageSlot::create(
1134 mmap.zero_offset(),
1135 HostAlignedByteCount::ZERO,
1136 4 << 20,
1137 None,
1138 );
1139 let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap());
1140 let initial = 64 << 10;
1141
1142 // Instantiate the image and test that memory remains accessible after
1143 // it's cleared.
1144 memfd
1145 .instantiate(
1146 initial,
1147 Some(&image),
1148 &ty,
1149 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1150 )
1151 .unwrap();
1152 assert!(memfd.has_image());
1153
1154 unsafe {
1155 with_slice_mut(&mmap, 0..(64 << 10) + page_size, move |slice| {
1156 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1157 slice[page_size] = 5;
1158 assert_eq!(&[5, 2, 3, 4], &slice[page_size..][..4]);
1159 });
1160 }
1161
1162 memfd
1163 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1164 .unwrap();
1165 let slice = unsafe { mmap.slice(0..(64 << 10) + page_size) };
1166 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1167
1168 // Re-instantiate make sure it preserves memory. Grow a bit and set data
1169 // beyond the initial size.
1170 memfd
1171 .instantiate(
1172 initial,
1173 Some(&image),
1174 &ty,
1175 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1176 )
1177 .unwrap();
1178 assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
1179
1180 memfd.set_heap_limit(initial * 2).unwrap();
1181
1182 unsafe {
1183 with_slice_mut(&mmap, 0..(64 << 10) + page_size, move |slice| {
1184 assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1185 slice[initial] = 100;
1186 assert_eq!(&[100, 0], &slice[initial..initial + 2]);
1187 });
1188 }
1189
1190 memfd
1191 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1192 .unwrap();
1193
1194 // Test that memory is still accessible, but it's been reset
1195 assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1196
1197 // Instantiate again, and again memory beyond the initial size should
1198 // still be accessible. Grow into it again and make sure it works.
1199 memfd
1200 .instantiate(
1201 initial,
1202 Some(&image),
1203 &ty,
1204 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1205 )
1206 .unwrap();
1207 assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1208 memfd.set_heap_limit(initial * 2).unwrap();
1209
1210 unsafe {
1211 with_slice_mut(&mmap, 0..(64 << 10) + page_size, move |slice| {
1212 assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1213 slice[initial] = 100;
1214 assert_eq!(&[100, 0], &slice[initial..initial + 2]);
1215 });
1216 }
1217
1218 memfd
1219 .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
1220 .unwrap();
1221
1222 // Reset the image to none and double-check everything is back to zero
1223 memfd
1224 .instantiate(
1225 64 << 10,
1226 None,
1227 &ty,
1228 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1229 )
1230 .unwrap();
1231 assert!(!memfd.has_image());
1232 assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);
1233 assert_eq!(&[0, 0], &slice[initial..initial + 2]);
1234 }
1235
1236 #[test]
1237 fn reset_with_pagemap() {
1238 let page_size = host_page_size();
1239 let ty = dummy_memory();
1240 let tunables = Tunables {
1241 memory_reservation: 100 << 16,
1242 ..Tunables::default_miri()
1243 };
1244 let mmap = mmap_4mib_inaccessible();
1245 let mmap_len = page_size * 9;
1246 let mut memfd = MemoryImageSlot::create(
1247 mmap.zero_offset(),
1248 HostAlignedByteCount::ZERO,
1249 mmap_len,
1250 None,
1251 );
1252 let pagemap = PageMap::new();
1253 let pagemap = pagemap.as_ref();
1254
1255 let mut data = vec![0; 3 * page_size];
1256 for (i, chunk) in data.chunks_mut(page_size).enumerate() {
1257 for slot in chunk {
1258 *slot = u8::try_from(i + 1).unwrap();
1259 }
1260 }
1261 let image = Arc::new(create_memfd_with_data(3 * page_size, &data).unwrap());
1262
1263 memfd
1264 .instantiate(
1265 mmap_len,
1266 Some(&image),
1267 &ty,
1268 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1269 )
1270 .unwrap();
1271
1272 let keep_resident = HostAlignedByteCount::new(mmap_len).unwrap();
1273 let assert_pristine_after_reset = |memfd: &mut MemoryImageSlot| unsafe {
1274 // Wipe the image, keeping some bytes resident.
1275 memfd
1276 .clear_and_remain_ready(pagemap, keep_resident, decommit)
1277 .unwrap();
1278
1279 // Double check that the contents of memory are as expected after
1280 // reset.
1281 with_slice_mut(&mmap, 0..mmap_len, move |slice| {
1282 for (i, chunk) in slice.chunks(page_size).enumerate() {
1283 let expected = match i {
1284 0..3 => 0,
1285 3..6 => u8::try_from(i).unwrap() - 2,
1286 6..9 => 0,
1287 _ => unreachable!(),
1288 };
1289 for slot in chunk {
1290 assert_eq!(*slot, expected);
1291 }
1292 }
1293 });
1294
1295 // Re-instantiate, but then wipe the image entirely by keeping
1296 // nothing resident.
1297 memfd
1298 .instantiate(
1299 mmap_len,
1300 Some(&image),
1301 &ty,
1302 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1303 )
1304 .unwrap();
1305 memfd
1306 .clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, decommit)
1307 .unwrap();
1308
1309 // Next re-instantiate a final time to get used for the next test.
1310 memfd
1311 .instantiate(
1312 mmap_len,
1313 Some(&image),
1314 &ty,
1315 &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
1316 )
1317 .unwrap();
1318 };
1319
1320 let write_page = |_memfd: &mut MemoryImageSlot, page: usize| unsafe {
1321 with_slice_mut(
1322 &mmap,
1323 page * page_size..(page + 1) * page_size,
1324 move |slice| slice.fill(0xff),
1325 );
1326 };
1327
1328 // Test various combinations of dirty pages and regions. For example
1329 // test a dirty region of memory entirely in the zero-initialized zone
1330 // before/after the image and also test when the dirty region straddles
1331 // just the start of the image, just the end of the image, both ends,
1332 // and is entirely contained in just the image.
1333 assert_pristine_after_reset(&mut memfd);
1334
1335 for i in 0..9 {
1336 write_page(&mut memfd, i);
1337 assert_pristine_after_reset(&mut memfd);
1338 }
1339 write_page(&mut memfd, 0);
1340 write_page(&mut memfd, 1);
1341 assert_pristine_after_reset(&mut memfd);
1342 write_page(&mut memfd, 1);
1343 assert_pristine_after_reset(&mut memfd);
1344 write_page(&mut memfd, 2);
1345 write_page(&mut memfd, 3);
1346 assert_pristine_after_reset(&mut memfd);
1347 write_page(&mut memfd, 3);
1348 write_page(&mut memfd, 4);
1349 write_page(&mut memfd, 5);
1350 assert_pristine_after_reset(&mut memfd);
1351 write_page(&mut memfd, 0);
1352 write_page(&mut memfd, 1);
1353 write_page(&mut memfd, 2);
1354 assert_pristine_after_reset(&mut memfd);
1355 write_page(&mut memfd, 0);
1356 write_page(&mut memfd, 3);
1357 write_page(&mut memfd, 6);
1358 assert_pristine_after_reset(&mut memfd);
1359 write_page(&mut memfd, 2);
1360 write_page(&mut memfd, 3);
1361 write_page(&mut memfd, 4);
1362 write_page(&mut memfd, 5);
1363 write_page(&mut memfd, 6);
1364 assert_pristine_after_reset(&mut memfd);
1365 write_page(&mut memfd, 4);
1366 write_page(&mut memfd, 5);
1367 write_page(&mut memfd, 6);
1368 write_page(&mut memfd, 7);
1369 assert_pristine_after_reset(&mut memfd);
1370 write_page(&mut memfd, 4);
1371 write_page(&mut memfd, 5);
1372 write_page(&mut memfd, 8);
1373 assert_pristine_after_reset(&mut memfd);
1374 }
1375}