wasmtime/runtime/vm/sys/unix/
vm.rs

1use crate::runtime::vm::sys::DecommitBehavior;
2use rustix::fd::AsRawFd;
3use rustix::mm::{mmap_anonymous, mprotect, MapFlags, MprotectFlags, ProtFlags};
4use std::fs::File;
5use std::io;
6#[cfg(feature = "std")]
7use std::sync::Arc;
8
9pub unsafe fn expose_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
10    mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE)?;
11    Ok(())
12}
13
14pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
15    mprotect(ptr.cast(), len, MprotectFlags::empty())?;
16    Ok(())
17}
18
19pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
20    let ret = mmap_anonymous(
21        ptr.cast(),
22        len,
23        ProtFlags::empty(),
24        MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
25    )?;
26    assert_eq!(ptr, ret.cast());
27    Ok(())
28}
29
30#[cfg(feature = "pooling-allocator")]
31pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {
32    // Pages are always READ | WRITE so there's nothing that needs to be done
33    // here.
34    Ok(())
35}
36
37#[cfg(feature = "pooling-allocator")]
38pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
39    if len == 0 {
40        return Ok(());
41    }
42
43    unsafe {
44        cfg_if::cfg_if! {
45            if #[cfg(target_os = "linux")] {
46                use rustix::mm::{madvise, Advice};
47
48                // On Linux, this is enough to cause the kernel to initialize
49                // the pages to 0 on next access
50                madvise(addr as _, len, Advice::LinuxDontNeed)?;
51            } else {
52                // By creating a new mapping at the same location, this will
53                // discard the mapping for the pages in the given range.
54                // The new mapping will be to the CoW zero page, so this
55                // effectively zeroes the pages.
56                mmap_anonymous(
57                    addr as _,
58                    len,
59                    ProtFlags::READ | ProtFlags::WRITE,
60                    MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
61                )?;
62            }
63        }
64    }
65
66    Ok(())
67}
68
69pub fn get_page_size() -> usize {
70    unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() }
71}
72
73pub fn decommit_behavior() -> DecommitBehavior {
74    if cfg!(target_os = "linux") {
75        DecommitBehavior::RestoreOriginalMapping
76    } else {
77        DecommitBehavior::Zero
78    }
79}
80
81#[derive(Debug)]
82pub enum MemoryImageSource {
83    #[cfg(feature = "std")]
84    Mmap(Arc<File>),
85    #[cfg(target_os = "linux")]
86    Memfd(memfd::Memfd),
87}
88
89impl MemoryImageSource {
90    #[cfg(feature = "std")]
91    pub fn from_file(file: &Arc<File>) -> Option<MemoryImageSource> {
92        Some(MemoryImageSource::Mmap(file.clone()))
93    }
94
95    #[cfg(not(target_os = "linux"))]
96    pub fn from_data(_data: &[u8]) -> io::Result<Option<MemoryImageSource>> {
97        Ok(None)
98    }
99
100    #[cfg(target_os = "linux")]
101    pub fn from_data(data: &[u8]) -> anyhow::Result<Option<MemoryImageSource>> {
102        // On Linux `memfd_create` is used to create an anonymous
103        // in-memory file to represent the heap image. This anonymous
104        // file is then used as the basis for further mmaps.
105
106        use std::io::{ErrorKind, Write};
107
108        // Create the memfd. It needs a name, but the documentation for
109        // `memfd_create()` says that names can be duplicated with no issues.
110        let memfd = match memfd::MemfdOptions::new()
111            .allow_sealing(true)
112            .create("wasm-memory-image")
113        {
114            Ok(memfd) => memfd,
115            // If this kernel is old enough to not support memfd then attempt to
116            // gracefully handle that and fall back to skipping the memfd
117            // optimization.
118            Err(memfd::Error::Create(err)) if err.kind() == ErrorKind::Unsupported => {
119                return Ok(None)
120            }
121            Err(e) => return Err(e.into()),
122        };
123        memfd.as_file().write_all(data)?;
124
125        // Seal the memfd's data and length.
126        //
127        // This is a defense-in-depth security mitigation. The
128        // memfd will serve as the starting point for the heap of
129        // every instance of this module. If anything were to
130        // write to this, it could affect every execution. The
131        // memfd object itself is owned by the machinery here and
132        // not exposed elsewhere, but it is still an ambient open
133        // file descriptor at the syscall level, so some other
134        // vulnerability that allowed writes to arbitrary fds
135        // could modify it. Or we could have some issue with the
136        // way that we map it into each instance. To be
137        // extra-super-sure that it never changes, and because
138        // this costs very little, we use the kernel's "seal" API
139        // to make the memfd image permanently read-only.
140        memfd.add_seals(&[
141            memfd::FileSeal::SealGrow,
142            memfd::FileSeal::SealShrink,
143            memfd::FileSeal::SealWrite,
144            memfd::FileSeal::SealSeal,
145        ])?;
146
147        Ok(Some(MemoryImageSource::Memfd(memfd)))
148    }
149
150    pub(super) fn as_file(&self) -> &File {
151        match *self {
152            #[cfg(feature = "std")]
153            MemoryImageSource::Mmap(ref file) => file,
154            #[cfg(target_os = "linux")]
155            MemoryImageSource::Memfd(ref memfd) => memfd.as_file(),
156        }
157    }
158
159    pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> io::Result<()> {
160        let ptr = mmap_anonymous(
161            base.cast(),
162            len,
163            ProtFlags::READ | ProtFlags::WRITE,
164            MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
165        )?;
166        assert_eq!(base, ptr.cast());
167        Ok(())
168    }
169}
170
171impl PartialEq for MemoryImageSource {
172    fn eq(&self, other: &MemoryImageSource) -> bool {
173        self.as_file().as_raw_fd() == other.as_file().as_raw_fd()
174    }
175}
wasmtime/runtime/vm/sys/unix/vm.rs

wasmtime/runtime/vm/sys/unix/
vm.rs
