wasmtime/runtime/vm/sys/unix/
vm.rs

1use crate::runtime::vm::sys::DecommitBehavior;
2use rustix::fd::AsRawFd;
3use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect};
4use std::fs::File;
5use std::io;
6#[cfg(feature = "std")]
7use std::sync::Arc;
8
9pub unsafe fn expose_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
10    mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE)?;
11    Ok(())
12}
13
14pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
15    mprotect(ptr.cast(), len, MprotectFlags::empty())?;
16    Ok(())
17}
18
19pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
20    let ret = mmap_anonymous(
21        ptr.cast(),
22        len,
23        ProtFlags::empty(),
24        MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
25    )?;
26    assert_eq!(ptr, ret.cast());
27    Ok(())
28}
29
30#[cfg(feature = "pooling-allocator")]
31pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {
32    // Pages are always READ | WRITE so there's nothing that needs to be done
33    // here.
34    Ok(())
35}
36
37#[cfg(feature = "pooling-allocator")]
38pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
39    if len == 0 {
40        return Ok(());
41    }
42
43    unsafe {
44        cfg_if::cfg_if! {
45            if #[cfg(target_os = "linux")] {
46                use rustix::mm::{madvise, Advice};
47
48                // On Linux, this is enough to cause the kernel to initialize
49                // the pages to 0 on next access
50                madvise(addr as _, len, Advice::LinuxDontNeed)?;
51            } else {
52                // By creating a new mapping at the same location, this will
53                // discard the mapping for the pages in the given range.
54                // The new mapping will be to the CoW zero page, so this
55                // effectively zeroes the pages.
56                mmap_anonymous(
57                    addr as _,
58                    len,
59                    ProtFlags::READ | ProtFlags::WRITE,
60                    MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
61                )?;
62            }
63        }
64    }
65
66    Ok(())
67}
68
69// NB: this function is duplicated in `crates/fiber/src/unix.rs` so if this
70// changes that should probably get updated as well.
71pub fn get_page_size() -> usize {
72    unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() }
73}
74
75pub fn decommit_behavior() -> DecommitBehavior {
76    if cfg!(target_os = "linux") {
77        DecommitBehavior::RestoreOriginalMapping
78    } else {
79        DecommitBehavior::Zero
80    }
81}
82
83#[derive(Debug)]
84pub enum MemoryImageSource {
85    #[cfg(feature = "std")]
86    Mmap(Arc<File>),
87    #[cfg(target_os = "linux")]
88    Memfd(memfd::Memfd),
89}
90
91impl MemoryImageSource {
92    #[cfg(feature = "std")]
93    pub fn from_file(file: &Arc<File>) -> Option<MemoryImageSource> {
94        Some(MemoryImageSource::Mmap(file.clone()))
95    }
96
97    #[cfg(not(target_os = "linux"))]
98    pub fn from_data(_data: &[u8]) -> io::Result<Option<MemoryImageSource>> {
99        Ok(None)
100    }
101
102    #[cfg(target_os = "linux")]
103    pub fn from_data(data: &[u8]) -> anyhow::Result<Option<MemoryImageSource>> {
104        // On Linux `memfd_create` is used to create an anonymous
105        // in-memory file to represent the heap image. This anonymous
106        // file is then used as the basis for further mmaps.
107
108        use std::io::{ErrorKind, Write};
109
110        // Create the memfd. It needs a name, but the documentation for
111        // `memfd_create()` says that names can be duplicated with no issues.
112        let memfd = match memfd::MemfdOptions::new()
113            .allow_sealing(true)
114            .create("wasm-memory-image")
115        {
116            Ok(memfd) => memfd,
117            // If this kernel is old enough to not support memfd then attempt to
118            // gracefully handle that and fall back to skipping the memfd
119            // optimization.
120            Err(memfd::Error::Create(err)) if err.kind() == ErrorKind::Unsupported => {
121                return Ok(None);
122            }
123            Err(e) => return Err(e.into()),
124        };
125        memfd.as_file().write_all(data)?;
126
127        // Seal the memfd's data and length.
128        //
129        // This is a defense-in-depth security mitigation. The
130        // memfd will serve as the starting point for the heap of
131        // every instance of this module. If anything were to
132        // write to this, it could affect every execution. The
133        // memfd object itself is owned by the machinery here and
134        // not exposed elsewhere, but it is still an ambient open
135        // file descriptor at the syscall level, so some other
136        // vulnerability that allowed writes to arbitrary fds
137        // could modify it. Or we could have some issue with the
138        // way that we map it into each instance. To be
139        // extra-super-sure that it never changes, and because
140        // this costs very little, we use the kernel's "seal" API
141        // to make the memfd image permanently read-only.
142        memfd.add_seals(&[
143            memfd::FileSeal::SealGrow,
144            memfd::FileSeal::SealShrink,
145            memfd::FileSeal::SealWrite,
146            memfd::FileSeal::SealSeal,
147        ])?;
148
149        Ok(Some(MemoryImageSource::Memfd(memfd)))
150    }
151
152    pub(super) fn as_file(&self) -> &File {
153        match *self {
154            #[cfg(feature = "std")]
155            MemoryImageSource::Mmap(ref file) => file,
156            #[cfg(target_os = "linux")]
157            MemoryImageSource::Memfd(ref memfd) => memfd.as_file(),
158        }
159    }
160
161    pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> io::Result<()> {
162        let ptr = mmap_anonymous(
163            base.cast(),
164            len,
165            ProtFlags::READ | ProtFlags::WRITE,
166            MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
167        )?;
168        assert_eq!(base, ptr.cast());
169        Ok(())
170    }
171}
172
173impl PartialEq for MemoryImageSource {
174    fn eq(&self, other: &MemoryImageSource) -> bool {
175        self.as_file().as_raw_fd() == other.as_file().as_raw_fd()
176    }
177}