wasmtime/runtime/vm/sys/unix/
mmap.rs

1use crate::prelude::*;
2use crate::runtime::vm::sys::vm::MemoryImageSource;
3use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};
4use rustix::mm::{mprotect, MprotectFlags};
5use std::ops::Range;
6use std::ptr::{self, NonNull};
7#[cfg(feature = "std")]
8use std::{fs::File, path::Path};
9
10/// Open a file so that it can be mmap'd for executing.
11#[cfg(feature = "std")]
12pub fn open_file_for_mmap(path: &Path) -> Result<File> {
13    File::open(path).context("failed to open file")
14}
15
16#[derive(Debug)]
17pub struct Mmap {
18    memory: SendSyncPtr<[u8]>,
19}
20
21cfg_if::cfg_if! {
22    if #[cfg(any(target_os = "illumos", target_os = "linux"))] {
23        // On illumos, by default, mmap reserves what it calls "swap space" ahead of time, so that
24        // memory accesses a`re guaranteed not to fail once mmap succeeds. NORESERVE is for cases
25        // where that memory is never meant to be accessed -- e.g. memory that's used as guard
26        // pages.
27        //
28        // This is less crucial on Linux because Linux tends to overcommit memory by default, but is
29        // still a good idea to pass in for large allocations that don't need to be backed by
30        // physical memory.
31        pub(super) const MMAP_NORESERVE_FLAG: rustix::mm::MapFlags =
32            rustix::mm::MapFlags::NORESERVE;
33    } else {
34        pub(super) const MMAP_NORESERVE_FLAG: rustix::mm::MapFlags = rustix::mm::MapFlags::empty();
35    }
36}
37
38impl Mmap {
39    pub fn new_empty() -> Mmap {
40        Mmap {
41            memory: crate::vm::sys::empty_mmap(),
42        }
43    }
44
45    pub fn new(size: HostAlignedByteCount) -> Result<Self> {
46        let ptr = unsafe {
47            rustix::mm::mmap_anonymous(
48                ptr::null_mut(),
49                size.byte_count(),
50                rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
51                rustix::mm::MapFlags::PRIVATE | MMAP_NORESERVE_FLAG,
52            )?
53        };
54        let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), size.byte_count());
55        let memory = SendSyncPtr::new(NonNull::new(memory).unwrap());
56        Ok(Mmap { memory })
57    }
58
59    pub fn reserve(size: HostAlignedByteCount) -> Result<Self> {
60        let ptr = unsafe {
61            rustix::mm::mmap_anonymous(
62                ptr::null_mut(),
63                size.byte_count(),
64                rustix::mm::ProtFlags::empty(),
65                // Astute readers might be wondering why a function called "reserve" passes in a
66                // NORESERVE flag. That's because "reserve" in this context means one of two
67                // different things.
68                //
69                // * This method is used to allocate virtual memory that starts off in a state where
70                //   it cannot be accessed (i.e. causes a segfault if accessed).
71                // * NORESERVE is meant for virtual memory space for which backing physical/swap
72                //   pages are reserved on first access.
73                //
74                // Virtual memory that cannot be accessed should not have a backing store reserved
75                // for it. Hence, passing in NORESERVE is correct here.
76                rustix::mm::MapFlags::PRIVATE | MMAP_NORESERVE_FLAG,
77            )?
78        };
79
80        let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), size.byte_count());
81        let memory = SendSyncPtr::new(NonNull::new(memory).unwrap());
82        Ok(Mmap { memory })
83    }
84
85    #[cfg(feature = "std")]
86    pub fn from_file(file: &File) -> Result<Self> {
87        let len = file
88            .metadata()
89            .context("failed to get file metadata")?
90            .len();
91        let len = usize::try_from(len).map_err(|_| anyhow::anyhow!("file too large to map"))?;
92        let ptr = unsafe {
93            rustix::mm::mmap(
94                ptr::null_mut(),
95                len,
96                rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
97                rustix::mm::MapFlags::PRIVATE,
98                &file,
99                0,
100            )
101            .context(format!("mmap failed to allocate {len:#x} bytes"))?
102        };
103        let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), len);
104        let memory = SendSyncPtr::new(NonNull::new(memory).unwrap());
105
106        Ok(Mmap { memory })
107    }
108
109    pub unsafe fn make_accessible(
110        &self,
111        start: HostAlignedByteCount,
112        len: HostAlignedByteCount,
113    ) -> Result<()> {
114        let ptr = self.memory.as_ptr();
115        unsafe {
116            mprotect(
117                ptr.byte_add(start.byte_count()).cast(),
118                len.byte_count(),
119                MprotectFlags::READ | MprotectFlags::WRITE,
120            )?;
121        }
122
123        Ok(())
124    }
125
126    #[inline]
127    pub fn as_send_sync_ptr(&self) -> SendSyncPtr<u8> {
128        self.memory.cast()
129    }
130
131    #[inline]
132    pub fn len(&self) -> usize {
133        // Note: while the start of memory is host page-aligned, the length might
134        // not be, and in particular is not aligned for file-backed mmaps. Be
135        // careful!
136        self.memory.as_ptr().len()
137    }
138
139    pub unsafe fn make_executable(
140        &self,
141        range: Range<usize>,
142        enable_branch_protection: bool,
143    ) -> Result<()> {
144        let base = self.memory.as_ptr().byte_add(range.start).cast();
145        let len = range.end - range.start;
146
147        let flags = MprotectFlags::READ | MprotectFlags::EXEC;
148        let flags = if enable_branch_protection {
149            #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
150            if std::arch::is_aarch64_feature_detected!("bti") {
151                MprotectFlags::from_bits_retain(flags.bits() | /* PROT_BTI */ 0x10)
152            } else {
153                flags
154            }
155
156            #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
157            flags
158        } else {
159            flags
160        };
161
162        mprotect(base, len, flags)?;
163
164        Ok(())
165    }
166
167    pub unsafe fn make_readonly(&self, range: Range<usize>) -> Result<()> {
168        let base = self.memory.as_ptr().byte_add(range.start).cast();
169        let len = range.end - range.start;
170
171        mprotect(base, len, MprotectFlags::READ)?;
172
173        Ok(())
174    }
175
176    pub unsafe fn map_image_at(
177        &self,
178        image_source: &MemoryImageSource,
179        source_offset: u64,
180        memory_offset: HostAlignedByteCount,
181        memory_len: HostAlignedByteCount,
182    ) -> Result<()> {
183        unsafe {
184            let map_base = self.memory.as_ptr().byte_add(memory_offset.byte_count());
185            let ptr = rustix::mm::mmap(
186                map_base.cast(),
187                memory_len.byte_count(),
188                rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
189                rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED,
190                image_source.as_file(),
191                source_offset,
192            )?;
193            assert_eq!(map_base.cast(), ptr);
194        };
195        Ok(())
196    }
197}
198
199impl Drop for Mmap {
200    fn drop(&mut self) {
201        unsafe {
202            let ptr = self.memory.as_ptr().cast();
203            let len = self.memory.as_ptr().len();
204            if len == 0 {
205                return;
206            }
207            rustix::mm::munmap(ptr, len).expect("munmap failed");
208        }
209    }
210}
wasmtime/runtime/vm/sys/unix/mmap.rs

wasmtime/runtime/vm/sys/unix/
mmap.rs
