wasmtime/runtime/vm/sys/unix/
mmap.rs1use crate::prelude::*;
2use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};
3use rustix::mm::{MprotectFlags, mprotect};
4use std::ops::Range;
5use std::ptr::{self, NonNull};
6#[cfg(feature = "std")]
7use std::{fs::File, path::Path};
8
9#[cfg(feature = "std")]
11pub fn open_file_for_mmap(path: &Path) -> Result<File> {
12 File::open(path).context("failed to open file")
13}
14
15#[derive(Debug)]
16pub struct Mmap {
17 memory: SendSyncPtr<[u8]>,
18}
19
20cfg_select! {
21 any(target_os = "illumos", target_os = "linux") => {
22 pub(super) const MMAP_NORESERVE_FLAG: rustix::mm::MapFlags =
31 rustix::mm::MapFlags::NORESERVE;
32 }
33 _ => {
34 pub(super) const MMAP_NORESERVE_FLAG: rustix::mm::MapFlags = rustix::mm::MapFlags::empty();
35 }
36}
37
38impl Mmap {
39 pub fn new_empty() -> Mmap {
40 Mmap {
41 memory: crate::vm::sys::empty_mmap(),
42 }
43 }
44
45 pub fn new(size: HostAlignedByteCount) -> Result<Self> {
46 let ptr = unsafe {
47 rustix::mm::mmap_anonymous(
48 ptr::null_mut(),
49 size.byte_count(),
50 rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
51 rustix::mm::MapFlags::PRIVATE | MMAP_NORESERVE_FLAG,
52 )?
53 };
54 let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), size.byte_count());
55 let memory = SendSyncPtr::new(NonNull::new(memory).unwrap());
56 Ok(Mmap { memory })
57 }
58
59 pub fn reserve(size: HostAlignedByteCount) -> Result<Self> {
60 let ptr = unsafe {
61 rustix::mm::mmap_anonymous(
62 ptr::null_mut(),
63 size.byte_count(),
64 rustix::mm::ProtFlags::empty(),
65 rustix::mm::MapFlags::PRIVATE | MMAP_NORESERVE_FLAG,
77 )?
78 };
79
80 let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), size.byte_count());
81 let memory = SendSyncPtr::new(NonNull::new(memory).unwrap());
82 Ok(Mmap { memory })
83 }
84
85 #[cfg(feature = "std")]
86 pub fn from_file(file: &File) -> Result<Self> {
87 let len = file
88 .metadata()
89 .context("failed to get file metadata")?
90 .len();
91 let len = usize::try_from(len).map_err(|_| crate::format_err!("file too large to map"))?;
92 let ptr = unsafe {
93 rustix::mm::mmap(
94 ptr::null_mut(),
95 len,
96 rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
97 rustix::mm::MapFlags::PRIVATE,
98 &file,
99 0,
100 )
101 .with_context(|| format!("mmap failed to allocate {len:#x} bytes"))?
102 };
103 let memory = std::ptr::slice_from_raw_parts_mut(ptr.cast(), len);
104 let memory = SendSyncPtr::new(NonNull::new(memory).unwrap());
105
106 Ok(Mmap { memory })
107 }
108
109 pub unsafe fn make_accessible(
110 &self,
111 start: HostAlignedByteCount,
112 len: HostAlignedByteCount,
113 ) -> Result<()> {
114 let ptr = self.memory.as_ptr();
115 unsafe {
116 mprotect(
117 ptr.byte_add(start.byte_count()).cast(),
118 len.byte_count(),
119 MprotectFlags::READ | MprotectFlags::WRITE,
120 )?;
121 }
122
123 Ok(())
124 }
125
126 #[inline]
127 pub fn as_send_sync_ptr(&self) -> SendSyncPtr<u8> {
128 self.memory.cast()
129 }
130
131 #[inline]
132 pub fn len(&self) -> usize {
133 self.memory.as_ptr().len()
137 }
138
139 pub unsafe fn make_executable(
140 &self,
141 range: Range<usize>,
142 enable_branch_protection: bool,
143 ) -> Result<()> {
144 let base = unsafe { self.memory.as_ptr().byte_add(range.start).cast() };
145 let len = range.end - range.start;
146
147 if !cfg!(feature = "std") {
148 bail!(
149 "with the `std` feature disabled at compile time \
150 there must be a custom implementation of publishing \
151 code memory, otherwise it's unknown how to do icache \
152 management"
153 );
154 }
155
156 #[cfg(feature = "std")]
163 unsafe {
164 wasmtime_jit_icache_coherence::clear_cache(base, len).context("failed cache clear")?;
165 }
166
167 let flags = MprotectFlags::READ | MprotectFlags::EXEC;
168 let flags = if enable_branch_protection {
169 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
170 if std::arch::is_aarch64_feature_detected!("bti") {
171 MprotectFlags::from_bits_retain(flags.bits() | 0x10)
172 } else {
173 flags
174 }
175
176 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
177 flags
178 } else {
179 flags
180 };
181
182 unsafe {
183 mprotect(base, len, flags)?;
184 }
185
186 #[cfg(feature = "std")]
188 wasmtime_jit_icache_coherence::pipeline_flush_mt().context("Failed pipeline flush")?;
189
190 Ok(())
191 }
192
193 unsafe fn change_protections(
194 &self,
195 range: Range<usize>,
196 permissions: MprotectFlags,
197 ) -> Result<()> {
198 let base = unsafe { self.memory.as_ptr().byte_add(range.start).cast() };
199 let len = range.end - range.start;
200
201 unsafe {
202 mprotect(base, len, permissions)?;
203 }
204
205 Ok(())
206 }
207
208 pub unsafe fn make_readonly(&self, range: Range<usize>) -> Result<()> {
209 unsafe { self.change_protections(range, MprotectFlags::READ) }
210 }
211
212 pub unsafe fn make_readwrite(&self, range: Range<usize>) -> Result<()> {
213 unsafe { self.change_protections(range, MprotectFlags::READ | MprotectFlags::WRITE) }
214 }
215
216 pub unsafe fn make_inaccessible(&self, range: Range<usize>) -> Result<()> {
217 unsafe { self.change_protections(range, MprotectFlags::empty()) }
218 }
219}
220
221impl Drop for Mmap {
222 fn drop(&mut self) {
223 unsafe {
224 let ptr = self.memory.as_ptr().cast();
225 let len = self.memory.as_ptr().len();
226 if len == 0 {
227 return;
228 }
229 rustix::mm::munmap(ptr, len).expect("munmap failed");
230 }
231 }
232}