wasmtime_jit_icache_coherence/
libc.rs

1use core::ffi::c_void;
2
3#[cfg(any(target_os = "linux", target_os = "android"))]
4extern crate std;
5#[cfg(any(target_os = "linux", target_os = "android"))]
6pub use std::io::Result;
7
8#[cfg(not(any(target_os = "linux", target_os = "android")))]
9pub use anyhow::Result;
10
11#[cfg(all(
12    target_arch = "aarch64",
13    any(target_os = "linux", target_os = "android")
14))]
15mod details {
16    extern crate std;
17
18    use super::*;
19    use libc::{syscall, EINVAL, EPERM};
20    use std::io::Error;
21
22    const MEMBARRIER_CMD_GLOBAL: libc::c_int = 1;
23    const MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 32;
24    const MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 64;
25
26    /// See docs on [crate::pipeline_flush_mt] for a description of what this function is trying to do.
27    #[inline]
28    pub(crate) fn pipeline_flush_mt() -> Result<()> {
29        // Ensure that no processor has fetched a stale instruction stream.
30        //
31        // On AArch64 we try to do this by executing a "broadcast" `ISB` which is not something
32        // that the architecture provides us but we can emulate it using the membarrier kernel
33        // interface.
34        //
35        // This behaviour was documented in a patch, however it seems that it hasn't been
36        // upstreamed yet Nevertheless it clearly explains the guarantees that the Linux kernel
37        // provides us regarding the membarrier interface, and how to use it for JIT contexts.
38        // https://lkml.kernel.org/lkml/07a8b963002cb955b7516e61bad19514a3acaa82.1623813516.git.luto@kernel.org/
39        //
40        // I couldn't find the follow up for that patch but there doesn't seem to be disagreement
41        // about that specific part in the replies.
42        // TODO: Check if the kernel has updated the membarrier documentation
43        //
44        // See the following issues for more info:
45        //  * https://github.com/bytecodealliance/wasmtime/pull/3426
46        //  * https://github.com/bytecodealliance/wasmtime/pull/4997
47        //
48        // TODO: x86 and s390x have coherent caches so they don't need this, but RISCV does not
49        // guarantee that, so we may need to do something similar for it. However as noted in the
50        // above kernel patch the SYNC_CORE membarrier has different guarantees on each
51        // architecture so we need follow up and check what it provides us.
52        // See: https://github.com/bytecodealliance/wasmtime/issues/5033
53        match membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE) {
54            Ok(_) => {}
55
56            // EPERM happens if the calling process hasn't yet called the register membarrier.
57            // We can call the register membarrier now, and then retry the actual membarrier,
58            //
59            // This does have some overhead since on the first time we call this function we
60            // actually execute three membarriers, but this only happens once per process and only
61            // one slow membarrier is actually executed (The last one, which actually generates an
62            // IPI).
63            Err(e) if e.raw_os_error().unwrap() == EPERM => {
64                membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)?;
65                membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE)?;
66            }
67
68            // On kernels older than 4.16 the above syscall does not exist, so we can
69            // fallback to MEMBARRIER_CMD_GLOBAL which is an alias for MEMBARRIER_CMD_SHARED
70            // that has existed since 4.3. GLOBAL is a lot slower, but allows us to have
71            // compatibility with older kernels.
72            Err(e) if e.raw_os_error().unwrap() == EINVAL => {
73                membarrier(MEMBARRIER_CMD_GLOBAL)?;
74            }
75
76            // In any other case we got an actual error, so lets propagate that up
77            e => e?,
78        }
79
80        Ok(())
81    }
82
83    fn membarrier(barrier: libc::c_int) -> Result<()> {
84        let flags: libc::c_int = 0;
85        let res = unsafe { syscall(libc::SYS_membarrier, barrier, flags) };
86        if res == 0 {
87            Ok(())
88        } else {
89            Err(Error::last_os_error())
90        }
91    }
92}
93
94#[cfg(not(all(
95    target_arch = "aarch64",
96    any(target_os = "linux", target_os = "android")
97)))]
98mod details {
99    // NB: this uses `anyhow::Result` instead of `std::io::Result` to compile on
100    // `no_std`.
101    pub(crate) fn pipeline_flush_mt() -> super::Result<()> {
102        Ok(())
103    }
104}
105
106#[cfg(all(target_arch = "riscv64", target_os = "linux"))]
107fn riscv_flush_icache(start: u64, end: u64) -> Result<()> {
108    cfg_if::cfg_if! {
109        if #[cfg(feature = "one-core")] {
110            use core::arch::asm;
111            let _ = (start, end);
112            unsafe {
113                asm!("fence.i");
114            };
115            Ok(())
116        } else {
117            extern crate std;
118
119            #[expect(non_upper_case_globals, reason = "matching C style")]
120            match unsafe {
121                libc::syscall(
122                    {
123                        // The syscall isn't defined in `libc`, so we define the syscall number here.
124                        // https://github.com/torvalds/linux/search?q=__NR_arch_specific_syscall
125                        const  __NR_arch_specific_syscall :i64 = 244;
126                        // https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/tools/arch/riscv/include/uapi/asm/unistd.h#L40
127                        const sys_riscv_flush_icache :i64 =  __NR_arch_specific_syscall + 15;
128                        sys_riscv_flush_icache
129                    },
130                    // Currently these parameters are not used, but they are still defined.
131                    start, // start
132                    end, // end
133                    {
134                        const SYS_RISCV_FLUSH_ICACHE_LOCAL :i64 = 1;
135                        const SYS_RISCV_FLUSH_ICACHE_ALL :i64 = SYS_RISCV_FLUSH_ICACHE_LOCAL;
136                        SYS_RISCV_FLUSH_ICACHE_ALL
137                    }, // flags
138                )
139            } {
140                0 => { Ok(()) }
141                _ => Err(std::io::Error::last_os_error()),
142            }
143        }
144    }
145}
146
147pub(crate) use details::*;
148
149/// See docs on [crate::clear_cache] for a description of what this function is trying to do.
150#[inline]
151pub(crate) fn clear_cache(_ptr: *const c_void, _len: usize) -> Result<()> {
152    // TODO: On AArch64 we currently rely on the `mprotect` call that switches the memory from W+R
153    // to R+X to do this for us, however that is an implementation detail and should not be relied
154    // upon.
155    // We should call some implementation of `clear_cache` here.
156    //
157    // See: https://github.com/bytecodealliance/wasmtime/issues/3310
158    #[cfg(all(target_arch = "riscv64", target_os = "linux"))]
159    riscv_flush_icache(_ptr as u64, (_ptr as u64) + (_len as u64))?;
160    Ok(())
161}
wasmtime_jit_icache_coherence/libc.rs

wasmtime_jit_icache_coherence/
libc.rs
