@@ -20,5 +20,7 @@
/* `bindgen` gets confused at certain things. */
const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN;
+const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE;
const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL;
const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO;
+const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM;
@@ -25,6 +25,8 @@
#include <linux/build_bug.h>
#include <linux/err.h>
#include <linux/errname.h>
+#include <linux/gfp.h>
+#include <linux/highmem.h>
#include <linux/mutex.h>
#include <linux/refcount.h>
#include <linux/sched/signal.h>
@@ -93,6 +95,24 @@ int rust_helper_signal_pending(struct task_struct *t)
}
EXPORT_SYMBOL_GPL(rust_helper_signal_pending);
+struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order)
+{
+ return alloc_pages(gfp_mask, order);
+}
+EXPORT_SYMBOL_GPL(rust_helper_alloc_pages);
+
+void *rust_helper_kmap_local_page(struct page *page)
+{
+ return kmap_local_page(page);
+}
+EXPORT_SYMBOL_GPL(rust_helper_kmap_local_page);
+
+void rust_helper_kunmap_local(const void *addr)
+{
+ kunmap_local(addr);
+}
+EXPORT_SYMBOL_GPL(rust_helper_kunmap_local);
+
refcount_t rust_helper_REFCOUNT_INIT(int n)
{
return (refcount_t)REFCOUNT_INIT(n);
@@ -39,6 +39,7 @@
pub mod kunit;
#[cfg(CONFIG_NET)]
pub mod net;
+pub mod page;
pub mod prelude;
pub mod print;
mod static_assert;
new file mode 100644
@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Kernel page allocation and management.
+
+use crate::{bindings, error::code::*, error::Result, uaccess::UserSliceReader};
+use core::{
+ alloc::AllocError,
+ ptr::{self, NonNull},
+};
+
+/// A bitwise shift for the page size.
+#[allow(clippy::unnecessary_cast)]
+pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;
+
+/// The number of bytes in a page.
+#[allow(clippy::unnecessary_cast)]
+pub const PAGE_SIZE: usize = bindings::PAGE_SIZE as usize;
+
+/// A bitmask that gives the page containing a given address.
+pub const PAGE_MASK: usize = !(PAGE_SIZE-1);
+
+/// Flags for the "get free page" function that underlies all memory allocations.
+pub mod flags {
+ /// gfp flags.
+ #[allow(non_camel_case_types)]
+ pub type gfp_t = bindings::gfp_t;
+
+ /// `GFP_KERNEL` is typical for kernel-internal allocations. The caller requires `ZONE_NORMAL`
+ /// or a lower zone for direct access but can direct reclaim.
+ pub const GFP_KERNEL: gfp_t = bindings::GFP_KERNEL;
+ /// `GFP_ZERO` returns a zeroed page on success.
+ pub const __GFP_ZERO: gfp_t = bindings::__GFP_ZERO;
+ /// `GFP_HIGHMEM` indicates that the allocated memory may be located in high memory.
+ pub const __GFP_HIGHMEM: gfp_t = bindings::__GFP_HIGHMEM;
+}
+
+/// A pointer to a page that owns the page allocation.
+///
+/// # Invariants
+///
+/// The pointer is valid, and has ownership over the page.
+pub struct Page {
+ page: NonNull<bindings::page>,
+}
+
+// SAFETY: Pages have no logic that relies on them staying on a given thread, so
+// moving them across threads is safe.
+unsafe impl Send for Page {}
+
+// SAFETY: Pages have no logic that relies on them not being accessed
+// concurrently, so accessing them concurrently is safe.
+unsafe impl Sync for Page {}
+
+impl Page {
+ /// Allocates a new page.
+ pub fn alloc_page(gfp_flags: flags::gfp_t) -> Result<Self, AllocError> {
+ // SAFETY: Depending on the value of `gfp_flags`, this call may sleep.
+ // Other than that, it is always safe to call this method.
+ let page = unsafe { bindings::alloc_pages(gfp_flags, 0) };
+ let page = NonNull::new(page).ok_or(AllocError)?;
+ // INVARIANT: We just successfully allocated a page, so we now have
+ // ownership of the newly allocated page. We transfer that ownership to
+ // the new `Page` object.
+ Ok(Self { page })
+ }
+
+ /// Returns a raw pointer to the page.
+ pub fn as_ptr(&self) -> *mut bindings::page {
+ self.page.as_ptr()
+ }
+
+ /// Runs a piece of code with this page mapped to an address.
+ ///
+ /// The page is unmapped when this call returns.
+ ///
+ /// # Using the raw pointer
+ ///
+ /// It is up to the caller to use the provided raw pointer correctly. The
+ /// pointer is valid for `PAGE_SIZE` bytes and for the duration in which the
+ /// closure is called. The pointer might only be mapped on the current
+ /// thread, and when that is the case, dereferencing it on other threads is
+ /// UB. Other than that, the usual rules for dereferencing a raw pointer
+ /// apply: don't cause data races, the memory may be uninitialized, and so
+ /// on.
+ ///
+ /// If multiple threads map the same page at the same time, then they may
+ /// reference with different addresses. However, even if the addresses are
+ /// different, the underlying memory is still the same for these purposes
+ /// (e.g., it's still a data race if they both write to the same underlying
+ /// byte at the same time).
+ fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
+ // SAFETY: `page` is valid due to the type invariants on `Page`.
+ let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };
+
+ let res = f(mapped_addr.cast());
+
+ // This unmaps the page mapped above.
+ //
+ // SAFETY: Since this API takes the user code as a closure, it can only
+ // be used in a manner where the pages are unmapped in reverse order.
+ // This is as required by `kunmap_local`.
+ //
+ // In other words, if this call to `kunmap_local` happens when a
+ // different page should be unmapped first, then there must necessarily
+ // be a call to `kmap_local_page` other than the call just above in
+ // `with_page_mapped` that made that possible. In this case, it is the
+ // unsafe block that wraps that other call that is incorrect.
+ unsafe { bindings::kunmap_local(mapped_addr) };
+
+ res
+ }
+
+ /// Runs a piece of code with a raw pointer to a slice of this page, with
+ /// bounds checking.
+ ///
+ /// If `f` is called, then it will be called with a pointer that points at
+ /// `off` bytes into the page, and the pointer will be valid for at least
+ /// `len` bytes. The pointer is only valid on this task, as this method uses
+ /// a local mapping.
+ ///
+ /// If `off` and `len` refers to a region outside of this page, then this
+ /// method returns `EINVAL` and does not call `f`.
+ ///
+ /// # Using the raw pointer
+ ///
+ /// It is up to the caller to use the provided raw pointer correctly. The
+ /// pointer is valid for `len` bytes and for the duration in which the
+ /// closure is called. The pointer might only be mapped on the current
+ /// thread, and when that is the case, dereferencing it on other threads is
+ /// UB. Other than that, the usual rules for dereferencing a raw pointer
+ /// apply: don't cause data races, the memory may be uninitialized, and so
+ /// on.
+ ///
+ /// If multiple threads map the same page at the same time, then they may
+ /// reference with different addresses. However, even if the addresses are
+ /// different, the underlying memory is still the same for these purposes
+ /// (e.g., it's still a data race if they both write to the same underlying
+ /// byte at the same time).
+ fn with_pointer_into_page<T>(
+ &self,
+ off: usize,
+ len: usize,
+ f: impl FnOnce(*mut u8) -> Result<T>,
+ ) -> Result<T> {
+ let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;
+
+ if bounds_ok {
+ self.with_page_mapped(move |page_addr| {
+ // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this
+ // pointer offset will result in a pointer that is in bounds or
+ // one off the end of the page.
+ f(unsafe { page_addr.add(off) })
+ })
+ } else {
+ Err(EINVAL)
+ }
+ }
+
+ /// Maps the page and reads from it into the given buffer.
+ ///
+ /// This method will perform bounds checks on the page offset. If `offset ..
+ /// offset+len` goes outside ot the page, then this call returns `EINVAL`.
+ ///
+ /// # Safety
+ ///
+ /// * Callers must ensure that `dst` is valid for writing `len` bytes.
+ /// * Callers must ensure that this call does not race with a write to the
+ /// same page that overlaps with this read.
+ pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
+ self.with_pointer_into_page(offset, len, move |src| {
+ // SAFETY: If `with_pointer_into_page` calls into this closure, then
+ // it has performed a bounds check and guarantees that `src` is
+ // valid for `len` bytes.
+ //
+ // There caller guarantees that there is no data race.
+ unsafe { ptr::copy_nonoverlapping(src, dst, len) };
+ Ok(())
+ })
+ }
+
+ /// Maps the page and writes into it from the given buffer.
+ ///
+ /// This method will perform bounds checks on the page offset. If `offset ..
+ /// offset+len` goes outside ot the page, then this call returns `EINVAL`.
+ ///
+ /// # Safety
+ ///
+ /// * Callers must ensure that `src` is valid for reading `len` bytes.
+ /// * Callers must ensure that this call does not race with a read or write
+ /// to the same page that overlaps with this write.
+ pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
+ self.with_pointer_into_page(offset, len, move |dst| {
+ // SAFETY: If `with_pointer_into_page` calls into this closure, then
+ // it has performed a bounds check and guarantees that `dst` is
+ // valid for `len` bytes.
+ //
+ // There caller guarantees that there is no data race.
+ unsafe { ptr::copy_nonoverlapping(src, dst, len) };
+ Ok(())
+ })
+ }
+
+ /// Maps the page and zeroes the given slice.
+ ///
+ /// This method will perform bounds checks on the page offset. If `offset ..
+ /// offset+len` goes outside ot the page, then this call returns `EINVAL`.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that this call does not race with a read or write to
+ /// the same page that overlaps with this write.
+ pub unsafe fn fill_zero(&self, offset: usize, len: usize) -> Result {
+ self.with_pointer_into_page(offset, len, move |dst| {
+ // SAFETY: If `with_pointer_into_page` calls into this closure, then
+ // it has performed a bounds check and guarantees that `dst` is
+ // valid for `len` bytes.
+ //
+ // There caller guarantees that there is no data race.
+ unsafe { ptr::write_bytes(dst, 0u8, len) };
+ Ok(())
+ })
+ }
+
+ /// Copies data from userspace into this page.
+ ///
+ /// This method will perform bounds checks on the page offset. If `offset ..
+ /// offset+len` goes outside ot the page, then this call returns `EINVAL`.
+ ///
+ /// Like the other `UserSliceReader` methods, data races are allowed on the
+ /// userspace address. However, they are not allowed on the page you are
+ /// copying into.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that this call does not race with a read or write to
+ /// the same page that overlaps with this write.
+ pub unsafe fn copy_from_user_slice(
+ &self,
+ reader: &mut UserSliceReader,
+ offset: usize,
+ len: usize,
+ ) -> Result {
+ self.with_pointer_into_page(offset, len, move |dst| {
+ // SAFETY: If `with_pointer_into_page` calls into this closure, then
+ // it has performed a bounds check and guarantees that `dst` is
+ // valid for `len` bytes. Furthermore, we have exclusive access to
+ // the slice since the caller guarantees that there are no races.
+ reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
+ })
+ }
+}
+
+impl Drop for Page {
+ fn drop(&mut self) {
+ // SAFETY: By the type invariants, we have ownership of the page and can
+ // free it.
+ unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
+ }
+}