From patchwork Wed May 3 09:06:58 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229893 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5DE9EC77B75 for ; Wed, 3 May 2023 09:07:30 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229865AbjECJH0 (ORCPT ); Wed, 3 May 2023 05:07:26 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45278 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229660AbjECJHX (ORCPT ); Wed, 3 May 2023 05:07:23 -0400 Received: from mail-wr1-x433.google.com (mail-wr1-x433.google.com [IPv6:2a00:1450:4864:20::433]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id CCACB40FD for ; Wed, 3 May 2023 02:07:18 -0700 (PDT) Received: by mail-wr1-x433.google.com with SMTP id ffacd0b85a97d-3063433fa66so1615289f8f.3 for ; Wed, 03 May 2023 02:07:18 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104837; x=1685696837; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=eyybpekGpHGVvvaVewFrSrZRoEZZ/l3dGyzewmhKKwg=; b=Hpob7q1r0p4V0caXE2crEE/51FttP5ZNpxWPBnl62DekIdsJcpMiZqQ+U+0jmvfFhs BuAsztkxherqbyie+iWYnyQMMPYONBuj/oH4n3bIkTFCW6YxZ1Qzi8QEu4wrtQeU4R3m 7pVC6wXg8s6dc6fWLdUSxUdlxle+M8WZmsl6xw4ADtWTPKY3Cr+OeVC1eL/tZD3HUO9D q8HSTwHe/phJrkywr0Cj2YrKfBXdA8V3FvuTAJaBlWK+ou8a65EmklXozWcnoVqQl1Nt SaOHMzoSKIqGjhbX5y3I97bbfDs7YJsYXztcWuw71U7QjlCUZXS0UIkgwyYoGaNAdEA8 IrdQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104837; x=1685696837; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=eyybpekGpHGVvvaVewFrSrZRoEZZ/l3dGyzewmhKKwg=; b=FVwAvdUNJrmiOU9/2i6fMogBLKvl+TGQsVuwaRelkug3/8Ou0ukFrBVodYJbL3gYx+ HVKkG3fQUmnxJDix5OS62ii5nEI3wyQbCwcWZkvwfkItnUrhrMkJSc82sQ67vYt7jbVO WGGkORKhlAAZAVmjktyUIXcsT4YBb9Fe/MC/t/iA1KzXEs+abx7Klt/8sQgWplMjmOaP xM46m3DPJ4uGs0gR3HsHUEUQfFh/erXKrChR3JNdLzS/BOj/4iyWYmmhMCVk9V6RsBR4 aIVCGTw817yIB7nAAqobd0Xkffr98N3By2S1rE/cniU+XQJmSGoq+jVajNX1XiWZX4C/ IcWw== X-Gm-Message-State: AC+VfDw/TeaL5iNqrxcR7OU3TSYQd/378BMf8UhJwJyYmJuoFjGFax1Y bpBh4Zvk5QNSZL3w94qUOpcegg== X-Google-Smtp-Source: ACHHUZ4c7pdGYmFDE+EObHEdf29jnpXn/rtwlDJWATkEBCQ9LPR+y1yLX/HtD1fPIL1dXO2oFBe9Lg== X-Received: by 2002:a5d:54d1:0:b0:306:2ff6:5cbf with SMTP id x17-20020a5d54d1000000b003062ff65cbfmr6873320wrv.24.1683104837081; Wed, 03 May 2023 02:07:17 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id e2-20020a056000120200b00306281cfa59sm9741741wrx.47.2023.05.03.02.07.16 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:16 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 01/11] rust: add radix tree abstraction Date: Wed, 3 May 2023 11:06:58 +0200 Message-Id: <20230503090708.2524310-2-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg Add abstractions for the C radix_tree. This abstraction allows Rust code to use the radix_tree as a map from `u64` keys to `ForeignOwnable` values. Signed-off-by: Andreas Hindborg --- rust/bindings/bindings_helper.h | 2 + rust/bindings/lib.rs | 1 + rust/helpers.c | 22 +++++ rust/kernel/lib.rs | 1 + rust/kernel/radix_tree.rs | 156 ++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+) create mode 100644 rust/kernel/radix_tree.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 50e7a76d5455..52834962b94d 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -10,7 +10,9 @@ #include #include #include +#include /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; +const gfp_t BINDINGS_GFP_ATOMIC = GFP_ATOMIC; const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 7b246454e009..62f36a9eb1f4 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -51,4 +51,5 @@ mod bindings_helper { pub use bindings_raw::*; pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL; +pub const GFP_ATOMIC: gfp_t = BINDINGS_GFP_ATOMIC; pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO; diff --git a/rust/helpers.c b/rust/helpers.c index 81e80261d597..5dd5e325b7cc 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -26,6 +26,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -128,6 +129,27 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); +void rust_helper_init_radix_tree(struct xarray *tree, gfp_t gfp_mask) +{ + INIT_RADIX_TREE(tree, gfp_mask); +} +EXPORT_SYMBOL_GPL(rust_helper_init_radix_tree); + +void **rust_helper_radix_tree_iter_init(struct radix_tree_iter *iter, + unsigned long start) +{ + return radix_tree_iter_init(iter, start); +} +EXPORT_SYMBOL_GPL(rust_helper_radix_tree_iter_init); + +void **rust_helper_radix_tree_next_slot(void **slot, + struct radix_tree_iter *iter, + unsigned flags) +{ + return radix_tree_next_slot(slot, iter, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_radix_tree_next_slot); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 676995d4e460..a85cb6aae8d6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -40,6 +40,7 @@ pub mod init; pub mod ioctl; pub mod prelude; pub mod print; +pub mod radix_tree; mod static_assert; #[doc(hidden)] pub mod std_vendor; diff --git a/rust/kernel/radix_tree.rs b/rust/kernel/radix_tree.rs new file mode 100644 index 000000000000..f659ab8b017c --- /dev/null +++ b/rust/kernel/radix_tree.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! RadixTree abstraction. +//! +//! C header: [`include/linux/radix_tree.h`](../../include/linux/radix_tree.h) + +use crate::error::to_result; +use crate::error::Result; +use crate::types::ForeignOwnable; +use crate::types::Opaque; +use crate::types::ScopeGuard; +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::pin::Pin; + +type Key = u64; + +/// A map of `u64` to `ForeignOwnable` +/// +/// # Invariants +/// +/// - `tree` always points to a valid and initialized `struct radix_tree`. +/// - Pointers stored in the tree are created by a call to `ForignOwnable::into_foreign()` +pub struct RadixTree { + tree: Pin>>, + _marker: PhantomData, +} + +impl RadixTree { + /// Create a new radix tree + /// + /// Note: This function allocates memory with `GFP_ATOMIC`. + pub fn new() -> Result { + let tree = Pin::from(Box::try_new(Opaque::uninit())?); + + // SAFETY: `tree` points to allocated but not initialized memory. This + // call will initialize the memory. + unsafe { bindings::init_radix_tree(tree.get(), bindings::GFP_ATOMIC) }; + + Ok(Self { + tree, + _marker: PhantomData, + }) + } + + /// Try to insert a value into the tree + pub fn try_insert(&mut self, key: Key, value: V) -> Result<()> { + // SAFETY: `self.tree` points to a valid and initialized `struct radix_tree` + let ret = + unsafe { bindings::radix_tree_insert(self.tree.get(), key, value.into_foreign() as _) }; + to_result(ret) + } + + /// Search for `key` in the map. Returns a reference to the associated + /// value if found. + pub fn get(&self, key: Key) -> Option> { + // SAFETY: `self.tree` points to a valid and initialized `struct radix_tree` + let item = + core::ptr::NonNull::new(unsafe { bindings::radix_tree_lookup(self.tree.get(), key) })?; + + // SAFETY: `item` was created by a call to + // `ForeignOwnable::into_foreign()`. As `get_mut()` and `remove()` takes + // a `&mut self`, no mutable borrows for `item` can exist and + // `ForeignOwnable::from_foreign()` cannot be called until this borrow + // is dropped. + Some(unsafe { V::borrow(item.as_ptr()) }) + } + + /// Search for `key` in the map. Return a mutable reference to the + /// associated value if found. + pub fn get_mut(&mut self, key: Key) -> Option> { + let item = + core::ptr::NonNull::new(unsafe { bindings::radix_tree_lookup(self.tree.get(), key) })?; + + // SAFETY: `item` was created by a call to + // `ForeignOwnable::into_foreign()`. As `get()` takes a `&self` and + // `remove()` takes a `&mut self`, no borrows for `item` can exist and + // `ForeignOwnable::from_foreign()` cannot be called until this borrow + // is dropped. + Some(MutBorrow { + guard: unsafe { V::borrow_mut(item.as_ptr()) }, + _marker: core::marker::PhantomData, + }) + } + + /// Search for `key` in the map. If `key` is found, the key and value is + /// removed from the map and the value is returned. + pub fn remove(&mut self, key: Key) -> Option { + // SAFETY: `self.tree` points to a valid and initialized `struct radix_tree` + let item = + core::ptr::NonNull::new(unsafe { bindings::radix_tree_delete(self.tree.get(), key) })?; + + // SAFETY: `item` was created by a call to + // `ForeignOwnable::into_foreign()` and no borrows to `item` can exist + // because this function takes a `&mut self`. + Some(unsafe { ForeignOwnable::from_foreign(item.as_ptr()) }) + } +} + +impl Drop for RadixTree { + fn drop(&mut self) { + let mut iter = bindings::radix_tree_iter { + index: 0, + next_index: 0, + tags: 0, + node: core::ptr::null_mut(), + }; + + // SAFETY: Iter is valid as we allocated it on the stack above + let mut slot = unsafe { bindings::radix_tree_iter_init(&mut iter, 0) }; + loop { + if slot.is_null() { + // SAFETY: Both `self.tree` and `iter` are valid + slot = unsafe { bindings::radix_tree_next_chunk(self.tree.get(), &mut iter, 0) }; + } + + if slot.is_null() { + break; + } + + // SAFETY: `self.tree` is valid and iter is managed by + // `radix_tree_next_chunk()` and `radix_tree_next_slot()` + let item = unsafe { bindings::radix_tree_delete(self.tree.get(), iter.index) }; + assert!(!item.is_null()); + + // SAFETY: All items in the tree are created by a call to + // `ForeignOwnable::into_foreign()`. + let _ = unsafe { V::from_foreign(item) }; + + // SAFETY: `self.tree` is valid and iter is managed by + // `radix_tree_next_chunk()` and `radix_tree_next_slot()`. Slot is + // not null. + slot = unsafe { bindings::radix_tree_next_slot(slot, &mut iter, 0) }; + } + } +} + +/// A mutable borrow of an object owned by a `RadixTree` +pub struct MutBorrow<'a, V: ForeignOwnable> { + guard: ScopeGuard, + _marker: core::marker::PhantomData<&'a mut V>, +} + +impl<'a, V: ForeignOwnable> core::ops::Deref for MutBorrow<'a, V> { + type Target = ScopeGuard; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +impl<'a, V: ForeignOwnable> core::ops::DerefMut for MutBorrow<'a, V> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard + } +} From patchwork Wed May 3 09:06:59 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229894 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 95F6DC77B7F for ; Wed, 3 May 2023 09:07:30 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229660AbjECJH0 (ORCPT ); Wed, 3 May 2023 05:07:26 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45308 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229741AbjECJHY (ORCPT ); Wed, 3 May 2023 05:07:24 -0400 Received: from mail-wr1-x434.google.com (mail-wr1-x434.google.com [IPv6:2a00:1450:4864:20::434]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D28D444A1 for ; Wed, 3 May 2023 02:07:19 -0700 (PDT) Received: by mail-wr1-x434.google.com with SMTP id ffacd0b85a97d-2fddb442d47so4548896f8f.2 for ; Wed, 03 May 2023 02:07:19 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104838; x=1685696838; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=s2OF8cEuaV6eZXvZ8+TxAkEcAWTLOcEvP8M1z8Cu3eg=; b=gxwZbpfudAGlkc9suPmtflL1jblQr4MK2hN1E3gaOGf/Plu3em+uZBGDw0rNUr7SUW EFB1a9htiLdSYQkjbn0JwIhIuQzJvNeRrzMlV0eXOLXXzIMN08VTuR5LYMKTn7m05OvH 9LHtC/oRDhdH3ACVUabMlgXW9Ewt8dD9XDI4Ez+21QfYADeNMEfiUmJzzu0XcckSKJuW Nj3Eh8mnhKzN9/T17uqbvi142/KhX1PBU6Qk/kU/ro0xuE5e8kN3AfwwdKqZvdZVSRvp 4qO3v7fHN7bd8dDrEFaz1iXrwav7RAETaSSc1oLICAvTlYivVQel8jdUiGT3J45aYlDO jWSQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104838; x=1685696838; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=s2OF8cEuaV6eZXvZ8+TxAkEcAWTLOcEvP8M1z8Cu3eg=; b=ZhIIcTSo7+MtO4qnCQHGY6XSlqo8tn1+pKvLHUBxz2RFGjeZ2cPSk7ii3C8XTpKOv4 cP1+5F+aQEk0vXRpKVCOsQRlTT2Rg03fb+gKRwA0ru/XqwNMwf92KSniUu67spE/NRwf t9QR4yxjEhVRqexyv/IhcTXH21+Nl+q6QfFR1WQ8vGCgoTEDAkGqeqVB5WoI9vNaecjs O32gAICJW1KntyOxgw4Umh1sqkN5OnQ8vRxutquJrpIFUWMITmh8H7ymo7cYnKzy81vo CJ9TLNHJgIMAyYFpAB6JEyeX9vy52Qne/KT3UMXfgsa9OgQInySApIeZzr6CW/OUQFJT qJUw== X-Gm-Message-State: AC+VfDy4YDcZ1CgPEZcIEOsqBED2QCC6iDWIx8pKLlQLz9oapMmNgyeH 7jnXU/Bf+ThcWbDphNB+BHkapw== X-Google-Smtp-Source: ACHHUZ7yrDB5gpNUkKE0X5q8MEbvZmkfVFN8rXKsdmGE0Vy4CSxFk+RB6D8+DyE/dyNCU1ItHJSB8g== X-Received: by 2002:a05:6000:1145:b0:306:2cf5:79dc with SMTP id d5-20020a056000114500b003062cf579dcmr6255554wrx.35.1683104838080; Wed, 03 May 2023 02:07:18 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id t15-20020adfe44f000000b002f00793bd7asm33035982wrm.27.2023.05.03.02.07.17 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:17 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 02/11] rust: add `pages` module for handling page allocation Date: Wed, 3 May 2023 11:06:59 +0200 Message-Id: <20230503090708.2524310-3-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg This patch adds support for working with pages of order 0. Support for pages with higher order is deferred. Page allocation flags are fixed in this patch. Future work might allow the user to specify allocation flags. This patch is a heavily modified version of code available in the rust tree [1], primarily adding support for multiple page mapping strategies. [1] https://github.com/rust-for-Linux/linux/tree/bc22545f38d74473cfef3e9fd65432733435b79f/rust/kernel/pages.rs Signed-off-by: Andreas Hindborg --- rust/helpers.c | 31 +++++ rust/kernel/lib.rs | 6 + rust/kernel/pages.rs | 284 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 rust/kernel/pages.rs diff --git a/rust/helpers.c b/rust/helpers.c index 5dd5e325b7cc..9bd9d95da951 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -27,6 +27,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -150,6 +151,36 @@ void **rust_helper_radix_tree_next_slot(void **slot, } EXPORT_SYMBOL_GPL(rust_helper_radix_tree_next_slot); +void *rust_helper_kmap(struct page *page) +{ + return kmap(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kmap); + +void rust_helper_kunmap(struct page *page) +{ + return kunmap(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kunmap); + +void *rust_helper_kmap_atomic(struct page *page) +{ + return kmap_atomic(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kmap_atomic); + +void rust_helper_kunmap_atomic(void *address) +{ + kunmap_atomic(address); +} +EXPORT_SYMBOL_GPL(rust_helper_kunmap_atomic); + +struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} +EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a85cb6aae8d6..8bef6686504b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -38,6 +38,7 @@ mod build_assert; pub mod error; pub mod init; pub mod ioctl; +pub mod pages; pub mod prelude; pub mod print; pub mod radix_tree; @@ -57,6 +58,11 @@ pub use uapi; #[doc(hidden)] pub use build_error::build_error; +/// Page size defined in terms of the `PAGE_SHIFT` macro from C. +/// +/// [`PAGE_SHIFT`]: ../../../include/asm-generic/page.h +pub const PAGE_SIZE: u32 = 1 << bindings::PAGE_SHIFT; + /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; diff --git a/rust/kernel/pages.rs b/rust/kernel/pages.rs new file mode 100644 index 000000000000..ed51b053dd5d --- /dev/null +++ b/rust/kernel/pages.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel page allocation and management. +//! +//! This module currently provides limited support. It supports pages of order 0 +//! for most operations. Page allocation flags are fixed. + +use crate::{bindings, error::code::*, error::Result, PAGE_SIZE}; +use core::{marker::PhantomData, ptr}; + +/// A set of physical pages. +/// +/// `Pages` holds a reference to a set of pages of order `ORDER`. Having the order as a generic +/// const allows the struct to have the same size as a pointer. +/// +/// # Invariants +/// +/// The pointer `Pages::pages` is valid and points to 2^ORDER pages. +pub struct Pages { + pub(crate) pages: *mut bindings::page, +} + +impl Pages { + /// Allocates a new set of contiguous pages. + pub fn new() -> Result { + let pages = unsafe { + bindings::alloc_pages( + bindings::GFP_KERNEL | bindings::__GFP_ZERO | bindings::___GFP_HIGHMEM, + ORDER, + ) + }; + if pages.is_null() { + return Err(ENOMEM); + } + // INVARIANTS: We checked that the allocation above succeeded. + // SAFETY: We allocated pages above + Ok(unsafe { Self::from_raw(pages) }) + } + + /// Create a `Pages` from a raw `struct page` pointer + /// + /// # Safety + /// + /// Caller must own the pages pointed to by `ptr` as these will be freed + /// when the returned `Pages` is dropped. + pub unsafe fn from_raw(ptr: *mut bindings::page) -> Self { + Self { pages: ptr } + } +} + +impl Pages<0> { + #[inline(always)] + fn check_offset_and_map( + &self, + offset: usize, + len: usize, + ) -> Result> + where + Pages<0>: MappingActions, + { + let end = offset.checked_add(len).ok_or(EINVAL)?; + if end as u32 > PAGE_SIZE { + return Err(EINVAL); + } + + let mapping = >::map(self); + + Ok(mapping) + } + + #[inline(always)] + unsafe fn read_internal( + &self, + dest: *mut u8, + offset: usize, + len: usize, + ) -> Result + where + Pages<0>: MappingActions, + { + let mapping = self.check_offset_and_map::(offset, len)?; + + unsafe { ptr::copy_nonoverlapping((mapping.ptr as *mut u8).add(offset), dest, len) }; + Ok(()) + } + + /// Maps the pages and reads from them into the given buffer. + /// + /// # Safety + /// + /// Callers must ensure that the destination buffer is valid for the given + /// length. Additionally, if the raw buffer is intended to be recast, they + /// must ensure that the data can be safely cast; + /// [`crate::io_buffer::ReadableFromBytes`] has more details about it. + /// `dest` may not point to the source page. + #[inline(always)] + pub unsafe fn read(&self, dest: *mut u8, offset: usize, len: usize) -> Result { + unsafe { self.read_internal::(dest, offset, len) } + } + + /// Maps the pages and reads from them into the given buffer. The page is + /// mapped atomically. + /// + /// # Safety + /// + /// Callers must ensure that the destination buffer is valid for the given + /// length. Additionally, if the raw buffer is intended to be recast, they + /// must ensure that the data can be safely cast; + /// [`crate::io_buffer::ReadableFromBytes`] has more details about it. + /// `dest` may not point to the source page. + #[inline(always)] + pub unsafe fn read_atomic(&self, dest: *mut u8, offset: usize, len: usize) -> Result { + unsafe { self.read_internal::(dest, offset, len) } + } + + #[inline(always)] + unsafe fn write_internal( + &self, + src: *const u8, + offset: usize, + len: usize, + ) -> Result + where + Pages<0>: MappingActions, + { + let mapping = self.check_offset_and_map::(offset, len)?; + + unsafe { ptr::copy_nonoverlapping(src, (mapping.ptr as *mut u8).add(offset), len) }; + Ok(()) + } + + /// Maps the pages and writes into them from the given buffer. + /// + /// # Safety + /// + /// Callers must ensure that the buffer is valid for the given length. + /// Additionally, if the page is (or will be) mapped by userspace, they must + /// ensure that no kernel data is leaked through padding if it was cast from + /// another type; [`crate::io_buffer::WritableToBytes`] has more details + /// about it. `src` must not point to the destination page. + #[inline(always)] + pub unsafe fn write(&self, src: *const u8, offset: usize, len: usize) -> Result { + unsafe { self.write_internal::(src, offset, len) } + } + + /// Maps the pages and writes into them from the given buffer. The page is + /// mapped atomically. + /// + /// # Safety + /// + /// Callers must ensure that the buffer is valid for the given length. + /// Additionally, if the page is (or will be) mapped by userspace, they must + /// ensure that no kernel data is leaked through padding if it was cast from + /// another type; [`crate::io_buffer::WritableToBytes`] has more details + /// about it. `src` must not point to the destination page. + #[inline(always)] + pub unsafe fn write_atomic(&self, src: *const u8, offset: usize, len: usize) -> Result { + unsafe { self.write_internal::(src, offset, len) } + } + + /// Maps the page at index 0. + #[inline(always)] + pub fn kmap(&self) -> PageMapping<'_, NormalMappingInfo> { + let ptr = unsafe { bindings::kmap(self.pages) }; + + PageMapping { + page: self.pages, + ptr, + _phantom: PhantomData, + _phantom2: PhantomData, + } + } + + /// Atomically Maps the page at index 0. + #[inline(always)] + pub fn kmap_atomic(&self) -> PageMapping<'_, AtomicMappingInfo> { + let ptr = unsafe { bindings::kmap_atomic(self.pages) }; + + PageMapping { + page: self.pages, + ptr, + _phantom: PhantomData, + _phantom2: PhantomData, + } + } +} + +impl Drop for Pages { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know the pages are allocated with the given order. + unsafe { bindings::__free_pages(self.pages, ORDER) }; + } +} + +/// Specifies the type of page mapping +pub trait MappingInfo {} + +/// Encapsulates methods to map and unmap pages +pub trait MappingActions +where + Pages<0>: MappingActions, +{ + /// Map a page into the kernel address scpace + fn map(pages: &Pages<0>) -> PageMapping<'_, I>; + + /// Unmap a page specified by `mapping` + /// + /// # Safety + /// + /// Must only be called by `PageMapping::drop()`. + unsafe fn unmap(mapping: &PageMapping<'_, I>); +} + +/// A type state indicating that pages were mapped atomically +pub struct AtomicMappingInfo; +impl MappingInfo for AtomicMappingInfo {} + +/// A type state indicating that pages were not mapped atomically +pub struct NormalMappingInfo; +impl MappingInfo for NormalMappingInfo {} + +impl MappingActions for Pages<0> { + #[inline(always)] + fn map(pages: &Pages<0>) -> PageMapping<'_, AtomicMappingInfo> { + pages.kmap_atomic() + } + + #[inline(always)] + unsafe fn unmap(mapping: &PageMapping<'_, AtomicMappingInfo>) { + // SAFETY: An instance of `PageMapping` is created only when `kmap` succeeded for the given + // page, so it is safe to unmap it here. + unsafe { bindings::kunmap_atomic(mapping.ptr) }; + } +} + +impl MappingActions for Pages<0> { + #[inline(always)] + fn map(pages: &Pages<0>) -> PageMapping<'_, NormalMappingInfo> { + pages.kmap() + } + + #[inline(always)] + unsafe fn unmap(mapping: &PageMapping<'_, NormalMappingInfo>) { + // SAFETY: An instance of `PageMapping` is created only when `kmap` succeeded for the given + // page, so it is safe to unmap it here. + unsafe { bindings::kunmap(mapping.page) }; + } +} + +/// An owned page mapping. When this struct is dropped, the page is unmapped. +pub struct PageMapping<'a, I: MappingInfo> +where + Pages<0>: MappingActions, +{ + page: *mut bindings::page, + ptr: *mut core::ffi::c_void, + _phantom: PhantomData<&'a i32>, + _phantom2: PhantomData, +} + +impl<'a, I: MappingInfo> PageMapping<'a, I> +where + Pages<0>: MappingActions, +{ + /// Return a pointer to the wrapped `struct page` + #[inline(always)] + pub fn get_ptr(&self) -> *mut core::ffi::c_void { + self.ptr + } +} + +// Because we do not have Drop specialization, we have to do this dance. Life +// would be much more simple if we could have `impl Drop for PageMapping<'_, +// Atomic>` and `impl Drop for PageMapping<'_, NotAtomic>` +impl Drop for PageMapping<'_, I> +where + Pages<0>: MappingActions, +{ + #[inline(always)] + fn drop(&mut self) { + // SAFETY: We are OK to call this because we are `PageMapping::drop()` + unsafe { as MappingActions>::unmap(self) } + } +} From patchwork Wed May 3 09:07:00 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229896 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 219B0C7EE2D for ; Wed, 3 May 2023 09:07:32 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229936AbjECJH3 (ORCPT ); Wed, 3 May 2023 05:07:29 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45344 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229825AbjECJHZ (ORCPT ); Wed, 3 May 2023 05:07:25 -0400 Received: from mail-wm1-x32e.google.com (mail-wm1-x32e.google.com [IPv6:2a00:1450:4864:20::32e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E82F746B7 for ; Wed, 3 May 2023 02:07:20 -0700 (PDT) Received: by mail-wm1-x32e.google.com with SMTP id 5b1f17b1804b1-3f178da21b5so31755885e9.3 for ; Wed, 03 May 2023 02:07:20 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104839; x=1685696839; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=oTm6JqVuE+nMnHE8FtSNpIwMZXEBmyO5CLHkBrzqYiM=; b=y8ejhwlgPFRLA58nqaLi6rcO3cTM2Bfu5R3HefR0o7cV9HTZoOxGUPasuvY46ZCwRh Sieo0Nw1Hzb4wY/iGDsNeHNIVg922qi3pacnA1K1Se735gJ7tsTrShXrb9Q3Tud15qCH pSi0ybeIHv49VuB2OI3rl+J+JTSb+9Q6ggrpcJFtEdi5YM3IsSttMHyzuBP0ZuAFksoN vicg6hsE+vaLEKi9gFV/LC2GyxdFyAnbn301c0wlUyeORPMq2Ykn/8QtIJHJeargKiXo g6ZYJxIn2Ojk9efoIi4W30BLPTSKqbSu7FqbbaUdRjKDx6DSjir7twQZkzXbwUjeKM5j +eEg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104839; x=1685696839; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oTm6JqVuE+nMnHE8FtSNpIwMZXEBmyO5CLHkBrzqYiM=; b=k73c8rjhm9oCD5J0jYCZSa38XJnaLYRsDqkbUWjfuAU2EkcBCI0QydKy/gAvMWXnku M4kY88KvCPCVoTqLJVPzdeud3MmsjFwl1UAE6l+gBSsrOfQ5lcR9tqmHLJz4OWABDVYm tVI4btYSI8xDyH+xRw6kl7CoDHw44RBRB7qoHR3XhMUqIg7bUdgfPgoHec8emaWDzayl q5215Q6B+c4p6rfCOo9bCXKA0GUG85OqSGkzh67u1NgwrYDuojZj5VLhGfoVFU2sBzeq DiZP9fXo1NU0mG8KuqfDPU21HjoZYrWNIdm5Rm8KSYxZ2J5FiJYafZkcQ2gdqys5r9vZ DhXw== X-Gm-Message-State: AC+VfDxrlmi8SE8mq+OG4x6CvwAaxqzAzDIcNjH7kz+4VBGGsMipG7FX 9s/juJfAtgv0CXE5SNOPT1oSQjHltJlGciM29XE= X-Google-Smtp-Source: ACHHUZ7NYkY5wjU5kOgvBlYNkrutaPEK/czR704t4GtHEI0ibjC9b/NRGmiwBmTmJwdWADfymkFv2Q== X-Received: by 2002:a7b:c383:0:b0:3f3:3a81:334 with SMTP id s3-20020a7bc383000000b003f33a810334mr6690102wmj.29.1683104839156; Wed, 03 May 2023 02:07:19 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id s7-20020a1cf207000000b003f1733feb3dsm1305041wmc.0.2023.05.03.02.07.18 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:18 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 03/11] rust: block: introduce `kernel::block::mq` module Date: Wed, 3 May 2023 11:07:00 +0200 Message-Id: <20230503090708.2524310-4-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg Add initial abstractions for working with blk-mq. This patch is a maintained, refactored subset of code originally published by Wedson Almeida Filho [1]. [1] https://github.com/wedsonaf/linux/tree/f2cfd2fe0e2ca4e90994f96afe268bbd4382a891/rust/kernel/blk/mq.rs Cc: Wedson Almeida Filho Signed-off-by: Andreas Hindborg --- rust/bindings/bindings_helper.h | 2 + rust/helpers.c | 22 +++ rust/kernel/block.rs | 5 + rust/kernel/block/mq.rs | 15 ++ rust/kernel/block/mq/gen_disk.rs | 133 +++++++++++++++ rust/kernel/block/mq/operations.rs | 260 +++++++++++++++++++++++++++++ rust/kernel/block/mq/raw_writer.rs | 30 ++++ rust/kernel/block/mq/request.rs | 71 ++++++++ rust/kernel/block/mq/tag_set.rs | 92 ++++++++++ rust/kernel/error.rs | 4 + rust/kernel/lib.rs | 1 + 11 files changed, 635 insertions(+) create mode 100644 rust/kernel/block.rs create mode 100644 rust/kernel/block/mq.rs create mode 100644 rust/kernel/block/mq/gen_disk.rs create mode 100644 rust/kernel/block/mq/operations.rs create mode 100644 rust/kernel/block/mq/raw_writer.rs create mode 100644 rust/kernel/block/mq/request.rs create mode 100644 rust/kernel/block/mq/tag_set.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 52834962b94d..86c07eeb1ba1 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; diff --git a/rust/helpers.c b/rust/helpers.c index 9bd9d95da951..a59341084774 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -18,6 +18,7 @@ * accidentally exposed. */ +#include #include #include #include @@ -28,6 +29,8 @@ #include #include #include +#include +#include __noreturn void rust_helper_BUG(void) { @@ -130,6 +133,25 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); +struct bio_vec rust_helper_req_bvec(struct request *rq) +{ + return req_bvec(rq); +} +EXPORT_SYMBOL_GPL(rust_helper_req_bvec); + +void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) +{ + return blk_mq_rq_to_pdu(rq); +} +EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_to_pdu); + +void rust_helper_bio_advance_iter_single(const struct bio *bio, + struct bvec_iter *iter, + unsigned int bytes) { + bio_advance_iter_single(bio, iter, bytes); +} +EXPORT_SYMBOL_GPL(rust_helper_bio_advance_iter_single); + void rust_helper_init_radix_tree(struct xarray *tree, gfp_t gfp_mask) { INIT_RADIX_TREE(tree, gfp_mask); diff --git a/rust/kernel/block.rs b/rust/kernel/block.rs new file mode 100644 index 000000000000..4c93317a568a --- /dev/null +++ b/rust/kernel/block.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with the block layer + +pub mod mq; diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs new file mode 100644 index 000000000000..5b40f6a73c0f --- /dev/null +++ b/rust/kernel/block/mq.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides types for implementing drivers that interface the +//! blk-mq subsystem + +mod gen_disk; +mod operations; +mod raw_writer; +mod request; +mod tag_set; + +pub use gen_disk::GenDisk; +pub use operations::Operations; +pub use request::Request; +pub use tag_set::TagSet; diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs new file mode 100644 index 000000000000..50496af15bbf --- /dev/null +++ b/rust/kernel/block/mq/gen_disk.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GenDisk abstraction +//! +//! C header: [`include/linux/blkdev.h`](../../include/linux/blkdev.h) +//! C header: [`include/linux/blk_mq.h`](../../include/linux/blk_mq.h) + +use crate::block::mq::{raw_writer::RawWriter, Operations, TagSet}; +use crate::{ + bindings, error::from_err_ptr, error::Result, sync::Arc, types::ForeignOwnable, + types::ScopeGuard, +}; +use core::fmt::{self, Write}; + +/// A generic block device +/// +/// # Invariants +/// +/// - `gendisk` must always point to an initialized and valid `struct gendisk`. +pub struct GenDisk { + _tagset: Arc>, + gendisk: *mut bindings::gendisk, +} + +// SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a +// `TagSet` It is safe to send this to other threads as long as T is Send. +unsafe impl Send for GenDisk {} + +impl GenDisk { + /// Try to create a new `GenDisk` + pub fn try_new(tagset: Arc>, queue_data: T::QueueData) -> Result { + let data = queue_data.into_foreign(); + let recover_data = ScopeGuard::new(|| { + // SAFETY: T::QueueData was created by the call to `into_foreign()` above + unsafe { T::QueueData::from_foreign(data) }; + }); + + let lock_class_key = crate::sync::LockClassKey::new(); + + // SAFETY: `tagset.raw_tag_set()` points to a valid and initialized tag set + let gendisk = from_err_ptr(unsafe { + bindings::__blk_mq_alloc_disk(tagset.raw_tag_set(), data as _, lock_class_key.as_ptr()) + })?; + + const TABLE: bindings::block_device_operations = bindings::block_device_operations { + submit_bio: None, + open: None, + release: None, + ioctl: None, + compat_ioctl: None, + check_events: None, + unlock_native_capacity: None, + getgeo: None, + set_read_only: None, + swap_slot_free_notify: None, + report_zones: None, + devnode: None, + alternative_gpt_sector: None, + get_unique_id: None, + owner: core::ptr::null_mut(), + pr_ops: core::ptr::null_mut(), + free_disk: None, + poll_bio: None, + }; + + // SAFETY: gendisk is a valid pointer as we initialized it above + unsafe { (*gendisk).fops = &TABLE }; + + recover_data.dismiss(); + Ok(Self { + _tagset: tagset, + gendisk, + }) + } + + /// Set the name of the device + pub fn set_name(&self, args: fmt::Arguments<'_>) -> Result { + let mut raw_writer = RawWriter::from_array(unsafe { &mut (*self.gendisk).disk_name }); + raw_writer.write_fmt(args)?; + raw_writer.write_char('\0')?; + Ok(()) + } + + /// Register the device with the kernel. When this function return, the + /// device is accessible from VFS. The kernel may issue reads to the device + /// during registration to discover partition infomation. + pub fn add(&self) -> Result { + crate::error::to_result(unsafe { + bindings::device_add_disk(core::ptr::null_mut(), self.gendisk, core::ptr::null_mut()) + }) + } + + /// Call to tell the block layer the capcacity of the device + pub fn set_capacity(&self, sectors: u64) { + unsafe { bindings::set_capacity(self.gendisk, sectors) }; + } + + /// Set the logical block size of the device + pub fn set_queue_logical_block_size(&self, size: u32) { + unsafe { bindings::blk_queue_logical_block_size((*self.gendisk).queue, size) }; + } + + /// Set the physical block size of the device + pub fn set_queue_physical_block_size(&self, size: u32) { + unsafe { bindings::blk_queue_physical_block_size((*self.gendisk).queue, size) }; + } + + /// Set the rotational media attribute for the device + pub fn set_rotational(&self, rotational: bool) { + if !rotational { + unsafe { + bindings::blk_queue_flag_set(bindings::QUEUE_FLAG_NONROT, (*self.gendisk).queue) + }; + } else { + unsafe { + bindings::blk_queue_flag_clear(bindings::QUEUE_FLAG_NONROT, (*self.gendisk).queue) + }; + } + } +} + +impl Drop for GenDisk { + fn drop(&mut self) { + let queue_data = unsafe { (*(*self.gendisk).queue).queuedata }; + + unsafe { bindings::del_gendisk(self.gendisk) }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a + // call to `ForeignOwnable::into_pointer()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called here. + let _queue_data = unsafe { T::QueueData::from_foreign(queue_data) }; + } +} diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs new file mode 100644 index 000000000000..fb1ab707d1f0 --- /dev/null +++ b/rust/kernel/block/mq/operations.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides an interface for blk-mq drivers to implement. +//! +//! C header: [`include/linux/blk-mq.h`](../../include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{tag_set::TagSetRef, Request}, + error::{from_result, Result}, + types::ForeignOwnable, +}; +use core::{marker::PhantomData, pin::Pin}; + +/// Implement this trait to interface blk-mq as block devices +#[macros::vtable] +pub trait Operations: Sized { + /// Data associated with a request. This data is located next to the request + /// structure. + type RequestData; + + /// Data associated with the `struct request_queue` that is allocated for + /// the `GenDisk` associated with this `Operations` implementation. + type QueueData: ForeignOwnable; + + /// Data associated with a dispatch queue. This is stored as a pointer in + /// `struct blk_mq_hw_ctx`. + type HwData: ForeignOwnable; + + /// Data associated with a tag set. This is stored as a pointer in `struct + /// blk_mq_tag_set`. + type TagSetData: ForeignOwnable; + + /// Called by the kernel to allocate a new `RequestData`. The structure will + /// eventually be pinned, so defer initialization to `init_request_data()` + fn new_request_data( + _tagset_data: ::Borrowed<'_>, + ) -> Result; + + /// Called by the kernel to initialize a previously allocated `RequestData` + fn init_request_data( + _tagset_data: ::Borrowed<'_>, + _data: Pin<&mut Self::RequestData>, + ) -> Result { + Ok(()) + } + + /// Called by the kernel to queue a request with the driver. If `is_last` is + /// `false`, the driver is allowed to defer commiting the request. + fn queue_rq( + hw_data: ::Borrowed<'_>, + queue_data: ::Borrowed<'_>, + rq: &Request, + is_last: bool, + ) -> Result; + + /// Called by the kernel to indicate that queued requests should be submitted + fn commit_rqs( + hw_data: ::Borrowed<'_>, + queue_data: ::Borrowed<'_>, + ); + + /// Called by the kernel when the request is completed + fn complete(_rq: &Request); + + /// Called by the kernel to allocate and initialize a driver specific hardware context data + fn init_hctx( + tagset_data: ::Borrowed<'_>, + hctx_idx: u32, + ) -> Result; + + /// Called by the kernel to poll the device for completed requests. Only used for poll queues. + fn poll(_hw_data: ::Borrowed<'_>) -> i32 { + unreachable!() + } + + /// Called by the kernel to map submission queues to CPU cores. + fn map_queues(_tag_set: &TagSetRef) { + unreachable!() + } + + // There is no need for exit_request() because `drop` will be called. +} + +pub(crate) struct OperationsVtable(PhantomData); + +impl OperationsVtable { + // # Safety + // + // The caller of this function must ensure that `hctx` and `bd` are valid + // and initialized. The pointees must outlive this function. Further + // `hctx->driver_data` must be a pointer created by a call to + // `Self::init_hctx_callback()` and the pointee must outlive this function. + // This function must not be called with a `hctx` for which + // `Self::exit_hctx_callback()` has been called. + unsafe extern "C" fn queue_rq_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + bd: *const bindings::blk_mq_queue_data, + ) -> bindings::blk_status_t { + // SAFETY: `bd` is valid as required by the safety requirement for this function. + let rq = unsafe { (*bd).rq }; + + // SAFETY: The safety requirement for this function ensure that + // `(*hctx).driver_data` was returned by a call to + // `Self::init_hctx_callback()`. That function uses + // `PointerWrapper::into_pointer()` to create `driver_data`. Further, + // the returned value does not outlive this function and + // `from_foreign()` is not called until `Self::exit_hctx_callback()` is + // called. By the safety requirement of this function and contract with + // the `blk-mq` API, `queue_rq_callback()` will not be called after that + // point. + let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) }; + + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a + // call to `ForeignOwnable::into_pointer()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + + // SAFETY: `bd` is valid as required by the safety requirement for this function. + let ret = T::queue_rq( + hw_data, + queue_data, + &unsafe { Request::from_ptr(rq) }, + unsafe { (*bd).last }, + ); + if let Err(e) = ret { + e.to_blk_status() + } else { + bindings::BLK_STS_OK as _ + } + } + + unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) { + let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) }; + + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a + // call to `ForeignOwnable::into_pointer()` to create `queuedata`. + // `ForeignOwnable::from_foreign()` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + T::commit_rqs(hw_data, queue_data) + } + + unsafe extern "C" fn complete_callback(rq: *mut bindings::request) { + T::complete(&unsafe { Request::from_ptr(rq) }); + } + + unsafe extern "C" fn poll_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + _iob: *mut bindings::io_comp_batch, + ) -> core::ffi::c_int { + let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) }; + T::poll(hw_data) + } + + unsafe extern "C" fn init_hctx_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + tagset_data: *mut core::ffi::c_void, + hctx_idx: core::ffi::c_uint, + ) -> core::ffi::c_int { + from_result(|| { + let tagset_data = unsafe { T::TagSetData::borrow(tagset_data) }; + let data = T::init_hctx(tagset_data, hctx_idx)?; + unsafe { (*hctx).driver_data = data.into_foreign() as _ }; + Ok(0) + }) + } + + unsafe extern "C" fn exit_hctx_callback( + hctx: *mut bindings::blk_mq_hw_ctx, + _hctx_idx: core::ffi::c_uint, + ) { + let ptr = unsafe { (*hctx).driver_data }; + unsafe { T::HwData::from_foreign(ptr) }; + } + + unsafe extern "C" fn init_request_callback( + set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: core::ffi::c_uint, + _numa_node: core::ffi::c_uint, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory + // for the request data. + let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) } as *mut T::RequestData; + let tagset_data = unsafe { T::TagSetData::borrow((*set).driver_data) }; + + let v = T::new_request_data(tagset_data)?; + + // SAFETY: `pdu` memory is valid, as it was allocated by the caller. + unsafe { pdu.write(v) }; + + let tagset_data = unsafe { T::TagSetData::borrow((*set).driver_data) }; + // SAFETY: `pdu` memory is valid and properly initialised. + T::init_request_data(tagset_data, unsafe { Pin::new_unchecked(&mut *pdu) })?; + + Ok(0) + }) + } + + unsafe extern "C" fn exit_request_callback( + _set: *mut bindings::blk_mq_tag_set, + rq: *mut bindings::request, + _hctx_idx: core::ffi::c_uint, + ) { + // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory + // for the request data. + let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) } as *mut T::RequestData; + + // SAFETY: `pdu` is valid for read and write and is properly initialised. + unsafe { core::ptr::drop_in_place(pdu) }; + } + + unsafe extern "C" fn map_queues_callback(tag_set_ptr: *mut bindings::blk_mq_tag_set) { + let tag_set = unsafe { TagSetRef::from_ptr(tag_set_ptr) }; + T::map_queues(&tag_set); + } + + const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops { + queue_rq: Some(Self::queue_rq_callback), + queue_rqs: None, + commit_rqs: Some(Self::commit_rqs_callback), + get_budget: None, + put_budget: None, + set_rq_budget_token: None, + get_rq_budget_token: None, + timeout: None, + poll: if T::HAS_POLL { + Some(Self::poll_callback) + } else { + None + }, + complete: Some(Self::complete_callback), + init_hctx: Some(Self::init_hctx_callback), + exit_hctx: Some(Self::exit_hctx_callback), + init_request: Some(Self::init_request_callback), + exit_request: Some(Self::exit_request_callback), + cleanup_rq: None, + busy: None, + map_queues: if T::HAS_MAP_QUEUES { + Some(Self::map_queues_callback) + } else { + None + }, + #[cfg(CONFIG_BLK_DEBUG_FS)] + show_rq: None, + }; + + pub(crate) const unsafe fn build() -> &'static bindings::blk_mq_ops { + &Self::VTABLE + } +} diff --git a/rust/kernel/block/mq/raw_writer.rs b/rust/kernel/block/mq/raw_writer.rs new file mode 100644 index 000000000000..25c16ee0b1f7 --- /dev/null +++ b/rust/kernel/block/mq/raw_writer.rs @@ -0,0 +1,30 @@ +use core::fmt::{self, Write}; + +pub(crate) struct RawWriter { + ptr: *mut u8, + len: usize, +} + +impl RawWriter { + unsafe fn new(ptr: *mut u8, len: usize) -> Self { + Self { ptr, len } + } + + pub(crate) fn from_array(a: &mut [core::ffi::c_char; N]) -> Self { + unsafe { Self::new(&mut a[0] as *mut _ as _, N) } + } +} + +impl Write for RawWriter { + fn write_str(&mut self, s: &str) -> fmt::Result { + let bytes = s.as_bytes(); + let len = bytes.len(); + if len > self.len { + return Err(fmt::Error); + } + unsafe { core::ptr::copy_nonoverlapping(&bytes[0], self.ptr, len) }; + self.ptr = unsafe { self.ptr.add(len) }; + self.len -= len; + Ok(()) + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs new file mode 100644 index 000000000000..e95ae3fd71ad --- /dev/null +++ b/rust/kernel/block/mq/request.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides a wrapper for the C `struct request` type. +//! +//! C header: [`include/linux/blk-mq.h`](../../include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::Operations, + error::{Error, Result}, +}; +use core::marker::PhantomData; + +/// A wrapper around a blk-mq `struct request`. This represents an IO request. +pub struct Request { + ptr: *mut bindings::request, + _p: PhantomData, +} + +impl Request { + pub(crate) unsafe fn from_ptr(ptr: *mut bindings::request) -> Self { + Self { + ptr, + _p: PhantomData, + } + } + + /// Get the command identifier for the request + pub fn command(&self) -> u32 { + unsafe { (*self.ptr).cmd_flags & ((1 << bindings::REQ_OP_BITS) - 1) } + } + + /// Call this to indicate to the kernel that the request has been issued by the driver + pub fn start(&self) { + unsafe { bindings::blk_mq_start_request(self.ptr) }; + } + + /// Call this to indicate to the kernel that the request has been completed without errors + // TODO: Consume rq so that we can't use it after ending it? + pub fn end_ok(&self) { + unsafe { bindings::blk_mq_end_request(self.ptr, bindings::BLK_STS_OK as _) }; + } + + /// Call this to indicate to the kernel that the request completed with an error + pub fn end_err(&self, err: Error) { + unsafe { bindings::blk_mq_end_request(self.ptr, err.to_blk_status()) }; + } + + /// Call this to indicate that the request completed with the status indicated by `status` + pub fn end(&self, status: Result) { + if let Err(e) = status { + self.end_err(e); + } else { + self.end_ok(); + } + } + + /// Call this to schedule defered completion of the request + // TODO: Consume rq so that we can't use it after completing it? + pub fn complete(&self) { + if !unsafe { bindings::blk_mq_complete_request_remote(self.ptr) } { + T::complete(&unsafe { Self::from_ptr(self.ptr) }); + } + } + + /// Get the target sector for the request + #[inline(always)] + pub fn sector(&self) -> usize { + unsafe { (*self.ptr).__sector as usize } + } +} diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs new file mode 100644 index 000000000000..d122db7f6d0e --- /dev/null +++ b/rust/kernel/block/mq/tag_set.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This module provides the `TagSet` struct to wrap the C `struct blk_mq_tag_set`. +//! +//! C header: [`include/linux/blk-mq.h`](../../include/linux/blk-mq.h) + +use crate::{ + bindings, + block::mq::{operations::OperationsVtable, Operations}, + error::{Error, Result}, + sync::Arc, + types::ForeignOwnable, +}; +use core::{cell::UnsafeCell, convert::TryInto, marker::PhantomData}; + +/// A wrapper for the C `struct blk_mq_tag_set` +pub struct TagSet { + inner: UnsafeCell, + _p: PhantomData, +} + +impl TagSet { + /// Try to create a new tag set + pub fn try_new( + nr_hw_queues: u32, + tagset_data: T::TagSetData, + num_tags: u32, + num_maps: u32, + ) -> Result> { + let tagset = Arc::try_new(Self { + inner: UnsafeCell::new(bindings::blk_mq_tag_set::default()), + _p: PhantomData, + })?; + + // SAFETY: We just allocated `tagset`, we know this is the only reference to it. + let inner = unsafe { &mut *tagset.inner.get() }; + + inner.ops = unsafe { OperationsVtable::::build() }; + inner.nr_hw_queues = nr_hw_queues; + inner.timeout = 0; // 0 means default which is 30 * HZ in C + inner.numa_node = bindings::NUMA_NO_NODE; + inner.queue_depth = num_tags; + inner.cmd_size = core::mem::size_of::().try_into()?; + inner.flags = bindings::BLK_MQ_F_SHOULD_MERGE; + inner.driver_data = tagset_data.into_foreign() as _; + inner.nr_maps = num_maps; + + // SAFETY: `inner` points to valid and initialised memory. + let ret = unsafe { bindings::blk_mq_alloc_tag_set(inner) }; + if ret < 0 { + // SAFETY: We created `driver_data` above with `into_foreign` + unsafe { T::TagSetData::from_foreign(inner.driver_data) }; + return Err(Error::from_errno(ret)); + } + + Ok(tagset) + } + + /// Return the pointer to the wrapped `struct blk_mq_tag_set` + pub(crate) fn raw_tag_set(&self) -> *mut bindings::blk_mq_tag_set { + self.inner.get() + } +} + +impl Drop for TagSet { + fn drop(&mut self) { + let tagset_data = unsafe { (*self.inner.get()).driver_data }; + + // SAFETY: `inner` is valid and has been properly initialised during construction. + unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) }; + + // SAFETY: `tagset_data` was created by a call to + // `ForeignOwnable::into_foreign` in `TagSet::try_new()` + unsafe { T::TagSetData::from_foreign(tagset_data) }; + } +} + +/// A tag set reference. Used to control lifetime and prevent drop of TagSet references passed to +/// `Operations::map_queues()` +pub struct TagSetRef { + ptr: *mut bindings::blk_mq_tag_set, +} + +impl TagSetRef { + pub(crate) unsafe fn from_ptr(tagset: *mut bindings::blk_mq_tag_set) -> Self { + Self { ptr: tagset } + } + + pub fn ptr(&self) -> *mut bindings::blk_mq_tag_set { + self.ptr + } +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 5f4114b30b94..421fef677321 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -107,6 +107,10 @@ impl Error { self.0 } + pub(crate) fn to_blk_status(self) -> bindings::blk_status_t { + unsafe { bindings::errno_to_blk_status(self.0) } + } + /// Returns the error encoded as a pointer. #[allow(dead_code)] pub(crate) fn to_ptr(self) -> *mut T { diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 8bef6686504b..cd798d12d97c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -34,6 +34,7 @@ extern crate self as kernel; #[cfg(not(test))] #[cfg(not(testlib))] mod allocator; +pub mod block; mod build_assert; pub mod error; pub mod init; From patchwork Wed May 3 09:07:01 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229897 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 78052C7EE2C for ; Wed, 3 May 2023 09:07:33 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229947AbjECJHb (ORCPT ); Wed, 3 May 2023 05:07:31 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45364 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229878AbjECJH0 (ORCPT ); Wed, 3 May 2023 05:07:26 -0400 Received: from mail-wr1-x429.google.com (mail-wr1-x429.google.com [IPv6:2a00:1450:4864:20::429]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id BE4714C02 for ; Wed, 3 May 2023 02:07:21 -0700 (PDT) Received: by mail-wr1-x429.google.com with SMTP id ffacd0b85a97d-30639daee76so1168172f8f.1 for ; Wed, 03 May 2023 02:07:21 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104840; x=1685696840; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=8Hg9EF+CTDGGAsWt0N5WyI6rk7QsJ8yExmPcN8MX0qQ=; b=d+lWf2ETGwZ6l0kF5yoMkucYwDf2kDuSHvqzafoL6LSaWDyChZFeb/y3Ask+uZL3ub Crm1FWg7HSXUUHQu4aZsOctpyLqbz0tvhgPgilALIxNAKTgLG9YZ5O7UUoSFUznFtAd7 8OtJqUv7m3XxiwrPol077Wc7De84xWLwlJ9zH1FgVk7LrKu7kj7KMolcbNdrGquUbXHA gx1o+RI6ewKReUGP7O57duA5S2sDgZdoARg8MtGBlJBT45+yVJtwMrDiwgD1CJt2xVPz ns0renNS1R/Xu68zuPNBQGUcpB38HrRkV/8/C94FuyxzQvT6lrnKR7AKdmaNVmf1WOts FeSw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104840; x=1685696840; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=8Hg9EF+CTDGGAsWt0N5WyI6rk7QsJ8yExmPcN8MX0qQ=; b=Gl6rJ08OYUmKE5dAlOKvOaHk9oL7gp8c+n6eK/rUfc0a6UJTfyOBtWKnuwpdSjzwk0 bsq/oXSd7P8LclYE+Jvhhww6IzcJa8dXPckg0QMb3dwr7fK/8bVMe1Qw6X55UzOxEG/B 2PKoPxkvGtXAx/zT9btmIoD4ky6EtnrKpqfeSZfpuGYjmIBFDKhOlPFwzqzMdul9Oq5R QboJhb44Gi+o/0Ng8APadHL6fqkOiTbg4WmMYX7nmgNTHPBEPb3JaZZqb0Nq6NWr1JZ8 7ndNHi3sU7niKI/6byssIzquda+8mdqT0e1OTxeMbXYsG5+arWgw3Efc8Em5X5Od6XB3 /Bfg== X-Gm-Message-State: AC+VfDzfloFq+QpcyhCTy/SRPXlVvwELmuJNTlz6K2CZzcTHW2c+NGkB LqHYKLj8X9M23SGuuE1tZwdlTA== X-Google-Smtp-Source: ACHHUZ6T2y7vjejoAr+gXXz7fkMQ4wlDgEasl/1jBzVe1iLoMPS+zptIdaYP9DejgMdUNDfXGrvBNg== X-Received: by 2002:adf:e484:0:b0:306:320b:5dbd with SMTP id i4-20020adfe484000000b00306320b5dbdmr5824254wrm.71.1683104840210; Wed, 03 May 2023 02:07:20 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id v12-20020adfe28c000000b0030497b3224bsm20046305wri.64.2023.05.03.02.07.19 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:19 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 04/11] rust: block: introduce `kernel::block::bio` module Date: Wed, 3 May 2023 11:07:01 +0200 Message-Id: <20230503090708.2524310-5-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg Add abstractions for working with `struct bio`. Signed-off-by: Andreas Hindborg --- rust/kernel/block.rs | 1 + rust/kernel/block/bio.rs | 93 ++++++++++++++++ rust/kernel/block/bio/vec.rs | 181 ++++++++++++++++++++++++++++++++ rust/kernel/block/mq/request.rs | 16 +++ 4 files changed, 291 insertions(+) create mode 100644 rust/kernel/block/bio.rs create mode 100644 rust/kernel/block/bio/vec.rs diff --git a/rust/kernel/block.rs b/rust/kernel/block.rs index 4c93317a568a..1797859551fd 100644 --- a/rust/kernel/block.rs +++ b/rust/kernel/block.rs @@ -2,4 +2,5 @@ //! Types for working with the block layer +pub mod bio; pub mod mq; diff --git a/rust/kernel/block/bio.rs b/rust/kernel/block/bio.rs new file mode 100644 index 000000000000..6e93e4420105 --- /dev/null +++ b/rust/kernel/block/bio.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with the bio layer. +//! +//! C header: [`include/linux/blk_types.h`](../../include/linux/blk_types.h) + +use core::fmt; +use core::ptr::NonNull; + +mod vec; + +pub use vec::BioSegmentIterator; +pub use vec::Segment; + +/// A wrapper around a `struct bio` pointer +/// +/// # Invariants +/// +/// First field must alwyas be a valid pointer to a valid `struct bio`. +pub struct Bio<'a>( + NonNull, + core::marker::PhantomData<&'a ()>, +); + +impl<'a> Bio<'a> { + /// Returns an iterator over segments in this `Bio`. Does not consider + /// segments of other bios in this bio chain. + #[inline(always)] + pub fn segment_iter(&'a self) -> BioSegmentIterator<'a> { + BioSegmentIterator::new(self) + } + + /// Get a pointer to the `bio_vec` off this bio + #[inline(always)] + fn io_vec(&self) -> *const bindings::bio_vec { + // SAFETY: By type invariant, get_raw() returns a valid pointer to a + // valid `struct bio` + unsafe { (*self.get_raw()).bi_io_vec } + } + + /// Return a copy of the `bvec_iter` for this `Bio` + #[inline(always)] + fn iter(&self) -> bindings::bvec_iter { + // SAFETY: self.0 is always a valid pointer + unsafe { (*self.get_raw()).bi_iter } + } + + /// Get the next `Bio` in the chain + #[inline(always)] + fn next(&self) -> Option> { + // SAFETY: self.0 is always a valid pointer + let next = unsafe { (*self.get_raw()).bi_next }; + Some(Self(NonNull::new(next)?, core::marker::PhantomData)) + } + + /// Return the raw pointer of the wrapped `struct bio` + #[inline(always)] + fn get_raw(&self) -> *const bindings::bio { + self.0.as_ptr() + } + + /// Create an instance of `Bio` from a raw pointer. Does check that the + /// pointer is not null. + #[inline(always)] + pub(crate) unsafe fn from_raw(ptr: *mut bindings::bio) -> Option> { + Some(Self(NonNull::new(ptr)?, core::marker::PhantomData)) + } +} + +impl core::fmt::Display for Bio<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Bio {:?}", self.0.as_ptr()) + } +} + +/// An iterator over `Bio` +pub struct BioIterator<'a> { + pub(crate) bio: Option>, +} + +impl<'a> core::iter::Iterator for BioIterator<'a> { + type Item = Bio<'a>; + + #[inline(always)] + fn next(&mut self) -> Option> { + if let Some(current) = self.bio.take() { + self.bio = current.next(); + Some(current) + } else { + None + } + } +} diff --git a/rust/kernel/block/bio/vec.rs b/rust/kernel/block/bio/vec.rs new file mode 100644 index 000000000000..acd328a6fe54 --- /dev/null +++ b/rust/kernel/block/bio/vec.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for working with `struct bio_vec` IO vectors +//! +//! C header: [`include/linux/bvec.h`](../../include/linux/bvec.h) + +use super::Bio; +use crate::error::Result; +use crate::pages::Pages; +use core::fmt; +use core::mem::ManuallyDrop; + +#[inline(always)] +fn mp_bvec_iter_offset(bvec: *const bindings::bio_vec, iter: &bindings::bvec_iter) -> u32 { + (unsafe { (*bvec_iter_bvec(bvec, iter)).bv_offset }) + iter.bi_bvec_done +} + +#[inline(always)] +fn mp_bvec_iter_page( + bvec: *const bindings::bio_vec, + iter: &bindings::bvec_iter, +) -> *mut bindings::page { + unsafe { (*bvec_iter_bvec(bvec, iter)).bv_page } +} + +#[inline(always)] +fn mp_bvec_iter_page_idx(bvec: *const bindings::bio_vec, iter: &bindings::bvec_iter) -> usize { + (mp_bvec_iter_offset(bvec, iter) / crate::PAGE_SIZE) as usize +} + +#[inline(always)] +fn mp_bvec_iter_len(bvec: *const bindings::bio_vec, iter: &bindings::bvec_iter) -> u32 { + iter.bi_size + .min(unsafe { (*bvec_iter_bvec(bvec, iter)).bv_len } - iter.bi_bvec_done) +} + +#[inline(always)] +fn bvec_iter_bvec( + bvec: *const bindings::bio_vec, + iter: &bindings::bvec_iter, +) -> *const bindings::bio_vec { + unsafe { bvec.add(iter.bi_idx as usize) } +} + +#[inline(always)] +fn bvec_iter_page( + bvec: *const bindings::bio_vec, + iter: &bindings::bvec_iter, +) -> *mut bindings::page { + unsafe { mp_bvec_iter_page(bvec, iter).add(mp_bvec_iter_page_idx(bvec, iter)) } +} + +#[inline(always)] +fn bvec_iter_len(bvec: *const bindings::bio_vec, iter: &bindings::bvec_iter) -> u32 { + mp_bvec_iter_len(bvec, iter).min(crate::PAGE_SIZE - bvec_iter_offset(bvec, iter)) +} + +#[inline(always)] +fn bvec_iter_offset(bvec: *const bindings::bio_vec, iter: &bindings::bvec_iter) -> u32 { + mp_bvec_iter_offset(bvec, iter) % crate::PAGE_SIZE +} + +/// A wrapper around a `strutct bio_vec` - a contiguous range of physical memory addresses +/// +/// # Invariants +/// +/// `bio_vec` must always be initialized and valid +pub struct Segment<'a> { + bio_vec: bindings::bio_vec, + _marker: core::marker::PhantomData<&'a ()>, +} + +impl Segment<'_> { + /// Get he lenght of the segment in bytes + #[inline(always)] + pub fn len(&self) -> usize { + self.bio_vec.bv_len as usize + } + + /// Returns true if the length of the segment is 0 + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get the offset field of the `bio_vec` + #[inline(always)] + pub fn offset(&self) -> usize { + self.bio_vec.bv_offset as usize + } + + /// Copy data of this segment into `page`. + #[inline(always)] + pub fn copy_to_page_atomic(&self, page: &mut Pages<0>) -> Result { + // SAFETY: self.bio_vec is valid and thus bv_page must be a valid + // pointer to a `struct page`. We do not own the page, but we prevent + // drop by wrapping the `Pages` in `ManuallyDrop`. + let our_page = ManuallyDrop::new(unsafe { Pages::<0>::from_raw(self.bio_vec.bv_page) }); + let our_map = our_page.kmap_atomic(); + + // TODO: Checck offset is within page - what guarantees does `bio_vec` provide? + let ptr = unsafe { (our_map.get_ptr() as *const u8).add(self.offset()) }; + + unsafe { page.write_atomic(ptr, self.offset(), self.len()) } + } + + /// Copy data from `page` into this segment + #[inline(always)] + pub fn copy_from_page_atomic(&mut self, page: &Pages<0>) -> Result { + // SAFETY: self.bio_vec is valid and thus bv_page must be a valid + // pointer to a `struct page`. We do not own the page, but we prevent + // drop by wrapping the `Pages` in `ManuallyDrop`. + let our_page = ManuallyDrop::new(unsafe { Pages::<0>::from_raw(self.bio_vec.bv_page) }); + let our_map = our_page.kmap_atomic(); + + // TODO: Checck offset is within page + let ptr = unsafe { (our_map.get_ptr() as *mut u8).add(self.offset()) }; + + unsafe { page.read_atomic(ptr, self.offset(), self.len()) } + } +} + +impl core::fmt::Display for Segment<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Segment {:?} len: {}", + self.bio_vec.bv_page, self.bio_vec.bv_len + ) + } +} + +/// An iterator over `Segment` +pub struct BioSegmentIterator<'a> { + bio: &'a Bio<'a>, + iter: bindings::bvec_iter, +} + +impl<'a> BioSegmentIterator<'a> { + #[inline(always)] + pub(crate) fn new(bio: &'a Bio<'a>) -> BioSegmentIterator<'_> { + Self { + bio, + iter: bio.iter(), + } + } +} + +impl<'a> core::iter::Iterator for BioSegmentIterator<'a> { + type Item = Segment<'a>; + + #[inline(always)] + fn next(&mut self) -> Option { + if self.iter.bi_size == 0 { + return None; + } + + // Macro + // bio_vec = bio_iter_iovec(bio, self.iter) + // bio_vec = bvec_iter_bvec(bio.bi_io_vec, self.iter); + let bio_vec_ret = bindings::bio_vec { + bv_page: bvec_iter_page(self.bio.io_vec(), &self.iter), + bv_len: bvec_iter_len(self.bio.io_vec(), &self.iter), + bv_offset: bvec_iter_offset(self.bio.io_vec(), &self.iter), + }; + + // Static C function + unsafe { + bindings::bio_advance_iter_single( + self.bio.get_raw(), + &mut self.iter as *mut bindings::bvec_iter, + bio_vec_ret.bv_len, + ) + }; + + Some(Segment { + bio_vec: bio_vec_ret, + _marker: core::marker::PhantomData, + }) + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index e95ae3fd71ad..ccb1033b64b6 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -11,6 +11,9 @@ use crate::{ }; use core::marker::PhantomData; +use crate::block::bio::Bio; +use crate::block::bio::BioIterator; + /// A wrapper around a blk-mq `struct request`. This represents an IO request. pub struct Request { ptr: *mut bindings::request, @@ -63,6 +66,19 @@ impl Request { } } + /// Get a wrapper for the first Bio in this request + #[inline(always)] + pub fn bio(&self) -> Option> { + let ptr = unsafe { (*self.ptr).bio }; + unsafe { Bio::from_raw(ptr) } + } + + /// Get an iterator over all bio structurs in this request + #[inline(always)] + pub fn bio_iter(&self) -> BioIterator<'_> { + BioIterator { bio: self.bio() } + } + /// Get the target sector for the request #[inline(always)] pub fn sector(&self) -> usize { From patchwork Wed May 3 09:07:02 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229900 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A4374C7EE2A for ; Wed, 3 May 2023 09:07:45 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229650AbjECJHn (ORCPT ); Wed, 3 May 2023 05:07:43 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45456 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229825AbjECJHb (ORCPT ); Wed, 3 May 2023 05:07:31 -0400 Received: from mail-wr1-x42e.google.com (mail-wr1-x42e.google.com [IPv6:2a00:1450:4864:20::42e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 6BA7EE4F for ; Wed, 3 May 2023 02:07:23 -0700 (PDT) Received: by mail-wr1-x42e.google.com with SMTP id ffacd0b85a97d-2f95231618aso2987458f8f.1 for ; Wed, 03 May 2023 02:07:23 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104841; x=1685696841; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=gSA28PAasqbQ6gihh6v56FFurfwvVr3++GQNYiXLBOM=; b=vQsE9+nWwjoc0CqWeJtb/ZfNGHy0rUvS1Y6KMIM/8XIALq5wndo94M0A9dNDG/sgx3 TntjBxyhGd3sPHq81BA5d+rCrZqPK+YW4IthsZPI1CtTP6bLLkexUwvgl31L4ic67Qf3 3a/12tbFt4/FvWOnftXK8qByjLSFlbpkmQMkf5C8I3hc6vlYpAnFzNFcDxdS2MRw4IEx wPbbwhJsNepi1C2Pis2b+onQe5ZIJJr4481QrdlPAHTxbbNDusKuXuoh7FCB5nas1ipJ GJZUZOXkGrlNsq2sAq9Ubuo5revBkkMADIqKLVPTnZWg7liUy4NgrQWnKD4vuzI/VlLI s34Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104841; x=1685696841; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=gSA28PAasqbQ6gihh6v56FFurfwvVr3++GQNYiXLBOM=; b=OAu9+q73mjUt7lLRvNyy6xCxtLjew7vpdSjBBkY3dfNrDXe3hySYJR0f8Tle7e/7wv Ol5pVV+NcP6eTykAHMzFVqE9tg97gupIFBU0WGNsyK4afplJUbcfO7/5U+XPVsiJWoXU u5kbtT1y8RoUG9yTZiDD+eXUcU8dN3q8b+yhIxNmbu4U115A/X24uow9cPeEzhgWF+84 XGqPMICC7chEPBxsueQpAdLAuXRTckY7gNAPPCc3EiXiFam1yLa3EdiBqH5pro3t7mGS MVR3ROZY4ookJl9xQXzcNAOMZCmSX2tYdpRlTocqxiufL/BlQhXRZaRLk4vEW2H/iPa2 1PVA== X-Gm-Message-State: AC+VfDyq6wf/BGHmU4dU5itE/prq1BhRlRmJxyISq4VRGOovSIYRw/ws rECn32G5IuTIHWj/kYC8yh+QBw== X-Google-Smtp-Source: ACHHUZ7Ksbe7Ooyt5nBUxHytKU58iAJRK1L03foLc8Hcc6DMHl/WjWoe8IHgUJ0g8IzAAH00aNw/ZQ== X-Received: by 2002:adf:f68c:0:b0:2f5:6430:35c with SMTP id v12-20020adff68c000000b002f56430035cmr13820637wrp.26.1683104841285; Wed, 03 May 2023 02:07:21 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id q11-20020a5d574b000000b003049d7b9f4csm19166645wrw.32.2023.05.03.02.07.20 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:21 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 05/11] RUST: add `module_params` macro Date: Wed, 3 May 2023 11:07:02 +0200 Message-Id: <20230503090708.2524310-6-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg This patch includes changes required for Rust kernel modules to utilize module parameters. Imported with minor changes from `rust` branch [1], see original code for attributions. [1] https://github.com/Rust-for-Linux/linux/tree/bc22545f38d74473cfef3e9fd65432733435b79f Cc: adamrk --- rust/kernel/lib.rs | 3 + rust/kernel/module_param.rs | 501 ++++++++++++++++++++++++++++++++++++ rust/macros/helpers.rs | 46 +++- rust/macros/module.rs | 402 +++++++++++++++++++++++++++-- 4 files changed, 920 insertions(+), 32 deletions(-) create mode 100644 rust/kernel/module_param.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index cd798d12d97c..a0bd0b0e2aef 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -22,6 +22,7 @@ #![feature(pin_macro)] #![feature(receiver_trait)] #![feature(unsize)] +#![feature(const_mut_refs)] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -39,6 +40,8 @@ mod build_assert; pub mod error; pub mod init; pub mod ioctl; +#[doc(hidden)] +pub mod module_param; pub mod pages; pub mod prelude; pub mod print; diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 000000000000..539decbdcd48 --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h) + +use crate::error::{code::*, from_result}; +use crate::str::{CStr, Formatter}; +use core::fmt::Write; + +/// Types that can be used for module parameters. +/// +/// Note that displaying the type in `sysfs` will fail if +/// [`alloc::string::ToString::to_string`] (as implemented through the +/// [`core::fmt::Display`] trait) writes more than [`PAGE_SIZE`] +/// bytes (including an additional null terminator). +/// +/// [`PAGE_SIZE`]: `crate::PAGE_SIZE` +pub trait ModuleParam: core::fmt::Display + core::marker::Sized { + /// The `ModuleParam` will be used by the kernel module through this type. + /// + /// This may differ from `Self` if, for example, `Self` needs to track + /// ownership without exposing it or allocate extra space for other possible + /// parameter values. See [`StringParam`] or [`ArrayParam`] for examples. + type Value: ?Sized; + + /// Whether the parameter is allowed to be set without an argument. + /// + /// Setting this to `true` allows the parameter to be passed without an + /// argument (e.g. just `module.param` instead of `module.param=foo`). + const NOARG_ALLOWED: bool; + + /// Convert a parameter argument into the parameter value. + /// + /// `None` should be returned when parsing of the argument fails. + /// `arg == None` indicates that the parameter was passed without an + /// argument. If `NOARG_ALLOWED` is set to `false` then `arg` is guaranteed + /// to always be `Some(_)`. + /// + /// Parameters passed at boot time will be set before [`kmalloc`] is + /// available (even if the module is loaded at a later time). However, in + /// this case, the argument buffer will be valid for the entire lifetime of + /// the kernel. So implementations of this method which need to allocate + /// should first check that the allocator is available (with + /// [`crate::bindings::slab_is_available`]) and when it is not available + /// provide an alternative implementation which doesn't allocate. In cases + /// where the allocator is not available it is safe to save references to + /// `arg` in `Self`, but in other cases a copy should be made. + /// + /// [`kmalloc`]: ../../../include/linux/slab.h + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option; + + /// Get the current value of the parameter for use in the kernel module. + /// + /// This function should not be used directly. Instead use the wrapper + /// `read` which will be generated by [`macros::module`]. + fn value(&self) -> &Self::Value; + + /// Set the module parameter from a string. + /// + /// Used to set the parameter value when loading the module or when set + /// through `sysfs`. + /// + /// # Safety + /// + /// If `val` is non-null then it must point to a valid null-terminated + /// string. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn set_param( + val: *const core::ffi::c_char, + param: *const crate::bindings::kernel_param, + ) -> core::ffi::c_int { + let arg = if val.is_null() { + None + } else { + Some(unsafe { CStr::from_char_ptr(val).as_bytes() }) + }; + match Self::try_from_param_arg(arg) { + Some(new_value) => { + let old_value = unsafe { (*param).__bindgen_anon_1.arg as *mut Self }; + let _ = unsafe { core::ptr::replace(old_value, new_value) }; + 0 + } + None => EINVAL.to_errno(), + } + } + + /// Write a string representation of the current parameter value to `buf`. + /// + /// Used for displaying the current parameter value in `sysfs`. + /// + /// # Safety + /// + /// `buf` must be a buffer of length at least `kernel::PAGE_SIZE` that is + /// writeable. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn get_param( + buf: *mut core::ffi::c_char, + param: *const crate::bindings::kernel_param, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C contracts guarantees that the buffer is at least `PAGE_SIZE` bytes. + let mut f = unsafe { Formatter::from_buffer(buf.cast(), crate::PAGE_SIZE as usize) }; + unsafe { write!(f, "{}\0", *((*param).__bindgen_anon_1.arg as *mut Self)) }?; + Ok(f.bytes_written().try_into()?) + }) + } + + /// Drop the parameter. + /// + /// Called when unloading a module. + /// + /// # Safety + /// + /// The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn free(arg: *mut core::ffi::c_void) { + unsafe { core::ptr::drop_in_place(arg as *mut Self) }; + } +} + +/// Trait for parsing integers. +/// +/// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtol +/// [`kstrtoul()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtoul +trait ParseInt: Sized { + fn from_str_radix(src: &str, radix: u32) -> Result; + fn checked_neg(self) -> Option; + + fn from_str_unsigned(src: &str) -> Result { + let (radix, digits) = if let Some(n) = src.strip_prefix("0x") { + (16, n) + } else if let Some(n) = src.strip_prefix("0X") { + (16, n) + } else if let Some(n) = src.strip_prefix("0o") { + (8, n) + } else if let Some(n) = src.strip_prefix("0O") { + (8, n) + } else if let Some(n) = src.strip_prefix("0b") { + (2, n) + } else if let Some(n) = src.strip_prefix("0B") { + (2, n) + } else if src.starts_with('0') { + (8, src) + } else { + (10, src) + }; + Self::from_str_radix(digits, radix) + } + + fn from_str(src: &str) -> Option { + match src.bytes().next() { + None => None, + Some(b'-') => Self::from_str_unsigned(&src[1..]).ok()?.checked_neg(), + Some(b'+') => Some(Self::from_str_unsigned(&src[1..]).ok()?), + Some(_) => Some(Self::from_str_unsigned(src).ok()?), + } + } +} + +macro_rules! impl_parse_int { + ($ty:ident) => { + impl ParseInt for $ty { + fn from_str_radix(src: &str, radix: u32) -> Result { + $ty::from_str_radix(src, radix) + } + + fn checked_neg(self) -> Option { + self.checked_neg() + } + } + }; +} + +impl_parse_int!(i8); +impl_parse_int!(u8); +impl_parse_int!(i16); +impl_parse_int!(u16); +impl_parse_int!(i32); +impl_parse_int!(u32); +impl_parse_int!(i64); +impl_parse_int!(u64); +impl_parse_int!(isize); +impl_parse_int!(usize); + +macro_rules! impl_module_param { + ($ty:ident) => { + impl ModuleParam for $ty { + type Value = $ty; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + let bytes = arg?; + let utf8 = core::str::from_utf8(bytes).ok()?; + <$ty as crate::module_param::ParseInt>::from_str(utf8) + } + + #[inline(always)] + fn value(&self) -> &Self::Value { + self + } + } + }; +} + +#[doc(hidden)] +#[macro_export] +/// Generate a static [`kernel_param_ops`](../../../include/linux/moduleparam.h) struct. +/// +/// # Examples +/// +/// ```ignore +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) => { + $crate::make_param_ops!( + #[doc=""] + $ops, + $ty + ); + }; + ($(#[$meta:meta])* $ops:ident, $ty:ty) => { + $(#[$meta])* + /// + /// Static [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// struct generated by [`make_param_ops`]. + pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops { + flags: if <$ty as $crate::module_param::ModuleParam>::NOARG_ALLOWED { + $crate::bindings::KERNEL_PARAM_OPS_FL_NOARG + } else { + 0 + }, + set: Some(<$ty as $crate::module_param::ModuleParam>::set_param), + get: Some(<$ty as $crate::module_param::ModuleParam>::get_param), + free: Some(<$ty as $crate::module_param::ModuleParam>::free), + }; + }; +} + +impl_module_param!(i8); +impl_module_param!(u8); +impl_module_param!(i16); +impl_module_param!(u16); +impl_module_param!(i32); +impl_module_param!(u32); +impl_module_param!(i64); +impl_module_param!(u64); +impl_module_param!(isize); +impl_module_param!(usize); + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i8`]. + PARAM_OPS_I8, + i8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u8`]. + PARAM_OPS_U8, + u8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i16`]. + PARAM_OPS_I16, + i16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u16`]. + PARAM_OPS_U16, + u16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i32`]. + PARAM_OPS_I32, + i32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u32`]. + PARAM_OPS_U32, + u32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i64`]. + PARAM_OPS_I64, + i64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u64`]. + PARAM_OPS_U64, + u64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`isize`]. + PARAM_OPS_ISIZE, + isize +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`usize`]. + PARAM_OPS_USIZE, + usize +); + +impl ModuleParam for bool { + type Value = bool; + + const NOARG_ALLOWED: bool = true; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + match arg { + None => Some(true), + Some(b"y") | Some(b"Y") | Some(b"1") | Some(b"true") => Some(true), + Some(b"n") | Some(b"N") | Some(b"0") | Some(b"false") => Some(false), + _ => None, + } + } + + #[inline(always)] + fn value(&self) -> &Self::Value { + self + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`bool`]. + PARAM_OPS_BOOL, + bool +); + +/// An array of at __most__ `N` values. +/// +/// # Invariant +/// +/// The first `self.used` elements of `self.values` are initialized. +pub struct ArrayParam { + values: [core::mem::MaybeUninit; N], + used: usize, +} + +impl ArrayParam { + fn values(&self) -> &[T] { + // SAFETY: The invariant maintained by `ArrayParam` allows us to cast + // the first `self.used` elements to `T`. + unsafe { + &*(&self.values[0..self.used] as *const [core::mem::MaybeUninit] as *const [T]) + } + } +} + +impl ArrayParam { + const fn new() -> Self { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + ArrayParam { + values: [core::mem::MaybeUninit::uninit(); N], + used: 0, + } + } + + const fn push(&mut self, val: T) { + if self.used < N { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + self.values[self.used] = core::mem::MaybeUninit::new(val); + self.used += 1; + } + } + + /// Create an instance of `ArrayParam` initialized with `vals`. + /// + /// This function is only meant to be used in the [`module::module`] macro. + pub const fn create(vals: &[T]) -> Self { + let mut result = ArrayParam::new(); + let mut i = 0; + while i < vals.len() { + result.push(vals[i]); + i += 1; + } + result + } +} + +impl core::fmt::Display for ArrayParam { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + for val in self.values() { + write!(f, "{},", val)?; + } + Ok(()) + } +} + +impl ModuleParam + for ArrayParam +{ + type Value = [T]; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + arg.and_then(|args| { + let mut result = Self::new(); + for arg in args.split(|b| *b == b',') { + result.push(T::try_from_param_arg(Some(arg))?); + } + Some(result) + }) + } + + fn value(&self) -> &Self::Value { + self.values() + } +} + +/// A C-style string parameter. +/// +/// The Rust version of the [`charp`] parameter. This type is meant to be +/// used by the [`macros::module`] macro, not handled directly. Instead use the +/// `read` method generated by that macro. +/// +/// [`charp`]: ../../../include/linux/moduleparam.h +pub enum StringParam { + /// A borrowed parameter value. + /// + /// Either the default value (which is static in the module) or borrowed + /// from the original argument buffer used to set the value. + Ref(&'static [u8]), + + /// A value that was allocated when the parameter was set. + /// + /// The value needs to be freed when the parameter is reset or the module is + /// unloaded. + Owned(alloc::vec::Vec), +} + +impl StringParam { + fn bytes(&self) -> &[u8] { + match self { + StringParam::Ref(bytes) => *bytes, + StringParam::Owned(vec) => &vec[..], + } + } +} + +impl core::fmt::Display for StringParam { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let bytes = self.bytes(); + match core::str::from_utf8(bytes) { + Ok(utf8) => write!(f, "{}", utf8), + Err(_) => write!(f, "{:?}", bytes), + } + } +} + +impl ModuleParam for StringParam { + type Value = [u8]; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + // SAFETY: It is always safe to call [`slab_is_available`](../../../include/linux/slab.h). + let slab_available = unsafe { crate::bindings::slab_is_available() }; + arg.and_then(|arg| { + if slab_available { + let mut vec = alloc::vec::Vec::new(); + vec.try_extend_from_slice(arg).ok()?; + Some(StringParam::Owned(vec)) + } else { + Some(StringParam::Ref(arg)) + } + }) + } + + fn value(&self) -> &Self::Value { + self.bytes() + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`StringParam`]. + PARAM_OPS_STR, + StringParam +); diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index b2bdd4d8c958..8b5f9bc414d7 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -18,6 +18,16 @@ pub(crate) fn try_literal(it: &mut token_stream::IntoIter) -> Option { } } +pub(crate) fn try_byte_string(it: &mut token_stream::IntoIter) -> Option { + try_literal(it).and_then(|byte_string| { + if byte_string.starts_with("b\"") && byte_string.ends_with('\"') { + Some(byte_string[2..byte_string.len() - 1].to_string()) + } else { + None + } + }) +} + pub(crate) fn try_string(it: &mut token_stream::IntoIter) -> Option { try_literal(it).and_then(|string| { if string.starts_with('\"') && string.ends_with('\"') { @@ -46,14 +56,8 @@ pub(crate) fn expect_punct(it: &mut token_stream::IntoIter) -> char { } } -pub(crate) fn expect_string(it: &mut token_stream::IntoIter) -> String { - try_string(it).expect("Expected string") -} - -pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> String { - let string = try_string(it).expect("Expected string"); - assert!(string.is_ascii(), "Expected ASCII string"); - string +pub(crate) fn expect_literal(it: &mut token_stream::IntoIter) -> String { + try_literal(it).expect("Expected Literal") } pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group { @@ -64,8 +68,34 @@ pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group { } } +pub(crate) fn expect_string(it: &mut token_stream::IntoIter) -> String { + try_string(it).expect("Expected string") +} + +pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> String { + let string = try_string(it).expect("Expected string"); + assert!(string.is_ascii(), "Expected ASCII string"); + string +} + pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { if it.next().is_some() { panic!("Expected end"); } } + +pub(crate) fn get_literal(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let literal = expect_literal(it); + assert_eq!(expect_punct(it), ','); + literal +} + +pub(crate) fn get_string(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let string = expect_string(it); + assert_eq!(expect_punct(it), ','); + string +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs index fb1244f8c2e6..172631e7fb05 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -1,25 +1,40 @@ // SPDX-License-Identifier: GPL-2.0 -use crate::helpers::*; -use proc_macro::{token_stream, Delimiter, Literal, TokenStream, TokenTree}; +use proc_macro::{token_stream, Delimiter, Group, Literal, TokenStream, TokenTree}; use std::fmt::Write; -fn expect_string_array(it: &mut token_stream::IntoIter) -> Vec { - let group = expect_group(it); - assert_eq!(group.delimiter(), Delimiter::Bracket); - let mut values = Vec::new(); - let mut it = group.stream().into_iter(); - - while let Some(val) = try_string(&mut it) { - assert!(val.is_ascii(), "Expected ASCII string"); - values.push(val); - match it.next() { - Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','), - None => break, - _ => panic!("Expected ',' or end of array"), +use crate::helpers::*; + +#[derive(Clone, PartialEq)] +enum ParamType { + Ident(String), + Array { vals: String, max_length: usize }, +} + +fn expect_array_fields(it: &mut token_stream::IntoIter) -> ParamType { + assert_eq!(expect_punct(it), '<'); + let vals = expect_ident(it); + assert_eq!(expect_punct(it), ','); + let max_length_str = expect_literal(it); + let max_length = max_length_str + .parse::() + .expect("Expected usize length"); + assert_eq!(expect_punct(it), '>'); + ParamType::Array { vals, max_length } +} + +fn expect_type(it: &mut token_stream::IntoIter) -> ParamType { + if let TokenTree::Ident(ident) = it + .next() + .expect("Reached end of token stream for param type") + { + match ident.to_string().as_ref() { + "ArrayParam" => expect_array_fields(it), + _ => ParamType::Ident(ident.to_string()), } + } else { + panic!("Expected Param Type") } - values } struct ModInfoBuilder<'a> { @@ -87,6 +102,113 @@ impl<'a> ModInfoBuilder<'a> { self.emit_only_builtin(field, content); self.emit_only_loadable(field, content); } + + fn emit_param(&mut self, field: &str, param: &str, content: &str) { + let content = format!("{param}:{content}", param = param, content = content); + self.emit(field, &content); + } +} + +fn permissions_are_readonly(perms: &str) -> bool { + let (radix, digits) = if let Some(n) = perms.strip_prefix("0x") { + (16, n) + } else if let Some(n) = perms.strip_prefix("0o") { + (8, n) + } else if let Some(n) = perms.strip_prefix("0b") { + (2, n) + } else { + (10, perms) + }; + match u32::from_str_radix(digits, radix) { + Ok(perms) => perms & 0o222 == 0, + Err(_) => false, + } +} + +fn param_ops_path(param_type: &str) -> &'static str { + match param_type { + "bool" => "kernel::module_param::PARAM_OPS_BOOL", + "i8" => "kernel::module_param::PARAM_OPS_I8", + "u8" => "kernel::module_param::PARAM_OPS_U8", + "i16" => "kernel::module_param::PARAM_OPS_I16", + "u16" => "kernel::module_param::PARAM_OPS_U16", + "i32" => "kernel::module_param::PARAM_OPS_I32", + "u32" => "kernel::module_param::PARAM_OPS_U32", + "i64" => "kernel::module_param::PARAM_OPS_I64", + "u64" => "kernel::module_param::PARAM_OPS_U64", + "isize" => "kernel::module_param::PARAM_OPS_ISIZE", + "usize" => "kernel::module_param::PARAM_OPS_USIZE", + "str" => "kernel::module_param::PARAM_OPS_STR", + t => panic!("Unrecognized type {}", t), + } +} + +#[allow(clippy::type_complexity)] +fn try_simple_param_val( + param_type: &str, +) -> Box Option> { + match param_type { + "bool" => Box::new(try_ident), + "str" => Box::new(|param_it| { + try_byte_string(param_it) + .map(|s| format!("kernel::module_param::StringParam::Ref(b\"{}\")", s)) + }), + _ => Box::new(try_literal), + } +} + +fn get_default(param_type: &ParamType, param_it: &mut token_stream::IntoIter) -> String { + let try_param_val = match param_type { + ParamType::Ident(ref param_type) + | ParamType::Array { + vals: ref param_type, + max_length: _, + } => try_simple_param_val(param_type), + }; + assert_eq!(expect_ident(param_it), "default"); + assert_eq!(expect_punct(param_it), ':'); + let default = match param_type { + ParamType::Ident(_) => try_param_val(param_it).expect("Expected default param value"), + ParamType::Array { + vals: _, + max_length: _, + } => { + let group = expect_group(param_it); + assert_eq!(group.delimiter(), Delimiter::Bracket); + let mut default_vals = Vec::new(); + let mut it = group.stream().into_iter(); + + while let Some(default_val) = try_param_val(&mut it) { + default_vals.push(default_val); + match it.next() { + Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','), + None => break, + _ => panic!("Expected ',' or end of array default values"), + } + } + + let mut default_array = "kernel::module_param::ArrayParam::create(&[".to_string(); + default_array.push_str( + &default_vals + .iter() + .map(|val| val.to_string()) + .collect::>() + .join(","), + ); + default_array.push_str("])"); + default_array + } + }; + assert_eq!(expect_punct(param_it), ','); + default +} + +fn generated_array_ops_name(vals: &str, max_length: usize) -> String { + format!( + "__generated_array_ops_{vals}_{max_length}", + vals = vals, + max_length = max_length + ) } #[derive(Debug, Default)] @@ -96,15 +218,24 @@ struct ModuleInfo { name: String, author: Option, description: Option, - alias: Option>, + alias: Option, + params: Option, } impl ModuleInfo { fn parse(it: &mut token_stream::IntoIter) -> Self { let mut info = ModuleInfo::default(); - const EXPECTED_KEYS: &[&str] = - &["type", "name", "author", "description", "license", "alias"]; + const EXPECTED_KEYS: &[&str] = &[ + "type", + "name", + "author", + "description", + "license", + "alias", + "alias_rtnl_link", + "params", + ]; const REQUIRED_KEYS: &[&str] = &["type", "name", "license"]; let mut seen_keys = Vec::new(); @@ -130,7 +261,11 @@ impl ModuleInfo { "author" => info.author = Some(expect_string(it)), "description" => info.description = Some(expect_string(it)), "license" => info.license = expect_string_ascii(it), - "alias" => info.alias = Some(expect_string_array(it)), + "alias" => info.alias = Some(expect_string_ascii(it)), + "alias_rtnl_link" => { + info.alias = Some(format!("rtnl-link-{}", expect_string_ascii(it))) + } + "params" => info.params = Some(expect_group(it)), _ => panic!( "Unknown key \"{}\". Valid keys are: {:?}.", key, EXPECTED_KEYS @@ -181,10 +316,8 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { modinfo.emit("description", &description); } modinfo.emit("license", &info.license); - if let Some(aliases) = info.alias { - for alias in aliases { - modinfo.emit("alias", &alias); - } + if let Some(alias) = info.alias { + modinfo.emit("alias", &alias); } // Built-in modules also export the `file` modinfo string. @@ -192,6 +325,195 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable"); modinfo.emit_only_builtin("file", &file); + let mut array_types_to_generate = Vec::new(); + if let Some(params) = info.params { + assert_eq!(params.delimiter(), Delimiter::Brace); + + let mut it = params.stream().into_iter(); + + loop { + let param_name = match it.next() { + Some(TokenTree::Ident(ident)) => ident.to_string(), + Some(_) => panic!("Expected Ident or end"), + None => break, + }; + + assert_eq!(expect_punct(&mut it), ':'); + let param_type = expect_type(&mut it); + let group = expect_group(&mut it); + assert_eq!(expect_punct(&mut it), ','); + + assert_eq!(group.delimiter(), Delimiter::Brace); + + let mut param_it = group.stream().into_iter(); + let param_default = get_default(¶m_type, &mut param_it); + let param_permissions = get_literal(&mut param_it, "permissions"); + let param_description = get_string(&mut param_it, "description"); + expect_end(&mut param_it); + + // TODO: More primitive types. + // TODO: Other kinds: unsafes, etc. + let (param_kernel_type, ops): (String, _) = match param_type { + ParamType::Ident(ref param_type) => ( + param_type.to_string(), + param_ops_path(param_type).to_string(), + ), + ParamType::Array { + ref vals, + max_length, + } => { + array_types_to_generate.push((vals.clone(), max_length)); + ( + format!("__rust_array_param_{}_{}", vals, max_length), + generated_array_ops_name(vals, max_length), + ) + } + }; + + modinfo.emit_param("parmtype", ¶m_name, ¶m_kernel_type); + modinfo.emit_param("parm", ¶m_name, ¶m_description); + let param_type_internal = match param_type { + ParamType::Ident(ref param_type) => match param_type.as_ref() { + "str" => "kernel::module_param::StringParam".to_string(), + other => other.to_string(), + }, + ParamType::Array { + ref vals, + max_length, + } => format!( + "kernel::module_param::ArrayParam<{vals}, {max_length}>", + vals = vals, + max_length = max_length + ), + }; + let read_func = if permissions_are_readonly(¶m_permissions) { + format!( + " + fn read(&self) + -> &<{param_type_internal} as kernel::module_param::ModuleParam>::Value {{ + // SAFETY: Parameters do not need to be locked because they are + // read only or sysfs is not enabled. + unsafe {{ + <{param_type_internal} as kernel::module_param::ModuleParam>::value( + &__{name}_{param_name}_value + ) + }} + }} + ", + name = info.name, + param_name = param_name, + param_type_internal = param_type_internal, + ) + } else { + format!( + " + fn read<'lck>(&self, lock: &'lck kernel::KParamGuard) + -> &'lck <{param_type_internal} as kernel::module_param::ModuleParam>::Value {{ + // SAFETY: Parameters are locked by `KParamGuard`. + unsafe {{ + <{param_type_internal} as kernel::module_param::ModuleParam>::value( + &__{name}_{param_name}_value + ) + }} + }} + ", + name = info.name, + param_name = param_name, + param_type_internal = param_type_internal, + ) + }; + let kparam = format!( + " + kernel::bindings::kernel_param__bindgen_ty_1 {{ + arg: unsafe {{ &__{name}_{param_name}_value }} + as *const _ as *mut core::ffi::c_void, + }}, + ", + name = info.name, + param_name = param_name, + ); + write!( + modinfo.buffer, + " + static mut __{name}_{param_name}_value: {param_type_internal} = {param_default}; + + struct __{name}_{param_name}; + + impl __{name}_{param_name} {{ {read_func} }} + + const {param_name}: __{name}_{param_name} = __{name}_{param_name}; + + // Note: the C macro that generates the static structs for the `__param` section + // asks for them to be `aligned(sizeof(void *))`. However, that was put in place + // in 2003 in commit 38d5b085d2a0 (\"[PATCH] Fix over-alignment problem on x86-64\") + // to undo GCC over-alignment of static structs of >32 bytes. It seems that is + // not the case anymore, so we simplify to a transparent representation here + // in the expectation that it is not needed anymore. + // TODO: Revisit this to confirm the above comment and remove it if it happened. + #[repr(transparent)] + struct __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param); + + unsafe impl Sync for __{name}_{param_name}_RacyKernelParam {{ + }} + + #[cfg(not(MODULE))] + const __{name}_{param_name}_name: *const core::ffi::c_char = + b\"{name}.{param_name}\\0\" as *const _ as *const core::ffi::c_char; + + #[cfg(MODULE)] + const __{name}_{param_name}_name: *const core::ffi::c_char = + b\"{param_name}\\0\" as *const _ as *const core::ffi::c_char; + + #[link_section = \"__param\"] + #[used] + static __{name}_{param_name}_struct: __{name}_{param_name}_RacyKernelParam = + __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param {{ + name: __{name}_{param_name}_name, + // SAFETY: `__this_module` is constructed by the kernel at load time + // and will not be freed until the module is unloaded. + #[cfg(MODULE)] + mod_: unsafe {{ &kernel::bindings::__this_module as *const _ as *mut _ }}, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: unsafe {{ &{ops} }} as *const kernel::bindings::kernel_param_ops, + perm: {permissions}, + level: -1, + flags: 0, + __bindgen_anon_1: {kparam} + }}); + ", + name = info.name, + param_type_internal = param_type_internal, + read_func = read_func, + param_default = param_default, + param_name = param_name, + ops = ops, + permissions = param_permissions, + kparam = kparam, + ) + .unwrap(); + } + } + + let mut generated_array_types = String::new(); + + for (vals, max_length) in array_types_to_generate { + let ops_name = generated_array_ops_name(&vals, max_length); + write!( + generated_array_types, + " + kernel::make_param_ops!( + {ops_name}, + kernel::module_param::ArrayParam<{vals}, {{ {max_length} }}> + ); + ", + ops_name = ops_name, + vals = vals, + max_length = max_length, + ) + .unwrap(); + } + format!( " /// The module name. @@ -291,12 +613,44 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { }} {modinfo} + + {generated_array_types} ", type_ = info.type_, name = info.name, modinfo = modinfo.buffer, + generated_array_types = generated_array_types, initcall_section = ".initcall6.init" ) .parse() .expect("Error parsing formatted string into token stream.") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_permissions_are_readonly() { + assert!(permissions_are_readonly("0b000000000")); + assert!(permissions_are_readonly("0o000")); + assert!(permissions_are_readonly("000")); + assert!(permissions_are_readonly("0x000")); + + assert!(!permissions_are_readonly("0b111111111")); + assert!(!permissions_are_readonly("0o777")); + assert!(!permissions_are_readonly("511")); + assert!(!permissions_are_readonly("0x1ff")); + + assert!(permissions_are_readonly("0o014")); + assert!(permissions_are_readonly("0o015")); + + assert!(!permissions_are_readonly("0o214")); + assert!(!permissions_are_readonly("0o024")); + assert!(!permissions_are_readonly("0o012")); + + assert!(!permissions_are_readonly("0o315")); + assert!(!permissions_are_readonly("0o065")); + assert!(!permissions_are_readonly("0o017")); + } +} From patchwork Wed May 3 09:07:03 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229898 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id BA381C7EE26 for ; Wed, 3 May 2023 09:07:42 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229972AbjECJHj (ORCPT ); Wed, 3 May 2023 05:07:39 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45426 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229748AbjECJH3 (ORCPT ); Wed, 3 May 2023 05:07:29 -0400 Received: from mail-wm1-x336.google.com (mail-wm1-x336.google.com [IPv6:2a00:1450:4864:20::336]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E226840FD for ; Wed, 3 May 2023 02:07:23 -0700 (PDT) Received: by mail-wm1-x336.google.com with SMTP id 5b1f17b1804b1-3f19afc4fbfso48655195e9.2 for ; Wed, 03 May 2023 02:07:23 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104842; x=1685696842; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=5cH5GZPThZX1LLdPnjlMEvQzzaSHQbdfYQa3aafCtFw=; b=Hgz+JsybFVjncw15HasJvie2Dl4VadDcEYJtGAk4kb/RQguZRxQO+WUSHbBzCchQfK 4fR/4HHzFj7feyZ/Ey1XOlfTNiuid/oK5BCfCTwEDhAS24mdDiPOWkT4Xnon05qvrKsI 9DROCY4yYimdZfsorqum1Kgn1FKSyFiupk9F01cUByEOnAhbB/Zk8PwTZ8IQQh3Ysld8 EQ/BMmUi7pFY1YFHuqo9s0suNHftuZROm2snpKuc0maQI6BsY8e6/z20fLheOG/FOaSj ck2bzsszlaJfYe3wOeCcoVvxNqXt9pxyYKS7bIBfx+nUkWsDba1tg8Zs6sA0UfeBQnJ9 LT3w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104842; x=1685696842; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=5cH5GZPThZX1LLdPnjlMEvQzzaSHQbdfYQa3aafCtFw=; b=BDvjaTRQ5mosclalkOiVQF9J/64rVgw0/hp2J6wb9cX4eWQ81S98gG2ZOATPEk0KmS 21chYIfWMTZDdG1anw8APKJdEyF6SdyrSvrbzjL9kLpeayTg94N0Ofa8VI6vEAp1REf6 UkTngsHgI4OZlusnvdW9nQfjcu8+ubSQ5Rxs6GrXCts5mTAdzs7hqPCjELwm/uX9Boh4 hxq8PAtPs6BTtjkQcn3Keq3Tges3DJP2OUKyBlGvvIPtQWRoI/AsTL8GzfM2pB/YBddI wyA8MLPYtlUFM6acw9eM3rt8SF/0gJnDaSy2w9pyfr9CEte9FH+QaWRhUfq3/3jPcjs2 T4Ug== X-Gm-Message-State: AC+VfDwA3lSFzMbeZQ4OIHxD4pEkDOSHnuZXsOcxGHvHCQkmdLOA4KiA yMZHYEaUstMtcb8GFwcTn4u0BQ== X-Google-Smtp-Source: ACHHUZ4EBxDh+2FdUjZNnlPIs7Z43merWOWOdSZYBzuMbtFHhqsjGSgkNu/agPTqqGu/gJ3ZmsuWWw== X-Received: by 2002:a7b:cb85:0:b0:3f1:7136:dd45 with SMTP id m5-20020a7bcb85000000b003f17136dd45mr14187268wmi.30.1683104842308; Wed, 03 May 2023 02:07:22 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id n15-20020a1c720f000000b003f339b2d06fsm1271659wmc.4.2023.05.03.02.07.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:22 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 06/11] rust: apply cache line padding for `SpinLock` Date: Wed, 3 May 2023 11:07:03 +0200 Message-Id: <20230503090708.2524310-7-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg The kernel `struct spinlock` is 4 bytes on x86 when lockdep is not enabled. The structure is not padded to fit a cache line. The effect of this for `SpinLock` is that the lock variable and the value protected by the lock will share a cache line, depending on the alignment requirements of the protected value. Aligning the lock variable and the protected value to a cache line yields a 20% performance increase for the Rust null block driver for sequential reads to memory backed devices at 6 concurrent readers. Signed-off-by: Andreas Hindborg --- rust/kernel/cache_padded.rs | 33 +++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 ++ rust/kernel/sync/lock.rs | 9 ++++++--- rust/kernel/sync/lock/spinlock.rs | 13 ++++++++---- 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 rust/kernel/cache_padded.rs diff --git a/rust/kernel/cache_padded.rs b/rust/kernel/cache_padded.rs new file mode 100644 index 000000000000..758678e71f50 --- /dev/null +++ b/rust/kernel/cache_padded.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 + +#[repr(align(64))] +pub struct CachePadded { + value: T, +} + +unsafe impl Send for CachePadded {} +unsafe impl Sync for CachePadded {} + +impl CachePadded { + /// Pads and aligns a value to 64 bytes. + #[inline(always)] + pub(crate) const fn new(t: T) -> CachePadded { + CachePadded:: { value: t } + } +} + +impl core::ops::Deref for CachePadded { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &self.value + } +} + +impl core::ops::DerefMut for CachePadded { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a0bd0b0e2aef..426e2dea0da6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -37,6 +37,7 @@ extern crate self as kernel; mod allocator; pub mod block; mod build_assert; +mod cache_padded; pub mod error; pub mod init; pub mod ioctl; @@ -56,6 +57,7 @@ pub mod types; #[doc(hidden)] pub use bindings; +pub(crate) use cache_padded::CachePadded; pub use macros; pub use uapi; diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index a2216325632d..1c584b1df30d 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -6,7 +6,9 @@ //! spinlocks, raw spinlocks) to be provided with minimal effort. use super::LockClassKey; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; +use crate::{ + bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard, CachePadded, +}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; @@ -87,7 +89,7 @@ pub struct Lock { _pin: PhantomPinned, /// The data protected by the lock. - pub(crate) data: UnsafeCell, + pub(crate) data: CachePadded>, } // SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can. @@ -102,7 +104,7 @@ impl Lock { #[allow(clippy::new_ret_no_self)] pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit { pin_init!(Self { - data: UnsafeCell::new(t), + data: CachePadded::new(UnsafeCell::new(t)), _pin: PhantomPinned, // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have // static lifetimes so they live indefinitely. @@ -115,6 +117,7 @@ impl Lock { impl Lock { /// Acquires the lock and gives the caller access to the data protected by it. + #[inline(always)] pub fn lock(&self) -> Guard<'_, T, B> { // SAFETY: The constructor of the type calls `init`, so the existence of the object proves // that `init` was called. diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 979b56464a4e..e39142a8148c 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,7 +4,10 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use core::ops::DerefMut; + use crate::bindings; +use crate::CachePadded; /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// @@ -90,7 +93,7 @@ pub struct SpinLockBackend; // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. unsafe impl super::Backend for SpinLockBackend { - type State = bindings::spinlock_t; + type State = CachePadded; type GuardState = (); unsafe fn init( @@ -100,18 +103,20 @@ unsafe impl super::Backend for SpinLockBackend { ) { // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and // `key` are valid for read indefinitely. - unsafe { bindings::__spin_lock_init(ptr, name, key) } + unsafe { bindings::__spin_lock_init((&mut *ptr).deref_mut(), name, key) } } + #[inline(always)] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `ptr` points to valid // memory, and that it has been initialised before. - unsafe { bindings::spin_lock(ptr) } + unsafe { bindings::spin_lock((&mut *ptr).deref_mut()) } } + #[inline(always)] unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the // caller is the owner of the mutex. - unsafe { bindings::spin_unlock(ptr) } + unsafe { bindings::spin_unlock((&mut *ptr).deref_mut()) } } } From patchwork Wed May 3 09:07:04 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229899 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0B058C77B7F for ; Wed, 3 May 2023 09:07:44 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229988AbjECJHl (ORCPT ); Wed, 3 May 2023 05:07:41 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45444 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229650AbjECJHa (ORCPT ); Wed, 3 May 2023 05:07:30 -0400 Received: from mail-wr1-x42a.google.com (mail-wr1-x42a.google.com [IPv6:2a00:1450:4864:20::42a]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C4026420B for ; Wed, 3 May 2023 02:07:24 -0700 (PDT) Received: by mail-wr1-x42a.google.com with SMTP id ffacd0b85a97d-2f6401ce8f8so3031804f8f.3 for ; Wed, 03 May 2023 02:07:24 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104843; x=1685696843; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=BtKpWEX99/tXFIUB1b2EGEDNutHLi8kX/M6s4oIe5M4=; b=CxSTo+18rMVa2ypTkZTy0wLUDTZw1QYq8jpoxdVhk/p84nZH5UHbiZq1swvBWxc3Xl r2kO5WRrEPvmzDQeUiP6rVG+UuPHdE+apzClMIY04+UBt5rRJASMvfHzn8yHzRZItfCa eICcSstADFeX/SdIHIB5B8+VFI4XL8BgekNIxDTO4LdHoRLMZqNr73unzM6ADYmaIyeq tJ6dsNUjxn+lkY8wiCDlibjPjlC8Y01rsjloOfV2tBlRm8G3dYPzQyejyyH9hyMJxKY2 Ga0zGhnlQF0t9qD5/DaEQMXU1PD13S+IC5523kVFJfpei0LQl8U9b9kNdjDOMCoLgVnw vA1A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104843; x=1685696843; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=BtKpWEX99/tXFIUB1b2EGEDNutHLi8kX/M6s4oIe5M4=; b=ZqKMLuTKYdKBGpGOkE5/KAWDKBFN9SdbyCjwETKgrV1yUtELsFoEOQv7eSTw9T3ZzN 4PFqlu15XAbSjSS+Pm11WanWUp6EiDR5B1nn+sG8sJZaJbof5iuTbDqeew4zYqFwcA4q Z0rKryKYfetl2YUOWkuJBmqr9hjobEWYFa+rtB/LRI34AxYiw5TgszNDEE5+stZdWTuk XcUU7RZ5g/8fWdA/T0c6LM3jCqdDhGg50EvdaJ7IlutQq/Ld67GYMJWnJbNGMfy81JO8 vj/xztudIZbhSEx6LBaLkTaq31Iw5thY5iD1thTS9v4IQ1EVXmRoeB325M1rijMVA4fG 1Iuw== X-Gm-Message-State: AC+VfDySPDe8MaLLCMCfdJER5zopFFJ0vRgg6hquFMt9M4r8+KSb4y7L b8o7plUX6vUVjLxxJgJLqJT0DQ== X-Google-Smtp-Source: ACHHUZ7KbXS+IwQVsR80XNjrwtE1ZUCnYctnByWWLdgm7ZrK387pgkawXMC7JKEwhn48tufrkT8hcQ== X-Received: by 2002:a5d:4577:0:b0:306:2b5a:d8db with SMTP id a23-20020a5d4577000000b003062b5ad8dbmr6680255wrc.23.1683104843273; Wed, 03 May 2023 02:07:23 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id x9-20020a5d60c9000000b002fbdb797483sm32900298wrt.49.2023.05.03.02.07.22 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:23 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Wedson Almeida Filho , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 07/11] rust: lock: add support for `Lock::lock_irqsave` Date: Wed, 3 May 2023 11:07:04 +0200 Message-Id: <20230503090708.2524310-8-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Wedson Almeida Filho This allows locks like spinlocks and raw spinlocks to expose a `lock_irqsave` variant in Rust that corresponds to the C version. Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho --- rust/kernel/sync/lock.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 1c584b1df30d..bb21af8a8377 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -72,6 +72,44 @@ pub unsafe trait Backend { } } +/// The "backend" of a lock that supports the irq-save variant. +/// +/// # Safety +/// +/// The same requirements wrt mutual exclusion in [`Backend`] apply for acquiring the lock via +/// [`IrqSaveBackend::lock_irqsave`]. +/// +/// Additionally, when [`IrqSaveBackend::lock_irqsave`] is used to acquire the lock, implementers +/// must disable interrupts on lock, and restore interrupt state on unlock. Implementers may use +/// [`Backend::GuardState`] to store state needed to keep track of the interrupt state. +pub unsafe trait IrqSaveBackend: Backend { + /// Acquires the lock, making the caller its owner. + /// + /// Before acquiring the lock, it disables interrupts, and returns the previous interrupt state + /// as its guard state so that the guard can restore it when it is dropped. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + #[must_use] + unsafe fn lock_irqsave(ptr: *mut Self::State) -> Self::GuardState; +} + +impl Lock { + /// Acquires the lock and gives the caller access to the data protected by it. + /// + /// Before acquiring the lock, it disables interrupts. When the guard is dropped, the interrupt + /// state (either enabled or disabled) is restored to its state before + /// [`lock_irqsave`](Self::lock_irqsave) was called. + pub fn lock_irqsave(&self) -> Guard<'_, T, B> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + let state = unsafe { B::lock_irqsave(self.state.get()) }; + // SAFETY: The lock was just acquired. + unsafe { Guard::new(self, state) } + } +} + /// A mutual exclusion primitive. /// /// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock backend From patchwork Wed May 3 09:07:05 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229901 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A66EDC7EE26 for ; Wed, 3 May 2023 09:07:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230020AbjECJHy (ORCPT ); Wed, 3 May 2023 05:07:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45888 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229760AbjECJHj (ORCPT ); Wed, 3 May 2023 05:07:39 -0400 Received: from mail-wr1-x429.google.com (mail-wr1-x429.google.com [IPv6:2a00:1450:4864:20::429]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 74B694691 for ; Wed, 3 May 2023 02:07:26 -0700 (PDT) Received: by mail-wr1-x429.google.com with SMTP id ffacd0b85a97d-2f40b891420so4700307f8f.0 for ; Wed, 03 May 2023 02:07:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104844; x=1685696844; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=pfKdjWdresPJSdXc5oUQnKkLhNP+u1b3Z6UA6/bvZEs=; b=zxf3V0pixeyAdrD64XjVTcoz5GBZP1Ez2NPEYfg4bJ+8OpHIgbs5IPYWoyRFF/oRls 1HiTMTqiw5oPzKcjXk85WtG2qWppkjvU02IYtCNv/P6lilDjZ3I/tCqie+hxdHS9fI72 TFUlh6Wb66UftjAJVkN8zXebQeRzUQEjndgJGChQSPHdwiL4EDhLk2wZR45z/mLS3qsS 0hle1vi+M/A5q3ANxKiVAd1H1NxXbl8yRYYvbayFD9yVys1oohANbByOMWC+mJ8fu1Aq h3If9GnckODucwV34nms5ONfQlshxGTXJ2YdkMkaVW/TPTSFD4UmrIVnUA6QvhXfqCL+ 1Ffw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104844; x=1685696844; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=pfKdjWdresPJSdXc5oUQnKkLhNP+u1b3Z6UA6/bvZEs=; b=H/x339Y7dYADKogt7iVamUu+ORIrKsQRuOTj30mkf2AHlYpmng3aEz6iNQ/2afHcGm ZN1qWhibMCFVic+ZV9ycdswJAjDkpnkC9IfdA6IK3vOzXM7FTeMFEsPcdrbzXMPd5G80 kRPQDTyBKZ9D7hfjkWqj4H52kw0QgpvjEuYoRX0ePy1pBLxwN4Z/fFMvc+tMUdVM1WYH MdoH178mbO7EBgdm65ahqsJ2BJxzaJknKAZTxllgPt8rAZiMVYdZOOMIkN8gGsqNAm2K NX/yQTBjTO2RD89OJIqKTx2aktzOToZsbLJIFdEd4NR3ytV9ca98E2iMwpf6+Zhnr5f6 k7Yg== X-Gm-Message-State: AC+VfDxYagOBKVFTQDxZjuxEvIN94abTLk/lWRqi/GRs7R9SJuVPoCl0 zEB7tVwIG4vqZtm2Ygw4u1A38A== X-Google-Smtp-Source: ACHHUZ5UM0xcnar/k/FZ+nJlonCpodkAJIUEXzZR9u3QQyFGMIWvcy7zXUt0eX7nCk//Wv22bv3Duw== X-Received: by 2002:adf:ee8d:0:b0:306:42e2:5ec3 with SMTP id b13-20020adfee8d000000b0030642e25ec3mr912726wro.6.1683104844271; Wed, 03 May 2023 02:07:24 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id k11-20020a7bc40b000000b003f173a2b2f6sm1248839wmi.12.2023.05.03.02.07.23 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:24 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Wedson Almeida Filho , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 08/11] rust: lock: implement `IrqSaveBackend` for `SpinLock` Date: Wed, 3 May 2023 11:07:05 +0200 Message-Id: <20230503090708.2524310-9-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Wedson Almeida Filho This allows Rust code to use the `lock_irqsave` variant of spinlocks. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Waiman Long Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho --- rust/helpers.c | 16 ++++++++++++ rust/kernel/sync/lock/spinlock.rs | 41 +++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/rust/helpers.c b/rust/helpers.c index a59341084774..7f5b95f652e1 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -203,6 +203,22 @@ struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) } EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); +unsigned long rust_helper_spin_lock_irqsave(spinlock_t *lock) +{ + unsigned long flags; + + spin_lock_irqsave(lock, flags); + + return flags; +} +EXPORT_SYMBOL_GPL(rust_helper_spin_lock_irqsave); + +void rust_helper_spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) +{ + spin_unlock_irqrestore(lock, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_unlock_irqrestore); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index e39142a8148c..50b8775bb49d 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -64,6 +64,8 @@ macro_rules! new_spinlock { /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); +/// assert_eq!(e.d.lock_irqsave().a, 20); +/// assert_eq!(e.d.lock_irqsave().b, 30); /// ``` /// /// The following example shows how to use interior mutability to modify the contents of a struct @@ -81,6 +83,12 @@ macro_rules! new_spinlock { /// let mut guard = m.lock(); /// guard.a += 10; /// guard.b += 20; +/// +/// fn example2(m: &SpinLock) { +/// let mut guard = m.lock_irqsave(); +/// guard.a += 10; +/// guard.b += 20; +/// } /// } /// ``` /// @@ -94,7 +102,7 @@ pub struct SpinLockBackend; // default implementation that always calls the same locking method. unsafe impl super::Backend for SpinLockBackend { type State = CachePadded; - type GuardState = (); + type GuardState = Option; unsafe fn init( ptr: *mut Self::State, @@ -110,13 +118,32 @@ unsafe impl super::Backend for SpinLockBackend { unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `ptr` points to valid // memory, and that it has been initialised before. - unsafe { bindings::spin_lock((&mut *ptr).deref_mut()) } + unsafe { bindings::spin_lock((&mut *ptr).deref_mut()) }; + None } - #[inline(always)] - unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { - // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the - // caller is the owner of the mutex. - unsafe { bindings::spin_unlock((&mut *ptr).deref_mut()) } + unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState) { + match guard_state { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that + // the caller is the owner of the mutex. + Some(flags) => unsafe { + bindings::spin_unlock_irqrestore((&mut *ptr).deref_mut(), *flags) + }, + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that + // the caller is the owner of the mutex. + None => unsafe { bindings::spin_unlock((&mut *ptr).deref_mut()) }, + } + } +} + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. We use the `irqsave` +// variant of the C lock acquisition functions to disable interrupts and retrieve the original +// interrupt state, and the `irqrestore` variant of the lock release functions to restore the state +// in `unlock` -- we use the guard context to determine which method was used to acquire the lock. +unsafe impl super::IrqSaveBackend for SpinLockBackend { + unsafe fn lock_irqsave(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + Some(unsafe { bindings::spin_lock_irqsave((&mut *ptr).deref_mut()) }) } } From patchwork Wed May 3 09:07:06 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229902 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 92A02C77B7F for ; Wed, 3 May 2023 09:08:02 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230024AbjECJIA (ORCPT ); Wed, 3 May 2023 05:08:00 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45818 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229694AbjECJHj (ORCPT ); Wed, 3 May 2023 05:07:39 -0400 Received: from mail-wm1-x331.google.com (mail-wm1-x331.google.com [IPv6:2a00:1450:4864:20::331]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 162E34EE0 for ; Wed, 3 May 2023 02:07:27 -0700 (PDT) Received: by mail-wm1-x331.google.com with SMTP id 5b1f17b1804b1-3f4000ec74aso1039245e9.3 for ; Wed, 03 May 2023 02:07:26 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104845; x=1685696845; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=n2aCenBwPL0i3vTawCb2L8YinJvZAslW/YJEjXhFLa8=; b=AkqTOFPVsQoS7ielM6GeSVWnFtz9yMESQff8kVfpSdZiy3bdaaWLpuG1t40bS0s7on TDgZIlo0iiLY/gG5xlQwmXkLkeWaeeMQho78fmcfwXr9vbm6f7BCJ5iI1WrR7q3Hq/B8 eriq0v/fAaOJFcSS6bHc2AuiUNCaQsbgtGUGWsqZWxzyvj4YwzKANe2QUZrEgauRlAi0 NXz0GQAAphVn7VtICawZK7usZ2YbQdSVM0vxmLmJsR+gx1YoAZPAl1wyrseet8M/HpaE 0YWtzJTugKmoNajsIAjcEDi5kVkiG0HUiPkLga9N2S03hYveh/RedYTApIvgbetEAu5S 8+zA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104845; x=1685696845; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=n2aCenBwPL0i3vTawCb2L8YinJvZAslW/YJEjXhFLa8=; b=NP1+46UXF9FZuCMkRjaj10UibowFB5b9OkFnLGr8UMg98v5Xjp+olg+Ic53GbR5vWx c958W6EYRA9i1C4gPtD3KGCDN1FqAY1kuijTacOuceAgCz3+WbZ3BWtK9id7a/aRvjy9 jLh+ZgqSbxZ6XRCZGy78YnVMds4BYDEwThZymG8HcUtpCjfdi9yvz8+/0QZpssyfDrAj uNtWDNij/lr9dPUUGr/IPBQlhAz0ws5fZfjaC2ygdRugkuQaccdZTyulZhwSNKbLplkW kzTp25KtxFQpLHQ2AAx1VXwXt1NOtsUMqAS4BhewO9+Y+0v6zRaQO9YR/Gd7zwwpRfx/ oPgA== X-Gm-Message-State: AC+VfDymQ0l9fBA+alANGGe0+zwBmATZjOZ2iBYSv3ZzzWJgXC/6jY56 jZDba8w9xTLOKJKo7hsteLYC3Q== X-Google-Smtp-Source: ACHHUZ6TOXRdg3DPcncxdnXc/lCXM7ZZOYLzK+Nmqw1ltFhclc2DCjmvwo383Z/Dy4ERb34WRGJ4yg== X-Received: by 2002:a1c:7916:0:b0:3f1:952c:3c70 with SMTP id l22-20020a1c7916000000b003f1952c3c70mr13719294wme.40.1683104845321; Wed, 03 May 2023 02:07:25 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id g10-20020a7bc4ca000000b003f046ad52efsm1235624wmk.31.2023.05.03.02.07.24 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:25 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 09/11] RUST: implement `ForeignOwnable` for `Pin` Date: Wed, 3 May 2023 11:07:06 +0200 Message-Id: <20230503090708.2524310-10-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg Implement `ForeignOwnable for Pin where T: ForeignOwnable + Deref`. Imported from rust tree [1] [1] https://github.com/Rust-for-Linux/linux/tree/bc22545f38d74473cfef3e9fd65432733435b79f Cc: Wedson Almeida Filho --- rust/kernel/types.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 29db59d6119a..98e71e96a7fc 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -9,6 +9,7 @@ use core::{ marker::PhantomData, mem::MaybeUninit, ops::{Deref, DerefMut}, + pin::Pin, ptr::NonNull, }; @@ -100,6 +101,29 @@ impl ForeignOwnable for () { unsafe fn from_foreign(_: *const core::ffi::c_void) -> Self {} } +impl ForeignOwnable for Pin { + type Borrowed<'a> = T::Borrowed<'a>; + + fn into_foreign(self) -> *const core::ffi::c_void { + // SAFETY: We continue to treat the pointer as pinned by returning just a pointer to it to + // the caller. + let inner = unsafe { Pin::into_inner_unchecked(self) }; + inner.into_foreign() + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Self::Borrowed<'a> { + // SAFETY: The safety requirements for this function are the same as the ones for + // `T::borrow`. + unsafe { T::borrow(ptr) } + } + + unsafe fn from_foreign(p: *const core::ffi::c_void) -> Self { + // SAFETY: The object was originally pinned. + // The passed pointer comes from a previous call to `T::into_foreign`. + unsafe { Pin::new_unchecked(T::from_foreign(p)) } + } +} + /// Runs a cleanup function/closure when dropped. /// /// The [`ScopeGuard::dismiss`] function prevents the cleanup function from running. From patchwork Wed May 3 09:07:07 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229903 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0504BC7EE29 for ; Wed, 3 May 2023 09:08:11 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230038AbjECJII (ORCPT ); Wed, 3 May 2023 05:08:08 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45960 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229976AbjECJHk (ORCPT ); Wed, 3 May 2023 05:07:40 -0400 Received: from mail-wm1-x32c.google.com (mail-wm1-x32c.google.com [IPv6:2a00:1450:4864:20::32c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D8B594EF5 for ; Wed, 3 May 2023 02:07:28 -0700 (PDT) Received: by mail-wm1-x32c.google.com with SMTP id 5b1f17b1804b1-3f19ab99540so47467925e9.2 for ; Wed, 03 May 2023 02:07:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104847; x=1685696847; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=NatBW5eIyTK1anrfiMCu2himhC60W7yfvOzay2vGAv8=; b=p4BdIUQKmjerjPm5oD73Ay9h/299qjSSXcgv4Bb+Qmp2g4owzjxFGY/dLkga6OOGno Nr2jHy7hW1KNXt2jFPKbJW4XgnrPwqb0NM4WY1MM+UiU7g1J3uHlVIYp9v2p258ltsrF t7OVqRC2Ti6RK+iAoEbzM+tW8O5ETUmXvIXQ0Nxl6UW1vz8hYm/rqZHDMKt3tb7YvkKb iSqk4PNYf+RSBTYroInYiqq9Vr6T8hmJUsoBUh2dCy7hB3MQazTr8tkOvGxRyX7V7C39 56UYzcYTSvClGccgLv6Ha/V3GAC/brvbNBumsb2Up+/YyAQ1fBFju2Cj4nuQOlnNUGET azOg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104847; x=1685696847; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=NatBW5eIyTK1anrfiMCu2himhC60W7yfvOzay2vGAv8=; b=AreTFDCG/Vs06OSDnaWO4/r3CR+HEavxEwgcxa16lADeRDhJs8pNiwelqLzClLDMn8 BVUSFS5vcudEwBpDGVkVumhrYbKxeTtygh+5apri8+41cJmFlYoIe/PJ2/6y9L6L+Wvj s/appalCRAYLEVvPGRnz4NqvQj3Ix/R8NnAtwqPYYWvJEGbE1DlbKyOQuaaRL59qrdiR FEZE1lQjYhkQGcYb8Cxo23eO2yVJTfy1uT+negixnbCZDh/7eG5URJqVCheogzI/n5PG hpV2D9UP77JJ3hkZngdyRGLv+OKWoIltuEKXy25XuBtZF6Nef7SCKdV48xLN9c/oud2F T4OQ== X-Gm-Message-State: AC+VfDw2wkIzofl3ghMoEtstDOIPh1XBJO43uF9KWUNkK7cNFNMbDmRW kM5VRmNKsVmeLDYmwGD9fT3pNA== X-Google-Smtp-Source: ACHHUZ4V+4yXzD+7jMqjGJI2nu+9smDJfF7r7MsYfyR2VAHsFx3pXLD9Y8eoC2R23SPbbiOVCzgrKw== X-Received: by 2002:a1c:4b0c:0:b0:3ee:b3bf:5f7c with SMTP id y12-20020a1c4b0c000000b003eeb3bf5f7cmr14680443wma.23.1683104846390; Wed, 03 May 2023 02:07:26 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id x2-20020a05600c21c200b003f182c11667sm1233908wmj.39.2023.05.03.02.07.25 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:26 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 10/11] rust: add null block driver Date: Wed, 3 May 2023 11:07:07 +0200 Message-Id: <20230503090708.2524310-11-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg The driver is implemented entirely using safe Rust. This initial version supports direct completion, read and write requests, blk-mq and optional memory backing. Signed-off-by: Andreas Hindborg --- drivers/block/Kconfig | 4 + drivers/block/Makefile | 4 + drivers/block/rnull-helpers.c | 60 ++++++++++++ drivers/block/rnull.rs | 177 ++++++++++++++++++++++++++++++++++ scripts/Makefile.build | 2 +- 5 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 drivers/block/rnull-helpers.c create mode 100644 drivers/block/rnull.rs diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index f79f20430ef7..644ef1bc7574 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -354,6 +354,10 @@ config VIRTIO_BLK This is the virtual block driver for virtio. It can be used with QEMU based VMMs (like KVM or Xen). Say Y or M. +config BLK_DEV_RS_NULL + tristate "Rust null block driver" + depends on RUST + config BLK_DEV_RBD tristate "Rados block device (RBD)" depends on INET && BLOCK diff --git a/drivers/block/Makefile b/drivers/block/Makefile index 101612cba303..cebbeece4bc2 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -9,6 +9,10 @@ # needed for trace events ccflags-y += -I$(src) +obj-$(CONFIG_BLK_DEV_RS_NULL) += rnull_mod.o +rnull_mod-y := rnull-helpers.o rnull.o +LLVM_LINK_FIX_drivers/block/rnull_mod.o := 1 + obj-$(CONFIG_MAC_FLOPPY) += swim3.o obj-$(CONFIG_BLK_DEV_SWIM) += swim_mod.o obj-$(CONFIG_BLK_DEV_FD) += floppy.o diff --git a/drivers/block/rnull-helpers.c b/drivers/block/rnull-helpers.c new file mode 100644 index 000000000000..826f2773ed93 --- /dev/null +++ b/drivers/block/rnull-helpers.c @@ -0,0 +1,60 @@ + +#include + +__attribute__((always_inline)) +void rust_helper_bio_advance_iter_single(const struct bio *bio, + struct bvec_iter *iter, unsigned int bytes) +{ + bio_advance_iter_single(bio, iter, bytes); +} + +__attribute__((always_inline)) void *rust_helper_kmap(struct page *page) +{ + return kmap(page); +} + +__attribute__((always_inline)) void rust_helper_kunmap(struct page *page) +{ + return kunmap(page); +} + +__attribute__((always_inline)) void *rust_helper_kmap_atomic(struct page *page) +{ + return kmap_atomic(page); +} + +__attribute__((always_inline)) void rust_helper_kunmap_atomic(void* address) +{ + kunmap_atomic(address); +} + +__attribute__((always_inline)) struct page * +rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} + +__attribute__((always_inline)) void rust_helper_spin_lock_irq(spinlock_t *lock) +{ + spin_lock_irq(lock); +} + +__attribute__((always_inline)) void +rust_helper_spin_unlock_irq(spinlock_t *lock) +{ + spin_unlock_irq(lock); +} +__attribute__((always_inline)) unsigned long +rust_helper_spin_lock_irqsave(spinlock_t *lock) +{ + unsigned long flags; + + spin_lock_irqsave(lock, flags); + + return flags; +} +__attribute__((always_inline)) void +rust_helper_spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) +{ + spin_unlock_irqrestore(lock, flags); +} diff --git a/drivers/block/rnull.rs b/drivers/block/rnull.rs new file mode 100644 index 000000000000..d95025664a60 --- /dev/null +++ b/drivers/block/rnull.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! This is a null block driver. It currently supports optional memory backing, +//! blk-mq interface and direct completion. The driver is configured at module +//! load time by parameters `memory_backed` and `capacity_mib`. + +use kernel::{ + bindings, + block::{ + bio::Segment, + mq::{self, GenDisk, Operations, TagSet}, + }, + error::Result, + macros::vtable, + new_mutex, new_spinlock, + pages::Pages, + pr_info, + prelude::*, + radix_tree::RadixTree, + sync::{Arc, Mutex, SpinLock}, + types::ForeignOwnable, +}; + +module! { + type: NullBlkModule, + name: "rs_null_blk", + author: "Andreas Hindborg", + license: "GPL v2", + params: { + memory_backed: bool { + default: true, + permissions: 0, + description: "Use memory backing", + }, + capacity_mib: u64 { + default: 4096, + permissions: 0, + description: "Device capacity in MiB", + }, + }, +} + +struct NullBlkModule { + _disk: Pin>>>, +} + +fn add_disk(tagset: Arc>) -> Result> { + let tree = RadixTree::new()?; + let queue_data = Box::pin_init(new_spinlock!(tree, "rnullb:mem"))?; + + let disk = GenDisk::try_new(tagset, queue_data)?; + disk.set_name(format_args!("rnullb{}", 0))?; + disk.set_capacity(*capacity_mib.read() << 11); + disk.set_queue_logical_block_size(4096); + disk.set_queue_physical_block_size(4096); + disk.set_rotational(false); + Ok(disk) +} + +impl kernel::Module for NullBlkModule { + fn init(_module: &'static ThisModule) -> Result { + pr_info!("Rust null_blk loaded\n"); + // Major device number? + let tagset = TagSet::try_new(1, (), 256, 1)?; + let disk = Box::pin_init(new_mutex!(add_disk(tagset)?, "nullb:disk"))?; + + disk.lock().add()?; + + Ok(Self { _disk: disk }) + } +} + +impl Drop for NullBlkModule { + fn drop(&mut self) { + pr_info!("Dropping rnullb\n"); + } +} + +struct NullBlkDevice; +type Tree = kernel::radix_tree::RadixTree>>; +type Data = Pin>>; + +impl NullBlkDevice { + #[inline(always)] + fn write(tree: &mut Tree, sector: usize, segment: &Segment<'_>) -> Result { + let idx = sector >> 3; // TODO: PAGE_SECTOR_SHIFT + let mut page = if let Some(page) = tree.get_mut(idx as u64) { + page + } else { + tree.try_insert(idx as u64, Box::try_new(Pages::new()?)?)?; + tree.get_mut(idx as u64).unwrap() + }; + + segment.copy_to_page_atomic(&mut page)?; + + Ok(()) + } + + #[inline(always)] + fn read(tree: &mut Tree, sector: usize, segment: &mut Segment<'_>) -> Result { + let idx = sector >> 3; // TODO: PAGE_SECTOR_SHIFT + if let Some(page) = tree.get(idx as u64) { + segment.copy_from_page_atomic(page)?; + } + + Ok(()) + } + + #[inline(never)] + fn transfer( + command: bindings::req_op, + tree: &mut Tree, + sector: usize, + segment: &mut Segment<'_>, + ) -> Result { + match command { + bindings::req_op_REQ_OP_WRITE => Self::write(tree, sector, segment)?, + bindings::req_op_REQ_OP_READ => Self::read(tree, sector, segment)?, + _ => (), + } + Ok(()) + } +} + +#[vtable] +impl Operations for NullBlkDevice { + type RequestData = (); + type QueueData = Data; + type HwData = (); + type TagSetData = (); + + fn new_request_data( + _tagset_data: ::Borrowed<'_>, + ) -> Result { + Ok(()) + } + + #[inline(always)] + fn queue_rq( + _hw_data: ::Borrowed<'_>, + queue_data: ::Borrowed<'_>, + rq: &mq::Request, + _is_last: bool, + ) -> Result { + rq.start(); + if *memory_backed.read() { + let mut tree = queue_data.lock_irqsave(); + + let mut sector = rq.sector(); + for bio in rq.bio_iter() { + for mut segment in bio.segment_iter() { + let _ = Self::transfer(rq.command(), &mut tree, sector, &mut segment); + sector += segment.len() >> 9; // TODO: SECTOR_SHIFT + } + } + } + rq.end_ok(); + Ok(()) + } + + fn commit_rqs( + _hw_data: ::Borrowed<'_>, + _queue_data: ::Borrowed<'_>, + ) { + } + + fn complete(_rq: &mq::Request) { + //rq.end_ok(); + } + + fn init_hctx( + _tagset_data: ::Borrowed<'_>, + _hctx_idx: u32, + ) -> Result { + Ok(()) + } +} diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 9f94fc83f086..94127fc3cf77 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := core_ffi_c,explicit_generic_args_with_impl_trait,new_uninit,pin_macro +rust_allowed_features := core_ffi_c,explicit_generic_args_with_impl_trait,new_uninit,pin_macro,allocator_api rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ From patchwork Wed May 3 09:07:08 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andreas Hindborg X-Patchwork-Id: 13229904 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3DB78C77B7F for ; Wed, 3 May 2023 09:08:22 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230037AbjECJIU (ORCPT ); Wed, 3 May 2023 05:08:20 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45972 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229748AbjECJHk (ORCPT ); Wed, 3 May 2023 05:07:40 -0400 Received: from mail-wr1-x436.google.com (mail-wr1-x436.google.com [IPv6:2a00:1450:4864:20::436]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id EBC144ED6 for ; Wed, 3 May 2023 02:07:29 -0700 (PDT) Received: by mail-wr1-x436.google.com with SMTP id ffacd0b85a97d-304935cc79bso4848708f8f.2 for ; Wed, 03 May 2023 02:07:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=metaspace-dk.20221208.gappssmtp.com; s=20221208; t=1683104848; x=1685696848; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=oHrN32lBaAC78cOIMsVaenJPrzKJjknoApgieLS2EVM=; b=oA2PVI7kXpD5OIbFur9s7uaOC1Nh/L72pwK/S62AuuQQaQp4WS6rCh6e1Qiu311wQM i+tmijmYwryRpAB+BQXDD6we65meeLaoR7RiUgcCodh6T1WdeG3rXHU3aZ7RyUOXMl3G TpBB2cGgmPdRWJt4X4inUYYm8i6bJcsAq9WphlZqTjKnuQu0J21VXA/d2UN7hiTp+25Y oBFKxT3L7/ViA4VeKCx5UI0YHrZqbgbOXKdIaytdI7oCcrlH1MOqviwi9jouVUUIxAd6 bxkaj0/ygyyrL74KdQ1C3LsOjbhlUrt5uX0Z8rNBRK89XnyS3Fh7uD9pLe/0c96ZDAgH tviw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1683104848; x=1685696848; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oHrN32lBaAC78cOIMsVaenJPrzKJjknoApgieLS2EVM=; b=ey4lUVheJM5ZsT3ZWK5bdFFaVHdwAJrR4BpOGneJur0tl/FedMfuaM5dmUYwb1iV7J qdwr7yg+5bw3KhAdNzIdUqNpqPdZsgTwvPGCpcbyVGCdQuNTrzfe/5WR4d/VFIpIywGz wLS6qxKOTgcdRXdGKbrxlHdb0NStZLRGEYat9Mj9IUerhs3zUS34I/XxGPC+LZzrvoYw rzIMKxSoEQBZsx2DypnnU21clozR5ZiNHqHsDAfH7qjYkdifh5hY+aQj+MwpepHae3B+ yKvx5AEW1tYwRM9/n3BdNGchS8H1qxXuSBozblNO6Kk9Alyicv7OEh/uOc+JWiFEtVWe i6+Q== X-Gm-Message-State: AC+VfDyxgWEO8cVePOT9JuX8KpR0+2TlAPIJ8jEdHFbUwj620B2Uq3yO qhzH06y9s2e+AfdUb7VE8RZggw== X-Google-Smtp-Source: ACHHUZ7P6M/jU8n50FGEjgTSgEYAcUdKtRM+n+M8XLe9UPXqN9jUX6WcfyARZKzQxoFqI3CdjYYuxQ== X-Received: by 2002:a05:6000:11c5:b0:2fb:600e:55bd with SMTP id i5-20020a05600011c500b002fb600e55bdmr15479168wrx.39.1683104848425; Wed, 03 May 2023 02:07:28 -0700 (PDT) Received: from localhost ([147.161.155.99]) by smtp.gmail.com with ESMTPSA id e22-20020a5d5956000000b003012030a0c6sm33106775wri.18.2023.05.03.02.07.27 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 03 May 2023 02:07:28 -0700 (PDT) From: Andreas Hindborg To: Jens Axboe , Christoph Hellwig , Keith Busch , Damien Le Moal , Hannes Reinecke , lsf-pc@lists.linux-foundation.org, rust-for-linux@vger.kernel.org, linux-block@vger.kernel.org Cc: Andreas Hindborg , Matthew Wilcox , Miguel Ojeda , Alex Gaynor , Wedson Almeida Filho , Boqun Feng , Gary Guo , =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= , Benno Lossin , Andreas Hindborg , linux-kernel@vger.kernel.org (open list), gost.dev@samsung.com Subject: [RFC PATCH 11/11] rust: inline a number of short functions Date: Wed, 3 May 2023 11:07:08 +0200 Message-Id: <20230503090708.2524310-12-nmi@metaspace.dk> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230503090708.2524310-1-nmi@metaspace.dk> References: <20230503090708.2524310-1-nmi@metaspace.dk> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org From: Andreas Hindborg The rust compiler will not inline functions that live in vmlinux when building modules. Add inline directives to these short functions to ensure that they are inlined when building modules. Signed-off-by: Andreas Hindborg --- rust/kernel/sync/lock.rs | 2 ++ rust/kernel/sync/lock/mutex.rs | 2 ++ rust/kernel/sync/lock/spinlock.rs | 2 ++ rust/kernel/types.rs | 6 ++++++ 4 files changed, 12 insertions(+) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index bb21af8a8377..4bfc2f5d9841 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -101,6 +101,7 @@ impl Lock { /// Before acquiring the lock, it disables interrupts. When the guard is dropped, the interrupt /// state (either enabled or disabled) is restored to its state before /// [`lock_irqsave`](Self::lock_irqsave) was called. + #[inline(always)] pub fn lock_irqsave(&self) -> Guard<'_, T, B> { // SAFETY: The constructor of the type calls `init`, so the existence of the object proves // that `init` was called. @@ -210,6 +211,7 @@ impl core::ops::DerefMut for Guard<'_, T, B> { } impl Drop for Guard<'_, T, B> { + #[inline(always)] fn drop(&mut self) { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 923472f04af4..5e8096811b98 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -104,12 +104,14 @@ unsafe impl super::Backend for MutexBackend { unsafe { bindings::__mutex_init(ptr, name, key) } } + #[inline(always)] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `ptr` points to valid // memory, and that it has been initialised before. unsafe { bindings::mutex_lock(ptr) }; } + #[inline(always)] unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the // caller is the owner of the mutex. diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 50b8775bb49d..23a973dab85c 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -122,6 +122,7 @@ unsafe impl super::Backend for SpinLockBackend { None } + #[inline(always)] unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState) { match guard_state { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that @@ -141,6 +142,7 @@ unsafe impl super::Backend for SpinLockBackend { // interrupt state, and the `irqrestore` variant of the lock release functions to restore the state // in `unlock` -- we use the guard context to determine which method was used to acquire the lock. unsafe impl super::IrqSaveBackend for SpinLockBackend { + #[inline(always)] unsafe fn lock_irqsave(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `ptr` points to valid // memory, and that it has been initialised before. diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 98e71e96a7fc..7be1f64bbde9 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -70,10 +70,12 @@ pub trait ForeignOwnable: Sized { impl ForeignOwnable for Box { type Borrowed<'a> = &'a T; + #[inline(always)] fn into_foreign(self) -> *const core::ffi::c_void { Box::into_raw(self) as _ } + #[inline(always)] unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> &'a T { // SAFETY: The safety requirements for this function ensure that the object is still alive, // so it is safe to dereference the raw pointer. @@ -82,6 +84,7 @@ impl ForeignOwnable for Box { unsafe { &*ptr.cast() } } + #[inline(always)] unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. @@ -92,12 +95,15 @@ impl ForeignOwnable for Box { impl ForeignOwnable for () { type Borrowed<'a> = (); + #[inline(always)] fn into_foreign(self) -> *const core::ffi::c_void { core::ptr::NonNull::dangling().as_ptr() } + #[inline(always)] unsafe fn borrow<'a>(_: *const core::ffi::c_void) -> Self::Borrowed<'a> {} + #[inline(always)] unsafe fn from_foreign(_: *const core::ffi::c_void) -> Self {} }