@@ -15,6 +15,7 @@
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
+/* Changes to this asm must be reflected in `rust/kernel/arch/arm64/jump_label.rs` */
#define JUMP_TABLE_ENTRY(key, label) \
".pushsection __jump_table, \"aw\"\n\t" \
".align 3\n\t" \
@@ -13,6 +13,7 @@
#define JUMP_LABEL_NOP_SIZE 4
+/* Changes to this asm must be reflected in `rust/kernel/arch/loongarch/jump_label.rs` */
#define JUMP_TABLE_ENTRY \
".pushsection __jump_table, \"aw\" \n\t" \
".align 3 \n\t" \
@@ -16,6 +16,7 @@
#define JUMP_LABEL_NOP_SIZE 4
+/* Changes to this asm must be reflected in `rust/kernel/arch/riscv/jump_label.rs` */
static __always_inline bool arch_static_branch(struct static_key * const key,
const bool branch)
{
@@ -12,6 +12,7 @@
#include <linux/stringify.h>
#include <linux/types.h>
+/* Changes to this asm must be reflected in `rust/kernel/arch/x86/jump_label.rs` */
#define JUMP_TABLE_ENTRY \
".pushsection __jump_table, \"aw\" \n\t" \
_ASM_ALIGN "\n\t" \
new file mode 100644
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Arm64 Rust implementation of jump_label.h
+
+/// arm64 implementation of arch_static_branch
+#[macro_export]
+#[cfg(target_arch = "aarch64")]
+macro_rules! arch_static_branch {
+ ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: {
+ core::arch::asm!(
+ r#"
+ 1: nop
+
+ .pushsection __jump_table, "aw"
+ .align 3
+ .long 1b - ., {0} - .
+ .quad {1} + {2} + {3} - .
+ .popsection
+ "#,
+ label {
+ break 'my_label true;
+ },
+ sym $key,
+ const ::core::mem::offset_of!($keytyp, $field),
+ const $crate::arch::bool_to_int($branch),
+ );
+
+ break 'my_label false;
+ }};
+}
+
+pub use arch_static_branch;
new file mode 100644
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Loongarch Rust implementation of jump_label.h
+
+/// loongarch implementation of arch_static_branch
+#[doc(hidden)]
+#[macro_export]
+#[cfg(target_arch = "loongarch64")]
+macro_rules! arch_static_branch {
+ ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: {
+ core::arch::asm!(
+ r#"
+ 1: nop
+
+ .pushsection __jump_table, "aw"
+ .align 3
+ .long 1b - ., {0} - .
+ .quad {1} + {2} + {3} - .
+ .popsection
+ "#,
+ label {
+ break 'my_label true;
+ },
+ sym $key,
+ const ::core::mem::offset_of!($keytyp, $field),
+ const $crate::arch::bool_to_int($branch),
+ );
+
+ break 'my_label false;
+ }};
+}
+
+pub use arch_static_branch;
new file mode 100644
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Architecture specific code.
+
+#[cfg_attr(target_arch = "aarch64", path = "arm64")]
+#[cfg_attr(target_arch = "x86_64", path = "x86")]
+#[cfg_attr(target_arch = "loongarch64", path = "loongarch")]
+#[cfg_attr(target_arch = "riscv64", path = "riscv")]
+mod inner {
+ pub mod jump_label;
+}
+
+pub use self::inner::*;
+
+/// A helper used by inline assembly to pass a boolean to as a `const` parameter.
+///
+/// Using this function instead of a cast lets you assert that the input is a boolean, rather than
+/// some other type that can be cast to an integer.
+#[doc(hidden)]
+pub const fn bool_to_int(b: bool) -> i32 {
+ b as i32
+}
new file mode 100644
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! RiscV Rust implementation of jump_label.h
+
+/// riscv implementation of arch_static_branch
+#[macro_export]
+#[cfg(target_arch = "riscv64")]
+macro_rules! arch_static_branch {
+ ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: {
+ core::arch::asm!(
+ r#"
+ .align 2
+ .option push
+ .option norelax
+ .option norvc
+ 1: nop
+ .option pop
+ .pushsection __jump_table, "aw"
+ .align 3
+ .long 1b - ., {0} - .
+ .dword {1} + {2} + {3} - .
+ .popsection
+ "#,
+ label {
+ break 'my_label true;
+ },
+ sym $key,
+ const ::core::mem::offset_of!($keytyp, $field),
+ const $crate::arch::bool_to_int($branch),
+ );
+
+ break 'my_label false;
+ }};
+}
+
+pub use arch_static_branch;
new file mode 100644
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! X86 Rust implementation of jump_label.h
+
+/// x86 implementation of arch_static_branch
+#[macro_export]
+#[cfg(target_arch = "x86_64")]
+macro_rules! arch_static_branch {
+ ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: {
+ core::arch::asm!(
+ r#"
+ 1: .byte 0x0f,0x1f,0x44,0x00,0x00
+
+ .pushsection __jump_table, "aw"
+ .balign 8
+ .long 1b - .
+ .long {0} - .
+ .quad {1} + {2} + {3} - .
+ .popsection
+ "#,
+ label {
+ break 'my_label true;
+ },
+ sym $key,
+ const ::core::mem::offset_of!($keytyp, $field),
+ const $crate::arch::bool_to_int($branch),
+ );
+
+ break 'my_label false;
+ }};
+}
+
+pub use arch_static_branch;
@@ -27,6 +27,7 @@
extern crate self as kernel;
pub mod alloc;
+pub mod arch;
#[cfg(CONFIG_BLOCK)]
pub mod block;
mod build_assert;
@@ -44,6 +45,7 @@
pub mod prelude;
pub mod print;
mod static_assert;
+pub mod static_key;
#[doc(hidden)]
pub mod std_vendor;
pub mod str;
new file mode 100644
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Logic for static keys.
+
+use crate::bindings::*;
+
+/// Branch based on a static key.
+///
+/// Takes three arguments:
+///
+/// * `key` - the path to the static variable containing the `static_key`.
+/// * `keytyp` - the type of `key`.
+/// * `field` - the name of the field of `key` that contains the `static_key`.
+#[macro_export]
+macro_rules! static_key_false {
+ ($key:path, $keytyp:ty, $field:ident) => {{
+ // Assert that `$key` has type `$keytyp` and that `$key.$field` has type `static_key`.
+ //
+ // SAFETY: We know that `$key` is a static because otherwise the inline assembly will not
+ // compile. The raw pointers created in this block are in-bounds of `$key`.
+ static _TY_ASSERT: () = unsafe {
+ let key: *const $keytyp = ::core::ptr::addr_of!($key);
+ let _: *const $crate::bindings::static_key = ::core::ptr::addr_of!((*key).$field);
+ };
+
+ $crate::arch::jump_label::arch_static_branch! { $key, $keytyp, $field, false }
+ }};
+}
+
+pub use static_key_false;
@@ -263,7 +263,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE
# Compile Rust sources (.rs)
# ---------------------------------------------------------------------------
-rust_allowed_features := new_uninit
+rust_allowed_features := asm_const,asm_goto,new_uninit
# `--out-dir` is required to avoid temporaries being created by `rustc` in the
# current working directory, which may be not accessible in the out-of-tree