@@ -87,6 +87,7 @@ extern int dynamic_might_resched(void);
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
extern void __might_resched(const char *file, int line, unsigned int offsets);
extern void __might_sleep(const char *file, int line);
+extern void __might_sleep_precision(const char *file, int len, int line);
extern void __cant_sleep(const char *file, int line, int preempt_offset);
extern void __cant_migrate(const char *file, int line);
@@ -145,6 +146,7 @@ extern void __cant_migrate(const char *file, int line);
static inline void __might_resched(const char *file, int line,
unsigned int offsets) { }
static inline void __might_sleep(const char *file, int line) { }
+static inline void __might_sleep_precision(const char *file, int len, int line) { }
# define might_sleep() do { might_resched(); } while (0)
# define cant_sleep() do { } while (0)
# define cant_migrate() do { } while (0)
@@ -8543,7 +8543,10 @@ void __init sched_init(void)
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
-void __might_sleep(const char *file, int line)
+extern inline void __might_resched_precision(const char *file, int len,
+ int line, unsigned int offsets);
+
+void __might_sleep_precision(const char *file, int len, int line)
{
unsigned int state = get_current_state();
/*
@@ -8557,7 +8560,14 @@ void __might_sleep(const char *file, int line)
(void *)current->task_state_change,
(void *)current->task_state_change);
- __might_resched(file, line, 0);
+ __might_resched_precision(file, len, line, 0);
+}
+
+void __might_sleep(const char *file, int line)
+{
+ long len = strlen(file);
+
+ __might_sleep_precision(file, len, line);
}
EXPORT_SYMBOL(__might_sleep);
@@ -8582,7 +8592,7 @@ static inline bool resched_offsets_ok(unsigned int offsets)
return nested == offsets;
}
-void __might_resched(const char *file, int line, unsigned int offsets)
+void __might_resched_precision(const char *file, int len, int line, unsigned int offsets)
{
/* Ratelimiting timestamp: */
static unsigned long prev_jiffy;
@@ -8605,8 +8615,8 @@ void __might_resched(const char *file, int line, unsigned int offsets)
/* Save this before calling printk(), since that will clobber it: */
preempt_disable_ip = get_preempt_disable_ip(current);
- pr_err("BUG: sleeping function called from invalid context at %s:%d\n",
- file, line);
+ pr_err("BUG: sleeping function called from invalid context at %.*s:%d\n",
+ len, file, line);
pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
in_atomic(), irqs_disabled(), current->non_block_count,
current->pid, current->comm);
@@ -8631,6 +8641,13 @@ void __might_resched(const char *file, int line, unsigned int offsets)
dump_stack();
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
}
+
+void __might_resched(const char *file, int line, unsigned int offsets)
+{
+ long len = strlen(file);
+
+ __might_resched_precision(file, len, line, offsets);
+}
EXPORT_SYMBOL(__might_resched);
void __cant_sleep(const char *file, int line, int preempt_offset)
@@ -12,6 +12,7 @@
#include "build_assert.c"
#include "build_bug.c"
#include "err.c"
+#include "kernel.c"
#include "kunit.c"
#include "mutex.c"
#include "page.c"
new file mode 100644
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/kernel.h>
+
+void rust_helper_cpu_relax(void)
+{
+ cpu_relax();
+}
+
+void rust_helper___might_sleep_precision(const char *file, int len, int line)
+{
+ __might_sleep_precision(file, len, line);
+}
new file mode 100644
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Processor related primitives.
+//!
+//! C header: [`include/linux/processor.h`](srctree/include/linux/processor.h).
+
+/// Lower CPU power consumption or yield to a hyperthreaded twin processor.
+///
+/// It also happens to serve as a compiler barrier.
+pub fn cpu_relax() {
+ // SAFETY: FFI call.
+ unsafe { bindings::cpu_relax() }
+}
@@ -58,6 +58,7 @@ macro_rules! declare_err {
declare_err!(EPIPE, "Broken pipe.");
declare_err!(EDOM, "Math argument out of domain of func.");
declare_err!(ERANGE, "Math result not representable.");
+ declare_err!(ETIMEDOUT, "Connection timed out.");
declare_err!(ERESTARTSYS, "Restart the system call.");
declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted.");
declare_err!(ERESTARTNOHAND, "Restart if no handler.");
new file mode 100644
@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Input and Output.
+
+pub mod poll;
new file mode 100644
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! IO polling.
+//!
+//! C header: [`include/linux/iopoll.h`](srctree/include/linux/iopoll.h).
+
+use crate::{
+ cpu::cpu_relax,
+ error::{code::*, Result},
+ time::{delay::fsleep, Delta, Instant},
+};
+
+use core::panic::Location;
+
+/// Polls periodically until a condition is met or a timeout is reached.
+///
+/// Public but hidden since it should only be used from public macros.
+///
+/// ```rust
+/// use kernel::io::poll::read_poll_timeout;
+/// use kernel::time::Delta;
+/// use kernel::sync::{SpinLock, new_spinlock};
+///
+/// let lock = KBox::pin_init(new_spinlock!(()), kernel::alloc::flags::GFP_KERNEL)?;
+/// let g = lock.lock();
+/// read_poll_timeout(|| Ok(()), |()| true, Delta::from_micros(42), Delta::from_micros(42));
+/// drop(g);
+///
+/// # Ok::<(), Error>(())
+/// ```
+#[track_caller]
+pub fn read_poll_timeout<Op, Cond, T: Copy>(
+ mut op: Op,
+ cond: Cond,
+ sleep_delta: Delta,
+ timeout_delta: Delta,
+) -> Result<T>
+where
+ Op: FnMut() -> Result<T>,
+ Cond: Fn(T) -> bool,
+{
+ let start = Instant::now();
+ let sleep = !sleep_delta.is_zero();
+ let timeout = !timeout_delta.is_zero();
+
+ might_sleep(Location::caller());
+
+ let val = loop {
+ let val = op()?;
+ if cond(val) {
+ // Unlike the C version, we immediately return.
+ // We know a condition is met so we don't need to check again.
+ return Ok(val);
+ }
+ if timeout && start.elapsed() > timeout_delta {
+ // Should we return Err(ETIMEDOUT) here instead of call op() again
+ // without a sleep between? But we follow the C version. op() could
+ // take some time so might be worth checking again.
+ break op()?;
+ }
+ if sleep {
+ fsleep(sleep_delta);
+ }
+ // fsleep() could be busy-wait loop so we always call cpu_relax().
+ cpu_relax();
+ };
+
+ if cond(val) {
+ Ok(val)
+ } else {
+ Err(ETIMEDOUT)
+ }
+}
+
+fn might_sleep(loc: &Location<'_>) {
+ // SAFETY: FFI call.
+ unsafe {
+ crate::bindings::__might_sleep_precision(
+ loc.file().as_ptr().cast(),
+ loc.file().len() as i32,
+ loc.line() as i32,
+ )
+ }
+}
@@ -30,11 +30,13 @@
#[cfg(CONFIG_BLOCK)]
pub mod block;
mod build_assert;
+pub mod cpu;
pub mod device;
pub mod error;
#[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)]
pub mod firmware;
pub mod init;
+pub mod io;
pub mod ioctl;
#[cfg(CONFIG_KUNIT)]
pub mod kunit;