[08/19] io-wq: add IO_WORKER_F_SUBMIT and its friends

Commit Message

Hao Xu Aug. 19, 2022, 3:27 p.m. UTC

From: Hao Xu <[e-mail]>

Add IO_WORKER_F_SUBMIT to indicate that a uringlet worker is submitting
sqes and thus we should do some scheduling when it blocks.

Signed-off-by: Hao Xu <[e-mail]>
---
 io_uring/io-wq.c | 20 ++++++++++++++++++++
 io_uring/io-wq.h |  1 +
 2 files changed, 21 insertions(+)

Patch

diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 55f1063f24c7..7e58bb5857ee 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -393,6 +393,21 @@  static inline bool io_wq_is_uringlet(struct io_wq *wq)
 	return wq->private;
 }
 
+static inline void io_worker_set_submit(struct io_worker *worker)
+{
+	worker->flags |= IO_WORKER_F_SUBMIT;
+}
+
+static inline void io_worker_clean_submit(struct io_worker *worker)
+{
+	worker->flags &= ~IO_WORKER_F_SUBMIT;
+}
+
+static inline bool io_worker_test_submit(struct io_worker *worker)
+{
+	return worker->flags & IO_WORKER_F_SUBMIT;
+}
+
 static void io_wqe_dec_running(struct io_worker *worker)
 {
 	struct io_wqe_acct *acct = io_wqe_get_acct(worker);
@@ -408,6 +423,9 @@  static void io_wqe_dec_running(struct io_worker *worker)
 	if (io_wq_is_uringlet(wq)) {
 		bool activated;
 
+		if (!io_worker_test_submit(worker))
+			return;
+
 		raw_spin_lock(&wqe->lock);
 		rcu_read_lock();
 		activated = io_wqe_activate_free_worker(wqe, acct);
@@ -688,7 +706,9 @@  static void io_wqe_worker_let(struct io_worker *worker)
 		do {
 			enum io_uringlet_state submit_state;
 
+			io_worker_set_submit(worker);
 			submit_state = wq->do_work(wq->private);
+			io_worker_clean_submit(worker);
 			if (submit_state == IO_URINGLET_SCHEDULED) {
 				empty_count = 0;
 				break;
diff --git a/io_uring/io-wq.h b/io_uring/io-wq.h
index 504a8a8e3fd8..1485e9009784 100644
--- a/io_uring/io-wq.h
+++ b/io_uring/io-wq.h
@@ -33,6 +33,7 @@  enum {
 	IO_WORKER_F_FREE	= 4,	/* worker on free list */
 	IO_WORKER_F_BOUND	= 8,	/* is doing bounded work */
 	IO_WORKER_F_SCHEDULED	= 16,	/* worker had been scheduled out before */
+	IO_WORKER_F_SUBMIT	= 32,	/* uringlet worker is submitting sqes */
 };
 
 typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *);