[05/19] io_uring: add io_uringler_offload() for uringlet mode

Commit Message

Hao Xu Aug. 19, 2022, 3:27 p.m. UTC

From: Hao Xu <[e-mail]>

In uringlet mode, a io_uring_enter call shouldn't do the sqe submission
work, but just offload it to io-workers.

Signed-off-by: Hao Xu <[e-mail]>
---
 io_uring/io-wq.c    | 18 ++++++++++++++++++
 io_uring/io-wq.h    |  1 +
 io_uring/io_uring.c | 21 ++++++++++++++-------
 3 files changed, 33 insertions(+), 7 deletions(-)

Patch

diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index b533db18d7c0..212ea16cbb5e 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -771,6 +771,24 @@  void io_wq_worker_sleeping(struct task_struct *tsk)
 	io_wqe_dec_running(worker);
 }
 
+int io_uringlet_offload(struct io_wq *wq)
+{
+	struct io_wqe *wqe = wq->wqes[numa_node_id()];
+	struct io_wqe_acct *acct = io_get_acct(wqe, true);
+	bool waken;
+
+	raw_spin_lock(&wqe->lock);
+	rcu_read_lock();
+	waken = io_wqe_activate_free_worker(wqe, acct);
+	rcu_read_unlock();
+	raw_spin_unlock(&wqe->lock);
+
+	if (waken)
+		return 0;
+
+	return io_wqe_create_worker(wqe, acct);
+}
+
 static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker,
 			       struct task_struct *tsk)
 {
diff --git a/io_uring/io-wq.h b/io_uring/io-wq.h
index b862b04e49ce..66d2aeb17951 100644
--- a/io_uring/io-wq.h
+++ b/io_uring/io-wq.h
@@ -90,4 +90,5 @@  static inline bool io_wq_current_is_worker(void)
 
 extern struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
 					struct task_struct *task);
+extern int io_uringlet_offload(struct io_wq *wq);
 #endif
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b57e9059a388..554041705e96 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3051,15 +3051,22 @@  SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 		if (unlikely(ret))
 			goto out;
 
-		mutex_lock(&ctx->uring_lock);
-		ret = io_submit_sqes(ctx, to_submit);
-		if (ret != to_submit) {
+		if (!(ctx->flags & IORING_SETUP_URINGLET)) {
+			mutex_lock(&ctx->uring_lock);
+			ret = io_submit_sqes(ctx, to_submit);
+			if (ret != to_submit) {
+				mutex_unlock(&ctx->uring_lock);
+				goto out;
+			}
+			if ((flags & IORING_ENTER_GETEVENTS) && ctx->syscall_iopoll)
+				goto iopoll_locked;
 			mutex_unlock(&ctx->uring_lock);
-			goto out;
+		} else {
+			ret = io_uringlet_offload(ctx->let);
+			if (ret)
+				goto out;
+			ret = to_submit;
 		}
-		if ((flags & IORING_ENTER_GETEVENTS) && ctx->syscall_iopoll)
-			goto iopoll_locked;
-		mutex_unlock(&ctx->uring_lock);
 	}
 	if (flags & IORING_ENTER_GETEVENTS) {
 		int ret2;