io_uring/sqpoll: fix io-wq affinity when IORING_SETUP_SQPOLL is used

Commit Message

Jens Axboe Aug. 13, 2023, 5:09 p.m. UTC

If we setup the ring with SQPOLL, then that polling thread has its
own io-wq setup. This means that if the application uses
IORING_REGISTER_IOWQ_AFF to set the io-wq affinity, we should not be
setting it for the invoking task, but rather the sqpoll task.

Add an sqpoll helper that parks the thread and updates the affinity,
and use that one if we're using SQPOLL.

Cc: [e-mail] # 5.10+
Link: https://github.com/axboe/liburing/discussions/884
Signed-off-by: Jens Axboe <[e-mail]>

---

Patch

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 93db3e4e7b68..4afee22b7bda 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -4200,13 +4200,9 @@  static int io_register_enable_rings(struct io_ring_ctx *ctx)
 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
 				       void __user *arg, unsigned len)
 {
-	struct io_uring_task *tctx = current->io_uring;
 	cpumask_var_t new_mask;
 	int ret;
 
-	if (!tctx || !tctx->io_wq)
-		return -EINVAL;
-
 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
 		return -ENOMEM;
 
@@ -4227,7 +4223,21 @@  static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
 		return -EFAULT;
 	}
 
-	ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
+	if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
+		struct io_uring_task *tctx = current->io_uring;
+
+		if (!tctx || !tctx->io_wq) {
+			free_cpumask_var(new_mask);
+			return -EINVAL;
+		}
+
+		ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
+	} else {
+		mutex_unlock(&ctx->uring_lock);
+		ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);
+		mutex_lock(&ctx->uring_lock);
+	}
+
 	free_cpumask_var(new_mask);
 	return ret;
 }
diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c
index 5e329e3cd470..79a41fdfb899 100644
--- a/io_uring/sqpoll.c
+++ b/io_uring/sqpoll.c
@@ -421,3 +421,21 @@  __cold int io_sq_offload_create(struct io_ring_ctx *ctx,
 	io_sq_thread_finish(ctx);
 	return ret;
 }
+
+__cold int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx,
+				     cpumask_var_t mask)
+{
+	struct io_sq_data *sqd = ctx->sq_data;
+	int ret = -EINVAL;
+
+	if (sqd) {
+		struct task_struct *task;
+
+		io_sq_thread_park(sqd);
+		task = sqd->thread;
+		ret = io_wq_cpu_affinity(task->io_uring->io_wq, mask);
+		io_sq_thread_unpark(sqd);
+	}
+
+	return ret;
+}
diff --git a/io_uring/sqpoll.h b/io_uring/sqpoll.h
index e1b8d508d22d..8df37e8c9149 100644
--- a/io_uring/sqpoll.h
+++ b/io_uring/sqpoll.h
@@ -27,3 +27,4 @@  void io_sq_thread_park(struct io_sq_data *sqd);
 void io_sq_thread_unpark(struct io_sq_data *sqd);
 void io_put_sq_data(struct io_sq_data *sqd);
 void io_sqpoll_wait_sq(struct io_ring_ctx *ctx);
+int io_sqpoll_wq_cpu_affinity(struct io_ring_ctx *ctx, cpumask_var_t mask);