diff mbox series

[8/9] io_uring/msg_ring: add basic wakeup batch support

Message ID 20240605141933.11975-9-axboe@kernel.dk (mailing list archive)
State New
Headers show
Series Improve MSG_RING DEFER_TASKRUN performance | expand

Commit Message

Jens Axboe June 5, 2024, 1:51 p.m. UTC 
Factor in the number of overflow entries waiting, on both the msg ring
and local task_work add side.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 io_uring/msg_ring.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

Comments

Pavel Begunkov June 5, 2024, 3:32 p.m. UTC | #1 
On 6/5/24 14:51, Jens Axboe wrote:
> Factor in the number of overflow entries waiting, on both the msg ring
> and local task_work add side.

Did you forget to add the local tw change to the patch?


> Signed-off-by: Jens Axboe <axboe@kernel.dk>
> ---
>   io_uring/msg_ring.c | 10 ++++++++--
>   1 file changed, 8 insertions(+), 2 deletions(-)
> 
> diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
> index 9a7c63f38c46..eeca1563ceed 100644
> --- a/io_uring/msg_ring.c
> +++ b/io_uring/msg_ring.c
> @@ -109,6 +109,8 @@ static void io_msg_add_overflow(struct io_msg *msg,
>   				u32 flags)
>   	__releases(&target_ctx->completion_lock)
>   {
> +	unsigned nr_prev, nr_wait;
> +
>   	if (list_empty(&target_ctx->cq_overflow_list)) {
>   		set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &target_ctx->check_cq);
>   		atomic_or(IORING_SQ_TASKRUN, &target_ctx->rings->sq_flags);
> @@ -117,10 +119,14 @@ static void io_msg_add_overflow(struct io_msg *msg,
>   	ocqe->cqe.user_data = msg->user_data;
>   	ocqe->cqe.res = ret;
>   	ocqe->cqe.flags = flags;
> -	target_ctx->nr_overflow++;
> +	nr_prev = target_ctx->nr_overflow++;
>   	list_add_tail(&ocqe->list, &target_ctx->cq_overflow_list);
>   	spin_unlock(&target_ctx->completion_lock);
> -	wake_up_state(target_ctx->submitter_task, TASK_INTERRUPTIBLE);
> +	rcu_read_lock();
> +	io_defer_tw_count(target_ctx, &nr_wait);
> +	nr_prev += nr_wait;
> +	io_defer_wake(target_ctx, nr_prev + 1, nr_prev);
> +	rcu_read_unlock();
>   }
>   
>   static int io_msg_fill_remote(struct io_msg *msg, unsigned int issue_flags,
Jens Axboe June 5, 2024, 3:50 p.m. UTC | #2 
On 6/5/24 9:32 AM, Pavel Begunkov wrote:
> On 6/5/24 14:51, Jens Axboe wrote:
>> Factor in the number of overflow entries waiting, on both the msg ring
>> and local task_work add side.
> 
> Did you forget to add the local tw change to the patch?

Gah yes, looked like that line got dropped while shuffling the
patches around...
diff mbox series

Patch

diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 9a7c63f38c46..eeca1563ceed 100644
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -109,6 +109,8 @@  static void io_msg_add_overflow(struct io_msg *msg,
 				u32 flags)
 	__releases(&target_ctx->completion_lock)
 {
+	unsigned nr_prev, nr_wait;
+
 	if (list_empty(&target_ctx->cq_overflow_list)) {
 		set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &target_ctx->check_cq);
 		atomic_or(IORING_SQ_TASKRUN, &target_ctx->rings->sq_flags);
@@ -117,10 +119,14 @@  static void io_msg_add_overflow(struct io_msg *msg,
 	ocqe->cqe.user_data = msg->user_data;
 	ocqe->cqe.res = ret;
 	ocqe->cqe.flags = flags;
-	target_ctx->nr_overflow++;
+	nr_prev = target_ctx->nr_overflow++;
 	list_add_tail(&ocqe->list, &target_ctx->cq_overflow_list);
 	spin_unlock(&target_ctx->completion_lock);
-	wake_up_state(target_ctx->submitter_task, TASK_INTERRUPTIBLE);
+	rcu_read_lock();
+	io_defer_tw_count(target_ctx, &nr_wait);
+	nr_prev += nr_wait;
+	io_defer_wake(target_ctx, nr_prev + 1, nr_prev);
+	rcu_read_unlock();
 }
 
 static int io_msg_fill_remote(struct io_msg *msg, unsigned int issue_flags,