| Message ID | 20240507091651.2720896-1-xue01.he@samsung.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [v3] io_uring: releasing CPU resources when polling | expand |
On 5/7/24 17:16, hexue wrote: >This patch is intended to release the CPU resources of io_uring in >polling mode. When IO is issued, the program immediately polls for >check completion, which is a waste of CPU resources when IO commands >are executed on the disk. > >I add the hybrid polling feature in io_uring, enables polling to >release a portion of CPU resources without affecting block layer. > >- Record the running time and context switching time of each > IO, and use these time to determine whether a process continue > to schedule. > >- Adaptive adjustment to different devices. Due to the real-time > nature of time recording, each device's IO processing speed is > different, so the CPU optimization effect will vary. > >- Set a interface (ctx->flag) enables application to choose whether > or not to use this feature. > >The CPU optimization in peak workload of patch is tested as follows: > set 8 poll queues > all CPU utilization of original polling is 100% for per CPU, after > optimization, the CPU utilization drop a lot (per CPU); > > read(128k, QD64, 1Job) 37% write(128k, QD64, 1Job) 40% > randread(4k, QD64, 16Job) 52% randwrite(4k, QD64, 16Job) 12% > > Compared to original polling, the optimised performance reduction > with peak workload within 1%. > > read 0.29% write 0.51% randread 0.09% randwrite 0% > >Signed-off-by: hexue <xue01.he@samsung.com> > >--- > >changes: >v2: > - extend hybrid poll to async polled io > >v1: > - initial version >--- > include/linux/io_uring_types.h | 14 ++++ > include/uapi/linux/io_uring.h | 1 + > io_uring/io_uring.c | 4 +- > io_uring/io_uring.h | 3 + > io_uring/rw.c | 115 ++++++++++++++++++++++++++++++++- > 5 files changed, 135 insertions(+), 2 deletions(-) > >diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h >index 854ad67a5f70..3a75b9904326 100644 >--- a/include/linux/io_uring_types.h >+++ b/include/linux/io_uring_types.h >@@ -224,6 +224,11 @@ struct io_alloc_cache { > size_t elem_size; > }; > >+struct iopoll_info { >+ long last_runtime; >+ long last_irqtime; >+}; >+ > struct io_ring_ctx { > /* const or read-mostly hot data */ > struct { >@@ -421,6 +426,7 @@ struct io_ring_ctx { > unsigned short n_sqe_pages; > struct page **ring_pages; > struct page **sqe_pages; >+ struct xarray poll_array; > }; > > struct io_tw_state { >@@ -571,6 +577,12 @@ static inline void io_kiocb_cmd_sz_check(size_t cmd_sz) > ) > #define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr) > >+struct hy_poll_time { >+ int poll_state; >+ struct timespec64 iopoll_start; >+ struct timespec64 iopoll_end; >+}; >+ > struct io_kiocb { > union { > /* >@@ -641,6 +653,8 @@ struct io_kiocb { > u64 extra1; > u64 extra2; > } big_cqe; >+ /* for hybrid iopoll */ >+ struct hy_poll_time *hy_poll; > }; > > struct io_overflow_cqe { >diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h >index 7a673b52827b..0038cdfec18f 100644 >--- a/include/uapi/linux/io_uring.h >+++ b/include/uapi/linux/io_uring.h >@@ -198,6 +198,7 @@ enum { > * Removes indirection through the SQ index array. > */ > #define IORING_SETUP_NO_SQARRAY (1U << 16) >+#define IORING_SETUP_HY_POLL (1U << 17) > > enum io_uring_op { > IORING_OP_NOP, >diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c >index cd9a137ad6ce..2c14768bbe27 100644 >--- a/io_uring/io_uring.c >+++ b/io_uring/io_uring.c >@@ -311,6 +311,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) > goto err; > > ctx->flags = p->flags; >+ xa_init(&ctx->poll_array); > atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT); > init_waitqueue_head(&ctx->sqo_sq_wait); > INIT_LIST_HEAD(&ctx->sqd_list); >@@ -2921,6 +2922,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) > kfree(ctx->cancel_table_locked.hbs); > kfree(ctx->io_bl); > xa_destroy(&ctx->io_bl_xa); >+ xa_destroy(&ctx->poll_array); > kfree(ctx); > } > >@@ -4050,7 +4052,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) > IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | > IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | > IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | >- IORING_SETUP_NO_SQARRAY)) >+ IORING_SETUP_NO_SQARRAY | IORING_SETUP_HY_POLL)) > return -EINVAL; > > return io_uring_create(entries, &p, params); >diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h >index d5495710c178..72d6a4c3b46d 100644 >--- a/io_uring/io_uring.h >+++ b/io_uring/io_uring.h >@@ -125,6 +125,9 @@ static inline void io_req_task_work_add(struct io_kiocb *req) > __io_req_task_work_add(req, 0); > } > >+/* if sleep time less than 1us, then do not do the schedule op */ >+#define MIN_SCHETIME 1000 >+ > #define io_for_each_link(pos, head) \ > for (pos = (head); pos; pos = pos->link) > >diff --git a/io_uring/rw.c b/io_uring/rw.c >index d5e79d9bdc71..29c7ce23ed71 100644 >--- a/io_uring/rw.c >+++ b/io_uring/rw.c >@@ -713,6 +713,46 @@ static bool need_complete_io(struct io_kiocb *req) > S_ISBLK(file_inode(req->file)->i_mode); > } > >+void init_hybrid_poll(struct io_ring_ctx *ctx, struct io_kiocb *req) >+{ >+ /* >+ * In multiple concurrency, a thread may operate several files >+ * under different file systems, the inode numbers may be >+ * duplicated. Each device has a different IO command processing >+ * capability, so using device number to record the running time >+ * of device >+ */ >+ u32 index = req->file->f_inode->i_rdev; >+ struct iopoll_info *entry = xa_load(&ctx->poll_array, index); >+ struct hy_poll_time *hpt = kmalloc(sizeof(struct hy_poll_time), GFP_KERNEL); >+ >+ /* if alloc fail, go to regular poll */ >+ if (!hpt) { >+ ctx->flags &= ~IORING_SETUP_HY_POLL; >+ return; >+ } >+ hpt->poll_state = 0; >+ req->hy_poll = hpt; >+ >+ if (!entry) { >+ entry = kmalloc(sizeof(struct iopoll_info), GFP_KERNEL); >+ if (!entry) { >+ ctx->flags &= ~IORING_SETUP_HY_POLL; >+ return; >+ } >+ entry->last_runtime = 0; >+ entry->last_irqtime = 0; >+ xa_store(&ctx->poll_array, index, entry, GFP_KERNEL); >+ } >+ >+ /* >+ * Here we need nanosecond timestamps, some ways of reading >+ * timestamps directly are only accurate to microseconds, so >+ * there's no better alternative here for now >+ */ >+ ktime_get_ts64(&hpt->iopoll_start); >+} >+ > static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) > { > struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); >@@ -750,6 +790,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) > kiocb->ki_flags |= IOCB_HIPRI; > kiocb->ki_complete = io_complete_rw_iopoll; > req->iopoll_completed = 0; >+ if (ctx->flags & IORING_SETUP_HY_POLL) >+ init_hybrid_poll(ctx, req); > } else { > if (kiocb->ki_flags & IOCB_HIPRI) > return -EINVAL; >@@ -1118,6 +1160,75 @@ void io_rw_fail(struct io_kiocb *req) > io_req_set_res(req, res, req->cqe.flags); > } > >+void io_delay(struct hy_poll_time *hpt, struct iopoll_info *entry) >+{ >+ struct hrtimer_sleeper timer; >+ struct timespec64 tc, oldtc; >+ enum hrtimer_mode mode; >+ ktime_t kt; >+ long sleep_ti; >+ >+ if (hpt->poll_state == 1) >+ return; >+ >+ if (entry->last_runtime <= entry->last_irqtime) >+ return; >+ >+ /* >+ * Avoid excessive scheduling time affecting performance >+ * by using only 25 per cent of the remaining time >+ */ >+ sleep_ti = (entry->last_runtime - entry->last_irqtime) / 4; >+ >+ /* >+ * If the time available for sleep is too short, i.e. the >+ * totle running time and the context switching loss time >+ * are very close to each other, the scheduling operation >+ * is not performed to avoid increasing latency >+ */ >+ if (sleep_ti < MIN_SCHETIME) >+ return; >+ >+ ktime_get_ts64(&oldtc); >+ kt = ktime_set(0, sleep_ti); >+ hpt->poll_state = 1; >+ >+ mode = HRTIMER_MODE_REL; >+ hrtimer_init_sleeper_on_stack(&timer, CLOCK_MONOTONIC, mode); >+ hrtimer_set_expires(&timer.timer, kt); >+ set_current_state(TASK_UNINTERRUPTIBLE); >+ hrtimer_sleeper_start_expires(&timer, mode); >+ >+ if (timer.task) >+ io_schedule(); >+ >+ hrtimer_cancel(&timer.timer); >+ mode = HRTIMER_MODE_ABS; >+ __set_current_state(TASK_RUNNING); >+ destroy_hrtimer_on_stack(&timer.timer); >+ >+ ktime_get_ts64(&tc); >+ entry->last_irqtime = tc.tv_nsec - oldtc.tv_nsec - sleep_ti; >+} >+ >+int io_uring_hybrid_poll(struct io_kiocb *req, >+ struct io_comp_batch *iob, unsigned int poll_flags) >+{ >+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); >+ struct io_ring_ctx *ctx = req->ctx; >+ struct hy_poll_time *hpt = req->hy_poll; >+ u32 index = req->file->f_inode->i_rdev; >+ struct iopoll_info *entry = xa_load(&ctx->poll_array, index); >+ int ret; >+ >+ io_delay(hpt, entry); >+ ret = req->file->f_op->iopoll(&rw->kiocb, iob, poll_flags); >+ >+ ktime_get_ts64(&hpt->iopoll_end); >+ entry->last_runtime = hpt->iopoll_end.tv_nsec - hpt->iopoll_start.tv_nsec; >+ return ret; >+} >+ > int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) > { > struct io_wq_work_node *pos, *start, *prev; >@@ -1145,7 +1256,9 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) > if (READ_ONCE(req->iopoll_completed)) > break; > >- if (req->opcode == IORING_OP_URING_CMD) { >+ if (ctx->flags & IORING_SETUP_HY_POLL) { >+ ret = io_uring_hybrid_poll(req, &iob, poll_flags); >+ } else if (req->opcode == IORING_OP_URING_CMD) { > struct io_uring_cmd *ioucmd; > > ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); Hi, Jens I have revised some of the code according to your suggestions, and added comments to the parts that were not modified. Do you have any other comments? -- Xue He
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 854ad67a5f70..3a75b9904326 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -224,6 +224,11 @@ struct io_alloc_cache { size_t elem_size; }; +struct iopoll_info { + long last_runtime; + long last_irqtime; +}; + struct io_ring_ctx { /* const or read-mostly hot data */ struct { @@ -421,6 +426,7 @@ struct io_ring_ctx { unsigned short n_sqe_pages; struct page **ring_pages; struct page **sqe_pages; + struct xarray poll_array; }; struct io_tw_state { @@ -571,6 +577,12 @@ static inline void io_kiocb_cmd_sz_check(size_t cmd_sz) ) #define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr) +struct hy_poll_time { + int poll_state; + struct timespec64 iopoll_start; + struct timespec64 iopoll_end; +}; + struct io_kiocb { union { /* @@ -641,6 +653,8 @@ struct io_kiocb { u64 extra1; u64 extra2; } big_cqe; + /* for hybrid iopoll */ + struct hy_poll_time *hy_poll; }; struct io_overflow_cqe { diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 7a673b52827b..0038cdfec18f 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -198,6 +198,7 @@ enum { * Removes indirection through the SQ index array. */ #define IORING_SETUP_NO_SQARRAY (1U << 16) +#define IORING_SETUP_HY_POLL (1U << 17) enum io_uring_op { IORING_OP_NOP, diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index cd9a137ad6ce..2c14768bbe27 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -311,6 +311,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) goto err; ctx->flags = p->flags; + xa_init(&ctx->poll_array); atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT); init_waitqueue_head(&ctx->sqo_sq_wait); INIT_LIST_HEAD(&ctx->sqd_list); @@ -2921,6 +2922,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) kfree(ctx->cancel_table_locked.hbs); kfree(ctx->io_bl); xa_destroy(&ctx->io_bl_xa); + xa_destroy(&ctx->poll_array); kfree(ctx); } @@ -4050,7 +4052,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY | - IORING_SETUP_NO_SQARRAY)) + IORING_SETUP_NO_SQARRAY | IORING_SETUP_HY_POLL)) return -EINVAL; return io_uring_create(entries, &p, params); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index d5495710c178..72d6a4c3b46d 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -125,6 +125,9 @@ static inline void io_req_task_work_add(struct io_kiocb *req) __io_req_task_work_add(req, 0); } +/* if sleep time less than 1us, then do not do the schedule op */ +#define MIN_SCHETIME 1000 + #define io_for_each_link(pos, head) \ for (pos = (head); pos; pos = pos->link) diff --git a/io_uring/rw.c b/io_uring/rw.c index d5e79d9bdc71..29c7ce23ed71 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -713,6 +713,46 @@ static bool need_complete_io(struct io_kiocb *req) S_ISBLK(file_inode(req->file)->i_mode); } +void init_hybrid_poll(struct io_ring_ctx *ctx, struct io_kiocb *req) +{ + /* + * In multiple concurrency, a thread may operate several files + * under different file systems, the inode numbers may be + * duplicated. Each device has a different IO command processing + * capability, so using device number to record the running time + * of device + */ + u32 index = req->file->f_inode->i_rdev; + struct iopoll_info *entry = xa_load(&ctx->poll_array, index); + struct hy_poll_time *hpt = kmalloc(sizeof(struct hy_poll_time), GFP_KERNEL); + + /* if alloc fail, go to regular poll */ + if (!hpt) { + ctx->flags &= ~IORING_SETUP_HY_POLL; + return; + } + hpt->poll_state = 0; + req->hy_poll = hpt; + + if (!entry) { + entry = kmalloc(sizeof(struct iopoll_info), GFP_KERNEL); + if (!entry) { + ctx->flags &= ~IORING_SETUP_HY_POLL; + return; + } + entry->last_runtime = 0; + entry->last_irqtime = 0; + xa_store(&ctx->poll_array, index, entry, GFP_KERNEL); + } + + /* + * Here we need nanosecond timestamps, some ways of reading + * timestamps directly are only accurate to microseconds, so + * there's no better alternative here for now + */ + ktime_get_ts64(&hpt->iopoll_start); +} + static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); @@ -750,6 +790,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) kiocb->ki_flags |= IOCB_HIPRI; kiocb->ki_complete = io_complete_rw_iopoll; req->iopoll_completed = 0; + if (ctx->flags & IORING_SETUP_HY_POLL) + init_hybrid_poll(ctx, req); } else { if (kiocb->ki_flags & IOCB_HIPRI) return -EINVAL; @@ -1118,6 +1160,75 @@ void io_rw_fail(struct io_kiocb *req) io_req_set_res(req, res, req->cqe.flags); } +void io_delay(struct hy_poll_time *hpt, struct iopoll_info *entry) +{ + struct hrtimer_sleeper timer; + struct timespec64 tc, oldtc; + enum hrtimer_mode mode; + ktime_t kt; + long sleep_ti; + + if (hpt->poll_state == 1) + return; + + if (entry->last_runtime <= entry->last_irqtime) + return; + + /* + * Avoid excessive scheduling time affecting performance + * by using only 25 per cent of the remaining time + */ + sleep_ti = (entry->last_runtime - entry->last_irqtime) / 4; + + /* + * If the time available for sleep is too short, i.e. the + * totle running time and the context switching loss time + * are very close to each other, the scheduling operation + * is not performed to avoid increasing latency + */ + if (sleep_ti < MIN_SCHETIME) + return; + + ktime_get_ts64(&oldtc); + kt = ktime_set(0, sleep_ti); + hpt->poll_state = 1; + + mode = HRTIMER_MODE_REL; + hrtimer_init_sleeper_on_stack(&timer, CLOCK_MONOTONIC, mode); + hrtimer_set_expires(&timer.timer, kt); + set_current_state(TASK_UNINTERRUPTIBLE); + hrtimer_sleeper_start_expires(&timer, mode); + + if (timer.task) + io_schedule(); + + hrtimer_cancel(&timer.timer); + mode = HRTIMER_MODE_ABS; + __set_current_state(TASK_RUNNING); + destroy_hrtimer_on_stack(&timer.timer); + + ktime_get_ts64(&tc); + entry->last_irqtime = tc.tv_nsec - oldtc.tv_nsec - sleep_ti; +} + +int io_uring_hybrid_poll(struct io_kiocb *req, + struct io_comp_batch *iob, unsigned int poll_flags) +{ + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + struct io_ring_ctx *ctx = req->ctx; + struct hy_poll_time *hpt = req->hy_poll; + u32 index = req->file->f_inode->i_rdev; + struct iopoll_info *entry = xa_load(&ctx->poll_array, index); + int ret; + + io_delay(hpt, entry); + ret = req->file->f_op->iopoll(&rw->kiocb, iob, poll_flags); + + ktime_get_ts64(&hpt->iopoll_end); + entry->last_runtime = hpt->iopoll_end.tv_nsec - hpt->iopoll_start.tv_nsec; + return ret; +} + int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_wq_work_node *pos, *start, *prev; @@ -1145,7 +1256,9 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (READ_ONCE(req->iopoll_completed)) break; - if (req->opcode == IORING_OP_URING_CMD) { + if (ctx->flags & IORING_SETUP_HY_POLL) { + ret = io_uring_hybrid_poll(req, &iob, poll_flags); + } else if (req->opcode == IORING_OP_URING_CMD) { struct io_uring_cmd *ioucmd; ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
This patch is intended to release the CPU resources of io_uring in polling mode. When IO is issued, the program immediately polls for check completion, which is a waste of CPU resources when IO commands are executed on the disk. I add the hybrid polling feature in io_uring, enables polling to release a portion of CPU resources without affecting block layer. - Record the running time and context switching time of each IO, and use these time to determine whether a process continue to schedule. - Adaptive adjustment to different devices. Due to the real-time nature of time recording, each device's IO processing speed is different, so the CPU optimization effect will vary. - Set a interface (ctx->flag) enables application to choose whether or not to use this feature. The CPU optimization in peak workload of patch is tested as follows: set 8 poll queues all CPU utilization of original polling is 100% for per CPU, after optimization, the CPU utilization drop a lot (per CPU); read(128k, QD64, 1Job) 37% write(128k, QD64, 1Job) 40% randread(4k, QD64, 16Job) 52% randwrite(4k, QD64, 16Job) 12% Compared to original polling, the optimised performance reduction with peak workload within 1%. read 0.29% write 0.51% randread 0.09% randwrite 0% Signed-off-by: hexue <xue01.he@samsung.com> --- changes: v2: - extend hybrid poll to async polled io v1: - initial version --- include/linux/io_uring_types.h | 14 ++++ include/uapi/linux/io_uring.h | 1 + io_uring/io_uring.c | 4 +- io_uring/io_uring.h | 3 + io_uring/rw.c | 115 ++++++++++++++++++++++++++++++++- 5 files changed, 135 insertions(+), 2 deletions(-)