| Message ID | 20211017020623.77815-4-axboe@kernel.dk (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | [1/6] block: add a struct io_comp_batch argument to fops->iopoll() | expand |
On Sat, Oct 16, 2021 at 08:06:20PM -0600, Jens Axboe wrote: > Instead of calling blk_mq_end_request() on a single request, add a helper > that takes the new struct io_comp_batch and completes any request stored > in there. > > Signed-off-by: Jens Axboe <axboe@kernel.dk> > --- > block/blk-mq-tag.c | 6 ++++ > block/blk-mq-tag.h | 1 + > block/blk-mq.c | 81 ++++++++++++++++++++++++++++++++---------- > include/linux/blk-mq.h | 29 +++++++++++++++ > 4 files changed, 98 insertions(+), 19 deletions(-) > > diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c > index c43b97201161..b94c3e8ef392 100644 > --- a/block/blk-mq-tag.c > +++ b/block/blk-mq-tag.c > @@ -207,6 +207,12 @@ void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, > } > } > > +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) > +{ > + sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags, > + tag_array, nr_tags); > +} > + > struct bt_iter_data { > struct blk_mq_hw_ctx *hctx; > busy_iter_fn *fn; > diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h > index e617c7220626..df787b5a23bd 100644 > --- a/block/blk-mq-tag.h > +++ b/block/blk-mq-tag.h > @@ -19,6 +19,7 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, > unsigned int *offset); > extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, > unsigned int tag); > +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags); > extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, > struct blk_mq_tags **tags, > unsigned int depth, bool can_grow); > diff --git a/block/blk-mq.c b/block/blk-mq.c > index 8eb80e70e8ea..58dc0c0c24ac 100644 > --- a/block/blk-mq.c > +++ b/block/blk-mq.c > @@ -292,15 +292,6 @@ void blk_mq_wake_waiters(struct request_queue *q) > blk_mq_tag_wakeup_all(hctx->tags, true); > } > > -/* > - * Only need start/end time stamping if we have iostat or > - * blk stats enabled, or using an IO scheduler. > - */ > -static inline bool blk_mq_need_time_stamp(struct request *rq) > -{ > - return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV)); > -} > - > static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, > unsigned int tag, u64 alloc_time_ns) > { > @@ -754,19 +745,21 @@ bool blk_update_request(struct request *req, blk_status_t error, > } > EXPORT_SYMBOL_GPL(blk_update_request); > > -inline void __blk_mq_end_request(struct request *rq, blk_status_t error) > +static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) > { > - if (blk_mq_need_time_stamp(rq)) { > - u64 now = ktime_get_ns(); > + if (rq->rq_flags & RQF_STATS) { > + blk_mq_poll_stats_start(rq->q); > + blk_stat_add(rq, now); > + } > > + blk_mq_sched_completed_request(rq, now); > + blk_account_io_done(rq, now); > +} > > - blk_mq_sched_completed_request(rq, now); > - blk_account_io_done(rq, now); > - } > +inline void __blk_mq_end_request(struct request *rq, blk_status_t error) > +{ > + if (blk_mq_need_time_stamp(rq)) > + __blk_mq_end_request_acct(rq, ktime_get_ns()); > > if (rq->end_io) { > rq_qos_done(rq->q, rq); > @@ -785,6 +778,56 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) > } > EXPORT_SYMBOL(blk_mq_end_request); > > +#define TAG_COMP_BATCH 32 > + > +static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx, > + int *tag_array, int nr_tags) > +{ > + struct request_queue *q = hctx->queue; > + > + blk_mq_put_tags(hctx->tags, tag_array, nr_tags); > + percpu_ref_put_many(&q->q_usage_counter, nr_tags); > +} > + > +void blk_mq_end_request_batch(struct io_comp_batch *iob) > +{ > + int tags[TAG_COMP_BATCH], nr_tags = 0; > + struct blk_mq_hw_ctx *last_hctx = NULL; > + struct request *rq; > + u64 now = 0; > + > + if (iob->need_ts) > + now = ktime_get_ns(); > + > + while ((rq = rq_list_pop(&iob->req_list)) != NULL) { > + prefetch(rq->bio); > + prefetch(rq->rq_next); > + > + blk_update_request(rq, BLK_STS_OK, blk_rq_bytes(rq)); > + __blk_mq_end_request_acct(rq, now); If iob->need_ts is not set we don't need to call __blk_mq_end_request_acct, do we? Otherwise looks good: Reviewed-by: Christoph Hellwig <hch@lst.de>
On 10/18/21 4:18 AM, Christoph Hellwig wrote: > On Sat, Oct 16, 2021 at 08:06:20PM -0600, Jens Axboe wrote: >> Instead of calling blk_mq_end_request() on a single request, add a helper >> that takes the new struct io_comp_batch and completes any request stored >> in there. >> >> Signed-off-by: Jens Axboe <axboe@kernel.dk> >> --- >> block/blk-mq-tag.c | 6 ++++ >> block/blk-mq-tag.h | 1 + >> block/blk-mq.c | 81 ++++++++++++++++++++++++++++++++---------- >> include/linux/blk-mq.h | 29 +++++++++++++++ >> 4 files changed, 98 insertions(+), 19 deletions(-) >> >> diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c >> index c43b97201161..b94c3e8ef392 100644 >> --- a/block/blk-mq-tag.c >> +++ b/block/blk-mq-tag.c >> @@ -207,6 +207,12 @@ void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, >> } >> } >> >> +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) >> +{ >> + sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags, >> + tag_array, nr_tags); >> +} >> + >> struct bt_iter_data { >> struct blk_mq_hw_ctx *hctx; >> busy_iter_fn *fn; >> diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h >> index e617c7220626..df787b5a23bd 100644 >> --- a/block/blk-mq-tag.h >> +++ b/block/blk-mq-tag.h >> @@ -19,6 +19,7 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, >> unsigned int *offset); >> extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, >> unsigned int tag); >> +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags); >> extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, >> struct blk_mq_tags **tags, >> unsigned int depth, bool can_grow); >> diff --git a/block/blk-mq.c b/block/blk-mq.c >> index 8eb80e70e8ea..58dc0c0c24ac 100644 >> --- a/block/blk-mq.c >> +++ b/block/blk-mq.c >> @@ -292,15 +292,6 @@ void blk_mq_wake_waiters(struct request_queue *q) >> blk_mq_tag_wakeup_all(hctx->tags, true); >> } >> >> -/* >> - * Only need start/end time stamping if we have iostat or >> - * blk stats enabled, or using an IO scheduler. >> - */ >> -static inline bool blk_mq_need_time_stamp(struct request *rq) >> -{ >> - return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV)); >> -} >> - >> static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, >> unsigned int tag, u64 alloc_time_ns) >> { >> @@ -754,19 +745,21 @@ bool blk_update_request(struct request *req, blk_status_t error, >> } >> EXPORT_SYMBOL_GPL(blk_update_request); >> >> -inline void __blk_mq_end_request(struct request *rq, blk_status_t error) >> +static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) >> { >> - if (blk_mq_need_time_stamp(rq)) { >> - u64 now = ktime_get_ns(); >> + if (rq->rq_flags & RQF_STATS) { >> + blk_mq_poll_stats_start(rq->q); >> + blk_stat_add(rq, now); >> + } >> >> + blk_mq_sched_completed_request(rq, now); >> + blk_account_io_done(rq, now); >> +} >> >> - blk_mq_sched_completed_request(rq, now); >> - blk_account_io_done(rq, now); >> - } >> +inline void __blk_mq_end_request(struct request *rq, blk_status_t error) >> +{ >> + if (blk_mq_need_time_stamp(rq)) >> + __blk_mq_end_request_acct(rq, ktime_get_ns()); >> >> if (rq->end_io) { >> rq_qos_done(rq->q, rq); >> @@ -785,6 +778,56 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) >> } >> EXPORT_SYMBOL(blk_mq_end_request); >> >> +#define TAG_COMP_BATCH 32 >> + >> +static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx, >> + int *tag_array, int nr_tags) >> +{ >> + struct request_queue *q = hctx->queue; >> + >> + blk_mq_put_tags(hctx->tags, tag_array, nr_tags); >> + percpu_ref_put_many(&q->q_usage_counter, nr_tags); >> +} >> + >> +void blk_mq_end_request_batch(struct io_comp_batch *iob) >> +{ >> + int tags[TAG_COMP_BATCH], nr_tags = 0; >> + struct blk_mq_hw_ctx *last_hctx = NULL; >> + struct request *rq; >> + u64 now = 0; >> + >> + if (iob->need_ts) >> + now = ktime_get_ns(); >> + >> + while ((rq = rq_list_pop(&iob->req_list)) != NULL) { >> + prefetch(rq->bio); >> + prefetch(rq->rq_next); >> + >> + blk_update_request(rq, BLK_STS_OK, blk_rq_bytes(rq)); >> + __blk_mq_end_request_acct(rq, now); > > If iob->need_ts is not set we don't need to call > __blk_mq_end_request_acct, do we? We don't strictly need to, I'll make that change.
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index c43b97201161..b94c3e8ef392 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -207,6 +207,12 @@ void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, } } +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) +{ + sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags, + tag_array, nr_tags); +} + struct bt_iter_data { struct blk_mq_hw_ctx *hctx; busy_iter_fn *fn; diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index e617c7220626..df787b5a23bd 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -19,6 +19,7 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, unsigned int *offset); extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, unsigned int tag); +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags); extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags **tags, unsigned int depth, bool can_grow); diff --git a/block/blk-mq.c b/block/blk-mq.c index 8eb80e70e8ea..58dc0c0c24ac 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -292,15 +292,6 @@ void blk_mq_wake_waiters(struct request_queue *q) blk_mq_tag_wakeup_all(hctx->tags, true); } -/* - * Only need start/end time stamping if we have iostat or - * blk stats enabled, or using an IO scheduler. - */ -static inline bool blk_mq_need_time_stamp(struct request *rq) -{ - return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV)); -} - static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, unsigned int tag, u64 alloc_time_ns) { @@ -754,19 +745,21 @@ bool blk_update_request(struct request *req, blk_status_t error, } EXPORT_SYMBOL_GPL(blk_update_request); -inline void __blk_mq_end_request(struct request *rq, blk_status_t error) +static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) { - if (blk_mq_need_time_stamp(rq)) { - u64 now = ktime_get_ns(); + if (rq->rq_flags & RQF_STATS) { + blk_mq_poll_stats_start(rq->q); + blk_stat_add(rq, now); + } - if (rq->rq_flags & RQF_STATS) { - blk_mq_poll_stats_start(rq->q); - blk_stat_add(rq, now); - } + blk_mq_sched_completed_request(rq, now); + blk_account_io_done(rq, now); +} - blk_mq_sched_completed_request(rq, now); - blk_account_io_done(rq, now); - } +inline void __blk_mq_end_request(struct request *rq, blk_status_t error) +{ + if (blk_mq_need_time_stamp(rq)) + __blk_mq_end_request_acct(rq, ktime_get_ns()); if (rq->end_io) { rq_qos_done(rq->q, rq); @@ -785,6 +778,56 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) } EXPORT_SYMBOL(blk_mq_end_request); +#define TAG_COMP_BATCH 32 + +static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx, + int *tag_array, int nr_tags) +{ + struct request_queue *q = hctx->queue; + + blk_mq_put_tags(hctx->tags, tag_array, nr_tags); + percpu_ref_put_many(&q->q_usage_counter, nr_tags); +} + +void blk_mq_end_request_batch(struct io_comp_batch *iob) +{ + int tags[TAG_COMP_BATCH], nr_tags = 0; + struct blk_mq_hw_ctx *last_hctx = NULL; + struct request *rq; + u64 now = 0; + + if (iob->need_ts) + now = ktime_get_ns(); + + while ((rq = rq_list_pop(&iob->req_list)) != NULL) { + prefetch(rq->bio); + prefetch(rq->rq_next); + + blk_update_request(rq, BLK_STS_OK, blk_rq_bytes(rq)); + __blk_mq_end_request_acct(rq, now); + + WRITE_ONCE(rq->state, MQ_RQ_IDLE); + if (!refcount_dec_and_test(&rq->ref)) + continue; + + blk_crypto_free_request(rq); + blk_pm_mark_last_busy(rq); + rq_qos_done(rq->q, rq); + + if (nr_tags == TAG_COMP_BATCH || + (last_hctx && last_hctx != rq->mq_hctx)) { + blk_mq_flush_tag_batch(last_hctx, tags, nr_tags); + nr_tags = 0; + } + tags[nr_tags++] = rq->tag; + last_hctx = rq->mq_hctx; + } + + if (nr_tags) + blk_mq_flush_tag_batch(last_hctx, tags, nr_tags); +} +EXPORT_SYMBOL_GPL(blk_mq_end_request_batch); + static void blk_complete_reqs(struct llist_head *list) { struct llist_node *entry = llist_reverse_order(llist_del_all(list)); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 938ca6e86556..4dbf7948b0e4 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -761,6 +761,35 @@ static inline void blk_mq_set_request_complete(struct request *rq) void blk_mq_start_request(struct request *rq); void blk_mq_end_request(struct request *rq, blk_status_t error); void __blk_mq_end_request(struct request *rq, blk_status_t error); +void blk_mq_end_request_batch(struct io_comp_batch *ib); + +/* + * Only need start/end time stamping if we have iostat or + * blk stats enabled, or using an IO scheduler. + */ +static inline bool blk_mq_need_time_stamp(struct request *rq) +{ + return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV)); +} + +/* + * Batched completions only work when there is no I/O error and no special + * ->end_io handler. + */ +static inline bool blk_mq_add_to_batch(struct request *req, + struct io_comp_batch *iob, int ioerror, + void (*complete)(struct io_comp_batch *)) +{ + if (!iob || (req->rq_flags & RQF_ELV) || req->end_io || ioerror) + return false; + if (!iob->complete) + iob->complete = complete; + else if (iob->complete != complete) + return false; + iob->need_ts |= blk_mq_need_time_stamp(req); + rq_list_add(&iob->req_list, req); + return true; +} void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list); void blk_mq_kick_requeue_list(struct request_queue *q);
Instead of calling blk_mq_end_request() on a single request, add a helper that takes the new struct io_comp_batch and completes any request stored in there. Signed-off-by: Jens Axboe <axboe@kernel.dk> --- block/blk-mq-tag.c | 6 ++++ block/blk-mq-tag.h | 1 + block/blk-mq.c | 81 ++++++++++++++++++++++++++++++++---------- include/linux/blk-mq.h | 29 +++++++++++++++ 4 files changed, 98 insertions(+), 19 deletions(-)