@@ -92,6 +92,8 @@ struct loop_cmd {
#define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ)
#define LOOP_DEFAULT_HW_Q_DEPTH 128
+static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd);
+
static DEFINE_IDR(loop_index_idr);
static DEFINE_MUTEX(loop_ctl_mutex);
static DEFINE_MUTEX(loop_validate_mutex);
@@ -380,8 +382,17 @@ static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
if (!atomic_dec_and_test(&cmd->ref))
return;
+
+ if (cmd->ret == -EAGAIN) {
+ struct loop_device *lo = rq->q->queuedata;
+
+ loop_queue_work(lo, cmd);
+ return;
+ }
+
kfree(cmd->bvec);
cmd->bvec = NULL;
+
if (likely(!blk_should_fake_timeout(rq->q)))
blk_mq_complete_request(rq);
}
@@ -478,16 +489,34 @@ static int lo_rw_aio_prep(struct loop_device *lo, struct loop_cmd *cmd,
}
static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, loff_t pos)
+{
+ unsigned int nr_bvec = lo_cmd_nr_bvec(cmd);
+ int ret;
+
+ cmd->iocb.ki_flags &= ~IOCB_NOWAIT;
+ ret = lo_submit_rw_aio(lo, cmd, pos, nr_bvec);
+ if (ret != -EIOCBQUEUED)
+ lo_rw_aio_complete(&cmd->iocb, ret);
+ return 0;
+}
+
+static int lo_rw_aio_nowait(struct loop_device *lo, struct loop_cmd *cmd, loff_t pos)
{
unsigned int nr_bvec = lo_cmd_nr_bvec(cmd);
int ret = lo_rw_aio_prep(lo, cmd, nr_bvec);
if (ret < 0)
return ret;
+
+ cmd->iocb.ki_flags |= IOCB_NOWAIT;
ret = lo_submit_rw_aio(lo, cmd, pos, nr_bvec);
- if (ret != -EIOCBQUEUED)
+ if (ret == -EIOCBQUEUED)
+ return 0;
+ if (ret != -EAGAIN) {
lo_rw_aio_complete(&cmd->iocb, ret);
- return 0;
+ return 0;
+ }
+ return ret;
}
static int do_req_filebacked(struct loop_device *lo, struct request *rq)
@@ -1926,6 +1955,17 @@ static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
break;
}
+ if (cmd->use_aio) {
+ loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
+ int ret = lo_rw_aio_nowait(lo, cmd, pos);
+
+ if (!ret)
+ return BLK_STS_OK;
+ if (ret != -EAGAIN)
+ return BLK_STS_IOERR;
+ /* fallback to workqueue for handling aio */
+ }
+
loop_queue_work(lo, cmd);
return BLK_STS_OK;
@@ -2076,7 +2116,8 @@ static int loop_add(int i)
lo->tag_set.queue_depth = hw_queue_depth;
lo->tag_set.numa_node = NUMA_NO_NODE;
lo->tag_set.cmd_size = sizeof(struct loop_cmd);
- lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT;
+ lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT |
+ BLK_MQ_F_BLOCKING;
lo->tag_set.driver_data = lo;
err = blk_mq_alloc_tag_set(&lo->tag_set);
Try to handle loop aio command via NOWAIT IO first, then we can avoid to queue the aio command into workqueue. Fallback to workqueue in case of -EAGAIN. BLK_MQ_F_BLOCKING has to be set for calling into .read_iter() or .write_iter() which might sleep even though it is NOWAIT. Signed-off-by: Ming Lei <ming.lei@redhat.com> --- drivers/block/loop.c | 47 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-)