@@ -582,23 +582,31 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd,
#define BFQ_LIMIT_INLINE_DEPTH 16
#ifdef CONFIG_BFQ_GROUP_IOSCHED
-static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit)
+static bool bfqq_request_over_limit(struct bfq_data *bfqd,
+ struct bfq_io_cq *bic, blk_opf_t opf,
+ unsigned int act_idx, int limit)
{
- struct bfq_data *bfqd = bfqq->bfqd;
- struct bfq_entity *entity = &bfqq->entity;
struct bfq_entity *inline_entities[BFQ_LIMIT_INLINE_DEPTH];
struct bfq_entity **entities = inline_entities;
- int depth, level, alloc_depth = BFQ_LIMIT_INLINE_DEPTH;
- int class_idx = bfqq->ioprio_class - 1;
+ int alloc_depth = BFQ_LIMIT_INLINE_DEPTH;
struct bfq_sched_data *sched_data;
+ struct bfq_entity *entity;
+ struct bfq_queue *bfqq;
unsigned long wsum;
bool ret = false;
-
- if (!entity->on_st_or_in_serv)
- return false;
+ int depth;
+ int level;
retry:
spin_lock_irq(&bfqd->lock);
+ bfqq = bic_to_bfqq(bic, op_is_sync(opf), act_idx);
+ if (!bfqq)
+ goto out;
+
+ entity = &bfqq->entity;
+ if (!entity->on_st_or_in_serv)
+ goto out;
+
/* +1 for bfqq entity, root cgroup not included */
depth = bfqg_to_blkg(bfqq_group(bfqq))->blkcg->css.cgroup->level + 1;
if (depth > alloc_depth) {
@@ -643,7 +651,7 @@ static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit)
* class.
*/
wsum = 0;
- for (i = 0; i <= class_idx; i++) {
+ for (i = 0; i <= bfqq->ioprio_class - 1; i++) {
wsum = wsum * IOPRIO_BE_NR +
sched_data->service_tree[i].wsum;
}
@@ -666,7 +674,9 @@ static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit)
return ret;
}
#else
-static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit)
+static bool bfqq_request_over_limit(struct bfq_data *bfqd,
+ struct bfq_io_cq *bic, blk_opf_t opf,
+ unsigned int act_idx, int limit)
{
return false;
}
@@ -704,8 +714,9 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data)
}
for (act_idx = 0; bic && act_idx < bfqd->num_actuators; act_idx++) {
- struct bfq_queue *bfqq =
- bic_to_bfqq(bic, op_is_sync(opf), act_idx);
+ /* Fast path to check if bfqq is already allocated. */
+ if (!bic_to_bfqq(bic, op_is_sync(opf), act_idx))
+ continue;
/*
* Does queue (or any parent entity) exceed number of
@@ -713,7 +724,7 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data)
* limit depth so that it cannot consume more
* available requests and thus starve other entities.
*/
- if (bfqq && bfqq_request_over_limit(bfqq, limit)) {
+ if (bfqq_request_over_limit(bfqd, bic, opf, act_idx, limit)) {
depth = 1;
break;
}