@@ -1145,6 +1145,10 @@ static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)
bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);
sq->nr_queued[rw]--;
+ /* Last throttled @bio is gone, clear congestion bit. */
+ if (rw && !sq->nr_queued[rw])
+ clear_wb_throttled(tg_to_blkg(tg)->wb_congested);
+
throtl_charge_bio(tg, bio);
/*
@@ -2215,6 +2219,10 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
* its @tg's disptime is not in the future.
*/
if (tg->flags & THROTL_TG_WAS_EMPTY) {
+ /* Next write will be throttled, set congestion bit. */
+ if (rw && blkg)
+ set_wb_throttled(blkg->wb_congested);
+
tg_update_disptime(tg);
throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true);
}
@@ -29,6 +29,7 @@ enum wb_state {
enum wb_congested_state {
WB_async_congested, /* The async (write) queue is getting full */
WB_sync_congested, /* The sync queue is getting full */
+ WB_write_throttled, /* Associated blkcg is throttled */
};
typedef int (congested_fn)(void *, int);
@@ -200,6 +201,16 @@ static inline void set_bdi_congested(struct backing_dev_info *bdi, int sync)
#ifdef CONFIG_CGROUP_WRITEBACK
+static inline void set_wb_throttled(struct bdi_writeback_congested *congested)
+{
+ set_bit(WB_write_throttled, &congested->state);
+}
+
+static inline void clear_wb_throttled(struct bdi_writeback_congested *congested)
+{
+ clear_bit(WB_write_throttled, &congested->state);
+}
+
/**
* wb_tryget - try to increment a wb's refcount
* @wb: bdi_writeback to get
@@ -244,6 +255,14 @@ static inline bool wb_dying(struct bdi_writeback *wb)
#else /* CONFIG_CGROUP_WRITEBACK */
+static inline void set_wb_throttled(struct bdi_writeback_congested *congested)
+{
+}
+
+static inline void clear_wb_throttled(struct bdi_writeback_congested *congested)
+{
+}
+
static inline bool wb_tryget(struct bdi_writeback *wb)
{
return true;
@@ -460,13 +460,15 @@ static inline int inode_read_congested(struct inode *inode)
static inline int inode_write_congested(struct inode *inode)
{
- return inode_congested(inode, 1 << WB_async_congested);
+ return inode_congested(inode, (1 << WB_async_congested) |
+ (1 << WB_write_throttled));
}
static inline int inode_rw_congested(struct inode *inode)
{
return inode_congested(inode, (1 << WB_sync_congested) |
- (1 << WB_async_congested));
+ (1 << WB_async_congested) |
+ (1 << WB_write_throttled));
}
static inline int bdi_congested(struct backing_dev_info *bdi, int cong_bits)
Throttler steals bio before allocating requests for them, thus throttled writeback never reaches congestion. This adds bit WB_write_throttled into per-cgroup bdi congestion control. It's set when write bandwidth limit is exceeded and throttler has at least one bio inside and cleared when last throttled bio is gone. Signed-off-by: Konstantin Khlebnikov <khlebnikov@yandex-team.ru> --- block/blk-throttle.c | 8 ++++++++ include/linux/backing-dev-defs.h | 19 +++++++++++++++++++ include/linux/backing-dev.h | 6 ++++-- 3 files changed, 31 insertions(+), 2 deletions(-)