| Message ID | 1612367638-3794-3-git-send-email-sergei.shtepa@veeam.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | block-layer interposer | expand |
On Wed, Feb 03 2021 at 10:53am -0500, Sergei Shtepa <sergei.shtepa@veeam.com> wrote: > blk_interposer allows to intercept bio requests, remap bio to another devices or add new bios. > > Signed-off-by: Sergei Shtepa <sergei.shtepa@veeam.com> > --- > block/bio.c | 2 + > block/blk-core.c | 33 ++++++++++++++++ > block/genhd.c | 82 +++++++++++++++++++++++++++++++++++++++ > include/linux/blk_types.h | 6 ++- > include/linux/genhd.h | 18 +++++++++ > 5 files changed, 139 insertions(+), 2 deletions(-) > > diff --git a/block/bio.c b/block/bio.c > index 1f2cc1fbe283..f6f135eb84b5 100644 > --- a/block/bio.c > +++ b/block/bio.c > @@ -684,6 +684,8 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) > bio_set_flag(bio, BIO_CLONED); > if (bio_flagged(bio_src, BIO_THROTTLED)) > bio_set_flag(bio, BIO_THROTTLED); > + if (bio_flagged(bio_src, BIO_INTERPOSED)) > + bio_set_flag(bio, BIO_INTERPOSED); > bio->bi_opf = bio_src->bi_opf; > bio->bi_ioprio = bio_src->bi_ioprio; > bio->bi_write_hint = bio_src->bi_write_hint; > diff --git a/block/blk-core.c b/block/blk-core.c > index 7663a9b94b80..c84bc42ba88b 100644 > --- a/block/blk-core.c > +++ b/block/blk-core.c > @@ -1032,6 +1032,32 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) > return ret; > } > > +static blk_qc_t __submit_bio_interposed(struct bio *bio) > +{ > + struct bio_list bio_list[2] = { }; > + blk_qc_t ret = BLK_QC_T_NONE; > + > + current->bio_list = bio_list; > + if (likely(bio_queue_enter(bio) == 0)) { > + struct gendisk *disk = bio->bi_disk; > + > + if (likely(blk_has_interposer(disk))) { > + bio_set_flag(bio, BIO_INTERPOSED); > + disk->interposer->ip_submit_bio(bio); > + } else /* interposer was removed */ > + bio_list_add(¤t->bio_list[0], bio); style nit: } else { /* interposer was removed */ bio_list_add(¤t->bio_list[0], bio); } > + > + blk_queue_exit(disk->queue); > + } > + current->bio_list = NULL; > + > + /* Resubmit remaining bios */ > + while ((bio = bio_list_pop(&bio_list[0]))) > + ret = submit_bio_noacct(bio); > + > + return ret; > +} > + > /** > * submit_bio_noacct - re-submit a bio to the block device layer for I/O > * @bio: The bio describing the location in memory and on the device. > @@ -1057,6 +1083,13 @@ blk_qc_t submit_bio_noacct(struct bio *bio) > return BLK_QC_T_NONE; > } > > + /* > + * Checking the BIO_INTERPOSED flag is necessary so that the bio > + * created by the blk_interposer do not get to it for processing. > + */ > + if (blk_has_interposer(bio->bi_disk) && > + !bio_flagged(bio, BIO_INTERPOSED)) > + return __submit_bio_interposed(bio); > if (!bio->bi_disk->fops->submit_bio) > return __submit_bio_noacct_mq(bio); > return __submit_bio_noacct(bio); > diff --git a/block/genhd.c b/block/genhd.c > index 419548e92d82..39785a3ef703 100644 > --- a/block/genhd.c > +++ b/block/genhd.c > @@ -30,6 +30,7 @@ > static struct kobject *block_depr; > > DECLARE_RWSEM(bdev_lookup_sem); > +DEFINE_MUTEX(bdev_interposer_mutex); Seems you're using this mutex to protect access to disk->interposer in attach/detach. This is to prevent attach/detach races to same device? Thankfully attach/detach isn't in the bio submission fast path but it'd be helpful to document what this mutex is protecting). A storm of attach or detach will all hit this global mutex though... Mike
The 02/03/2021 11:18, Mike Snitzer wrote: > On Wed, Feb 03 2021 at 10:53am -0500, > Sergei Shtepa <sergei.shtepa@veeam.com> wrote: > > > blk_interposer allows to intercept bio requests, remap bio to another devices or add new bios. > > > > Signed-off-by: Sergei Shtepa <sergei.shtepa@veeam.com> > > --- > > block/bio.c | 2 + > > block/blk-core.c | 33 ++++++++++++++++ > > block/genhd.c | 82 +++++++++++++++++++++++++++++++++++++++ > > include/linux/blk_types.h | 6 ++- > > include/linux/genhd.h | 18 +++++++++ > > 5 files changed, 139 insertions(+), 2 deletions(-) > > > > diff --git a/block/bio.c b/block/bio.c > > index 1f2cc1fbe283..f6f135eb84b5 100644 > > --- a/block/bio.c > > +++ b/block/bio.c > > @@ -684,6 +684,8 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) > > bio_set_flag(bio, BIO_CLONED); > > if (bio_flagged(bio_src, BIO_THROTTLED)) > > bio_set_flag(bio, BIO_THROTTLED); > > + if (bio_flagged(bio_src, BIO_INTERPOSED)) > > + bio_set_flag(bio, BIO_INTERPOSED); > > bio->bi_opf = bio_src->bi_opf; > > bio->bi_ioprio = bio_src->bi_ioprio; > > bio->bi_write_hint = bio_src->bi_write_hint; > > diff --git a/block/blk-core.c b/block/blk-core.c > > index 7663a9b94b80..c84bc42ba88b 100644 > > --- a/block/blk-core.c > > +++ b/block/blk-core.c > > @@ -1032,6 +1032,32 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) > > return ret; > > } > > > > +static blk_qc_t __submit_bio_interposed(struct bio *bio) > > +{ > > + struct bio_list bio_list[2] = { }; > > + blk_qc_t ret = BLK_QC_T_NONE; > > + > > + current->bio_list = bio_list; > > + if (likely(bio_queue_enter(bio) == 0)) { > > + struct gendisk *disk = bio->bi_disk; > > + > > + if (likely(blk_has_interposer(disk))) { > > + bio_set_flag(bio, BIO_INTERPOSED); > > + disk->interposer->ip_submit_bio(bio); > > + } else /* interposer was removed */ > > + bio_list_add(¤t->bio_list[0], bio); > > style nit: > > } else { > /* interposer was removed */ > bio_list_add(¤t->bio_list[0], bio); > } > > > + > > + blk_queue_exit(disk->queue); > > + } > > + current->bio_list = NULL; > > + > > + /* Resubmit remaining bios */ > > + while ((bio = bio_list_pop(&bio_list[0]))) > > + ret = submit_bio_noacct(bio); > > + > > + return ret; > > +} > > + > > /** > > * submit_bio_noacct - re-submit a bio to the block device layer for I/O > > * @bio: The bio describing the location in memory and on the device. > > @@ -1057,6 +1083,13 @@ blk_qc_t submit_bio_noacct(struct bio *bio) > > return BLK_QC_T_NONE; > > } > > > > + /* > > + * Checking the BIO_INTERPOSED flag is necessary so that the bio > > + * created by the blk_interposer do not get to it for processing. > > + */ > > + if (blk_has_interposer(bio->bi_disk) && > > + !bio_flagged(bio, BIO_INTERPOSED)) > > + return __submit_bio_interposed(bio); > > if (!bio->bi_disk->fops->submit_bio) > > return __submit_bio_noacct_mq(bio); > > return __submit_bio_noacct(bio); > > diff --git a/block/genhd.c b/block/genhd.c > > index 419548e92d82..39785a3ef703 100644 > > --- a/block/genhd.c > > +++ b/block/genhd.c > > @@ -30,6 +30,7 @@ > > static struct kobject *block_depr; > > > > DECLARE_RWSEM(bdev_lookup_sem); > > +DEFINE_MUTEX(bdev_interposer_mutex); > > Seems you're using this mutex to protect access to disk->interposer in > attach/detach. This is to prevent attach/detach races to same device? Yes. There is a probability of 0.00...01% that two different modules will try to attach/detach to the same disk at the same time. Since the attach/detach operation is infrequent, using mutex is quite appropriate. > > Thankfully attach/detach isn't in the bio submission fast path but it'd > be helpful to document what this mutex is protecting). I'll think about the name of this mutex and add a comment. > > A storm of attach or detach will all hit this global mutex though... > > Mike > Thank you for the review. I am very interested in your opinion about [PATCH v4 4/6] and [PATCH v4 5/6]. However, the kernel test robot has already found something there on sparc.
diff --git a/block/bio.c b/block/bio.c index 1f2cc1fbe283..f6f135eb84b5 100644 --- a/block/bio.c +++ b/block/bio.c @@ -684,6 +684,8 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) bio_set_flag(bio, BIO_CLONED); if (bio_flagged(bio_src, BIO_THROTTLED)) bio_set_flag(bio, BIO_THROTTLED); + if (bio_flagged(bio_src, BIO_INTERPOSED)) + bio_set_flag(bio, BIO_INTERPOSED); bio->bi_opf = bio_src->bi_opf; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; diff --git a/block/blk-core.c b/block/blk-core.c index 7663a9b94b80..c84bc42ba88b 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1032,6 +1032,32 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) return ret; } +static blk_qc_t __submit_bio_interposed(struct bio *bio) +{ + struct bio_list bio_list[2] = { }; + blk_qc_t ret = BLK_QC_T_NONE; + + current->bio_list = bio_list; + if (likely(bio_queue_enter(bio) == 0)) { + struct gendisk *disk = bio->bi_disk; + + if (likely(blk_has_interposer(disk))) { + bio_set_flag(bio, BIO_INTERPOSED); + disk->interposer->ip_submit_bio(bio); + } else /* interposer was removed */ + bio_list_add(¤t->bio_list[0], bio); + + blk_queue_exit(disk->queue); + } + current->bio_list = NULL; + + /* Resubmit remaining bios */ + while ((bio = bio_list_pop(&bio_list[0]))) + ret = submit_bio_noacct(bio); + + return ret; +} + /** * submit_bio_noacct - re-submit a bio to the block device layer for I/O * @bio: The bio describing the location in memory and on the device. @@ -1057,6 +1083,13 @@ blk_qc_t submit_bio_noacct(struct bio *bio) return BLK_QC_T_NONE; } + /* + * Checking the BIO_INTERPOSED flag is necessary so that the bio + * created by the blk_interposer do not get to it for processing. + */ + if (blk_has_interposer(bio->bi_disk) && + !bio_flagged(bio, BIO_INTERPOSED)) + return __submit_bio_interposed(bio); if (!bio->bi_disk->fops->submit_bio) return __submit_bio_noacct_mq(bio); return __submit_bio_noacct(bio); diff --git a/block/genhd.c b/block/genhd.c index 419548e92d82..39785a3ef703 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -30,6 +30,7 @@ static struct kobject *block_depr; DECLARE_RWSEM(bdev_lookup_sem); +DEFINE_MUTEX(bdev_interposer_mutex); /* for extended dynamic devt allocation, currently only one major is used */ #define NR_EXT_DEVT (1 << MINORBITS) @@ -2148,3 +2149,84 @@ static void disk_release_events(struct gendisk *disk) WARN_ON_ONCE(disk->ev && disk->ev->block != 1); kfree(disk->ev); } + +/** + * blk_interposer_attach - Attach interposer to disk + * @disk: target disk + * @interposer: block device interposer + * @ip_submit_bio: hook for submit_bio() + * + * Returns: + * -EINVAL if @interposer is NULL. + * -EPERM if queue is not frozen. + * -EBUSY if the block device already has @interposer. + * -EALREADY if the block device already has @interposer with same callback. + * + * Disk must be frozen by blk_mq_freeze_queue(). + */ +int blk_interposer_attach(struct gendisk *disk, struct blk_interposer *interposer, + const ip_submit_bio_t ip_submit_bio) +{ + int ret = 0; + + if (WARN_ON(!interposer)) + return -EINVAL; + + if (!blk_mq_is_queue_frozen(disk->queue)) + return -EPERM; + + mutex_lock(&bdev_interposer_mutex); + if (blk_has_interposer(disk)) { + if (disk->interposer->ip_submit_bio == ip_submit_bio) + ret = -EALREADY; + else + ret = -EBUSY; + goto out; + } + + interposer->ip_submit_bio = ip_submit_bio; + interposer->disk = disk; + + disk->interposer = interposer; +out: + mutex_unlock(&bdev_interposer_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(blk_interposer_attach); + +/** + * blk_interposer_detach - Detach interposer from disk + * @interposer: block device interposer + * @ip_submit_bio: hook for submit_bio() + * + * Disk must be frozen by blk_mq_freeze_queue(). + */ +void blk_interposer_detach(struct blk_interposer *interposer, + const ip_submit_bio_t ip_submit_bio) +{ + struct gendisk *disk; + + if (WARN_ON(!interposer)) + return; + + mutex_lock(&bdev_interposer_mutex); + + /* Check if the interposer is still active. */ + disk = interposer->disk; + if (WARN_ON(!disk)) + goto out; + + if (WARN_ON(!blk_mq_is_queue_frozen(disk->queue))) + goto out; + + /* Check if it is really our interposer. */ + if (WARN_ON(disk->interposer->ip_submit_bio != ip_submit_bio)) + goto out; + + disk->interposer = NULL; + interposer->disk = NULL; +out: + mutex_unlock(&bdev_interposer_mutex); +} +EXPORT_SYMBOL_GPL(blk_interposer_detach); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 866f74261b3b..6c1351d7b73f 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -227,7 +227,7 @@ struct bio { * top bits REQ_OP. Use * accessors. */ - unsigned short bi_flags; /* status, etc and bvec pool number */ + unsigned int bi_flags; /* status, etc and bvec pool number */ unsigned short bi_ioprio; unsigned short bi_write_hint; blk_status_t bi_status; @@ -304,6 +304,8 @@ enum { * of this bio. */ BIO_CGROUP_ACCT, /* has been accounted to a cgroup */ BIO_TRACKED, /* set if bio goes through the rq_qos path */ + BIO_INTERPOSED, /* bio has been interposed and can be moved to + * a different disk */ BIO_FLAG_LAST }; @@ -322,7 +324,7 @@ enum { * freed. */ #define BVEC_POOL_BITS (3) -#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS) +#define BVEC_POOL_OFFSET (32 - BVEC_POOL_BITS) #define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET) #if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1) # error "BVEC_POOL_BITS is too small" diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 809aaa32d53c..f68c8e83b4f1 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -134,6 +134,13 @@ struct blk_integrity { unsigned char tag_size; }; +typedef void (*ip_submit_bio_t) (struct bio *bio); + +struct blk_interposer { + ip_submit_bio_t ip_submit_bio; + struct gendisk *disk; +}; + struct gendisk { /* major, first_minor and minors are input parameters only, * don't use directly. Use disk_devt() and disk_max_parts(). @@ -158,6 +165,7 @@ struct gendisk { const struct block_device_operations *fops; struct request_queue *queue; + struct blk_interposer *interposer; void *private_data; int flags; @@ -346,4 +354,14 @@ static inline void printk_all_partitions(void) } #endif /* CONFIG_BLOCK */ +/* + * block layer interposer + */ +#define blk_has_interposer(d) ((d)->interposer != NULL) + +int blk_interposer_attach(struct gendisk *disk, struct blk_interposer *interposer, + const ip_submit_bio_t ip_submit_bio); +void blk_interposer_detach(struct blk_interposer *interposer, + const ip_submit_bio_t ip_submit_bio); + #endif /* _LINUX_GENHD_H */
blk_interposer allows to intercept bio requests, remap bio to another devices or add new bios. Signed-off-by: Sergei Shtepa <sergei.shtepa@veeam.com> --- block/bio.c | 2 + block/blk-core.c | 33 ++++++++++++++++ block/genhd.c | 82 +++++++++++++++++++++++++++++++++++++++ include/linux/blk_types.h | 6 ++- include/linux/genhd.h | 18 +++++++++ 5 files changed, 139 insertions(+), 2 deletions(-)