| Message ID | 20230828020021.2489641-4-yukuai1@huaweicloud.com (mailing list archive) |
|---|---|
| State | New, archived |
| Delegated to: | Song Liu |
| Headers | show |
| Series | md: synchronize io with array reconfiguration | expand |
On Mon, Aug 28, 2023 at 10:04 AM Yu Kuai <yukuai1@huaweicloud.com> wrote: > > From: Yu Kuai <yukuai3@huawei.com> > > Advantages for new apis: > - reconfig_mutex is not required; > - the weird logical that suspend array hold 'reconfig_mutex' for > mddev_check_recovery() to update superblock is not needed; > - the specail handling, 'pers->prepare_suspend', for raid456 is not > needed; > - It's safe to be called at any time once mddev is allocated, and it's > designed to be used from slow path where array configuration is changed; > > Signed-off-by: Yu Kuai <yukuai3@huawei.com> > --- > drivers/md/md.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++-- > drivers/md/md.h | 3 ++ > 2 files changed, 86 insertions(+), 2 deletions(-) > > diff --git a/drivers/md/md.c b/drivers/md/md.c > index 7fa311a14317..6236e2e395c1 100644 > --- a/drivers/md/md.c > +++ b/drivers/md/md.c > @@ -443,12 +443,22 @@ void mddev_suspend(struct mddev *mddev) > lockdep_is_held(&mddev->reconfig_mutex)); > > WARN_ON_ONCE(thread && current == thread->tsk); > - if (mddev->suspended++) > + > + /* can't concurrent with __mddev_suspend() and __mddev_resume() */ > + mutex_lock(&mddev->suspend_mutex); > + if (mddev->suspended++) { > + mutex_unlock(&mddev->suspend_mutex); > return; > + } > + > wake_up(&mddev->sb_wait); > set_bit(MD_ALLOW_SB_UPDATE, &mddev->flags); > percpu_ref_kill(&mddev->active_io); > > + /* > + * TODO: cleanup 'pers->prepare_suspend after all callers are replaced > + * by __mddev_suspend(). > + */ > if (mddev->pers && mddev->pers->prepare_suspend) > mddev->pers->prepare_suspend(mddev); > > @@ -459,14 +469,21 @@ void mddev_suspend(struct mddev *mddev) > del_timer_sync(&mddev->safemode_timer); > /* restrict memory reclaim I/O during raid array is suspend */ > mddev->noio_flag = memalloc_noio_save(); > + > + mutex_unlock(&mddev->suspend_mutex); > } > EXPORT_SYMBOL_GPL(mddev_suspend); > > void mddev_resume(struct mddev *mddev) > { > lockdep_assert_held(&mddev->reconfig_mutex); > - if (--mddev->suspended) > + > + /* can't concurrent with __mddev_suspend() and __mddev_resume() */ > + mutex_lock(&mddev->suspend_mutex); > + if (--mddev->suspended) { > + mutex_unlock(&mddev->suspend_mutex); > return; > + } > > /* entred the memalloc scope from mddev_suspend() */ > memalloc_noio_restore(mddev->noio_flag); > @@ -477,9 +494,72 @@ void mddev_resume(struct mddev *mddev) > set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); > md_wakeup_thread(mddev->thread); > md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */ > + > + mutex_unlock(&mddev->suspend_mutex); > } > EXPORT_SYMBOL_GPL(mddev_resume); > > +void __mddev_suspend(struct mddev *mddev) > +{ > + > + /* > + * hold reconfig_mutex to wait for normal io will deadlock, because > + * other context can't update super_block, and normal io can rely on > + * updating super_block. > + */ > + lockdep_assert_not_held(&mddev->reconfig_mutex); > + > + mutex_lock(&mddev->suspend_mutex); > + > + if (mddev->suspended) { > + WRITE_ONCE(mddev->suspended, mddev->suspended + 1); > + mutex_unlock(&mddev->suspend_mutex); > + return; > + } > + > + percpu_ref_kill(&mddev->active_io); > + wait_event(mddev->sb_wait, percpu_ref_is_zero(&mddev->active_io)); > + > + /* > + * For raid456, io might be waiting for reshape to make progress, > + * allow new reshape to start while waiting for io to be done to > + * prevent deadlock. > + */ > + WRITE_ONCE(mddev->suspended, mddev->suspended + 1); It changes the order of setting suspended and checking active_io. suspended is used to stop I/O. Now it checks active_io first and then adds suspended, if the i/o doesn't stop, it looks like active_io can't be 0. So it will stuck at waiting active_io to be 0? Best Regards Xiao > + > + del_timer_sync(&mddev->safemode_timer); > + /* restrict memory reclaim I/O during raid array is suspend */ > + mddev->noio_flag = memalloc_noio_save(); > + > + mutex_unlock(&mddev->suspend_mutex); > +} > +EXPORT_SYMBOL_GPL(__mddev_suspend); > + > +void __mddev_resume(struct mddev *mddev) > +{ > + lockdep_assert_not_held(&mddev->reconfig_mutex); > + > + mutex_lock(&mddev->suspend_mutex); > + WRITE_ONCE(mddev->suspended, mddev->suspended - 1); > + if (mddev->suspended) { > + mutex_unlock(&mddev->suspend_mutex); > + return; > + } > + > + /* entred the memalloc scope from __mddev_suspend() */ > + memalloc_noio_restore(mddev->noio_flag); > + > + percpu_ref_resurrect(&mddev->active_io); > + wake_up(&mddev->sb_wait); > + > + set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); > + md_wakeup_thread(mddev->thread); > + md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */ > + > + mutex_unlock(&mddev->suspend_mutex); > +} > +EXPORT_SYMBOL_GPL(__mddev_resume); > + > /* > * Generic flush handling for md > */ > @@ -667,6 +747,7 @@ int mddev_init(struct mddev *mddev) > mutex_init(&mddev->open_mutex); > mutex_init(&mddev->reconfig_mutex); > mutex_init(&mddev->sync_mutex); > + mutex_init(&mddev->suspend_mutex); > mutex_init(&mddev->bitmap_info.mutex); > INIT_LIST_HEAD(&mddev->disks); > INIT_LIST_HEAD(&mddev->all_mddevs); > diff --git a/drivers/md/md.h b/drivers/md/md.h > index fb3b123f16dd..1103e6b08ad9 100644 > --- a/drivers/md/md.h > +++ b/drivers/md/md.h > @@ -316,6 +316,7 @@ struct mddev { > unsigned long sb_flags; > > int suspended; > + struct mutex suspend_mutex; > struct percpu_ref active_io; > int ro; > int sysfs_active; /* set when sysfs deletes > @@ -811,6 +812,8 @@ extern void md_rdev_clear(struct md_rdev *rdev); > extern void md_handle_request(struct mddev *mddev, struct bio *bio); > extern void mddev_suspend(struct mddev *mddev); > extern void mddev_resume(struct mddev *mddev); > +extern void __mddev_suspend(struct mddev *mddev); > +extern void __mddev_resume(struct mddev *mddev); > > extern void md_reload_sb(struct mddev *mddev, int raid_disk); > extern void md_update_sb(struct mddev *mddev, int force); > -- > 2.39.2 >
On Mon, Sep 25, 2023 at 3:21 PM Xiao Ni <xni@redhat.com> wrote: > > On Mon, Aug 28, 2023 at 10:04 AM Yu Kuai <yukuai1@huaweicloud.com> wrote: > > > > From: Yu Kuai <yukuai3@huawei.com> > > > > Advantages for new apis: > > - reconfig_mutex is not required; > > - the weird logical that suspend array hold 'reconfig_mutex' for > > mddev_check_recovery() to update superblock is not needed; > > - the specail handling, 'pers->prepare_suspend', for raid456 is not > > needed; > > - It's safe to be called at any time once mddev is allocated, and it's > > designed to be used from slow path where array configuration is changed; > > > > Signed-off-by: Yu Kuai <yukuai3@huawei.com> > > --- > > drivers/md/md.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++-- > > drivers/md/md.h | 3 ++ > > 2 files changed, 86 insertions(+), 2 deletions(-) > > > > diff --git a/drivers/md/md.c b/drivers/md/md.c > > index 7fa311a14317..6236e2e395c1 100644 > > --- a/drivers/md/md.c > > +++ b/drivers/md/md.c > > @@ -443,12 +443,22 @@ void mddev_suspend(struct mddev *mddev) > > lockdep_is_held(&mddev->reconfig_mutex)); > > > > WARN_ON_ONCE(thread && current == thread->tsk); > > - if (mddev->suspended++) > > + > > + /* can't concurrent with __mddev_suspend() and __mddev_resume() */ > > + mutex_lock(&mddev->suspend_mutex); > > + if (mddev->suspended++) { > > + mutex_unlock(&mddev->suspend_mutex); > > return; > > + } > > + > > wake_up(&mddev->sb_wait); > > set_bit(MD_ALLOW_SB_UPDATE, &mddev->flags); > > percpu_ref_kill(&mddev->active_io); > > > > + /* > > + * TODO: cleanup 'pers->prepare_suspend after all callers are replaced > > + * by __mddev_suspend(). > > + */ > > if (mddev->pers && mddev->pers->prepare_suspend) > > mddev->pers->prepare_suspend(mddev); > > > > @@ -459,14 +469,21 @@ void mddev_suspend(struct mddev *mddev) > > del_timer_sync(&mddev->safemode_timer); > > /* restrict memory reclaim I/O during raid array is suspend */ > > mddev->noio_flag = memalloc_noio_save(); > > + > > + mutex_unlock(&mddev->suspend_mutex); > > } > > EXPORT_SYMBOL_GPL(mddev_suspend); > > > > void mddev_resume(struct mddev *mddev) > > { > > lockdep_assert_held(&mddev->reconfig_mutex); > > - if (--mddev->suspended) > > + > > + /* can't concurrent with __mddev_suspend() and __mddev_resume() */ > > + mutex_lock(&mddev->suspend_mutex); > > + if (--mddev->suspended) { > > + mutex_unlock(&mddev->suspend_mutex); > > return; > > + } > > > > /* entred the memalloc scope from mddev_suspend() */ > > memalloc_noio_restore(mddev->noio_flag); > > @@ -477,9 +494,72 @@ void mddev_resume(struct mddev *mddev) > > set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); > > md_wakeup_thread(mddev->thread); > > md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */ > > + > > + mutex_unlock(&mddev->suspend_mutex); > > } > > EXPORT_SYMBOL_GPL(mddev_resume); > > > > +void __mddev_suspend(struct mddev *mddev) > > +{ > > + > > + /* > > + * hold reconfig_mutex to wait for normal io will deadlock, because > > + * other context can't update super_block, and normal io can rely on > > + * updating super_block. > > + */ > > + lockdep_assert_not_held(&mddev->reconfig_mutex); > > + > > + mutex_lock(&mddev->suspend_mutex); > > + > > + if (mddev->suspended) { > > + WRITE_ONCE(mddev->suspended, mddev->suspended + 1); > > + mutex_unlock(&mddev->suspend_mutex); > > + return; > > + } > > + > > + percpu_ref_kill(&mddev->active_io); > > + wait_event(mddev->sb_wait, percpu_ref_is_zero(&mddev->active_io)); > > + > > + /* > > + * For raid456, io might be waiting for reshape to make progress, > > + * allow new reshape to start while waiting for io to be done to > > + * prevent deadlock. > > + */ > > + WRITE_ONCE(mddev->suspended, mddev->suspended + 1); > > It changes the order of setting suspended and checking active_io. > suspended is used to stop I/O. Now it checks active_io first and then > adds suspended, if the i/o doesn't stop, it looks like active_io can't > be 0. So it will stuck at waiting active_io to be 0? Ah, I c, you add the state of active_io to judge if a raid is suspended. Regards Xiao > > Best Regards > Xiao > > > + > > + del_timer_sync(&mddev->safemode_timer); > > + /* restrict memory reclaim I/O during raid array is suspend */ > > + mddev->noio_flag = memalloc_noio_save(); > > + > > + mutex_unlock(&mddev->suspend_mutex); > > +} > > +EXPORT_SYMBOL_GPL(__mddev_suspend); > > + > > +void __mddev_resume(struct mddev *mddev) > > +{ > > + lockdep_assert_not_held(&mddev->reconfig_mutex); > > + > > + mutex_lock(&mddev->suspend_mutex); > > + WRITE_ONCE(mddev->suspended, mddev->suspended - 1); > > + if (mddev->suspended) { > > + mutex_unlock(&mddev->suspend_mutex); > > + return; > > + } > > + > > + /* entred the memalloc scope from __mddev_suspend() */ > > + memalloc_noio_restore(mddev->noio_flag); > > + > > + percpu_ref_resurrect(&mddev->active_io); > > + wake_up(&mddev->sb_wait); > > + > > + set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); > > + md_wakeup_thread(mddev->thread); > > + md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */ > > + > > + mutex_unlock(&mddev->suspend_mutex); > > +} > > +EXPORT_SYMBOL_GPL(__mddev_resume); > > + > > /* > > * Generic flush handling for md > > */ > > @@ -667,6 +747,7 @@ int mddev_init(struct mddev *mddev) > > mutex_init(&mddev->open_mutex); > > mutex_init(&mddev->reconfig_mutex); > > mutex_init(&mddev->sync_mutex); > > + mutex_init(&mddev->suspend_mutex); > > mutex_init(&mddev->bitmap_info.mutex); > > INIT_LIST_HEAD(&mddev->disks); > > INIT_LIST_HEAD(&mddev->all_mddevs); > > diff --git a/drivers/md/md.h b/drivers/md/md.h > > index fb3b123f16dd..1103e6b08ad9 100644 > > --- a/drivers/md/md.h > > +++ b/drivers/md/md.h > > @@ -316,6 +316,7 @@ struct mddev { > > unsigned long sb_flags; > > > > int suspended; > > + struct mutex suspend_mutex; > > struct percpu_ref active_io; > > int ro; > > int sysfs_active; /* set when sysfs deletes > > @@ -811,6 +812,8 @@ extern void md_rdev_clear(struct md_rdev *rdev); > > extern void md_handle_request(struct mddev *mddev, struct bio *bio); > > extern void mddev_suspend(struct mddev *mddev); > > extern void mddev_resume(struct mddev *mddev); > > +extern void __mddev_suspend(struct mddev *mddev); > > +extern void __mddev_resume(struct mddev *mddev); > > > > extern void md_reload_sb(struct mddev *mddev, int raid_disk); > > extern void md_update_sb(struct mddev *mddev, int force); > > -- > > 2.39.2 > >
diff --git a/drivers/md/md.c b/drivers/md/md.c index 7fa311a14317..6236e2e395c1 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -443,12 +443,22 @@ void mddev_suspend(struct mddev *mddev) lockdep_is_held(&mddev->reconfig_mutex)); WARN_ON_ONCE(thread && current == thread->tsk); - if (mddev->suspended++) + + /* can't concurrent with __mddev_suspend() and __mddev_resume() */ + mutex_lock(&mddev->suspend_mutex); + if (mddev->suspended++) { + mutex_unlock(&mddev->suspend_mutex); return; + } + wake_up(&mddev->sb_wait); set_bit(MD_ALLOW_SB_UPDATE, &mddev->flags); percpu_ref_kill(&mddev->active_io); + /* + * TODO: cleanup 'pers->prepare_suspend after all callers are replaced + * by __mddev_suspend(). + */ if (mddev->pers && mddev->pers->prepare_suspend) mddev->pers->prepare_suspend(mddev); @@ -459,14 +469,21 @@ void mddev_suspend(struct mddev *mddev) del_timer_sync(&mddev->safemode_timer); /* restrict memory reclaim I/O during raid array is suspend */ mddev->noio_flag = memalloc_noio_save(); + + mutex_unlock(&mddev->suspend_mutex); } EXPORT_SYMBOL_GPL(mddev_suspend); void mddev_resume(struct mddev *mddev) { lockdep_assert_held(&mddev->reconfig_mutex); - if (--mddev->suspended) + + /* can't concurrent with __mddev_suspend() and __mddev_resume() */ + mutex_lock(&mddev->suspend_mutex); + if (--mddev->suspended) { + mutex_unlock(&mddev->suspend_mutex); return; + } /* entred the memalloc scope from mddev_suspend() */ memalloc_noio_restore(mddev->noio_flag); @@ -477,9 +494,72 @@ void mddev_resume(struct mddev *mddev) set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); md_wakeup_thread(mddev->thread); md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */ + + mutex_unlock(&mddev->suspend_mutex); } EXPORT_SYMBOL_GPL(mddev_resume); +void __mddev_suspend(struct mddev *mddev) +{ + + /* + * hold reconfig_mutex to wait for normal io will deadlock, because + * other context can't update super_block, and normal io can rely on + * updating super_block. + */ + lockdep_assert_not_held(&mddev->reconfig_mutex); + + mutex_lock(&mddev->suspend_mutex); + + if (mddev->suspended) { + WRITE_ONCE(mddev->suspended, mddev->suspended + 1); + mutex_unlock(&mddev->suspend_mutex); + return; + } + + percpu_ref_kill(&mddev->active_io); + wait_event(mddev->sb_wait, percpu_ref_is_zero(&mddev->active_io)); + + /* + * For raid456, io might be waiting for reshape to make progress, + * allow new reshape to start while waiting for io to be done to + * prevent deadlock. + */ + WRITE_ONCE(mddev->suspended, mddev->suspended + 1); + + del_timer_sync(&mddev->safemode_timer); + /* restrict memory reclaim I/O during raid array is suspend */ + mddev->noio_flag = memalloc_noio_save(); + + mutex_unlock(&mddev->suspend_mutex); +} +EXPORT_SYMBOL_GPL(__mddev_suspend); + +void __mddev_resume(struct mddev *mddev) +{ + lockdep_assert_not_held(&mddev->reconfig_mutex); + + mutex_lock(&mddev->suspend_mutex); + WRITE_ONCE(mddev->suspended, mddev->suspended - 1); + if (mddev->suspended) { + mutex_unlock(&mddev->suspend_mutex); + return; + } + + /* entred the memalloc scope from __mddev_suspend() */ + memalloc_noio_restore(mddev->noio_flag); + + percpu_ref_resurrect(&mddev->active_io); + wake_up(&mddev->sb_wait); + + set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); + md_wakeup_thread(mddev->thread); + md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */ + + mutex_unlock(&mddev->suspend_mutex); +} +EXPORT_SYMBOL_GPL(__mddev_resume); + /* * Generic flush handling for md */ @@ -667,6 +747,7 @@ int mddev_init(struct mddev *mddev) mutex_init(&mddev->open_mutex); mutex_init(&mddev->reconfig_mutex); mutex_init(&mddev->sync_mutex); + mutex_init(&mddev->suspend_mutex); mutex_init(&mddev->bitmap_info.mutex); INIT_LIST_HEAD(&mddev->disks); INIT_LIST_HEAD(&mddev->all_mddevs); diff --git a/drivers/md/md.h b/drivers/md/md.h index fb3b123f16dd..1103e6b08ad9 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -316,6 +316,7 @@ struct mddev { unsigned long sb_flags; int suspended; + struct mutex suspend_mutex; struct percpu_ref active_io; int ro; int sysfs_active; /* set when sysfs deletes @@ -811,6 +812,8 @@ extern void md_rdev_clear(struct md_rdev *rdev); extern void md_handle_request(struct mddev *mddev, struct bio *bio); extern void mddev_suspend(struct mddev *mddev); extern void mddev_resume(struct mddev *mddev); +extern void __mddev_suspend(struct mddev *mddev); +extern void __mddev_resume(struct mddev *mddev); extern void md_reload_sb(struct mddev *mddev, int raid_disk); extern void md_update_sb(struct mddev *mddev, int force);