Re: [PATCH -next v2 03/28] md: add new helpers to suspend/resume array

[Xiao Ni]

On Mon, Aug 28, 2023 at 10:04 AM Yu Kuai <yukuai1@xxxxxxxxxxxxxxx> wrote:
>
> From: Yu Kuai <yukuai3@xxxxxxxxxx>
>
> Advantages for new apis:
>  - reconfig_mutex is not required;
>  - the weird logical that suspend array hold 'reconfig_mutex' for
>    mddev_check_recovery() to update superblock is not needed;
>  - the specail handling, 'pers->prepare_suspend', for raid456 is not
>    needed;
>  - It's safe to be called at any time once mddev is allocated, and it's
>    designed to be used from slow path where array configuration is changed;
>
> Signed-off-by: Yu Kuai <yukuai3@xxxxxxxxxx>
> ---
>  drivers/md/md.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++--
>  drivers/md/md.h |  3 ++
>  2 files changed, 86 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index 7fa311a14317..6236e2e395c1 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -443,12 +443,22 @@ void mddev_suspend(struct mddev *mddev)
>                         lockdep_is_held(&mddev->reconfig_mutex));
>
>         WARN_ON_ONCE(thread && current == thread->tsk);
> -       if (mddev->suspended++)
> +
> +       /* can't concurrent with __mddev_suspend() and __mddev_resume() */
> +       mutex_lock(&mddev->suspend_mutex);
> +       if (mddev->suspended++) {
> +               mutex_unlock(&mddev->suspend_mutex);
>                 return;
> +       }
> +
>         wake_up(&mddev->sb_wait);
>         set_bit(MD_ALLOW_SB_UPDATE, &mddev->flags);
>         percpu_ref_kill(&mddev->active_io);
>
> +       /*
> +        * TODO: cleanup 'pers->prepare_suspend after all callers are replaced
> +        * by __mddev_suspend().
> +        */
>         if (mddev->pers && mddev->pers->prepare_suspend)
>                 mddev->pers->prepare_suspend(mddev);
>
> @@ -459,14 +469,21 @@ void mddev_suspend(struct mddev *mddev)
>         del_timer_sync(&mddev->safemode_timer);
>         /* restrict memory reclaim I/O during raid array is suspend */
>         mddev->noio_flag = memalloc_noio_save();
> +
> +       mutex_unlock(&mddev->suspend_mutex);
>  }
>  EXPORT_SYMBOL_GPL(mddev_suspend);
>
>  void mddev_resume(struct mddev *mddev)
>  {
>         lockdep_assert_held(&mddev->reconfig_mutex);
> -       if (--mddev->suspended)
> +
> +       /* can't concurrent with __mddev_suspend() and __mddev_resume() */
> +       mutex_lock(&mddev->suspend_mutex);
> +       if (--mddev->suspended) {
> +               mutex_unlock(&mddev->suspend_mutex);
>                 return;
> +       }
>
>         /* entred the memalloc scope from mddev_suspend() */
>         memalloc_noio_restore(mddev->noio_flag);
> @@ -477,9 +494,72 @@ void mddev_resume(struct mddev *mddev)
>         set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
>         md_wakeup_thread(mddev->thread);
>         md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
> +
> +       mutex_unlock(&mddev->suspend_mutex);
>  }
>  EXPORT_SYMBOL_GPL(mddev_resume);
>
> +void __mddev_suspend(struct mddev *mddev)
> +{
> +
> +       /*
> +        * hold reconfig_mutex to wait for normal io will deadlock, because
> +        * other context can't update super_block, and normal io can rely on
> +        * updating super_block.
> +        */
> +       lockdep_assert_not_held(&mddev->reconfig_mutex);
> +
> +       mutex_lock(&mddev->suspend_mutex);
> +
> +       if (mddev->suspended) {
> +               WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
> +               mutex_unlock(&mddev->suspend_mutex);
> +               return;
> +       }
> +
> +       percpu_ref_kill(&mddev->active_io);
> +       wait_event(mddev->sb_wait, percpu_ref_is_zero(&mddev->active_io));
> +
> +       /*
> +        * For raid456, io might be waiting for reshape to make progress,
> +        * allow new reshape to start while waiting for io to be done to
> +        * prevent deadlock.
> +        */
> +       WRITE_ONCE(mddev->suspended, mddev->suspended + 1);

It changes the order of setting suspended and checking active_io.
suspended is used to stop I/O. Now it checks active_io first and then
adds suspended, if the i/o doesn't stop, it looks like active_io can't
be 0. So it will stuck at waiting active_io to be 0?

Best Regards
Xiao

> +
> +       del_timer_sync(&mddev->safemode_timer);
> +       /* restrict memory reclaim I/O during raid array is suspend */
> +       mddev->noio_flag = memalloc_noio_save();
> +
> +       mutex_unlock(&mddev->suspend_mutex);
> +}
> +EXPORT_SYMBOL_GPL(__mddev_suspend);
> +
> +void __mddev_resume(struct mddev *mddev)
> +{
> +       lockdep_assert_not_held(&mddev->reconfig_mutex);
> +
> +       mutex_lock(&mddev->suspend_mutex);
> +       WRITE_ONCE(mddev->suspended, mddev->suspended - 1);
> +       if (mddev->suspended) {
> +               mutex_unlock(&mddev->suspend_mutex);
> +               return;
> +       }
> +
> +       /* entred the memalloc scope from __mddev_suspend() */
> +       memalloc_noio_restore(mddev->noio_flag);
> +
> +       percpu_ref_resurrect(&mddev->active_io);
> +       wake_up(&mddev->sb_wait);
> +
> +       set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
> +       md_wakeup_thread(mddev->thread);
> +       md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
> +
> +       mutex_unlock(&mddev->suspend_mutex);
> +}
> +EXPORT_SYMBOL_GPL(__mddev_resume);
> +
>  /*
>   * Generic flush handling for md
>   */
> @@ -667,6 +747,7 @@ int mddev_init(struct mddev *mddev)
>         mutex_init(&mddev->open_mutex);
>         mutex_init(&mddev->reconfig_mutex);
>         mutex_init(&mddev->sync_mutex);
> +       mutex_init(&mddev->suspend_mutex);
>         mutex_init(&mddev->bitmap_info.mutex);
>         INIT_LIST_HEAD(&mddev->disks);
>         INIT_LIST_HEAD(&mddev->all_mddevs);
> diff --git a/drivers/md/md.h b/drivers/md/md.h
> index fb3b123f16dd..1103e6b08ad9 100644
> --- a/drivers/md/md.h
> +++ b/drivers/md/md.h
> @@ -316,6 +316,7 @@ struct mddev {
>         unsigned long                   sb_flags;
>
>         int                             suspended;
> +       struct mutex                    suspend_mutex;
>         struct percpu_ref               active_io;
>         int                             ro;
>         int                             sysfs_active; /* set when sysfs deletes
> @@ -811,6 +812,8 @@ extern void md_rdev_clear(struct md_rdev *rdev);
>  extern void md_handle_request(struct mddev *mddev, struct bio *bio);
>  extern void mddev_suspend(struct mddev *mddev);
>  extern void mddev_resume(struct mddev *mddev);
> +extern void __mddev_suspend(struct mddev *mddev);
> +extern void __mddev_resume(struct mddev *mddev);
>
>  extern void md_reload_sb(struct mddev *mddev, int raid_disk);
>  extern void md_update_sb(struct mddev *mddev, int force);
> --
> 2.39.2
>