Re: [PATCH BUGFIX] block, bfq: access and cache blkg data only when safe

[Paolo Valente]

> Il giorno 05 giu 2017, alle ore 10:11, Paolo Valente <paolo.valente@xxxxxxxxxx> ha scritto:
> 
> In blk-cgroup, operations on blkg objects are protected with the
> request_queue lock. This is no more the lock that protects
> I/O-scheduler operations in blk-mq. In fact, the latter are now
> protected with a finer-grained per-scheduler-instance lock. As a
> consequence, although blkg lookups are also rcu-protected, blk-mq I/O
> schedulers may see inconsistent data when they access blkg and
> blkg-related objects. BFQ does access these objects, and does incur
> this problem, in the following case.
> 
> The blkg_lookup performed in bfq_get_queue, being protected (only)
> through rcu, may happen to return the address of a copy of the
> original blkg. If this is the case, then the blkg_get performed in
> bfq_get_queue, to pin down the blkg, is useless: it does not prevent
> blk-cgroup code from destroying both the original blkg and all objects
> directly or indirectly referred by the copy of the blkg. BFQ accesses
> these objects, which typically causes a crash for NULL-pointer
> dereference of memory-protection violation.
> 
> Some additional protection mechanism should be added to blk-cgroup to
> address this issue. In the meantime, this commit provides a quick
> temporary fix for BFQ: cache (when safe) blkg data that might
> disappear right after a blkg_lookup.
> 
> In particular, this commit exploits the following facts to achieve its
> goal without introducing further locks.  Destroy operations on a blkg
> invoke, as a first step, hooks of the scheduler associated with the
> blkg. And these hooks are executed with bfqd->lock held for BFQ. As a
> consequence, for any blkg associated with the request queue an
> instance of BFQ is attached to, we are guaranteed that such a blkg is
> not destroyed, and that all the pointers it contains are consistent,
> while that instance is holding its bfqd->lock. A blkg_lookup performed
> with bfqd->lock held then returns a fully consistent blkg, which
> remains consistent until this lock is held. In more detail, this holds
> even if the returned blkg is a copy of the original one.
> 
> Finally, also the object describing a group inside BFQ needs to be
> protected from destruction on the blkg_free of the original blkg
> (which invokes bfq_pd_free). This commit adds private refcounting for
> this object, to let it disappear only after no bfq_queue refers to it
> any longer.
> 
> This commit also removes or updates some stale comments on locking
> issues related to blk-cgroup operations.
> 
> Reported-by: Tomas Konir <tomas.konir@xxxxxxxxx>
> Reported-by: Lee Tibbert <lee.tibbert@xxxxxxxxx>
> Reported-by: Marco Piazza <mpiazza@xxxxxxxxx>
> Signed-off-by: Paolo Valente <paolo.valente@xxxxxxxxxx>
> Tested-by: Tomas Konir <tomas.konir@xxxxxxxxx>
> Tested-by: Lee Tibbert <lee.tibbert@xxxxxxxxx>
> Tested-by: Marco Piazza <mpiazza@xxxxxxxxx>

Hi Jens,
are you waiting for some further review/ack on this, or is it just in
your queue of patches to check?  Sorry for bothering you, but this bug
is causing problems to users.

Thanks,
Paolo

> ---
> block/bfq-cgroup.c  | 116 +++++++++++++++++++++++++++++++++++++++++-----------
> block/bfq-iosched.c |   2 +-
> block/bfq-iosched.h |  23 +++++------
> 3 files changed, 105 insertions(+), 36 deletions(-)
> 
> diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
> index c8a32fb..78b2e0d 100644
> --- a/block/bfq-cgroup.c
> +++ b/block/bfq-cgroup.c
> @@ -52,7 +52,7 @@ BFQG_FLAG_FNS(idling)
> BFQG_FLAG_FNS(empty)
> #undef BFQG_FLAG_FNS
> 
> -/* This should be called with the queue_lock held. */
> +/* This should be called with the scheduler lock held. */
> static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
> {
> 	unsigned long long now;
> @@ -67,7 +67,7 @@ static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
> 	bfqg_stats_clear_waiting(stats);
> }
> 
> -/* This should be called with the queue_lock held. */
> +/* This should be called with the scheduler lock held. */
> static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
> 						 struct bfq_group *curr_bfqg)
> {
> @@ -81,7 +81,7 @@ static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
> 	bfqg_stats_mark_waiting(stats);
> }
> 
> -/* This should be called with the queue_lock held. */
> +/* This should be called with the scheduler lock held. */
> static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
> {
> 	unsigned long long now;
> @@ -203,12 +203,30 @@ struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
> 
> static void bfqg_get(struct bfq_group *bfqg)
> {
> -	return blkg_get(bfqg_to_blkg(bfqg));
> +	bfqg->ref++;
> }
> 
> void bfqg_put(struct bfq_group *bfqg)
> {
> -	return blkg_put(bfqg_to_blkg(bfqg));
> +	bfqg->ref--;
> +
> +	if (bfqg->ref == 0)
> +		kfree(bfqg);
> +}
> +
> +static void bfqg_and_blkg_get(struct bfq_group *bfqg)
> +{
> +	/* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
> +	bfqg_get(bfqg);
> +
> +	blkg_get(bfqg_to_blkg(bfqg));
> +}
> +
> +void bfqg_and_blkg_put(struct bfq_group *bfqg)
> +{
> +	bfqg_put(bfqg);
> +
> +	blkg_put(bfqg_to_blkg(bfqg));
> }
> 
> void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
> @@ -312,7 +330,11 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
> 	if (bfqq) {
> 		bfqq->ioprio = bfqq->new_ioprio;
> 		bfqq->ioprio_class = bfqq->new_ioprio_class;
> -		bfqg_get(bfqg);
> +		/*
> +		 * Make sure that bfqg and its associated blkg do not
> +		 * disappear before entity.
> +		 */
> +		bfqg_and_blkg_get(bfqg);
> 	}
> 	entity->parent = bfqg->my_entity; /* NULL for root group */
> 	entity->sched_data = &bfqg->sched_data;
> @@ -399,6 +421,8 @@ struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
> 		return NULL;
> 	}
> 
> +	/* see comments in bfq_bic_update_cgroup for why refcounting */
> +	bfqg_get(bfqg);
> 	return &bfqg->pd;
> }
> 
> @@ -426,7 +450,7 @@ void bfq_pd_free(struct blkg_policy_data *pd)
> 	struct bfq_group *bfqg = pd_to_bfqg(pd);
> 
> 	bfqg_stats_exit(&bfqg->stats);
> -	return kfree(bfqg);
> +	bfqg_put(bfqg);
> }
> 
> void bfq_pd_reset_stats(struct blkg_policy_data *pd)
> @@ -496,9 +520,10 @@ struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
>  * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
>  * it on the new one.  Avoid putting the entity on the old group idle tree.
>  *
> - * Must be called under the queue lock; the cgroup owning @bfqg must
> - * not disappear (by now this just means that we are called under
> - * rcu_read_lock()).
> + * Must be called under the scheduler lock, to make sure that the blkg
> + * owning @bfqg does not disappear (see comments in
> + * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
> + * objects).
>  */
> void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
> 		   struct bfq_group *bfqg)
> @@ -519,16 +544,12 @@ void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
> 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
> 	else if (entity->on_st)
> 		bfq_put_idle_entity(bfq_entity_service_tree(entity), entity);
> -	bfqg_put(bfqq_group(bfqq));
> +	bfqg_and_blkg_put(bfqq_group(bfqq));
> 
> -	/*
> -	 * Here we use a reference to bfqg.  We don't need a refcounter
> -	 * as the cgroup reference will not be dropped, so that its
> -	 * destroy() callback will not be invoked.
> -	 */
> 	entity->parent = bfqg->my_entity;
> 	entity->sched_data = &bfqg->sched_data;
> -	bfqg_get(bfqg);
> +	/* pin down bfqg and its associated blkg  */
> +	bfqg_and_blkg_get(bfqg);
> 
> 	if (bfq_bfqq_busy(bfqq)) {
> 		bfq_pos_tree_add_move(bfqd, bfqq);
> @@ -545,8 +566,9 @@ void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
>  * @bic: the bic to move.
>  * @blkcg: the blk-cgroup to move to.
>  *
> - * Move bic to blkcg, assuming that bfqd->queue is locked; the caller
> - * has to make sure that the reference to cgroup is valid across the call.
> + * Move bic to blkcg, assuming that bfqd->lock is held; which makes
> + * sure that the reference to cgroup is valid across the call (see
> + * comments in bfq_bic_update_cgroup on this issue)
>  *
>  * NOTE: an alternative approach might have been to store the current
>  * cgroup in bfqq and getting a reference to it, reducing the lookup
> @@ -604,6 +626,57 @@ void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
> 		goto out;
> 
> 	bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio));
> +	/*
> +	 * Update blkg_path for bfq_log_* functions. We cache this
> +	 * path, and update it here, for the following
> +	 * reasons. Operations on blkg objects in blk-cgroup are
> +	 * protected with the request_queue lock, and not with the
> +	 * lock that protects the instances of this scheduler
> +	 * (bfqd->lock). This exposes BFQ to the following sort of
> +	 * race.
> +	 *
> +	 * The blkg_lookup performed in bfq_get_queue, protected
> +	 * through rcu, may happen to return the address of a copy of
> +	 * the original blkg. If this is the case, then the
> +	 * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
> +	 * the blkg, is useless: it does not prevent blk-cgroup code
> +	 * from destroying both the original blkg and all objects
> +	 * directly or indirectly referred by the copy of the
> +	 * blkg.
> +	 *
> +	 * On the bright side, destroy operations on a blkg invoke, as
> +	 * a first step, hooks of the scheduler associated with the
> +	 * blkg. And these hooks are executed with bfqd->lock held for
> +	 * BFQ. As a consequence, for any blkg associated with the
> +	 * request queue this instance of the scheduler is attached
> +	 * to, we are guaranteed that such a blkg is not destroyed, and
> +	 * that all the pointers it contains are consistent, while we
> +	 * are holding bfqd->lock. A blkg_lookup performed with
> +	 * bfqd->lock held then returns a fully consistent blkg, which
> +	 * remains consistent until this lock is held.
> +	 *
> +	 * Thanks to the last fact, and to the fact that: (1) bfqg has
> +	 * been obtained through a blkg_lookup in the above
> +	 * assignment, and (2) bfqd->lock is being held, here we can
> +	 * safely use the policy data for the involved blkg (i.e., the
> +	 * field bfqg->pd) to get to the blkg associated with bfqg,
> +	 * and then we can safely use any field of blkg. After we
> +	 * release bfqd->lock, even just getting blkg through this
> +	 * bfqg may cause dangling references to be traversed, as
> +	 * bfqg->pd may not exist any more.
> +	 *
> +	 * In view of the above facts, here we cache, in the bfqg, any
> +	 * blkg data we may need for this bic, and for its associated
> +	 * bfq_queue. As of now, we need to cache only the path of the
> +	 * blkg, which is used in the bfq_log_* functions.
> +	 *
> +	 * Finally, note that bfqg itself needs to be protected from
> +	 * destruction on the blkg_free of the original blkg (which
> +	 * invokes bfq_pd_free). We use an additional private
> +	 * refcounter for bfqg, to let it disappear only after no
> +	 * bfq_queue refers to it any longer.
> +	 */
> +	blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));
> 	bic->blkcg_serial_nr = serial_nr;
> out:
> 	rcu_read_unlock();
> @@ -640,8 +713,6 @@ static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
>  * @bfqd: the device data structure with the root group.
>  * @bfqg: the group to move from.
>  * @st: the service tree with the entities.
> - *
> - * Needs queue_lock to be taken and reference to be valid over the call.
>  */
> static void bfq_reparent_active_entities(struct bfq_data *bfqd,
> 					 struct bfq_group *bfqg,
> @@ -692,8 +763,7 @@ void bfq_pd_offline(struct blkg_policy_data *pd)
> 		/*
> 		 * The idle tree may still contain bfq_queues belonging
> 		 * to exited task because they never migrated to a different
> -		 * cgroup from the one being destroyed now.  No one else
> -		 * can access them so it's safe to act without any lock.
> +		 * cgroup from the one being destroyed now.
> 		 */
> 		bfq_flush_idle_tree(st);
> 
> diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
> index 08ce450..ed93da2 100644
> --- a/block/bfq-iosched.c
> +++ b/block/bfq-iosched.c
> @@ -3665,7 +3665,7 @@ void bfq_put_queue(struct bfq_queue *bfqq)
> 
> 	kmem_cache_free(bfq_pool, bfqq);
> #ifdef CONFIG_BFQ_GROUP_IOSCHED
> -	bfqg_put(bfqg);
> +	bfqg_and_blkg_put(bfqg);
> #endif
> }
> 
> diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
> index ae783c0..5c3bf98 100644
> --- a/block/bfq-iosched.h
> +++ b/block/bfq-iosched.h
> @@ -759,6 +759,12 @@ struct bfq_group {
> 	/* must be the first member */
> 	struct blkg_policy_data pd;
> 
> +	/* cached path for this blkg (see comments in bfq_bic_update_cgroup) */
> +	char blkg_path[128];
> +
> +	/* reference counter (see comments in bfq_bic_update_cgroup) */
> +	int ref;
> +
> 	struct bfq_entity entity;
> 	struct bfq_sched_data sched_data;
> 
> @@ -838,7 +844,7 @@ struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
> struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg);
> struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
> struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node);
> -void bfqg_put(struct bfq_group *bfqg);
> +void bfqg_and_blkg_put(struct bfq_group *bfqg);
> 
> #ifdef CONFIG_BFQ_GROUP_IOSCHED
> extern struct cftype bfq_blkcg_legacy_files[];
> @@ -910,20 +916,13 @@ void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq);
> struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
> 
> #define bfq_log_bfqq(bfqd, bfqq, fmt, args...)	do {			\
> -	char __pbuf[128];						\
> -									\
> -	blkg_path(bfqg_to_blkg(bfqq_group(bfqq)), __pbuf, sizeof(__pbuf)); \
> -	blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s " fmt, (bfqq)->pid, \
> +	blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s " fmt, (bfqq)->pid,\
> 			bfq_bfqq_sync((bfqq)) ? 'S' : 'A',		\
> -			  __pbuf, ##args);				\
> +			bfqq_group(bfqq)->blkg_path, ##args);		\
> } while (0)
> 
> -#define bfq_log_bfqg(bfqd, bfqg, fmt, args...)	do {			\
> -	char __pbuf[128];						\
> -									\
> -	blkg_path(bfqg_to_blkg(bfqg), __pbuf, sizeof(__pbuf));		\
> -	blk_add_trace_msg((bfqd)->queue, "%s " fmt, __pbuf, ##args);	\
> -} while (0)
> +#define bfq_log_bfqg(bfqd, bfqg, fmt, args...)	\
> +	blk_add_trace_msg((bfqd)->queue, "%s " fmt, (bfqg)->blkg_path, ##args)
> 
> #else /* CONFIG_BFQ_GROUP_IOSCHED */
> 
> -- 
> 2.10.0
>