1 /* SPDX-License-Identifier: GPL-2.0 */
17 #include <linux/blk-cgroup.h>
20 #include <linux/blk-mq.h>
28 /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
63 	/* all non-root blkcg_gq's are guaranteed to have access to parent */
75 	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];  member
130  * information per blkcg - q pair.
139 	/* the blkg and policy id this per-policy data belongs to */
146  * Policies that need to keep per-blkcg data which is independent from any
150  * cpd_init() is invoked to let each policy handle per-blkcg data.
153 	/* the blkcg and policy id this per-policy data belongs to */
164 typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
165 typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
166 typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
167 typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
168 typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
169 typedef void (blkcg_pol_stat_pd_fn)(struct blkg_policy_data *pd,
211 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
227  * bio_issue_as_root_blkg - see if this bio needs to be issued as root blkg
239 	return (bio->bi_opf & (REQ_META | REQ_SWAP)) != 0;  in bio_issue_as_root_blkg()
243  * blkg_lookup - lookup blkg for the specified blkcg - q pair
247  * Lookup blkg for the @blkcg - @q pair.
257 		return q->root_blkg;  in blkg_lookup()
259 	blkg = rcu_dereference_check(blkcg->blkg_hint,  in blkg_lookup()
260 			lockdep_is_held(&q->queue_lock));  in blkg_lookup()
261 	if (blkg && blkg->q == q)  in blkg_lookup()
264 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in blkg_lookup()
265 	if (blkg && blkg->q != q)  in blkg_lookup()
271  * blkg_to_pdata - get policy private data
275  * Return pointer to private data associated with the @blkg-@pol pair.
280 	return blkg ? blkg->pd[pol->plid] : NULL;  in blkg_to_pd()
286 	return blkcg ? blkcg->cpd[pol->plid] : NULL;  in blkcg_to_cpd()
290  * pdata_to_blkg - get blkg associated with policy private data
291  * @pd: policy private data of interest
293  * @pd is policy private data.  Determine the blkg it's associated with.
295 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)  in pd_to_blkg()  argument
297 	return pd ? pd->blkg : NULL;  in pd_to_blkg()
302 	return cpd ? cpd->blkcg : NULL;  in cpd_to_blkcg()
306  * blkg_get - get a blkg reference
313 	percpu_ref_get(&blkg->refcnt);  in blkg_get()
317  * blkg_tryget - try and get a blkg reference
325 	return blkg && percpu_ref_tryget(&blkg->refcnt);  in blkg_tryget()
329  * blkg_put - put a blkg reference
334 	percpu_ref_put(&blkg->refcnt);  in blkg_put()
338  * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
350 	css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css)	\
352 					    (p_blkg)->q)))
355  * blkg_for_each_descendant_post - post-order walk of a blkg's descendants
360  * Similar to blkg_for_each_descendant_pre() but performs post-order
365 	css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css)	\
367 					    (p_blkg)->q)))
371 	bio_issue_init(&bio->bi_issue, bio_sectors(bio));  in blkcg_bio_issue_init()
376 	if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))  in blkcg_use_delay()
378 	if (atomic_add_return(1, &blkg->use_delay) == 1)  in blkcg_use_delay()
379 		atomic_inc(&blkg->blkcg->congestion_count);  in blkcg_use_delay()
384 	int old = atomic_read(&blkg->use_delay);  in blkcg_unuse_delay()
398 	while (old && !atomic_try_cmpxchg(&blkg->use_delay, &old, old - 1))  in blkcg_unuse_delay()
404 		atomic_dec(&blkg->blkcg->congestion_count);  in blkcg_unuse_delay()
409  * blkcg_set_delay - Enable allocator delay mechanism with the specified delay amount
419 	int old = atomic_read(&blkg->use_delay);  in blkcg_set_delay()
422 	if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))  in blkcg_set_delay()
423 		atomic_inc(&blkg->blkcg->congestion_count);  in blkcg_set_delay()
425 	atomic64_set(&blkg->delay_nsec, delay);  in blkcg_set_delay()
429  * blkcg_clear_delay - Disable allocator delay mechanism
432  * Disable use_delay mechanism. See blkcg_set_delay().
436 	int old = atomic_read(&blkg->use_delay);  in blkcg_clear_delay()
439 	if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))  in blkcg_clear_delay()
440 		atomic_dec(&blkg->blkcg->congestion_count);  in blkcg_clear_delay()
444  * blk_cgroup_mergeable - Determine whether to allow or disallow merges
454 	return rq->bio->bi_blkg == bio->bi_blkg &&  in blk_cgroup_mergeable()
455 		bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);  in blk_cgroup_mergeable()
487 static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }  in pd_to_blkg()  argument
495 	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)