1 /* SPDX-License-Identifier: GPL-2.0
2  *
3  * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
4  * Do not use in new code.
5  */
6 #ifndef _BLK_CGROUP_RWSTAT_H
7 #define _BLK_CGROUP_RWSTAT_H
8 
9 #include "blk-cgroup.h"
10 
11 enum blkg_rwstat_type {
12 	BLKG_RWSTAT_READ,
13 	BLKG_RWSTAT_WRITE,
14 	BLKG_RWSTAT_SYNC,
15 	BLKG_RWSTAT_ASYNC,
16 	BLKG_RWSTAT_DISCARD,
17 
18 	BLKG_RWSTAT_NR,
19 	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
20 };
21 
22 /*
23  * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
24  * recursive.  Used to carry stats of dead children.
25  */
26 struct blkg_rwstat {
27 	struct percpu_counter		cpu_cnt[BLKG_RWSTAT_NR];
28 	atomic64_t			aux_cnt[BLKG_RWSTAT_NR];
29 };
30 
31 struct blkg_rwstat_sample {
32 	u64				cnt[BLKG_RWSTAT_NR];
33 };
34 
blkg_rwstat_read_counter(struct blkg_rwstat * rwstat,unsigned int idx)35 static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
36 		unsigned int idx)
37 {
38 	return atomic64_read(&rwstat->aux_cnt[idx]) +
39 		percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
40 }
41 
42 int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
43 void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
44 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
45 			 const struct blkg_rwstat_sample *rwstat);
46 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
47 		       int off);
48 void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
49 		int off, struct blkg_rwstat_sample *sum);
50 
51 
52 /**
53  * blkg_rwstat_add - add a value to a blkg_rwstat
54  * @rwstat: target blkg_rwstat
55  * @op: REQ_OP and flags
56  * @val: value to add
57  *
58  * Add @val to @rwstat.  The counters are chosen according to @rw.  The
59  * caller is responsible for synchronizing calls to this function.
60  */
blkg_rwstat_add(struct blkg_rwstat * rwstat,blk_opf_t opf,uint64_t val)61 static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
62 				   blk_opf_t opf, uint64_t val)
63 {
64 	struct percpu_counter *cnt;
65 
66 	if (op_is_discard(opf))
67 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
68 	else if (op_is_write(opf))
69 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
70 	else
71 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
72 
73 	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
74 
75 	if (op_is_sync(opf))
76 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
77 	else
78 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
79 
80 	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
81 }
82 
83 /**
84  * blkg_rwstat_read - read the current values of a blkg_rwstat
85  * @rwstat: blkg_rwstat to read
86  *
87  * Read the current snapshot of @rwstat and return it in the aux counts.
88  */
blkg_rwstat_read(struct blkg_rwstat * rwstat,struct blkg_rwstat_sample * result)89 static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
90 		struct blkg_rwstat_sample *result)
91 {
92 	int i;
93 
94 	for (i = 0; i < BLKG_RWSTAT_NR; i++)
95 		result->cnt[i] =
96 			percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
97 }
98 
99 /**
100  * blkg_rwstat_total - read the total count of a blkg_rwstat
101  * @rwstat: blkg_rwstat to read
102  *
103  * Return the total count of @rwstat regardless of the IO direction.  This
104  * function can be called without synchronization and takes care of u64
105  * atomicity.
106  */
blkg_rwstat_total(struct blkg_rwstat * rwstat)107 static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
108 {
109 	struct blkg_rwstat_sample tmp = { };
110 
111 	blkg_rwstat_read(rwstat, &tmp);
112 	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
113 }
114 
115 /**
116  * blkg_rwstat_reset - reset a blkg_rwstat
117  * @rwstat: blkg_rwstat to reset
118  */
blkg_rwstat_reset(struct blkg_rwstat * rwstat)119 static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
120 {
121 	int i;
122 
123 	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
124 		percpu_counter_set(&rwstat->cpu_cnt[i], 0);
125 		atomic64_set(&rwstat->aux_cnt[i], 0);
126 	}
127 }
128 
129 /**
130  * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
131  * @to: the destination blkg_rwstat
132  * @from: the source
133  *
134  * Add @from's count including the aux one to @to's aux count.
135  */
blkg_rwstat_add_aux(struct blkg_rwstat * to,struct blkg_rwstat * from)136 static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
137 				       struct blkg_rwstat *from)
138 {
139 	u64 sum[BLKG_RWSTAT_NR];
140 	int i;
141 
142 	for (i = 0; i < BLKG_RWSTAT_NR; i++)
143 		sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
144 
145 	for (i = 0; i < BLKG_RWSTAT_NR; i++)
146 		atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
147 			     &to->aux_cnt[i]);
148 }
149 #endif	/* _BLK_CGROUP_RWSTAT_H */
150