1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* delayacct.h - per-task delay accounting
3  *
4  * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5  */
6 
7 #ifndef _LINUX_DELAYACCT_H
8 #define _LINUX_DELAYACCT_H
9 
10 #include <uapi/linux/taskstats.h>
11 
12 #ifdef CONFIG_TASK_DELAY_ACCT
13 struct task_delay_info {
14 	raw_spinlock_t	lock;
15 
16 	/* For each stat XXX, add following, aligned appropriately
17 	 *
18 	 * struct timespec XXX_start, XXX_end;
19 	 * u64 XXX_delay;
20 	 * u32 XXX_count;
21 	 *
22 	 * Atomicity of updates to XXX_delay, XXX_count protected by
23 	 * single lock above (split into XXX_lock if contention is an issue).
24 	 */
25 
26 	/*
27 	 * XXX_count is incremented on every XXX operation, the delay
28 	 * associated with the operation is added to XXX_delay.
29 	 * XXX_delay contains the accumulated delay time in nanoseconds.
30 	 */
31 	u64 blkio_start;
32 	u64 blkio_delay;	/* wait for sync block io completion */
33 	u64 swapin_start;
34 	u64 swapin_delay;	/* wait for swapin */
35 	u32 blkio_count;	/* total count of the number of sync block */
36 				/* io operations performed */
37 	u32 swapin_count;	/* total count of swapin */
38 
39 	u64 freepages_start;
40 	u64 freepages_delay;	/* wait for memory reclaim */
41 
42 	u64 thrashing_start;
43 	u64 thrashing_delay;	/* wait for thrashing page */
44 
45 	u64 compact_start;
46 	u64 compact_delay;	/* wait for memory compact */
47 
48 	u64 wpcopy_start;
49 	u64 wpcopy_delay;	/* wait for write-protect copy */
50 
51 	u32 freepages_count;	/* total count of memory reclaim */
52 	u32 thrashing_count;	/* total count of thrash waits */
53 	u32 compact_count;	/* total count of memory compact */
54 	u32 wpcopy_count;	/* total count of write-protect copy */
55 };
56 #endif
57 
58 #include <linux/sched.h>
59 #include <linux/slab.h>
60 #include <linux/jump_label.h>
61 
62 #ifdef CONFIG_TASK_DELAY_ACCT
63 DECLARE_STATIC_KEY_FALSE(delayacct_key);
64 extern int delayacct_on;	/* Delay accounting turned on/off */
65 extern struct kmem_cache *delayacct_cache;
66 extern void delayacct_init(void);
67 
68 extern void __delayacct_tsk_init(struct task_struct *);
69 extern void __delayacct_tsk_exit(struct task_struct *);
70 extern void __delayacct_blkio_start(void);
71 extern void __delayacct_blkio_end(struct task_struct *);
72 extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
73 extern __u64 __delayacct_blkio_ticks(struct task_struct *);
74 extern void __delayacct_freepages_start(void);
75 extern void __delayacct_freepages_end(void);
76 extern void __delayacct_thrashing_start(bool *in_thrashing);
77 extern void __delayacct_thrashing_end(bool *in_thrashing);
78 extern void __delayacct_swapin_start(void);
79 extern void __delayacct_swapin_end(void);
80 extern void __delayacct_compact_start(void);
81 extern void __delayacct_compact_end(void);
82 extern void __delayacct_wpcopy_start(void);
83 extern void __delayacct_wpcopy_end(void);
84 
delayacct_tsk_init(struct task_struct * tsk)85 static inline void delayacct_tsk_init(struct task_struct *tsk)
86 {
87 	/* reinitialize in case parent's non-null pointer was dup'ed*/
88 	tsk->delays = NULL;
89 	if (delayacct_on)
90 		__delayacct_tsk_init(tsk);
91 }
92 
93 /* Free tsk->delays. Called from bad fork and __put_task_struct
94  * where there's no risk of tsk->delays being accessed elsewhere
95  */
delayacct_tsk_free(struct task_struct * tsk)96 static inline void delayacct_tsk_free(struct task_struct *tsk)
97 {
98 	if (tsk->delays)
99 		kmem_cache_free(delayacct_cache, tsk->delays);
100 	tsk->delays = NULL;
101 }
102 
delayacct_blkio_start(void)103 static inline void delayacct_blkio_start(void)
104 {
105 	if (!static_branch_unlikely(&delayacct_key))
106 		return;
107 
108 	if (current->delays)
109 		__delayacct_blkio_start();
110 }
111 
delayacct_blkio_end(struct task_struct * p)112 static inline void delayacct_blkio_end(struct task_struct *p)
113 {
114 	if (!static_branch_unlikely(&delayacct_key))
115 		return;
116 
117 	if (p->delays)
118 		__delayacct_blkio_end(p);
119 }
120 
delayacct_blkio_ticks(struct task_struct * tsk)121 static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
122 {
123 	if (tsk->delays)
124 		return __delayacct_blkio_ticks(tsk);
125 	return 0;
126 }
127 
delayacct_freepages_start(void)128 static inline void delayacct_freepages_start(void)
129 {
130 	if (!static_branch_unlikely(&delayacct_key))
131 		return;
132 
133 	if (current->delays)
134 		__delayacct_freepages_start();
135 }
136 
delayacct_freepages_end(void)137 static inline void delayacct_freepages_end(void)
138 {
139 	if (!static_branch_unlikely(&delayacct_key))
140 		return;
141 
142 	if (current->delays)
143 		__delayacct_freepages_end();
144 }
145 
delayacct_thrashing_start(bool * in_thrashing)146 static inline void delayacct_thrashing_start(bool *in_thrashing)
147 {
148 	if (!static_branch_unlikely(&delayacct_key))
149 		return;
150 
151 	if (current->delays)
152 		__delayacct_thrashing_start(in_thrashing);
153 }
154 
delayacct_thrashing_end(bool * in_thrashing)155 static inline void delayacct_thrashing_end(bool *in_thrashing)
156 {
157 	if (!static_branch_unlikely(&delayacct_key))
158 		return;
159 
160 	if (current->delays)
161 		__delayacct_thrashing_end(in_thrashing);
162 }
163 
delayacct_swapin_start(void)164 static inline void delayacct_swapin_start(void)
165 {
166 	if (!static_branch_unlikely(&delayacct_key))
167 		return;
168 
169 	if (current->delays)
170 		__delayacct_swapin_start();
171 }
172 
delayacct_swapin_end(void)173 static inline void delayacct_swapin_end(void)
174 {
175 	if (!static_branch_unlikely(&delayacct_key))
176 		return;
177 
178 	if (current->delays)
179 		__delayacct_swapin_end();
180 }
181 
delayacct_compact_start(void)182 static inline void delayacct_compact_start(void)
183 {
184 	if (!static_branch_unlikely(&delayacct_key))
185 		return;
186 
187 	if (current->delays)
188 		__delayacct_compact_start();
189 }
190 
delayacct_compact_end(void)191 static inline void delayacct_compact_end(void)
192 {
193 	if (!static_branch_unlikely(&delayacct_key))
194 		return;
195 
196 	if (current->delays)
197 		__delayacct_compact_end();
198 }
199 
delayacct_wpcopy_start(void)200 static inline void delayacct_wpcopy_start(void)
201 {
202 	if (!static_branch_unlikely(&delayacct_key))
203 		return;
204 
205 	if (current->delays)
206 		__delayacct_wpcopy_start();
207 }
208 
delayacct_wpcopy_end(void)209 static inline void delayacct_wpcopy_end(void)
210 {
211 	if (!static_branch_unlikely(&delayacct_key))
212 		return;
213 
214 	if (current->delays)
215 		__delayacct_wpcopy_end();
216 }
217 
218 #else
delayacct_init(void)219 static inline void delayacct_init(void)
220 {}
delayacct_tsk_init(struct task_struct * tsk)221 static inline void delayacct_tsk_init(struct task_struct *tsk)
222 {}
delayacct_tsk_free(struct task_struct * tsk)223 static inline void delayacct_tsk_free(struct task_struct *tsk)
224 {}
delayacct_blkio_start(void)225 static inline void delayacct_blkio_start(void)
226 {}
delayacct_blkio_end(struct task_struct * p)227 static inline void delayacct_blkio_end(struct task_struct *p)
228 {}
delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk)229 static inline int delayacct_add_tsk(struct taskstats *d,
230 					struct task_struct *tsk)
231 { return 0; }
delayacct_blkio_ticks(struct task_struct * tsk)232 static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
233 { return 0; }
delayacct_is_task_waiting_on_io(struct task_struct * p)234 static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
235 { return 0; }
delayacct_freepages_start(void)236 static inline void delayacct_freepages_start(void)
237 {}
delayacct_freepages_end(void)238 static inline void delayacct_freepages_end(void)
239 {}
delayacct_thrashing_start(bool * in_thrashing)240 static inline void delayacct_thrashing_start(bool *in_thrashing)
241 {}
delayacct_thrashing_end(bool * in_thrashing)242 static inline void delayacct_thrashing_end(bool *in_thrashing)
243 {}
delayacct_swapin_start(void)244 static inline void delayacct_swapin_start(void)
245 {}
delayacct_swapin_end(void)246 static inline void delayacct_swapin_end(void)
247 {}
delayacct_compact_start(void)248 static inline void delayacct_compact_start(void)
249 {}
delayacct_compact_end(void)250 static inline void delayacct_compact_end(void)
251 {}
delayacct_wpcopy_start(void)252 static inline void delayacct_wpcopy_start(void)
253 {}
delayacct_wpcopy_end(void)254 static inline void delayacct_wpcopy_end(void)
255 {}
256 
257 #endif /* CONFIG_TASK_DELAY_ACCT */
258 
259 #endif
260