1  /* SPDX-License-Identifier: GPL-2.0-or-later */
2  /* delayacct.h - per-task delay accounting
3   *
4   * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5   */
6  
7  #ifndef _LINUX_DELAYACCT_H
8  #define _LINUX_DELAYACCT_H
9  
10  #include <uapi/linux/taskstats.h>
11  
12  #ifdef CONFIG_TASK_DELAY_ACCT
13  struct task_delay_info {
14  	raw_spinlock_t	lock;
15  
16  	/* For each stat XXX, add following, aligned appropriately
17  	 *
18  	 * struct timespec XXX_start, XXX_end;
19  	 * u64 XXX_delay;
20  	 * u32 XXX_count;
21  	 *
22  	 * Atomicity of updates to XXX_delay, XXX_count protected by
23  	 * single lock above (split into XXX_lock if contention is an issue).
24  	 */
25  
26  	/*
27  	 * XXX_count is incremented on every XXX operation, the delay
28  	 * associated with the operation is added to XXX_delay.
29  	 * XXX_delay contains the accumulated delay time in nanoseconds.
30  	 */
31  	u64 blkio_start;
32  	u64 blkio_delay;	/* wait for sync block io completion */
33  	u64 swapin_start;
34  	u64 swapin_delay;	/* wait for swapin */
35  	u32 blkio_count;	/* total count of the number of sync block */
36  				/* io operations performed */
37  	u32 swapin_count;	/* total count of swapin */
38  
39  	u64 freepages_start;
40  	u64 freepages_delay;	/* wait for memory reclaim */
41  
42  	u64 thrashing_start;
43  	u64 thrashing_delay;	/* wait for thrashing page */
44  
45  	u64 compact_start;
46  	u64 compact_delay;	/* wait for memory compact */
47  
48  	u64 wpcopy_start;
49  	u64 wpcopy_delay;	/* wait for write-protect copy */
50  
51  	u64 irq_delay;	/* wait for IRQ/SOFTIRQ */
52  
53  	u32 freepages_count;	/* total count of memory reclaim */
54  	u32 thrashing_count;	/* total count of thrash waits */
55  	u32 compact_count;	/* total count of memory compact */
56  	u32 wpcopy_count;	/* total count of write-protect copy */
57  	u32 irq_count;	/* total count of IRQ/SOFTIRQ */
58  };
59  #endif
60  
61  #include <linux/sched.h>
62  #include <linux/slab.h>
63  #include <linux/jump_label.h>
64  
65  #ifdef CONFIG_TASK_DELAY_ACCT
66  DECLARE_STATIC_KEY_FALSE(delayacct_key);
67  extern int delayacct_on;	/* Delay accounting turned on/off */
68  extern struct kmem_cache *delayacct_cache;
69  extern void delayacct_init(void);
70  
71  extern void __delayacct_tsk_init(struct task_struct *);
72  extern void __delayacct_tsk_exit(struct task_struct *);
73  extern void __delayacct_blkio_start(void);
74  extern void __delayacct_blkio_end(struct task_struct *);
75  extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
76  extern __u64 __delayacct_blkio_ticks(struct task_struct *);
77  extern void __delayacct_freepages_start(void);
78  extern void __delayacct_freepages_end(void);
79  extern void __delayacct_thrashing_start(bool *in_thrashing);
80  extern void __delayacct_thrashing_end(bool *in_thrashing);
81  extern void __delayacct_swapin_start(void);
82  extern void __delayacct_swapin_end(void);
83  extern void __delayacct_compact_start(void);
84  extern void __delayacct_compact_end(void);
85  extern void __delayacct_wpcopy_start(void);
86  extern void __delayacct_wpcopy_end(void);
87  extern void __delayacct_irq(struct task_struct *task, u32 delta);
88  
delayacct_tsk_init(struct task_struct * tsk)89  static inline void delayacct_tsk_init(struct task_struct *tsk)
90  {
91  	/* reinitialize in case parent's non-null pointer was dup'ed*/
92  	tsk->delays = NULL;
93  	if (delayacct_on)
94  		__delayacct_tsk_init(tsk);
95  }
96  
97  /* Free tsk->delays. Called from bad fork and __put_task_struct
98   * where there's no risk of tsk->delays being accessed elsewhere
99   */
delayacct_tsk_free(struct task_struct * tsk)100  static inline void delayacct_tsk_free(struct task_struct *tsk)
101  {
102  	if (tsk->delays)
103  		kmem_cache_free(delayacct_cache, tsk->delays);
104  	tsk->delays = NULL;
105  }
106  
delayacct_blkio_start(void)107  static inline void delayacct_blkio_start(void)
108  {
109  	if (!static_branch_unlikely(&delayacct_key))
110  		return;
111  
112  	if (current->delays)
113  		__delayacct_blkio_start();
114  }
115  
delayacct_blkio_end(struct task_struct * p)116  static inline void delayacct_blkio_end(struct task_struct *p)
117  {
118  	if (!static_branch_unlikely(&delayacct_key))
119  		return;
120  
121  	if (p->delays)
122  		__delayacct_blkio_end(p);
123  }
124  
delayacct_blkio_ticks(struct task_struct * tsk)125  static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
126  {
127  	if (tsk->delays)
128  		return __delayacct_blkio_ticks(tsk);
129  	return 0;
130  }
131  
delayacct_freepages_start(void)132  static inline void delayacct_freepages_start(void)
133  {
134  	if (!static_branch_unlikely(&delayacct_key))
135  		return;
136  
137  	if (current->delays)
138  		__delayacct_freepages_start();
139  }
140  
delayacct_freepages_end(void)141  static inline void delayacct_freepages_end(void)
142  {
143  	if (!static_branch_unlikely(&delayacct_key))
144  		return;
145  
146  	if (current->delays)
147  		__delayacct_freepages_end();
148  }
149  
delayacct_thrashing_start(bool * in_thrashing)150  static inline void delayacct_thrashing_start(bool *in_thrashing)
151  {
152  	if (!static_branch_unlikely(&delayacct_key))
153  		return;
154  
155  	if (current->delays)
156  		__delayacct_thrashing_start(in_thrashing);
157  }
158  
delayacct_thrashing_end(bool * in_thrashing)159  static inline void delayacct_thrashing_end(bool *in_thrashing)
160  {
161  	if (!static_branch_unlikely(&delayacct_key))
162  		return;
163  
164  	if (current->delays)
165  		__delayacct_thrashing_end(in_thrashing);
166  }
167  
delayacct_swapin_start(void)168  static inline void delayacct_swapin_start(void)
169  {
170  	if (!static_branch_unlikely(&delayacct_key))
171  		return;
172  
173  	if (current->delays)
174  		__delayacct_swapin_start();
175  }
176  
delayacct_swapin_end(void)177  static inline void delayacct_swapin_end(void)
178  {
179  	if (!static_branch_unlikely(&delayacct_key))
180  		return;
181  
182  	if (current->delays)
183  		__delayacct_swapin_end();
184  }
185  
delayacct_compact_start(void)186  static inline void delayacct_compact_start(void)
187  {
188  	if (!static_branch_unlikely(&delayacct_key))
189  		return;
190  
191  	if (current->delays)
192  		__delayacct_compact_start();
193  }
194  
delayacct_compact_end(void)195  static inline void delayacct_compact_end(void)
196  {
197  	if (!static_branch_unlikely(&delayacct_key))
198  		return;
199  
200  	if (current->delays)
201  		__delayacct_compact_end();
202  }
203  
delayacct_wpcopy_start(void)204  static inline void delayacct_wpcopy_start(void)
205  {
206  	if (!static_branch_unlikely(&delayacct_key))
207  		return;
208  
209  	if (current->delays)
210  		__delayacct_wpcopy_start();
211  }
212  
delayacct_wpcopy_end(void)213  static inline void delayacct_wpcopy_end(void)
214  {
215  	if (!static_branch_unlikely(&delayacct_key))
216  		return;
217  
218  	if (current->delays)
219  		__delayacct_wpcopy_end();
220  }
221  
delayacct_irq(struct task_struct * task,u32 delta)222  static inline void delayacct_irq(struct task_struct *task, u32 delta)
223  {
224  	if (!static_branch_unlikely(&delayacct_key))
225  		return;
226  
227  	if (task->delays)
228  		__delayacct_irq(task, delta);
229  }
230  
231  #else
delayacct_init(void)232  static inline void delayacct_init(void)
233  {}
delayacct_tsk_init(struct task_struct * tsk)234  static inline void delayacct_tsk_init(struct task_struct *tsk)
235  {}
delayacct_tsk_free(struct task_struct * tsk)236  static inline void delayacct_tsk_free(struct task_struct *tsk)
237  {}
delayacct_blkio_start(void)238  static inline void delayacct_blkio_start(void)
239  {}
delayacct_blkio_end(struct task_struct * p)240  static inline void delayacct_blkio_end(struct task_struct *p)
241  {}
delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk)242  static inline int delayacct_add_tsk(struct taskstats *d,
243  					struct task_struct *tsk)
244  { return 0; }
delayacct_blkio_ticks(struct task_struct * tsk)245  static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
246  { return 0; }
delayacct_is_task_waiting_on_io(struct task_struct * p)247  static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
248  { return 0; }
delayacct_freepages_start(void)249  static inline void delayacct_freepages_start(void)
250  {}
delayacct_freepages_end(void)251  static inline void delayacct_freepages_end(void)
252  {}
delayacct_thrashing_start(bool * in_thrashing)253  static inline void delayacct_thrashing_start(bool *in_thrashing)
254  {}
delayacct_thrashing_end(bool * in_thrashing)255  static inline void delayacct_thrashing_end(bool *in_thrashing)
256  {}
delayacct_swapin_start(void)257  static inline void delayacct_swapin_start(void)
258  {}
delayacct_swapin_end(void)259  static inline void delayacct_swapin_end(void)
260  {}
delayacct_compact_start(void)261  static inline void delayacct_compact_start(void)
262  {}
delayacct_compact_end(void)263  static inline void delayacct_compact_end(void)
264  {}
delayacct_wpcopy_start(void)265  static inline void delayacct_wpcopy_start(void)
266  {}
delayacct_wpcopy_end(void)267  static inline void delayacct_wpcopy_end(void)
268  {}
delayacct_irq(struct task_struct * task,u32 delta)269  static inline void delayacct_irq(struct task_struct *task, u32 delta)
270  {}
271  
272  #endif /* CONFIG_TASK_DELAY_ACCT */
273  
274  #endif
275