1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * page allocation tagging
4 */
5 #ifndef _LINUX_PGALLOC_TAG_H
6 #define _LINUX_PGALLOC_TAG_H
7
8 #include <linux/alloc_tag.h>
9
10 #ifdef CONFIG_MEM_ALLOC_PROFILING
11
12 #include <linux/page_ext.h>
13
14 extern struct page_ext_operations page_alloc_tagging_ops;
15
codetag_ref_from_page_ext(struct page_ext * page_ext)16 static inline union codetag_ref *codetag_ref_from_page_ext(struct page_ext *page_ext)
17 {
18 return (union codetag_ref *)page_ext_data(page_ext, &page_alloc_tagging_ops);
19 }
20
page_ext_from_codetag_ref(union codetag_ref * ref)21 static inline struct page_ext *page_ext_from_codetag_ref(union codetag_ref *ref)
22 {
23 return (void *)ref - page_alloc_tagging_ops.offset;
24 }
25
26 /* Should be called only if mem_alloc_profiling_enabled() */
get_page_tag_ref(struct page * page)27 static inline union codetag_ref *get_page_tag_ref(struct page *page)
28 {
29 if (page) {
30 struct page_ext *page_ext = page_ext_get(page);
31
32 if (page_ext)
33 return codetag_ref_from_page_ext(page_ext);
34 }
35 return NULL;
36 }
37
put_page_tag_ref(union codetag_ref * ref)38 static inline void put_page_tag_ref(union codetag_ref *ref)
39 {
40 if (WARN_ON(!ref))
41 return;
42
43 page_ext_put(page_ext_from_codetag_ref(ref));
44 }
45
clear_page_tag_ref(struct page * page)46 static inline void clear_page_tag_ref(struct page *page)
47 {
48 if (mem_alloc_profiling_enabled()) {
49 union codetag_ref *ref = get_page_tag_ref(page);
50
51 if (ref) {
52 set_codetag_empty(ref);
53 put_page_tag_ref(ref);
54 }
55 }
56 }
57
pgalloc_tag_add(struct page * page,struct task_struct * task,unsigned int nr)58 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
59 unsigned int nr)
60 {
61 if (mem_alloc_profiling_enabled()) {
62 union codetag_ref *ref = get_page_tag_ref(page);
63
64 if (ref) {
65 alloc_tag_add(ref, task->alloc_tag, PAGE_SIZE * nr);
66 put_page_tag_ref(ref);
67 }
68 }
69 }
70
pgalloc_tag_sub(struct page * page,unsigned int nr)71 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
72 {
73 if (mem_alloc_profiling_enabled()) {
74 union codetag_ref *ref = get_page_tag_ref(page);
75
76 if (ref) {
77 alloc_tag_sub(ref, PAGE_SIZE * nr);
78 put_page_tag_ref(ref);
79 }
80 }
81 }
82
pgalloc_tag_get(struct page * page)83 static inline struct alloc_tag *pgalloc_tag_get(struct page *page)
84 {
85 struct alloc_tag *tag = NULL;
86
87 if (mem_alloc_profiling_enabled()) {
88 union codetag_ref *ref = get_page_tag_ref(page);
89
90 alloc_tag_sub_check(ref);
91 if (ref) {
92 if (ref->ct)
93 tag = ct_to_alloc_tag(ref->ct);
94 put_page_tag_ref(ref);
95 }
96 }
97
98 return tag;
99 }
100
pgalloc_tag_sub_pages(struct alloc_tag * tag,unsigned int nr)101 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
102 {
103 if (mem_alloc_profiling_enabled() && tag)
104 this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr);
105 }
106
107 #else /* CONFIG_MEM_ALLOC_PROFILING */
108
get_page_tag_ref(struct page * page)109 static inline union codetag_ref *get_page_tag_ref(struct page *page) { return NULL; }
put_page_tag_ref(union codetag_ref * ref)110 static inline void put_page_tag_ref(union codetag_ref *ref) {}
clear_page_tag_ref(struct page * page)111 static inline void clear_page_tag_ref(struct page *page) {}
pgalloc_tag_add(struct page * page,struct task_struct * task,unsigned int nr)112 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
113 unsigned int nr) {}
pgalloc_tag_sub(struct page * page,unsigned int nr)114 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
pgalloc_tag_get(struct page * page)115 static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; }
pgalloc_tag_sub_pages(struct alloc_tag * tag,unsigned int nr)116 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
117
118 #endif /* CONFIG_MEM_ALLOC_PROFILING */
119
120 #endif /* _LINUX_PGALLOC_TAG_H */
121