7 #include <linux/page-flags.h>
29 extern void __page_ref_set(struct page *page, int v);
30 extern void __page_ref_mod(struct page *page, int v);
31 extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
32 extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
33 extern void __page_ref_mod_unless(struct page *page, int v, int u);
34 extern void __page_ref_freeze(struct page *page, int v, int ret);
35 extern void __page_ref_unfreeze(struct page *page, int v);
41 static inline void __page_ref_set(struct page *page, int v)  in __page_ref_set()  argument
44 static inline void __page_ref_mod(struct page *page, int v)  in __page_ref_mod()  argument
47 static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)  in __page_ref_mod_and_test()  argument
50 static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)  in __page_ref_mod_and_return()  argument
53 static inline void __page_ref_mod_unless(struct page *page, int v, int u)  in __page_ref_mod_unless()  argument
56 static inline void __page_ref_freeze(struct page *page, int v, int ret)  in __page_ref_freeze()  argument
59 static inline void __page_ref_unfreeze(struct page *page, int v)  in __page_ref_unfreeze()  argument
65 static inline int page_ref_count(const struct page *page)  in page_ref_count()  argument
67 	return atomic_read(&page->_refcount);  in page_ref_count()
78  * - Each reference from a page table
79  * - The page cache
83  * - Direct IO which references this page in the process address space
89 	return page_ref_count(&folio->page);  in folio_ref_count()
92 static inline int page_count(const struct page *page)  in page_count()  argument
94 	return folio_ref_count(page_folio(page));  in page_count()
97 static inline void set_page_count(struct page *page, int v)  in set_page_count()  argument
99 	atomic_set(&page->_refcount, v);  in set_page_count()
101 		__page_ref_set(page, v);  in set_page_count()
106 	set_page_count(&folio->page, v);  in folio_set_count()
110  * Setup the page count before being freed into the page allocator for
113 static inline void init_page_count(struct page *page)  in init_page_count()  argument
115 	set_page_count(page, 1);  in init_page_count()
118 static inline void page_ref_add(struct page *page, int nr)  in page_ref_add()  argument
120 	atomic_add(nr, &page->_refcount);  in page_ref_add()
122 		__page_ref_mod(page, nr);  in page_ref_add()
127 	page_ref_add(&folio->page, nr);  in folio_ref_add()
130 static inline void page_ref_sub(struct page *page, int nr)  in page_ref_sub()  argument
132 	atomic_sub(nr, &page->_refcount);  in page_ref_sub()
134 		__page_ref_mod(page, -nr);  in page_ref_sub()
139 	page_ref_sub(&folio->page, nr);  in folio_ref_sub()
147 		__page_ref_mod_and_return(&folio->page, -nr, ret);  in folio_ref_sub_return()
151 static inline void page_ref_inc(struct page *page)  in page_ref_inc()  argument
153 	atomic_inc(&page->_refcount);  in page_ref_inc()
155 		__page_ref_mod(page, 1);  in page_ref_inc()
160 	page_ref_inc(&folio->page);  in folio_ref_inc()
163 static inline void page_ref_dec(struct page *page)  in page_ref_dec()  argument
165 	atomic_dec(&page->_refcount);  in page_ref_dec()
167 		__page_ref_mod(page, -1);  in page_ref_dec()
172 	page_ref_dec(&folio->page);  in folio_ref_dec()
175 static inline int page_ref_sub_and_test(struct page *page, int nr)  in page_ref_sub_and_test()  argument
177 	int ret = atomic_sub_and_test(nr, &page->_refcount);  in page_ref_sub_and_test()
180 		__page_ref_mod_and_test(page, -nr, ret);  in page_ref_sub_and_test()
186 	return page_ref_sub_and_test(&folio->page, nr);  in folio_ref_sub_and_test()
189 static inline int page_ref_inc_return(struct page *page)  in page_ref_inc_return()  argument
191 	int ret = atomic_inc_return(&page->_refcount);  in page_ref_inc_return()
194 		__page_ref_mod_and_return(page, 1, ret);  in page_ref_inc_return()
200 	return page_ref_inc_return(&folio->page);  in folio_ref_inc_return()
203 static inline int page_ref_dec_and_test(struct page *page)  in page_ref_dec_and_test()  argument
205 	int ret = atomic_dec_and_test(&page->_refcount);  in page_ref_dec_and_test()
208 		__page_ref_mod_and_test(page, -1, ret);  in page_ref_dec_and_test()
214 	return page_ref_dec_and_test(&folio->page);  in folio_ref_dec_and_test()
217 static inline int page_ref_dec_return(struct page *page)  in page_ref_dec_return()  argument
219 	int ret = atomic_dec_return(&page->_refcount);  in page_ref_dec_return()
222 		__page_ref_mod_and_return(page, -1, ret);  in page_ref_dec_return()
228 	return page_ref_dec_return(&folio->page);  in folio_ref_dec_return()
231 static inline bool page_ref_add_unless(struct page *page, int nr, int u)  in page_ref_add_unless()  argument
237 	if (!page_is_fake_head(page) && page_ref_count(page) != u)  in page_ref_add_unless()
238 		ret = atomic_add_unless(&page->_refcount, nr, u);  in page_ref_add_unless()
242 		__page_ref_mod_unless(page, nr, ret);  in page_ref_add_unless()
248 	return page_ref_add_unless(&folio->page, nr, u);  in folio_ref_add_unless()
272 static inline int page_ref_freeze(struct page *page, int count)  in page_ref_freeze()  argument
274 	int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);  in page_ref_freeze()
277 		__page_ref_freeze(page, count, ret);  in page_ref_freeze()
283 	return page_ref_freeze(&folio->page, count);  in folio_ref_freeze()
286 static inline void page_ref_unfreeze(struct page *page, int count)  in page_ref_unfreeze()  argument
288 	VM_BUG_ON_PAGE(page_count(page) != 0, page);  in page_ref_unfreeze()
291 	atomic_set_release(&page->_refcount, count);  in page_ref_unfreeze()
293 		__page_ref_unfreeze(page, count);  in page_ref_unfreeze()
298 	page_ref_unfreeze(&folio->page, count);  in folio_ref_unfreeze()