1 /* SPDX-License-Identifier: GPL-2.0 */
5 #include <linux/radix-tree.h>
18  * the low-order bits.
20  * We arrange the `type' and `offset' fields so that `type' is at the six
21  * high-order bits of the swp_entry_t and `offset' is right-aligned in the
25  * swp_entry_t's are *never* stored anywhere in their arch-dependent format.
27 #define SWP_TYPE_SHIFT	(BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
28 #define SWP_OFFSET_MASK	((1UL << SWP_TYPE_SHIFT) - 1)
36 #define SWP_PFN_BITS		(MAX_PHYSMEM_BITS - PAGE_SHIFT)
39 				      sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \
42 #define SWP_PFN_MASK		(BIT(SWP_PFN_BITS) - 1)
47  *   |----------+--------------------|
49  *   |----------+--------+-+-+-------|
51  *   |----------+--------+-+-+-------|
53  * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A)
54  * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D)
57  * free bits in arch specific swp offset.  By default we'll ignore A/D bits
58  * when migrating a page.  Please refer to migration_entry_supports_ad()
84  * Store a type+offset into a swp_entry_t in an arch-independent format
86 static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)  in swp_entry()  argument
90 	ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK);  in swp_entry()
96  * arch-independent format
104  * Extract the `offset' field from a swp_entry_t.  The swp_entry_t is in
105  * arch-independent format
130  * Convert the arch-dependent pte representation of a swp_entry_t into an
131  * arch-independent swp_entry_t.
143  * Convert the arch-independent representation of a swp_entry_t into the
144  * arch-dependent pte representation.
168 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)  in make_readable_device_private_entry()  argument
170 	return swp_entry(SWP_DEVICE_READ, offset);  in make_readable_device_private_entry()
173 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)  in make_writable_device_private_entry()  argument
175 	return swp_entry(SWP_DEVICE_WRITE, offset);  in make_writable_device_private_entry()
189 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset)  in make_readable_device_exclusive_entry()  argument
191 	return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset);  in make_readable_device_exclusive_entry()
194 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset)  in make_writable_device_exclusive_entry()  argument
196 	return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset);  in make_writable_device_exclusive_entry()
210 static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)  in make_readable_device_private_entry()  argument
215 static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)  in make_writable_device_private_entry()  argument
230 static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset)  in make_readable_device_exclusive_entry()  argument
235 static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset)  in make_writable_device_exclusive_entry()  argument
274 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)  in make_readable_migration_entry()  argument
276 	return swp_entry(SWP_MIGRATION_READ, offset);  in make_readable_migration_entry()
279 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)  in make_readable_exclusive_migration_entry()  argument
281 	return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset);  in make_readable_exclusive_migration_entry()
284 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)  in make_writable_migration_entry()  argument
286 	return swp_entry(SWP_MIGRATION_WRITE, offset);  in make_writable_migration_entry()
290  * Returns whether the host has large enough swap offset field to support
291  * carrying over pgtable A/D bits for page migrations.  The result is
315 	/* Keep the old behavior of aging page after migration */  in is_migration_entry_young()
331 	/* Keep the old behavior of clean page after migration */  in is_migration_entry_dirty()
339 static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)  in make_readable_migration_entry()  argument
344 static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)  in make_readable_exclusive_migration_entry()  argument
349 static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)  in make_writable_migration_entry()  argument
398 static inline swp_entry_t make_hwpoison_entry(struct page *page)  in make_hwpoison_entry()  argument
400 	BUG_ON(!PageLocked(page));  in make_hwpoison_entry()
401 	return swp_entry(SWP_HWPOISON, page_to_pfn(page));  in make_hwpoison_entry()
411 static inline swp_entry_t make_hwpoison_entry(struct page *page)  in make_hwpoison_entry()  argument
431 #define  PTE_MARKER_MASK			(BIT(2) - 1)
473  * onto the none pte so it becomes not-none any more.
475  * It should be used when the pte is file-backed, ram-based and backing
478  * memory, kernel-only memory (including when the system is during-boot),
479  * non-ram based generic file-system.  It's fine to be used even there, but the
487 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)  in pfn_swap_entry_to_page()
489 	struct page *p = pfn_to_page(swp_offset_pfn(entry));  in pfn_swap_entry_to_page()
493 	 * corresponding page is locked  in pfn_swap_entry_to_page()
515  * in the swap offset. They can either be used to represent unaddressable device
516  * memory, to restrict access to a page undergoing migration or to represent a
521 	/* Make sure the swp offset can always store the needed fields */  in is_pfn_swap_entry()
532 		struct page *page);
535 		struct page *new);
565 		struct page *page)  in set_pmd_migration_entry()  argument
571 		struct page *new)  in remove_migration_pmd()