1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_PERCPU_H 3 #define __LINUX_PERCPU_H 4 5 #include <linux/alloc_tag.h> 6 #include <linux/mmdebug.h> 7 #include <linux/preempt.h> 8 #include <linux/smp.h> 9 #include <linux/pfn.h> 10 #include <linux/init.h> 11 #include <linux/cleanup.h> 12 #include <linux/sched.h> 13 14 #include <asm/percpu.h> 15 16 /* enough to cover all DEFINE_PER_CPUs in modules */ 17 #ifdef CONFIG_MODULES 18 #ifdef CONFIG_MEM_ALLOC_PROFILING 19 #define PERCPU_MODULE_RESERVE (8 << 13) 20 #else 21 #define PERCPU_MODULE_RESERVE (8 << 10) 22 #endif 23 #else 24 #define PERCPU_MODULE_RESERVE 0 25 #endif 26 27 /* minimum unit size, also is the maximum supported allocation size */ 28 #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) 29 30 /* minimum allocation size and shift in bytes */ 31 #define PCPU_MIN_ALLOC_SHIFT 2 32 #define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT) 33 34 /* 35 * The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as the 36 * updating of hints is used to manage the nr_empty_pop_pages in both 37 * the chunk and globally. 38 */ 39 #define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE 40 #define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \ 41 PCPU_MIN_ALLOC_SHIFT) 42 43 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 44 # if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PAGE_SIZE_4KB) 45 # define PERCPU_DYNAMIC_SIZE_SHIFT 13 46 # else 47 # define PERCPU_DYNAMIC_SIZE_SHIFT 12 48 #endif /* LOCKDEP and PAGE_SIZE > 4KiB */ 49 #else 50 #define PERCPU_DYNAMIC_SIZE_SHIFT 10 51 #endif 52 53 /* 54 * Percpu allocator can serve percpu allocations before slab is 55 * initialized which allows slab to depend on the percpu allocator. 56 * The following parameter decide how much resource to preallocate 57 * for this. Keep PERCPU_DYNAMIC_RESERVE equal to or larger than 58 * PERCPU_DYNAMIC_EARLY_SIZE. 59 */ 60 #define PERCPU_DYNAMIC_EARLY_SIZE (20 << PERCPU_DYNAMIC_SIZE_SHIFT) 61 62 /* 63 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 64 * back on the first chunk for dynamic percpu allocation if arch is 65 * manually allocating and mapping it for faster access (as a part of 66 * large page mapping for example). 67 * 68 * The following values give between one and two pages of free space 69 * after typical minimal boot (2-way SMP, single disk and NIC) with 70 * both defconfig and a distro config on x86_64 and 32. More 71 * intelligent way to determine this would be nice. 72 */ 73 #if BITS_PER_LONG > 32 74 #define PERCPU_DYNAMIC_RESERVE (28 << PERCPU_DYNAMIC_SIZE_SHIFT) 75 #else 76 #define PERCPU_DYNAMIC_RESERVE (20 << PERCPU_DYNAMIC_SIZE_SHIFT) 77 #endif 78 79 extern void *pcpu_base_addr; 80 extern const unsigned long *pcpu_unit_offsets; 81 82 struct pcpu_group_info { 83 int nr_units; /* aligned # of units */ 84 unsigned long base_offset; /* base address offset */ 85 unsigned int *cpu_map; /* unit->cpu map, empty 86 * entries contain NR_CPUS */ 87 }; 88 89 struct pcpu_alloc_info { 90 size_t static_size; 91 size_t reserved_size; 92 size_t dyn_size; 93 size_t unit_size; 94 size_t atom_size; 95 size_t alloc_size; 96 size_t __ai_size; /* internal, don't use */ 97 int nr_groups; /* 0 if grouping unnecessary */ 98 struct pcpu_group_info groups[]; 99 }; 100 101 enum pcpu_fc { 102 PCPU_FC_AUTO, 103 PCPU_FC_EMBED, 104 PCPU_FC_PAGE, 105 106 PCPU_FC_NR, 107 }; 108 extern const char * const pcpu_fc_names[PCPU_FC_NR]; 109 110 extern enum pcpu_fc pcpu_chosen_fc; 111 112 typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); 113 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 114 115 extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, 116 int nr_units); 117 extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai); 118 119 extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, 120 void *base_addr); 121 122 extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 123 size_t atom_size, 124 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 125 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 126 127 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 128 void __init pcpu_populate_pte(unsigned long addr); 129 extern int __init pcpu_page_first_chunk(size_t reserved_size, 130 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 131 #endif 132 133 extern bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr); 134 extern bool is_kernel_percpu_address(unsigned long addr); 135 136 #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) 137 extern void __init setup_per_cpu_areas(void); 138 #endif 139 140 extern void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, 141 gfp_t gfp) __alloc_size(1); 142 143 #define __alloc_percpu_gfp(_size, _align, _gfp) \ 144 alloc_hooks(pcpu_alloc_noprof(_size, _align, false, _gfp)) 145 #define __alloc_percpu(_size, _align) \ 146 alloc_hooks(pcpu_alloc_noprof(_size, _align, false, GFP_KERNEL)) 147 #define __alloc_reserved_percpu(_size, _align) \ 148 alloc_hooks(pcpu_alloc_noprof(_size, _align, true, GFP_KERNEL)) 149 150 #define alloc_percpu_gfp(type, gfp) \ 151 (typeof(type) __percpu *)__alloc_percpu_gfp(sizeof(type), \ 152 __alignof__(type), gfp) 153 #define alloc_percpu(type) \ 154 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ 155 __alignof__(type)) 156 #define alloc_percpu_noprof(type) \ 157 ((typeof(type) __percpu *)pcpu_alloc_noprof(sizeof(type), \ 158 __alignof__(type), false, GFP_KERNEL)) 159 160 extern void free_percpu(void __percpu *__pdata); 161 162 DEFINE_FREE(free_percpu, void __percpu *, free_percpu(_T)) 163 164 extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 165 166 extern unsigned long pcpu_nr_pages(void); 167 168 #endif /* __LINUX_PERCPU_H */ 169