1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * NUMA support, based on the x86 implementation.
4  *
5  * Copyright (C) 2015 Cavium Inc.
6  * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
7  */
8 
9 #define pr_fmt(fmt) "NUMA: " fmt
10 
11 #include <linux/acpi.h>
12 #include <linux/memblock.h>
13 #include <linux/module.h>
14 #include <linux/of.h>
15 #include <linux/numa_memblks.h>
16 
17 #include <asm/sections.h>
18 
19 static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
20 
21 bool numa_off;
22 
numa_parse_early_param(char * opt)23 static __init int numa_parse_early_param(char *opt)
24 {
25 	if (!opt)
26 		return -EINVAL;
27 	if (str_has_prefix(opt, "off"))
28 		numa_off = true;
29 	if (!strncmp(opt, "fake=", 5))
30 		return numa_emu_cmdline(opt + 5);
31 
32 	return 0;
33 }
34 early_param("numa", numa_parse_early_param);
35 
36 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
37 EXPORT_SYMBOL(node_to_cpumask_map);
38 
39 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
40 
41 /*
42  * Returns a pointer to the bitmask of CPUs on Node 'node'.
43  */
cpumask_of_node(int node)44 const struct cpumask *cpumask_of_node(int node)
45 {
46 
47 	if (node == NUMA_NO_NODE)
48 		return cpu_all_mask;
49 
50 	if (WARN_ON(node < 0 || node >= nr_node_ids))
51 		return cpu_none_mask;
52 
53 	if (WARN_ON(node_to_cpumask_map[node] == NULL))
54 		return cpu_online_mask;
55 
56 	return node_to_cpumask_map[node];
57 }
58 EXPORT_SYMBOL(cpumask_of_node);
59 
60 #endif
61 
62 #ifndef CONFIG_NUMA_EMU
numa_update_cpu(unsigned int cpu,bool remove)63 static void numa_update_cpu(unsigned int cpu, bool remove)
64 {
65 	int nid = cpu_to_node(cpu);
66 
67 	if (nid == NUMA_NO_NODE)
68 		return;
69 
70 	if (remove)
71 		cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
72 	else
73 		cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
74 }
75 
numa_add_cpu(unsigned int cpu)76 void numa_add_cpu(unsigned int cpu)
77 {
78 	numa_update_cpu(cpu, false);
79 }
80 
numa_remove_cpu(unsigned int cpu)81 void numa_remove_cpu(unsigned int cpu)
82 {
83 	numa_update_cpu(cpu, true);
84 }
85 #endif
86 
numa_clear_node(unsigned int cpu)87 void numa_clear_node(unsigned int cpu)
88 {
89 	numa_remove_cpu(cpu);
90 	set_cpu_numa_node(cpu, NUMA_NO_NODE);
91 }
92 
93 /*
94  * Allocate node_to_cpumask_map based on number of available nodes
95  * Requires node_possible_map to be valid.
96  *
97  * Note: cpumask_of_node() is not valid until after this is done.
98  * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
99  */
setup_node_to_cpumask_map(void)100 static void __init setup_node_to_cpumask_map(void)
101 {
102 	int node;
103 
104 	/* setup nr_node_ids if not done yet */
105 	if (nr_node_ids == MAX_NUMNODES)
106 		setup_nr_node_ids();
107 
108 	/* allocate and clear the mapping */
109 	for (node = 0; node < nr_node_ids; node++) {
110 		alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
111 		cpumask_clear(node_to_cpumask_map[node]);
112 	}
113 
114 	/* cpumask_of_node() will now work */
115 	pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids);
116 }
117 
118 /*
119  * Set the cpu to node and mem mapping
120  */
numa_store_cpu_info(unsigned int cpu)121 void numa_store_cpu_info(unsigned int cpu)
122 {
123 	set_cpu_numa_node(cpu, cpu_to_node_map[cpu]);
124 }
125 
early_map_cpu_to_node(unsigned int cpu,int nid)126 void __init early_map_cpu_to_node(unsigned int cpu, int nid)
127 {
128 	/* fallback to node 0 */
129 	if (nid < 0 || nid >= MAX_NUMNODES || numa_off)
130 		nid = 0;
131 
132 	cpu_to_node_map[cpu] = nid;
133 
134 	/*
135 	 * We should set the numa node of cpu0 as soon as possible, because it
136 	 * has already been set up online before. cpu_to_node(0) will soon be
137 	 * called.
138 	 */
139 	if (!cpu)
140 		set_cpu_numa_node(cpu, nid);
141 }
142 
143 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
144 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
145 EXPORT_SYMBOL(__per_cpu_offset);
146 
early_cpu_to_node(int cpu)147 int early_cpu_to_node(int cpu)
148 {
149 	return cpu_to_node_map[cpu];
150 }
151 
pcpu_cpu_distance(unsigned int from,unsigned int to)152 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
153 {
154 	return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));
155 }
156 
setup_per_cpu_areas(void)157 void __init setup_per_cpu_areas(void)
158 {
159 	unsigned long delta;
160 	unsigned int cpu;
161 	int rc = -EINVAL;
162 
163 	if (pcpu_chosen_fc != PCPU_FC_PAGE) {
164 		/*
165 		 * Always reserve area for module percpu variables.  That's
166 		 * what the legacy allocator did.
167 		 */
168 		rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
169 					    PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
170 					    pcpu_cpu_distance,
171 					    early_cpu_to_node);
172 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
173 		if (rc < 0)
174 			pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n",
175 				   pcpu_fc_names[pcpu_chosen_fc], rc);
176 #endif
177 	}
178 
179 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
180 	if (rc < 0)
181 		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);
182 #endif
183 	if (rc < 0)
184 		panic("Failed to initialize percpu areas (err=%d).", rc);
185 
186 	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
187 	for_each_possible_cpu(cpu)
188 		__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
189 }
190 #endif
191 
192 /*
193  * Initialize NODE_DATA for a node on the local memory
194  */
setup_node_data(int nid,u64 start_pfn,u64 end_pfn)195 static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
196 {
197 	if (start_pfn >= end_pfn)
198 		pr_info("Initmem setup node %d [<memory-less node>]\n", nid);
199 
200 	alloc_node_data(nid);
201 
202 	NODE_DATA(nid)->node_id = nid;
203 	NODE_DATA(nid)->node_start_pfn = start_pfn;
204 	NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
205 }
206 
numa_register_nodes(void)207 static int __init numa_register_nodes(void)
208 {
209 	int nid;
210 
211 	/* Finally register nodes. */
212 	for_each_node_mask(nid, numa_nodes_parsed) {
213 		unsigned long start_pfn, end_pfn;
214 
215 		get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
216 		setup_node_data(nid, start_pfn, end_pfn);
217 		node_set_online(nid);
218 	}
219 
220 	/* Setup online nodes to actual nodes*/
221 	node_possible_map = numa_nodes_parsed;
222 
223 	return 0;
224 }
225 
numa_init(int (* init_func)(void))226 static int __init numa_init(int (*init_func)(void))
227 {
228 	int ret;
229 
230 	nodes_clear(numa_nodes_parsed);
231 	nodes_clear(node_possible_map);
232 	nodes_clear(node_online_map);
233 
234 	ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false);
235 	if (ret < 0)
236 		goto out_free_distance;
237 
238 	if (nodes_empty(numa_nodes_parsed)) {
239 		pr_info("No NUMA configuration found\n");
240 		ret = -EINVAL;
241 		goto out_free_distance;
242 	}
243 
244 	ret = numa_register_nodes();
245 	if (ret < 0)
246 		goto out_free_distance;
247 
248 	setup_node_to_cpumask_map();
249 
250 	return 0;
251 out_free_distance:
252 	numa_reset_distance();
253 	return ret;
254 }
255 
256 /**
257  * dummy_numa_init() - Fallback dummy NUMA init
258  *
259  * Used if there's no underlying NUMA architecture, NUMA initialization
260  * fails, or NUMA is disabled on the command line.
261  *
262  * Must online at least one node (node 0) and add memory blocks that cover all
263  * allowed memory. It is unlikely that this function fails.
264  *
265  * Return: 0 on success, -errno on failure.
266  */
dummy_numa_init(void)267 static int __init dummy_numa_init(void)
268 {
269 	phys_addr_t start = memblock_start_of_DRAM();
270 	phys_addr_t end = memblock_end_of_DRAM() - 1;
271 	int ret;
272 
273 	if (numa_off)
274 		pr_info("NUMA disabled\n"); /* Forced off on command line. */
275 	pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end);
276 
277 	ret = numa_add_memblk(0, start, end + 1);
278 	if (ret) {
279 		pr_err("NUMA init failed\n");
280 		return ret;
281 	}
282 	node_set(0, numa_nodes_parsed);
283 
284 	numa_off = true;
285 	return 0;
286 }
287 
288 #ifdef CONFIG_ACPI_NUMA
arch_acpi_numa_init(void)289 static int __init arch_acpi_numa_init(void)
290 {
291 	int ret;
292 
293 	ret = acpi_numa_init();
294 	if (ret) {
295 		pr_debug("Failed to initialise from firmware\n");
296 		return ret;
297 	}
298 
299 	return srat_disabled() ? -EINVAL : 0;
300 }
301 #else
arch_acpi_numa_init(void)302 static int __init arch_acpi_numa_init(void)
303 {
304 	return -EOPNOTSUPP;
305 }
306 #endif
307 
308 /**
309  * arch_numa_init() - Initialize NUMA
310  *
311  * Try each configured NUMA initialization method until one succeeds. The
312  * last fallback is dummy single node config encompassing whole memory.
313  */
arch_numa_init(void)314 void __init arch_numa_init(void)
315 {
316 	if (!numa_off) {
317 		if (!acpi_disabled && !numa_init(arch_acpi_numa_init))
318 			return;
319 		if (acpi_disabled && !numa_init(of_numa_init))
320 			return;
321 	}
322 
323 	numa_init(dummy_numa_init);
324 }
325 
326 #ifdef CONFIG_NUMA_EMU
numa_emu_update_cpu_to_node(int * emu_nid_to_phys,unsigned int nr_emu_nids)327 void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys,
328 					unsigned int nr_emu_nids)
329 {
330 	int i, j;
331 
332 	/*
333 	 * Transform cpu_to_node_map table to use emulated nids by
334 	 * reverse-mapping phys_nid.  The maps should always exist but fall
335 	 * back to zero just in case.
336 	 */
337 	for (i = 0; i < ARRAY_SIZE(cpu_to_node_map); i++) {
338 		if (cpu_to_node_map[i] == NUMA_NO_NODE)
339 			continue;
340 		for (j = 0; j < nr_emu_nids; j++)
341 			if (cpu_to_node_map[i] == emu_nid_to_phys[j])
342 				break;
343 		cpu_to_node_map[i] = j < nr_emu_nids ? j : 0;
344 	}
345 }
346 
numa_emu_dma_end(void)347 u64 __init numa_emu_dma_end(void)
348 {
349 	return memblock_start_of_DRAM() + SZ_4G;
350 }
351 
debug_cpumask_set_cpu(unsigned int cpu,int node,bool enable)352 void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable)
353 {
354 	struct cpumask *mask;
355 
356 	if (node == NUMA_NO_NODE)
357 		return;
358 
359 	mask = node_to_cpumask_map[node];
360 	if (!cpumask_available(mask)) {
361 		pr_err("node_to_cpumask_map[%i] NULL\n", node);
362 		dump_stack();
363 		return;
364 	}
365 
366 	if (enable)
367 		cpumask_set_cpu(cpu, mask);
368 	else
369 		cpumask_clear_cpu(cpu, mask);
370 
371 	pr_debug("%s cpu %d node %d: mask now %*pbl\n",
372 		 enable ? "numa_add_cpu" : "numa_remove_cpu",
373 		 cpu, node, cpumask_pr_args(mask));
374 }
375 #endif /* CONFIG_NUMA_EMU */
376