1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Common code for 32 and 64-bit NUMA */
3 #include <linux/acpi.h>
4 #include <linux/kernel.h>
5 #include <linux/mm.h>
6 #include <linux/of.h>
7 #include <linux/string.h>
8 #include <linux/init.h>
9 #include <linux/memblock.h>
10 #include <linux/mmzone.h>
11 #include <linux/ctype.h>
12 #include <linux/nodemask.h>
13 #include <linux/sched.h>
14 #include <linux/topology.h>
15 #include <linux/sort.h>
16 #include <linux/numa_memblks.h>
17
18 #include <asm/e820/api.h>
19 #include <asm/proto.h>
20 #include <asm/dma.h>
21 #include <asm/amd_nb.h>
22
23 #include "numa_internal.h"
24
25 int numa_off;
26
numa_setup(char * opt)27 static __init int numa_setup(char *opt)
28 {
29 if (!opt)
30 return -EINVAL;
31 if (!strncmp(opt, "off", 3))
32 numa_off = 1;
33 if (!strncmp(opt, "fake=", 5))
34 return numa_emu_cmdline(opt + 5);
35 if (!strncmp(opt, "noacpi", 6))
36 disable_srat();
37 if (!strncmp(opt, "nohmat", 6))
38 disable_hmat();
39 return 0;
40 }
41 early_param("numa", numa_setup);
42
43 /*
44 * apicid, cpu, node mappings
45 */
46 s16 __apicid_to_node[MAX_LOCAL_APIC] = {
47 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
48 };
49
numa_cpu_node(int cpu)50 int numa_cpu_node(int cpu)
51 {
52 u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
53
54 if (apicid != BAD_APICID)
55 return __apicid_to_node[apicid];
56 return NUMA_NO_NODE;
57 }
58
59 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
60 EXPORT_SYMBOL(node_to_cpumask_map);
61
62 /*
63 * Map cpu index to node index
64 */
65 DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
66 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
67
numa_set_node(int cpu,int node)68 void numa_set_node(int cpu, int node)
69 {
70 int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
71
72 /* early setting, no percpu area yet */
73 if (cpu_to_node_map) {
74 cpu_to_node_map[cpu] = node;
75 return;
76 }
77
78 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
79 if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
80 printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
81 dump_stack();
82 return;
83 }
84 #endif
85 per_cpu(x86_cpu_to_node_map, cpu) = node;
86
87 set_cpu_numa_node(cpu, node);
88 }
89
numa_clear_node(int cpu)90 void numa_clear_node(int cpu)
91 {
92 numa_set_node(cpu, NUMA_NO_NODE);
93 }
94
95 /*
96 * Allocate node_to_cpumask_map based on number of available nodes
97 * Requires node_possible_map to be valid.
98 *
99 * Note: cpumask_of_node() is not valid until after this is done.
100 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
101 */
setup_node_to_cpumask_map(void)102 void __init setup_node_to_cpumask_map(void)
103 {
104 unsigned int node;
105
106 /* setup nr_node_ids if not done yet */
107 if (nr_node_ids == MAX_NUMNODES)
108 setup_nr_node_ids();
109
110 /* allocate the map */
111 for (node = 0; node < nr_node_ids; node++)
112 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
113
114 /* cpumask_of_node() will now work */
115 pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids);
116 }
117
numa_register_nodes(void)118 static int __init numa_register_nodes(void)
119 {
120 int nid;
121
122 if (!memblock_validate_numa_coverage(SZ_1M))
123 return -EINVAL;
124
125 /* Finally register nodes. */
126 for_each_node_mask(nid, node_possible_map) {
127 unsigned long start_pfn, end_pfn;
128
129 /*
130 * Note, get_pfn_range_for_nid() depends on
131 * memblock_set_node() having already happened
132 */
133 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
134 if (start_pfn >= end_pfn)
135 continue;
136
137 alloc_node_data(nid);
138 node_set_online(nid);
139 }
140
141 /* Dump memblock with node info and return. */
142 memblock_dump_all();
143 return 0;
144 }
145
146 /*
147 * There are unfortunately some poorly designed mainboards around that
148 * only connect memory to a single CPU. This breaks the 1:1 cpu->node
149 * mapping. To avoid this fill in the mapping for all possible CPUs,
150 * as the number of CPUs is not known yet. We round robin the existing
151 * nodes.
152 */
numa_init_array(void)153 static void __init numa_init_array(void)
154 {
155 int rr, i;
156
157 rr = first_node(node_online_map);
158 for (i = 0; i < nr_cpu_ids; i++) {
159 if (early_cpu_to_node(i) != NUMA_NO_NODE)
160 continue;
161 numa_set_node(i, rr);
162 rr = next_node_in(rr, node_online_map);
163 }
164 }
165
numa_init(int (* init_func)(void))166 static int __init numa_init(int (*init_func)(void))
167 {
168 int i;
169 int ret;
170
171 for (i = 0; i < MAX_LOCAL_APIC; i++)
172 set_apicid_to_node(i, NUMA_NO_NODE);
173
174 ret = numa_memblks_init(init_func, /* memblock_force_top_down */ true);
175 if (ret < 0)
176 return ret;
177
178 ret = numa_register_nodes();
179 if (ret < 0)
180 return ret;
181
182 for (i = 0; i < nr_cpu_ids; i++) {
183 int nid = early_cpu_to_node(i);
184
185 if (nid == NUMA_NO_NODE)
186 continue;
187 if (!node_online(nid))
188 numa_clear_node(i);
189 }
190 numa_init_array();
191
192 return 0;
193 }
194
195 /**
196 * dummy_numa_init - Fallback dummy NUMA init
197 *
198 * Used if there's no underlying NUMA architecture, NUMA initialization
199 * fails, or NUMA is disabled on the command line.
200 *
201 * Must online at least one node and add memory blocks that cover all
202 * allowed memory. This function must not fail.
203 */
dummy_numa_init(void)204 static int __init dummy_numa_init(void)
205 {
206 printk(KERN_INFO "%s\n",
207 numa_off ? "NUMA turned off" : "No NUMA configuration found");
208 printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n",
209 0LLU, PFN_PHYS(max_pfn) - 1);
210
211 node_set(0, numa_nodes_parsed);
212 numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
213
214 return 0;
215 }
216
217 /**
218 * x86_numa_init - Initialize NUMA
219 *
220 * Try each configured NUMA initialization method until one succeeds. The
221 * last fallback is dummy single node config encompassing whole memory and
222 * never fails.
223 */
x86_numa_init(void)224 void __init x86_numa_init(void)
225 {
226 if (!numa_off) {
227 #ifdef CONFIG_ACPI_NUMA
228 if (!numa_init(x86_acpi_numa_init))
229 return;
230 #endif
231 #ifdef CONFIG_AMD_NUMA
232 if (!numa_init(amd_numa_init))
233 return;
234 #endif
235 if (acpi_disabled && !numa_init(of_numa_init))
236 return;
237 }
238
239 numa_init(dummy_numa_init);
240 }
241
242
243 /*
244 * A node may exist which has one or more Generic Initiators but no CPUs and no
245 * memory.
246 *
247 * This function must be called after init_cpu_to_node(), to ensure that any
248 * memoryless CPU nodes have already been brought online, and before the
249 * node_data[nid] is needed for zone list setup in build_all_zonelists().
250 *
251 * When this function is called, any nodes containing either memory and/or CPUs
252 * will already be online and there is no need to do anything extra, even if
253 * they also contain one or more Generic Initiators.
254 */
init_gi_nodes(void)255 void __init init_gi_nodes(void)
256 {
257 int nid;
258
259 /*
260 * Exclude this node from
261 * bringup_nonboot_cpus
262 * cpu_up
263 * __try_online_node
264 * register_one_node
265 * because node_subsys is not initialized yet.
266 * TODO remove dependency on node_online
267 */
268 for_each_node_state(nid, N_GENERIC_INITIATOR)
269 if (!node_online(nid))
270 node_set_online(nid);
271 }
272
273 /*
274 * Setup early cpu_to_node.
275 *
276 * Populate cpu_to_node[] only if x86_cpu_to_apicid[],
277 * and apicid_to_node[] tables have valid entries for a CPU.
278 * This means we skip cpu_to_node[] initialisation for NUMA
279 * emulation and faking node case (when running a kernel compiled
280 * for NUMA on a non NUMA box), which is OK as cpu_to_node[]
281 * is already initialized in a round robin manner at numa_init_array,
282 * prior to this call, and this initialization is good enough
283 * for the fake NUMA cases.
284 *
285 * Called before the per_cpu areas are setup.
286 */
init_cpu_to_node(void)287 void __init init_cpu_to_node(void)
288 {
289 int cpu;
290 u32 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
291
292 BUG_ON(cpu_to_apicid == NULL);
293
294 for_each_possible_cpu(cpu) {
295 int node = numa_cpu_node(cpu);
296
297 if (node == NUMA_NO_NODE)
298 continue;
299
300 /*
301 * Exclude this node from
302 * bringup_nonboot_cpus
303 * cpu_up
304 * __try_online_node
305 * register_one_node
306 * because node_subsys is not initialized yet.
307 * TODO remove dependency on node_online
308 */
309 if (!node_online(node))
310 node_set_online(node);
311
312 numa_set_node(cpu, node);
313 }
314 }
315
316 #ifndef CONFIG_DEBUG_PER_CPU_MAPS
317
318 # ifndef CONFIG_NUMA_EMU
numa_add_cpu(unsigned int cpu)319 void numa_add_cpu(unsigned int cpu)
320 {
321 cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
322 }
323
numa_remove_cpu(unsigned int cpu)324 void numa_remove_cpu(unsigned int cpu)
325 {
326 cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
327 }
328 # endif /* !CONFIG_NUMA_EMU */
329
330 #else /* !CONFIG_DEBUG_PER_CPU_MAPS */
331
__cpu_to_node(int cpu)332 int __cpu_to_node(int cpu)
333 {
334 if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
335 printk(KERN_WARNING
336 "cpu_to_node(%d): usage too early!\n", cpu);
337 dump_stack();
338 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
339 }
340 return per_cpu(x86_cpu_to_node_map, cpu);
341 }
342 EXPORT_SYMBOL(__cpu_to_node);
343
344 /*
345 * Same function as cpu_to_node() but used if called before the
346 * per_cpu areas are setup.
347 */
early_cpu_to_node(int cpu)348 int early_cpu_to_node(int cpu)
349 {
350 if (early_per_cpu_ptr(x86_cpu_to_node_map))
351 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
352
353 if (!cpu_possible(cpu)) {
354 printk(KERN_WARNING
355 "early_cpu_to_node(%d): no per_cpu area!\n", cpu);
356 dump_stack();
357 return NUMA_NO_NODE;
358 }
359 return per_cpu(x86_cpu_to_node_map, cpu);
360 }
361
debug_cpumask_set_cpu(unsigned int cpu,int node,bool enable)362 void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable)
363 {
364 struct cpumask *mask;
365
366 if (node == NUMA_NO_NODE) {
367 /* early_cpu_to_node() already emits a warning and trace */
368 return;
369 }
370 mask = node_to_cpumask_map[node];
371 if (!cpumask_available(mask)) {
372 pr_err("node_to_cpumask_map[%i] NULL\n", node);
373 dump_stack();
374 return;
375 }
376
377 if (enable)
378 cpumask_set_cpu(cpu, mask);
379 else
380 cpumask_clear_cpu(cpu, mask);
381
382 printk(KERN_DEBUG "%s cpu %d node %d: mask now %*pbl\n",
383 enable ? "numa_add_cpu" : "numa_remove_cpu",
384 cpu, node, cpumask_pr_args(mask));
385 return;
386 }
387
388 # ifndef CONFIG_NUMA_EMU
numa_set_cpumask(int cpu,bool enable)389 static void numa_set_cpumask(int cpu, bool enable)
390 {
391 debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable);
392 }
393
numa_add_cpu(unsigned int cpu)394 void numa_add_cpu(unsigned int cpu)
395 {
396 numa_set_cpumask(cpu, true);
397 }
398
numa_remove_cpu(unsigned int cpu)399 void numa_remove_cpu(unsigned int cpu)
400 {
401 numa_set_cpumask(cpu, false);
402 }
403 # endif /* !CONFIG_NUMA_EMU */
404
405 /*
406 * Returns a pointer to the bitmask of CPUs on Node 'node'.
407 */
cpumask_of_node(int node)408 const struct cpumask *cpumask_of_node(int node)
409 {
410 if ((unsigned)node >= nr_node_ids) {
411 printk(KERN_WARNING
412 "cpumask_of_node(%d): (unsigned)node >= nr_node_ids(%u)\n",
413 node, nr_node_ids);
414 dump_stack();
415 return cpu_none_mask;
416 }
417 if (!cpumask_available(node_to_cpumask_map[node])) {
418 printk(KERN_WARNING
419 "cpumask_of_node(%d): no node_to_cpumask_map!\n",
420 node);
421 dump_stack();
422 return cpu_online_mask;
423 }
424 return node_to_cpumask_map[node];
425 }
426 EXPORT_SYMBOL(cpumask_of_node);
427
428 #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
429
430 #ifdef CONFIG_NUMA_EMU
numa_emu_update_cpu_to_node(int * emu_nid_to_phys,unsigned int nr_emu_nids)431 void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys,
432 unsigned int nr_emu_nids)
433 {
434 int i, j;
435
436 /*
437 * Transform __apicid_to_node table to use emulated nids by
438 * reverse-mapping phys_nid. The maps should always exist but fall
439 * back to zero just in case.
440 */
441 for (i = 0; i < ARRAY_SIZE(__apicid_to_node); i++) {
442 if (__apicid_to_node[i] == NUMA_NO_NODE)
443 continue;
444 for (j = 0; j < nr_emu_nids; j++)
445 if (__apicid_to_node[i] == emu_nid_to_phys[j])
446 break;
447 __apicid_to_node[i] = j < nr_emu_nids ? j : 0;
448 }
449 }
450
numa_emu_dma_end(void)451 u64 __init numa_emu_dma_end(void)
452 {
453 return PFN_PHYS(MAX_DMA32_PFN);
454 }
455 #endif /* CONFIG_NUMA_EMU */
456