1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/efi.h>
4 #include <linux/memblock.h>
5 #include <linux/spinlock.h>
6 #include <linux/crash_dump.h>
7 #include <linux/nmi.h>
8 #include <asm/unaccepted_memory.h>
9
10 /* Protects unaccepted memory bitmap and accepting_list */
11 static DEFINE_SPINLOCK(unaccepted_memory_lock);
12
13 struct accept_range {
14 struct list_head list;
15 unsigned long start;
16 unsigned long end;
17 };
18
19 static LIST_HEAD(accepting_list);
20
21 /*
22 * accept_memory() -- Consult bitmap and accept the memory if needed.
23 *
24 * Only memory that is explicitly marked as unaccepted in the bitmap requires
25 * an action. All the remaining memory is implicitly accepted and doesn't need
26 * acceptance.
27 *
28 * No need to accept:
29 * - anything if the system has no unaccepted table;
30 * - memory that is below phys_base;
31 * - memory that is above the memory that addressable by the bitmap;
32 */
accept_memory(phys_addr_t start,unsigned long size)33 void accept_memory(phys_addr_t start, unsigned long size)
34 {
35 struct efi_unaccepted_memory *unaccepted;
36 unsigned long range_start, range_end;
37 struct accept_range range, *entry;
38 phys_addr_t end = start + size;
39 unsigned long flags;
40 u64 unit_size;
41
42 unaccepted = efi_get_unaccepted_table();
43 if (!unaccepted)
44 return;
45
46 unit_size = unaccepted->unit_size;
47
48 /*
49 * Only care for the part of the range that is represented
50 * in the bitmap.
51 */
52 if (start < unaccepted->phys_base)
53 start = unaccepted->phys_base;
54 if (end < unaccepted->phys_base)
55 return;
56
57 /* Translate to offsets from the beginning of the bitmap */
58 start -= unaccepted->phys_base;
59 end -= unaccepted->phys_base;
60
61 /*
62 * load_unaligned_zeropad() can lead to unwanted loads across page
63 * boundaries. The unwanted loads are typically harmless. But, they
64 * might be made to totally unrelated or even unmapped memory.
65 * load_unaligned_zeropad() relies on exception fixup (#PF, #GP and now
66 * #VE) to recover from these unwanted loads.
67 *
68 * But, this approach does not work for unaccepted memory. For TDX, a
69 * load from unaccepted memory will not lead to a recoverable exception
70 * within the guest. The guest will exit to the VMM where the only
71 * recourse is to terminate the guest.
72 *
73 * There are two parts to fix this issue and comprehensively avoid
74 * access to unaccepted memory. Together these ensure that an extra
75 * "guard" page is accepted in addition to the memory that needs to be
76 * used:
77 *
78 * 1. Implicitly extend the range_contains_unaccepted_memory(start, size)
79 * checks up to the next unit_size if 'start+size' is aligned on a
80 * unit_size boundary.
81 *
82 * 2. Implicitly extend accept_memory(start, size) to the next unit_size
83 * if 'size+end' is aligned on a unit_size boundary. (immediately
84 * following this comment)
85 */
86 if (!(end % unit_size))
87 end += unit_size;
88
89 /* Make sure not to overrun the bitmap */
90 if (end > unaccepted->size * unit_size * BITS_PER_BYTE)
91 end = unaccepted->size * unit_size * BITS_PER_BYTE;
92
93 range.start = start / unit_size;
94 range.end = DIV_ROUND_UP(end, unit_size);
95 retry:
96 spin_lock_irqsave(&unaccepted_memory_lock, flags);
97
98 /*
99 * Check if anybody works on accepting the same range of the memory.
100 *
101 * The check is done with unit_size granularity. It is crucial to catch
102 * all accept requests to the same unit_size block, even if they don't
103 * overlap on physical address level.
104 */
105 list_for_each_entry(entry, &accepting_list, list) {
106 if (entry->end <= range.start)
107 continue;
108 if (entry->start >= range.end)
109 continue;
110
111 /*
112 * Somebody else accepting the range. Or at least part of it.
113 *
114 * Drop the lock and retry until it is complete.
115 */
116 spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
117 goto retry;
118 }
119
120 /*
121 * Register that the range is about to be accepted.
122 * Make sure nobody else will accept it.
123 */
124 list_add(&range.list, &accepting_list);
125
126 range_start = range.start;
127 for_each_set_bitrange_from(range_start, range_end, unaccepted->bitmap,
128 range.end) {
129 unsigned long phys_start, phys_end;
130 unsigned long len = range_end - range_start;
131
132 phys_start = range_start * unit_size + unaccepted->phys_base;
133 phys_end = range_end * unit_size + unaccepted->phys_base;
134
135 /*
136 * Keep interrupts disabled until the accept operation is
137 * complete in order to prevent deadlocks.
138 *
139 * Enabling interrupts before calling arch_accept_memory()
140 * creates an opportunity for an interrupt handler to request
141 * acceptance for the same memory. The handler will continuously
142 * spin with interrupts disabled, preventing other task from
143 * making progress with the acceptance process.
144 */
145 spin_unlock(&unaccepted_memory_lock);
146
147 arch_accept_memory(phys_start, phys_end);
148
149 spin_lock(&unaccepted_memory_lock);
150 bitmap_clear(unaccepted->bitmap, range_start, len);
151 }
152
153 list_del(&range.list);
154
155 touch_softlockup_watchdog();
156
157 spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
158 }
159
range_contains_unaccepted_memory(phys_addr_t start,unsigned long size)160 bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size)
161 {
162 struct efi_unaccepted_memory *unaccepted;
163 phys_addr_t end = start + size;
164 unsigned long flags;
165 bool ret = false;
166 u64 unit_size;
167
168 unaccepted = efi_get_unaccepted_table();
169 if (!unaccepted)
170 return false;
171
172 unit_size = unaccepted->unit_size;
173
174 /*
175 * Only care for the part of the range that is represented
176 * in the bitmap.
177 */
178 if (start < unaccepted->phys_base)
179 start = unaccepted->phys_base;
180 if (end < unaccepted->phys_base)
181 return false;
182
183 /* Translate to offsets from the beginning of the bitmap */
184 start -= unaccepted->phys_base;
185 end -= unaccepted->phys_base;
186
187 /*
188 * Also consider the unaccepted state of the *next* page. See fix #1 in
189 * the comment on load_unaligned_zeropad() in accept_memory().
190 */
191 if (!(end % unit_size))
192 end += unit_size;
193
194 /* Make sure not to overrun the bitmap */
195 if (end > unaccepted->size * unit_size * BITS_PER_BYTE)
196 end = unaccepted->size * unit_size * BITS_PER_BYTE;
197
198 spin_lock_irqsave(&unaccepted_memory_lock, flags);
199 while (start < end) {
200 if (test_bit(start / unit_size, unaccepted->bitmap)) {
201 ret = true;
202 break;
203 }
204
205 start += unit_size;
206 }
207 spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
208
209 return ret;
210 }
211
212 #ifdef CONFIG_PROC_VMCORE
unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb * cb,unsigned long pfn)213 static bool unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb *cb,
214 unsigned long pfn)
215 {
216 return !pfn_is_unaccepted_memory(pfn);
217 }
218
219 static struct vmcore_cb vmcore_cb = {
220 .pfn_is_ram = unaccepted_memory_vmcore_pfn_is_ram,
221 };
222
unaccepted_memory_init_kdump(void)223 static int __init unaccepted_memory_init_kdump(void)
224 {
225 register_vmcore_cb(&vmcore_cb);
226 return 0;
227 }
228 core_initcall(unaccepted_memory_init_kdump);
229 #endif /* CONFIG_PROC_VMCORE */
230