1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_IDTENTRY_H
3 #define _ASM_X86_IDTENTRY_H
4
5 /* Interrupts/Exceptions */
6 #include <asm/trapnr.h>
7
8 #define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT))
9
10 #ifndef __ASSEMBLY__
11 #include <linux/entry-common.h>
12 #include <linux/hardirq.h>
13
14 #include <asm/irq_stack.h>
15
16 typedef void (*idtentry_t)(struct pt_regs *regs);
17
18 /**
19 * DECLARE_IDTENTRY - Declare functions for simple IDT entry points
20 * No error code pushed by hardware
21 * @vector: Vector number (ignored for C)
22 * @func: Function name of the entry point
23 *
24 * Declares four functions:
25 * - The ASM entry point: asm_##func
26 * - The XEN PV trap entry point: xen_##func (maybe unused)
27 * - The C handler called from the FRED event dispatcher (maybe unused)
28 * - The C handler called from the ASM entry point
29 *
30 * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it
31 * declares the entry points for usage in C code. There is an ASM variant
32 * as well which is used to emit the entry stubs in entry_32/64.S.
33 */
34 #define DECLARE_IDTENTRY(vector, func) \
35 asmlinkage void asm_##func(void); \
36 asmlinkage void xen_asm_##func(void); \
37 void fred_##func(struct pt_regs *regs); \
38 __visible void func(struct pt_regs *regs)
39
40 /**
41 * DEFINE_IDTENTRY - Emit code for simple IDT entry points
42 * @func: Function name of the entry point
43 *
44 * @func is called from ASM entry code with interrupts disabled.
45 *
46 * The macro is written so it acts as function definition. Append the
47 * body with a pair of curly brackets.
48 *
49 * irqentry_enter() contains common code which has to be invoked before
50 * arbitrary code in the body. irqentry_exit() contains common code
51 * which has to run before returning to the low level assembly code.
52 */
53 #define DEFINE_IDTENTRY(func) \
54 static __always_inline void __##func(struct pt_regs *regs); \
55 \
56 __visible noinstr void func(struct pt_regs *regs) \
57 { \
58 irqentry_state_t state = irqentry_enter(regs); \
59 \
60 instrumentation_begin(); \
61 __##func (regs); \
62 instrumentation_end(); \
63 irqentry_exit(regs, state); \
64 } \
65 \
66 static __always_inline void __##func(struct pt_regs *regs)
67
68 /* Special case for 32bit IRET 'trap' */
69 #define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY
70 #define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY
71
72 /**
73 * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points
74 * Error code pushed by hardware
75 * @vector: Vector number (ignored for C)
76 * @func: Function name of the entry point
77 *
78 * Declares three functions:
79 * - The ASM entry point: asm_##func
80 * - The XEN PV trap entry point: xen_##func (maybe unused)
81 * - The C handler called from the ASM entry point
82 *
83 * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the
84 * C-handler.
85 */
86 #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
87 asmlinkage void asm_##func(void); \
88 asmlinkage void xen_asm_##func(void); \
89 __visible void func(struct pt_regs *regs, unsigned long error_code)
90
91 /**
92 * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points
93 * Error code pushed by hardware
94 * @func: Function name of the entry point
95 *
96 * Same as DEFINE_IDTENTRY, but has an extra error_code argument
97 */
98 #define DEFINE_IDTENTRY_ERRORCODE(func) \
99 static __always_inline void __##func(struct pt_regs *regs, \
100 unsigned long error_code); \
101 \
102 __visible noinstr void func(struct pt_regs *regs, \
103 unsigned long error_code) \
104 { \
105 irqentry_state_t state = irqentry_enter(regs); \
106 \
107 instrumentation_begin(); \
108 __##func (regs, error_code); \
109 instrumentation_end(); \
110 irqentry_exit(regs, state); \
111 } \
112 \
113 static __always_inline void __##func(struct pt_regs *regs, \
114 unsigned long error_code)
115
116 /**
117 * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points
118 * No error code pushed by hardware
119 * @vector: Vector number (ignored for C)
120 * @func: Function name of the entry point
121 *
122 * Maps to DECLARE_IDTENTRY().
123 */
124 #define DECLARE_IDTENTRY_RAW(vector, func) \
125 DECLARE_IDTENTRY(vector, func)
126
127 /**
128 * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points
129 * @func: Function name of the entry point
130 *
131 * @func is called from ASM entry code with interrupts disabled.
132 *
133 * The macro is written so it acts as function definition. Append the
134 * body with a pair of curly brackets.
135 *
136 * Contrary to DEFINE_IDTENTRY() this does not invoke the
137 * idtentry_enter/exit() helpers before and after the body invocation. This
138 * needs to be done in the body itself if applicable. Use if extra work
139 * is required before the enter/exit() helpers are invoked.
140 */
141 #define DEFINE_IDTENTRY_RAW(func) \
142 __visible noinstr void func(struct pt_regs *regs)
143
144 /**
145 * DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points
146 * @func: Function name of the entry point
147 *
148 * @func is called from the FRED event dispatcher with interrupts disabled.
149 *
150 * See @DEFINE_IDTENTRY_RAW for further details.
151 */
152 #define DEFINE_FREDENTRY_RAW(func) \
153 noinstr void fred_##func(struct pt_regs *regs)
154
155 /**
156 * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points
157 * Error code pushed by hardware
158 * @vector: Vector number (ignored for C)
159 * @func: Function name of the entry point
160 *
161 * Maps to DECLARE_IDTENTRY_ERRORCODE()
162 */
163 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \
164 DECLARE_IDTENTRY_ERRORCODE(vector, func)
165
166 /**
167 * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points
168 * @func: Function name of the entry point
169 *
170 * @func is called from ASM entry code with interrupts disabled.
171 *
172 * The macro is written so it acts as function definition. Append the
173 * body with a pair of curly brackets.
174 *
175 * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the
176 * irqentry_enter/exit() helpers before and after the body invocation. This
177 * needs to be done in the body itself if applicable. Use if extra work
178 * is required before the enter/exit() helpers are invoked.
179 */
180 #define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \
181 __visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
182
183 /**
184 * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
185 * points (common/spurious)
186 * @vector: Vector number (ignored for C)
187 * @func: Function name of the entry point
188 *
189 * Maps to DECLARE_IDTENTRY_ERRORCODE()
190 */
191 #define DECLARE_IDTENTRY_IRQ(vector, func) \
192 DECLARE_IDTENTRY_ERRORCODE(vector, func)
193
194 /**
195 * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points
196 * @func: Function name of the entry point
197 *
198 * The vector number is pushed by the low level entry stub and handed
199 * to the function as error_code argument which needs to be truncated
200 * to an u8 because the push is sign extending.
201 *
202 * irq_enter/exit_rcu() are invoked before the function body and the
203 * KVM L1D flush request is set. Stack switching to the interrupt stack
204 * has to be done in the function body if necessary.
205 */
206 #define DEFINE_IDTENTRY_IRQ(func) \
207 static void __##func(struct pt_regs *regs, u32 vector); \
208 \
209 __visible noinstr void func(struct pt_regs *regs, \
210 unsigned long error_code) \
211 { \
212 irqentry_state_t state = irqentry_enter(regs); \
213 u32 vector = (u32)(u8)error_code; \
214 \
215 kvm_set_cpu_l1tf_flush_l1d(); \
216 instrumentation_begin(); \
217 run_irq_on_irqstack_cond(__##func, regs, vector); \
218 instrumentation_end(); \
219 irqentry_exit(regs, state); \
220 } \
221 \
222 static noinline void __##func(struct pt_regs *regs, u32 vector)
223
224 /**
225 * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points
226 * @vector: Vector number (ignored for C)
227 * @func: Function name of the entry point
228 *
229 * Declares three functions:
230 * - The ASM entry point: asm_##func
231 * - The XEN PV trap entry point: xen_##func (maybe unused)
232 * - The C handler called from the ASM entry point
233 *
234 * Maps to DECLARE_IDTENTRY().
235 */
236 #define DECLARE_IDTENTRY_SYSVEC(vector, func) \
237 DECLARE_IDTENTRY(vector, func)
238
239 /**
240 * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points
241 * @func: Function name of the entry point
242 *
243 * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the
244 * function body. KVM L1D flush request is set.
245 *
246 * Runs the function on the interrupt stack if the entry hit kernel mode
247 */
248 #define DEFINE_IDTENTRY_SYSVEC(func) \
249 static void __##func(struct pt_regs *regs); \
250 \
251 static __always_inline void instr_##func(struct pt_regs *regs) \
252 { \
253 run_sysvec_on_irqstack_cond(__##func, regs); \
254 } \
255 \
256 __visible noinstr void func(struct pt_regs *regs) \
257 { \
258 irqentry_state_t state = irqentry_enter(regs); \
259 \
260 kvm_set_cpu_l1tf_flush_l1d(); \
261 instrumentation_begin(); \
262 instr_##func (regs); \
263 instrumentation_end(); \
264 irqentry_exit(regs, state); \
265 } \
266 \
267 void fred_##func(struct pt_regs *regs) \
268 { \
269 instr_##func (regs); \
270 } \
271 \
272 static noinline void __##func(struct pt_regs *regs)
273
274 /**
275 * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT
276 * entry points
277 * @func: Function name of the entry point
278 *
279 * Runs the function on the interrupted stack. No switch to IRQ stack and
280 * only the minimal __irq_enter/exit() handling.
281 *
282 * Only use for 'empty' vectors like reschedule IPI and KVM posted
283 * interrupt vectors.
284 */
285 #define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \
286 static __always_inline void __##func(struct pt_regs *regs); \
287 \
288 static __always_inline void instr_##func(struct pt_regs *regs) \
289 { \
290 __irq_enter_raw(); \
291 __##func (regs); \
292 __irq_exit_raw(); \
293 } \
294 \
295 __visible noinstr void func(struct pt_regs *regs) \
296 { \
297 irqentry_state_t state = irqentry_enter(regs); \
298 \
299 kvm_set_cpu_l1tf_flush_l1d(); \
300 instrumentation_begin(); \
301 instr_##func (regs); \
302 instrumentation_end(); \
303 irqentry_exit(regs, state); \
304 } \
305 \
306 void fred_##func(struct pt_regs *regs) \
307 { \
308 instr_##func (regs); \
309 } \
310 \
311 static __always_inline void __##func(struct pt_regs *regs)
312
313 /**
314 * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point
315 * @vector: Vector number (ignored for C)
316 * @func: Function name of the entry point
317 *
318 * Declares three functions:
319 * - The ASM entry point: asm_##func
320 * - The XEN PV trap entry point: xen_##func (maybe unused)
321 * - The C handler called from the ASM entry point
322 *
323 * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit
324 * difference
325 */
326 #define DECLARE_IDTENTRY_XENCB(vector, func) \
327 DECLARE_IDTENTRY(vector, func)
328
329 #ifdef CONFIG_X86_64
330 /**
331 * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points
332 * @vector: Vector number (ignored for C)
333 * @func: Function name of the entry point
334 *
335 * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler
336 * which is called from the ASM entry point on user mode entry
337 */
338 #define DECLARE_IDTENTRY_IST(vector, func) \
339 DECLARE_IDTENTRY_RAW(vector, func); \
340 __visible void noist_##func(struct pt_regs *regs)
341
342 /**
343 * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
344 * @vector: Vector number (ignored for C)
345 * @func: Function name of the entry point
346 *
347 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the
348 * safe_stack C handler.
349 */
350 #define DECLARE_IDTENTRY_VC(vector, func) \
351 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \
352 __visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \
353 __visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code)
354
355 /**
356 * DEFINE_IDTENTRY_IST - Emit code for IST entry points
357 * @func: Function name of the entry point
358 *
359 * Maps to DEFINE_IDTENTRY_RAW
360 */
361 #define DEFINE_IDTENTRY_IST(func) \
362 DEFINE_IDTENTRY_RAW(func)
363
364 /**
365 * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which
366 * belong to a IST entry point (MCE, DB)
367 * @func: Function name of the entry point. Must be the same as
368 * the function name of the corresponding IST variant
369 *
370 * Maps to DEFINE_IDTENTRY_RAW().
371 */
372 #define DEFINE_IDTENTRY_NOIST(func) \
373 DEFINE_IDTENTRY_RAW(noist_##func)
374
375 /**
376 * DECLARE_IDTENTRY_DF - Declare functions for double fault
377 * @vector: Vector number (ignored for C)
378 * @func: Function name of the entry point
379 *
380 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE
381 */
382 #define DECLARE_IDTENTRY_DF(vector, func) \
383 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)
384
385 /**
386 * DEFINE_IDTENTRY_DF - Emit code for double fault
387 * @func: Function name of the entry point
388 *
389 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
390 */
391 #define DEFINE_IDTENTRY_DF(func) \
392 DEFINE_IDTENTRY_RAW_ERRORCODE(func)
393
394 /**
395 * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
396 when raised from kernel mode
397 * @func: Function name of the entry point
398 *
399 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
400 */
401 #define DEFINE_IDTENTRY_VC_KERNEL(func) \
402 DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func)
403
404 /**
405 * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
406 when raised from user mode
407 * @func: Function name of the entry point
408 *
409 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
410 */
411 #define DEFINE_IDTENTRY_VC_USER(func) \
412 DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func)
413
414 #else /* CONFIG_X86_64 */
415
416 /**
417 * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant
418 * @vector: Vector number (ignored for C)
419 * @func: Function name of the entry point
420 *
421 * Declares two functions:
422 * - The ASM entry point: asm_##func
423 * - The C handler called from the C shim
424 */
425 #define DECLARE_IDTENTRY_DF(vector, func) \
426 asmlinkage void asm_##func(void); \
427 __visible void func(struct pt_regs *regs, \
428 unsigned long error_code, \
429 unsigned long address)
430
431 /**
432 * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit
433 * @func: Function name of the entry point
434 *
435 * This is called through the doublefault shim which already provides
436 * cr2 in the address argument.
437 */
438 #define DEFINE_IDTENTRY_DF(func) \
439 __visible noinstr void func(struct pt_regs *regs, \
440 unsigned long error_code, \
441 unsigned long address)
442
443 #endif /* !CONFIG_X86_64 */
444
445 /* C-Code mapping */
446 #define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW
447 #define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW
448 #define DEFINE_FREDENTRY_NMI DEFINE_FREDENTRY_RAW
449
450 #ifdef CONFIG_X86_64
451 #define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST
452 #define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST
453 #define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST
454 #define DEFINE_FREDENTRY_MCE DEFINE_FREDENTRY_RAW
455
456 #define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST
457 #define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST
458 #define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST
459 #define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW
460 #endif
461
462 void idt_install_sysvec(unsigned int n, const void *function);
463
464 #ifdef CONFIG_X86_FRED
465 void fred_install_sysvec(unsigned int vector, const idtentry_t function);
466 #else
fred_install_sysvec(unsigned int vector,const idtentry_t function)467 static inline void fred_install_sysvec(unsigned int vector, const idtentry_t function) { }
468 #endif
469
470 #define sysvec_install(vector, function) { \
471 if (cpu_feature_enabled(X86_FEATURE_FRED)) \
472 fred_install_sysvec(vector, function); \
473 else \
474 idt_install_sysvec(vector, asm_##function); \
475 }
476
477 #else /* !__ASSEMBLY__ */
478
479 /*
480 * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs.
481 */
482 #define DECLARE_IDTENTRY(vector, func) \
483 idtentry vector asm_##func func has_error_code=0
484
485 #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
486 idtentry vector asm_##func func has_error_code=1
487
488 /* Special case for 32bit IRET 'trap'. Do not emit ASM code */
489 #define DECLARE_IDTENTRY_SW(vector, func)
490
491 #define DECLARE_IDTENTRY_RAW(vector, func) \
492 DECLARE_IDTENTRY(vector, func)
493
494 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \
495 DECLARE_IDTENTRY_ERRORCODE(vector, func)
496
497 /* Entries for common/spurious (device) interrupts */
498 #define DECLARE_IDTENTRY_IRQ(vector, func) \
499 idtentry_irq vector func
500
501 /* System vector entries */
502 #define DECLARE_IDTENTRY_SYSVEC(vector, func) \
503 DECLARE_IDTENTRY(vector, func)
504
505 #ifdef CONFIG_X86_64
506 # define DECLARE_IDTENTRY_MCE(vector, func) \
507 idtentry_mce_db vector asm_##func func
508
509 # define DECLARE_IDTENTRY_DEBUG(vector, func) \
510 idtentry_mce_db vector asm_##func func
511
512 # define DECLARE_IDTENTRY_DF(vector, func) \
513 idtentry_df vector asm_##func func
514
515 # define DECLARE_IDTENTRY_XENCB(vector, func) \
516 DECLARE_IDTENTRY(vector, func)
517
518 # define DECLARE_IDTENTRY_VC(vector, func) \
519 idtentry_vc vector asm_##func func
520
521 #else
522 # define DECLARE_IDTENTRY_MCE(vector, func) \
523 DECLARE_IDTENTRY(vector, func)
524
525 /* No ASM emitted for DF as this goes through a C shim */
526 # define DECLARE_IDTENTRY_DF(vector, func)
527
528 /* No ASM emitted for XEN hypervisor callback */
529 # define DECLARE_IDTENTRY_XENCB(vector, func)
530
531 #endif
532
533 /* No ASM code emitted for NMI */
534 #define DECLARE_IDTENTRY_NMI(vector, func)
535
536 /*
537 * ASM code to emit the common vector entry stubs where each stub is
538 * packed into IDT_ALIGN bytes.
539 *
540 * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
541 * GCC treats the local vector variable as unsigned int and would expand
542 * all vectors above 0x7F to a 5 byte push. The original code did an
543 * adjustment of the vector number to be in the signed byte range to avoid
544 * this. While clever it's mindboggling counterintuitive and requires the
545 * odd conversion back to a real vector number in the C entry points. Using
546 * .byte achieves the same thing and the only fixup needed in the C entry
547 * point is to mask off the bits above bit 7 because the push is sign
548 * extending.
549 */
550 .align IDT_ALIGN
551 SYM_CODE_START(irq_entries_start)
552 vector=FIRST_EXTERNAL_VECTOR
553 .rept NR_EXTERNAL_VECTORS
554 UNWIND_HINT_IRET_REGS
555 0 :
556 ENDBR
557 .byte 0x6a, vector
558 jmp asm_common_interrupt
559 /* Ensure that the above is IDT_ALIGN bytes max */
560 .fill 0b + IDT_ALIGN - ., 1, 0xcc
561 vector = vector+1
562 .endr
563 SYM_CODE_END(irq_entries_start)
564
565 #ifdef CONFIG_X86_LOCAL_APIC
566 .align IDT_ALIGN
567 SYM_CODE_START(spurious_entries_start)
568 vector=FIRST_SYSTEM_VECTOR
569 .rept NR_SYSTEM_VECTORS
570 UNWIND_HINT_IRET_REGS
571 0 :
572 ENDBR
573 .byte 0x6a, vector
574 jmp asm_spurious_interrupt
575 /* Ensure that the above is IDT_ALIGN bytes max */
576 .fill 0b + IDT_ALIGN - ., 1, 0xcc
577 vector = vector+1
578 .endr
579 SYM_CODE_END(spurious_entries_start)
580 #endif
581
582 #endif /* __ASSEMBLY__ */
583
584 /*
585 * The actual entry points. Note that DECLARE_IDTENTRY*() serves two
586 * purposes:
587 * - provide the function declarations when included from C-Code
588 * - emit the ASM stubs when included from entry_32/64.S
589 *
590 * This avoids duplicate defines and ensures that everything is consistent.
591 */
592
593 /*
594 * Dummy trap number so the low level ASM macro vector number checks do not
595 * match which results in emitting plain IDTENTRY stubs without bells and
596 * whistles.
597 */
598 #define X86_TRAP_OTHER 0xFFFF
599
600 /* Simple exception entry points. No hardware error code */
601 DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error);
602 DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow);
603 DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds);
604 DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available);
605 DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun);
606 DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug);
607 DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error);
608 DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error);
609
610 /* 32bit software IRET trap. Do not emit ASM code */
611 DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error);
612
613 /* Simple exception entries with error code pushed by hardware */
614 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss);
615 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present);
616 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment);
617 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection);
618 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check);
619
620 /* Raw exception entries which need extra work */
621 DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op);
622 DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3);
623 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault);
624
625 #if defined(CONFIG_IA32_EMULATION)
626 DECLARE_IDTENTRY_RAW(IA32_SYSCALL_VECTOR, int80_emulation);
627 #endif
628
629 #ifdef CONFIG_X86_MCE
630 #ifdef CONFIG_X86_64
631 DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check);
632 #else
633 DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check);
634 #endif
635 #ifdef CONFIG_XEN_PV
636 DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check);
637 #endif
638 #endif
639
640 /* NMI */
641
642 #if IS_ENABLED(CONFIG_KVM_INTEL)
643 /*
644 * Special entry point for VMX which invokes this on the kernel stack, even for
645 * 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work
646 * correctly vs. the NMI 'executing' marker. Used for 32-bit kernels as well
647 * to avoid more ifdeffery.
648 */
649 DECLARE_IDTENTRY(X86_TRAP_NMI, exc_nmi_kvm_vmx);
650 #endif
651
652 DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi);
653 #ifdef CONFIG_XEN_PV
654 DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi);
655 #endif
656
657 /* #DB */
658 #ifdef CONFIG_X86_64
659 DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug);
660 #else
661 DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug);
662 #endif
663 #ifdef CONFIG_XEN_PV
664 DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug);
665 #endif
666
667 /* #DF */
668 DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault);
669 #ifdef CONFIG_XEN_PV
670 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault);
671 #endif
672
673 /* #CP */
674 #ifdef CONFIG_X86_CET
675 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection);
676 #endif
677
678 /* #VC */
679 #ifdef CONFIG_AMD_MEM_ENCRYPT
680 DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication);
681 #endif
682
683 #ifdef CONFIG_XEN_PV
684 DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback);
685 DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER, exc_xen_unknown_trap);
686 #endif
687
688 #ifdef CONFIG_INTEL_TDX_GUEST
689 DECLARE_IDTENTRY(X86_TRAP_VE, exc_virtualization_exception);
690 #endif
691
692 /* Device interrupts common/spurious */
693 DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt);
694 #ifdef CONFIG_X86_LOCAL_APIC
695 DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt);
696 #endif
697
698 /* System vector entry points */
699 #ifdef CONFIG_X86_LOCAL_APIC
700 DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt);
701 DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt);
702 DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt);
703 DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi);
704 #endif
705
706 #ifdef CONFIG_SMP
707 DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi);
708 DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot);
709 DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single);
710 DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function);
711 #else
712 # define fred_sysvec_reschedule_ipi NULL
713 # define fred_sysvec_reboot NULL
714 # define fred_sysvec_call_function_single NULL
715 # define fred_sysvec_call_function NULL
716 #endif
717
718 #ifdef CONFIG_X86_LOCAL_APIC
719 # ifdef CONFIG_X86_MCE_THRESHOLD
720 DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold);
721 # else
722 # define fred_sysvec_threshold NULL
723 # endif
724
725 # ifdef CONFIG_X86_MCE_AMD
726 DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error);
727 # else
728 # define fred_sysvec_deferred_error NULL
729 # endif
730
731 # ifdef CONFIG_X86_THERMAL_VECTOR
732 DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal);
733 # else
734 # define fred_sysvec_thermal NULL
735 # endif
736
737 # ifdef CONFIG_IRQ_WORK
738 DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work);
739 # else
740 # define fred_sysvec_irq_work NULL
741 # endif
742 #endif
743
744 #if IS_ENABLED(CONFIG_KVM)
745 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi);
746 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi);
747 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi);
748 #else
749 # define fred_sysvec_kvm_posted_intr_ipi NULL
750 # define fred_sysvec_kvm_posted_intr_wakeup_ipi NULL
751 # define fred_sysvec_kvm_posted_intr_nested_ipi NULL
752 #endif
753
754 # ifdef CONFIG_X86_POSTED_MSI
755 DECLARE_IDTENTRY_SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, sysvec_posted_msi_notification);
756 #else
757 # define fred_sysvec_posted_msi_notification NULL
758 # endif
759
760 #if IS_ENABLED(CONFIG_HYPERV)
761 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback);
762 DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment);
763 DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0);
764 #endif
765
766 #if IS_ENABLED(CONFIG_ACRN_GUEST)
767 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback);
768 #endif
769
770 #ifdef CONFIG_XEN_PVHVM
771 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback);
772 #endif
773
774 #ifdef CONFIG_KVM_GUEST
775 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt);
776 #endif
777
778 #undef X86_TRAP_OTHER
779
780 #endif
781