1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/perf_event.h>
10 #include <linux/irq.h>
11 #include <linux/stringify.h>
12
13 #include <asm/processor.h>
14 #include <asm/ptrace.h>
15 #include <asm/csr.h>
16 #include <asm/entry-common.h>
17 #include <asm/hwprobe.h>
18 #include <asm/cpufeature.h>
19
20 #define INSN_MATCH_LB 0x3
21 #define INSN_MASK_LB 0x707f
22 #define INSN_MATCH_LH 0x1003
23 #define INSN_MASK_LH 0x707f
24 #define INSN_MATCH_LW 0x2003
25 #define INSN_MASK_LW 0x707f
26 #define INSN_MATCH_LD 0x3003
27 #define INSN_MASK_LD 0x707f
28 #define INSN_MATCH_LBU 0x4003
29 #define INSN_MASK_LBU 0x707f
30 #define INSN_MATCH_LHU 0x5003
31 #define INSN_MASK_LHU 0x707f
32 #define INSN_MATCH_LWU 0x6003
33 #define INSN_MASK_LWU 0x707f
34 #define INSN_MATCH_SB 0x23
35 #define INSN_MASK_SB 0x707f
36 #define INSN_MATCH_SH 0x1023
37 #define INSN_MASK_SH 0x707f
38 #define INSN_MATCH_SW 0x2023
39 #define INSN_MASK_SW 0x707f
40 #define INSN_MATCH_SD 0x3023
41 #define INSN_MASK_SD 0x707f
42
43 #define INSN_MATCH_FLW 0x2007
44 #define INSN_MASK_FLW 0x707f
45 #define INSN_MATCH_FLD 0x3007
46 #define INSN_MASK_FLD 0x707f
47 #define INSN_MATCH_FLQ 0x4007
48 #define INSN_MASK_FLQ 0x707f
49 #define INSN_MATCH_FSW 0x2027
50 #define INSN_MASK_FSW 0x707f
51 #define INSN_MATCH_FSD 0x3027
52 #define INSN_MASK_FSD 0x707f
53 #define INSN_MATCH_FSQ 0x4027
54 #define INSN_MASK_FSQ 0x707f
55
56 #define INSN_MATCH_C_LD 0x6000
57 #define INSN_MASK_C_LD 0xe003
58 #define INSN_MATCH_C_SD 0xe000
59 #define INSN_MASK_C_SD 0xe003
60 #define INSN_MATCH_C_LW 0x4000
61 #define INSN_MASK_C_LW 0xe003
62 #define INSN_MATCH_C_SW 0xc000
63 #define INSN_MASK_C_SW 0xe003
64 #define INSN_MATCH_C_LDSP 0x6002
65 #define INSN_MASK_C_LDSP 0xe003
66 #define INSN_MATCH_C_SDSP 0xe002
67 #define INSN_MASK_C_SDSP 0xe003
68 #define INSN_MATCH_C_LWSP 0x4002
69 #define INSN_MASK_C_LWSP 0xe003
70 #define INSN_MATCH_C_SWSP 0xc002
71 #define INSN_MASK_C_SWSP 0xe003
72
73 #define INSN_MATCH_C_FLD 0x2000
74 #define INSN_MASK_C_FLD 0xe003
75 #define INSN_MATCH_C_FLW 0x6000
76 #define INSN_MASK_C_FLW 0xe003
77 #define INSN_MATCH_C_FSD 0xa000
78 #define INSN_MASK_C_FSD 0xe003
79 #define INSN_MATCH_C_FSW 0xe000
80 #define INSN_MASK_C_FSW 0xe003
81 #define INSN_MATCH_C_FLDSP 0x2002
82 #define INSN_MASK_C_FLDSP 0xe003
83 #define INSN_MATCH_C_FSDSP 0xa002
84 #define INSN_MASK_C_FSDSP 0xe003
85 #define INSN_MATCH_C_FLWSP 0x6002
86 #define INSN_MASK_C_FLWSP 0xe003
87 #define INSN_MATCH_C_FSWSP 0xe002
88 #define INSN_MASK_C_FSWSP 0xe003
89
90 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
91
92 #if defined(CONFIG_64BIT)
93 #define LOG_REGBYTES 3
94 #define XLEN 64
95 #else
96 #define LOG_REGBYTES 2
97 #define XLEN 32
98 #endif
99 #define REGBYTES (1 << LOG_REGBYTES)
100 #define XLEN_MINUS_16 ((XLEN) - 16)
101
102 #define SH_RD 7
103 #define SH_RS1 15
104 #define SH_RS2 20
105 #define SH_RS2C 2
106
107 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
108 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
109 (RV_X(x, 10, 3) << 3) | \
110 (RV_X(x, 5, 1) << 6))
111 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
112 (RV_X(x, 5, 2) << 6))
113 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
114 (RV_X(x, 12, 1) << 5) | \
115 (RV_X(x, 2, 2) << 6))
116 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
117 (RV_X(x, 12, 1) << 5) | \
118 (RV_X(x, 2, 3) << 6))
119 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
120 (RV_X(x, 7, 2) << 6))
121 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
122 (RV_X(x, 7, 3) << 6))
123 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
124 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
125 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
126
127 #define SHIFT_RIGHT(x, y) \
128 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
129
130 #define REG_MASK \
131 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
132
133 #define REG_OFFSET(insn, pos) \
134 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
135
136 #define REG_PTR(insn, pos, regs) \
137 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
138
139 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
140 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
141 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
142 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
143 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
144 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
145 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
146 #define IMM_I(insn) ((s32)(insn) >> 20)
147 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
148 (s32)(((insn) >> 7) & 0x1f))
149 #define MASK_FUNCT3 0x7000
150
151 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
152 #define GET_RM(insn) (((insn) >> 12) & 7)
153 #define PRECISION_S 0
154 #define PRECISION_D 1
155
156 #ifdef CONFIG_FPU
157
158 #define FP_GET_RD(insn) (insn >> 7 & 0x1F)
159
160 extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
161
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)162 static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
163 unsigned long val)
164 {
165 unsigned long fp_reg = FP_GET_RD(insn);
166
167 put_f32_reg(fp_reg, val);
168 regs->status |= SR_FS_DIRTY;
169
170 return 0;
171 }
172
173 extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
174
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)175 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
176 {
177 unsigned long fp_reg = FP_GET_RD(insn);
178 unsigned long value;
179
180 #if __riscv_xlen == 32
181 value = (unsigned long) &val;
182 #else
183 value = val;
184 #endif
185 put_f64_reg(fp_reg, value);
186 regs->status |= SR_FS_DIRTY;
187
188 return 0;
189 }
190
191 #if __riscv_xlen == 32
192 extern void get_f64_reg(unsigned long fp_reg, u64 *value);
193
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)194 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
195 struct pt_regs *regs)
196 {
197 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
198 u64 val;
199
200 get_f64_reg(fp_reg, &val);
201 regs->status |= SR_FS_DIRTY;
202
203 return val;
204 }
205 #else
206
207 extern unsigned long get_f64_reg(unsigned long fp_reg);
208
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)209 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
210 struct pt_regs *regs)
211 {
212 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
213 unsigned long val;
214
215 val = get_f64_reg(fp_reg);
216 regs->status |= SR_FS_DIRTY;
217
218 return val;
219 }
220
221 #endif
222
223 extern unsigned long get_f32_reg(unsigned long fp_reg);
224
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)225 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
226 struct pt_regs *regs)
227 {
228 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
229 unsigned long val;
230
231 val = get_f32_reg(fp_reg);
232 regs->status |= SR_FS_DIRTY;
233
234 return val;
235 }
236
237 #else /* CONFIG_FPU */
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)238 static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
239 unsigned long val) {}
240
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)241 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
242
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)243 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
244 struct pt_regs *regs)
245 {
246 return 0;
247 }
248
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)249 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
250 struct pt_regs *regs)
251 {
252 return 0;
253 }
254
255 #endif
256
257 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
258 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
259 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
260
261 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
262 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
263 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
264
265 #define __read_insn(regs, insn, insn_addr, type) \
266 ({ \
267 int __ret; \
268 \
269 if (user_mode(regs)) { \
270 __ret = __get_user(insn, (type __user *) insn_addr); \
271 } else { \
272 insn = *(type *)insn_addr; \
273 __ret = 0; \
274 } \
275 \
276 __ret; \
277 })
278
get_insn(struct pt_regs * regs,ulong epc,ulong * r_insn)279 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
280 {
281 ulong insn = 0;
282
283 if (epc & 0x2) {
284 ulong tmp = 0;
285
286 if (__read_insn(regs, insn, epc, u16))
287 return -EFAULT;
288 /* __get_user() uses regular "lw" which sign extend the loaded
289 * value make sure to clear higher order bits in case we "or" it
290 * below with the upper 16 bits half.
291 */
292 insn &= GENMASK(15, 0);
293 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
294 *r_insn = insn;
295 return 0;
296 }
297 epc += sizeof(u16);
298 if (__read_insn(regs, tmp, epc, u16))
299 return -EFAULT;
300 *r_insn = (tmp << 16) | insn;
301
302 return 0;
303 } else {
304 if (__read_insn(regs, insn, epc, u32))
305 return -EFAULT;
306 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
307 *r_insn = insn;
308 return 0;
309 }
310 insn &= GENMASK(15, 0);
311 *r_insn = insn;
312
313 return 0;
314 }
315 }
316
317 union reg_data {
318 u8 data_bytes[8];
319 ulong data_ulong;
320 u64 data_u64;
321 };
322
323 static bool unaligned_ctl __read_mostly;
324
325 /* sysctl hooks */
326 int unaligned_enabled __read_mostly = 1; /* Enabled by default */
327
handle_misaligned_load(struct pt_regs * regs)328 int handle_misaligned_load(struct pt_regs *regs)
329 {
330 union reg_data val;
331 unsigned long epc = regs->epc;
332 unsigned long insn;
333 unsigned long addr = regs->badaddr;
334 int fp = 0, shift = 0, len = 0;
335
336 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
337
338 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
339 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
340 #endif
341
342 if (!unaligned_enabled)
343 return -1;
344
345 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
346 return -1;
347
348 if (get_insn(regs, epc, &insn))
349 return -1;
350
351 regs->epc = 0;
352
353 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
354 len = 4;
355 shift = 8 * (sizeof(unsigned long) - len);
356 #if defined(CONFIG_64BIT)
357 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
358 len = 8;
359 shift = 8 * (sizeof(unsigned long) - len);
360 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
361 len = 4;
362 #endif
363 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
364 fp = 1;
365 len = 8;
366 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
367 fp = 1;
368 len = 4;
369 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
370 len = 2;
371 shift = 8 * (sizeof(unsigned long) - len);
372 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
373 len = 2;
374 #if defined(CONFIG_64BIT)
375 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
376 len = 8;
377 shift = 8 * (sizeof(unsigned long) - len);
378 insn = RVC_RS2S(insn) << SH_RD;
379 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
380 ((insn >> SH_RD) & 0x1f)) {
381 len = 8;
382 shift = 8 * (sizeof(unsigned long) - len);
383 #endif
384 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
385 len = 4;
386 shift = 8 * (sizeof(unsigned long) - len);
387 insn = RVC_RS2S(insn) << SH_RD;
388 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
389 ((insn >> SH_RD) & 0x1f)) {
390 len = 4;
391 shift = 8 * (sizeof(unsigned long) - len);
392 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
393 fp = 1;
394 len = 8;
395 insn = RVC_RS2S(insn) << SH_RD;
396 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
397 fp = 1;
398 len = 8;
399 #if defined(CONFIG_32BIT)
400 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
401 fp = 1;
402 len = 4;
403 insn = RVC_RS2S(insn) << SH_RD;
404 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
405 fp = 1;
406 len = 4;
407 #endif
408 } else {
409 regs->epc = epc;
410 return -1;
411 }
412
413 if (!IS_ENABLED(CONFIG_FPU) && fp)
414 return -EOPNOTSUPP;
415
416 val.data_u64 = 0;
417 if (user_mode(regs)) {
418 if (copy_from_user(&val, (u8 __user *)addr, len))
419 return -1;
420 } else {
421 memcpy(&val, (u8 *)addr, len);
422 }
423
424 if (!fp)
425 SET_RD(insn, regs, val.data_ulong << shift >> shift);
426 else if (len == 8)
427 set_f64_rd(insn, regs, val.data_u64);
428 else
429 set_f32_rd(insn, regs, val.data_ulong);
430
431 regs->epc = epc + INSN_LEN(insn);
432
433 return 0;
434 }
435
handle_misaligned_store(struct pt_regs * regs)436 int handle_misaligned_store(struct pt_regs *regs)
437 {
438 union reg_data val;
439 unsigned long epc = regs->epc;
440 unsigned long insn;
441 unsigned long addr = regs->badaddr;
442 int len = 0, fp = 0;
443
444 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
445
446 if (!unaligned_enabled)
447 return -1;
448
449 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
450 return -1;
451
452 if (get_insn(regs, epc, &insn))
453 return -1;
454
455 regs->epc = 0;
456
457 val.data_ulong = GET_RS2(insn, regs);
458
459 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
460 len = 4;
461 #if defined(CONFIG_64BIT)
462 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
463 len = 8;
464 #endif
465 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
466 fp = 1;
467 len = 8;
468 val.data_u64 = GET_F64_RS2(insn, regs);
469 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
470 fp = 1;
471 len = 4;
472 val.data_ulong = GET_F32_RS2(insn, regs);
473 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
474 len = 2;
475 #if defined(CONFIG_64BIT)
476 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
477 len = 8;
478 val.data_ulong = GET_RS2S(insn, regs);
479 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
480 len = 8;
481 val.data_ulong = GET_RS2C(insn, regs);
482 #endif
483 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
484 len = 4;
485 val.data_ulong = GET_RS2S(insn, regs);
486 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
487 len = 4;
488 val.data_ulong = GET_RS2C(insn, regs);
489 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
490 fp = 1;
491 len = 8;
492 val.data_u64 = GET_F64_RS2S(insn, regs);
493 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
494 fp = 1;
495 len = 8;
496 val.data_u64 = GET_F64_RS2C(insn, regs);
497 #if !defined(CONFIG_64BIT)
498 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
499 fp = 1;
500 len = 4;
501 val.data_ulong = GET_F32_RS2S(insn, regs);
502 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
503 fp = 1;
504 len = 4;
505 val.data_ulong = GET_F32_RS2C(insn, regs);
506 #endif
507 } else {
508 regs->epc = epc;
509 return -1;
510 }
511
512 if (!IS_ENABLED(CONFIG_FPU) && fp)
513 return -EOPNOTSUPP;
514
515 if (user_mode(regs)) {
516 if (copy_to_user((u8 __user *)addr, &val, len))
517 return -1;
518 } else {
519 memcpy((u8 *)addr, &val, len);
520 }
521
522 regs->epc = epc + INSN_LEN(insn);
523
524 return 0;
525 }
526
check_unaligned_access_emulated(int cpu)527 static bool check_unaligned_access_emulated(int cpu)
528 {
529 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
530 unsigned long tmp_var, tmp_val;
531 bool misaligned_emu_detected;
532
533 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
534
535 __asm__ __volatile__ (
536 " "REG_L" %[tmp], 1(%[ptr])\n"
537 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
538
539 misaligned_emu_detected = (*mas_ptr == RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED);
540 /*
541 * If unaligned_ctl is already set, this means that we detected that all
542 * CPUS uses emulated misaligned access at boot time. If that changed
543 * when hotplugging the new cpu, this is something we don't handle.
544 */
545 if (unlikely(unaligned_ctl && !misaligned_emu_detected)) {
546 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
547 while (true)
548 cpu_relax();
549 }
550
551 return misaligned_emu_detected;
552 }
553
check_unaligned_access_emulated_all_cpus(void)554 bool check_unaligned_access_emulated_all_cpus(void)
555 {
556 int cpu;
557
558 /*
559 * We can only support PR_UNALIGN controls if all CPUs have misaligned
560 * accesses emulated since tasks requesting such control can run on any
561 * CPU.
562 */
563 for_each_online_cpu(cpu)
564 if (!check_unaligned_access_emulated(cpu))
565 return false;
566
567 unaligned_ctl = true;
568 return true;
569 }
570
unaligned_ctl_available(void)571 bool unaligned_ctl_available(void)
572 {
573 return unaligned_ctl;
574 }
575