1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright 2022-2024 Rivos, Inc
4 */
5
6 #ifndef _ASM_CPUFEATURE_H
7 #define _ASM_CPUFEATURE_H
8
9 #include <linux/bitmap.h>
10 #include <linux/jump_label.h>
11 #include <asm/hwcap.h>
12 #include <asm/alternative-macros.h>
13 #include <asm/errno.h>
14
15 /*
16 * These are probed via a device_initcall(), via either the SBI or directly
17 * from the corresponding CSRs.
18 */
19 struct riscv_cpuinfo {
20 unsigned long mvendorid;
21 unsigned long marchid;
22 unsigned long mimpid;
23 };
24
25 struct riscv_isainfo {
26 DECLARE_BITMAP(isa, RISCV_ISA_EXT_MAX);
27 };
28
29 DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo);
30
31 /* Per-cpu ISA extensions. */
32 extern struct riscv_isainfo hart_isa[NR_CPUS];
33
34 void riscv_user_isa_enable(void);
35
36 #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \
37 .name = #_name, \
38 .property = #_name, \
39 .id = _id, \
40 .subset_ext_ids = _subset_exts, \
41 .subset_ext_size = _subset_exts_size, \
42 .validate = _validate \
43 }
44
45 #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, NULL)
46
47 #define __RISCV_ISA_EXT_DATA_VALIDATE(_name, _id, _validate) \
48 _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, _validate)
49
50 /* Used to declare pure "lasso" extension (Zk for instance) */
51 #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \
52 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
53 ARRAY_SIZE(_bundled_exts), NULL)
54
55 /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */
56 #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \
57 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), NULL)
58 #define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
59 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
60
61 #if defined(CONFIG_RISCV_MISALIGNED)
62 bool check_unaligned_access_emulated_all_cpus(void);
63 void unaligned_emulation_finish(void);
64 bool unaligned_ctl_available(void);
65 DECLARE_PER_CPU(long, misaligned_access_speed);
66 #else
unaligned_ctl_available(void)67 static inline bool unaligned_ctl_available(void)
68 {
69 return false;
70 }
71 #endif
72
73 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
74 DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
75
has_fast_unaligned_accesses(void)76 static __always_inline bool has_fast_unaligned_accesses(void)
77 {
78 return static_branch_likely(&fast_unaligned_access_speed_key);
79 }
80 #else
has_fast_unaligned_accesses(void)81 static __always_inline bool has_fast_unaligned_accesses(void)
82 {
83 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
84 return true;
85 else
86 return false;
87 }
88 #endif
89
90 unsigned long riscv_get_elf_hwcap(void);
91
92 struct riscv_isa_ext_data {
93 const unsigned int id;
94 const char *name;
95 const char *property;
96 const unsigned int *subset_ext_ids;
97 const unsigned int subset_ext_size;
98 int (*validate)(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap);
99 };
100
101 extern const struct riscv_isa_ext_data riscv_isa_ext[];
102 extern const size_t riscv_isa_ext_count;
103 extern bool riscv_isa_fallback;
104
105 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap);
106
107 #define STANDARD_EXT 0
108
109 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit);
110 #define riscv_isa_extension_available(isa_bitmap, ext) \
111 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
112
__riscv_has_extension_likely(const unsigned long vendor,const unsigned long ext)113 static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor,
114 const unsigned long ext)
115 {
116 asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
117 :
118 : [vendor] "i" (vendor), [ext] "i" (ext)
119 :
120 : l_no);
121
122 return true;
123 l_no:
124 return false;
125 }
126
__riscv_has_extension_unlikely(const unsigned long vendor,const unsigned long ext)127 static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor,
128 const unsigned long ext)
129 {
130 asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
131 :
132 : [vendor] "i" (vendor), [ext] "i" (ext)
133 :
134 : l_yes);
135
136 return false;
137 l_yes:
138 return true;
139 }
140
riscv_has_extension_unlikely(const unsigned long ext)141 static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext)
142 {
143 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
144
145 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
146 return __riscv_has_extension_unlikely(STANDARD_EXT, ext);
147
148 return __riscv_isa_extension_available(NULL, ext);
149 }
150
riscv_has_extension_likely(const unsigned long ext)151 static __always_inline bool riscv_has_extension_likely(const unsigned long ext)
152 {
153 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
154
155 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
156 return __riscv_has_extension_likely(STANDARD_EXT, ext);
157
158 return __riscv_isa_extension_available(NULL, ext);
159 }
160
riscv_cpu_has_extension_likely(int cpu,const unsigned long ext)161 static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)
162 {
163 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
164
165 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
166 __riscv_has_extension_likely(STANDARD_EXT, ext))
167 return true;
168
169 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
170 }
171
riscv_cpu_has_extension_unlikely(int cpu,const unsigned long ext)172 static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext)
173 {
174 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
175
176 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
177 __riscv_has_extension_unlikely(STANDARD_EXT, ext))
178 return true;
179
180 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
181 }
182
183 #endif
184