1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * Copyright (C) 2020 SiFive
4 */
5
6 #ifndef __ASM_RISCV_VECTOR_H
7 #define __ASM_RISCV_VECTOR_H
8
9 #include <linux/types.h>
10 #include <uapi/asm-generic/errno.h>
11
12 #ifdef CONFIG_RISCV_ISA_V
13
14 #include <linux/stringify.h>
15 #include <linux/sched.h>
16 #include <linux/sched/task_stack.h>
17 #include <asm/ptrace.h>
18 #include <asm/cpufeature.h>
19 #include <asm/csr.h>
20 #include <asm/asm.h>
21
22 extern unsigned long riscv_v_vsize;
23 int riscv_v_setup_vsize(void);
24 bool riscv_v_first_use_handler(struct pt_regs *regs);
25 void kernel_vector_begin(void);
26 void kernel_vector_end(void);
27 void get_cpu_vector_context(void);
28 void put_cpu_vector_context(void);
29 void riscv_v_thread_free(struct task_struct *tsk);
30 void __init riscv_v_setup_ctx_cache(void);
31 void riscv_v_thread_alloc(struct task_struct *tsk);
32
riscv_v_flags(void)33 static inline u32 riscv_v_flags(void)
34 {
35 return READ_ONCE(current->thread.riscv_v_flags);
36 }
37
has_vector(void)38 static __always_inline bool has_vector(void)
39 {
40 return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X);
41 }
42
__riscv_v_vstate_clean(struct pt_regs * regs)43 static inline void __riscv_v_vstate_clean(struct pt_regs *regs)
44 {
45 regs->status = (regs->status & ~SR_VS) | SR_VS_CLEAN;
46 }
47
__riscv_v_vstate_dirty(struct pt_regs * regs)48 static inline void __riscv_v_vstate_dirty(struct pt_regs *regs)
49 {
50 regs->status = (regs->status & ~SR_VS) | SR_VS_DIRTY;
51 }
52
riscv_v_vstate_off(struct pt_regs * regs)53 static inline void riscv_v_vstate_off(struct pt_regs *regs)
54 {
55 regs->status = (regs->status & ~SR_VS) | SR_VS_OFF;
56 }
57
riscv_v_vstate_on(struct pt_regs * regs)58 static inline void riscv_v_vstate_on(struct pt_regs *regs)
59 {
60 regs->status = (regs->status & ~SR_VS) | SR_VS_INITIAL;
61 }
62
riscv_v_vstate_query(struct pt_regs * regs)63 static inline bool riscv_v_vstate_query(struct pt_regs *regs)
64 {
65 return (regs->status & SR_VS) != 0;
66 }
67
riscv_v_enable(void)68 static __always_inline void riscv_v_enable(void)
69 {
70 csr_set(CSR_SSTATUS, SR_VS);
71 }
72
riscv_v_disable(void)73 static __always_inline void riscv_v_disable(void)
74 {
75 csr_clear(CSR_SSTATUS, SR_VS);
76 }
77
__vstate_csr_save(struct __riscv_v_ext_state * dest)78 static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest)
79 {
80 asm volatile (
81 "csrr %0, " __stringify(CSR_VSTART) "\n\t"
82 "csrr %1, " __stringify(CSR_VTYPE) "\n\t"
83 "csrr %2, " __stringify(CSR_VL) "\n\t"
84 "csrr %3, " __stringify(CSR_VCSR) "\n\t"
85 "csrr %4, " __stringify(CSR_VLENB) "\n\t"
86 : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl),
87 "=r" (dest->vcsr), "=r" (dest->vlenb) : :);
88 }
89
__vstate_csr_restore(struct __riscv_v_ext_state * src)90 static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src)
91 {
92 asm volatile (
93 ".option push\n\t"
94 ".option arch, +zve32x\n\t"
95 "vsetvl x0, %2, %1\n\t"
96 ".option pop\n\t"
97 "csrw " __stringify(CSR_VSTART) ", %0\n\t"
98 "csrw " __stringify(CSR_VCSR) ", %3\n\t"
99 : : "r" (src->vstart), "r" (src->vtype), "r" (src->vl),
100 "r" (src->vcsr) :);
101 }
102
__riscv_v_vstate_save(struct __riscv_v_ext_state * save_to,void * datap)103 static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
104 void *datap)
105 {
106 unsigned long vl;
107
108 riscv_v_enable();
109 __vstate_csr_save(save_to);
110 asm volatile (
111 ".option push\n\t"
112 ".option arch, +zve32x\n\t"
113 "vsetvli %0, x0, e8, m8, ta, ma\n\t"
114 "vse8.v v0, (%1)\n\t"
115 "add %1, %1, %0\n\t"
116 "vse8.v v8, (%1)\n\t"
117 "add %1, %1, %0\n\t"
118 "vse8.v v16, (%1)\n\t"
119 "add %1, %1, %0\n\t"
120 "vse8.v v24, (%1)\n\t"
121 ".option pop\n\t"
122 : "=&r" (vl) : "r" (datap) : "memory");
123 riscv_v_disable();
124 }
125
__riscv_v_vstate_restore(struct __riscv_v_ext_state * restore_from,void * datap)126 static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from,
127 void *datap)
128 {
129 unsigned long vl;
130
131 riscv_v_enable();
132 asm volatile (
133 ".option push\n\t"
134 ".option arch, +zve32x\n\t"
135 "vsetvli %0, x0, e8, m8, ta, ma\n\t"
136 "vle8.v v0, (%1)\n\t"
137 "add %1, %1, %0\n\t"
138 "vle8.v v8, (%1)\n\t"
139 "add %1, %1, %0\n\t"
140 "vle8.v v16, (%1)\n\t"
141 "add %1, %1, %0\n\t"
142 "vle8.v v24, (%1)\n\t"
143 ".option pop\n\t"
144 : "=&r" (vl) : "r" (datap) : "memory");
145 __vstate_csr_restore(restore_from);
146 riscv_v_disable();
147 }
148
__riscv_v_vstate_discard(void)149 static inline void __riscv_v_vstate_discard(void)
150 {
151 unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1);
152
153 riscv_v_enable();
154 asm volatile (
155 ".option push\n\t"
156 ".option arch, +zve32x\n\t"
157 "vsetvli %0, x0, e8, m8, ta, ma\n\t"
158 "vmv.v.i v0, -1\n\t"
159 "vmv.v.i v8, -1\n\t"
160 "vmv.v.i v16, -1\n\t"
161 "vmv.v.i v24, -1\n\t"
162 "vsetvl %0, x0, %1\n\t"
163 ".option pop\n\t"
164 : "=&r" (vl) : "r" (vtype_inval) : "memory");
165 riscv_v_disable();
166 }
167
riscv_v_vstate_discard(struct pt_regs * regs)168 static inline void riscv_v_vstate_discard(struct pt_regs *regs)
169 {
170 if ((regs->status & SR_VS) == SR_VS_OFF)
171 return;
172
173 __riscv_v_vstate_discard();
174 __riscv_v_vstate_dirty(regs);
175 }
176
riscv_v_vstate_save(struct __riscv_v_ext_state * vstate,struct pt_regs * regs)177 static inline void riscv_v_vstate_save(struct __riscv_v_ext_state *vstate,
178 struct pt_regs *regs)
179 {
180 if ((regs->status & SR_VS) == SR_VS_DIRTY) {
181 __riscv_v_vstate_save(vstate, vstate->datap);
182 __riscv_v_vstate_clean(regs);
183 }
184 }
185
riscv_v_vstate_restore(struct __riscv_v_ext_state * vstate,struct pt_regs * regs)186 static inline void riscv_v_vstate_restore(struct __riscv_v_ext_state *vstate,
187 struct pt_regs *regs)
188 {
189 if ((regs->status & SR_VS) != SR_VS_OFF) {
190 __riscv_v_vstate_restore(vstate, vstate->datap);
191 __riscv_v_vstate_clean(regs);
192 }
193 }
194
riscv_v_vstate_set_restore(struct task_struct * task,struct pt_regs * regs)195 static inline void riscv_v_vstate_set_restore(struct task_struct *task,
196 struct pt_regs *regs)
197 {
198 if ((regs->status & SR_VS) != SR_VS_OFF) {
199 set_tsk_thread_flag(task, TIF_RISCV_V_DEFER_RESTORE);
200 riscv_v_vstate_on(regs);
201 }
202 }
203
204 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
riscv_preempt_v_dirty(struct task_struct * task)205 static inline bool riscv_preempt_v_dirty(struct task_struct *task)
206 {
207 return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_DIRTY);
208 }
209
riscv_preempt_v_restore(struct task_struct * task)210 static inline bool riscv_preempt_v_restore(struct task_struct *task)
211 {
212 return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_NEED_RESTORE);
213 }
214
riscv_preempt_v_clear_dirty(struct task_struct * task)215 static inline void riscv_preempt_v_clear_dirty(struct task_struct *task)
216 {
217 barrier();
218 task->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_DIRTY;
219 }
220
riscv_preempt_v_set_restore(struct task_struct * task)221 static inline void riscv_preempt_v_set_restore(struct task_struct *task)
222 {
223 barrier();
224 task->thread.riscv_v_flags |= RISCV_PREEMPT_V_NEED_RESTORE;
225 }
226
riscv_preempt_v_started(struct task_struct * task)227 static inline bool riscv_preempt_v_started(struct task_struct *task)
228 {
229 return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V);
230 }
231
232 #else /* !CONFIG_RISCV_ISA_V_PREEMPTIVE */
riscv_preempt_v_dirty(struct task_struct * task)233 static inline bool riscv_preempt_v_dirty(struct task_struct *task) { return false; }
riscv_preempt_v_restore(struct task_struct * task)234 static inline bool riscv_preempt_v_restore(struct task_struct *task) { return false; }
riscv_preempt_v_started(struct task_struct * task)235 static inline bool riscv_preempt_v_started(struct task_struct *task) { return false; }
236 #define riscv_preempt_v_clear_dirty(tsk) do {} while (0)
237 #define riscv_preempt_v_set_restore(tsk) do {} while (0)
238 #endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */
239
__switch_to_vector(struct task_struct * prev,struct task_struct * next)240 static inline void __switch_to_vector(struct task_struct *prev,
241 struct task_struct *next)
242 {
243 struct pt_regs *regs;
244
245 if (riscv_preempt_v_started(prev)) {
246 if (riscv_preempt_v_dirty(prev)) {
247 __riscv_v_vstate_save(&prev->thread.kernel_vstate,
248 prev->thread.kernel_vstate.datap);
249 riscv_preempt_v_clear_dirty(prev);
250 }
251 } else {
252 regs = task_pt_regs(prev);
253 riscv_v_vstate_save(&prev->thread.vstate, regs);
254 }
255
256 if (riscv_preempt_v_started(next))
257 riscv_preempt_v_set_restore(next);
258 else
259 riscv_v_vstate_set_restore(next, task_pt_regs(next));
260 }
261
262 void riscv_v_vstate_ctrl_init(struct task_struct *tsk);
263 bool riscv_v_vstate_ctrl_user_allowed(void);
264
265 #else /* ! CONFIG_RISCV_ISA_V */
266
267 struct pt_regs;
268
riscv_v_setup_vsize(void)269 static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }
has_vector(void)270 static __always_inline bool has_vector(void) { return false; }
riscv_v_first_use_handler(struct pt_regs * regs)271 static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; }
riscv_v_vstate_query(struct pt_regs * regs)272 static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; }
riscv_v_vstate_ctrl_user_allowed(void)273 static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }
274 #define riscv_v_vsize (0)
275 #define riscv_v_vstate_discard(regs) do {} while (0)
276 #define riscv_v_vstate_save(vstate, regs) do {} while (0)
277 #define riscv_v_vstate_restore(vstate, regs) do {} while (0)
278 #define __switch_to_vector(__prev, __next) do {} while (0)
279 #define riscv_v_vstate_off(regs) do {} while (0)
280 #define riscv_v_vstate_on(regs) do {} while (0)
281 #define riscv_v_thread_free(tsk) do {} while (0)
282 #define riscv_v_setup_ctx_cache() do {} while (0)
283 #define riscv_v_thread_alloc(tsk) do {} while (0)
284
285 #endif /* CONFIG_RISCV_ISA_V */
286
287 /*
288 * Return the implementation's vlen value.
289 *
290 * riscv_v_vsize contains the value of "32 vector registers with vlenb length"
291 * so rebuild the vlen value in bits from it.
292 */
riscv_vector_vlen(void)293 static inline int riscv_vector_vlen(void)
294 {
295 return riscv_v_vsize / 32 * 8;
296 }
297
298 #endif /* ! __ASM_RISCV_VECTOR_H */
299