1  /* SPDX-License-Identifier: GPL-2.0-or-later */
2  /*
3   * Copyright (C) 2020 SiFive
4   */
5  
6  #ifndef __ASM_RISCV_VECTOR_H
7  #define __ASM_RISCV_VECTOR_H
8  
9  #include <linux/types.h>
10  #include <uapi/asm-generic/errno.h>
11  
12  #ifdef CONFIG_RISCV_ISA_V
13  
14  #include <linux/stringify.h>
15  #include <linux/sched.h>
16  #include <linux/sched/task_stack.h>
17  #include <asm/ptrace.h>
18  #include <asm/cpufeature.h>
19  #include <asm/csr.h>
20  #include <asm/asm.h>
21  
22  extern unsigned long riscv_v_vsize;
23  int riscv_v_setup_vsize(void);
24  bool riscv_v_first_use_handler(struct pt_regs *regs);
25  void kernel_vector_begin(void);
26  void kernel_vector_end(void);
27  void get_cpu_vector_context(void);
28  void put_cpu_vector_context(void);
29  void riscv_v_thread_free(struct task_struct *tsk);
30  void __init riscv_v_setup_ctx_cache(void);
31  void riscv_v_thread_alloc(struct task_struct *tsk);
32  
riscv_v_flags(void)33  static inline u32 riscv_v_flags(void)
34  {
35  	return READ_ONCE(current->thread.riscv_v_flags);
36  }
37  
has_vector(void)38  static __always_inline bool has_vector(void)
39  {
40  	return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X);
41  }
42  
__riscv_v_vstate_clean(struct pt_regs * regs)43  static inline void __riscv_v_vstate_clean(struct pt_regs *regs)
44  {
45  	regs->status = (regs->status & ~SR_VS) | SR_VS_CLEAN;
46  }
47  
__riscv_v_vstate_dirty(struct pt_regs * regs)48  static inline void __riscv_v_vstate_dirty(struct pt_regs *regs)
49  {
50  	regs->status = (regs->status & ~SR_VS) | SR_VS_DIRTY;
51  }
52  
riscv_v_vstate_off(struct pt_regs * regs)53  static inline void riscv_v_vstate_off(struct pt_regs *regs)
54  {
55  	regs->status = (regs->status & ~SR_VS) | SR_VS_OFF;
56  }
57  
riscv_v_vstate_on(struct pt_regs * regs)58  static inline void riscv_v_vstate_on(struct pt_regs *regs)
59  {
60  	regs->status = (regs->status & ~SR_VS) | SR_VS_INITIAL;
61  }
62  
riscv_v_vstate_query(struct pt_regs * regs)63  static inline bool riscv_v_vstate_query(struct pt_regs *regs)
64  {
65  	return (regs->status & SR_VS) != 0;
66  }
67  
riscv_v_enable(void)68  static __always_inline void riscv_v_enable(void)
69  {
70  	csr_set(CSR_SSTATUS, SR_VS);
71  }
72  
riscv_v_disable(void)73  static __always_inline void riscv_v_disable(void)
74  {
75  	csr_clear(CSR_SSTATUS, SR_VS);
76  }
77  
__vstate_csr_save(struct __riscv_v_ext_state * dest)78  static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest)
79  {
80  	asm volatile (
81  		"csrr	%0, " __stringify(CSR_VSTART) "\n\t"
82  		"csrr	%1, " __stringify(CSR_VTYPE) "\n\t"
83  		"csrr	%2, " __stringify(CSR_VL) "\n\t"
84  		"csrr	%3, " __stringify(CSR_VCSR) "\n\t"
85  		"csrr	%4, " __stringify(CSR_VLENB) "\n\t"
86  		: "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl),
87  		  "=r" (dest->vcsr), "=r" (dest->vlenb) : :);
88  }
89  
__vstate_csr_restore(struct __riscv_v_ext_state * src)90  static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src)
91  {
92  	asm volatile (
93  		".option push\n\t"
94  		".option arch, +zve32x\n\t"
95  		"vsetvl	 x0, %2, %1\n\t"
96  		".option pop\n\t"
97  		"csrw	" __stringify(CSR_VSTART) ", %0\n\t"
98  		"csrw	" __stringify(CSR_VCSR) ", %3\n\t"
99  		: : "r" (src->vstart), "r" (src->vtype), "r" (src->vl),
100  		    "r" (src->vcsr) :);
101  }
102  
__riscv_v_vstate_save(struct __riscv_v_ext_state * save_to,void * datap)103  static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
104  					 void *datap)
105  {
106  	unsigned long vl;
107  
108  	riscv_v_enable();
109  	__vstate_csr_save(save_to);
110  	asm volatile (
111  		".option push\n\t"
112  		".option arch, +zve32x\n\t"
113  		"vsetvli	%0, x0, e8, m8, ta, ma\n\t"
114  		"vse8.v		v0, (%1)\n\t"
115  		"add		%1, %1, %0\n\t"
116  		"vse8.v		v8, (%1)\n\t"
117  		"add		%1, %1, %0\n\t"
118  		"vse8.v		v16, (%1)\n\t"
119  		"add		%1, %1, %0\n\t"
120  		"vse8.v		v24, (%1)\n\t"
121  		".option pop\n\t"
122  		: "=&r" (vl) : "r" (datap) : "memory");
123  	riscv_v_disable();
124  }
125  
__riscv_v_vstate_restore(struct __riscv_v_ext_state * restore_from,void * datap)126  static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from,
127  					    void *datap)
128  {
129  	unsigned long vl;
130  
131  	riscv_v_enable();
132  	asm volatile (
133  		".option push\n\t"
134  		".option arch, +zve32x\n\t"
135  		"vsetvli	%0, x0, e8, m8, ta, ma\n\t"
136  		"vle8.v		v0, (%1)\n\t"
137  		"add		%1, %1, %0\n\t"
138  		"vle8.v		v8, (%1)\n\t"
139  		"add		%1, %1, %0\n\t"
140  		"vle8.v		v16, (%1)\n\t"
141  		"add		%1, %1, %0\n\t"
142  		"vle8.v		v24, (%1)\n\t"
143  		".option pop\n\t"
144  		: "=&r" (vl) : "r" (datap) : "memory");
145  	__vstate_csr_restore(restore_from);
146  	riscv_v_disable();
147  }
148  
__riscv_v_vstate_discard(void)149  static inline void __riscv_v_vstate_discard(void)
150  {
151  	unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1);
152  
153  	riscv_v_enable();
154  	asm volatile (
155  		".option push\n\t"
156  		".option arch, +zve32x\n\t"
157  		"vsetvli	%0, x0, e8, m8, ta, ma\n\t"
158  		"vmv.v.i	v0, -1\n\t"
159  		"vmv.v.i	v8, -1\n\t"
160  		"vmv.v.i	v16, -1\n\t"
161  		"vmv.v.i	v24, -1\n\t"
162  		"vsetvl		%0, x0, %1\n\t"
163  		".option pop\n\t"
164  		: "=&r" (vl) : "r" (vtype_inval) : "memory");
165  	riscv_v_disable();
166  }
167  
riscv_v_vstate_discard(struct pt_regs * regs)168  static inline void riscv_v_vstate_discard(struct pt_regs *regs)
169  {
170  	if ((regs->status & SR_VS) == SR_VS_OFF)
171  		return;
172  
173  	__riscv_v_vstate_discard();
174  	__riscv_v_vstate_dirty(regs);
175  }
176  
riscv_v_vstate_save(struct __riscv_v_ext_state * vstate,struct pt_regs * regs)177  static inline void riscv_v_vstate_save(struct __riscv_v_ext_state *vstate,
178  				       struct pt_regs *regs)
179  {
180  	if ((regs->status & SR_VS) == SR_VS_DIRTY) {
181  		__riscv_v_vstate_save(vstate, vstate->datap);
182  		__riscv_v_vstate_clean(regs);
183  	}
184  }
185  
riscv_v_vstate_restore(struct __riscv_v_ext_state * vstate,struct pt_regs * regs)186  static inline void riscv_v_vstate_restore(struct __riscv_v_ext_state *vstate,
187  					  struct pt_regs *regs)
188  {
189  	if ((regs->status & SR_VS) != SR_VS_OFF) {
190  		__riscv_v_vstate_restore(vstate, vstate->datap);
191  		__riscv_v_vstate_clean(regs);
192  	}
193  }
194  
riscv_v_vstate_set_restore(struct task_struct * task,struct pt_regs * regs)195  static inline void riscv_v_vstate_set_restore(struct task_struct *task,
196  					      struct pt_regs *regs)
197  {
198  	if ((regs->status & SR_VS) != SR_VS_OFF) {
199  		set_tsk_thread_flag(task, TIF_RISCV_V_DEFER_RESTORE);
200  		riscv_v_vstate_on(regs);
201  	}
202  }
203  
204  #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
riscv_preempt_v_dirty(struct task_struct * task)205  static inline bool riscv_preempt_v_dirty(struct task_struct *task)
206  {
207  	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_DIRTY);
208  }
209  
riscv_preempt_v_restore(struct task_struct * task)210  static inline bool riscv_preempt_v_restore(struct task_struct *task)
211  {
212  	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_NEED_RESTORE);
213  }
214  
riscv_preempt_v_clear_dirty(struct task_struct * task)215  static inline void riscv_preempt_v_clear_dirty(struct task_struct *task)
216  {
217  	barrier();
218  	task->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_DIRTY;
219  }
220  
riscv_preempt_v_set_restore(struct task_struct * task)221  static inline void riscv_preempt_v_set_restore(struct task_struct *task)
222  {
223  	barrier();
224  	task->thread.riscv_v_flags |= RISCV_PREEMPT_V_NEED_RESTORE;
225  }
226  
riscv_preempt_v_started(struct task_struct * task)227  static inline bool riscv_preempt_v_started(struct task_struct *task)
228  {
229  	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V);
230  }
231  
232  #else /* !CONFIG_RISCV_ISA_V_PREEMPTIVE */
riscv_preempt_v_dirty(struct task_struct * task)233  static inline bool riscv_preempt_v_dirty(struct task_struct *task) { return false; }
riscv_preempt_v_restore(struct task_struct * task)234  static inline bool riscv_preempt_v_restore(struct task_struct *task) { return false; }
riscv_preempt_v_started(struct task_struct * task)235  static inline bool riscv_preempt_v_started(struct task_struct *task) { return false; }
236  #define riscv_preempt_v_clear_dirty(tsk)	do {} while (0)
237  #define riscv_preempt_v_set_restore(tsk)	do {} while (0)
238  #endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */
239  
__switch_to_vector(struct task_struct * prev,struct task_struct * next)240  static inline void __switch_to_vector(struct task_struct *prev,
241  				      struct task_struct *next)
242  {
243  	struct pt_regs *regs;
244  
245  	if (riscv_preempt_v_started(prev)) {
246  		if (riscv_preempt_v_dirty(prev)) {
247  			__riscv_v_vstate_save(&prev->thread.kernel_vstate,
248  					      prev->thread.kernel_vstate.datap);
249  			riscv_preempt_v_clear_dirty(prev);
250  		}
251  	} else {
252  		regs = task_pt_regs(prev);
253  		riscv_v_vstate_save(&prev->thread.vstate, regs);
254  	}
255  
256  	if (riscv_preempt_v_started(next))
257  		riscv_preempt_v_set_restore(next);
258  	else
259  		riscv_v_vstate_set_restore(next, task_pt_regs(next));
260  }
261  
262  void riscv_v_vstate_ctrl_init(struct task_struct *tsk);
263  bool riscv_v_vstate_ctrl_user_allowed(void);
264  
265  #else /* ! CONFIG_RISCV_ISA_V  */
266  
267  struct pt_regs;
268  
riscv_v_setup_vsize(void)269  static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }
has_vector(void)270  static __always_inline bool has_vector(void) { return false; }
riscv_v_first_use_handler(struct pt_regs * regs)271  static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; }
riscv_v_vstate_query(struct pt_regs * regs)272  static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; }
riscv_v_vstate_ctrl_user_allowed(void)273  static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }
274  #define riscv_v_vsize (0)
275  #define riscv_v_vstate_discard(regs)		do {} while (0)
276  #define riscv_v_vstate_save(vstate, regs)	do {} while (0)
277  #define riscv_v_vstate_restore(vstate, regs)	do {} while (0)
278  #define __switch_to_vector(__prev, __next)	do {} while (0)
279  #define riscv_v_vstate_off(regs)		do {} while (0)
280  #define riscv_v_vstate_on(regs)			do {} while (0)
281  #define riscv_v_thread_free(tsk)		do {} while (0)
282  #define  riscv_v_setup_ctx_cache()		do {} while (0)
283  #define riscv_v_thread_alloc(tsk)		do {} while (0)
284  
285  #endif /* CONFIG_RISCV_ISA_V */
286  
287  /*
288   * Return the implementation's vlen value.
289   *
290   * riscv_v_vsize contains the value of "32 vector registers with vlenb length"
291   * so rebuild the vlen value in bits from it.
292   */
riscv_vector_vlen(void)293  static inline int riscv_vector_vlen(void)
294  {
295  	return riscv_v_vsize / 32 * 8;
296  }
297  
298  #endif /* ! __ASM_RISCV_VECTOR_H */
299