1  /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2  
3  #include "rseq-bits-template.h"
4  
5  #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6  	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7  
8  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)9  int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
10  {
11  	RSEQ_INJECT_C(9)
12  
13  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
14  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
15  #ifdef RSEQ_COMPARE_TWICE
16  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
17  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
18  #endif
19  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
20  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
21  				  RSEQ_INJECT_ASM(3)
22  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
23  				  RSEQ_INJECT_ASM(4)
24  #ifdef RSEQ_COMPARE_TWICE
25  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
26  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
27  #endif
28  				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
29  				  RSEQ_INJECT_ASM(5)
30  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
31  				  : /* gcc asm goto does not allow outputs */
32  				  : [cpu_id]		"r" (cpu),
33  				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
34  				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
35  				    [v]			"m" (*v),
36  				    [expect]		"r" (expect),
37  				    [newv]		"r" (newv)
38  				    RSEQ_INJECT_INPUT
39  				  : "memory", RSEQ_ASM_TMP_REG_1
40  				    RSEQ_INJECT_CLOBBER
41  				  : abort, cmpfail
42  #ifdef RSEQ_COMPARE_TWICE
43  				    , error1, error2
44  #endif
45  	);
46  
47  	return 0;
48  abort:
49  	RSEQ_INJECT_FAILED
50  	return -1;
51  cmpfail:
52  	return 1;
53  #ifdef RSEQ_COMPARE_TWICE
54  error1:
55  	rseq_bug("cpu_id comparison failed");
56  error2:
57  	rseq_bug("expected value comparison failed");
58  #endif
59  }
60  
61  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)62  int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
63  			       off_t voffp, intptr_t *load, int cpu)
64  {
65  	RSEQ_INJECT_C(9)
66  
67  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
68  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
69  #ifdef RSEQ_COMPARE_TWICE
70  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
71  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
72  #endif
73  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
74  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
75  				  RSEQ_INJECT_ASM(3)
76  				  RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
77  				  RSEQ_INJECT_ASM(4)
78  #ifdef RSEQ_COMPARE_TWICE
79  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
80  				  RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
81  #endif
82  				  RSEQ_ASM_OP_R_LOAD(v)
83  				  RSEQ_ASM_OP_R_STORE(load)
84  				  RSEQ_ASM_OP_R_LOAD_OFF(voffp)
85  				  RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
86  				  RSEQ_INJECT_ASM(5)
87  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
88  				  : /* gcc asm goto does not allow outputs */
89  				  : [cpu_id]		"r" (cpu),
90  				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
91  				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
92  				    [v]			"m" (*v),
93  				    [expectnot]		"r" (expectnot),
94  				    [load]		"m" (*load),
95  				    [voffp]		"r" (voffp)
96  				    RSEQ_INJECT_INPUT
97  				  : "memory", RSEQ_ASM_TMP_REG_1
98  				    RSEQ_INJECT_CLOBBER
99  				  : abort, cmpfail
100  #ifdef RSEQ_COMPARE_TWICE
101  				    , error1, error2
102  #endif
103  	);
104  	return 0;
105  abort:
106  	RSEQ_INJECT_FAILED
107  	return -1;
108  cmpfail:
109  	return 1;
110  #ifdef RSEQ_COMPARE_TWICE
111  error1:
112  	rseq_bug("cpu_id comparison failed");
113  error2:
114  	rseq_bug("expected value comparison failed");
115  #endif
116  }
117  
118  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)119  int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
120  {
121  	RSEQ_INJECT_C(9)
122  
123  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
124  #ifdef RSEQ_COMPARE_TWICE
125  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
126  #endif
127  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
128  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
129  				  RSEQ_INJECT_ASM(3)
130  #ifdef RSEQ_COMPARE_TWICE
131  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
132  #endif
133  				  RSEQ_ASM_OP_R_LOAD(v)
134  				  RSEQ_ASM_OP_R_ADD(count)
135  				  RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
136  				  RSEQ_INJECT_ASM(4)
137  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
138  				  : /* gcc asm goto does not allow outputs */
139  				  : [cpu_id]		"r" (cpu),
140  				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
141  				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
142  				    [v]			"m" (*v),
143  				    [count]		"r" (count)
144  				    RSEQ_INJECT_INPUT
145  				  : "memory", RSEQ_ASM_TMP_REG_1
146  				    RSEQ_INJECT_CLOBBER
147  				  : abort
148  #ifdef RSEQ_COMPARE_TWICE
149  				    , error1
150  #endif
151  	);
152  	return 0;
153  abort:
154  	RSEQ_INJECT_FAILED
155  	return -1;
156  #ifdef RSEQ_COMPARE_TWICE
157  error1:
158  	rseq_bug("cpu_id comparison failed");
159  #endif
160  }
161  
162  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)163  int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
164  			      intptr_t *v2, intptr_t expect2,
165  			      intptr_t newv, int cpu)
166  {
167  	RSEQ_INJECT_C(9)
168  
169  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
170  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
171  #ifdef RSEQ_COMPARE_TWICE
172  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
173  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
174  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
175  #endif
176  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
177  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
178  				  RSEQ_INJECT_ASM(3)
179  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
180  				  RSEQ_INJECT_ASM(4)
181  				  RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
182  				  RSEQ_INJECT_ASM(5)
183  #ifdef RSEQ_COMPARE_TWICE
184  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
185  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
186  				  RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
187  #endif
188  				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
189  				  RSEQ_INJECT_ASM(6)
190  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
191  				  : /* gcc asm goto does not allow outputs */
192  				  : [cpu_id]		"r" (cpu),
193  				    [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
194  				    [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
195  				    [v]			"m" (*v),
196  				    [expect]		"r" (expect),
197  				    [v2]			"m" (*v2),
198  				    [expect2]		"r" (expect2),
199  				    [newv]		"r" (newv)
200  				    RSEQ_INJECT_INPUT
201  				  : "memory", RSEQ_ASM_TMP_REG_1
202  				    RSEQ_INJECT_CLOBBER
203  				  : abort, cmpfail
204  #ifdef RSEQ_COMPARE_TWICE
205  				    , error1, error2, error3
206  #endif
207  	);
208  
209  	return 0;
210  abort:
211  	RSEQ_INJECT_FAILED
212  	return -1;
213  cmpfail:
214  	return 1;
215  #ifdef RSEQ_COMPARE_TWICE
216  error1:
217  	rseq_bug("cpu_id comparison failed");
218  error2:
219  	rseq_bug("expected value comparison failed");
220  error3:
221  	rseq_bug("2nd expected value comparison failed");
222  #endif
223  }
224  
225  #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
226  
227  /*
228   *   pval = *(ptr+off)
229   *  *pval += inc;
230   */
231  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)232  int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
233  {
234  	RSEQ_INJECT_C(9)
235  
236  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
237  #ifdef RSEQ_COMPARE_TWICE
238  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
239  #endif
240  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
241  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
242  				  RSEQ_INJECT_ASM(3)
243  #ifdef RSEQ_COMPARE_TWICE
244  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
245  #endif
246  				  RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
247  				  RSEQ_INJECT_ASM(4)
248  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
249  				  : /* gcc asm goto does not allow outputs */
250  				  : [cpu_id]			"r" (cpu),
251  				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
252  				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
253  				    [ptr]			"r" (ptr),
254  				    [off]			"er" (off),
255  				    [inc]			"er" (inc)
256  				    RSEQ_INJECT_INPUT
257  				  : "memory", RSEQ_ASM_TMP_REG_1
258  				    RSEQ_INJECT_CLOBBER
259  				  : abort
260  #ifdef RSEQ_COMPARE_TWICE
261  				    , error1
262  #endif
263  	);
264  	return 0;
265  abort:
266  	RSEQ_INJECT_FAILED
267  	return -1;
268  #ifdef RSEQ_COMPARE_TWICE
269  error1:
270  	rseq_bug("cpu_id comparison failed");
271  #endif
272  }
273  
274  #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
275  	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
276  
277  #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
278  	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
279  
280  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)281  int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
282  				 intptr_t *v2, intptr_t newv2,
283  				 intptr_t newv, int cpu)
284  {
285  	RSEQ_INJECT_C(9)
286  
287  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
288  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
289  #ifdef RSEQ_COMPARE_TWICE
290  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
291  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
292  #endif
293  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
294  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
295  				  RSEQ_INJECT_ASM(3)
296  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
297  				  RSEQ_INJECT_ASM(4)
298  #ifdef RSEQ_COMPARE_TWICE
299  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
300  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
301  #endif
302  				  RSEQ_ASM_OP_STORE(newv2, v2)
303  				  RSEQ_INJECT_ASM(5)
304  #ifdef RSEQ_TEMPLATE_MO_RELEASE
305  				  RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
306  #else
307  				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
308  #endif
309  				  RSEQ_INJECT_ASM(6)
310  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
311  				  : /* gcc asm goto does not allow outputs */
312  				  : [cpu_id]			"r" (cpu),
313  				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
314  				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
315  				    [expect]			"r" (expect),
316  				    [v]				"m" (*v),
317  				    [newv]			"r" (newv),
318  				    [v2]			"m" (*v2),
319  				    [newv2]			"r" (newv2)
320  				    RSEQ_INJECT_INPUT
321  				  : "memory", RSEQ_ASM_TMP_REG_1
322  				    RSEQ_INJECT_CLOBBER
323  				  : abort, cmpfail
324  #ifdef RSEQ_COMPARE_TWICE
325  				    , error1, error2
326  #endif
327  	);
328  
329  	return 0;
330  abort:
331  	RSEQ_INJECT_FAILED
332  	return -1;
333  cmpfail:
334  	return 1;
335  #ifdef RSEQ_COMPARE_TWICE
336  error1:
337  	rseq_bug("cpu_id comparison failed");
338  error2:
339  	rseq_bug("expected value comparison failed");
340  #endif
341  }
342  
343  static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)344  int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
345  				 void *dst, void *src, size_t len,
346  				 intptr_t newv, int cpu)
347  {
348  	RSEQ_INJECT_C(9)
349  	__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
350  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
351  #ifdef RSEQ_COMPARE_TWICE
352  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
353  				  RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
354  #endif
355  				  RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
356  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
357  				  RSEQ_INJECT_ASM(3)
358  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
359  				  RSEQ_INJECT_ASM(4)
360  #ifdef RSEQ_COMPARE_TWICE
361  				  RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
362  				  RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
363  #endif
364  				  RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
365  				  RSEQ_INJECT_ASM(5)
366  #ifdef RSEQ_TEMPLATE_MO_RELEASE
367  				  RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
368  #else
369  				  RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
370  #endif
371  				  RSEQ_INJECT_ASM(6)
372  				  RSEQ_ASM_DEFINE_ABORT(4, abort)
373  				  : /* gcc asm goto does not allow outputs */
374  				  : [cpu_id]			"r" (cpu),
375  				    [current_cpu_id]		"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
376  				    [rseq_cs]			"m" (rseq_get_abi()->rseq_cs.arch.ptr),
377  				    [expect]			"r" (expect),
378  				    [v]				"m" (*v),
379  				    [newv]			"r" (newv),
380  				    [dst]			"r" (dst),
381  				    [src]			"r" (src),
382  				    [len]			"r" (len)
383  				    RSEQ_INJECT_INPUT
384  				  : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
385  				    RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
386  				    RSEQ_INJECT_CLOBBER
387  				  : abort, cmpfail
388  #ifdef RSEQ_COMPARE_TWICE
389  				    , error1, error2
390  #endif
391  	);
392  
393  	return 0;
394  abort:
395  	RSEQ_INJECT_FAILED
396  	return -1;
397  cmpfail:
398  	return 1;
399  #ifdef RSEQ_COMPARE_TWICE
400  error1:
401  	rseq_bug("cpu_id comparison failed");
402  error2:
403  	rseq_bug("expected value comparison failed");
404  #endif
405  }
406  
407  #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
408  	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
409  
410  #include "rseq-bits-reset.h"
411