1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Atomic operations.
4 *
5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6 */
7 #ifndef _ASM_ATOMIC_H
8 #define _ASM_ATOMIC_H
9
10 #include <linux/types.h>
11 #include <asm/barrier.h>
12 #include <asm/cmpxchg.h>
13
14 #if __SIZEOF_LONG__ == 4
15 #define __LL "ll.w "
16 #define __SC "sc.w "
17 #define __AMADD "amadd.w "
18 #define __AMOR "amor.w "
19 #define __AMAND_DB "amand_db.w "
20 #define __AMOR_DB "amor_db.w "
21 #define __AMXOR_DB "amxor_db.w "
22 #elif __SIZEOF_LONG__ == 8
23 #define __LL "ll.d "
24 #define __SC "sc.d "
25 #define __AMADD "amadd.d "
26 #define __AMOR "amor.d "
27 #define __AMAND_DB "amand_db.d "
28 #define __AMOR_DB "amor_db.d "
29 #define __AMXOR_DB "amxor_db.d "
30 #endif
31
32 #define ATOMIC_INIT(i) { (i) }
33
34 #define arch_atomic_read(v) READ_ONCE((v)->counter)
35 #define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))
36
37 #define ATOMIC_OP(op, I, asm_op) \
38 static inline void arch_atomic_##op(int i, atomic_t *v) \
39 { \
40 __asm__ __volatile__( \
41 "am"#asm_op".w" " $zero, %1, %0 \n" \
42 : "+ZB" (v->counter) \
43 : "r" (I) \
44 : "memory"); \
45 }
46
47 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
48 static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
49 { \
50 int result; \
51 \
52 __asm__ __volatile__( \
53 "am"#asm_op#mb".w" " %1, %2, %0 \n" \
54 : "+ZB" (v->counter), "=&r" (result) \
55 : "r" (I) \
56 : "memory"); \
57 \
58 return result c_op I; \
59 }
60
61 #define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \
62 static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \
63 { \
64 int result; \
65 \
66 __asm__ __volatile__( \
67 "am"#asm_op#mb".w" " %1, %2, %0 \n" \
68 : "+ZB" (v->counter), "=&r" (result) \
69 : "r" (I) \
70 : "memory"); \
71 \
72 return result; \
73 }
74
75 #define ATOMIC_OPS(op, I, asm_op, c_op) \
76 ATOMIC_OP(op, I, asm_op) \
77 ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \
78 ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
79 ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
80 ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
81
82 ATOMIC_OPS(add, i, add, +)
83 ATOMIC_OPS(sub, -i, add, +)
84
85 #define arch_atomic_add_return arch_atomic_add_return
86 #define arch_atomic_add_return_acquire arch_atomic_add_return
87 #define arch_atomic_add_return_release arch_atomic_add_return
88 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
89 #define arch_atomic_sub_return arch_atomic_sub_return
90 #define arch_atomic_sub_return_acquire arch_atomic_sub_return
91 #define arch_atomic_sub_return_release arch_atomic_sub_return
92 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
93 #define arch_atomic_fetch_add arch_atomic_fetch_add
94 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
95 #define arch_atomic_fetch_add_release arch_atomic_fetch_add
96 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
97 #define arch_atomic_fetch_sub arch_atomic_fetch_sub
98 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
99 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
100 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
101
102 #undef ATOMIC_OPS
103
104 #define ATOMIC_OPS(op, I, asm_op) \
105 ATOMIC_OP(op, I, asm_op) \
106 ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
107 ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
108
ATOMIC_OPS(and,i,and)109 ATOMIC_OPS(and, i, and)
110 ATOMIC_OPS(or, i, or)
111 ATOMIC_OPS(xor, i, xor)
112
113 #define arch_atomic_fetch_and arch_atomic_fetch_and
114 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
115 #define arch_atomic_fetch_and_release arch_atomic_fetch_and
116 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
117 #define arch_atomic_fetch_or arch_atomic_fetch_or
118 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
119 #define arch_atomic_fetch_or_release arch_atomic_fetch_or
120 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
121 #define arch_atomic_fetch_xor arch_atomic_fetch_xor
122 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
123 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
124 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
125
126 #undef ATOMIC_OPS
127 #undef ATOMIC_FETCH_OP
128 #undef ATOMIC_OP_RETURN
129 #undef ATOMIC_OP
130
131 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
132 {
133 int prev, rc;
134
135 __asm__ __volatile__ (
136 "0: ll.w %[p], %[c]\n"
137 " beq %[p], %[u], 1f\n"
138 " add.w %[rc], %[p], %[a]\n"
139 " sc.w %[rc], %[c]\n"
140 " beqz %[rc], 0b\n"
141 " b 2f\n"
142 "1:\n"
143 __WEAK_LLSC_MB
144 "2:\n"
145 : [p]"=&r" (prev), [rc]"=&r" (rc),
146 [c]"=ZB" (v->counter)
147 : [a]"r" (a), [u]"r" (u)
148 : "memory");
149
150 return prev;
151 }
152 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
153
arch_atomic_sub_if_positive(int i,atomic_t * v)154 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
155 {
156 int result;
157 int temp;
158
159 if (__builtin_constant_p(i)) {
160 __asm__ __volatile__(
161 "1: ll.w %1, %2 # atomic_sub_if_positive\n"
162 " addi.w %0, %1, %3 \n"
163 " move %1, %0 \n"
164 " bltz %0, 2f \n"
165 " sc.w %1, %2 \n"
166 " beqz %1, 1b \n"
167 "2: \n"
168 __WEAK_LLSC_MB
169 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
170 : "I" (-i));
171 } else {
172 __asm__ __volatile__(
173 "1: ll.w %1, %2 # atomic_sub_if_positive\n"
174 " sub.w %0, %1, %3 \n"
175 " move %1, %0 \n"
176 " bltz %0, 2f \n"
177 " sc.w %1, %2 \n"
178 " beqz %1, 1b \n"
179 "2: \n"
180 __WEAK_LLSC_MB
181 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
182 : "r" (i));
183 }
184
185 return result;
186 }
187
188 #define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v)
189
190 #ifdef CONFIG_64BIT
191
192 #define ATOMIC64_INIT(i) { (i) }
193
194 #define arch_atomic64_read(v) READ_ONCE((v)->counter)
195 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
196
197 #define ATOMIC64_OP(op, I, asm_op) \
198 static inline void arch_atomic64_##op(long i, atomic64_t *v) \
199 { \
200 __asm__ __volatile__( \
201 "am"#asm_op".d " " $zero, %1, %0 \n" \
202 : "+ZB" (v->counter) \
203 : "r" (I) \
204 : "memory"); \
205 }
206
207 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
208 static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \
209 { \
210 long result; \
211 __asm__ __volatile__( \
212 "am"#asm_op#mb".d " " %1, %2, %0 \n" \
213 : "+ZB" (v->counter), "=&r" (result) \
214 : "r" (I) \
215 : "memory"); \
216 \
217 return result c_op I; \
218 }
219
220 #define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \
221 static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \
222 { \
223 long result; \
224 \
225 __asm__ __volatile__( \
226 "am"#asm_op#mb".d " " %1, %2, %0 \n" \
227 : "+ZB" (v->counter), "=&r" (result) \
228 : "r" (I) \
229 : "memory"); \
230 \
231 return result; \
232 }
233
234 #define ATOMIC64_OPS(op, I, asm_op, c_op) \
235 ATOMIC64_OP(op, I, asm_op) \
236 ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \
237 ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
238 ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
239 ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
240
241 ATOMIC64_OPS(add, i, add, +)
242 ATOMIC64_OPS(sub, -i, add, +)
243
244 #define arch_atomic64_add_return arch_atomic64_add_return
245 #define arch_atomic64_add_return_acquire arch_atomic64_add_return
246 #define arch_atomic64_add_return_release arch_atomic64_add_return
247 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
248 #define arch_atomic64_sub_return arch_atomic64_sub_return
249 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
250 #define arch_atomic64_sub_return_release arch_atomic64_sub_return
251 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
252 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
253 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
254 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
255 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
256 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
257 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
258 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
259 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
260
261 #undef ATOMIC64_OPS
262
263 #define ATOMIC64_OPS(op, I, asm_op) \
264 ATOMIC64_OP(op, I, asm_op) \
265 ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
266 ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
267
ATOMIC64_OPS(and,i,and)268 ATOMIC64_OPS(and, i, and)
269 ATOMIC64_OPS(or, i, or)
270 ATOMIC64_OPS(xor, i, xor)
271
272 #define arch_atomic64_fetch_and arch_atomic64_fetch_and
273 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
274 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
275 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
276 #define arch_atomic64_fetch_or arch_atomic64_fetch_or
277 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
278 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
279 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
280 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
281 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
282 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
283 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
284
285 #undef ATOMIC64_OPS
286 #undef ATOMIC64_FETCH_OP
287 #undef ATOMIC64_OP_RETURN
288 #undef ATOMIC64_OP
289
290 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
291 {
292 long prev, rc;
293
294 __asm__ __volatile__ (
295 "0: ll.d %[p], %[c]\n"
296 " beq %[p], %[u], 1f\n"
297 " add.d %[rc], %[p], %[a]\n"
298 " sc.d %[rc], %[c]\n"
299 " beqz %[rc], 0b\n"
300 " b 2f\n"
301 "1:\n"
302 __WEAK_LLSC_MB
303 "2:\n"
304 : [p]"=&r" (prev), [rc]"=&r" (rc),
305 [c] "=ZB" (v->counter)
306 : [a]"r" (a), [u]"r" (u)
307 : "memory");
308
309 return prev;
310 }
311 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
312
arch_atomic64_sub_if_positive(long i,atomic64_t * v)313 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
314 {
315 long result;
316 long temp;
317
318 if (__builtin_constant_p(i)) {
319 __asm__ __volatile__(
320 "1: ll.d %1, %2 # atomic64_sub_if_positive \n"
321 " addi.d %0, %1, %3 \n"
322 " move %1, %0 \n"
323 " bltz %0, 2f \n"
324 " sc.d %1, %2 \n"
325 " beqz %1, 1b \n"
326 "2: \n"
327 __WEAK_LLSC_MB
328 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
329 : "I" (-i));
330 } else {
331 __asm__ __volatile__(
332 "1: ll.d %1, %2 # atomic64_sub_if_positive \n"
333 " sub.d %0, %1, %3 \n"
334 " move %1, %0 \n"
335 " bltz %0, 2f \n"
336 " sc.d %1, %2 \n"
337 " beqz %1, 1b \n"
338 "2: \n"
339 __WEAK_LLSC_MB
340 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
341 : "r" (i));
342 }
343
344 return result;
345 }
346
347 #define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v)
348
349 #endif /* CONFIG_64BIT */
350
351 #endif /* _ASM_ATOMIC_H */
352