1  /* SPDX-License-Identifier: GPL-2.0 */
2  #ifndef __ASM_ARM_CMPXCHG_H
3  #define __ASM_ARM_CMPXCHG_H
4  
5  #include <linux/irqflags.h>
6  #include <linux/prefetch.h>
7  #include <asm/barrier.h>
8  #include <linux/cmpxchg-emu.h>
9  
10  #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
11  /*
12   * On the StrongARM, "swp" is terminally broken since it bypasses the
13   * cache totally.  This means that the cache becomes inconsistent, and,
14   * since we use normal loads/stores as well, this is really bad.
15   * Typically, this causes oopsen in filp_close, but could have other,
16   * more disastrous effects.  There are two work-arounds:
17   *  1. Disable interrupts and emulate the atomic swap
18   *  2. Clean the cache, perform atomic swap, flush the cache
19   *
20   * We choose (1) since its the "easiest" to achieve here and is not
21   * dependent on the processor type.
22   *
23   * NOTE that this solution won't work on an SMP system, so explcitly
24   * forbid it here.
25   */
26  #define swp_is_buggy
27  #endif
28  
29  static inline unsigned long
__arch_xchg(unsigned long x,volatile void * ptr,int size)30  __arch_xchg(unsigned long x, volatile void *ptr, int size)
31  {
32  	extern void __bad_xchg(volatile void *, int);
33  	unsigned long ret;
34  #ifdef swp_is_buggy
35  	unsigned long flags;
36  #endif
37  #if __LINUX_ARM_ARCH__ >= 6
38  	unsigned int tmp;
39  #endif
40  
41  	prefetchw((const void *)ptr);
42  
43  	switch (size) {
44  #if __LINUX_ARM_ARCH__ >= 6
45  #ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */
46  	case 1:
47  		asm volatile("@	__xchg1\n"
48  		"1:	ldrexb	%0, [%3]\n"
49  		"	strexb	%1, %2, [%3]\n"
50  		"	teq	%1, #0\n"
51  		"	bne	1b"
52  			: "=&r" (ret), "=&r" (tmp)
53  			: "r" (x), "r" (ptr)
54  			: "memory", "cc");
55  		break;
56  	case 2:
57  		asm volatile("@	__xchg2\n"
58  		"1:	ldrexh	%0, [%3]\n"
59  		"	strexh	%1, %2, [%3]\n"
60  		"	teq	%1, #0\n"
61  		"	bne	1b"
62  			: "=&r" (ret), "=&r" (tmp)
63  			: "r" (x), "r" (ptr)
64  			: "memory", "cc");
65  		break;
66  #endif
67  	case 4:
68  		asm volatile("@	__xchg4\n"
69  		"1:	ldrex	%0, [%3]\n"
70  		"	strex	%1, %2, [%3]\n"
71  		"	teq	%1, #0\n"
72  		"	bne	1b"
73  			: "=&r" (ret), "=&r" (tmp)
74  			: "r" (x), "r" (ptr)
75  			: "memory", "cc");
76  		break;
77  #elif defined(swp_is_buggy)
78  #ifdef CONFIG_SMP
79  #error SMP is not supported on this platform
80  #endif
81  	case 1:
82  		raw_local_irq_save(flags);
83  		ret = *(volatile unsigned char *)ptr;
84  		*(volatile unsigned char *)ptr = x;
85  		raw_local_irq_restore(flags);
86  		break;
87  
88  	case 4:
89  		raw_local_irq_save(flags);
90  		ret = *(volatile unsigned long *)ptr;
91  		*(volatile unsigned long *)ptr = x;
92  		raw_local_irq_restore(flags);
93  		break;
94  #else
95  	case 1:
96  		asm volatile("@	__xchg1\n"
97  		"	swpb	%0, %1, [%2]"
98  			: "=&r" (ret)
99  			: "r" (x), "r" (ptr)
100  			: "memory", "cc");
101  		break;
102  	case 4:
103  		asm volatile("@	__xchg4\n"
104  		"	swp	%0, %1, [%2]"
105  			: "=&r" (ret)
106  			: "r" (x), "r" (ptr)
107  			: "memory", "cc");
108  		break;
109  #endif
110  	default:
111  		/* Cause a link-time error, the xchg() size is not supported */
112  		__bad_xchg(ptr, size), ret = 0;
113  		break;
114  	}
115  
116  	return ret;
117  }
118  
119  #define arch_xchg_relaxed(ptr, x) ({					\
120  	(__typeof__(*(ptr)))__arch_xchg((unsigned long)(x), (ptr),	\
121  					sizeof(*(ptr)));		\
122  })
123  
124  #include <asm-generic/cmpxchg-local.h>
125  
126  #if __LINUX_ARM_ARCH__ < 6
127  /* min ARCH < ARMv6 */
128  
129  #ifdef CONFIG_SMP
130  #error "SMP is not supported on this platform"
131  #endif
132  
133  #define arch_xchg arch_xchg_relaxed
134  
135  /*
136   * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
137   * them available.
138   */
139  #define arch_cmpxchg_local(ptr, o, n) ({				\
140  	(__typeof(*ptr))__generic_cmpxchg_local((ptr),			\
141  					        (unsigned long)(o),	\
142  					        (unsigned long)(n),	\
143  					        sizeof(*(ptr)));	\
144  })
145  
146  #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
147  
148  #include <asm-generic/cmpxchg.h>
149  
150  #else	/* min ARCH >= ARMv6 */
151  
152  extern void __bad_cmpxchg(volatile void *ptr, int size);
153  
154  /*
155   * cmpxchg only support 32-bits operands on ARMv6.
156   */
157  
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,int size)158  static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
159  				      unsigned long new, int size)
160  {
161  	unsigned long oldval, res;
162  
163  	prefetchw((const void *)ptr);
164  
165  	switch (size) {
166  #ifdef CONFIG_CPU_V6	/* ARCH == ARMv6 */
167  	case 1:
168  		oldval = cmpxchg_emu_u8((volatile u8 *)ptr, old, new);
169  		break;
170  #else /* min ARCH > ARMv6 */
171  	case 1:
172  		do {
173  			asm volatile("@ __cmpxchg1\n"
174  			"	ldrexb	%1, [%2]\n"
175  			"	mov	%0, #0\n"
176  			"	teq	%1, %3\n"
177  			"	strexbeq %0, %4, [%2]\n"
178  				: "=&r" (res), "=&r" (oldval)
179  				: "r" (ptr), "Ir" (old), "r" (new)
180  				: "memory", "cc");
181  		} while (res);
182  		break;
183  	case 2:
184  		do {
185  			asm volatile("@ __cmpxchg1\n"
186  			"	ldrexh	%1, [%2]\n"
187  			"	mov	%0, #0\n"
188  			"	teq	%1, %3\n"
189  			"	strexheq %0, %4, [%2]\n"
190  				: "=&r" (res), "=&r" (oldval)
191  				: "r" (ptr), "Ir" (old), "r" (new)
192  				: "memory", "cc");
193  		} while (res);
194  		break;
195  #endif
196  	case 4:
197  		do {
198  			asm volatile("@ __cmpxchg4\n"
199  			"	ldrex	%1, [%2]\n"
200  			"	mov	%0, #0\n"
201  			"	teq	%1, %3\n"
202  			"	strexeq %0, %4, [%2]\n"
203  				: "=&r" (res), "=&r" (oldval)
204  				: "r" (ptr), "Ir" (old), "r" (new)
205  				: "memory", "cc");
206  		} while (res);
207  		break;
208  	default:
209  		__bad_cmpxchg(ptr, size);
210  		oldval = 0;
211  	}
212  
213  	return oldval;
214  }
215  
216  #define arch_cmpxchg_relaxed(ptr,o,n) ({				\
217  	(__typeof__(*(ptr)))__cmpxchg((ptr),				\
218  				      (unsigned long)(o),		\
219  				      (unsigned long)(n),		\
220  				      sizeof(*(ptr)));			\
221  })
222  
__cmpxchg_local(volatile void * ptr,unsigned long old,unsigned long new,int size)223  static inline unsigned long __cmpxchg_local(volatile void *ptr,
224  					    unsigned long old,
225  					    unsigned long new, int size)
226  {
227  	unsigned long ret;
228  
229  	switch (size) {
230  #ifdef CONFIG_CPU_V6	/* min ARCH == ARMv6 */
231  	case 1:
232  	case 2:
233  		ret = __generic_cmpxchg_local(ptr, old, new, size);
234  		break;
235  #endif
236  	default:
237  		ret = __cmpxchg(ptr, old, new, size);
238  	}
239  
240  	return ret;
241  }
242  
243  #define arch_cmpxchg_local(ptr, o, n) ({				\
244  	(__typeof(*ptr))__cmpxchg_local((ptr),				\
245  				        (unsigned long)(o),		\
246  				        (unsigned long)(n),		\
247  				        sizeof(*(ptr)));		\
248  })
249  
__cmpxchg64(unsigned long long * ptr,unsigned long long old,unsigned long long new)250  static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
251  					     unsigned long long old,
252  					     unsigned long long new)
253  {
254  	unsigned long long oldval;
255  	unsigned long res;
256  
257  	prefetchw(ptr);
258  
259  	__asm__ __volatile__(
260  "1:	ldrexd		%1, %H1, [%3]\n"
261  "	teq		%1, %4\n"
262  "	teqeq		%H1, %H4\n"
263  "	bne		2f\n"
264  "	strexd		%0, %5, %H5, [%3]\n"
265  "	teq		%0, #0\n"
266  "	bne		1b\n"
267  "2:"
268  	: "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
269  	: "r" (ptr), "r" (old), "r" (new)
270  	: "cc");
271  
272  	return oldval;
273  }
274  
275  #define arch_cmpxchg64_relaxed(ptr, o, n) ({				\
276  	(__typeof__(*(ptr)))__cmpxchg64((ptr),				\
277  					(unsigned long long)(o),	\
278  					(unsigned long long)(n));	\
279  })
280  
281  #define arch_cmpxchg64_local(ptr, o, n) arch_cmpxchg64_relaxed((ptr), (o), (n))
282  
283  #endif	/* __LINUX_ARM_ARCH__ >= 6 */
284  
285  #endif /* __ASM_ARM_CMPXCHG_H */
286