17 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val)	\
21 	off = (unsigned long)p % sizeof(u32);			\
23 	p -= off;						\
25 	prev_mask = (u32)(type)-1 << bitoff;			\
28 "1:	lwarx   %0,0,%3\n"					\
29 "	andc	%1,%0,%5\n"					\
30 "	or	%1,%1,%4\n"					\
31 "	stwcx.	%1,0,%3\n"					\
32 "	bne-	1b\n"						\
33 	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
34 	: "r" (p), "r" (val), "r" (prev_mask)			\
42 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new)	\
46 	off = (unsigned long)p % sizeof(u32);			\
48 	p -= off;						\
51 	prev_mask = (u32)(type)-1 << bitoff;			\
55 "1:	lwarx   %0,0,%3\n"					\
56 "	and	%1,%0,%6\n"					\
57 "	cmpw	0,%1,%4\n"					\
59 "	andc	%1,%0,%6\n"					\
60 "	or	%1,%1,%5\n"					\
61 "	stwcx.  %1,0,%3\n"					\
62 "	bne-    1b\n"						\
66 	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
67 	: "r" (p), "r" (old), "r" (new), "r" (prev_mask)	\
76  * Changes the memory location '*p' to be val and returns
87 __xchg_u8_local(volatile void *p, unsigned long val)  in __xchg_u8_local()  argument
92 "1:	lbarx	%0,0,%2		# __xchg_u8_local\n"  in __xchg_u8_local()
94 "	bne-	1b"  in __xchg_u8_local()
95 	: "=&r" (prev), "+m" (*(volatile unsigned char *)p)  in __xchg_u8_local()
96 	: "r" (p), "r" (val)  in __xchg_u8_local()
103 __xchg_u8_relaxed(u8 *p, unsigned long val)  in __xchg_u8_relaxed()  argument
108 "1:	lbarx	%0,0,%2		# __xchg_u8_relaxed\n"  in __xchg_u8_relaxed()
110 "	bne-	1b"  in __xchg_u8_relaxed()
111 	: "=&r" (prev), "+m" (*p)  in __xchg_u8_relaxed()
112 	: "r" (p), "r" (val)  in __xchg_u8_relaxed()
119 __xchg_u16_local(volatile void *p, unsigned long val)  in __xchg_u16_local()  argument
124 "1:	lharx	%0,0,%2		# __xchg_u16_local\n"  in __xchg_u16_local()
126 "	bne-	1b"  in __xchg_u16_local()
127 	: "=&r" (prev), "+m" (*(volatile unsigned short *)p)  in __xchg_u16_local()
128 	: "r" (p), "r" (val)  in __xchg_u16_local()
135 __xchg_u16_relaxed(u16 *p, unsigned long val)  in __xchg_u16_relaxed()  argument
140 "1:	lharx	%0,0,%2		# __xchg_u16_relaxed\n"  in __xchg_u16_relaxed()
142 "	bne-	1b"  in __xchg_u16_relaxed()
143 	: "=&r" (prev), "+m" (*p)  in __xchg_u16_relaxed()
144 	: "r" (p), "r" (val)  in __xchg_u16_relaxed()
152 __xchg_u32_local(volatile void *p, unsigned long val)  in __xchg_u32_local()  argument
157 "1:	lwarx	%0,0,%2 \n"  in __xchg_u32_local()
159 	bne-	1b"  in __xchg_u32_local()
160 	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)  in __xchg_u32_local()
161 	: "r" (p), "r" (val)  in __xchg_u32_local()
168 __xchg_u32_relaxed(u32 *p, unsigned long val)  in __xchg_u32_relaxed()  argument
173 "1:	lwarx	%0,0,%2\n"  in __xchg_u32_relaxed()
175 "	bne-	1b"  in __xchg_u32_relaxed()
176 	: "=&r" (prev), "+m" (*p)  in __xchg_u32_relaxed()
177 	: "r" (p), "r" (val)  in __xchg_u32_relaxed()
185 __xchg_u64_local(volatile void *p, unsigned long val)  in __xchg_u64_local()  argument
190 "1:	ldarx	%0,0,%2 \n"  in __xchg_u64_local()
192 	bne-	1b"  in __xchg_u64_local()
193 	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)  in __xchg_u64_local()
194 	: "r" (p), "r" (val)  in __xchg_u64_local()
201 __xchg_u64_relaxed(u64 *p, unsigned long val)  in __xchg_u64_relaxed()  argument
206 "1:	ldarx	%0,0,%2\n"  in __xchg_u64_relaxed()
208 "	bne-	1b"  in __xchg_u64_relaxed()
209 	: "=&r" (prev), "+m" (*p)  in __xchg_u64_relaxed()
210 	: "r" (p), "r" (val)  in __xchg_u64_relaxed()
221 	case 1:  in __xchg_local()
232 	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");  in __xchg_local()
240 	case 1:  in __xchg_relaxed()
251 	BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed");  in __xchg_relaxed()
269  * Compare and exchange - if *p == old, set it to new,
270  * and return the old value of *p.
283 __cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)  in __cmpxchg_u8()  argument
289 "1:	lbarx	%0,0,%2		# __cmpxchg_u8\n"  in __cmpxchg_u8()
293 "	bne-	1b"  in __cmpxchg_u8()
297 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u8()
298 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u8()
305 __cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,  in __cmpxchg_u8_local()  argument
311 "1:	lbarx	%0,0,%2		# __cmpxchg_u8_local\n"  in __cmpxchg_u8_local()
315 "	bne-	1b\n"  in __cmpxchg_u8_local()
317 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u8_local()
318 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u8_local()
325 __cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)  in __cmpxchg_u8_relaxed()  argument
330 "1:	lbarx	%0,0,%2		# __cmpxchg_u8_relaxed\n"  in __cmpxchg_u8_relaxed()
334 "	bne-	1b\n"  in __cmpxchg_u8_relaxed()
336 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u8_relaxed()
337 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u8_relaxed()
344 __cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)  in __cmpxchg_u8_acquire()  argument
349 "1:	lbarx	%0,0,%2		# __cmpxchg_u8_acquire\n"  in __cmpxchg_u8_acquire()
353 "	bne-	1b\n"  in __cmpxchg_u8_acquire()
356 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u8_acquire()
357 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u8_acquire()
364 __cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)  in __cmpxchg_u16()  argument
370 "1:	lharx	%0,0,%2		# __cmpxchg_u16\n"  in __cmpxchg_u16()
374 "	bne-	1b\n"  in __cmpxchg_u16()
377 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u16()
378 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u16()
385 __cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,  in __cmpxchg_u16_local()  argument
391 "1:	lharx	%0,0,%2		# __cmpxchg_u16_local\n"  in __cmpxchg_u16_local()
395 "	bne-	1b"  in __cmpxchg_u16_local()
397 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u16_local()
398 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u16_local()
405 __cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)  in __cmpxchg_u16_relaxed()  argument
410 "1:	lharx	%0,0,%2		# __cmpxchg_u16_relaxed\n"  in __cmpxchg_u16_relaxed()
414 "	bne-	1b\n"  in __cmpxchg_u16_relaxed()
416 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u16_relaxed()
417 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u16_relaxed()
424 __cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)  in __cmpxchg_u16_acquire()  argument
429 "1:	lharx	%0,0,%2		# __cmpxchg_u16_acquire\n"  in __cmpxchg_u16_acquire()
433 "	bne-	1b\n"  in __cmpxchg_u16_acquire()
436 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u16_acquire()
437 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u16_acquire()
445 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)  in __cmpxchg_u32()  argument
451 "1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\  in __cmpxchg_u32()
455 	bne-	1b"  in __cmpxchg_u32()
459 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u32()
460 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u32()
467 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,  in __cmpxchg_u32_local()  argument
473 "1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\  in __cmpxchg_u32_local()
477 	bne-	1b"  in __cmpxchg_u32_local()
480 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u32_local()
481 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u32_local()
488 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)  in __cmpxchg_u32_relaxed()  argument
493 "1:	lwarx	%0,0,%2		# __cmpxchg_u32_relaxed\n"  in __cmpxchg_u32_relaxed()
497 "	bne-	1b\n"  in __cmpxchg_u32_relaxed()
499 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u32_relaxed()
500 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u32_relaxed()
515 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)  in __cmpxchg_u32_acquire()  argument
520 "1:	lwarx	%0,0,%2		# __cmpxchg_u32_acquire\n"  in __cmpxchg_u32_acquire()
524 "	bne-	1b\n"  in __cmpxchg_u32_acquire()
528 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u32_acquire()
529 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u32_acquire()
537 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)  in __cmpxchg_u64()  argument
543 "1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\  in __cmpxchg_u64()
547 	bne-	1b"  in __cmpxchg_u64()
551 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u64()
552 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u64()
559 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,  in __cmpxchg_u64_local()  argument
565 "1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\  in __cmpxchg_u64_local()
569 	bne-	1b"  in __cmpxchg_u64_local()
572 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u64_local()
573 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u64_local()
580 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)  in __cmpxchg_u64_relaxed()  argument
585 "1:	ldarx	%0,0,%2		# __cmpxchg_u64_relaxed\n"  in __cmpxchg_u64_relaxed()
589 "	bne-	1b\n"  in __cmpxchg_u64_relaxed()
591 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u64_relaxed()
592 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u64_relaxed()
599 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)  in __cmpxchg_u64_acquire()  argument
604 "1:	ldarx	%0,0,%2		# __cmpxchg_u64_acquire\n"  in __cmpxchg_u64_acquire()
608 "	bne-	1b\n"  in __cmpxchg_u64_acquire()
612 	: "=&r" (prev), "+m" (*p)  in __cmpxchg_u64_acquire()
613 	: "r" (p), "r" (old), "r" (new)  in __cmpxchg_u64_acquire()
625 	case 1:  in __cmpxchg()
636 	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");  in __cmpxchg()
645 	case 1:  in __cmpxchg_local()
656 	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");  in __cmpxchg_local()
665 	case 1:  in __cmpxchg_relaxed()
676 	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");  in __cmpxchg_relaxed()
685 	case 1:  in __cmpxchg_acquire()
696 	BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");  in __cmpxchg_acquire()