44 static inline unsigned long *
45 __bitops_word(unsigned long nr, const volatile unsigned long *ptr)  in __bitops_word()
47 	unsigned long addr;  in __bitops_word()
49 	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);  in __bitops_word()
50 	return (unsigned long *)addr;  in __bitops_word()
53 static inline unsigned long __bitops_mask(unsigned long nr)  in __bitops_mask()
58 static __always_inline void arch_set_bit(unsigned long nr, volatile unsigned long *ptr)  in arch_set_bit()
60 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_set_bit()
61 	unsigned long mask = __bitops_mask(nr);  in arch_set_bit()
63 	__atomic64_or(mask, (long *)addr);  in arch_set_bit()
66 static __always_inline void arch_clear_bit(unsigned long nr, volatile unsigned long *ptr)  in arch_clear_bit()
68 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_clear_bit()
69 	unsigned long mask = __bitops_mask(nr);  in arch_clear_bit()
71 	__atomic64_and(~mask, (long *)addr);  in arch_clear_bit()
74 static __always_inline void arch_change_bit(unsigned long nr,  in arch_change_bit()
75 					    volatile unsigned long *ptr)  in arch_change_bit()
77 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_change_bit()
78 	unsigned long mask = __bitops_mask(nr);  in arch_change_bit()
80 	__atomic64_xor(mask, (long *)addr);  in arch_change_bit()
83 static inline bool arch_test_and_set_bit(unsigned long nr,  in arch_test_and_set_bit()
84 					 volatile unsigned long *ptr)  in arch_test_and_set_bit()
86 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_and_set_bit()
87 	unsigned long mask = __bitops_mask(nr);  in arch_test_and_set_bit()
88 	unsigned long old;  in arch_test_and_set_bit()
90 	old = __atomic64_or_barrier(mask, (long *)addr);  in arch_test_and_set_bit()
94 static inline bool arch_test_and_clear_bit(unsigned long nr,  in arch_test_and_clear_bit()
95 					   volatile unsigned long *ptr)  in arch_test_and_clear_bit()
97 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_and_clear_bit()
98 	unsigned long mask = __bitops_mask(nr);  in arch_test_and_clear_bit()
99 	unsigned long old;  in arch_test_and_clear_bit()
101 	old = __atomic64_and_barrier(~mask, (long *)addr);  in arch_test_and_clear_bit()
105 static inline bool arch_test_and_change_bit(unsigned long nr,  in arch_test_and_change_bit()
106 					    volatile unsigned long *ptr)  in arch_test_and_change_bit()
108 	unsigned long *addr = __bitops_word(nr, ptr);  in arch_test_and_change_bit()
109 	unsigned long mask = __bitops_mask(nr);  in arch_test_and_change_bit()
110 	unsigned long old;  in arch_test_and_change_bit()
112 	old = __atomic64_xor_barrier(mask, (long *)addr);  in arch_test_and_change_bit()
117 arch___set_bit(unsigned long nr, volatile unsigned long *addr)  in arch___set_bit()
119 	unsigned long *p = __bitops_word(nr, addr);  in arch___set_bit()
120 	unsigned long mask = __bitops_mask(nr);  in arch___set_bit()
126 arch___clear_bit(unsigned long nr, volatile unsigned long *addr)  in arch___clear_bit()
128 	unsigned long *p = __bitops_word(nr, addr);  in arch___clear_bit()
129 	unsigned long mask = __bitops_mask(nr);  in arch___clear_bit()
135 arch___change_bit(unsigned long nr, volatile unsigned long *addr)  in arch___change_bit()
137 	unsigned long *p = __bitops_word(nr, addr);  in arch___change_bit()
138 	unsigned long mask = __bitops_mask(nr);  in arch___change_bit()
144 arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)  in arch___test_and_set_bit()
146 	unsigned long *p = __bitops_word(nr, addr);  in arch___test_and_set_bit()
147 	unsigned long mask = __bitops_mask(nr);  in arch___test_and_set_bit()
148 	unsigned long old;  in arch___test_and_set_bit()
156 arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)  in arch___test_and_clear_bit()
158 	unsigned long *p = __bitops_word(nr, addr);  in arch___test_and_clear_bit()
159 	unsigned long mask = __bitops_mask(nr);  in arch___test_and_clear_bit()
160 	unsigned long old;  in arch___test_and_clear_bit()
168 arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)  in arch___test_and_change_bit()
170 	unsigned long *p = __bitops_word(nr, addr);  in arch___test_and_change_bit()
171 	unsigned long mask = __bitops_mask(nr);  in arch___test_and_change_bit()
172 	unsigned long old;  in arch___test_and_change_bit()
182 static inline bool arch_test_and_set_bit_lock(unsigned long nr,  in arch_test_and_set_bit_lock()
183 					      volatile unsigned long *ptr)  in arch_test_and_set_bit_lock()
190 static inline void arch_clear_bit_unlock(unsigned long nr,  in arch_clear_bit_unlock()
191 					 volatile unsigned long *ptr)  in arch_clear_bit_unlock()
197 static inline void arch___clear_bit_unlock(unsigned long nr,  in arch___clear_bit_unlock()
198 					   volatile unsigned long *ptr)  in arch___clear_bit_unlock()
204 static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,  in arch_xor_unlock_is_negative_byte()
205 		volatile unsigned long *ptr)  in arch_xor_unlock_is_negative_byte()
207 	unsigned long old;  in arch_xor_unlock_is_negative_byte()
209 	old = __atomic64_xor_barrier(mask, (long *)ptr);  in arch_xor_unlock_is_negative_byte()
223 unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
224 unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
225 				unsigned long offset);
232 static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in set_bit_inv()
237 static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in clear_bit_inv()
242 static inline bool test_and_clear_bit_inv(unsigned long nr,  in test_and_clear_bit_inv()
243 					  volatile unsigned long *ptr)  in test_and_clear_bit_inv()
248 static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in __set_bit_inv()
253 static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)  in __clear_bit_inv()
258 static inline bool test_bit_inv(unsigned long nr,  in test_bit_inv()
259 				const volatile unsigned long *ptr)  in test_bit_inv()
272 static inline unsigned char __flogr(unsigned long word)  in __flogr()
275 		unsigned long bit = 0;  in __flogr()
321 static inline unsigned long __ffs(unsigned long word)  in __ffs()
335 	unsigned long mask = 2 * BITS_PER_LONG - 1;  in ffs()
342  * __fls - find last (most-significant) set bit in a long word
347 static inline unsigned long __fls(unsigned long word)  in __fls()
363 static inline int fls64(unsigned long word)  in fls64()
365 	unsigned long mask = 2 * BITS_PER_LONG - 1;  in fls64()