27 #include <trace/events/lock.h>
36 					struct rt_mutex *lock,  in __ww_mutex_add_waiter()  argument
42 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock,  in __ww_mutex_check_waiters()  argument
47 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock,  in ww_mutex_lock_acquired()  argument
52 static inline int __ww_mutex_check_kill(struct rt_mutex *lock,  in __ww_mutex_check_kill()  argument
66  * lock->owner state tracking:
68  * lock->owner holds the task_struct pointer of the owner. Bit 0
69  * is used to keep track of the "lock has waiters" state.
72  * NULL		0	lock is free (fast acquire possible)
73  * NULL		1	lock is free and has waiters and the top waiter
74  *				is going to take the lock*
75  * taskpointer	0	lock is held (fast release possible)
76  * taskpointer	1	lock is held and has waiters**
79  * possible when bit 0 of lock->owner is 0.
81  * (*) It also can be a transitional state when grabbing the lock
82  * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
83  * we need to set the bit0 before looking at the lock, and the owner may be
87  * waiters. This can happen when grabbing the lock in the slow path.
88  * To prevent a cmpxchg of the owner releasing the lock, we need to
89  * set this bit before looking at the lock.
93 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_owner_encode()  argument
97 	if (rt_mutex_has_waiters(lock))  in rt_mutex_owner_encode()
104 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
107 	 * lock->wait_lock is held but explicit acquire semantics are needed  in rt_mutex_set_owner()
108 	 * for a new lock owner so WRITE_ONCE is insufficient.  in rt_mutex_set_owner()
110 	xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner));  in rt_mutex_set_owner()
113 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock)  in rt_mutex_clear_owner()  argument
115 	/* lock->wait_lock is held so the unlock provides release semantics. */  in rt_mutex_clear_owner()
116 	WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));  in rt_mutex_clear_owner()
119 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)  in clear_rt_mutex_waiters()  argument
121 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
122 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
126 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock)  in fixup_rt_mutex_waiters()  argument
128 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in fixup_rt_mutex_waiters()
130 	if (rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
135 	 * lock->owner still has the waiters bit set, otherwise the  in fixup_rt_mutex_waiters()
141 	 *		lock(l->lock)  in fixup_rt_mutex_waiters()
145 	 *		  unlock(l->lock)  in fixup_rt_mutex_waiters()
149 	 *				lock(l->lock)  in fixup_rt_mutex_waiters()
153 	 *				  unlock(l->lock)  in fixup_rt_mutex_waiters()
156 	 *		lock(l->lock)  in fixup_rt_mutex_waiters()
159 	 *		  unlock(l->lock)  in fixup_rt_mutex_waiters()
160 	 *				lock(l->lock)  in fixup_rt_mutex_waiters()
164 	 *				 unlock(l->lock)  in fixup_rt_mutex_waiters()
165 	 *		lock(l->lock)  in fixup_rt_mutex_waiters()
172 	 *				lock(l->lock)  in fixup_rt_mutex_waiters()
185 	 * serialized by l->lock, so nothing else can modify the waiters  in fixup_rt_mutex_waiters()
199 		 * in case that the lock acquisition failed it might  in fixup_rt_mutex_waiters()
214 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
218 	return try_cmpxchg_acquire(&lock->owner, &old, new);  in rt_mutex_cmpxchg_acquire()
221 static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)  in rt_mutex_try_acquire()  argument
223 	return rt_mutex_cmpxchg_acquire(lock, NULL, current);  in rt_mutex_try_acquire()
226 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
230 	return try_cmpxchg_release(&lock->owner, &old, new);  in rt_mutex_cmpxchg_release()
235  * all future threads that attempt to [Rmw] the lock to the slowpath. As such
238 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
240 	unsigned long *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
259  * 2) Drop lock->wait_lock
260  * 3) Try to unlock the lock with cmpxchg
262 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
264 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
266 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()
268 	clear_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
269 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
275 	 *					lock(wait_lock);  in unlock_rt_mutex_safe()
277 	 *					mark_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
278 	 *					acquire(lock);  in unlock_rt_mutex_safe()
282 	 *					lock(wait_lock);  in unlock_rt_mutex_safe()
283 	 *					mark_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
288 	 * lock(wait_lock);  in unlock_rt_mutex_safe()
291 	 *					lock(wait_lock);  in unlock_rt_mutex_safe()
292 	 *					acquire(lock);  in unlock_rt_mutex_safe()
294 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
298 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
306 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock);
308 static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)  in rt_mutex_try_acquire()  argument
317 	return rt_mutex_slowtrylock(lock);  in rt_mutex_try_acquire()
320 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
327 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
329 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
330 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
334  * Simple slow path only version: lock->owner is protected by lock->wait_lock.
336 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
338 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
340 	lock->owner = NULL;  in unlock_rt_mutex_safe()
341 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
362 	lockdep_assert_held(&waiter->lock->wait_lock);  in waiter_update_prio()
375 	lockdep_assert_held(&waiter->lock->wait_lock);  in waiter_clone_prio()
477 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
479 	lockdep_assert_held(&lock->wait_lock);  in rt_mutex_enqueue()
481 	rb_add_cached(&waiter->tree.entry, &lock->waiters, __waiter_less);  in rt_mutex_enqueue()
485 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
487 	lockdep_assert_held(&lock->wait_lock);  in rt_mutex_dequeue()
492 	rb_erase_cached(&waiter->tree.entry, &lock->waiters);  in rt_mutex_dequeue()
524 static __always_inline void rt_mutex_adjust_prio(struct rt_mutex_base *lock,  in rt_mutex_adjust_prio()  argument
529 	lockdep_assert_held(&lock->wait_lock);  in rt_mutex_adjust_prio()
530 	lockdep_assert(rt_mutex_owner(lock) == p);  in rt_mutex_adjust_prio()
599 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;  in task_blocked_on_lock()
614  *		comparison to detect lock chain changes.
648  * [1]	  lock(task->pi_lock);			[R] acquire [P1]
651  * [4]	  lock = waiter->lock;			[P1]
652  * [5]	  if (!try_lock(lock->wait_lock)) {	[P1] try to acquire [L]
657  * [7]	  requeue_lock_waiter(lock, waiter);	[P1] + [L]
661  * [10]	  task = owner(lock);			[L]
663  *	  lock(task->pi_lock);			[L] acquire [P2]
664  * [11]	  requeue_pi_waiter(tsk, waiters(lock));[P2] + [L]
667  *	  unlock(lock->wait_lock);		release [L]
670  * Where P1 is the blocking task and P2 is the lock owner; going up one step
685 	struct rt_mutex_base *lock;  in rt_mutex_adjust_prio_chain()  local
699 	 * We limit the lock chain length for each invocation.  in rt_mutex_adjust_prio_chain()
710 			printk(KERN_WARNING "Maximum lock depth %d reached "  in rt_mutex_adjust_prio_chain()
750 	 * the previous owner of the lock might have released the lock.  in rt_mutex_adjust_prio_chain()
757 	 * the task might have moved on in the lock chain or even left  in rt_mutex_adjust_prio_chain()
758 	 * the chain completely and blocks now on an unrelated lock or  in rt_mutex_adjust_prio_chain()
761 	 * We stored the lock on which @task was blocked in @next_lock,  in rt_mutex_adjust_prio_chain()
764 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
768 	 * There could be 'spurious' loops in the lock graph due to ww_mutex,  in rt_mutex_adjust_prio_chain()
829 	 * [4] Get the next lock; per holding task->pi_lock we can't unblock  in rt_mutex_adjust_prio_chain()
830 	 * and guarantee @lock's existence.  in rt_mutex_adjust_prio_chain()
832 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
835 	 * which is the reverse lock order versus the other rtmutex  in rt_mutex_adjust_prio_chain()
838 	 * Per the above, holding task->pi_lock guarantees lock exists, so  in rt_mutex_adjust_prio_chain()
839 	 * inverting this lock order is infeasible from a life-time  in rt_mutex_adjust_prio_chain()
842 	if (!raw_spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
850 	 * lock->wait_lock.  in rt_mutex_adjust_prio_chain()
852 	 * Deadlock detection. If the lock is the same as the original  in rt_mutex_adjust_prio_chain()
853 	 * lock which caused us to walk the lock chain or if the  in rt_mutex_adjust_prio_chain()
854 	 * current lock is owned by the task which initiated the chain  in rt_mutex_adjust_prio_chain()
857 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
872 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
877 	 * If we just follow the lock chain for deadlock detection, no  in rt_mutex_adjust_prio_chain()
890 		 * [9] check_exit_conditions_3 protected by lock->wait_lock.  in rt_mutex_adjust_prio_chain()
891 		 * If there is no owner of the lock, end of chain.  in rt_mutex_adjust_prio_chain()
893 		if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
894 			raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
898 		/* [10] Grab the next task, i.e. owner of @lock */  in rt_mutex_adjust_prio_chain()
899 		task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
912 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
916 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
926 	 * operation on @lock. We need it for the boost/deboost  in rt_mutex_adjust_prio_chain()
929 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
931 	/* [7] Requeue the waiter in the lock waiter tree. */  in rt_mutex_adjust_prio_chain()
932 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
947 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
953 	 * Since we hold lock->waiter_lock, task cannot unblock, even if we  in rt_mutex_adjust_prio_chain()
960 	 * [9] check_exit_conditions_3 protected by lock->wait_lock.  in rt_mutex_adjust_prio_chain()
962 	 * We must abort the chain walk if there is no lock owner even  in rt_mutex_adjust_prio_chain()
963 	 * in the dead lock detection case, as we have nothing to  in rt_mutex_adjust_prio_chain()
966 	if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
970 		 * to get the lock.  in rt_mutex_adjust_prio_chain()
972 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
975 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
980 	 * [10] Grab the next task, i.e. the owner of @lock  in rt_mutex_adjust_prio_chain()
982 	 * Per holding lock->wait_lock and checking for !owner above, there  in rt_mutex_adjust_prio_chain()
985 	task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
989 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
992 		 * waiter on the lock. Replace the previous top waiter  in rt_mutex_adjust_prio_chain()
999 		rt_mutex_adjust_prio(lock, task);  in rt_mutex_adjust_prio_chain()
1003 		 * The waiter was the top waiter on the lock, but is  in rt_mutex_adjust_prio_chain()
1013 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
1016 		rt_mutex_adjust_prio(lock, task);  in rt_mutex_adjust_prio_chain()
1026 	 * and lock->wait_lock. The actual decisions are made after we  in rt_mutex_adjust_prio_chain()
1029 	 * Check whether the task which owns the current lock is pi  in rt_mutex_adjust_prio_chain()
1030 	 * blocked itself. If yes we store a pointer to the lock for  in rt_mutex_adjust_prio_chain()
1031 	 * the lock chain change detection above. After we dropped  in rt_mutex_adjust_prio_chain()
1036 	 * Store the top waiter of @lock for the end of chain walk  in rt_mutex_adjust_prio_chain()
1039 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
1043 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
1049 	 * We reached the end of the lock chain. Stop right here. No  in rt_mutex_adjust_prio_chain()
1056 	 * If the current waiter is not the top waiter on the lock,  in rt_mutex_adjust_prio_chain()
1076  * Must be called with lock->wait_lock held and interrupts disabled
1078  * @lock:   The lock to be acquired.
1079  * @task:   The task which wants to acquire the lock
1080  * @waiter: The waiter that is queued to the lock's wait tree if the
1084 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
1087 	lockdep_assert_held(&lock->wait_lock);  in try_to_take_rt_mutex()
1090 	 * Before testing whether we can acquire @lock, we set the  in try_to_take_rt_mutex()
1091 	 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all  in try_to_take_rt_mutex()
1092 	 * other tasks which try to modify @lock into the slow path  in try_to_take_rt_mutex()
1093 	 * and they serialize on @lock->wait_lock.  in try_to_take_rt_mutex()
1098 	 * - There is a lock owner. The caller must fixup the  in try_to_take_rt_mutex()
1099 	 *   transient state if it does a trylock or leaves the lock  in try_to_take_rt_mutex()
1102 	 * - @task acquires the lock and there are no other  in try_to_take_rt_mutex()
1106 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
1109 	 * If @lock has an owner, give up.  in try_to_take_rt_mutex()
1111 	if (rt_mutex_owner(lock))  in try_to_take_rt_mutex()
1116 	 * into @lock waiter tree. If @waiter == NULL then this is a  in try_to_take_rt_mutex()
1120 		struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock);  in try_to_take_rt_mutex()
1123 		 * If waiter is the highest priority waiter of @lock,  in try_to_take_rt_mutex()
1128 			 * We can acquire the lock. Remove the waiter from the  in try_to_take_rt_mutex()
1129 			 * lock waiters tree.  in try_to_take_rt_mutex()
1131 			rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
1137 		 * If the lock has waiters already we check whether @task is  in try_to_take_rt_mutex()
1138 		 * eligible to take over the lock.  in try_to_take_rt_mutex()
1141 		 * the lock.  @task->pi_blocked_on is NULL, so it does  in try_to_take_rt_mutex()
1144 		if (rt_mutex_has_waiters(lock)) {  in try_to_take_rt_mutex()
1147 					    rt_mutex_top_waiter(lock)))  in try_to_take_rt_mutex()
1152 			 * don't have to change anything in the lock  in try_to_take_rt_mutex()
1157 			 * No waiters. Take the lock without the  in try_to_take_rt_mutex()
1175 	 * Finish the lock acquisition. @task is the new owner. If  in try_to_take_rt_mutex()
1179 	if (rt_mutex_has_waiters(lock))  in try_to_take_rt_mutex()
1180 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
1188 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
1194  * Task blocks on lock.
1198  * This must be called with lock->wait_lock held and interrupts disabled
1200 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,  in task_blocks_on_rt_mutex()  argument
1206 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
1211 	lockdep_assert_held(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1230 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
1234 	/* Get the top priority waiter on the lock */  in task_blocks_on_rt_mutex()
1235 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
1236 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
1237 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
1247 		rtm = container_of(lock, struct rt_mutex, rtmutex);  in task_blocks_on_rt_mutex()
1251 			rt_mutex_dequeue(lock, waiter);  in task_blocks_on_rt_mutex()
1262 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1266 		rt_mutex_adjust_prio(lock, owner);  in task_blocks_on_rt_mutex()
1273 	/* Store the lock on which owner is blocked or NULL */  in task_blocks_on_rt_mutex()
1286 	 * The owner can't disappear while holding a lock,  in task_blocks_on_rt_mutex()
1292 	raw_spin_unlock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1294 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
1297 	raw_spin_lock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1306  * Called with lock->wait_lock held and interrupts disabled.
1309 					    struct rt_mutex_base *lock)  in mark_wakeup_next_waiter()  argument
1313 	lockdep_assert_held(&lock->wait_lock);  in mark_wakeup_next_waiter()
1317 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1327 	rt_mutex_adjust_prio(lock, current);  in mark_wakeup_next_waiter()
1331 	 * queued on the lock until it gets the lock, this lock  in mark_wakeup_next_waiter()
1335 	 * the top waiter can steal this lock.  in mark_wakeup_next_waiter()
1337 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1354 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in __rt_mutex_slowtrylock()  argument
1356 	int ret = try_to_take_rt_mutex(lock, current, NULL);  in __rt_mutex_slowtrylock()
1359 	 * try_to_take_rt_mutex() sets the lock waiters bit  in __rt_mutex_slowtrylock()
1362 	fixup_rt_mutex_waiters(lock, true);  in __rt_mutex_slowtrylock()
1368  * Slow path try-lock function:
1370 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in rt_mutex_slowtrylock()  argument
1376 	 * If the lock already has an owner we fail to get the lock.  in rt_mutex_slowtrylock()
1377 	 * This can be done without taking the @lock->wait_lock as  in rt_mutex_slowtrylock()
1380 	if (rt_mutex_owner(lock))  in rt_mutex_slowtrylock()
1384 	 * The mutex has currently no owner. Lock the wait lock and try to  in rt_mutex_slowtrylock()
1385 	 * acquire the lock. We use irqsave here to support early boot calls.  in rt_mutex_slowtrylock()
1387 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1389 	ret = __rt_mutex_slowtrylock(lock);  in rt_mutex_slowtrylock()
1391 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1396 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)  in __rt_mutex_trylock()  argument
1398 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_trylock()
1401 	return rt_mutex_slowtrylock(lock);  in __rt_mutex_trylock()
1407 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)  in rt_mutex_slowunlock()  argument
1413 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1415 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
1422 	 * foo->lock->owner = NULL;  in rt_mutex_slowunlock()
1423 	 *			rtmutex_lock(foo->lock);   <- fast path  in rt_mutex_slowunlock()
1425 	 *			rtmutex_unlock(foo->lock); <- fast path  in rt_mutex_slowunlock()
1428 	 * raw_spin_unlock(foo->lock->wait_lock);  in rt_mutex_slowunlock()
1433 	 * lock->wait_lock. So we do the following sequence:  in rt_mutex_slowunlock()
1435 	 *	owner = rt_mutex_owner(lock);  in rt_mutex_slowunlock()
1436 	 *	clear_rt_mutex_waiters(lock);  in rt_mutex_slowunlock()
1437 	 *	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_slowunlock()
1438 	 *	if (cmpxchg(&lock->owner, owner, 0) == owner)  in rt_mutex_slowunlock()
1443 	 * lock->owner is serialized by lock->wait_lock:  in rt_mutex_slowunlock()
1445 	 *	lock->owner = NULL;  in rt_mutex_slowunlock()
1446 	 *	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_slowunlock()
1448 	while (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
1449 		/* Drops lock->wait_lock ! */  in rt_mutex_slowunlock()
1450 		if (unlock_rt_mutex_safe(lock, flags) == true)  in rt_mutex_slowunlock()
1453 		raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1462 	mark_wakeup_next_waiter(&wqh, lock);  in rt_mutex_slowunlock()
1463 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1468 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)  in __rt_mutex_unlock()  argument
1470 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))  in __rt_mutex_unlock()
1473 	rt_mutex_slowunlock(lock);  in __rt_mutex_unlock()
1477 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1486 		if (owner != rt_mutex_owner(lock))  in rtmutex_spin_on_owner()
1490 		 * the lock owner still matches @owner. If that fails,  in rtmutex_spin_on_owner()
1497 		 *  - the lock owner has been scheduled out  in rtmutex_spin_on_owner()
1504 		    !rt_mutex_waiter_is_top_waiter(lock, waiter)) {  in rtmutex_spin_on_owner()
1514 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1530  * Remove a waiter from a lock and give up
1532  * Must be called with lock->wait_lock held and interrupts disabled. It must
1535 static void __sched remove_waiter(struct rt_mutex_base *lock,  in remove_waiter()  argument
1538 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1539 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
1542 	lockdep_assert_held(&lock->wait_lock);  in remove_waiter()
1545 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1551 	 * waiter of the lock and there is an owner to update.  in remove_waiter()
1560 	if (rt_mutex_has_waiters(lock))  in remove_waiter()
1561 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1563 	rt_mutex_adjust_prio(lock, owner);  in remove_waiter()
1565 	/* Store the lock on which owner is blocked or NULL */  in remove_waiter()
1580 	raw_spin_unlock_irq(&lock->wait_lock);  in remove_waiter()
1582 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1585 	raw_spin_lock_irq(&lock->wait_lock);  in remove_waiter()
1590  * @lock:		 the rt_mutex to take
1597  * Must be called with lock->wait_lock held and interrupts disabled
1599 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,  in rt_mutex_slowlock_block()  argument
1605 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in rt_mutex_slowlock_block()
1610 		/* Try to acquire the lock: */  in rt_mutex_slowlock_block()
1611 		if (try_to_take_rt_mutex(lock, current, waiter))  in rt_mutex_slowlock_block()
1629 		if (waiter == rt_mutex_top_waiter(lock))  in rt_mutex_slowlock_block()
1630 			owner = rt_mutex_owner(lock);  in rt_mutex_slowlock_block()
1633 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1635 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))  in rt_mutex_slowlock_block()
1638 		raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1647 					     struct rt_mutex_base *lock,  in rt_mutex_handle_deadlock()  argument
1660 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_handle_deadlock()
1671  * __rt_mutex_slowlock - Locking slowpath invoked with lock::wait_lock held
1672  * @lock:	The rtmutex to block lock
1678 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,  in __rt_mutex_slowlock()  argument
1684 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in __rt_mutex_slowlock()
1688 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_slowlock()
1690 	/* Try to acquire the lock again: */  in __rt_mutex_slowlock()
1691 	if (try_to_take_rt_mutex(lock, current, NULL)) {  in __rt_mutex_slowlock()
1701 	trace_contention_begin(lock, LCB_F_RT);  in __rt_mutex_slowlock()
1703 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);  in __rt_mutex_slowlock()
1705 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);  in __rt_mutex_slowlock()
1708 		/* acquired the lock */  in __rt_mutex_slowlock()
1716 		remove_waiter(lock, waiter);  in __rt_mutex_slowlock()
1717 		rt_mutex_handle_deadlock(ret, chwalk, lock, waiter);  in __rt_mutex_slowlock()
1724 	fixup_rt_mutex_waiters(lock, true);  in __rt_mutex_slowlock()
1726 	trace_contention_end(lock, ret);  in __rt_mutex_slowlock()
1731 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,  in __rt_mutex_slowlock_locked()  argument
1741 	ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK,  in __rt_mutex_slowlock_locked()
1750  * @lock:	The rtmutex to block lock
1754 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,  in rt_mutex_slowlock()  argument
1774 	 * rtmutex with lock->wait_lock held. But we cannot unconditionally  in rt_mutex_slowlock()
1778 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1779 	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);  in rt_mutex_slowlock()
1780 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1786 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,  in __rt_mutex_lock()  argument
1791 	if (likely(rt_mutex_try_acquire(lock)))  in __rt_mutex_lock()
1794 	return rt_mutex_slowlock(lock, NULL, state);  in __rt_mutex_lock()
1804  * rtlock_slowlock_locked - Slow path lock acquisition for RT locks
1805  * @lock:	The underlying RT mutex
1807 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)  in rtlock_slowlock_locked()  argument
1812 	lockdep_assert_held(&lock->wait_lock);  in rtlock_slowlock_locked()
1814 	if (try_to_take_rt_mutex(lock, current, NULL))  in rtlock_slowlock_locked()
1822 	trace_contention_begin(lock, LCB_F_RT);  in rtlock_slowlock_locked()
1824 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);  in rtlock_slowlock_locked()
1827 		/* Try to acquire the lock again */  in rtlock_slowlock_locked()
1828 		if (try_to_take_rt_mutex(lock, current, &waiter))  in rtlock_slowlock_locked()
1831 		if (&waiter == rt_mutex_top_waiter(lock))  in rtlock_slowlock_locked()
1832 			owner = rt_mutex_owner(lock);  in rtlock_slowlock_locked()
1835 		raw_spin_unlock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1837 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))  in rtlock_slowlock_locked()
1840 		raw_spin_lock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1851 	fixup_rt_mutex_waiters(lock, true);  in rtlock_slowlock_locked()
1854 	trace_contention_end(lock, 0);  in rtlock_slowlock_locked()
1857 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)  in rtlock_slowlock()  argument
1861 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rtlock_slowlock()
1862 	rtlock_slowlock_locked(lock);  in rtlock_slowlock()
1863 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rtlock_slowlock()