9 __ww_waiter_first(struct mutex *lock)  in __ww_waiter_first()  argument
13 	w = list_first_entry(&lock->wait_list, struct mutex_waiter, list);  in __ww_waiter_first()
14 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_first()
21 __ww_waiter_next(struct mutex *lock, struct mutex_waiter *w)  in __ww_waiter_next()  argument
24 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_next()
31 __ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w)  in __ww_waiter_prev()  argument
34 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_prev()
41 __ww_waiter_last(struct mutex *lock)  in __ww_waiter_last()  argument
45 	w = list_last_entry(&lock->wait_list, struct mutex_waiter, list);  in __ww_waiter_last()
46 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_last()
53 __ww_waiter_add(struct mutex *lock, struct mutex_waiter *waiter, struct mutex_waiter *pos)  in __ww_waiter_add()  argument
55 	struct list_head *p = &lock->wait_list;  in __ww_waiter_add()
58 	__mutex_add_waiter(lock, waiter, p);  in __ww_waiter_add()
62 __ww_mutex_owner(struct mutex *lock)  in __ww_mutex_owner()  argument
64 	return __mutex_owner(lock);  in __ww_mutex_owner()
68 __ww_mutex_has_waiters(struct mutex *lock)  in __ww_mutex_has_waiters()  argument
70 	return atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS;  in __ww_mutex_has_waiters()
73 static inline void lock_wait_lock(struct mutex *lock)  in lock_wait_lock()  argument
75 	raw_spin_lock(&lock->wait_lock);  in lock_wait_lock()
78 static inline void unlock_wait_lock(struct mutex *lock)  in unlock_wait_lock()  argument
80 	raw_spin_unlock(&lock->wait_lock);  in unlock_wait_lock()
83 static inline void lockdep_assert_wait_lock_held(struct mutex *lock)  in lockdep_assert_wait_lock_held()  argument
85 	lockdep_assert_held(&lock->wait_lock);  in lockdep_assert_wait_lock_held()
94 __ww_waiter_first(struct rt_mutex *lock)  in __ww_waiter_first()  argument
96 	struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root);  in __ww_waiter_first()
103 __ww_waiter_next(struct rt_mutex *lock, struct rt_mutex_waiter *w)  in __ww_waiter_next()  argument
112 __ww_waiter_prev(struct rt_mutex *lock, struct rt_mutex_waiter *w)  in __ww_waiter_prev()  argument
121 __ww_waiter_last(struct rt_mutex *lock)  in __ww_waiter_last()  argument
123 	struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root);  in __ww_waiter_last()
130 __ww_waiter_add(struct rt_mutex *lock, struct rt_mutex_waiter *waiter, struct rt_mutex_waiter *pos)  in __ww_waiter_add()  argument
136 __ww_mutex_owner(struct rt_mutex *lock)  in __ww_mutex_owner()  argument
138 	return rt_mutex_owner(&lock->rtmutex);  in __ww_mutex_owner()
142 __ww_mutex_has_waiters(struct rt_mutex *lock)  in __ww_mutex_has_waiters()  argument
144 	return rt_mutex_has_waiters(&lock->rtmutex);  in __ww_mutex_has_waiters()
147 static inline void lock_wait_lock(struct rt_mutex *lock)  in lock_wait_lock()  argument
149 	raw_spin_lock(&lock->rtmutex.wait_lock);  in lock_wait_lock()
152 static inline void unlock_wait_lock(struct rt_mutex *lock)  in unlock_wait_lock()  argument
154 	raw_spin_unlock(&lock->rtmutex.wait_lock);  in unlock_wait_lock()
157 static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock)  in lockdep_assert_wait_lock_held()  argument
159 	lockdep_assert_held(&lock->rtmutex.wait_lock);  in lockdep_assert_wait_lock_held()
167  *     It (the new transaction) makes a request for a lock being held
172  *     An older transaction makes a request for a lock being held by
277 __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter,  in __ww_mutex_die()  argument
285 		debug_mutex_wake_waiter(lock, waiter);  in __ww_mutex_die()
294  * Wound-Wait; wound a lesser @hold_ctx if it holds the lock.
296  * Wound the lock holder if there are waiters with more important transactions
297  * than the lock holders. Even if multiple waiters may wound the lock holder,
300 static bool __ww_mutex_wound(struct MUTEX *lock,  in __ww_mutex_wound()  argument
304 	struct task_struct *owner = __ww_mutex_owner(lock);  in __ww_mutex_wound()
306 	lockdep_assert_wait_lock_held(lock);  in __ww_mutex_wound()
343  * We just acquired @lock under @ww_ctx, if there are more important contexts
355 __ww_mutex_check_waiters(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_check_waiters()  argument
359 	lockdep_assert_wait_lock_held(lock);  in __ww_mutex_check_waiters()
361 	for (cur = __ww_waiter_first(lock); cur;  in __ww_mutex_check_waiters()
362 	     cur = __ww_waiter_next(lock, cur)) {  in __ww_mutex_check_waiters()
367 		if (__ww_mutex_die(lock, cur, ww_ctx) ||  in __ww_mutex_check_waiters()
368 		    __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx))  in __ww_mutex_check_waiters()
374  * After acquiring lock with fastpath, where we do not hold wait_lock, set ctx
378 ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_set_context_fastpath()  argument
380 	ww_mutex_lock_acquired(lock, ctx);  in ww_mutex_set_context_fastpath()
383 	 * The lock->ctx update should be visible on all cores before  in ww_mutex_set_context_fastpath()
400 	if (likely(!__ww_mutex_has_waiters(&lock->base)))  in ww_mutex_set_context_fastpath()
407 	lock_wait_lock(&lock->base);  in ww_mutex_set_context_fastpath()
408 	__ww_mutex_check_waiters(&lock->base, ctx);  in ww_mutex_set_context_fastpath()
409 	unlock_wait_lock(&lock->base);  in ww_mutex_set_context_fastpath()
413 __ww_mutex_kill(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_kill()  argument
419 		ww = container_of(lock, struct ww_mutex, base);  in __ww_mutex_kill()
430  * Check the wound condition for the current lock acquire.
434  * Wait-Die: If we're trying to acquire a lock already held by an older
441 __ww_mutex_check_kill(struct MUTEX *lock, struct MUTEX_WAITER *waiter,  in __ww_mutex_check_kill()  argument
444 	struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);  in __ww_mutex_check_kill()
453 			return __ww_mutex_kill(lock, ctx);  in __ww_mutex_check_kill()
459 		return __ww_mutex_kill(lock, ctx);  in __ww_mutex_check_kill()
465 	for (cur = __ww_waiter_prev(lock, waiter); cur;  in __ww_mutex_check_kill()
466 	     cur = __ww_waiter_prev(lock, cur)) {  in __ww_mutex_check_kill()
471 		return __ww_mutex_kill(lock, ctx);  in __ww_mutex_check_kill()
479  * first. Such that older contexts are preferred to acquire the lock over
490 		      struct MUTEX *lock,  in __ww_mutex_add_waiter()  argument
497 		__ww_waiter_add(lock, waiter, NULL);  in __ww_mutex_add_waiter()
508 	 * may wound the lock holder.  in __ww_mutex_add_waiter()
510 	for (cur = __ww_waiter_last(lock); cur;  in __ww_mutex_add_waiter()
511 	     cur = __ww_waiter_prev(lock, cur)) {  in __ww_mutex_add_waiter()
520 			 * die the moment it would acquire the lock.  in __ww_mutex_add_waiter()
523 				int ret = __ww_mutex_kill(lock, ww_ctx);  in __ww_mutex_add_waiter()
535 		__ww_mutex_die(lock, cur, ww_ctx);  in __ww_mutex_add_waiter()
538 	__ww_waiter_add(lock, waiter, pos);  in __ww_mutex_add_waiter()
545 		struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);  in __ww_mutex_add_waiter()
553 		__ww_mutex_wound(lock, ww_ctx, ww->ctx);  in __ww_mutex_add_waiter()
559 static inline void __ww_mutex_unlock(struct ww_mutex *lock)  in __ww_mutex_unlock()  argument
561 	if (lock->ctx) {  in __ww_mutex_unlock()
563 		DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);  in __ww_mutex_unlock()
565 		if (lock->ctx->acquired > 0)  in __ww_mutex_unlock()
566 			lock->ctx->acquired--;  in __ww_mutex_unlock()
567 		lock->ctx = NULL;  in __ww_mutex_unlock()