1 /* SPDX-License-Identifier: GPL-2.0-only */
13 	w = list_first_entry(&lock->wait_list, struct mutex_waiter, list);  in __ww_waiter_first()
14 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_first()
24 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_next()
34 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_prev()
45 	w = list_last_entry(&lock->wait_list, struct mutex_waiter, list);  in __ww_waiter_last()
46 	if (list_entry_is_head(w, &lock->wait_list, list))  in __ww_waiter_last()
55 	struct list_head *p = &lock->wait_list;  in __ww_waiter_add()
57 		p = &pos->list;  in __ww_waiter_add()
70 	return atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS;  in __ww_mutex_has_waiters()
75 	raw_spin_lock(&lock->wait_lock);  in lock_wait_lock()
80 	raw_spin_unlock(&lock->wait_lock);  in unlock_wait_lock()
85 	lockdep_assert_held(&lock->wait_lock);  in lockdep_assert_wait_lock_held()
96 	struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root);  in __ww_waiter_first()
105 	struct rb_node *n = rb_next(&w->tree.entry);  in __ww_waiter_next()
114 	struct rb_node *n = rb_prev(&w->tree.entry);  in __ww_waiter_prev()
123 	struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root);  in __ww_waiter_last()
132 	/* RT unconditionally adds the waiter first and then removes it on error */  in __ww_waiter_add()
138 	return rt_mutex_owner(&lock->rtmutex);  in __ww_mutex_owner()
144 	return rt_mutex_has_waiters(&lock->rtmutex);  in __ww_mutex_has_waiters()
149 	raw_spin_lock(&lock->rtmutex.wait_lock);  in lock_wait_lock()
154 	raw_spin_unlock(&lock->rtmutex.wait_lock);  in unlock_wait_lock()
159 	lockdep_assert_held(&lock->rtmutex.wait_lock);  in lockdep_assert_wait_lock_held()
165  * Wait-Die:
170  * Wound-Wait:
190 	DEBUG_LOCKS_WARN_ON(ww->ctx);  in ww_mutex_lock_acquired()
195 	DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);  in ww_mutex_lock_acquired()
197 	if (ww_ctx->contending_lock) {  in ww_mutex_lock_acquired()
199 		 * After -EDEADLK you tried to  in ww_mutex_lock_acquired()
202 		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);  in ww_mutex_lock_acquired()
205 		 * You called ww_mutex_lock after receiving -EDEADLK,  in ww_mutex_lock_acquired()
208 		DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);  in ww_mutex_lock_acquired()
209 		ww_ctx->contending_lock = NULL;  in ww_mutex_lock_acquired()
215 	DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);  in ww_mutex_lock_acquired()
217 	ww_ctx->acquired++;  in ww_mutex_lock_acquired()
218 	ww->ctx = ww_ctx;  in ww_mutex_lock_acquired()
225  * Depending on the algorithm, @a will either need to wait for @b, or die.
231  * Can only do the RT prio for WW_RT, because task->prio isn't stable due to PI,  in __ww_ctx_less()
232  * so the wait_list ordering will go wobbly. rt_mutex re-queues the waiter and  in __ww_ctx_less()
237 	int a_prio = a->task->prio;  in __ww_ctx_less()
238 	int b_prio = b->task->prio;  in __ww_ctx_less()
251 			if (dl_time_before(b->task->dl.deadline,  in __ww_ctx_less()
252 					   a->task->dl.deadline))  in __ww_ctx_less()
255 			if (dl_time_before(a->task->dl.deadline,  in __ww_ctx_less()
256 					   b->task->dl.deadline))  in __ww_ctx_less()
264 	/* FIFO order tie break -- bigger is younger */  in __ww_ctx_less()
265 	return (signed long)(a->stamp - b->stamp) > 0;  in __ww_ctx_less()
269  * Wait-Die; wake a lesser waiter context (when locks held) such that it can
270  * die.
273  * already (ctx->acquired > 0), because __ww_mutex_add_waiter() and
280 	if (!ww_ctx->is_wait_die)  in __ww_mutex_die()
283 	if (waiter->ww_ctx->acquired > 0 && __ww_ctx_less(waiter->ww_ctx, ww_ctx)) {  in __ww_mutex_die()
287 		wake_up_process(waiter->task);  in __ww_mutex_die()
294  * Wound-Wait; wound a lesser @hold_ctx if it holds the lock.
324 	if (ww_ctx->acquired > 0 && __ww_ctx_less(hold_ctx, ww_ctx)) {  in __ww_mutex_wound()
325 		hold_ctx->wounded = 1;  in __ww_mutex_wound()
331 		 * wakeup pending to re-read the wounded state.  in __ww_mutex_wound()
344  * waiting behind us on the wait-list, check if they need to die, or wound us.
346  * See __ww_mutex_add_waiter() for the list-order construction; basically the
349  * This relies on never mixing wait-die/wound-wait on the same wait-list;
352  * The current task must not be on the wait list.
364 		if (!cur->ww_ctx)  in __ww_mutex_check_waiters()
368 		    __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx))  in __ww_mutex_check_waiters()
383 	 * The lock->ctx update should be visible on all cores before  in ww_mutex_set_context_fastpath()
392 	 * [W] ww->ctx = ctx	    [W] MUTEX_FLAG_WAITERS  in ww_mutex_set_context_fastpath()
394 	 * [R] MUTEX_FLAG_WAITERS   [R] ww->ctx  in ww_mutex_set_context_fastpath()
397 	 * __ww_mutex_add_waiter() and makes sure we either observe ww->ctx  in ww_mutex_set_context_fastpath()
400 	if (likely(!__ww_mutex_has_waiters(&lock->base)))  in ww_mutex_set_context_fastpath()
405 	 * die or wound us.  in ww_mutex_set_context_fastpath()
407 	lock_wait_lock(&lock->base);  in ww_mutex_set_context_fastpath()
408 	__ww_mutex_check_waiters(&lock->base, ctx);  in ww_mutex_set_context_fastpath()
409 	unlock_wait_lock(&lock->base);  in ww_mutex_set_context_fastpath()
415 	if (ww_ctx->acquired > 0) {  in __ww_mutex_kill()
420 		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock);  in __ww_mutex_kill()
421 		ww_ctx->contending_lock = ww;  in __ww_mutex_kill()
423 		return -EDEADLK;  in __ww_mutex_kill()
432  * Wound-Wait: If we're wounded, kill ourself.
434  * Wait-Die: If we're trying to acquire a lock already held by an older
437  * Since __ww_mutex_add_waiter() orders the wait-list on stamp, we only have to
438  * look at waiters before us in the wait-list.
445 	struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);  in __ww_mutex_check_kill()
448 	if (ctx->acquired == 0)  in __ww_mutex_check_kill()
451 	if (!ctx->is_wait_die) {  in __ww_mutex_check_kill()
452 		if (ctx->wounded)  in __ww_mutex_check_kill()
468 		if (!cur->ww_ctx)  in __ww_mutex_check_kill()
478  * Add @waiter to the wait-list, keep the wait-list ordered by stamp, smallest
484  * Furthermore, for Wait-Die kill ourself immediately when possible (there are
486  * Wound-Wait ensure we wound the owning context when it is younger.
501 	is_wait_die = ww_ctx->is_wait_die;  in __ww_mutex_add_waiter()
506 	 * them. Wait-Die waiters may die here. Wound-Wait waiters  in __ww_mutex_add_waiter()
507 	 * never die here, but they are sorted in stamp order and  in __ww_mutex_add_waiter()
513 		if (!cur->ww_ctx)  in __ww_mutex_add_waiter()
516 		if (__ww_ctx_less(ww_ctx, cur->ww_ctx)) {  in __ww_mutex_add_waiter()
518 			 * Wait-Die: if we find an older context waiting, there  in __ww_mutex_add_waiter()
520 			 * die the moment it would acquire the lock.  in __ww_mutex_add_waiter()
534 		/* Wait-Die: ensure younger waiters die. */  in __ww_mutex_add_waiter()
541 	 * Wound-Wait: if we're blocking on a mutex owned by a younger context,  in __ww_mutex_add_waiter()
549 		 * MUTEX_FLAG_WAITERS vs the ww->ctx load,  in __ww_mutex_add_waiter()
550 		 * such that either we or the fastpath will wound @ww->ctx.  in __ww_mutex_add_waiter()
553 		__ww_mutex_wound(lock, ww_ctx, ww->ctx);  in __ww_mutex_add_waiter()
561 	if (lock->ctx) {  in __ww_mutex_unlock()
563 		DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);  in __ww_mutex_unlock()
565 		if (lock->ctx->acquired > 0)  in __ww_mutex_unlock()
566 			lock->ctx->acquired--;  in __ww_mutex_unlock()
567 		lock->ctx = NULL;  in __ww_mutex_unlock()