Lines Matching +full:lock +full:- +full:- +full:- +full:-
1 /* SPDX-License-Identifier: GPL-2.0-only */
9 __ww_waiter_first(struct mutex *lock) in __ww_waiter_first() argument
13 w = list_first_entry(&lock->wait_list, struct mutex_waiter, list); in __ww_waiter_first()
14 if (list_entry_is_head(w, &lock->wait_list, list)) in __ww_waiter_first()
21 __ww_waiter_next(struct mutex *lock, struct mutex_waiter *w) in __ww_waiter_next() argument
24 if (list_entry_is_head(w, &lock->wait_list, list)) in __ww_waiter_next()
31 __ww_waiter_prev(struct mutex *lock, struct mutex_waiter *w) in __ww_waiter_prev() argument
34 if (list_entry_is_head(w, &lock->wait_list, list)) in __ww_waiter_prev()
41 __ww_waiter_last(struct mutex *lock) in __ww_waiter_last() argument
45 w = list_last_entry(&lock->wait_list, struct mutex_waiter, list); in __ww_waiter_last()
46 if (list_entry_is_head(w, &lock->wait_list, list)) in __ww_waiter_last()
53 __ww_waiter_add(struct mutex *lock, struct mutex_waiter *waiter, struct mutex_waiter *pos) in __ww_waiter_add() argument
55 struct list_head *p = &lock->wait_list; in __ww_waiter_add()
57 p = &pos->list; in __ww_waiter_add()
58 __mutex_add_waiter(lock, waiter, p); in __ww_waiter_add()
62 __ww_mutex_owner(struct mutex *lock) in __ww_mutex_owner() argument
64 return __mutex_owner(lock); in __ww_mutex_owner()
68 __ww_mutex_has_waiters(struct mutex *lock) in __ww_mutex_has_waiters() argument
70 return atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS; in __ww_mutex_has_waiters()
73 static inline void lock_wait_lock(struct mutex *lock) in lock_wait_lock() argument
75 raw_spin_lock(&lock->wait_lock); in lock_wait_lock()
78 static inline void unlock_wait_lock(struct mutex *lock) in unlock_wait_lock() argument
80 raw_spin_unlock(&lock->wait_lock); in unlock_wait_lock()
83 static inline void lockdep_assert_wait_lock_held(struct mutex *lock) in lockdep_assert_wait_lock_held() argument
85 lockdep_assert_held(&lock->wait_lock); in lockdep_assert_wait_lock_held()
94 __ww_waiter_first(struct rt_mutex *lock) in __ww_waiter_first() argument
96 struct rb_node *n = rb_first(&lock->rtmutex.waiters.rb_root); in __ww_waiter_first()
103 __ww_waiter_next(struct rt_mutex *lock, struct rt_mutex_waiter *w) in __ww_waiter_next() argument
105 struct rb_node *n = rb_next(&w->tree.entry); in __ww_waiter_next()
112 __ww_waiter_prev(struct rt_mutex *lock, struct rt_mutex_waiter *w) in __ww_waiter_prev() argument
114 struct rb_node *n = rb_prev(&w->tree.entry); in __ww_waiter_prev()
121 __ww_waiter_last(struct rt_mutex *lock) in __ww_waiter_last() argument
123 struct rb_node *n = rb_last(&lock->rtmutex.waiters.rb_root); in __ww_waiter_last()
130 __ww_waiter_add(struct rt_mutex *lock, struct rt_mutex_waiter *waiter, struct rt_mutex_waiter *pos) in __ww_waiter_add() argument
136 __ww_mutex_owner(struct rt_mutex *lock) in __ww_mutex_owner() argument
138 return rt_mutex_owner(&lock->rtmutex); in __ww_mutex_owner()
142 __ww_mutex_has_waiters(struct rt_mutex *lock) in __ww_mutex_has_waiters() argument
144 return rt_mutex_has_waiters(&lock->rtmutex); in __ww_mutex_has_waiters()
147 static inline void lock_wait_lock(struct rt_mutex *lock) in lock_wait_lock() argument
149 raw_spin_lock(&lock->rtmutex.wait_lock); in lock_wait_lock()
152 static inline void unlock_wait_lock(struct rt_mutex *lock) in unlock_wait_lock() argument
154 raw_spin_unlock(&lock->rtmutex.wait_lock); in unlock_wait_lock()
157 static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock) in lockdep_assert_wait_lock_held() argument
159 lockdep_assert_held(&lock->rtmutex.wait_lock); in lockdep_assert_wait_lock_held()
165 * Wait-Die:
167 * It (the new transaction) makes a request for a lock being held
170 * Wound-Wait:
172 * An older transaction makes a request for a lock being held by
190 DEBUG_LOCKS_WARN_ON(ww->ctx); in ww_mutex_lock_acquired()
195 DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire); in ww_mutex_lock_acquired()
197 if (ww_ctx->contending_lock) { in ww_mutex_lock_acquired()
199 * After -EDEADLK you tried to in ww_mutex_lock_acquired()
202 DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww); in ww_mutex_lock_acquired()
205 * You called ww_mutex_lock after receiving -EDEADLK, in ww_mutex_lock_acquired()
208 DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0); in ww_mutex_lock_acquired()
209 ww_ctx->contending_lock = NULL; in ww_mutex_lock_acquired()
215 DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class); in ww_mutex_lock_acquired()
217 ww_ctx->acquired++; in ww_mutex_lock_acquired()
218 ww->ctx = ww_ctx; in ww_mutex_lock_acquired()
231 * Can only do the RT prio for WW_RT, because task->prio isn't stable due to PI, in __ww_ctx_less()
232 * so the wait_list ordering will go wobbly. rt_mutex re-queues the waiter and in __ww_ctx_less()
237 int a_prio = a->task->prio; in __ww_ctx_less()
238 int b_prio = b->task->prio; in __ww_ctx_less()
251 if (dl_time_before(b->task->dl.deadline, in __ww_ctx_less()
252 a->task->dl.deadline)) in __ww_ctx_less()
255 if (dl_time_before(a->task->dl.deadline, in __ww_ctx_less()
256 b->task->dl.deadline)) in __ww_ctx_less()
264 /* FIFO order tie break -- bigger is younger */ in __ww_ctx_less()
265 return (signed long)(a->stamp - b->stamp) > 0; in __ww_ctx_less()
269 * Wait-Die; wake a lesser waiter context (when locks held) such that it can
273 * already (ctx->acquired > 0), because __ww_mutex_add_waiter() and
277 __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter, in __ww_mutex_die() argument
280 if (!ww_ctx->is_wait_die) in __ww_mutex_die()
283 if (waiter->ww_ctx->acquired > 0 && __ww_ctx_less(waiter->ww_ctx, ww_ctx)) { in __ww_mutex_die()
285 debug_mutex_wake_waiter(lock, waiter); in __ww_mutex_die()
287 wake_up_process(waiter->task); in __ww_mutex_die()
294 * Wound-Wait; wound a lesser @hold_ctx if it holds the lock.
296 * Wound the lock holder if there are waiters with more important transactions
297 * than the lock holders. Even if multiple waiters may wound the lock holder,
300 static bool __ww_mutex_wound(struct MUTEX *lock, in __ww_mutex_wound() argument
304 struct task_struct *owner = __ww_mutex_owner(lock); in __ww_mutex_wound()
306 lockdep_assert_wait_lock_held(lock); in __ww_mutex_wound()
324 if (ww_ctx->acquired > 0 && __ww_ctx_less(hold_ctx, ww_ctx)) { in __ww_mutex_wound()
325 hold_ctx->wounded = 1; in __ww_mutex_wound()
331 * wakeup pending to re-read the wounded state. in __ww_mutex_wound()
343 * We just acquired @lock under @ww_ctx, if there are more important contexts
344 * waiting behind us on the wait-list, check if they need to die, or wound us.
346 * See __ww_mutex_add_waiter() for the list-order construction; basically the
349 * This relies on never mixing wait-die/wound-wait on the same wait-list;
355 __ww_mutex_check_waiters(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx) in __ww_mutex_check_waiters() argument
359 lockdep_assert_wait_lock_held(lock); in __ww_mutex_check_waiters()
361 for (cur = __ww_waiter_first(lock); cur; in __ww_mutex_check_waiters()
362 cur = __ww_waiter_next(lock, cur)) { in __ww_mutex_check_waiters()
364 if (!cur->ww_ctx) in __ww_mutex_check_waiters()
367 if (__ww_mutex_die(lock, cur, ww_ctx) || in __ww_mutex_check_waiters()
368 __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx)) in __ww_mutex_check_waiters()
374 * After acquiring lock with fastpath, where we do not hold wait_lock, set ctx
378 ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in ww_mutex_set_context_fastpath() argument
380 ww_mutex_lock_acquired(lock, ctx); in ww_mutex_set_context_fastpath()
383 * The lock->ctx update should be visible on all cores before in ww_mutex_set_context_fastpath()
392 * [W] ww->ctx = ctx [W] MUTEX_FLAG_WAITERS in ww_mutex_set_context_fastpath()
394 * [R] MUTEX_FLAG_WAITERS [R] ww->ctx in ww_mutex_set_context_fastpath()
397 * __ww_mutex_add_waiter() and makes sure we either observe ww->ctx in ww_mutex_set_context_fastpath()
400 if (likely(!__ww_mutex_has_waiters(&lock->base))) in ww_mutex_set_context_fastpath()
407 lock_wait_lock(&lock->base); in ww_mutex_set_context_fastpath()
408 __ww_mutex_check_waiters(&lock->base, ctx); in ww_mutex_set_context_fastpath()
409 unlock_wait_lock(&lock->base); in ww_mutex_set_context_fastpath()
413 __ww_mutex_kill(struct MUTEX *lock, struct ww_acquire_ctx *ww_ctx) in __ww_mutex_kill() argument
415 if (ww_ctx->acquired > 0) { in __ww_mutex_kill()
419 ww = container_of(lock, struct ww_mutex, base); in __ww_mutex_kill()
420 DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock); in __ww_mutex_kill()
421 ww_ctx->contending_lock = ww; in __ww_mutex_kill()
423 return -EDEADLK; in __ww_mutex_kill()
430 * Check the wound condition for the current lock acquire.
432 * Wound-Wait: If we're wounded, kill ourself.
434 * Wait-Die: If we're trying to acquire a lock already held by an older
437 * Since __ww_mutex_add_waiter() orders the wait-list on stamp, we only have to
438 * look at waiters before us in the wait-list.
441 __ww_mutex_check_kill(struct MUTEX *lock, struct MUTEX_WAITER *waiter, in __ww_mutex_check_kill() argument
444 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); in __ww_mutex_check_kill()
445 struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx); in __ww_mutex_check_kill()
448 if (ctx->acquired == 0) in __ww_mutex_check_kill()
451 if (!ctx->is_wait_die) { in __ww_mutex_check_kill()
452 if (ctx->wounded) in __ww_mutex_check_kill()
453 return __ww_mutex_kill(lock, ctx); in __ww_mutex_check_kill()
459 return __ww_mutex_kill(lock, ctx); in __ww_mutex_check_kill()
465 for (cur = __ww_waiter_prev(lock, waiter); cur; in __ww_mutex_check_kill()
466 cur = __ww_waiter_prev(lock, cur)) { in __ww_mutex_check_kill()
468 if (!cur->ww_ctx) in __ww_mutex_check_kill()
471 return __ww_mutex_kill(lock, ctx); in __ww_mutex_check_kill()
478 * Add @waiter to the wait-list, keep the wait-list ordered by stamp, smallest
479 * first. Such that older contexts are preferred to acquire the lock over
484 * Furthermore, for Wait-Die kill ourself immediately when possible (there are
486 * Wound-Wait ensure we wound the owning context when it is younger.
490 struct MUTEX *lock, in __ww_mutex_add_waiter() argument
497 __ww_waiter_add(lock, waiter, NULL); in __ww_mutex_add_waiter()
501 is_wait_die = ww_ctx->is_wait_die; in __ww_mutex_add_waiter()
506 * them. Wait-Die waiters may die here. Wound-Wait waiters in __ww_mutex_add_waiter()
508 * may wound the lock holder. in __ww_mutex_add_waiter()
510 for (cur = __ww_waiter_last(lock); cur; in __ww_mutex_add_waiter()
511 cur = __ww_waiter_prev(lock, cur)) { in __ww_mutex_add_waiter()
513 if (!cur->ww_ctx) in __ww_mutex_add_waiter()
516 if (__ww_ctx_less(ww_ctx, cur->ww_ctx)) { in __ww_mutex_add_waiter()
518 * Wait-Die: if we find an older context waiting, there in __ww_mutex_add_waiter()
520 * die the moment it would acquire the lock. in __ww_mutex_add_waiter()
523 int ret = __ww_mutex_kill(lock, ww_ctx); in __ww_mutex_add_waiter()
534 /* Wait-Die: ensure younger waiters die. */ in __ww_mutex_add_waiter()
535 __ww_mutex_die(lock, cur, ww_ctx); in __ww_mutex_add_waiter()
538 __ww_waiter_add(lock, waiter, pos); in __ww_mutex_add_waiter()
541 * Wound-Wait: if we're blocking on a mutex owned by a younger context, in __ww_mutex_add_waiter()
545 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base); in __ww_mutex_add_waiter()
549 * MUTEX_FLAG_WAITERS vs the ww->ctx load, in __ww_mutex_add_waiter()
550 * such that either we or the fastpath will wound @ww->ctx. in __ww_mutex_add_waiter()
553 __ww_mutex_wound(lock, ww_ctx, ww->ctx); in __ww_mutex_add_waiter()
559 static inline void __ww_mutex_unlock(struct ww_mutex *lock) in __ww_mutex_unlock() argument
561 if (lock->ctx) { in __ww_mutex_unlock()
563 DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired); in __ww_mutex_unlock()
565 if (lock->ctx->acquired > 0) in __ww_mutex_unlock()
566 lock->ctx->acquired--; in __ww_mutex_unlock()
567 lock->ctx = NULL; in __ww_mutex_unlock()