Lines Matching +full:lock +full:- +full:- +full:- +full:-
1 // SPDX-License-Identifier: GPL-2.0-only
17 * Debug aware fast / slowpath lock,trylock,unlock
22 static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, in __rt_mutex_lock_common() argument
30 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_); in __rt_mutex_lock_common()
31 ret = __rt_mutex_lock(&lock->rtmutex, state); in __rt_mutex_lock_common()
33 mutex_release(&lock->dep_map, _RET_IP_); in __rt_mutex_lock_common()
45 * rt_mutex_lock_nested - lock a rt_mutex
47 * @lock: the rt_mutex to be locked
50 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) in rt_mutex_lock_nested() argument
52 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass); in rt_mutex_lock_nested()
56 void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) in _rt_mutex_lock_nest_lock() argument
58 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0); in _rt_mutex_lock_nest_lock()
65 * rt_mutex_lock - lock a rt_mutex
67 * @lock: the rt_mutex to be locked
69 void __sched rt_mutex_lock(struct rt_mutex *lock) in rt_mutex_lock() argument
71 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0); in rt_mutex_lock()
77 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
79 * @lock: the rt_mutex to be locked
83 * -EINTR when interrupted by a signal
85 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) in rt_mutex_lock_interruptible() argument
87 return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, NULL, 0); in rt_mutex_lock_interruptible()
92 * rt_mutex_lock_killable - lock a rt_mutex killable
94 * @lock: the rt_mutex to be locked
98 * -EINTR when interrupted by a signal
100 int __sched rt_mutex_lock_killable(struct rt_mutex *lock) in rt_mutex_lock_killable() argument
102 return __rt_mutex_lock_common(lock, TASK_KILLABLE, NULL, 0); in rt_mutex_lock_killable()
107 * rt_mutex_trylock - try to lock a rt_mutex
109 * @lock: the rt_mutex to be locked
118 int __sched rt_mutex_trylock(struct rt_mutex *lock) in rt_mutex_trylock() argument
125 ret = __rt_mutex_trylock(&lock->rtmutex); in rt_mutex_trylock()
127 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); in rt_mutex_trylock()
134 * rt_mutex_unlock - unlock a rt_mutex
136 * @lock: the rt_mutex to be unlocked
138 void __sched rt_mutex_unlock(struct rt_mutex *lock) in rt_mutex_unlock() argument
140 mutex_release(&lock->dep_map, _RET_IP_); in rt_mutex_unlock()
141 __rt_mutex_unlock(&lock->rtmutex); in rt_mutex_unlock()
148 int __sched rt_mutex_futex_trylock(struct rt_mutex_base *lock) in rt_mutex_futex_trylock() argument
150 return rt_mutex_slowtrylock(lock); in rt_mutex_futex_trylock()
153 int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock) in __rt_mutex_futex_trylock() argument
155 return __rt_mutex_slowtrylock(lock); in __rt_mutex_futex_trylock()
159 * __rt_mutex_futex_unlock - Futex variant, that since futex variants
160 * do not use the fast-path, can be simple and will not need to retry.
162 * @lock: The rt_mutex to be unlocked
163 * @wqh: The wake queue head from which to get the next lock waiter
165 bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock, in __rt_mutex_futex_unlock() argument
168 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_futex_unlock()
170 debug_rt_mutex_unlock(lock); in __rt_mutex_futex_unlock()
172 if (!rt_mutex_has_waiters(lock)) { in __rt_mutex_futex_unlock()
173 lock->owner = NULL; in __rt_mutex_futex_unlock()
183 mark_wakeup_next_waiter(wqh, lock); in __rt_mutex_futex_unlock()
188 void __sched rt_mutex_futex_unlock(struct rt_mutex_base *lock) in rt_mutex_futex_unlock() argument
194 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
195 postunlock = __rt_mutex_futex_unlock(lock, &wqh); in rt_mutex_futex_unlock()
196 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
203 * __rt_mutex_init - initialize the rt_mutex
205 * @lock: The rt_mutex to be initialized
206 * @name: The lock name used for debugging
207 * @key: The lock class key used for debugging
213 void __sched __rt_mutex_init(struct rt_mutex *lock, const char *name, in __rt_mutex_init() argument
216 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); in __rt_mutex_init()
217 __rt_mutex_base_init(&lock->rtmutex); in __rt_mutex_init()
218 lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP); in __rt_mutex_init()
223 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
226 * @lock: the rt_mutex to be locked
231 * Special API call for PI-futex support. This initializes the rtmutex and
236 void __sched rt_mutex_init_proxy_locked(struct rt_mutex_base *lock, in rt_mutex_init_proxy_locked() argument
241 __rt_mutex_base_init(lock); in rt_mutex_init_proxy_locked()
245 * some of the futex functions invoke spin_unlock(&hb->lock) with in rt_mutex_init_proxy_locked()
248 * the spinlock is based, which makes lockdep notice a lock in rt_mutex_init_proxy_locked()
251 lockdep_set_class(&lock->wait_lock, &pi_futex_key); in rt_mutex_init_proxy_locked()
252 rt_mutex_set_owner(lock, proxy_owner); in rt_mutex_init_proxy_locked()
256 * rt_mutex_proxy_unlock - release a lock on behalf of owner
258 * @lock: the rt_mutex to be locked
262 * Special API call for PI-futex support. This just cleans up the rtmutex
267 void __sched rt_mutex_proxy_unlock(struct rt_mutex_base *lock) in rt_mutex_proxy_unlock() argument
269 debug_rt_mutex_proxy_unlock(lock); in rt_mutex_proxy_unlock()
270 rt_mutex_clear_owner(lock); in rt_mutex_proxy_unlock()
274 * __rt_mutex_start_proxy_lock() - Start lock acquisition for another task
275 * @lock: the rt_mutex to take
276 * @waiter: the pre-initialized rt_mutex_waiter
286 * 0 - task blocked on lock
287 * 1 - acquired the lock for task, caller should wake it up
288 * <0 - error
290 * Special API call for PI-futex support.
292 int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, in __rt_mutex_start_proxy_lock() argument
298 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_start_proxy_lock()
300 if (try_to_take_rt_mutex(lock, task, NULL)) in __rt_mutex_start_proxy_lock()
304 ret = task_blocks_on_rt_mutex(lock, waiter, task, NULL, in __rt_mutex_start_proxy_lock()
307 if (ret && !rt_mutex_owner(lock)) { in __rt_mutex_start_proxy_lock()
310 * returned with -EDEADLK and the owner in __rt_mutex_start_proxy_lock()
311 * released the lock while we were walking the in __rt_mutex_start_proxy_lock()
321 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
322 * @lock: the rt_mutex to take
323 * @waiter: the pre-initialized rt_mutex_waiter
333 * 0 - task blocked on lock
334 * 1 - acquired the lock for task, caller should wake it up
335 * <0 - error
337 * Special API call for PI-futex support.
339 int __sched rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, in rt_mutex_start_proxy_lock() argument
345 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
346 ret = __rt_mutex_start_proxy_lock(lock, waiter, task); in rt_mutex_start_proxy_lock()
348 remove_waiter(lock, waiter); in rt_mutex_start_proxy_lock()
349 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
355 * rt_mutex_wait_proxy_lock() - Wait for lock acquisition
356 * @lock: the rt_mutex we were woken on
359 * @waiter: the pre-initialized rt_mutex_waiter
361 * Wait for the lock acquisition started on our behalf by
366 * 0 - success
367 * <0 - error, one of -EINTR, -ETIMEDOUT
369 * Special API call for PI-futex support
371 int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, in rt_mutex_wait_proxy_lock() argument
377 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
380 ret = rt_mutex_slowlock_block(lock, NULL, TASK_INTERRUPTIBLE, to, waiter); in rt_mutex_wait_proxy_lock()
385 fixup_rt_mutex_waiters(lock, true); in rt_mutex_wait_proxy_lock()
386 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
392 * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition
393 * @lock: the rt_mutex we were woken on
394 * @waiter: the pre-initialized rt_mutex_waiter
399 * Unless we acquired the lock; we're still enqueued on the wait-list and can
405 * true - did the cleanup, we done.
406 * false - we acquired the lock after rt_mutex_wait_proxy_lock() returned,
409 * Special API call for PI-futex support
411 bool __sched rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock, in rt_mutex_cleanup_proxy_lock() argument
416 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()
418 * Do an unconditional try-lock, this deals with the lock stealing in rt_mutex_cleanup_proxy_lock()
419 * state where __rt_mutex_futex_unlock() -> mark_wakeup_next_waiter() in rt_mutex_cleanup_proxy_lock()
424 * we will own the lock and it will have removed the waiter. If we in rt_mutex_cleanup_proxy_lock()
428 try_to_take_rt_mutex(lock, current, waiter); in rt_mutex_cleanup_proxy_lock()
433 if (rt_mutex_owner(lock) != current) { in rt_mutex_cleanup_proxy_lock()
434 remove_waiter(lock, waiter); in rt_mutex_cleanup_proxy_lock()
441 fixup_rt_mutex_waiters(lock, false); in rt_mutex_cleanup_proxy_lock()
443 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()
459 raw_spin_lock_irqsave(&task->pi_lock, flags); in rt_mutex_adjust_pi()
461 waiter = task->pi_blocked_on; in rt_mutex_adjust_pi()
462 if (!waiter || rt_waiter_node_equal(&waiter->tree, task_to_waiter_node(task))) { in rt_mutex_adjust_pi()
463 raw_spin_unlock_irqrestore(&task->pi_lock, flags); in rt_mutex_adjust_pi()
466 next_lock = waiter->lock; in rt_mutex_adjust_pi()
467 raw_spin_unlock_irqrestore(&task->pi_lock, flags); in rt_mutex_adjust_pi()
477 * Performs the wakeup of the top-waiter and re-enables preemption.
487 DEBUG_LOCKS_WARN_ON(!RB_EMPTY_ROOT(&task->pi_waiters.rb_root)); in rt_mutex_debug_task_free()
488 DEBUG_LOCKS_WARN_ON(task->pi_blocked_on); in rt_mutex_debug_task_free()
498 lockdep_init_map_wait(&mutex->dep_map, name, key, 0, LD_WAIT_SLEEP); in __mutex_rt_init()
502 static __always_inline int __mutex_lock_common(struct mutex *lock, in __mutex_lock_common() argument
511 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); in __mutex_lock_common()
512 ret = __rt_mutex_lock(&lock->rtmutex, state); in __mutex_lock_common()
514 mutex_release(&lock->dep_map, ip); in __mutex_lock_common()
516 lock_acquired(&lock->dep_map, ip); in __mutex_lock_common()
521 void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass) in mutex_lock_nested() argument
523 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); in mutex_lock_nested()
527 void __sched _mutex_lock_nest_lock(struct mutex *lock, in _mutex_lock_nest_lock() argument
530 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest_lock, _RET_IP_); in _mutex_lock_nest_lock()
534 int __sched mutex_lock_interruptible_nested(struct mutex *lock, in mutex_lock_interruptible_nested() argument
537 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_); in mutex_lock_interruptible_nested()
541 int __sched mutex_lock_killable_nested(struct mutex *lock, in mutex_lock_killable_nested() argument
544 return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_); in mutex_lock_killable_nested()
548 void __sched mutex_lock_io_nested(struct mutex *lock, unsigned int subclass) in mutex_lock_io_nested() argument
555 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); in mutex_lock_io_nested()
562 void __sched mutex_lock(struct mutex *lock) in mutex_lock() argument
564 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); in mutex_lock()
568 int __sched mutex_lock_interruptible(struct mutex *lock) in mutex_lock_interruptible() argument
570 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_); in mutex_lock_interruptible()
574 int __sched mutex_lock_killable(struct mutex *lock) in mutex_lock_killable() argument
576 return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_); in mutex_lock_killable()
580 void __sched mutex_lock_io(struct mutex *lock) in mutex_lock_io() argument
584 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); in mutex_lock_io()
590 int __sched mutex_trylock(struct mutex *lock) in mutex_trylock() argument
597 ret = __rt_mutex_trylock(&lock->rtmutex); in mutex_trylock()
599 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); in mutex_trylock()
605 void __sched mutex_unlock(struct mutex *lock) in mutex_unlock() argument
607 mutex_release(&lock->dep_map, _RET_IP_); in mutex_unlock()
608 __rt_mutex_unlock(&lock->rtmutex); in mutex_unlock()