Lines Matching +full:fault +full:- +full:q
1 // SPDX-License-Identifier: GPL-2.0-or-later
17 if (likely(current->pi_state_cache)) in refill_pi_state_cache()
23 return -ENOMEM; in refill_pi_state_cache()
25 INIT_LIST_HEAD(&pi_state->list); in refill_pi_state_cache()
27 pi_state->owner = NULL; in refill_pi_state_cache()
28 refcount_set(&pi_state->refcount, 1); in refill_pi_state_cache()
29 pi_state->key = FUTEX_KEY_INIT; in refill_pi_state_cache()
31 current->pi_state_cache = pi_state; in refill_pi_state_cache()
38 struct futex_pi_state *pi_state = current->pi_state_cache; in alloc_pi_state()
41 current->pi_state_cache = NULL; in alloc_pi_state()
49 struct task_struct *old_owner = pi_state->owner; in pi_state_update_owner()
51 lockdep_assert_held(&pi_state->pi_mutex.wait_lock); in pi_state_update_owner()
54 raw_spin_lock(&old_owner->pi_lock); in pi_state_update_owner()
55 WARN_ON(list_empty(&pi_state->list)); in pi_state_update_owner()
56 list_del_init(&pi_state->list); in pi_state_update_owner()
57 raw_spin_unlock(&old_owner->pi_lock); in pi_state_update_owner()
61 raw_spin_lock(&new_owner->pi_lock); in pi_state_update_owner()
62 WARN_ON(!list_empty(&pi_state->list)); in pi_state_update_owner()
63 list_add(&pi_state->list, &new_owner->pi_state_list); in pi_state_update_owner()
64 pi_state->owner = new_owner; in pi_state_update_owner()
65 raw_spin_unlock(&new_owner->pi_lock); in pi_state_update_owner()
71 WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount)); in get_pi_state()
83 if (!refcount_dec_and_test(&pi_state->refcount)) in put_pi_state()
87 * If pi_state->owner is NULL, the owner is most probably dying in put_pi_state()
90 if (pi_state->owner) { in put_pi_state()
93 raw_spin_lock_irqsave(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
95 rt_mutex_proxy_unlock(&pi_state->pi_mutex); in put_pi_state()
96 raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); in put_pi_state()
99 if (current->pi_state_cache) { in put_pi_state()
103 * pi_state->list is already empty. in put_pi_state()
104 * clear pi_state->owner. in put_pi_state()
105 * refcount is at 0 - put it back to 1. in put_pi_state()
107 pi_state->owner = NULL; in put_pi_state()
108 refcount_set(&pi_state->refcount, 1); in put_pi_state()
109 current->pi_state_cache = pi_state; in put_pi_state()
116 * Waiter | pi_state | pi->owner | uTID | uODIED | ?
118 * [1] NULL | --- | --- | 0 | 0/1 | Valid
119 * [2] NULL | --- | --- | >0 | 0/1 | Valid
121 * [3] Found | NULL | -- | Any | 0/1 | Invalid
151 * [7] pi_state->owner can only be NULL when the OWNER_DIED bit is set.
166 * hb->lock:
168 * hb -> futex_q, relation
169 * futex_q -> pi_state, relation
174 * pi_mutex->wait_lock:
180 * p->pi_lock:
182 * p->pi_state_list -> pi_state->list, relation
183 * pi_mutex->owner -> pi_state->owner, relation
185 * pi_state->refcount:
192 * hb->lock
193 * pi_mutex->wait_lock
194 * p->pi_lock
212 * Userspace might have messed up non-PI and PI futexes [3] in attach_to_pi_state()
215 return -EINVAL; in attach_to_pi_state()
218 * We get here with hb->lock held, and having found a in attach_to_pi_state()
220 * has dropped the hb->lock in between futex_queue() and futex_unqueue_pi(), in attach_to_pi_state()
229 WARN_ON(!refcount_read(&pi_state->refcount)); in attach_to_pi_state()
235 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
256 * pi_state->rt_mutex will fixup owner. in attach_to_pi_state()
258 if (!pi_state->owner) { in attach_to_pi_state()
286 if (!pi_state->owner) in attach_to_pi_state()
295 if (pid != task_pid_vnr(pi_state->owner)) in attach_to_pi_state()
300 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
305 ret = -EINVAL; in attach_to_pi_state()
309 ret = -EAGAIN; in attach_to_pi_state()
313 ret = -EFAULT; in attach_to_pi_state()
317 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in attach_to_pi_state()
330 if (tsk && tsk->futex_state != FUTEX_STATE_DEAD) in handle_exit_race()
331 return -EBUSY; in handle_exit_race()
342 * tsk->flags |= PF_EXITING; *uaddr == 0x00000PID in handle_exit_race()
347 * } if (!tsk->flags & PF_EXITING) { in handle_exit_race()
349 * tsk->futex_state = } else { in handle_exit_race()
350 * FUTEX_STATE_DEAD; if (tsk->futex_state != in handle_exit_race()
352 * return -EAGAIN; in handle_exit_race()
353 * return -ESRCH; <--- FAIL in handle_exit_race()
363 return -EFAULT; in handle_exit_race()
367 return -EAGAIN; in handle_exit_race()
374 return -ESRCH; in handle_exit_race()
383 * This creates pi_state, we have hb->lock held, this means nothing can in __attach_to_pi_owner()
392 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); in __attach_to_pi_owner()
395 pi_state->key = *key; in __attach_to_pi_owner()
397 WARN_ON(!list_empty(&pi_state->list)); in __attach_to_pi_owner()
398 list_add(&pi_state->list, &p->pi_state_list); in __attach_to_pi_owner()
400 * Assignment without holding pi_state->pi_mutex.wait_lock is safe in __attach_to_pi_owner()
403 pi_state->owner = p; in __attach_to_pi_owner()
419 * We are the first waiter - try to look up the real owner and attach in attach_to_pi_owner()
426 return -EAGAIN; in attach_to_pi_owner()
431 if (unlikely(p->flags & PF_KTHREAD)) { in attach_to_pi_owner()
433 return -EPERM; in attach_to_pi_owner()
439 * in futex_exit_release(), we do this protected by p->pi_lock: in attach_to_pi_owner()
441 raw_spin_lock_irq(&p->pi_lock); in attach_to_pi_owner()
442 if (unlikely(p->futex_state != FUTEX_STATE_OK)) { in attach_to_pi_owner()
450 raw_spin_unlock_irq(&p->pi_lock); in attach_to_pi_owner()
460 if (ret == -EBUSY) in attach_to_pi_owner()
468 raw_spin_unlock_irq(&p->pi_lock); in attach_to_pi_owner()
481 return -EFAULT; in lock_pi_update_atomic()
488 return curval != uval ? -EAGAIN : 0; in lock_pi_update_atomic()
492 * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
505 * - 0 - ready to wait;
506 * - 1 - acquired the lock;
507 * - <0 - error
509 * The hb->lock must be held by the caller.
511 * @exiting is only set when the return value is -EBUSY. If so, this holds
531 return -EFAULT; in futex_lock_pi_atomic()
534 return -EFAULT; in futex_lock_pi_atomic()
540 return -EDEADLK; in futex_lock_pi_atomic()
543 return -EDEADLK; in futex_lock_pi_atomic()
551 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); in futex_lock_pi_atomic()
587 raw_spin_lock_irq(&task->pi_lock); in futex_lock_pi_atomic()
589 raw_spin_unlock_irq(&task->pi_lock); in futex_lock_pi_atomic()
597 * the kernel and blocked on hb->lock. in futex_lock_pi_atomic()
624 new_owner = top_waiter->task; in wake_futex_pi()
634 ret = -EFAULT; in wake_futex_pi()
642 * try the TID->0 transition) raced with a waiter setting the in wake_futex_pi()
647 ret = -EAGAIN; in wake_futex_pi()
649 ret = -EINVAL; in wake_futex_pi()
659 postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wqh); in wake_futex_pi()
663 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in wake_futex_pi()
671 static int __fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, in __fixup_pi_state_owner() argument
674 struct futex_pi_state *pi_state = q->pi_state; in __fixup_pi_state_owner()
679 oldowner = pi_state->owner; in __fixup_pi_state_owner()
684 * - we stole the lock and pi_state->owner needs updating to reflect in __fixup_pi_state_owner()
689 * - someone stole our lock and we need to fix things to point to the in __fixup_pi_state_owner()
696 * because we can fault here. Imagine swapped out pages or a fork in __fixup_pi_state_owner()
700 * pi_state in an inconsistent state when we fault here, because we in __fixup_pi_state_owner()
701 * need to drop the locks to handle the fault. This might be observed in __fixup_pi_state_owner()
714 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) { in __fixup_pi_state_owner()
723 newowner = rt_mutex_owner(&pi_state->pi_mutex); in __fixup_pi_state_owner()
733 err = -EAGAIN; in __fixup_pi_state_owner()
750 if (!pi_state->owner) in __fixup_pi_state_owner()
778 * In order to reschedule or handle a page fault, we need to drop the in __fixup_pi_state_owner()
779 * locks here. In the case of a fault, this gives the other task in __fixup_pi_state_owner()
782 * are back from handling the fault we need to check the pi_state after in __fixup_pi_state_owner()
786 * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely in __fixup_pi_state_owner()
787 * drop hb->lock since the caller owns the hb -> futex_q relation. in __fixup_pi_state_owner()
788 * Dropping the pi_mutex->wait_lock requires the state revalidate. in __fixup_pi_state_owner()
791 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
792 spin_unlock(q->lock_ptr); in __fixup_pi_state_owner()
795 case -EFAULT: in __fixup_pi_state_owner()
799 case -EAGAIN: in __fixup_pi_state_owner()
809 spin_lock(q->lock_ptr); in __fixup_pi_state_owner()
810 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in __fixup_pi_state_owner()
815 if (pi_state->owner != oldowner) in __fixup_pi_state_owner()
818 /* Retry if err was -EAGAIN or the fault in succeeded */ in __fixup_pi_state_owner()
835 * The rtmutex has an owner - either current or some other in __fixup_pi_state_owner()
838 pi_state_update_owner(pi_state, rt_mutex_owner(&pi_state->pi_mutex)); in __fixup_pi_state_owner()
843 static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, in fixup_pi_state_owner() argument
846 struct futex_pi_state *pi_state = q->pi_state; in fixup_pi_state_owner()
849 lockdep_assert_held(q->lock_ptr); in fixup_pi_state_owner()
851 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
852 ret = __fixup_pi_state_owner(uaddr, q, argowner); in fixup_pi_state_owner()
853 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in fixup_pi_state_owner()
858 * fixup_pi_owner() - Post lock pi_state and corner case management
860 * @q: futex_q (contains pi_state and access to the rt_mutex)
868 * - 1 - success, lock taken;
869 * - 0 - success, lock not taken;
870 * - <0 - on error (-EFAULT)
872 int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked) in fixup_pi_owner() argument
877 * did a lock-steal - fix up the PI-state in that case: in fixup_pi_owner()
879 * Speculative pi_state->owner read (we don't hold wait_lock); in fixup_pi_owner()
880 * since we own the lock pi_state->owner == current is the in fixup_pi_owner()
883 if (q->pi_state->owner != current) in fixup_pi_owner()
884 return fixup_pi_state_owner(uaddr, q, current); in fixup_pi_owner()
893 * Another speculative read; pi_state->owner == current is unstable in fixup_pi_owner()
896 if (q->pi_state->owner == current) in fixup_pi_owner()
897 return fixup_pi_state_owner(uaddr, q, NULL); in fixup_pi_owner()
903 if (WARN_ON_ONCE(rt_mutex_owner(&q->pi_state->pi_mutex) == current)) in fixup_pi_owner()
904 return fixup_pi_state_owner(uaddr, q, current); in fixup_pi_owner()
910 * Userspace tried a 0 -> TID atomic transition of the futex value
913 * on rt-mutexes, it does PI, etc. (Due to races the kernel might see
924 struct futex_q q = futex_q_init; in futex_lock_pi() local
928 return -ENOSYS; in futex_lock_pi()
931 return -ENOMEM; in futex_lock_pi()
936 ret = get_futex_key(uaddr, flags, &q.key, FUTEX_WRITE); in futex_lock_pi()
941 hb = futex_q_lock(&q); in futex_lock_pi()
943 ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, in futex_lock_pi()
955 case -EFAULT: in futex_lock_pi()
957 case -EBUSY: in futex_lock_pi()
958 case -EAGAIN: in futex_lock_pi()
961 * - EBUSY: Task is exiting and we just wait for the in futex_lock_pi()
963 * - EAGAIN: The user space value changed. in futex_lock_pi()
979 WARN_ON(!q.pi_state); in futex_lock_pi()
984 __futex_queue(&q, hb); in futex_lock_pi()
987 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex); in futex_lock_pi()
989 ret = ret ? 0 : -EWOULDBLOCK; in futex_lock_pi()
1002 * On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not in futex_lock_pi()
1004 * include hb->lock in the blocking chain, even through we'll not in in futex_lock_pi()
1005 * fact hold it while blocking. This will lead it to report -EDEADLK in futex_lock_pi()
1008 * Therefore acquire wait_lock while holding hb->lock, but drop the in futex_lock_pi()
1010 * interleaves with futex_unlock_pi() -- which does a similar lock in futex_lock_pi()
1011 * handoff -- such that the latter can observe the futex_q::pi_state in futex_lock_pi()
1014 raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
1015 spin_unlock(q.lock_ptr); in futex_lock_pi()
1021 ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current); in futex_lock_pi()
1022 raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock); in futex_lock_pi()
1033 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter); in futex_lock_pi()
1038 * must unwind the above, however we canont lock hb->lock because in futex_lock_pi()
1039 * rt_mutex already has a waiter enqueued and hb->lock can itself try in futex_lock_pi()
1042 * Doing the cleanup without holding hb->lock can cause inconsistent in futex_lock_pi()
1054 if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter)) in futex_lock_pi()
1062 spin_lock(q.lock_ptr); in futex_lock_pi()
1072 res = fixup_pi_owner(uaddr, &q, !ret); in futex_lock_pi()
1075 * the lock, clear our -ETIMEDOUT or -EINTR. in futex_lock_pi()
1080 futex_unqueue_pi(&q); in futex_lock_pi()
1081 spin_unlock(q.lock_ptr); in futex_lock_pi()
1089 hrtimer_cancel(&to->timer); in futex_lock_pi()
1090 destroy_hrtimer_on_stack(&to->timer); in futex_lock_pi()
1092 return ret != -EINTR ? ret : -ERESTARTNOINTR; in futex_lock_pi()
1108 * Userspace attempted a TID -> 0 atomic transition, and failed.
1109 * This is the in-kernel slowpath: we look up the PI state (if any),
1110 * and do the rt-mutex unlock.
1121 return -ENOSYS; in futex_unlock_pi()
1125 return -EFAULT; in futex_unlock_pi()
1130 return -EPERM; in futex_unlock_pi()
1137 spin_lock(&hb->lock); in futex_unlock_pi()
1147 struct futex_pi_state *pi_state = top_waiter->pi_state; in futex_unlock_pi()
1150 ret = -EINVAL; in futex_unlock_pi()
1158 if (pi_state->owner != current) in futex_unlock_pi()
1162 * By taking wait_lock while still holding hb->lock, we ensure in futex_unlock_pi()
1167 * rt_waiter without holding hb->lock, it is possible for in futex_unlock_pi()
1176 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); in futex_unlock_pi()
1179 * Futex vs rt_mutex waiter state -- if there are no rt_mutex in futex_unlock_pi()
1182 * new futex_lock_pi() is not using this stale PI-state while in futex_unlock_pi()
1187 rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex); in futex_unlock_pi()
1190 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); in futex_unlock_pi()
1195 spin_unlock(&hb->lock); in futex_unlock_pi()
1197 /* drops pi_state->pi_mutex.wait_lock */ in futex_unlock_pi()
1209 * pagefault, so retry the user-access and the wakeup: in futex_unlock_pi()
1211 if (ret == -EFAULT) in futex_unlock_pi()
1217 if (ret == -EAGAIN) in futex_unlock_pi()
1229 * on hb->lock. So we can safely ignore them. We do neither in futex_unlock_pi()
1234 spin_unlock(&hb->lock); in futex_unlock_pi()
1236 case -EFAULT: in futex_unlock_pi()
1239 case -EAGAIN: in futex_unlock_pi()
1251 ret = (curval == uval) ? 0 : -EAGAIN; in futex_unlock_pi()
1254 spin_unlock(&hb->lock); in futex_unlock_pi()