203 static int attach_to_pi_state(u32 __user *uaddr, u32 uval,  in attach_to_pi_state()  argument
207 	pid_t pid = uval & FUTEX_TID_MASK;  in attach_to_pi_state()
246 	if (uval != uval2)  in attach_to_pi_state()
252 	if (uval & FUTEX_OWNER_DIED) {  in attach_to_pi_state()
321 static int handle_exit_race(u32 __user *uaddr, u32 uval,  in handle_exit_race()  argument
366 	if (uval2 != uval)  in handle_exit_race()
411 static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,  in attach_to_pi_owner()  argument
415 	pid_t pid = uval & FUTEX_TID_MASK;  in attach_to_pi_owner()
429 		return handle_exit_race(uaddr, uval, NULL);  in attach_to_pi_owner()
448 		int ret = handle_exit_race(uaddr, uval, p);  in attach_to_pi_owner()
475 static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)  in lock_pi_update_atomic()  argument
483 	err = futex_cmpxchg_value_locked(&curval, uaddr, uval, newval);  in lock_pi_update_atomic()
488 	return curval != uval ? -EAGAIN : 0;  in lock_pi_update_atomic()
522 	u32 uval, newval, vpid = task_pid_vnr(task);  in futex_lock_pi_atomic()  local
530 	if (futex_get_value_locked(&uval, uaddr))  in futex_lock_pi_atomic()
539 	if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))  in futex_lock_pi_atomic()
551 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
559 	if (!(uval & FUTEX_TID_MASK)) {  in futex_lock_pi_atomic()
564 		newval = uval & FUTEX_OWNER_DIED;  in futex_lock_pi_atomic()
571 		ret = lock_pi_update_atomic(uaddr, uval, newval);  in futex_lock_pi_atomic()
599 	newval = uval | FUTEX_WAITERS;  in futex_lock_pi_atomic()
600 	ret = lock_pi_update_atomic(uaddr, uval, newval);  in futex_lock_pi_atomic()
614 static int wake_futex_pi(u32 __user *uaddr, u32 uval,  in wake_futex_pi()  argument
638 	ret = futex_cmpxchg_value_locked(&curval, uaddr, uval, newval);  in wake_futex_pi()
639 	if (!ret && (curval != uval)) {  in wake_futex_pi()
646 		if ((FUTEX_TID_MASK & curval) == uval)  in wake_futex_pi()
676 	u32 uval, curval, newval, newtid;  in __fixup_pi_state_owner()  local
753 	err = futex_get_value_locked(&uval, uaddr);  in __fixup_pi_state_owner()
758 		newval = (uval & FUTEX_OWNER_DIED) | newtid;  in __fixup_pi_state_owner()
760 		err = futex_cmpxchg_value_locked(&curval, uaddr, uval, newval);  in __fixup_pi_state_owner()
764 		if (curval == uval)  in __fixup_pi_state_owner()
766 		uval = curval;  in __fixup_pi_state_owner()
1114 	u32 curval, uval, vpid = task_pid_vnr(current);  in futex_unlock_pi()  local
1124 	if (get_user(uval, uaddr))  in futex_unlock_pi()
1129 	if ((uval & FUTEX_TID_MASK) != vpid)  in futex_unlock_pi()
1198 		ret = wake_futex_pi(uaddr, uval, pi_state, rt_waiter);  in futex_unlock_pi()
1233 	if ((ret = futex_cmpxchg_value_locked(&curval, uaddr, uval, 0))) {  in futex_unlock_pi()
1251 	ret = (curval == uval) ? 0 : -EAGAIN;  in futex_unlock_pi()