59 	struct kcm_mux *mux = psock->mux;  in kcm_abort_tx_psock()  local
63 	spin_lock_bh(&mux->lock);  in kcm_abort_tx_psock()
66 		spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
87 	spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
93 /* RX mux lock held. */
94 static void kcm_update_rx_mux_stats(struct kcm_mux *mux,  in kcm_update_rx_mux_stats()  argument
97 	STRP_STATS_ADD(mux->stats.rx_bytes,  in kcm_update_rx_mux_stats()
100 	mux->stats.rx_msgs +=  in kcm_update_rx_mux_stats()
106 static void kcm_update_tx_mux_stats(struct kcm_mux *mux,  in kcm_update_tx_mux_stats()  argument
109 	KCM_STATS_ADD(mux->stats.tx_bytes,  in kcm_update_tx_mux_stats()
111 	mux->stats.tx_msgs +=  in kcm_update_tx_mux_stats()
121  * pending ready messages on a psock. RX mux lock held.
125 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()  local
132 	while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {  in kcm_rcv_ready()
135 			skb_queue_head(&mux->rx_hold_queue, skb);  in kcm_rcv_ready()
141 	while (!list_empty(&mux->psocks_ready)) {  in kcm_rcv_ready()
142 		psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,  in kcm_rcv_ready()
165 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
174 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()  local
185 		spin_lock_bh(&mux->rx_lock);  in kcm_rfree()
187 		spin_unlock_bh(&mux->rx_lock);  in kcm_rfree()
219  * RX mux lock held.
221 static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)  in requeue_rx_msgs()  argument
231 		if (list_empty(&mux->kcm_rx_waiters)) {  in requeue_rx_msgs()
232 			skb_queue_tail(&mux->rx_hold_queue, skb);  in requeue_rx_msgs()
236 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
257 	struct kcm_mux *mux = psock->mux;  in reserve_rx_kcm()  local
265 	spin_lock_bh(&mux->rx_lock);  in reserve_rx_kcm()
268 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
272 	kcm_update_rx_mux_stats(mux, psock);  in reserve_rx_kcm()
274 	if (list_empty(&mux->kcm_rx_waiters)) {  in reserve_rx_kcm()
278 			      &mux->psocks_ready);  in reserve_rx_kcm()
279 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
283 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
293 	spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
310 	struct kcm_mux *mux = psock->mux;  in unreserve_rx_kcm()  local
315 	spin_lock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
327 		spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
338 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
345 	spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
418 	struct kcm_mux *mux;  in psock_write_space()  local
426 	mux = psock->mux;  in psock_write_space()
428 	spin_lock_bh(&mux->lock);  in psock_write_space()
435 	spin_unlock_bh(&mux->lock);  in psock_write_space()
445 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()  local
460 	spin_lock_bh(&mux->lock);  in reserve_psock()
468 		spin_unlock_bh(&mux->lock);  in reserve_psock()
472 	if (!list_empty(&mux->psocks_avail)) {  in reserve_psock()
473 		psock = list_first_entry(&mux->psocks_avail,  in reserve_psock()
486 			      &mux->kcm_tx_waiters);  in reserve_psock()
490 	spin_unlock_bh(&mux->lock);  in reserve_psock()
495 /* mux lock held */
498 	struct kcm_mux *mux = psock->mux;  in psock_now_avail()  local
501 	if (list_empty(&mux->kcm_tx_waiters)) {  in psock_now_avail()
503 			      &mux->psocks_avail);  in psock_now_avail()
505 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
527 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()  local
529 	spin_lock_bh(&mux->lock);  in unreserve_psock()
534 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
540 	kcm_update_tx_mux_stats(mux, psock);  in unreserve_psock()
552 			mux->psocks_cnt--;  in unreserve_psock()
560 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
567 	spin_unlock_bh(&mux->lock);  in unreserve_psock()
572 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()  local
574 	spin_lock_bh(&mux->lock);  in kcm_report_tx_retry()
575 	KCM_STATS_INCR(mux->stats.tx_retries);  in kcm_report_tx_retry()
576 	spin_unlock_bh(&mux->lock);  in kcm_report_tx_retry()
1074 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()  local
1079 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_disable()
1091 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1094 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_disable()
1100 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()  local
1105 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_enable()
1110 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_enable()
1179 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1191 	/* Add to mux's kcm sockets list */  in init_kcm_sock()
1192 	kcm->mux = mux;  in init_kcm_sock()
1193 	spin_lock_bh(&mux->lock);  in init_kcm_sock()
1195 	head = &mux->kcm_socks;  in init_kcm_sock()
1196 	list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {  in init_kcm_sock()
1206 	mux->kcm_socks_cnt++;  in init_kcm_sock()
1207 	spin_unlock_bh(&mux->lock);  in init_kcm_sock()
1212 	spin_lock_bh(&mux->rx_lock);  in init_kcm_sock()
1214 	spin_unlock_bh(&mux->rx_lock);  in init_kcm_sock()
1221 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()  local
1258 	psock->mux = mux;  in kcm_attach()
1293 	/* Finished initialization, now add the psock to the MUX. */  in kcm_attach()
1294 	spin_lock_bh(&mux->lock);  in kcm_attach()
1295 	head = &mux->psocks;  in kcm_attach()
1296 	list_for_each_entry(tpsock, &mux->psocks, psock_list) {  in kcm_attach()
1306 	KCM_STATS_INCR(mux->stats.psock_attach);  in kcm_attach()
1307 	mux->psocks_cnt++;  in kcm_attach()
1309 	spin_unlock_bh(&mux->lock);  in kcm_attach()
1353 	struct kcm_mux *mux = psock->mux;  in kcm_unattach()  local
1373 	spin_lock_bh(&mux->rx_lock);  in kcm_unattach()
1382 		KCM_STATS_INCR(mux->stats.rx_ready_drops);  in kcm_unattach()
1385 	spin_unlock_bh(&mux->rx_lock);  in kcm_unattach()
1396 	spin_lock_bh(&mux->lock);  in kcm_unattach()
1398 	aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);  in kcm_unattach()
1399 	save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);  in kcm_unattach()
1401 	KCM_STATS_INCR(mux->stats.psock_unattach);  in kcm_unattach()
1408 		KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);  in kcm_unattach()
1409 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1413 		 * to do this without the mux lock.  in kcm_unattach()
1417 		spin_lock_bh(&mux->lock);  in kcm_unattach()
1419 			/* psock now unreserved in window mux was unlocked */  in kcm_unattach()
1429 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1435 		mux->psocks_cnt--;  in kcm_unattach()
1436 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1449 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()  local
1467 	spin_lock_bh(&mux->lock);  in kcm_unattach_ioctl()
1469 	list_for_each_entry(psock, &mux->psocks, psock_list) {  in kcm_unattach_ioctl()
1482 		spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1491 	spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1526 	init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);  in kcm_clone()
1589 	struct kcm_mux *mux = container_of(rcu,  in free_mux()  local
1592 	kmem_cache_free(kcm_muxp, mux);  in free_mux()
1595 static void release_mux(struct kcm_mux *mux)  in release_mux()  argument
1597 	struct kcm_net *knet = mux->knet;  in release_mux()
1602 				 &mux->psocks, psock_list) {  in release_mux()
1607 	if (WARN_ON(mux->psocks_cnt))  in release_mux()
1610 	__skb_queue_purge(&mux->rx_hold_queue);  in release_mux()
1613 	aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);  in release_mux()
1614 	aggregate_psock_stats(&mux->aggregate_psock_stats,  in release_mux()
1616 	aggregate_strp_stats(&mux->aggregate_strp_stats,  in release_mux()
1618 	list_del_rcu(&mux->kcm_mux_list);  in release_mux()
1622 	call_rcu(&mux->rcu, free_mux);  in release_mux()
1627 	struct kcm_mux *mux = kcm->mux;  in kcm_done()  local
1631 	spin_lock_bh(&mux->rx_lock);  in kcm_done()
1637 		spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1647 	requeue_rx_msgs(mux, &sk->sk_receive_queue);  in kcm_done()
1649 	spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1654 	/* Detach from MUX */  in kcm_done()
1655 	spin_lock_bh(&mux->lock);  in kcm_done()
1658 	mux->kcm_socks_cnt--;  in kcm_done()
1659 	socks_cnt = mux->kcm_socks_cnt;  in kcm_done()
1661 	spin_unlock_bh(&mux->lock);  in kcm_done()
1664 		/* We are done with the mux now. */  in kcm_done()
1665 		release_mux(mux);  in kcm_done()
1674  * If this is the last KCM socket on the MUX, destroy the MUX.
1680 	struct kcm_mux *mux;  in kcm_release()  local
1687 	mux = kcm->mux;  in kcm_release()
1707 	spin_lock_bh(&mux->lock);  in kcm_release()
1715 	spin_unlock_bh(&mux->lock);  in kcm_release()
1793 	struct kcm_mux *mux;  in kcm_create()  local
1813 	/* Allocate a kcm mux, shared between KCM sockets */  in kcm_create()
1814 	mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);  in kcm_create()
1815 	if (!mux) {  in kcm_create()
1820 	spin_lock_init(&mux->lock);  in kcm_create()
1821 	spin_lock_init(&mux->rx_lock);  in kcm_create()
1822 	INIT_LIST_HEAD(&mux->kcm_socks);  in kcm_create()
1823 	INIT_LIST_HEAD(&mux->kcm_rx_waiters);  in kcm_create()
1824 	INIT_LIST_HEAD(&mux->kcm_tx_waiters);  in kcm_create()
1826 	INIT_LIST_HEAD(&mux->psocks);  in kcm_create()
1827 	INIT_LIST_HEAD(&mux->psocks_ready);  in kcm_create()
1828 	INIT_LIST_HEAD(&mux->psocks_avail);  in kcm_create()
1830 	mux->knet = knet;  in kcm_create()
1832 	/* Add new MUX to list */  in kcm_create()
1834 	list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);  in kcm_create()
1838 	skb_queue_head_init(&mux->rx_hold_queue);  in kcm_create()
1842 	init_kcm_sock(kcm_sk(sk), mux);  in kcm_create()