158 	struct svc_sock *svsk =  in svc_set_cmsg_data()  local
160 	switch (svsk->sk_sk->sk_family) {  in svc_set_cmsg_data()
196 static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining)  in svc_one_sock_name()  argument
198 	const struct sock *sk = svsk->sk_sk;  in svc_one_sock_name()
260 svc_tcp_sock_recv_cmsg(struct svc_sock *svsk, struct msghdr *msg)  in svc_tcp_sock_recv_cmsg()  argument
266 	struct socket *sock = svsk->sk_sock;  in svc_tcp_sock_recv_cmsg()
304 	struct svc_sock *svsk =  in svc_tcp_read_msg()  local
312 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_read_msg()
324 	len = svc_tcp_sock_recv_cmsg(svsk, &msg);  in svc_tcp_read_msg()
332 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_read_msg()
340 static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs)  in svc_sock_setbufsize()  argument
342 	unsigned int max_mesg = svsk->sk_xprt.xpt_server->sv_max_mesg;  in svc_sock_setbufsize()
343 	struct socket *sock = svsk->sk_sock;  in svc_sock_setbufsize()
367 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_data_ready()  local
371 	if (svsk) {  in svc_data_ready()
374 		svsk->sk_odata(sk);  in svc_data_ready()
375 		trace_svcsock_data_ready(&svsk->sk_xprt, 0);  in svc_data_ready()
376 		if (test_bit(XPT_HANDSHAKE, &svsk->sk_xprt.xpt_flags))  in svc_data_ready()
378 		if (!test_and_set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags))  in svc_data_ready()
379 			svc_xprt_enqueue(&svsk->sk_xprt);  in svc_data_ready()
388 	struct svc_sock	*svsk = (struct svc_sock *)(sk->sk_user_data);  in svc_write_space()  local
390 	if (svsk) {  in svc_write_space()
393 		trace_svcsock_write_space(&svsk->sk_xprt, 0);  in svc_write_space()
394 		svsk->sk_owspace(sk);  in svc_write_space()
395 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_write_space()
401 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_has_wspace()  local
405 	return !test_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_tcp_has_wspace()
410 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_kill_temp_xprt()  local
412 	sock_no_linger(svsk->sk_sock->sk);  in svc_tcp_kill_temp_xprt()
429 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_handshake_done()  local
437 	complete_all(&svsk->sk_handshake_done);  in svc_tcp_handshake_done()
447 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_handshake()  local
448 	struct sock *sk = svsk->sk_sock->sk;  in svc_tcp_handshake()
450 		.ta_sock	= svsk->sk_sock,  in svc_tcp_handshake()
459 	init_completion(&svsk->sk_handshake_done);  in svc_tcp_handshake()
467 	ret = wait_for_completion_interruptible_timeout(&svsk->sk_handshake_done,  in svc_tcp_handshake()
564 	struct svc_sock	*svsk =  in svc_udp_recvfrom()  local
566 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_udp_recvfrom()
582 	if (test_and_clear_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags))  in svc_udp_recvfrom()
591 	    svc_sock_setbufsize(svsk, serv->sv_nrthreads + 3);  in svc_udp_recvfrom()
593 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_recvfrom()
594 	err = kernel_recvmsg(svsk->sk_sock, &msg, NULL,  in svc_udp_recvfrom()
598 	skb = skb_recv_udp(svsk->sk_sk, MSG_DONTWAIT, &err);  in svc_udp_recvfrom()
609 	sock_write_timestamp(svsk->sk_sk, skb->tstamp);  in svc_udp_recvfrom()
610 	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); /* there may be more data... */  in svc_udp_recvfrom()
614 	trace_svcsock_udp_recv(&svsk->sk_xprt, len);  in svc_udp_recvfrom()
660 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_recvfrom()
662 	trace_svcsock_udp_recv_err(&svsk->sk_xprt, err);  in svc_udp_recvfrom()
689 	struct svc_sock	*svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_udp_sendto()  local
721 	err = sock_sendmsg(svsk->sk_sock, &msg);  in svc_udp_sendto()
726 		err = sock_sendmsg(svsk->sk_sock, &msg);  in svc_udp_sendto()
741 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_udp_has_wspace()  local
749 	set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_udp_has_wspace()
750 	required = atomic_read(&svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg;  in svc_udp_has_wspace()
751 	if (required*2 > sock_wspace(svsk->sk_sk))  in svc_udp_has_wspace()
753 	clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_udp_has_wspace()
796 static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv)  in svc_udp_init()  argument
798 	svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_udp_class,  in svc_udp_init()
799 		      &svsk->sk_xprt, serv);  in svc_udp_init()
800 	clear_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
801 	svsk->sk_sk->sk_data_ready = svc_data_ready;  in svc_udp_init()
802 	svsk->sk_sk->sk_write_space = svc_write_space;  in svc_udp_init()
808 	svc_sock_setbufsize(svsk, 3);  in svc_udp_init()
811 	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
812 	set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
815 	switch (svsk->sk_sk->sk_family) {  in svc_udp_init()
817 		ip_sock_set_pktinfo(svsk->sk_sock->sk);  in svc_udp_init()
820 		ip6_sock_set_recvpktinfo(svsk->sk_sock->sk);  in svc_udp_init()
833 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_listen_data_ready()  local
851 	if (svsk) {  in svc_tcp_listen_data_ready()
854 		svsk->sk_odata(sk);  in svc_tcp_listen_data_ready()
855 		set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_listen_data_ready()
856 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_tcp_listen_data_ready()
865 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_state_change()  local
867 	if (svsk) {  in svc_tcp_state_change()
870 		svsk->sk_ostate(sk);  in svc_tcp_state_change()
871 		trace_svcsock_tcp_state(&svsk->sk_xprt, svsk->sk_sock);  in svc_tcp_state_change()
873 			svc_xprt_deferred_close(&svsk->sk_xprt);  in svc_tcp_state_change()
882 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_accept()  local
885 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_accept()
886 	struct socket	*sock = svsk->sk_sock;  in svc_tcp_accept()
894 	clear_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_accept()
904 	set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_accept()
914 	newsock->sk->sk_state_change = svsk->sk_ostate;  in svc_tcp_accept()
915 	newsock->sk->sk_data_ready = svsk->sk_odata;  in svc_tcp_accept()
916 	newsock->sk->sk_write_space = svsk->sk_owspace;  in svc_tcp_accept()
948 static size_t svc_tcp_restore_pages(struct svc_sock *svsk,  in svc_tcp_restore_pages()  argument
951 	size_t len = svsk->sk_datalen;  in svc_tcp_restore_pages()
960 		BUG_ON(svsk->sk_pages[i] == NULL);  in svc_tcp_restore_pages()
961 		rqstp->rq_pages[i] = svsk->sk_pages[i];  in svc_tcp_restore_pages()
962 		svsk->sk_pages[i] = NULL;  in svc_tcp_restore_pages()
968 static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)  in svc_tcp_save_pages()  argument
972 	if (svsk->sk_datalen == 0)  in svc_tcp_save_pages()
974 	len = svsk->sk_datalen;  in svc_tcp_save_pages()
977 		svsk->sk_pages[i] = rqstp->rq_pages[i];  in svc_tcp_save_pages()
982 static void svc_tcp_clear_pages(struct svc_sock *svsk)  in svc_tcp_clear_pages()  argument
986 	if (svsk->sk_datalen == 0)  in svc_tcp_clear_pages()
988 	len = svsk->sk_datalen;  in svc_tcp_clear_pages()
991 		if (svsk->sk_pages[i] == NULL) {  in svc_tcp_clear_pages()
995 		put_page(svsk->sk_pages[i]);  in svc_tcp_clear_pages()
996 		svsk->sk_pages[i] = NULL;  in svc_tcp_clear_pages()
999 	svsk->sk_tcplen = 0;  in svc_tcp_clear_pages()
1000 	svsk->sk_datalen = 0;  in svc_tcp_clear_pages()
1006 static ssize_t svc_tcp_read_marker(struct svc_sock *svsk,  in svc_tcp_read_marker()  argument
1014 	if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) {  in svc_tcp_read_marker()
1018 		want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;  in svc_tcp_read_marker()
1019 		iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen;  in svc_tcp_read_marker()
1022 		len = svc_tcp_sock_recv_cmsg(svsk, &msg);  in svc_tcp_read_marker()
1025 		svsk->sk_tcplen += len;  in svc_tcp_read_marker()
1030 		trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker);  in svc_tcp_read_marker()
1031 		if (svc_sock_reclen(svsk) + svsk->sk_datalen >  in svc_tcp_read_marker()
1032 		    svsk->sk_xprt.xpt_server->sv_max_mesg)  in svc_tcp_read_marker()
1035 	return svc_sock_reclen(svsk);  in svc_tcp_read_marker()
1039 			       __func__, svsk->sk_xprt.xpt_server->sv_name,  in svc_tcp_read_marker()
1040 			       svc_sock_reclen(svsk));  in svc_tcp_read_marker()
1041 	svc_xprt_deferred_close(&svsk->sk_xprt);  in svc_tcp_read_marker()
1046 static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)  in receive_cb_reply()  argument
1048 	struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt;  in receive_cb_reply()
1081 static void svc_tcp_fragment_received(struct svc_sock *svsk)  in svc_tcp_fragment_received()  argument
1084 	svsk->sk_tcplen = 0;  in svc_tcp_fragment_received()
1085 	svsk->sk_marker = xdr_zero;  in svc_tcp_fragment_received()
1088 	tcp_set_rcvlowat(svsk->sk_sk, 1);  in svc_tcp_fragment_received()
1111 	struct svc_sock	*svsk =  in svc_tcp_recvfrom()  local
1113 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_recvfrom()
1119 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_recvfrom()
1120 	len = svc_tcp_read_marker(svsk, rqstp);  in svc_tcp_recvfrom()
1124 	base = svc_tcp_restore_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1125 	want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr));  in svc_tcp_recvfrom()
1128 		trace_svcsock_tcp_recv(&svsk->sk_xprt, len);  in svc_tcp_recvfrom()
1129 		svsk->sk_tcplen += len;  in svc_tcp_recvfrom()
1130 		svsk->sk_datalen += len;  in svc_tcp_recvfrom()
1132 	if (len != want || !svc_sock_final_rec(svsk))  in svc_tcp_recvfrom()
1134 	if (svsk->sk_datalen < 8)  in svc_tcp_recvfrom()
1137 	rqstp->rq_arg.len = svsk->sk_datalen;  in svc_tcp_recvfrom()
1147 	if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags))  in svc_tcp_recvfrom()
1155 		len = receive_cb_reply(svsk, rqstp);  in svc_tcp_recvfrom()
1158 	svsk->sk_datalen = 0;  in svc_tcp_recvfrom()
1159 	svc_tcp_fragment_received(svsk);  in svc_tcp_recvfrom()
1164 	svc_xprt_copy_addrs(rqstp, &svsk->sk_xprt);  in svc_tcp_recvfrom()
1173 	svc_tcp_save_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1177 		svc_tcp_fragment_received(svsk);  in svc_tcp_recvfrom()
1182 		tcp_set_rcvlowat(svsk->sk_sk,  in svc_tcp_recvfrom()
1183 				 svc_sock_reclen(svsk) - svsk->sk_tcplen);  in svc_tcp_recvfrom()
1185 		trace_svcsock_tcp_recv_short(&svsk->sk_xprt,  in svc_tcp_recvfrom()
1186 				svc_sock_reclen(svsk),  in svc_tcp_recvfrom()
1187 				svsk->sk_tcplen - sizeof(rpc_fraghdr));  in svc_tcp_recvfrom()
1193 	trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0);  in svc_tcp_recvfrom()
1196 	svsk->sk_datalen = 0;  in svc_tcp_recvfrom()
1198 	trace_svcsock_tcp_recv_err(&svsk->sk_xprt, len);  in svc_tcp_recvfrom()
1199 	svc_xprt_deferred_close(&svsk->sk_xprt);  in svc_tcp_recvfrom()
1210 static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp,  in svc_tcp_sendmsg()  argument
1225 	buf = page_frag_alloc(&svsk->sk_frag_cache, sizeof(marker),  in svc_tcp_sendmsg()
1237 	ret = sock_sendmsg(svsk->sk_sock, &msg);  in svc_tcp_sendmsg()
1257 	struct svc_sock	*svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_sendto()  local
1270 	err = svc_tcp_sendmsg(svsk, rqstp, marker, &sent);  in svc_tcp_sendto()
1332 static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv)  in svc_tcp_init()  argument
1334 	struct sock	*sk = svsk->sk_sk;  in svc_tcp_init()
1336 	svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_tcp_class,  in svc_tcp_init()
1337 		      &svsk->sk_xprt, serv);  in svc_tcp_init()
1338 	set_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1339 	set_bit(XPT_CONG_CTRL, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1341 		strcpy(svsk->sk_xprt.xpt_remotebuf, "listener");  in svc_tcp_init()
1342 		set_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1344 		set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1350 		svsk->sk_marker = xdr_zero;  in svc_tcp_init()
1351 		svsk->sk_tcplen = 0;  in svc_tcp_init()
1352 		svsk->sk_datalen = 0;  in svc_tcp_init()
1353 		memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages));  in svc_tcp_init()
1357 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1363 			svc_xprt_deferred_close(&svsk->sk_xprt);  in svc_tcp_init()
1374 	struct svc_sock *svsk;  in svc_sock_update_bufs()  local
1377 	list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list)  in svc_sock_update_bufs()
1378 		set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags);  in svc_sock_update_bufs()
1389 	struct svc_sock	*svsk;  in svc_setup_socket()  local
1393 	svsk = kzalloc(sizeof(*svsk), GFP_KERNEL);  in svc_setup_socket()
1394 	if (!svsk)  in svc_setup_socket()
1406 			kfree(svsk);  in svc_setup_socket()
1411 	svsk->sk_sock = sock;  in svc_setup_socket()
1412 	svsk->sk_sk = inet;  in svc_setup_socket()
1413 	svsk->sk_ostate = inet->sk_state_change;  in svc_setup_socket()
1414 	svsk->sk_odata = inet->sk_data_ready;  in svc_setup_socket()
1415 	svsk->sk_owspace = inet->sk_write_space;  in svc_setup_socket()
1422 	inet->sk_user_data = svsk;  in svc_setup_socket()
1426 		svc_udp_init(svsk, serv);  in svc_setup_socket()
1428 		svc_tcp_init(svsk, serv);  in svc_setup_socket()
1430 	trace_svcsock_new(svsk, sock);  in svc_setup_socket()
1431 	return svsk;  in svc_setup_socket()
1452 	struct svc_sock *svsk = NULL;  in svc_addsock()  local
1475 	svsk = svc_setup_socket(serv, so, SVC_SOCK_DEFAULTS);  in svc_addsock()
1476 	if (IS_ERR(svsk)) {  in svc_addsock()
1478 		err = PTR_ERR(svsk);  in svc_addsock()
1481 	salen = kernel_getsockname(svsk->sk_sock, sin);  in svc_addsock()
1483 		svc_xprt_set_local(&svsk->sk_xprt, sin, salen);  in svc_addsock()
1484 	svsk->sk_xprt.xpt_cred = get_cred(cred);  in svc_addsock()
1485 	svc_add_new_perm_xprt(serv, &svsk->sk_xprt);  in svc_addsock()
1486 	return svc_one_sock_name(svsk, name_return, len);  in svc_addsock()
1502 	struct svc_sock	*svsk;  in svc_create_socket()  local
1558 	svsk = svc_setup_socket(serv, sock, flags);  in svc_create_socket()
1559 	if (IS_ERR(svsk)) {  in svc_create_socket()
1560 		error = PTR_ERR(svsk);  in svc_create_socket()
1563 	svc_xprt_set_local(&svsk->sk_xprt, newsin, newlen);  in svc_create_socket()
1564 	return (struct svc_xprt *)svsk;  in svc_create_socket()
1576 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_sock_detach()  local
1577 	struct sock *sk = svsk->sk_sk;  in svc_sock_detach()
1581 	sk->sk_state_change = svsk->sk_ostate;  in svc_sock_detach()
1582 	sk->sk_data_ready = svsk->sk_odata;  in svc_sock_detach()
1583 	sk->sk_write_space = svsk->sk_owspace;  in svc_sock_detach()
1593 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_sock_detach()  local
1595 	tls_handshake_close(svsk->sk_sock);  in svc_tcp_sock_detach()
1600 		svc_tcp_clear_pages(svsk);  in svc_tcp_sock_detach()
1601 		kernel_sock_shutdown(svsk->sk_sock, SHUT_RDWR);  in svc_tcp_sock_detach()
1610 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_sock_free()  local
1611 	struct page_frag_cache *pfc = &svsk->sk_frag_cache;  in svc_sock_free()
1612 	struct socket *sock = svsk->sk_sock;  in svc_sock_free()
1614 	trace_svcsock_free(svsk, sock);  in svc_sock_free()
1624 	kfree(svsk);  in svc_sock_free()