Lines Matching full:smc
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
19 #define KMSG_COMPONENT "smc"
34 #include <net/smc.h>
41 #include "smc.h"
88 sock_net(skb->sk)->smc.limit_smc_hs)) in smc_nl_dump_hs_limitation()
102 sock_net(skb->sk)->smc.limit_smc_hs = true; in smc_nl_enable_hs_limitation()
108 sock_net(skb->sk)->smc.limit_smc_hs = false; in smc_nl_disable_hs_limitation()
114 struct smc_sock *smc = smc_sk(sk); in smc_set_keepalive() local
116 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); in smc_set_keepalive()
126 struct smc_sock *smc; in smc_tcp_syn_recv_sock() local
129 smc = smc_clcsock_user_data(sk); in smc_tcp_syn_recv_sock()
131 if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) > in smc_tcp_syn_recv_sock()
135 if (sk_acceptq_is_full(&smc->sk)) { in smc_tcp_syn_recv_sock()
141 child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash, in smc_tcp_syn_recv_sock()
143 /* child must not inherit smc or its ops */ in smc_tcp_syn_recv_sock()
149 inet_csk(child)->icsk_af_ops = smc->ori_af_ops; in smc_tcp_syn_recv_sock()
161 const struct smc_sock *smc; in smc_hs_congested() local
163 smc = smc_clcsock_user_data(sk); in smc_hs_congested()
165 if (!smc) in smc_hs_congested()
213 struct smc_sock *smc = smc_sk(sk); in smc_release_cb() local
215 if (smc->conn.tx_in_release_sock) { in smc_release_cb()
216 smc_tx_pending(&smc->conn); in smc_release_cb()
217 smc->conn.tx_in_release_sock = false; in smc_release_cb()
222 .name = "SMC",
247 static void smc_fback_restore_callbacks(struct smc_sock *smc) in smc_fback_restore_callbacks() argument
249 struct sock *clcsk = smc->clcsock->sk; in smc_fback_restore_callbacks()
254 smc_clcsock_restore_cb(&clcsk->sk_state_change, &smc->clcsk_state_change); in smc_fback_restore_callbacks()
255 smc_clcsock_restore_cb(&clcsk->sk_data_ready, &smc->clcsk_data_ready); in smc_fback_restore_callbacks()
256 smc_clcsock_restore_cb(&clcsk->sk_write_space, &smc->clcsk_write_space); in smc_fback_restore_callbacks()
257 smc_clcsock_restore_cb(&clcsk->sk_error_report, &smc->clcsk_error_report); in smc_fback_restore_callbacks()
262 static void smc_restore_fallback_changes(struct smc_sock *smc) in smc_restore_fallback_changes() argument
264 if (smc->clcsock->file) { /* non-accepted sockets have no file yet */ in smc_restore_fallback_changes()
265 smc->clcsock->file->private_data = smc->sk.sk_socket; in smc_restore_fallback_changes()
266 smc->clcsock->file = NULL; in smc_restore_fallback_changes()
267 smc_fback_restore_callbacks(smc); in smc_restore_fallback_changes()
271 static int __smc_release(struct smc_sock *smc) in __smc_release() argument
273 struct sock *sk = &smc->sk; in __smc_release()
276 if (!smc->use_fallback) { in __smc_release()
277 rc = smc_close_active(smc); in __smc_release()
287 rc = kernel_sock_shutdown(smc->clcsock, in __smc_release()
293 smc_restore_fallback_changes(smc); in __smc_release()
299 if (smc->clcsock) { in __smc_release()
301 smc_clcsock_release(smc); in __smc_release()
304 if (!smc->use_fallback) in __smc_release()
305 smc_conn_free(&smc->conn); in __smc_release()
314 struct smc_sock *smc; in smc_release() local
321 smc = smc_sk(sk); in smc_release()
326 if (smc->connect_nonblock && old_state == SMC_INIT) in smc_release()
327 tcp_abort(smc->clcsock->sk, ECONNABORTED); in smc_release()
329 if (cancel_work_sync(&smc->connect_work)) in smc_release()
330 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */ in smc_release()
341 !smc->use_fallback) in smc_release()
342 smc_close_active_abort(smc); in smc_release()
344 rc = __smc_release(smc); in smc_release()
367 struct smc_sock *smc = smc_sk(sk); in smc_sk_init() local
372 WRITE_ONCE(sk->sk_sndbuf, 2 * READ_ONCE(net->smc.sysctl_wmem)); in smc_sk_init()
373 WRITE_ONCE(sk->sk_rcvbuf, 2 * READ_ONCE(net->smc.sysctl_rmem)); in smc_sk_init()
374 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); in smc_sk_init()
375 INIT_WORK(&smc->connect_work, smc_connect_work); in smc_sk_init()
376 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work); in smc_sk_init()
377 INIT_LIST_HEAD(&smc->accept_q); in smc_sk_init()
378 spin_lock_init(&smc->accept_q_lock); in smc_sk_init()
379 spin_lock_init(&smc->conn.send_lock); in smc_sk_init()
381 mutex_init(&smc->clcsock_release_lock); in smc_sk_init()
382 smc_init_saved_callbacks(smc); in smc_sk_init()
383 smc->limit_smc_hs = net->smc.limit_smc_hs; in smc_sk_init()
384 smc->use_fallback = false; /* assume rdma capability first */ in smc_sk_init()
385 smc->fallback_rsn = 0; in smc_sk_init()
410 struct smc_sock *smc; in smc_bind() local
413 smc = smc_sk(sk); in smc_bind()
434 if (sk->sk_state != SMC_INIT || smc->connect_nonblock) in smc_bind()
437 smc->clcsock->sk->sk_reuse = sk->sk_reuse; in smc_bind()
438 smc->clcsock->sk->sk_reuseport = sk->sk_reuseport; in smc_bind()
439 rc = kernel_bind(smc->clcsock, uaddr, addr_len); in smc_bind()
447 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
448 * clc socket (since smc is not called for these options from net/core)
467 /* if set, use value set by setsockopt() - else use IPv4 or SMC sysctl value */
497 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) in smc_copy_sock_settings_to_clc() argument
499 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); in smc_copy_sock_settings_to_clc()
506 /* copy only settings and flags relevant for smc from clc to smc socket */
507 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) in smc_copy_sock_settings_to_smc() argument
509 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); in smc_copy_sock_settings_to_smc()
585 static int smcr_clnt_conf_first_link(struct smc_sock *smc) in smcr_clnt_conf_first_link() argument
587 struct smc_link *link = smc->conn.lnk; in smcr_clnt_conf_first_link()
601 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_clnt_conf_first_link()
618 if (smc->conn.sndbuf_desc->is_vm) { in smcr_clnt_conf_first_link()
619 if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc)) in smcr_clnt_conf_first_link()
624 if (smcr_link_reg_buf(link, smc->conn.rmb_desc)) in smcr_clnt_conf_first_link()
628 smc->conn.rmb_desc->is_conf_rkey = true; in smcr_clnt_conf_first_link()
645 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_clnt_conf_first_link()
667 static void smc_conn_save_peer_info_fce(struct smc_sock *smc, in smc_conn_save_peer_info_fce() argument
677 if (smc->conn.lgr->is_smcd) { in smc_conn_save_peer_info_fce()
678 memcpy(smc->conn.lgr->negotiated_eid, clc->d1.eid, in smc_conn_save_peer_info_fce()
682 memcpy(smc->conn.lgr->negotiated_eid, clc->r1.eid, in smc_conn_save_peer_info_fce()
687 smc->conn.lgr->peer_os = fce->os_type; in smc_conn_save_peer_info_fce()
688 smc->conn.lgr->peer_smc_release = fce->release; in smc_conn_save_peer_info_fce()
690 memcpy(smc->conn.lgr->peer_hostname, fce->hostname, in smc_conn_save_peer_info_fce()
694 static void smcr_conn_save_peer_info(struct smc_sock *smc, in smcr_conn_save_peer_info() argument
699 smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx; in smcr_conn_save_peer_info()
700 smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token); in smcr_conn_save_peer_info()
701 smc->conn.peer_rmbe_size = bufsize; in smcr_conn_save_peer_info()
702 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); in smcr_conn_save_peer_info()
703 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1); in smcr_conn_save_peer_info()
706 static void smcd_conn_save_peer_info(struct smc_sock *smc, in smcd_conn_save_peer_info() argument
711 smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx; in smcd_conn_save_peer_info()
712 smc->conn.peer_token = ntohll(clc->d0.token); in smcd_conn_save_peer_info()
714 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg); in smcd_conn_save_peer_info()
715 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); in smcd_conn_save_peer_info()
716 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx; in smcd_conn_save_peer_info()
719 static void smc_conn_save_peer_info(struct smc_sock *smc, in smc_conn_save_peer_info() argument
722 if (smc->conn.lgr->is_smcd) in smc_conn_save_peer_info()
723 smcd_conn_save_peer_info(smc, clc); in smc_conn_save_peer_info()
725 smcr_conn_save_peer_info(smc, clc); in smc_conn_save_peer_info()
726 smc_conn_save_peer_info_fce(smc, clc); in smc_conn_save_peer_info()
740 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc, in smc_stat_inc_fback_rsn_cnt() argument
746 if (fback_arr[cnt].fback_code == smc->fallback_rsn) { in smc_stat_inc_fback_rsn_cnt()
751 fback_arr[cnt].fback_code = smc->fallback_rsn; in smc_stat_inc_fback_rsn_cnt()
758 static void smc_stat_fallback(struct smc_sock *smc) in smc_stat_fallback() argument
760 struct net *net = sock_net(&smc->sk); in smc_stat_fallback()
762 mutex_lock(&net->smc.mutex_fback_rsn); in smc_stat_fallback()
763 if (smc->listen_smc) { in smc_stat_fallback()
764 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv); in smc_stat_fallback()
765 net->smc.fback_rsn->srv_fback_cnt++; in smc_stat_fallback()
767 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt); in smc_stat_fallback()
768 net->smc.fback_rsn->clnt_fback_cnt++; in smc_stat_fallback()
770 mutex_unlock(&net->smc.mutex_fback_rsn); in smc_stat_fallback()
774 static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key) in smc_fback_wakeup_waitqueue() argument
779 wq = rcu_dereference(smc->sk.sk_wq); in smc_fback_wakeup_waitqueue()
783 /* wake up smc sk->sk_wq */ in smc_fback_wakeup_waitqueue()
809 static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk, in smc_fback_forward_wakeup() argument
826 smc_fback_wakeup_waitqueue(smc, mark.key); in smc_fback_forward_wakeup()
833 struct smc_sock *smc; in smc_fback_state_change() local
836 smc = smc_clcsock_user_data(clcsk); in smc_fback_state_change()
837 if (smc) in smc_fback_state_change()
838 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_state_change()
839 smc->clcsk_state_change); in smc_fback_state_change()
845 struct smc_sock *smc; in smc_fback_data_ready() local
848 smc = smc_clcsock_user_data(clcsk); in smc_fback_data_ready()
849 if (smc) in smc_fback_data_ready()
850 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_data_ready()
851 smc->clcsk_data_ready); in smc_fback_data_ready()
857 struct smc_sock *smc; in smc_fback_write_space() local
860 smc = smc_clcsock_user_data(clcsk); in smc_fback_write_space()
861 if (smc) in smc_fback_write_space()
862 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_write_space()
863 smc->clcsk_write_space); in smc_fback_write_space()
869 struct smc_sock *smc; in smc_fback_error_report() local
872 smc = smc_clcsock_user_data(clcsk); in smc_fback_error_report()
873 if (smc) in smc_fback_error_report()
874 smc_fback_forward_wakeup(smc, clcsk, in smc_fback_error_report()
875 smc->clcsk_error_report); in smc_fback_error_report()
879 static void smc_fback_replace_callbacks(struct smc_sock *smc) in smc_fback_replace_callbacks() argument
881 struct sock *clcsk = smc->clcsock->sk; in smc_fback_replace_callbacks()
884 clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); in smc_fback_replace_callbacks()
887 &smc->clcsk_state_change); in smc_fback_replace_callbacks()
889 &smc->clcsk_data_ready); in smc_fback_replace_callbacks()
891 &smc->clcsk_write_space); in smc_fback_replace_callbacks()
893 &smc->clcsk_error_report); in smc_fback_replace_callbacks()
898 static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code) in smc_switch_to_fallback() argument
902 mutex_lock(&smc->clcsock_release_lock); in smc_switch_to_fallback()
903 if (!smc->clcsock) { in smc_switch_to_fallback()
908 smc->use_fallback = true; in smc_switch_to_fallback()
909 smc->fallback_rsn = reason_code; in smc_switch_to_fallback()
910 smc_stat_fallback(smc); in smc_switch_to_fallback()
911 trace_smc_switch_to_fallback(smc, reason_code); in smc_switch_to_fallback()
912 if (smc->sk.sk_socket && smc->sk.sk_socket->file) { in smc_switch_to_fallback()
913 smc->clcsock->file = smc->sk.sk_socket->file; in smc_switch_to_fallback()
914 smc->clcsock->file->private_data = smc->clcsock; in smc_switch_to_fallback()
915 smc->clcsock->wq.fasync_list = in smc_switch_to_fallback()
916 smc->sk.sk_socket->wq.fasync_list; in smc_switch_to_fallback()
917 smc->sk.sk_socket->wq.fasync_list = NULL; in smc_switch_to_fallback()
920 * in smc sk->sk_wq and they should be woken up in smc_switch_to_fallback()
923 smc_fback_replace_callbacks(smc); in smc_switch_to_fallback()
926 mutex_unlock(&smc->clcsock_release_lock); in smc_switch_to_fallback()
931 static int smc_connect_fallback(struct smc_sock *smc, int reason_code) in smc_connect_fallback() argument
933 struct net *net = sock_net(&smc->sk); in smc_connect_fallback()
936 rc = smc_switch_to_fallback(smc, reason_code); in smc_connect_fallback()
938 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_fallback()
939 if (smc->sk.sk_state == SMC_INIT) in smc_connect_fallback()
940 sock_put(&smc->sk); /* passive closing */ in smc_connect_fallback()
943 smc_copy_sock_settings_to_clc(smc); in smc_connect_fallback()
944 smc->connect_nonblock = 0; in smc_connect_fallback()
945 if (smc->sk.sk_state == SMC_INIT) in smc_connect_fallback()
946 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_fallback()
951 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code, in smc_connect_decline_fallback() argument
954 struct net *net = sock_net(&smc->sk); in smc_connect_decline_fallback()
958 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_decline_fallback()
959 if (smc->sk.sk_state == SMC_INIT) in smc_connect_decline_fallback()
960 sock_put(&smc->sk); /* passive closing */ in smc_connect_decline_fallback()
964 rc = smc_clc_send_decline(smc, reason_code, version); in smc_connect_decline_fallback()
966 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); in smc_connect_decline_fallback()
967 if (smc->sk.sk_state == SMC_INIT) in smc_connect_decline_fallback()
968 sock_put(&smc->sk); /* passive closing */ in smc_connect_decline_fallback()
972 return smc_connect_fallback(smc, reason_code); in smc_connect_decline_fallback()
975 static void smc_conn_abort(struct smc_sock *smc, int local_first) in smc_conn_abort() argument
977 struct smc_connection *conn = &smc->conn; in smc_conn_abort()
991 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini) in smc_find_rdma_device() argument
997 smc_pnet_find_roce_resource(smc->clcsock->sk, ini); in smc_find_rdma_device()
1007 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini) in smc_find_ism_device() argument
1010 smc_pnet_find_ism_resource(smc->clcsock->sk, ini); in smc_find_ism_device()
1033 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc, in smc_find_ism_v2_device_clnt() argument
1053 smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) { in smc_find_ism_v2_device_clnt()
1056 * Proposal SMC-Dv2 extension, but an Emulated- in smc_find_ism_v2_device_clnt()
1080 static int smc_connect_ism_vlan_setup(struct smc_sock *smc, in smc_connect_ism_vlan_setup() argument
1088 static int smc_find_proposal_devices(struct smc_sock *smc, in smc_find_proposal_devices() argument
1095 smc_find_ism_device(smc, ini) || in smc_find_proposal_devices()
1096 smc_connect_ism_vlan_setup(smc, ini)) in smc_find_proposal_devices()
1102 smc_find_rdma_device(smc, ini)) in smc_find_proposal_devices()
1112 smc_find_ism_v2_device_clnt(smc, ini)) in smc_find_proposal_devices()
1117 ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr; in smc_find_proposal_devices()
1119 smc->clcsock->sk->sk_family != AF_INET || in smc_find_proposal_devices()
1121 smc_find_rdma_device(smc, ini)) in smc_find_proposal_devices()
1138 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, in smc_connect_ism_vlan_cleanup() argument
1154 static int smc_connect_clc(struct smc_sock *smc, in smc_connect_clc() argument
1161 rc = smc_clc_send_proposal(smc, ini); in smc_connect_clc()
1164 /* receive SMC Accept CLC message */ in smc_connect_clc()
1165 return smc_clc_wait_msg(smc, aclc, SMC_CLC_MAX_ACCEPT_LEN, in smc_connect_clc()
1197 static int smc_connect_rdma_v2_prepare(struct smc_sock *smc, in smc_connect_rdma_v2_prepare() argument
1203 struct net *net = sock_net(&smc->sk); in smc_connect_rdma_v2_prepare()
1213 if (smc_ib_find_route(net, smc->clcsock->sk->sk_rcv_saddr, in smc_connect_rdma_v2_prepare()
1233 static int smc_connect_rdma(struct smc_sock *smc, in smc_connect_rdma() argument
1250 reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini); in smc_connect_rdma()
1255 reason_code = smc_conn_create(smc, ini); in smc_connect_rdma()
1261 smc_conn_save_peer_info(smc, aclc); in smc_connect_rdma()
1264 link = smc->conn.lnk; in smc_connect_rdma()
1269 struct smc_link *l = &smc->conn.lgr->lnk[i]; in smc_connect_rdma()
1285 smc_switch_link_and_count(&smc->conn, link); in smc_connect_rdma()
1289 if (smc_buf_create(smc, false)) { in smc_connect_rdma()
1297 if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) { in smc_connect_rdma()
1302 smc_close_init(smc); in smc_connect_rdma()
1303 smc_rx_init(smc); in smc_connect_rdma()
1312 if (smc->conn.sndbuf_desc->is_vm) { in smc_connect_rdma()
1313 if (smcr_lgr_reg_sndbufs(link, smc->conn.sndbuf_desc)) { in smc_connect_rdma()
1318 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) { in smc_connect_rdma()
1331 reason_code = smc_clc_send_confirm(smc, ini->first_contact_local, in smc_connect_rdma()
1336 smc_tx_init(smc); in smc_connect_rdma()
1341 reason_code = smcr_clnt_conf_first_link(smc); in smc_connect_rdma()
1348 smc_copy_sock_settings_to_clc(smc); in smc_connect_rdma()
1349 smc->connect_nonblock = 0; in smc_connect_rdma()
1350 if (smc->sk.sk_state == SMC_INIT) in smc_connect_rdma()
1351 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_rdma()
1355 smc_conn_abort(smc, ini->first_contact_local); in smc_connect_rdma()
1357 smc->connect_nonblock = 0; in smc_connect_rdma()
1382 static int smc_connect_ism(struct smc_sock *smc, in smc_connect_ism() argument
1414 /* there is only one lgr role for SMC-D; use server lock */ in smc_connect_ism()
1416 rc = smc_conn_create(smc, ini); in smc_connect_ism()
1423 rc = smc_buf_create(smc, true); in smc_connect_ism()
1429 smc_conn_save_peer_info(smc, aclc); in smc_connect_ism()
1431 if (smc_ism_support_dmb_nocopy(smc->conn.lgr->smcd)) { in smc_connect_ism()
1432 rc = smcd_buf_attach(smc); in smc_connect_ism()
1438 smc_close_init(smc); in smc_connect_ism()
1439 smc_rx_init(smc); in smc_connect_ism()
1440 smc_tx_init(smc); in smc_connect_ism()
1445 rc = smc_clc_send_confirm(smc, ini->first_contact_local, in smc_connect_ism()
1451 smc_copy_sock_settings_to_clc(smc); in smc_connect_ism()
1452 smc->connect_nonblock = 0; in smc_connect_ism()
1453 if (smc->sk.sk_state == SMC_INIT) in smc_connect_ism()
1454 smc->sk.sk_state = SMC_ACTIVE; in smc_connect_ism()
1458 smc_conn_abort(smc, ini->first_contact_local); in smc_connect_ism()
1460 smc->connect_nonblock = 0; in smc_connect_ism()
1487 static int __smc_connect(struct smc_sock *smc) in __smc_connect() argument
1495 if (smc->use_fallback) in __smc_connect()
1496 return smc_connect_fallback(smc, smc->fallback_rsn); in __smc_connect()
1498 /* if peer has not signalled SMC-capability, fall back */ in __smc_connect()
1499 if (!tcp_sk(smc->clcsock->sk)->syn_smc) in __smc_connect()
1500 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC); in __smc_connect()
1502 /* IPSec connections opt out of SMC optimizations */ in __smc_connect()
1503 if (using_ipsec(smc)) in __smc_connect()
1504 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC, in __smc_connect()
1509 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM, in __smc_connect()
1518 if (smc_vlan_by_tcpsk(smc->clcsock, ini)) { in __smc_connect()
1524 rc = smc_find_proposal_devices(smc, ini); in __smc_connect()
1536 rc = smc_connect_clc(smc, aclc, ini); in __smc_connect()
1541 smc->sk.sk_err = ETIMEDOUT; in __smc_connect()
1546 /* check if smc modes and versions of CLC proposal and accept match */ in __smc_connect()
1555 rc = smc_connect_rdma(smc, aclc, ini); in __smc_connect()
1558 rc = smc_connect_ism(smc, aclc, ini); in __smc_connect()
1563 SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc); in __smc_connect()
1564 smc_connect_ism_vlan_cleanup(smc, ini); in __smc_connect()
1570 smc_connect_ism_vlan_cleanup(smc, ini); in __smc_connect()
1574 return smc_connect_decline_fallback(smc, rc, version); in __smc_connect()
1579 struct smc_sock *smc = container_of(work, struct smc_sock, in smc_connect_work() local
1581 long timeo = smc->sk.sk_sndtimeo; in smc_connect_work()
1586 lock_sock(smc->clcsock->sk); in smc_connect_work()
1587 if (smc->clcsock->sk->sk_err) { in smc_connect_work()
1588 smc->sk.sk_err = smc->clcsock->sk->sk_err; in smc_connect_work()
1589 } else if ((1 << smc->clcsock->sk->sk_state) & in smc_connect_work()
1591 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo); in smc_connect_work()
1593 ((1 << smc->clcsock->sk->sk_state) & in smc_connect_work()
1597 release_sock(smc->clcsock->sk); in smc_connect_work()
1598 lock_sock(&smc->sk); in smc_connect_work()
1599 if (rc != 0 || smc->sk.sk_err) { in smc_connect_work()
1600 smc->sk.sk_state = SMC_CLOSED; in smc_connect_work()
1602 smc->sk.sk_err = EPIPE; in smc_connect_work()
1604 smc->sk.sk_err = ECONNREFUSED; in smc_connect_work()
1606 smc->sk.sk_err = -sock_intr_errno(timeo); in smc_connect_work()
1607 sock_put(&smc->sk); /* passive closing */ in smc_connect_work()
1611 rc = __smc_connect(smc); in smc_connect_work()
1613 smc->sk.sk_err = -rc; in smc_connect_work()
1616 if (!sock_flag(&smc->sk, SOCK_DEAD)) { in smc_connect_work()
1617 if (smc->sk.sk_err) { in smc_connect_work()
1618 smc->sk.sk_state_change(&smc->sk); in smc_connect_work()
1620 smc->clcsock->sk->sk_write_space(smc->clcsock->sk); in smc_connect_work()
1621 smc->sk.sk_write_space(&smc->sk); in smc_connect_work()
1624 release_sock(&smc->sk); in smc_connect_work()
1631 struct smc_sock *smc; in smc_connect() local
1634 smc = smc_sk(sk); in smc_connect()
1636 /* separate smc parameter checking to be safe */ in smc_connect()
1673 smc_copy_sock_settings_to_clc(smc); in smc_connect()
1674 tcp_sk(smc->clcsock->sk)->syn_smc = 1; in smc_connect()
1675 if (smc->connect_nonblock) { in smc_connect()
1679 rc = kernel_connect(smc->clcsock, addr, alen, flags); in smc_connect()
1683 if (smc->use_fallback) { in smc_connect()
1687 sock_hold(&smc->sk); /* sock put in passive closing */ in smc_connect()
1689 if (queue_work(smc_hs_wq, &smc->connect_work)) in smc_connect()
1690 smc->connect_nonblock = 1; in smc_connect()
1694 rc = __smc_connect(smc); in smc_connect()
1744 /* new clcsock has inherited the smc listen-specific sk_data_ready in smc_clcsock_accept()
1830 struct smc_sock *smc = smc_sk(sk); in smc_close_non_accepted() local
1837 __smc_release(smc); in smc_close_non_accepted()
1843 static int smcr_serv_conf_first_link(struct smc_sock *smc) in smcr_serv_conf_first_link() argument
1845 struct smc_link *link = smc->conn.lnk; in smcr_serv_conf_first_link()
1850 if (smc->conn.sndbuf_desc->is_vm) { in smcr_serv_conf_first_link()
1851 if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc)) in smcr_serv_conf_first_link()
1856 if (smcr_link_reg_buf(link, smc->conn.rmb_desc)) in smcr_serv_conf_first_link()
1870 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), in smcr_serv_conf_first_link()
1881 smc->conn.rmb_desc->is_conf_rkey = true; in smcr_serv_conf_first_link()
1934 this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt); in smc_listen_out_err()
2061 /* listen worker: initialize connection and buffers for SMC-D */
2433 /* check if peer is smc capable */ in smc_listen_work()
2444 * wait for and receive SMC Proposal CLC message in smc_listen_work()
2460 /* IPSec connections opt out of SMC optimizations */ in smc_listen_work()
2491 /* send SMC Accept CLC message */ in smc_listen_work()
2498 /* SMC-D does not need this lock any more */ in smc_listen_work()
2502 /* receive SMC Confirm CLC message */ in smc_listen_work()
2520 /* fce smc release version is needed in smc_listen_rdma_finish, in smc_listen_work()
2612 struct smc_sock *smc; in smc_listen() local
2615 smc = smc_sk(sk); in smc_listen()
2620 smc->connect_nonblock || sock->state != SS_UNCONNECTED) in smc_listen()
2629 * them to the clc socket -- copy smc socket options to clc socket in smc_listen()
2631 smc_copy_sock_settings_to_clc(smc); in smc_listen()
2632 if (!smc->use_fallback) in smc_listen()
2633 tcp_sk(smc->clcsock->sk)->syn_smc = 1; in smc_listen()
2636 * smc-specific sk_data_ready function in smc_listen()
2638 write_lock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2639 smc->clcsock->sk->sk_user_data = in smc_listen()
2640 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); in smc_listen()
2641 smc_clcsock_replace_cb(&smc->clcsock->sk->sk_data_ready, in smc_listen()
2642 smc_clcsock_data_ready, &smc->clcsk_data_ready); in smc_listen()
2643 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2646 smc->ori_af_ops = inet_csk(smc->clcsock->sk)->icsk_af_ops; in smc_listen()
2648 smc->af_ops = *smc->ori_af_ops; in smc_listen()
2649 smc->af_ops.syn_recv_sock = smc_tcp_syn_recv_sock; in smc_listen()
2651 inet_csk(smc->clcsock->sk)->icsk_af_ops = &smc->af_ops; in smc_listen()
2653 if (smc->limit_smc_hs) in smc_listen()
2654 tcp_sk(smc->clcsock->sk)->smc_hs_congested = smc_hs_congested; in smc_listen()
2656 rc = kernel_listen(smc->clcsock, backlog); in smc_listen()
2658 write_lock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2659 smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, in smc_listen()
2660 &smc->clcsk_data_ready); in smc_listen()
2661 smc->clcsock->sk->sk_user_data = NULL; in smc_listen()
2662 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); in smc_listen()
2747 struct smc_sock *smc; in smc_getname() local
2753 smc = smc_sk(sock->sk); in smc_getname()
2755 return smc->clcsock->ops->getname(smc->clcsock, addr, peer); in smc_getname()
2761 struct smc_sock *smc; in smc_sendmsg() local
2764 smc = smc_sk(sk); in smc_sendmsg()
2767 /* SMC does not support connect with fastopen */ in smc_sendmsg()
2770 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { in smc_sendmsg()
2771 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); in smc_sendmsg()
2785 if (smc->use_fallback) { in smc_sendmsg()
2786 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len); in smc_sendmsg()
2788 rc = smc_tx_sendmsg(smc, msg, len); in smc_sendmsg()
2789 SMC_STAT_TX_PAYLOAD(smc, len, rc); in smc_sendmsg()
2800 struct smc_sock *smc; in smc_recvmsg() local
2803 smc = smc_sk(sk); in smc_recvmsg()
2820 if (smc->use_fallback) { in smc_recvmsg()
2821 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags); in smc_recvmsg()
2824 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags); in smc_recvmsg()
2825 SMC_STAT_RX_PAYLOAD(smc, rc, rc); in smc_recvmsg()
2850 struct smc_sock *smc; in smc_poll() local
2856 smc = smc_sk(sock->sk); in smc_poll()
2857 if (smc->use_fallback) { in smc_poll()
2859 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait); in smc_poll()
2860 sk->sk_err = smc->clcsock->sk->sk_err; in smc_poll()
2872 } else if (smc->use_fallback) { /* as result of connect_work()*/ in smc_poll()
2873 mask |= smc->clcsock->ops->poll(file, smc->clcsock, in smc_poll()
2875 sk->sk_err = smc->clcsock->sk->sk_err; in smc_poll()
2878 atomic_read(&smc->conn.sndbuf_space)) || in smc_poll()
2885 if (atomic_read(&smc->conn.bytes_to_rcv)) in smc_poll()
2891 if (smc->conn.urg_state == SMC_URG_VALID) in smc_poll()
2903 struct smc_sock *smc; in smc_shutdown() local
2908 smc = smc_sk(sk); in smc_shutdown()
2934 if (smc->use_fallback) { in smc_shutdown()
2935 rc = kernel_sock_shutdown(smc->clcsock, how); in smc_shutdown()
2936 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown; in smc_shutdown()
2947 rc = smc_close_active(smc); in smc_shutdown()
2953 rc = smc_close_shutdown_write(smc); in smc_shutdown()
2960 if (do_shutdown && smc->clcsock) in smc_shutdown()
2961 rc1 = kernel_sock_shutdown(smc->clcsock, how); in smc_shutdown()
2977 struct smc_sock *smc; in __smc_getsockopt() local
2980 smc = smc_sk(sock->sk); in __smc_getsockopt()
2992 val = smc->limit_smc_hs; in __smc_getsockopt()
3010 struct smc_sock *smc; in __smc_setsockopt() local
3013 smc = smc_sk(sk); in __smc_setsockopt()
3027 smc->limit_smc_hs = !!val; in __smc_setsockopt()
3043 struct smc_sock *smc; in smc_setsockopt() local
3051 smc = smc_sk(sk); in smc_setsockopt()
3056 mutex_lock(&smc->clcsock_release_lock); in smc_setsockopt()
3057 if (!smc->clcsock) { in smc_setsockopt()
3058 mutex_unlock(&smc->clcsock_release_lock); in smc_setsockopt()
3061 if (unlikely(!smc->clcsock->ops->setsockopt)) in smc_setsockopt()
3064 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname, in smc_setsockopt()
3066 if (smc->clcsock->sk->sk_err) { in smc_setsockopt()
3067 sk->sk_err = smc->clcsock->sk->sk_err; in smc_setsockopt()
3070 mutex_unlock(&smc->clcsock_release_lock); in smc_setsockopt()
3078 if (rc || smc->use_fallback) in smc_setsockopt()
3085 /* option not supported by SMC */ in smc_setsockopt()
3086 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { in smc_setsockopt()
3087 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); in smc_setsockopt()
3097 SMC_STAT_INC(smc, ndly_cnt); in smc_setsockopt()
3098 smc_tx_pending(&smc->conn); in smc_setsockopt()
3099 cancel_delayed_work(&smc->conn.tx_work); in smc_setsockopt()
3108 SMC_STAT_INC(smc, cork_cnt); in smc_setsockopt()
3109 smc_tx_pending(&smc->conn); in smc_setsockopt()
3110 cancel_delayed_work(&smc->conn.tx_work); in smc_setsockopt()
3115 smc->sockopt_defer_accept = val; in smc_setsockopt()
3129 struct smc_sock *smc; in smc_getsockopt() local
3135 smc = smc_sk(sock->sk); in smc_getsockopt()
3136 mutex_lock(&smc->clcsock_release_lock); in smc_getsockopt()
3137 if (!smc->clcsock) { in smc_getsockopt()
3138 mutex_unlock(&smc->clcsock_release_lock); in smc_getsockopt()
3142 if (unlikely(!smc->clcsock->ops->getsockopt)) { in smc_getsockopt()
3143 mutex_unlock(&smc->clcsock_release_lock); in smc_getsockopt()
3146 rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname, in smc_getsockopt()
3148 mutex_unlock(&smc->clcsock_release_lock); in smc_getsockopt()
3157 struct smc_sock *smc; in smc_ioctl() local
3160 smc = smc_sk(sock->sk); in smc_ioctl()
3161 conn = &smc->conn; in smc_ioctl()
3162 lock_sock(&smc->sk); in smc_ioctl()
3163 if (smc->use_fallback) { in smc_ioctl()
3164 if (!smc->clcsock) { in smc_ioctl()
3165 release_sock(&smc->sk); in smc_ioctl()
3168 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg); in smc_ioctl()
3169 release_sock(&smc->sk); in smc_ioctl()
3174 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3175 release_sock(&smc->sk); in smc_ioctl()
3178 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3179 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
3182 answ = atomic_read(&smc->conn.bytes_to_rcv); in smc_ioctl()
3186 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3187 release_sock(&smc->sk); in smc_ioctl()
3190 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3191 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
3194 answ = smc->conn.sndbuf_desc->len - in smc_ioctl()
3195 atomic_read(&smc->conn.sndbuf_space); in smc_ioctl()
3199 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3200 release_sock(&smc->sk); in smc_ioctl()
3203 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3204 smc->sk.sk_state == SMC_CLOSED) in smc_ioctl()
3207 answ = smc_tx_prepared_sends(&smc->conn); in smc_ioctl()
3210 if (smc->sk.sk_state == SMC_LISTEN) { in smc_ioctl()
3211 release_sock(&smc->sk); in smc_ioctl()
3214 if (smc->sk.sk_state == SMC_INIT || in smc_ioctl()
3215 smc->sk.sk_state == SMC_CLOSED) { in smc_ioctl()
3225 release_sock(&smc->sk); in smc_ioctl()
3228 release_sock(&smc->sk); in smc_ioctl()
3244 struct smc_sock *smc; in smc_splice_read() local
3247 smc = smc_sk(sk); in smc_splice_read()
3264 if (smc->use_fallback) { in smc_splice_read()
3265 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos, in smc_splice_read()
3276 SMC_STAT_INC(smc, splice_cnt); in smc_splice_read()
3277 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags); in smc_splice_read()
3309 struct smc_sock *smc = smc_sk(sk); in smc_create_clcsk() local
3313 &smc->clcsock); in smc_create_clcsk()
3317 /* smc_clcsock_release() does not wait smc->clcsock->sk's in smc_create_clcsk()
3319 * smc->sk is close()d, and TCP timers can be fired later, in smc_create_clcsk()
3322 sk = smc->clcsock->sk; in smc_create_clcsk()
3334 struct smc_sock *smc; in __smc_create() local
3354 smc = smc_sk(sk); in __smc_create()
3358 smc->clcsock = clcsock; in __smc_create()
3414 /* replace tcp socket to smc */ in smc_ulp_init()
3434 .name = "smc",
3643 MODULE_DESCRIPTION("smc socket address family");
3646 MODULE_ALIAS_TCP_ULP("smc");