Lines Matching +full:tx +full:- +full:termination +full:- +full:fix
1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
62 if (lgr->is_smcd) { in smc_lgr_list_head()
63 *lgr_lock = &lgr->smcd->lgr_lock; in smc_lgr_list_head()
64 return &lgr->smcd->lgr_list; in smc_lgr_list_head()
73 atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]); in smc_ibdev_cnt_inc()
78 atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]); in smc_ibdev_cnt_dec()
85 * otherwise there is a risk of out-of-sync link groups. in smc_lgr_schedule_free_work()
87 if (!lgr->freeing) { in smc_lgr_schedule_free_work()
88 mod_delayed_work(system_wq, &lgr->free_work, in smc_lgr_schedule_free_work()
89 (!lgr->is_smcd && lgr->role == SMC_CLNT) ? in smc_lgr_schedule_free_work()
104 u32 token = conn->alert_token_local; in smc_lgr_add_alert_token()
106 link = &conn->lgr->conns_all.rb_node; in smc_lgr_add_alert_token()
112 if (cur->alert_token_local > token) in smc_lgr_add_alert_token()
113 link = &parent->rb_left; in smc_lgr_add_alert_token()
115 link = &parent->rb_right; in smc_lgr_add_alert_token()
118 rb_link_node(&conn->alert_node, parent, link); in smc_lgr_add_alert_token()
119 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); in smc_lgr_add_alert_token()
122 /* assign an SMC-R link to the connection */
130 conn->lnk = NULL; /* reset conn->lnk first */ in smcr_lgr_conn_assign_link()
132 struct smc_link *lnk = &conn->lgr->lnk[i]; in smcr_lgr_conn_assign_link()
134 if (lnk->state != expected || lnk->link_is_asym) in smcr_lgr_conn_assign_link()
136 if (conn->lgr->role == SMC_CLNT) { in smcr_lgr_conn_assign_link()
137 conn->lnk = lnk; /* temporary, SMC server assigns link*/ in smcr_lgr_conn_assign_link()
140 if (conn->lgr->conns_num % 2) { in smcr_lgr_conn_assign_link()
144 lnk2 = &conn->lgr->lnk[j]; in smcr_lgr_conn_assign_link()
145 if (lnk2->state == expected && in smcr_lgr_conn_assign_link()
146 !lnk2->link_is_asym) { in smcr_lgr_conn_assign_link()
147 conn->lnk = lnk2; in smcr_lgr_conn_assign_link()
152 if (!conn->lnk) in smcr_lgr_conn_assign_link()
153 conn->lnk = lnk; in smcr_lgr_conn_assign_link()
156 if (!conn->lnk) in smcr_lgr_conn_assign_link()
158 atomic_inc(&conn->lnk->conn_cnt); in smcr_lgr_conn_assign_link()
173 if (!conn->lgr->is_smcd) { in smc_lgr_register_conn()
176 conn->lgr = NULL; in smc_lgr_register_conn()
183 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ in smc_lgr_register_conn()
184 while (!conn->alert_token_local) { in smc_lgr_register_conn()
185 conn->alert_token_local = atomic_inc_return(&nexttoken); in smc_lgr_register_conn()
186 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) in smc_lgr_register_conn()
187 conn->alert_token_local = 0; in smc_lgr_register_conn()
190 conn->lgr->conns_num++; in smc_lgr_register_conn()
199 struct smc_link_group *lgr = conn->lgr; in __smc_lgr_unregister_conn()
201 rb_erase(&conn->alert_node, &lgr->conns_all); in __smc_lgr_unregister_conn()
202 if (conn->lnk) in __smc_lgr_unregister_conn()
203 atomic_dec(&conn->lnk->conn_cnt); in __smc_lgr_unregister_conn()
204 lgr->conns_num--; in __smc_lgr_unregister_conn()
205 conn->alert_token_local = 0; in __smc_lgr_unregister_conn()
206 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ in __smc_lgr_unregister_conn()
213 struct smc_link_group *lgr = conn->lgr; in smc_lgr_unregister_conn()
217 write_lock_bh(&lgr->conns_lock); in smc_lgr_unregister_conn()
218 if (conn->alert_token_local) { in smc_lgr_unregister_conn()
221 write_unlock_bh(&lgr->conns_lock); in smc_lgr_unregister_conn()
229 list_add(&buf_desc->list, buf_list); in smc_lgr_buf_list_add()
231 lgr->alloc_rmbs += buf_desc->len; in smc_lgr_buf_list_add()
232 lgr->alloc_rmbs += in smc_lgr_buf_list_add()
233 lgr->is_smcd ? sizeof(struct smcd_cdc_msg) : 0; in smc_lgr_buf_list_add()
235 lgr->alloc_sndbufs += buf_desc->len; in smc_lgr_buf_list_add()
243 list_del(&buf_desc->list); in smc_lgr_buf_list_del()
245 lgr->alloc_rmbs -= buf_desc->len; in smc_lgr_buf_list_del()
246 lgr->alloc_rmbs -= in smc_lgr_buf_list_del()
247 lgr->is_smcd ? sizeof(struct smcd_cdc_msg) : 0; in smc_lgr_buf_list_del()
249 lgr->alloc_sndbufs -= buf_desc->len; in smc_lgr_buf_list_del()
263 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, in smc_nl_get_sys_info()
268 if (cb_ctx->pos[0]) in smc_nl_get_sys_info()
297 cb_ctx->pos[0] = 1; in smc_nl_get_sys_info()
298 return skb->len; in smc_nl_get_sys_info()
305 return skb->len; in smc_nl_get_sys_info()
317 if (nla_put_u8(skb, SMC_NLA_LGR_V2_VER, lgr->smc_version)) in smc_nl_fill_lgr_v2_common()
319 if (nla_put_u8(skb, SMC_NLA_LGR_V2_REL, lgr->peer_smc_release)) in smc_nl_fill_lgr_v2_common()
321 if (nla_put_u8(skb, SMC_NLA_LGR_V2_OS, lgr->peer_os)) in smc_nl_fill_lgr_v2_common()
323 memcpy(smc_host, lgr->peer_hostname, SMC_MAX_HOSTNAME_LEN); in smc_nl_fill_lgr_v2_common()
327 memcpy(smc_eid, lgr->negotiated_eid, SMC_MAX_EID_LEN); in smc_nl_fill_lgr_v2_common()
337 return -EMSGSIZE; in smc_nl_fill_lgr_v2_common()
349 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_DIRECT, !lgr->uses_gateway)) in smc_nl_fill_smcr_lgr_v2()
351 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_MAX_CONNS, lgr->max_conns)) in smc_nl_fill_smcr_lgr_v2()
353 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_MAX_LINKS, lgr->max_links)) in smc_nl_fill_smcr_lgr_v2()
362 return -EMSGSIZE; in smc_nl_fill_smcr_lgr_v2()
376 if (nla_put_u32(skb, SMC_NLA_LGR_R_ID, *((u32 *)&lgr->id))) in smc_nl_fill_lgr()
378 if (nla_put_u32(skb, SMC_NLA_LGR_R_CONNS_NUM, lgr->conns_num)) in smc_nl_fill_lgr()
380 if (nla_put_u8(skb, SMC_NLA_LGR_R_ROLE, lgr->role)) in smc_nl_fill_lgr()
382 if (nla_put_u8(skb, SMC_NLA_LGR_R_TYPE, lgr->type)) in smc_nl_fill_lgr()
384 if (nla_put_u8(skb, SMC_NLA_LGR_R_BUF_TYPE, lgr->buf_type)) in smc_nl_fill_lgr()
386 if (nla_put_u8(skb, SMC_NLA_LGR_R_VLAN_ID, lgr->vlan_id)) in smc_nl_fill_lgr()
389 lgr->net->net_cookie, SMC_NLA_LGR_R_PAD)) in smc_nl_fill_lgr()
391 memcpy(smc_target, lgr->pnet_id, SMC_MAX_PNETID_LEN); in smc_nl_fill_lgr()
395 if (nla_put_uint(skb, SMC_NLA_LGR_R_SNDBUF_ALLOC, lgr->alloc_sndbufs)) in smc_nl_fill_lgr()
397 if (nla_put_uint(skb, SMC_NLA_LGR_R_RMB_ALLOC, lgr->alloc_rmbs)) in smc_nl_fill_lgr()
399 if (lgr->smc_version > SMC_V1) { in smc_nl_fill_lgr()
414 return -EMSGSIZE; in smc_nl_fill_lgr()
428 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, in smc_nl_fill_lgr_link()
438 if (nla_put_u8(skb, SMC_NLA_LINK_ID, link->link_id)) in smc_nl_fill_lgr_link()
440 if (nla_put_u32(skb, SMC_NLA_LINK_STATE, link->state)) in smc_nl_fill_lgr_link()
443 atomic_read(&link->conn_cnt))) in smc_nl_fill_lgr_link()
445 if (nla_put_u8(skb, SMC_NLA_LINK_IB_PORT, link->ibport)) in smc_nl_fill_lgr_link()
447 if (nla_put_u32(skb, SMC_NLA_LINK_NET_DEV, link->ndev_ifidx)) in smc_nl_fill_lgr_link()
449 snprintf(smc_ibname, sizeof(smc_ibname), "%s", link->ibname); in smc_nl_fill_lgr_link()
452 memcpy(&link_uid, link->link_uid, sizeof(link_uid)); in smc_nl_fill_lgr_link()
455 memcpy(&link_uid, link->peer_link_uid, sizeof(link_uid)); in smc_nl_fill_lgr_link()
459 smc_gid_be16_convert(smc_gid_target, link->gid); in smc_nl_fill_lgr_link()
463 smc_gid_be16_convert(smc_gid_target, link->peer_gid); in smc_nl_fill_lgr_link()
475 return -EMSGSIZE; in smc_nl_fill_lgr_link()
486 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, in smc_nl_handle_lgr()
498 if (!smc_link_usable(&lgr->lnk[i])) in smc_nl_handle_lgr()
500 if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb)) in smc_nl_handle_lgr()
509 return -EMSGSIZE; in smc_nl_handle_lgr()
519 int snum = cb_ctx->pos[0]; in smc_nl_fill_lgr_list()
522 spin_lock_bh(&smc_lgr->lock); in smc_nl_fill_lgr_list()
523 list_for_each_entry(lgr, &smc_lgr->list, list) { in smc_nl_fill_lgr_list()
532 spin_unlock_bh(&smc_lgr->lock); in smc_nl_fill_lgr_list()
533 cb_ctx->pos[0] = num; in smc_nl_fill_lgr_list()
541 struct smcd_dev *smcd = lgr->smcd; in smc_nl_fill_smcd_lgr()
546 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, in smc_nl_fill_smcd_lgr()
556 if (nla_put_u32(skb, SMC_NLA_LGR_D_ID, *((u32 *)&lgr->id))) in smc_nl_fill_smcd_lgr()
558 smcd->ops->get_local_gid(smcd, &smcd_gid); in smc_nl_fill_smcd_lgr()
565 if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_GID, lgr->peer_gid.gid, in smc_nl_fill_smcd_lgr()
569 lgr->peer_gid.gid_ext, SMC_NLA_LGR_D_PAD)) in smc_nl_fill_smcd_lgr()
571 if (nla_put_u8(skb, SMC_NLA_LGR_D_VLAN_ID, lgr->vlan_id)) in smc_nl_fill_smcd_lgr()
573 if (nla_put_u32(skb, SMC_NLA_LGR_D_CONNS_NUM, lgr->conns_num)) in smc_nl_fill_smcd_lgr()
575 if (nla_put_u32(skb, SMC_NLA_LGR_D_CHID, smc_ism_get_chid(lgr->smcd))) in smc_nl_fill_smcd_lgr()
577 if (nla_put_uint(skb, SMC_NLA_LGR_D_SNDBUF_ALLOC, lgr->alloc_sndbufs)) in smc_nl_fill_smcd_lgr()
579 if (nla_put_uint(skb, SMC_NLA_LGR_D_DMB_ALLOC, lgr->alloc_rmbs)) in smc_nl_fill_smcd_lgr()
581 memcpy(smc_pnet, lgr->smcd->pnetid, SMC_MAX_PNETID_LEN); in smc_nl_fill_smcd_lgr()
585 if (lgr->smc_version > SMC_V1) { in smc_nl_fill_smcd_lgr()
603 return -EMSGSIZE; in smc_nl_fill_smcd_lgr()
612 int snum = cb_ctx->pos[1]; in smc_nl_handle_smcd_lgr()
615 spin_lock_bh(&dev->lgr_lock); in smc_nl_handle_smcd_lgr()
616 list_for_each_entry(lgr, &dev->lgr_list, list) { in smc_nl_handle_smcd_lgr()
617 if (!lgr->is_smcd) in smc_nl_handle_smcd_lgr()
628 spin_unlock_bh(&dev->lgr_lock); in smc_nl_handle_smcd_lgr()
629 cb_ctx->pos[1] = num; in smc_nl_handle_smcd_lgr()
639 int snum = cb_ctx->pos[0]; in smc_nl_fill_smcd_dev()
642 mutex_lock(&dev_list->mutex); in smc_nl_fill_smcd_dev()
643 list_for_each_entry(smcd_dev, &dev_list->list, list) { in smc_nl_fill_smcd_dev()
644 if (list_empty(&smcd_dev->lgr_list)) in smc_nl_fill_smcd_dev()
655 mutex_unlock(&dev_list->mutex); in smc_nl_fill_smcd_dev()
656 cb_ctx->pos[0] = num; in smc_nl_fill_smcd_dev()
665 return skb->len; in smcr_nl_get_lgr()
673 return skb->len; in smcr_nl_get_link()
679 return skb->len; in smcd_nl_get_lgr()
692 if (!list_empty(&lgr->list)) in smc_lgr_cleanup_early()
693 list_del_init(&lgr->list); in smc_lgr_cleanup_early()
703 struct smc_link *lnk = &lgr->lnk[i]; in smcr_lgr_link_deactivate_all()
706 lnk->state = SMC_LNK_INACTIVE; in smcr_lgr_link_deactivate_all()
708 wake_up_all(&lgr->llc_msg_waiter); in smcr_lgr_link_deactivate_all()
709 wake_up_all(&lgr->llc_flow_waiter); in smcr_lgr_link_deactivate_all()
724 if (lgr->freeing) { in smc_lgr_free_work()
728 read_lock_bh(&lgr->conns_lock); in smc_lgr_free_work()
729 conns = RB_EMPTY_ROOT(&lgr->conns_all); in smc_lgr_free_work()
730 read_unlock_bh(&lgr->conns_lock); in smc_lgr_free_work()
735 list_del_init(&lgr->list); /* remove from smc_lgr_list */ in smc_lgr_free_work()
736 lgr->freeing = 1; /* this instance does the freeing, no new schedule */ in smc_lgr_free_work()
738 cancel_delayed_work(&lgr->free_work); in smc_lgr_free_work()
740 if (!lgr->is_smcd && !lgr->terminating) in smc_lgr_free_work()
743 if (lgr->is_smcd && !lgr->terminating) in smc_lgr_free_work()
745 if (!lgr->is_smcd) in smc_lgr_free_work()
766 link_id = ++lgr->next_link_id; in smcr_next_link_id()
768 link_id = ++lgr->next_link_id; in smcr_next_link_id()
770 if (smc_link_usable(&lgr->lnk[i]) && in smcr_next_link_id()
771 lgr->lnk[i].link_id == link_id) in smcr_next_link_id()
781 struct smc_ib_device *smcibdev = link->smcibdev; in smcr_copy_dev_info_to_link()
783 snprintf(link->ibname, sizeof(link->ibname), "%s", in smcr_copy_dev_info_to_link()
784 smcibdev->ibdev->name); in smcr_copy_dev_info_to_link()
785 link->ndev_ifidx = smcibdev->ndev_ifidx[link->ibport - 1]; in smcr_copy_dev_info_to_link()
795 if (lgr->smc_version == SMC_V2) { in smcr_link_init()
796 lnk->smcibdev = ini->smcrv2.ib_dev_v2; in smcr_link_init()
797 lnk->ibport = ini->smcrv2.ib_port_v2; in smcr_link_init()
799 lnk->smcibdev = ini->ib_dev; in smcr_link_init()
800 lnk->ibport = ini->ib_port; in smcr_link_init()
802 get_device(&lnk->smcibdev->ibdev->dev); in smcr_link_init()
803 atomic_inc(&lnk->smcibdev->lnk_cnt); in smcr_link_init()
804 refcount_set(&lnk->refcnt, 1); /* link refcnt is set to 1 */ in smcr_link_init()
805 lnk->clearing = 0; in smcr_link_init()
806 lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu; in smcr_link_init()
807 lnk->link_id = smcr_next_link_id(lgr); in smcr_link_init()
808 lnk->lgr = lgr; in smcr_link_init()
810 lnk->link_idx = link_idx; in smcr_link_init()
811 lnk->wr_rx_id_compl = 0; in smcr_link_init()
814 atomic_set(&lnk->conn_cnt, 0); in smcr_link_init()
816 INIT_WORK(&lnk->link_down_wrk, smc_link_down_work); in smcr_link_init()
817 if (!lnk->smcibdev->initialized) { in smcr_link_init()
818 rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev); in smcr_link_init()
823 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + in smcr_link_init()
825 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport, in smcr_link_init()
826 ini->vlan_id, lnk->gid, &lnk->sgid_index, in smcr_link_init()
827 lgr->smc_version == SMC_V2 ? in smcr_link_init()
828 &ini->smcrv2 : NULL); in smcr_link_init()
846 lnk->state = SMC_LNK_ACTIVATING; in smcr_link_init()
859 put_device(&lnk->smcibdev->ibdev->dev); in smcr_link_init()
860 smcibdev = lnk->smcibdev; in smcr_link_init()
862 lnk->state = SMC_LNK_UNUSED; in smcr_link_init()
863 if (!atomic_dec_return(&smcibdev->lnk_cnt)) in smcr_link_init()
864 wake_up(&smcibdev->lnks_deleted); in smcr_link_init()
881 if (ini->is_smcd && ini->vlan_id) { in smc_lgr_create()
882 if (smc_ism_get_vlan(ini->ism_dev[ini->ism_selected], in smc_lgr_create()
883 ini->vlan_id)) { in smc_lgr_create()
894 lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", 0, 0, in smc_lgr_create()
895 SMC_LGR_ID_SIZE, &lgr->id); in smc_lgr_create()
896 if (!lgr->tx_wq) { in smc_lgr_create()
897 rc = -ENOMEM; in smc_lgr_create()
900 lgr->is_smcd = ini->is_smcd; in smc_lgr_create()
901 lgr->sync_err = 0; in smc_lgr_create()
902 lgr->terminating = 0; in smc_lgr_create()
903 lgr->freeing = 0; in smc_lgr_create()
904 lgr->vlan_id = ini->vlan_id; in smc_lgr_create()
905 refcount_set(&lgr->refcnt, 1); /* set lgr refcnt to 1 */ in smc_lgr_create()
906 init_rwsem(&lgr->sndbufs_lock); in smc_lgr_create()
907 init_rwsem(&lgr->rmbs_lock); in smc_lgr_create()
908 rwlock_init(&lgr->conns_lock); in smc_lgr_create()
910 INIT_LIST_HEAD(&lgr->sndbufs[i]); in smc_lgr_create()
911 INIT_LIST_HEAD(&lgr->rmbs[i]); in smc_lgr_create()
913 lgr->next_link_id = 0; in smc_lgr_create()
915 memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE); in smc_lgr_create()
916 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); in smc_lgr_create()
917 INIT_WORK(&lgr->terminate_work, smc_lgr_terminate_work); in smc_lgr_create()
918 lgr->conns_all = RB_ROOT; in smc_lgr_create()
919 if (ini->is_smcd) { in smc_lgr_create()
920 /* SMC-D specific settings */ in smc_lgr_create()
921 smcd = ini->ism_dev[ini->ism_selected]; in smc_lgr_create()
922 get_device(smcd->ops->get_dev(smcd)); in smc_lgr_create()
923 lgr->peer_gid.gid = in smc_lgr_create()
924 ini->ism_peer_gid[ini->ism_selected].gid; in smc_lgr_create()
925 lgr->peer_gid.gid_ext = in smc_lgr_create()
926 ini->ism_peer_gid[ini->ism_selected].gid_ext; in smc_lgr_create()
927 lgr->smcd = ini->ism_dev[ini->ism_selected]; in smc_lgr_create()
928 lgr_list = &ini->ism_dev[ini->ism_selected]->lgr_list; in smc_lgr_create()
929 lgr_lock = &lgr->smcd->lgr_lock; in smc_lgr_create()
930 lgr->smc_version = ini->smcd_version; in smc_lgr_create()
931 lgr->peer_shutdown = 0; in smc_lgr_create()
932 atomic_inc(&ini->ism_dev[ini->ism_selected]->lgr_cnt); in smc_lgr_create()
934 /* SMC-R specific settings */ in smc_lgr_create()
938 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; in smc_lgr_create()
939 lgr->smc_version = ini->smcr_version; in smc_lgr_create()
940 memcpy(lgr->peer_systemid, ini->peer_systemid, in smc_lgr_create()
942 if (lgr->smc_version == SMC_V2) { in smc_lgr_create()
943 ibdev = ini->smcrv2.ib_dev_v2; in smc_lgr_create()
944 ibport = ini->smcrv2.ib_port_v2; in smc_lgr_create()
945 lgr->saddr = ini->smcrv2.saddr; in smc_lgr_create()
946 lgr->uses_gateway = ini->smcrv2.uses_gateway; in smc_lgr_create()
947 memcpy(lgr->nexthop_mac, ini->smcrv2.nexthop_mac, in smc_lgr_create()
949 lgr->max_conns = ini->max_conns; in smc_lgr_create()
950 lgr->max_links = ini->max_links; in smc_lgr_create()
952 ibdev = ini->ib_dev; in smc_lgr_create()
953 ibport = ini->ib_port; in smc_lgr_create()
954 lgr->max_conns = SMC_CONN_PER_LGR_MAX; in smc_lgr_create()
955 lgr->max_links = SMC_LINKS_ADD_LNK_MAX; in smc_lgr_create()
957 memcpy(lgr->pnet_id, ibdev->pnetid[ibport - 1], in smc_lgr_create()
965 lnk = &lgr->lnk[link_idx]; in smc_lgr_create()
971 lgr->net = smc_ib_net(lnk->smcibdev); in smc_lgr_create()
974 lgr->buf_type = lgr->net->smc.sysctl_smcr_buf_type; in smc_lgr_create()
977 smc->conn.lgr = lgr; in smc_lgr_create()
979 list_add_tail(&lgr->list, lgr_list); in smc_lgr_create()
984 destroy_workqueue(lgr->tx_wq); in smc_lgr_create()
988 if (ini->is_smcd && ini->vlan_id) in smc_lgr_create()
989 smc_ism_put_vlan(ini->ism_dev[ini->ism_selected], ini->vlan_id); in smc_lgr_create()
992 if (rc == -ENOMEM) in smc_lgr_create()
1002 int buffer_len = conn->peer_rmbe_size; in smc_write_space()
1007 smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn); in smc_write_space()
1008 smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn); in smc_write_space()
1010 space = buffer_len - smc_curs_diff(buffer_len, &cons, &prod); in smc_write_space()
1017 struct smc_connection *conn = &smc->conn; in smc_switch_cursor()
1022 smc_curs_copy(&conn->tx_curs_sent, &conn->tx_curs_fin, conn); in smc_switch_cursor()
1023 smc_curs_copy(&fin, &conn->local_tx_ctrl_fin, conn); in smc_switch_cursor()
1025 smc_curs_copy(&conn->local_tx_ctrl.prod, &fin, conn); in smc_switch_cursor()
1026 smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn); in smc_switch_cursor()
1028 if (smc_curs_comp(conn->peer_rmbe_size, &cons, &fin) < 0) { in smc_switch_cursor()
1030 * fin above, so now prod is smaller than cons. Fix that. in smc_switch_cursor()
1032 diff = smc_curs_diff(conn->peer_rmbe_size, &fin, &cons); in smc_switch_cursor()
1033 smc_curs_add(conn->sndbuf_desc->len, in smc_switch_cursor()
1034 &conn->tx_curs_sent, diff); in smc_switch_cursor()
1035 smc_curs_add(conn->sndbuf_desc->len, in smc_switch_cursor()
1036 &conn->tx_curs_fin, diff); in smc_switch_cursor()
1039 atomic_add(diff, &conn->sndbuf_space); in smc_switch_cursor()
1042 smc_curs_add(conn->peer_rmbe_size, in smc_switch_cursor()
1043 &conn->local_tx_ctrl.prod, diff); in smc_switch_cursor()
1044 smc_curs_add(conn->peer_rmbe_size, in smc_switch_cursor()
1045 &conn->local_tx_ctrl_fin, diff); in smc_switch_cursor()
1048 atomic_set(&smc->conn.peer_rmbe_space, smc_write_space(conn)); in smc_switch_cursor()
1050 if (smc->sk.sk_state != SMC_INIT && in smc_switch_cursor()
1051 smc->sk.sk_state != SMC_CLOSED) { in smc_switch_cursor()
1054 queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work, 0); in smc_switch_cursor()
1055 smc->sk.sk_data_ready(&smc->sk); in smc_switch_cursor()
1058 smc_wr_tx_put_slot(conn->lnk, in smc_switch_cursor()
1067 atomic_dec(&conn->lnk->conn_cnt); in smc_switch_link_and_count()
1069 smcr_link_put(conn->lnk); in smc_switch_link_and_count()
1070 conn->lnk = to_lnk; in smc_switch_link_and_count()
1071 atomic_inc(&conn->lnk->conn_cnt); in smc_switch_link_and_count()
1073 smcr_link_hold(conn->lnk); in smc_switch_link_and_count()
1087 /* link is inactive, wake up tx waiters */ in smc_switch_conns()
1091 if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx) in smc_switch_conns()
1093 if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev && in smc_switch_conns()
1094 from_lnk->ibport == lgr->lnk[i].ibport) { in smc_switch_conns()
1097 to_lnk = &lgr->lnk[i]; in smc_switch_conns()
1105 read_lock_bh(&lgr->conns_lock); in smc_switch_conns()
1106 for (node = rb_first(&lgr->conns_all); node; node = rb_next(node)) { in smc_switch_conns()
1108 if (conn->lnk != from_lnk) in smc_switch_conns()
1111 /* conn->lnk not yet set in SMC_INIT state */ in smc_switch_conns()
1112 if (smc->sk.sk_state == SMC_INIT) in smc_switch_conns()
1114 if (smc->sk.sk_state == SMC_CLOSED || in smc_switch_conns()
1115 smc->sk.sk_state == SMC_PEERCLOSEWAIT1 || in smc_switch_conns()
1116 smc->sk.sk_state == SMC_PEERCLOSEWAIT2 || in smc_switch_conns()
1117 smc->sk.sk_state == SMC_APPFINCLOSEWAIT || in smc_switch_conns()
1118 smc->sk.sk_state == SMC_APPCLOSEWAIT1 || in smc_switch_conns()
1119 smc->sk.sk_state == SMC_APPCLOSEWAIT2 || in smc_switch_conns()
1120 smc->sk.sk_state == SMC_PEERFINCLOSEWAIT || in smc_switch_conns()
1121 smc->sk.sk_state == SMC_PEERABORTWAIT || in smc_switch_conns()
1122 smc->sk.sk_state == SMC_PROCESSABORT) { in smc_switch_conns()
1123 spin_lock_bh(&conn->send_lock); in smc_switch_conns()
1125 spin_unlock_bh(&conn->send_lock); in smc_switch_conns()
1128 sock_hold(&smc->sk); in smc_switch_conns()
1129 read_unlock_bh(&lgr->conns_lock); in smc_switch_conns()
1130 /* pre-fetch buffer outside of send_lock, might sleep */ in smc_switch_conns()
1135 spin_lock_bh(&conn->send_lock); in smc_switch_conns()
1138 spin_unlock_bh(&conn->send_lock); in smc_switch_conns()
1139 sock_put(&smc->sk); in smc_switch_conns()
1144 read_unlock_bh(&lgr->conns_lock); in smc_switch_conns()
1160 if (is_rmb && buf_desc->is_conf_rkey && !list_empty(&lgr->list)) { in smcr_buf_unuse()
1165 down_read(&lgr->llc_conf_mutex); in smcr_buf_unuse()
1167 buf_desc->is_conf_rkey = false; in smcr_buf_unuse()
1168 up_read(&lgr->llc_conf_mutex); in smcr_buf_unuse()
1169 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); in smcr_buf_unuse()
1173 if (buf_desc->is_reg_err) { in smcr_buf_unuse()
1175 lock = is_rmb ? &lgr->rmbs_lock : in smcr_buf_unuse()
1176 &lgr->sndbufs_lock; in smcr_buf_unuse()
1184 memzero_explicit(buf_desc->cpu_addr, buf_desc->len); in smcr_buf_unuse()
1185 WRITE_ONCE(buf_desc->used, 0); in smcr_buf_unuse()
1191 struct smcd_dev *smcd = conn->lgr->smcd; in smcd_buf_detach()
1192 u64 peer_token = conn->peer_token; in smcd_buf_detach()
1194 if (!conn->sndbuf_desc) in smcd_buf_detach()
1199 kfree(conn->sndbuf_desc); in smcd_buf_detach()
1200 conn->sndbuf_desc = NULL; in smcd_buf_detach()
1207 bool is_smcd = lgr->is_smcd; in smc_buf_unuse()
1210 if (conn->sndbuf_desc) { in smc_buf_unuse()
1211 bufsize = conn->sndbuf_desc->len; in smc_buf_unuse()
1212 if (!is_smcd && conn->sndbuf_desc->is_vm) { in smc_buf_unuse()
1213 smcr_buf_unuse(conn->sndbuf_desc, false, lgr); in smc_buf_unuse()
1215 memzero_explicit(conn->sndbuf_desc->cpu_addr, bufsize); in smc_buf_unuse()
1216 WRITE_ONCE(conn->sndbuf_desc->used, 0); in smc_buf_unuse()
1220 if (conn->rmb_desc) { in smc_buf_unuse()
1221 bufsize = conn->rmb_desc->len; in smc_buf_unuse()
1223 smcr_buf_unuse(conn->rmb_desc, true, lgr); in smc_buf_unuse()
1226 memzero_explicit(conn->rmb_desc->cpu_addr, bufsize); in smc_buf_unuse()
1227 WRITE_ONCE(conn->rmb_desc->used, 0); in smc_buf_unuse()
1236 struct smc_link_group *lgr = conn->lgr; in smc_conn_free()
1238 if (!lgr || conn->freed) in smc_conn_free()
1244 conn->freed = 1; in smc_conn_free()
1251 if (lgr->is_smcd) { in smc_conn_free()
1252 if (!list_empty(&lgr->list)) in smc_conn_free()
1254 if (smc_ism_support_dmb_nocopy(lgr->smcd)) in smc_conn_free()
1256 tasklet_kill(&conn->rx_tsklet); in smc_conn_free()
1259 if (current_work() != &conn->abort_work) in smc_conn_free()
1260 cancel_work_sync(&conn->abort_work); in smc_conn_free()
1262 if (!list_empty(&lgr->list)) { in smc_conn_free()
1267 if (!lgr->conns_num) in smc_conn_free()
1270 if (!lgr->is_smcd) in smc_conn_free()
1271 smcr_link_put(conn->lnk); /* link_hold in smc_conn_create() */ in smc_conn_free()
1279 if (is_rmb || buf_desc->is_vm) in smcr_buf_unmap_link()
1280 buf_desc->is_reg_mr[lnk->link_idx] = false; in smcr_buf_unmap_link()
1281 if (!buf_desc->is_map_ib[lnk->link_idx]) in smcr_buf_unmap_link()
1284 if ((is_rmb || buf_desc->is_vm) && in smcr_buf_unmap_link()
1285 buf_desc->mr[lnk->link_idx]) { in smcr_buf_unmap_link()
1286 smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]); in smcr_buf_unmap_link()
1287 buf_desc->mr[lnk->link_idx] = NULL; in smcr_buf_unmap_link()
1294 sg_free_table(&buf_desc->sgt[lnk->link_idx]); in smcr_buf_unmap_link()
1295 buf_desc->is_map_ib[lnk->link_idx] = false; in smcr_buf_unmap_link()
1301 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_unmap_lgr()
1306 down_write(&lgr->rmbs_lock); in smcr_buf_unmap_lgr()
1307 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) in smcr_buf_unmap_lgr()
1309 up_write(&lgr->rmbs_lock); in smcr_buf_unmap_lgr()
1311 down_write(&lgr->sndbufs_lock); in smcr_buf_unmap_lgr()
1312 list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], in smcr_buf_unmap_lgr()
1315 up_write(&lgr->sndbufs_lock); in smcr_buf_unmap_lgr()
1321 struct smc_link_group *lgr = lnk->lgr; in smcr_rtoken_clear_link()
1325 lgr->rtokens[i][lnk->link_idx].rkey = 0; in smcr_rtoken_clear_link()
1326 lgr->rtokens[i][lnk->link_idx].dma_addr = 0; in smcr_rtoken_clear_link()
1332 struct smc_link_group *lgr = lnk->lgr; in __smcr_link_clear()
1337 put_device(&lnk->smcibdev->ibdev->dev); in __smcr_link_clear()
1338 smcibdev = lnk->smcibdev; in __smcr_link_clear()
1340 lnk->state = SMC_LNK_UNUSED; in __smcr_link_clear()
1341 if (!atomic_dec_return(&smcibdev->lnk_cnt)) in __smcr_link_clear()
1342 wake_up(&smcibdev->lnks_deleted); in __smcr_link_clear()
1346 /* must be called under lgr->llc_conf_mutex lock */
1349 if (!lnk->lgr || lnk->clearing || in smcr_link_clear()
1350 lnk->state == SMC_LNK_UNUSED) in smcr_link_clear()
1352 lnk->clearing = 1; in smcr_link_clear()
1353 lnk->peer_qpn = 0; in smcr_link_clear()
1366 refcount_inc(&lnk->refcnt); in smcr_link_hold()
1371 if (refcount_dec_and_test(&lnk->refcnt)) in smcr_link_put()
1381 smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]); in smcr_buf_free()
1383 if (!buf_desc->is_vm && buf_desc->pages) in smcr_buf_free()
1384 __free_pages(buf_desc->pages, buf_desc->order); in smcr_buf_free()
1385 else if (buf_desc->is_vm && buf_desc->cpu_addr) in smcr_buf_free()
1386 vfree(buf_desc->cpu_addr); in smcr_buf_free()
1395 buf_desc->len += sizeof(struct smcd_cdc_msg); in smcd_buf_free()
1396 smc_ism_unregister_dmb(lgr->smcd, buf_desc); in smcd_buf_free()
1398 kfree(buf_desc->cpu_addr); in smcd_buf_free()
1406 if (lgr->is_smcd) in smc_buf_free()
1420 buf_list = &lgr->rmbs[i]; in __smc_lgr_free_bufs()
1422 buf_list = &lgr->sndbufs[i]; in __smc_lgr_free_bufs()
1443 if (lgr->is_smcd) { in __smc_lgr_free()
1444 if (!atomic_dec_return(&lgr->smcd->lgr_cnt)) in __smc_lgr_free()
1445 wake_up(&lgr->smcd->lgrs_deleted); in __smc_lgr_free()
1459 if (!lgr->is_smcd) { in smc_lgr_free()
1460 down_write(&lgr->llc_conf_mutex); in smc_lgr_free()
1462 if (lgr->lnk[i].state != SMC_LNK_UNUSED) in smc_lgr_free()
1463 smcr_link_clear(&lgr->lnk[i], false); in smc_lgr_free()
1465 up_write(&lgr->llc_conf_mutex); in smc_lgr_free()
1469 destroy_workqueue(lgr->tx_wq); in smc_lgr_free()
1470 if (lgr->is_smcd) { in smc_lgr_free()
1471 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id); in smc_lgr_free()
1472 put_device(lgr->smcd->ops->get_dev(lgr->smcd)); in smc_lgr_free()
1479 refcount_inc(&lgr->refcnt); in smc_lgr_hold()
1484 if (refcount_dec_and_test(&lgr->refcnt)) in smc_lgr_put()
1490 smc->sk.sk_write_space(&smc->sk); in smc_sk_wake_ups()
1491 smc->sk.sk_data_ready(&smc->sk); in smc_sk_wake_ups()
1492 smc->sk.sk_state_change(&smc->sk); in smc_sk_wake_ups()
1500 if (conn->lgr->is_smcd && conn->lgr->peer_shutdown) in smc_conn_kill()
1501 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; in smc_conn_kill()
1504 conn->killed = 1; in smc_conn_kill()
1505 smc->sk.sk_err = ECONNABORTED; in smc_conn_kill()
1507 if (conn->lgr->is_smcd) { in smc_conn_kill()
1509 if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) in smc_conn_kill()
1512 tasklet_kill(&conn->rx_tsklet); in smc_conn_kill()
1514 tasklet_unlock_wait(&conn->rx_tsklet); in smc_conn_kill()
1524 if (lgr->is_smcd) { in smc_lgr_cleanup()
1527 u32 rsn = lgr->llc_termination_rsn; in smc_lgr_cleanup()
1546 if (lgr->terminating) in __smc_lgr_terminate()
1548 /* cancel free_work sync, will terminate when lgr->freeing is set */ in __smc_lgr_terminate()
1549 cancel_delayed_work(&lgr->free_work); in __smc_lgr_terminate()
1550 lgr->terminating = 1; in __smc_lgr_terminate()
1553 read_lock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1554 node = rb_first(&lgr->conns_all); in __smc_lgr_terminate()
1556 read_unlock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1559 sock_hold(&smc->sk); /* sock_put below */ in __smc_lgr_terminate()
1560 lock_sock(&smc->sk); in __smc_lgr_terminate()
1562 release_sock(&smc->sk); in __smc_lgr_terminate()
1563 sock_put(&smc->sk); /* sock_hold above */ in __smc_lgr_terminate()
1564 read_lock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1565 node = rb_first(&lgr->conns_all); in __smc_lgr_terminate()
1567 read_unlock_bh(&lgr->conns_lock); in __smc_lgr_terminate()
1572 /* unlink link group and schedule termination */
1579 if (list_empty(&lgr->list) || lgr->terminating || lgr->freeing) { in smc_lgr_terminate_sched()
1583 list_del_init(&lgr->list); in smc_lgr_terminate_sched()
1584 lgr->freeing = 1; in smc_lgr_terminate_sched()
1586 schedule_work(&lgr->terminate_work); in smc_lgr_terminate_sched()
1597 spin_lock_bh(&dev->lgr_lock); in smc_smcd_terminate()
1598 list_for_each_entry_safe(lgr, l, &dev->lgr_list, list) { in smc_smcd_terminate()
1599 if ((!peer_gid->gid || in smc_smcd_terminate()
1600 (lgr->peer_gid.gid == peer_gid->gid && in smc_smcd_terminate()
1602 lgr->peer_gid.gid_ext == peer_gid->gid_ext)) && in smc_smcd_terminate()
1603 (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) { in smc_smcd_terminate()
1604 if (peer_gid->gid) /* peer triggered termination */ in smc_smcd_terminate()
1605 lgr->peer_shutdown = 1; in smc_smcd_terminate()
1606 list_move(&lgr->list, &lgr_free_list); in smc_smcd_terminate()
1607 lgr->freeing = 1; in smc_smcd_terminate()
1610 spin_unlock_bh(&dev->lgr_lock); in smc_smcd_terminate()
1614 list_del_init(&lgr->list); in smc_smcd_terminate()
1615 schedule_work(&lgr->terminate_work); in smc_smcd_terminate()
1625 spin_lock_bh(&smcd->lgr_lock); in smc_smcd_terminate_all()
1626 list_splice_init(&smcd->lgr_list, &lgr_free_list); in smc_smcd_terminate_all()
1628 lgr->freeing = 1; in smc_smcd_terminate_all()
1629 spin_unlock_bh(&smcd->lgr_lock); in smc_smcd_terminate_all()
1632 list_del_init(&lgr->list); in smc_smcd_terminate_all()
1636 if (atomic_read(&smcd->lgr_cnt)) in smc_smcd_terminate_all()
1637 wait_event(smcd->lgrs_deleted, !atomic_read(&smcd->lgr_cnt)); in smc_smcd_terminate_all()
1654 lgr->freeing = 1; in smc_smcr_terminate_all()
1658 if (lgr->lnk[i].smcibdev == smcibdev) in smc_smcr_terminate_all()
1659 smcr_link_down_cond_sched(&lgr->lnk[i]); in smc_smcr_terminate_all()
1666 list_del_init(&lgr->list); in smc_smcr_terminate_all()
1672 if (atomic_read(&smcibdev->lnk_cnt)) in smc_smcr_terminate_all()
1673 wait_event(smcibdev->lnks_deleted, in smc_smcr_terminate_all()
1674 !atomic_read(&smcibdev->lnk_cnt)); in smc_smcr_terminate_all()
1688 if (smc_link_usable(&lgr->lnk[i])) in smcr_lgr_set_type()
1689 lgr->lnk[i].link_is_asym = false; in smcr_lgr_set_type()
1690 if (lgr->type == new_type) in smcr_lgr_set_type()
1692 lgr->type = new_type; in smcr_lgr_set_type()
1694 switch (lgr->type) { in smcr_lgr_set_type()
1711 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu state changed: " in smcr_lgr_set_type()
1712 "%s, pnetid %.16s\n", SMC_LGR_ID_SIZE, &lgr->id, in smcr_lgr_set_type()
1713 lgr->net->net_cookie, lgr_type, lgr->pnet_id); in smcr_lgr_set_type()
1721 lgr->lnk[asym_lnk_idx].link_is_asym = true; in smcr_lgr_set_type_asym()
1732 lock_sock(&smc->sk); in smc_conn_abort_work()
1734 release_sock(&smc->sk); in smc_conn_abort_work()
1735 sock_put(&smc->sk); /* sock_hold done by schedulers of abort_work */ in smc_conn_abort_work()
1746 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, in smcr_port_add()
1748 lgr->type == SMC_LGR_SYMMETRIC || in smcr_port_add()
1749 lgr->type == SMC_LGR_ASYMMETRIC_PEER || in smcr_port_add()
1750 !rdma_dev_access_netns(smcibdev->ibdev, lgr->net)) in smcr_port_add()
1753 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) in smcr_port_add()
1764 /* link is down - switch connections to alternate link,
1765 * must be called under lgr->llc_conf_mutex lock
1769 struct smc_link_group *lgr = lnk->lgr; in smcr_link_down()
1773 if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list)) in smcr_link_down()
1782 del_link_id = lnk->link_id; in smcr_link_down()
1784 if (lgr->role == SMC_SERV) { in smcr_link_down()
1788 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { in smcr_link_down()
1790 up_write(&lgr->llc_conf_mutex); in smcr_link_down()
1791 wait_event_timeout(lgr->llc_flow_waiter, in smcr_link_down()
1792 (list_empty(&lgr->list) || in smcr_link_down()
1793 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE), in smcr_link_down()
1795 down_write(&lgr->llc_conf_mutex); in smcr_link_down()
1797 if (!list_empty(&lgr->list)) { in smcr_link_down()
1803 wake_up(&lgr->llc_flow_waiter); /* wake up next waiter */ in smcr_link_down()
1807 /* must be called under lgr->llc_conf_mutex lock */
1810 if (smc_link_downing(&lnk->state)) { in smcr_link_down_cond()
1816 /* will get the lgr->llc_conf_mutex lock */
1819 if (smc_link_downing(&lnk->state)) { in smcr_link_down_cond_sched()
1821 schedule_work(&lnk->link_down_wrk); in smcr_link_down_cond_sched()
1831 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, in smcr_port_err()
1834 if (list_empty(&lgr->list)) in smcr_port_err()
1837 struct smc_link *lnk = &lgr->lnk[i]; in smcr_port_err()
1840 lnk->smcibdev == smcibdev && lnk->ibport == ibport) in smcr_port_err()
1850 struct smc_link_group *lgr = link->lgr; in smc_link_down_work()
1852 if (list_empty(&lgr->list)) in smc_link_down_work()
1854 wake_up_all(&lgr->llc_msg_waiter); in smc_link_down_work()
1855 down_write(&lgr->llc_conf_mutex); in smc_link_down_work()
1857 up_write(&lgr->llc_conf_mutex); in smc_link_down_work()
1863 unsigned short *vlan_id = (unsigned short *)priv->data; in smc_vlan_by_tcpsk_walk()
1876 struct dst_entry *dst = sk_dst_get(clcsock->sk); in smc_vlan_by_tcpsk()
1881 ini->vlan_id = 0; in smc_vlan_by_tcpsk()
1883 rc = -ENOTCONN; in smc_vlan_by_tcpsk()
1886 if (!dst->dev) { in smc_vlan_by_tcpsk()
1887 rc = -ENODEV; in smc_vlan_by_tcpsk()
1891 ndev = dst->dev; in smc_vlan_by_tcpsk()
1893 ini->vlan_id = vlan_dev_vlan_id(ndev); in smc_vlan_by_tcpsk()
1897 priv.data = (void *)&ini->vlan_id; in smc_vlan_by_tcpsk()
1918 if (memcmp(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN) || in smcr_lgr_match()
1919 lgr->role != role) in smcr_lgr_match()
1923 lnk = &lgr->lnk[i]; in smcr_lgr_match()
1927 /* use verbs API to check netns, instead of lgr->net */ in smcr_lgr_match()
1928 if (!rdma_dev_access_netns(lnk->smcibdev->ibdev, net)) in smcr_lgr_match()
1930 if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) && in smcr_lgr_match()
1931 !memcmp(lnk->peer_gid, peer_gid, SMC_GID_SIZE) && in smcr_lgr_match()
1933 !memcmp(lnk->peer_mac, peer_mac_v1, ETH_ALEN))) in smcr_lgr_match()
1943 if (lgr->peer_gid.gid != peer_gid->gid || in smcd_lgr_match()
1944 lgr->smcd != smcismdev) in smcd_lgr_match()
1948 lgr->peer_gid.gid_ext != peer_gid->gid_ext) in smcd_lgr_match()
1957 struct smc_connection *conn = &smc->conn; in smc_conn_create()
1958 struct net *net = sock_net(&smc->sk); in smc_conn_create()
1965 lgr_list = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_list : in smc_conn_create()
1967 lgr_lock = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_lock : in smc_conn_create()
1969 ini->first_contact_local = 1; in smc_conn_create()
1970 role = smc->listen_smc ? SMC_SERV : SMC_CLNT; in smc_conn_create()
1971 if (role == SMC_CLNT && ini->first_contact_peer) in smc_conn_create()
1978 write_lock_bh(&lgr->conns_lock); in smc_conn_create()
1979 if ((ini->is_smcd ? in smc_conn_create()
1980 smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected], in smc_conn_create()
1981 &ini->ism_peer_gid[ini->ism_selected]) : in smc_conn_create()
1982 smcr_lgr_match(lgr, ini->smcr_version, in smc_conn_create()
1983 ini->peer_systemid, in smc_conn_create()
1984 ini->peer_gid, ini->peer_mac, role, in smc_conn_create()
1985 ini->ib_clcqpn, net)) && in smc_conn_create()
1986 !lgr->sync_err && in smc_conn_create()
1987 (ini->smcd_version == SMC_V2 || in smc_conn_create()
1988 lgr->vlan_id == ini->vlan_id) && in smc_conn_create()
1989 (role == SMC_CLNT || ini->is_smcd || in smc_conn_create()
1990 (lgr->conns_num < lgr->max_conns && in smc_conn_create()
1991 !bitmap_full(lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX)))) { in smc_conn_create()
1993 ini->first_contact_local = 0; in smc_conn_create()
1994 conn->lgr = lgr; in smc_conn_create()
1996 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
1997 if (!rc && delayed_work_pending(&lgr->free_work)) in smc_conn_create()
1998 cancel_delayed_work(&lgr->free_work); in smc_conn_create()
2001 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
2007 if (role == SMC_CLNT && !ini->first_contact_peer && in smc_conn_create()
2008 ini->first_contact_local) { in smc_conn_create()
2017 if (ini->first_contact_local) { in smc_conn_create()
2021 lgr = conn->lgr; in smc_conn_create()
2022 write_lock_bh(&lgr->conns_lock); in smc_conn_create()
2024 write_unlock_bh(&lgr->conns_lock); in smc_conn_create()
2030 smc_lgr_hold(conn->lgr); /* lgr_put in smc_conn_free() */ in smc_conn_create()
2031 if (!conn->lgr->is_smcd) in smc_conn_create()
2032 smcr_link_hold(conn->lnk); /* link_put in smc_conn_free() */ in smc_conn_create()
2033 conn->freed = 0; in smc_conn_create()
2034 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; in smc_conn_create()
2035 conn->local_tx_ctrl.len = SMC_WR_TX_SIZE; in smc_conn_create()
2036 conn->urg_state = SMC_URG_READ; in smc_conn_create()
2037 init_waitqueue_head(&conn->cdc_pend_tx_wq); in smc_conn_create()
2038 INIT_WORK(&smc->conn.abort_work, smc_conn_abort_work); in smc_conn_create()
2039 if (ini->is_smcd) { in smc_conn_create()
2040 conn->rx_off = sizeof(struct smcd_cdc_msg); in smc_conn_create()
2043 conn->rx_off = 0; in smc_conn_create()
2046 spin_lock_init(&conn->acurs_lock); in smc_conn_create()
2053 #define SMCD_DMBE_SIZES 6 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */
2054 #define SMCR_RMBE_SIZES 15 /* 0 -> 16KB, 1 -> 32KB, .. 15 -> 512MB */
2068 size = (size - 1) >> 14; /* convert to 16K multiple */ in smc_compress_bufsize()
2101 if (cmpxchg(&buf_slot->used, 0, 1) == 0) { in smc_buf_get_slot()
2127 if (buf_desc->is_map_ib[lnk->link_idx]) in smcr_buf_map_link()
2130 if (buf_desc->is_vm) { in smcr_buf_map_link()
2131 buf = buf_desc->cpu_addr; in smcr_buf_map_link()
2132 buf_size = buf_desc->len; in smcr_buf_map_link()
2133 offset = offset_in_page(buf_desc->cpu_addr); in smcr_buf_map_link()
2139 rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL); in smcr_buf_map_link()
2143 if (buf_desc->is_vm) { in smcr_buf_map_link()
2145 for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) { in smcr_buf_map_link()
2146 size = min_t(int, PAGE_SIZE - offset, buf_size); in smcr_buf_map_link()
2149 buf_size -= size; in smcr_buf_map_link()
2154 sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl, in smcr_buf_map_link()
2155 buf_desc->cpu_addr, buf_desc->len); in smcr_buf_map_link()
2163 rc = -EAGAIN; in smcr_buf_map_link()
2167 buf_desc->is_dma_need_sync |= in smcr_buf_map_link()
2168 smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx; in smcr_buf_map_link()
2170 if (is_rmb || buf_desc->is_vm) { in smcr_buf_map_link()
2176 rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags, in smcr_buf_map_link()
2177 buf_desc, lnk->link_idx); in smcr_buf_map_link()
2183 buf_desc->is_map_ib[lnk->link_idx] = true; in smcr_buf_map_link()
2190 sg_free_table(&buf_desc->sgt[lnk->link_idx]); in smcr_buf_map_link()
2195 * must be called under lgr->llc_conf_mutex lock
2199 if (list_empty(&link->lgr->list)) in smcr_link_reg_buf()
2200 return -ENOLINK; in smcr_link_reg_buf()
2201 if (!buf_desc->is_reg_mr[link->link_idx]) { in smcr_link_reg_buf()
2203 if (buf_desc->is_vm) in smcr_link_reg_buf()
2204 buf_desc->mr[link->link_idx]->iova = in smcr_link_reg_buf()
2205 (uintptr_t)buf_desc->cpu_addr; in smcr_link_reg_buf()
2206 if (smc_wr_reg_send(link, buf_desc->mr[link->link_idx])) { in smcr_link_reg_buf()
2207 buf_desc->is_reg_err = true; in smcr_link_reg_buf()
2208 return -EFAULT; in smcr_link_reg_buf()
2210 buf_desc->is_reg_mr[link->link_idx] = true; in smcr_link_reg_buf()
2223 if (!buf_desc->used) in _smcr_buf_map_lgr()
2237 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_map_lgr()
2241 rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock, in smcr_buf_map_lgr()
2242 &lgr->rmbs[i], true); in smcr_buf_map_lgr()
2245 rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock, in smcr_buf_map_lgr()
2246 &lgr->sndbufs[i], false); in smcr_buf_map_lgr()
2254 * must be called under lgr->llc_conf_mutex lock
2258 struct smc_link_group *lgr = lnk->lgr; in smcr_buf_reg_lgr()
2263 down_write(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
2265 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) { in smcr_buf_reg_lgr()
2266 if (!buf_desc->used) in smcr_buf_reg_lgr()
2270 up_write(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
2275 up_write(&lgr->rmbs_lock); in smcr_buf_reg_lgr()
2277 if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) in smcr_buf_reg_lgr()
2281 down_write(&lgr->sndbufs_lock); in smcr_buf_reg_lgr()
2283 list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], list) { in smcr_buf_reg_lgr()
2284 if (!buf_desc->used || !buf_desc->is_vm) in smcr_buf_reg_lgr()
2288 up_write(&lgr->sndbufs_lock); in smcr_buf_reg_lgr()
2293 up_write(&lgr->sndbufs_lock); in smcr_buf_reg_lgr()
2305 return ERR_PTR(-ENOMEM); in smcr_new_buf_create()
2307 switch (lgr->buf_type) { in smcr_new_buf_create()
2310 buf_desc->order = get_order(bufsize); in smcr_new_buf_create()
2311 buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN | in smcr_new_buf_create()
2314 buf_desc->order); in smcr_new_buf_create()
2315 if (buf_desc->pages) { in smcr_new_buf_create()
2316 buf_desc->cpu_addr = in smcr_new_buf_create()
2317 (void *)page_address(buf_desc->pages); in smcr_new_buf_create()
2318 buf_desc->len = bufsize; in smcr_new_buf_create()
2319 buf_desc->is_vm = false; in smcr_new_buf_create()
2322 if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) in smcr_new_buf_create()
2326 buf_desc->order = get_order(bufsize); in smcr_new_buf_create()
2327 buf_desc->cpu_addr = vzalloc(PAGE_SIZE << buf_desc->order); in smcr_new_buf_create()
2328 if (!buf_desc->cpu_addr) in smcr_new_buf_create()
2330 buf_desc->pages = NULL; in smcr_new_buf_create()
2331 buf_desc->len = bufsize; in smcr_new_buf_create()
2332 buf_desc->is_vm = true; in smcr_new_buf_create()
2339 return ERR_PTR(-EAGAIN); in smcr_new_buf_create()
2351 down_read(&lgr->llc_conf_mutex); in smcr_buf_map_usable_links()
2353 struct smc_link *lnk = &lgr->lnk[i]; in smcr_buf_map_usable_links()
2358 rc = -ENOMEM; in smcr_buf_map_usable_links()
2364 up_read(&lgr->llc_conf_mutex); in smcr_buf_map_usable_links()
2366 rc = -EINVAL; in smcr_buf_map_usable_links()
2379 return ERR_PTR(-ENOMEM); in smcd_new_buf_create()
2384 if (rc == -ENOMEM) in smcd_new_buf_create()
2385 return ERR_PTR(-EAGAIN); in smcd_new_buf_create()
2386 if (rc == -ENOSPC) in smcd_new_buf_create()
2387 return ERR_PTR(-ENOSPC); in smcd_new_buf_create()
2388 return ERR_PTR(-EIO); in smcd_new_buf_create()
2390 buf_desc->pages = virt_to_page(buf_desc->cpu_addr); in smcd_new_buf_create()
2392 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg); in smcd_new_buf_create()
2394 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL | in smcd_new_buf_create()
2397 if (!buf_desc->cpu_addr) { in smcd_new_buf_create()
2399 return ERR_PTR(-EAGAIN); in smcd_new_buf_create()
2401 buf_desc->len = bufsize; in smcd_new_buf_create()
2408 struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM); in __smc_buf_create()
2409 struct smc_connection *conn = &smc->conn; in __smc_buf_create()
2410 struct smc_link_group *lgr = conn->lgr; in __smc_buf_create()
2418 bufsize = smc->sk.sk_rcvbuf / 2; in __smc_buf_create()
2421 bufsize = smc->sk.sk_sndbuf / 2; in __smc_buf_create()
2424 bufsize_comp >= 0; bufsize_comp--) { in __smc_buf_create()
2426 lock = &lgr->rmbs_lock; in __smc_buf_create()
2427 buf_list = &lgr->rmbs[bufsize_comp]; in __smc_buf_create()
2429 lock = &lgr->sndbufs_lock; in __smc_buf_create()
2430 buf_list = &lgr->sndbufs[bufsize_comp]; in __smc_buf_create()
2437 buf_desc->is_dma_need_sync = 0; in __smc_buf_create()
2448 if (PTR_ERR(buf_desc) == -ENOMEM) in __smc_buf_create()
2460 buf_desc->used = 1; in __smc_buf_create()
2473 return -ENOMEM; in __smc_buf_create()
2478 conn->rmb_desc = buf_desc; in __smc_buf_create()
2479 conn->rmbe_size_comp = bufsize_comp; in __smc_buf_create()
2480 smc->sk.sk_rcvbuf = bufsize * 2; in __smc_buf_create()
2481 atomic_set(&conn->bytes_to_rcv, 0); in __smc_buf_create()
2482 conn->rmbe_update_limit = in __smc_buf_create()
2483 smc_rmb_wnd_update_limit(buf_desc->len); in __smc_buf_create()
2487 conn->sndbuf_desc = buf_desc; in __smc_buf_create()
2488 smc->sk.sk_sndbuf = bufsize * 2; in __smc_buf_create()
2489 atomic_set(&conn->sndbuf_space, bufsize); in __smc_buf_create()
2496 if (!conn->sndbuf_desc->is_dma_need_sync) in smc_sndbuf_sync_sg_for_device()
2498 if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd || in smc_sndbuf_sync_sg_for_device()
2499 !smc_link_active(conn->lnk)) in smc_sndbuf_sync_sg_for_device()
2501 smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE); in smc_sndbuf_sync_sg_for_device()
2508 if (!conn->rmb_desc->is_dma_need_sync) in smc_rmb_sync_sg_for_cpu()
2510 if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd) in smc_rmb_sync_sg_for_cpu()
2513 if (!smc_link_active(&conn->lgr->lnk[i])) in smc_rmb_sync_sg_for_cpu()
2515 smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc, in smc_rmb_sync_sg_for_cpu()
2522 * (even though the SMC protocol allows more than one RMB-element per RMB,
2523 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
2532 smc_ism_support_dmb_nocopy(smc->conn.lgr->smcd)) in smc_buf_create()
2542 if (rc && smc->conn.sndbuf_desc) { in smc_buf_create()
2543 down_write(&smc->conn.lgr->sndbufs_lock); in smc_buf_create()
2544 smc_lgr_buf_list_del(smc->conn.lgr, false, in smc_buf_create()
2545 smc->conn.sndbuf_desc); in smc_buf_create()
2546 up_write(&smc->conn.lgr->sndbufs_lock); in smc_buf_create()
2547 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc); in smc_buf_create()
2548 smc->conn.sndbuf_desc = NULL; in smc_buf_create()
2555 struct smc_connection *conn = &smc->conn; in smcd_buf_attach()
2556 struct smcd_dev *smcd = conn->lgr->smcd; in smcd_buf_attach()
2557 u64 peer_token = conn->peer_token; in smcd_buf_attach()
2563 return -ENOMEM; in smcd_buf_attach()
2574 smc->sk.sk_sndbuf = buf_desc->len; in smcd_buf_attach()
2575 buf_desc->cpu_addr = in smcd_buf_attach()
2576 (u8 *)buf_desc->cpu_addr + sizeof(struct smcd_cdc_msg); in smcd_buf_attach()
2577 buf_desc->len -= sizeof(struct smcd_cdc_msg); in smcd_buf_attach()
2578 conn->sndbuf_desc = buf_desc; in smcd_buf_attach()
2579 conn->sndbuf_desc->used = 1; in smcd_buf_attach()
2580 atomic_set(&conn->sndbuf_space, conn->sndbuf_desc->len); in smcd_buf_attach()
2592 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { in smc_rmb_reserve_rtoken_idx()
2593 if (!test_and_set_bit(i, lgr->rtokens_used_mask)) in smc_rmb_reserve_rtoken_idx()
2596 return -ENOSPC; in smc_rmb_reserve_rtoken_idx()
2605 if (test_bit(i, lgr->rtokens_used_mask) && in smc_rtoken_find_by_link()
2606 lgr->rtokens[i][lnk_idx].rkey == rkey) in smc_rtoken_find_by_link()
2609 return -ENOENT; in smc_rtoken_find_by_link()
2619 if (rtok_idx == -ENOENT) in smc_rtoken_set()
2621 lgr->rtokens[rtok_idx][link_idx_new].rkey = ntohl(nw_rkey); in smc_rtoken_set()
2622 lgr->rtokens[rtok_idx][link_idx_new].dma_addr = be64_to_cpu(nw_vaddr); in smc_rtoken_set()
2635 if (lgr->lnk[link_idx].link_id == link_id) { in smc_rtoken_set2()
2642 lgr->rtokens[rtok_idx][link_idx].rkey = rkey; in smc_rtoken_set2()
2643 lgr->rtokens[rtok_idx][link_idx].dma_addr = dma_addr; in smc_rtoken_set2()
2655 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_add()
2656 lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr && in smc_rtoken_add()
2657 test_bit(i, lgr->rtokens_used_mask)) { in smc_rtoken_add()
2665 lgr->rtokens[i][lnk->link_idx].rkey = rkey; in smc_rtoken_add()
2666 lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr; in smc_rtoken_add()
2678 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && in smc_rtoken_delete()
2679 test_bit(i, lgr->rtokens_used_mask)) { in smc_rtoken_delete()
2681 lgr->rtokens[i][j].rkey = 0; in smc_rtoken_delete()
2682 lgr->rtokens[i][j].dma_addr = 0; in smc_rtoken_delete()
2684 clear_bit(i, lgr->rtokens_used_mask); in smc_rtoken_delete()
2688 return -ENOENT; in smc_rtoken_delete()
2696 conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr, in smc_rmb_rtoken_handling()
2697 clc->r0.rmb_rkey); in smc_rmb_rtoken_handling()
2698 if (conn->rtoken_idx < 0) in smc_rmb_rtoken_handling()
2699 return conn->rtoken_idx; in smc_rmb_rtoken_handling()
2713 set_bit(i, smcibdev->ports_going_away); in smc_core_going_away()
2719 smcd->going_away = 1; in smc_core_going_away()