15 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_query_device()  local
16 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_query_device()
17 	struct pci_dev *pcidev = iwdev->rf->pcidev;  in irdma_query_device()
25 			    iwdev->netdev->dev_addr);  in irdma_query_device()
75 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_query_port()  local
76 	struct net_device *netdev = iwdev->netdev;  in irdma_query_port()
106 	props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;  in irdma_query_port()
128 	pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] +  in irdma_mmap_legacy()
129 	       pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;  in irdma_mmap_legacy()
187 		ibdev_dbg(&ucontext->iwdev->ibdev,  in irdma_mmap()
194 	ibdev_dbg(&ucontext->iwdev->ibdev,  in irdma_mmap()
199 	      pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;  in irdma_mmap()
217 		ibdev_dbg(&ucontext->iwdev->ibdev,  in irdma_mmap()
233 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_alloc_push_page()  local
237 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_alloc_push_page()
249 	cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp;  in irdma_alloc_push_page()
252 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_alloc_push_page()
254 	    iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) {  in irdma_alloc_push_page()
259 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_alloc_push_page()
276 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_alloc_ucontext()  local
280 	struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;  in irdma_alloc_ucontext()
292 	ucontext->iwdev = iwdev;  in irdma_alloc_ucontext()
304 		uresp.max_qps = iwdev->rf->max_qp;  in irdma_alloc_ucontext()
305 		uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;  in irdma_alloc_ucontext()
306 		uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;  in irdma_alloc_ucontext()
312 		u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];  in irdma_alloc_ucontext()
350 	ibdev_err(&iwdev->ibdev,  in irdma_alloc_ucontext()
376 	struct irdma_device *iwdev = to_iwdev(pd->device);  in irdma_alloc_pd()  local
377 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_alloc_pd()
378 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_alloc_pd()
423 	struct irdma_device *iwdev = to_iwdev(ibpd->device);  in irdma_dealloc_pd()  local
425 	irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);  in irdma_dealloc_pd()
484 	struct irdma_device *iwdev = ucontext->iwdev;  in irdma_setup_push_mmap_entries()  local
488 	bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];  in irdma_setup_push_mmap_entries()
520 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_destroy_qp()  local
541 	irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp);  in irdma_destroy_qp()
555 static void irdma_setup_virt_qp(struct irdma_device *iwdev,  in irdma_setup_virt_qp()  argument
583 				struct irdma_device *iwdev,  in irdma_setup_umode_qp()  argument
598 		ibdev_dbg(&iwdev->ibdev, "VERBS: ib_copy_from_data fail\n");  in irdma_setup_umode_qp()
613 			ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n");  in irdma_setup_umode_qp()
650 	irdma_setup_virt_qp(iwdev, iwqp, info);  in irdma_setup_umode_qp()
662 static int irdma_setup_kmode_qp(struct irdma_device *iwdev,  in irdma_setup_kmode_qp()  argument
703 	mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size,  in irdma_setup_kmode_qp()
733 	struct irdma_pci_f *rf = iwqp->iwdev->rf;  in irdma_cqp_create_qp_cmd()
763 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_roce_fill_and_set_qpctx_info()  local
764 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_roce_fill_and_set_qpctx_info()
769 	udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu));  in irdma_roce_fill_and_set_qpctx_info()
770 	udp_info->cwnd = iwdev->roce_cwnd;  in irdma_roce_fill_and_set_qpctx_info()
776 	ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr);  in irdma_roce_fill_and_set_qpctx_info()
784 	roce_info->ack_credits = iwdev->roce_ackcreds;  in irdma_roce_fill_and_set_qpctx_info()
803 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_iw_fill_and_set_qpctx_info()  local
804 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_iw_fill_and_set_qpctx_info()
808 	ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr);  in irdma_iw_fill_and_set_qpctx_info()
831 				   struct irdma_device *iwdev)  in irdma_validate_qp_attrs()  argument
833 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_validate_qp_attrs()
846 	if (rdma_protocol_roce(&iwdev->ibdev, 1)) {  in irdma_validate_qp_attrs()
881 	struct irdma_device *iwdev = to_iwdev(ibpd->device);  in irdma_create_qp()  local
882 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_create_qp()
893 	err_code = irdma_validate_qp_attrs(init_attr, iwdev);  in irdma_create_qp()
901 	init_info.vsi = &iwdev->vsi;  in irdma_create_qp()
913 	iwqp->iwdev = iwdev;  in irdma_create_qp()
947 	if (!rdma_protocol_roce(&iwdev->ibdev, 1))  in irdma_create_qp()
955 		err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info,  in irdma_create_qp()
960 		err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr);  in irdma_create_qp()
964 		ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n");  in irdma_create_qp()
968 	if (rdma_protocol_roce(&iwdev->ibdev, 1)) {  in irdma_create_qp()
989 		ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n");  in irdma_create_qp()
997 	if (rdma_protocol_roce(&iwdev->ibdev, 1))  in irdma_create_qp()
1012 	if (rdma_protocol_roce(&iwdev->ibdev, 1)) {  in irdma_create_qp()
1013 		if (dev->ws_add(&iwdev->vsi, 0)) {  in irdma_create_qp()
1028 			if (rdma_protocol_iwarp(&iwdev->ibdev, 1))  in irdma_create_qp()
1039 			ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n");  in irdma_create_qp()
1163 	if (!wait_event_timeout(iwqp->iwdev->suspend_wq,  in irdma_wait_for_suspend()
1167 		ibdev_warn(&iwqp->iwdev->ibdev,  in irdma_wait_for_suspend()
1190 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_modify_qp_roce()  local
1191 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_modify_qp_roce()
1285 		if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)  in irdma_modify_qp_roce()
1329 			irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4,  in irdma_modify_qp_roce()
1335 			ibdev_err(&iwdev->ibdev,  in irdma_modify_qp_roce()
1348 			ibdev_err(&iwdev->ibdev,  in irdma_modify_qp_roce()
1369 	ibdev_dbg(&iwdev->ibdev,  in irdma_modify_qp_roce()
1378 			ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",  in irdma_modify_qp_roce()
1420 			if (iwdev->push_mode && udata &&  in irdma_modify_qp_roce()
1478 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))  in irdma_modify_qp_roce()
1519 				ibdev_dbg(&iwdev->ibdev,  in irdma_modify_qp_roce()
1546 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_modify_qp()  local
1547 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_modify_qp()
1573 	ibdev_dbg(&iwdev->ibdev,  in irdma_modify_qp()
1594 			if (iwdev->push_mode && udata &&  in irdma_modify_qp()
1671 			    iwdev->iw_status &&  in irdma_modify_qp()
1707 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))  in irdma_modify_qp()
1731 			spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);  in irdma_modify_qp()
1735 				spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);  in irdma_modify_qp()
1744 				spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);  in irdma_modify_qp()
1766 			ibdev_dbg(&iwdev->ibdev,  in irdma_modify_qp()
1822 				     struct irdma_device *iwdev,  in irdma_process_resize_list()  argument
1835 		queue_work(iwdev->cleanup_wq, &cq_buf->work);  in irdma_process_resize_list()
1849 	struct irdma_device *iwdev = to_iwdev(ib_cq->device);  in irdma_destroy_cq()  local
1861 		irdma_process_resize_list(iwcq, iwdev, NULL);  in irdma_destroy_cq()
1867 	irdma_cq_wq_destroy(iwdev->rf, cq);  in irdma_destroy_cq()
1872 	irdma_cq_free_rsrc(iwdev->rf, iwcq);  in irdma_destroy_cq()
1896 	struct irdma_device *iwdev;  in irdma_resize_cq()  local
1902 	iwdev = to_iwdev(ibcq->device);  in irdma_resize_cq()
1903 	rf = iwdev->rf;  in irdma_resize_cq()
2048 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_create_cq()  local
2049 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_create_cq()
2090 	info.vsi = &iwdev->vsi;  in irdma_create_cq()
2193 		ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n");  in irdma_create_cq()
2222 			ibdev_dbg(&iwdev->ibdev,  in irdma_create_cq()
2268 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag)  in irdma_free_stag()  argument
2272 	stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S;  in irdma_free_stag()
2273 	irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx);  in irdma_free_stag()
2280 static u32 irdma_create_stag(struct irdma_device *iwdev)  in irdma_create_stag()  argument
2293 	driver_key = random & ~iwdev->rf->mr_stagmask;  in irdma_create_stag()
2294 	next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8;  in irdma_create_stag()
2295 	next_stag_index %= iwdev->rf->max_mr;  in irdma_create_stag()
2297 	ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs,  in irdma_create_stag()
2298 			       iwdev->rf->max_mr, &stag_index,  in irdma_create_stag()
2458 static int irdma_handle_q_mem(struct irdma_device *iwdev,  in irdma_handle_q_mem()  argument
2473 	err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);  in irdma_handle_q_mem()
2521 		ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n");  in irdma_handle_q_mem()
2526 		irdma_free_pble(iwdev->rf->pble_rsrc, palloc);  in irdma_handle_q_mem()
2538 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)  in irdma_hw_alloc_mw()  argument
2546 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hw_alloc_mw()
2562 	cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev;  in irdma_hw_alloc_mw()
2564 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hw_alloc_mw()
2565 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hw_alloc_mw()
2577 	struct irdma_device *iwdev = to_iwdev(ibmw->device);  in irdma_alloc_mw()  local
2582 	stag = irdma_create_stag(iwdev);  in irdma_alloc_mw()
2589 	err_code = irdma_hw_alloc_mw(iwdev, iwmr);  in irdma_alloc_mw()
2591 		irdma_free_stag(iwdev, stag);  in irdma_alloc_mw()
2607 	struct irdma_device *iwdev = to_iwdev(ibmw->device);  in irdma_dealloc_mw()  local
2612 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_dealloc_mw()
2624 	cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;  in irdma_dealloc_mw()
2626 	irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_dealloc_mw()
2627 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_dealloc_mw()
2628 	irdma_free_stag(iwdev, iwmr->stag);  in irdma_dealloc_mw()
2638 static int irdma_hw_alloc_stag(struct irdma_device *iwdev,  in irdma_hw_alloc_stag()  argument
2648 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hw_alloc_stag()
2663 	cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev;  in irdma_hw_alloc_stag()
2665 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hw_alloc_stag()
2666 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hw_alloc_stag()
2683 	struct irdma_device *iwdev = to_iwdev(pd->device);  in irdma_alloc_mr()  local
2694 	stag = irdma_create_stag(iwdev);  in irdma_alloc_mr()
2712 	err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,  in irdma_alloc_mr()
2717 	err_code = irdma_hw_alloc_stag(iwdev, iwmr);  in irdma_alloc_mr()
2725 	irdma_free_pble(iwdev->rf->pble_rsrc, palloc);  in irdma_alloc_mr()
2727 	irdma_free_stag(iwdev, stag);  in irdma_alloc_mr()
2786 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,  in irdma_hwreg_mr()  argument
2798 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hwreg_mr()
2832 	cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev;  in irdma_hwreg_mr()
2834 	ret = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hwreg_mr()
2835 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hwreg_mr()
2846 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_reg_user_mr_type_mem()  local
2854 	err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);  in irdma_reg_user_mr_type_mem()
2862 			irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);  in irdma_reg_user_mr_type_mem()
2868 		stag = irdma_create_stag(iwdev);  in irdma_reg_user_mr_type_mem()
2879 	err = irdma_hwreg_mr(iwdev, iwmr, access);  in irdma_reg_user_mr_type_mem()
2887 		irdma_free_stag(iwdev, stag);  in irdma_reg_user_mr_type_mem()
2891 		irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);  in irdma_reg_user_mr_type_mem()
2900 	struct irdma_device *iwdev = to_iwdev(pd->device);  in irdma_alloc_iwmr()  local
2918 		iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K;  in irdma_alloc_iwmr()
2942 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_reg_user_mr_type_qp()  local
2951 	if (!rdma_protocol_roce(&iwdev->ibdev, 1) &&  in irdma_reg_user_mr_type_qp()
2961 	err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);  in irdma_reg_user_mr_type_qp()
2979 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_reg_user_mr_type_cq()  local
2988 	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)  in irdma_reg_user_mr_type_cq()
2995 	err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);  in irdma_reg_user_mr_type_cq()
3023 	struct irdma_device *iwdev = to_iwdev(pd->device);  in irdma_reg_user_mr()  local
3029 	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)  in irdma_reg_user_mr()
3038 		ibdev_dbg(&iwdev->ibdev,  in irdma_reg_user_mr()
3090 	struct irdma_device *iwdev = to_iwdev(pd->device);  in irdma_reg_user_mr_dmabuf()  local
3095 	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)  in irdma_reg_user_mr_dmabuf()
3101 		ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err);  in irdma_reg_user_mr_dmabuf()
3128 	struct irdma_device *iwdev = to_iwdev(ib_mr->device);  in irdma_hwdereg_mr()  local
3143 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hwdereg_mr()
3158 	cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;  in irdma_hwdereg_mr()
3160 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hwdereg_mr()
3161 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hwdereg_mr()
3182 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_rereg_mr_trans()  local
3196 				iwdev->rf->sc_dev.hw_attrs.page_size_cap,  in irdma_rereg_mr_trans()
3238 	struct irdma_device *iwdev = to_iwdev(ib_mr->device);  in irdma_rereg_user_mr()  local
3243 	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)  in irdma_rereg_user_mr()
3263 			irdma_free_pble(iwdev->rf->pble_rsrc,  in irdma_rereg_user_mr()
3274 		ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);  in irdma_rereg_user_mr()
3292 	struct irdma_device *iwdev = to_iwdev(pd->device);  in irdma_reg_phys_mr()  local
3308 	stag = irdma_create_stag(iwdev);  in irdma_reg_phys_mr()
3322 	ret = irdma_hwreg_mr(iwdev, iwmr, access);  in irdma_reg_phys_mr()
3324 		irdma_free_stag(iwdev, stag);  in irdma_reg_phys_mr()
3389 	struct irdma_device *iwdev = to_iwdev(ib_mr->device);  in irdma_dereg_mr()  local
3409 	irdma_free_stag(iwdev, iwmr->stag);  in irdma_dereg_mr()
3412 		irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);  in irdma_dereg_mr()
3443 	dev = &iwqp->iwdev->rf->sc_dev;  in irdma_post_send()
3568 			ibdev_dbg(&iwqp->iwdev->ibdev,  in irdma_post_send()
3585 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_post_send()
3620 			ibdev_dbg(&iwqp->iwdev->ibdev,  in irdma_post_recv()
3631 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_post_recv()
3780 	struct irdma_device *iwdev;  in __irdma_poll_cq()  local
3786 	iwdev = to_iwdev(iwcq->ibcq.device);  in __irdma_poll_cq()
3841 		resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL);  in __irdma_poll_cq()
3844 		resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf);  in __irdma_poll_cq()
3851 	ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n",  in __irdma_poll_cq()
4007 	struct irdma_device *iwdev = to_iwdev(dev);  in irdma_get_dev_fw_str()  local
4010 		 irdma_fw_major_ver(&iwdev->rf->sc_dev),  in irdma_get_dev_fw_str()
4011 		 irdma_fw_minor_ver(&iwdev->rf->sc_dev));  in irdma_get_dev_fw_str()
4022 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_alloc_hw_port_stats()  local
4023 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_alloc_hw_port_stats()
4043 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_get_hw_stats()  local
4044 	struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats;  in irdma_get_hw_stats()
4046 	if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)  in irdma_get_hw_stats()
4047 		irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true);  in irdma_get_hw_stats()
4049 		irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat);  in irdma_get_hw_stats()
4066 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_query_gid()  local
4069 	ether_addr_copy(gid->raw, iwdev->netdev->dev_addr);  in irdma_query_gid()
4124 static int irdma_mcast_cqp_op(struct irdma_device *iwdev,  in irdma_mcast_cqp_op()  argument
4131 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_mcast_cqp_op()
4140 	cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp;  in irdma_mcast_cqp_op()
4141 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_mcast_cqp_op()
4142 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_mcast_cqp_op()
4189 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_attach_mcast()  local
4190 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_attach_mcast()
4210 		ibdev_dbg(&iwdev->ibdev,  in irdma_attach_mcast()
4219 		ibdev_dbg(&iwdev->ibdev,  in irdma_attach_mcast()
4265 		mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id;  in irdma_attach_mcast()
4287 		ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,  in irdma_attach_mcast()
4290 		ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,  in irdma_attach_mcast()
4329 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_detach_mcast()  local
4330 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_detach_mcast()
4349 		ibdev_dbg(&iwdev->ibdev,  in irdma_detach_mcast()
4359 		ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,  in irdma_detach_mcast()
4362 			ibdev_dbg(&iwdev->ibdev,  in irdma_detach_mcast()
4380 		ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,  in irdma_detach_mcast()
4383 			ibdev_dbg(&iwdev->ibdev,  in irdma_detach_mcast()
4392 static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep)  in irdma_create_hw_ah()  argument
4394 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_create_hw_ah()
4406 		ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail");  in irdma_create_hw_ah()
4419 			ibdev_dbg(&iwdev->ibdev, "VERBS: CQP create AH timed out");  in irdma_create_hw_ah()
4427 	irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx);  in irdma_create_hw_ah()
4438 	struct irdma_device *iwdev = to_iwdev(ibah->pd->device);  in irdma_setup_ah()  local
4439 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_setup_ah()
4448 	sc_ah->ah_info.vsi = &iwdev->vsi;  in irdma_setup_ah()
4496 	ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr,  in irdma_setup_ah()
4502 	if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode)  in irdma_setup_ah()
4524 static bool irdma_ah_exists(struct irdma_device *iwdev,  in irdma_ah_exists()  argument
4533 	hash_for_each_possible(iwdev->ah_hash_tbl, ah, list, key) {  in irdma_ah_exists()
4555 	struct irdma_device *iwdev = to_iwdev(ibah->device);  in irdma_destroy_ah()  local
4559 		mutex_lock(&iwdev->ah_tbl_lock);  in irdma_destroy_ah()
4561 			mutex_unlock(&iwdev->ah_tbl_lock);  in irdma_destroy_ah()
4566 		mutex_unlock(&iwdev->ah_tbl_lock);  in irdma_destroy_ah()
4569 	irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,  in irdma_destroy_ah()
4572 	irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,  in irdma_destroy_ah()
4592 	struct irdma_device *iwdev = to_iwdev(ibah->pd->device);  in irdma_create_user_ah()  local
4603 	mutex_lock(&iwdev->ah_tbl_lock);  in irdma_create_user_ah()
4604 	if (!irdma_ah_exists(iwdev, ah)) {  in irdma_create_user_ah()
4605 		err = irdma_create_hw_ah(iwdev, ah, true);  in irdma_create_user_ah()
4607 			mutex_unlock(&iwdev->ah_tbl_lock);  in irdma_create_user_ah()
4619 			hash_add(iwdev->ah_hash_tbl, &parent_ah->list, key);  in irdma_create_user_ah()
4623 	mutex_unlock(&iwdev->ah_tbl_lock);  in irdma_create_user_ah()
4645 	struct irdma_device *iwdev = to_iwdev(ibah->pd->device);  in irdma_create_ah()  local
4651 	err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE);  in irdma_create_ah()
4761 static void irdma_init_roce_device(struct irdma_device *iwdev)  in irdma_init_roce_device()  argument
4763 	iwdev->ibdev.node_type = RDMA_NODE_IB_CA;  in irdma_init_roce_device()
4764 	addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid,  in irdma_init_roce_device()
4765 			    iwdev->netdev->dev_addr);  in irdma_init_roce_device()
4766 	ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops);  in irdma_init_roce_device()
4773 static void irdma_init_iw_device(struct irdma_device *iwdev)  in irdma_init_iw_device()  argument
4775 	struct net_device *netdev = iwdev->netdev;  in irdma_init_iw_device()
4777 	iwdev->ibdev.node_type = RDMA_NODE_RNIC;  in irdma_init_iw_device()
4778 	addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid,  in irdma_init_iw_device()
4780 	memcpy(iwdev->ibdev.iw_ifname, netdev->name,  in irdma_init_iw_device()
4781 	       sizeof(iwdev->ibdev.iw_ifname));  in irdma_init_iw_device()
4782 	ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops);  in irdma_init_iw_device()
4789 static void irdma_init_rdma_device(struct irdma_device *iwdev)  in irdma_init_rdma_device()  argument
4791 	struct pci_dev *pcidev = iwdev->rf->pcidev;  in irdma_init_rdma_device()
4793 	if (iwdev->roce_mode)  in irdma_init_rdma_device()
4794 		irdma_init_roce_device(iwdev);  in irdma_init_rdma_device()
4796 		irdma_init_iw_device(iwdev);  in irdma_init_rdma_device()
4798 	iwdev->ibdev.phys_port_cnt = 1;  in irdma_init_rdma_device()
4799 	iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count;  in irdma_init_rdma_device()
4800 	iwdev->ibdev.dev.parent = &pcidev->dev;  in irdma_init_rdma_device()
4801 	ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops);  in irdma_init_rdma_device()
4808 void irdma_port_ibevent(struct irdma_device *iwdev)  in irdma_port_ibevent()  argument
4812 	event.device = &iwdev->ibdev;  in irdma_port_ibevent()
4815 		iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;  in irdma_port_ibevent()
4824 void irdma_ib_unregister_device(struct irdma_device *iwdev)  in irdma_ib_unregister_device()  argument
4826 	iwdev->iw_status = 0;  in irdma_ib_unregister_device()
4827 	irdma_port_ibevent(iwdev);  in irdma_ib_unregister_device()
4828 	ib_unregister_device(&iwdev->ibdev);  in irdma_ib_unregister_device()
4835 int irdma_ib_register_device(struct irdma_device *iwdev)  in irdma_ib_register_device()  argument
4839 	irdma_init_rdma_device(iwdev);  in irdma_ib_register_device()
4841 	ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1);  in irdma_ib_register_device()
4844 	dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX);  in irdma_ib_register_device()
4845 	ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device);  in irdma_ib_register_device()
4849 	iwdev->iw_status = 1;  in irdma_ib_register_device()
4850 	irdma_port_ibevent(iwdev);  in irdma_ib_register_device()
4856 		ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n");  in irdma_ib_register_device()
4870 	struct irdma_device *iwdev = to_iwdev(ibdev);  in irdma_ib_dealloc_device()  local
4872 	irdma_rt_deinit_hw(iwdev);  in irdma_ib_dealloc_device()
4873 	irdma_ctrl_deinit_hw(iwdev->rf);  in irdma_ib_dealloc_device()
4874 	kfree(iwdev->rf);  in irdma_ib_dealloc_device()