Lines Matching +full:cm +full:- +full:poll +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
6 * irdma_arp_table -manage arp table
25 spin_lock_irqsave(&rf->arp_lock, flags); in irdma_arp_table()
26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { in irdma_arp_table()
27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) in irdma_arp_table()
33 if (arp_index != rf->arp_table_size) { in irdma_arp_table()
34 arp_index = -1; in irdma_arp_table()
39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, in irdma_arp_table()
40 (u32 *)&arp_index, &rf->next_arp_index)) { in irdma_arp_table()
41 arp_index = -1; in irdma_arp_table()
45 memcpy(rf->arp_table[arp_index].ip_addr, ip, in irdma_arp_table()
46 sizeof(rf->arp_table[arp_index].ip_addr)); in irdma_arp_table()
47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); in irdma_arp_table()
50 if (arp_index == rf->arp_table_size) in irdma_arp_table()
51 arp_index = -1; in irdma_arp_table()
54 if (arp_index == rf->arp_table_size) { in irdma_arp_table()
55 arp_index = -1; in irdma_arp_table()
59 memset(rf->arp_table[arp_index].ip_addr, 0, in irdma_arp_table()
60 sizeof(rf->arp_table[arp_index].ip_addr)); in irdma_arp_table()
61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); in irdma_arp_table()
62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); in irdma_arp_table()
65 arp_index = -1; in irdma_arp_table()
69 spin_unlock_irqrestore(&rf->arp_lock, flags); in irdma_arp_table()
74 * irdma_add_arp - add a new arp entry if needed
86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) in irdma_add_arp()
89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, in irdma_add_arp()
99 * wr32 - write 32 bits to hw register
106 writel(val, hw->hw_addr + reg); in wr32()
110 * rd32 - read a 32 bit hw register
118 return readl(hw->hw_addr + reg); in rd32()
122 * rd64 - read a 64 bit hw register
130 return readq(hw->hw_addr + reg); in rd64()
144 * irdma_inetaddr_event - system notifier for ipv4 addr events
153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; in irdma_inetaddr_event()
167 local_ipaddr = ntohl(ifa->ifa_address); in irdma_inetaddr_event()
168 ibdev_dbg(&iwdev->ibdev, in irdma_inetaddr_event()
170 event, &local_ipaddr, real_dev->dev_addr); in irdma_inetaddr_event()
173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, in irdma_inetaddr_event()
176 irdma_gid_change_event(&iwdev->ibdev); in irdma_inetaddr_event()
180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr); in irdma_inetaddr_event()
182 irdma_gid_change_event(&iwdev->ibdev); in irdma_inetaddr_event()
194 * irdma_inet6addr_event - system notifier for ipv6 addr events
203 struct net_device *real_dev, *netdev = ifa->idev->dev; in irdma_inet6addr_event()
217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); in irdma_inet6addr_event()
218 ibdev_dbg(&iwdev->ibdev, in irdma_inet6addr_event()
220 event, local_ipaddr6, real_dev->dev_addr); in irdma_inet6addr_event()
223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, in irdma_inet6addr_event()
226 irdma_gid_change_event(&iwdev->ibdev); in irdma_inet6addr_event()
230 irdma_add_arp(iwdev->rf, local_ipaddr6, false, in irdma_inet6addr_event()
231 real_dev->dev_addr); in irdma_inet6addr_event()
233 irdma_gid_change_event(&iwdev->ibdev); in irdma_inet6addr_event()
245 * irdma_net_event - system notifier for net events
254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; in irdma_net_event()
271 p = (__be32 *)neigh->primary_key; in irdma_net_event()
272 if (neigh->tbl->family == AF_INET6) { in irdma_net_event()
279 ibdev_dbg(&iwdev->ibdev, in irdma_net_event()
281 iwdev->netdev, neigh->nud_state, local_ipaddr, in irdma_net_event()
282 neigh->ha); in irdma_net_event()
284 if (neigh->nud_state & NUD_VALID) in irdma_net_event()
285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); in irdma_net_event()
288 irdma_manage_arp_cache(iwdev->rf, neigh->ha, in irdma_net_event()
301 * irdma_netdevice_event - system notifier for netdev events
318 iwdev->iw_status = 1; in irdma_netdevice_event()
321 iwdev->iw_status = 0; in irdma_netdevice_event()
335 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
348 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || in irdma_add_ipv6_addr()
349 ip_dev == iwdev->netdev) && in irdma_add_ipv6_addr()
350 (READ_ONCE(ip_dev->flags) & IFF_UP)) { in irdma_add_ipv6_addr()
353 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); in irdma_add_ipv6_addr()
356 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, in irdma_add_ipv6_addr()
358 ibdev_dbg(&iwdev->ibdev, in irdma_add_ipv6_addr()
360 &ifp->addr, in irdma_add_ipv6_addr()
362 ip_dev->dev_addr); in irdma_add_ipv6_addr()
365 ifp->addr.in6_u.u6_addr32); in irdma_add_ipv6_addr()
366 irdma_manage_arp_cache(iwdev->rf, in irdma_add_ipv6_addr()
367 ip_dev->dev_addr, in irdma_add_ipv6_addr()
377 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
389 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || in irdma_add_ipv4_addr()
390 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { in irdma_add_ipv4_addr()
398 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", in irdma_add_ipv4_addr()
399 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), in irdma_add_ipv4_addr()
400 dev->dev_addr); in irdma_add_ipv4_addr()
402 ip_addr = ntohl(ifa->ifa_address); in irdma_add_ipv4_addr()
403 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, in irdma_add_ipv4_addr()
413 * irdma_add_ip - add ip addresses
425 * irdma_alloc_and_get_cqp_request - get cqp struct
427 * @wait: cqp to be used in wait mode
435 spin_lock_irqsave(&cqp->req_lock, flags); in irdma_alloc_and_get_cqp_request()
436 if (!list_empty(&cqp->cqp_avail_reqs)) { in irdma_alloc_and_get_cqp_request()
437 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, in irdma_alloc_and_get_cqp_request()
439 list_del_init(&cqp_request->list); in irdma_alloc_and_get_cqp_request()
441 spin_unlock_irqrestore(&cqp->req_lock, flags); in irdma_alloc_and_get_cqp_request()
445 cqp_request->dynamic = true; in irdma_alloc_and_get_cqp_request()
447 init_waitqueue_head(&cqp_request->waitq); in irdma_alloc_and_get_cqp_request()
451 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); in irdma_alloc_and_get_cqp_request()
455 cqp_request->waiting = wait; in irdma_alloc_and_get_cqp_request()
456 refcount_set(&cqp_request->refcnt, 1); in irdma_alloc_and_get_cqp_request()
457 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); in irdma_alloc_and_get_cqp_request()
463 * irdma_get_cqp_request - increase refcount for cqp_request
468 refcount_inc(&cqp_request->refcnt); in irdma_get_cqp_request()
472 * irdma_free_cqp_request - free cqp request
481 if (cqp_request->dynamic) { in irdma_free_cqp_request()
484 WRITE_ONCE(cqp_request->request_done, false); in irdma_free_cqp_request()
485 cqp_request->callback_fcn = NULL; in irdma_free_cqp_request()
486 cqp_request->waiting = false; in irdma_free_cqp_request()
488 spin_lock_irqsave(&cqp->req_lock, flags); in irdma_free_cqp_request()
489 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); in irdma_free_cqp_request()
490 spin_unlock_irqrestore(&cqp->req_lock, flags); in irdma_free_cqp_request()
492 wake_up(&cqp->remove_wq); in irdma_free_cqp_request()
496 * irdma_put_cqp_request - dec ref count and free if 0
503 if (refcount_dec_and_test(&cqp_request->refcnt)) in irdma_put_cqp_request()
508 * irdma_free_pending_cqp_request -free pending cqp request objs
516 if (cqp_request->waiting) { in irdma_free_pending_cqp_request()
517 cqp_request->compl_info.error = true; in irdma_free_pending_cqp_request()
518 WRITE_ONCE(cqp_request->request_done, true); in irdma_free_pending_cqp_request()
519 wake_up(&cqp_request->waitq); in irdma_free_pending_cqp_request()
521 wait_event_timeout(cqp->remove_wq, in irdma_free_pending_cqp_request()
522 refcount_read(&cqp_request->refcnt) == 1, 1000); in irdma_free_pending_cqp_request()
527 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
533 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_cleanup_pending_cqp_op()
534 struct irdma_cqp *cqp = &rf->cqp; in irdma_cleanup_pending_cqp_op()
539 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
540 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
543 cqp->scratch_array[wqe_idx]; in irdma_cleanup_pending_cqp_op()
546 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
549 while (!list_empty(&dev->cqp_cmd_head)) { in irdma_cleanup_pending_cqp_op()
559 * irdma_wait_event - wait for completion
570 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); in irdma_wait_event()
572 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); in irdma_wait_event()
573 if (wait_event_timeout(cqp_request->waitq, in irdma_wait_event()
574 READ_ONCE(cqp_request->request_done), in irdma_wait_event()
578 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); in irdma_wait_event()
583 if (!rf->reset) { in irdma_wait_event()
584 rf->reset = true; in irdma_wait_event()
585 rf->gen_ops.request_reset(rf); in irdma_wait_event()
587 return -ETIMEDOUT; in irdma_wait_event()
590 cqp_error = cqp_request->compl_info.error; in irdma_wait_event()
592 err_code = -EIO; in irdma_wait_event()
593 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { in irdma_wait_event()
594 if (cqp_request->compl_info.min_err_code == 0x8002) in irdma_wait_event()
595 err_code = -EBUSY; in irdma_wait_event()
596 else if (cqp_request->compl_info.min_err_code == 0x8029) { in irdma_wait_event()
597 if (!rf->reset) { in irdma_wait_event()
598 rf->reset = true; in irdma_wait_event()
599 rf->gen_ops.request_reset(rf); in irdma_wait_event()
623 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
648 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
666 * irdma_cqp_crit_err - check if CQP error is critical
692 * irdma_handle_cqp_op - process cqp command
699 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_handle_cqp_op()
700 struct cqp_cmds_info *info = &cqp_request->info; in irdma_handle_cqp_op()
704 if (rf->reset) in irdma_handle_cqp_op()
705 return -EBUSY; in irdma_handle_cqp_op()
712 if (cqp_request->waiting) { in irdma_handle_cqp_op()
722 if (irdma_cqp_crit_err(dev, info->cqp_cmd, in irdma_handle_cqp_op()
723 cqp_request->compl_info.maj_err_code, in irdma_handle_cqp_op()
724 cqp_request->compl_info.min_err_code)) in irdma_handle_cqp_op()
725 ibdev_err(&rf->iwdev->ibdev, in irdma_handle_cqp_op()
727 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, in irdma_handle_cqp_op()
728 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, in irdma_handle_cqp_op()
729 cqp_request->compl_info.min_err_code); in irdma_handle_cqp_op()
732 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_handle_cqp_op()
741 refcount_inc(&iwqp->refcnt); in irdma_qp_add_ref()
747 struct irdma_device *iwdev = iwqp->iwdev; in irdma_qp_rem_ref()
751 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
752 if (!refcount_dec_and_test(&iwqp->refcnt)) { in irdma_qp_rem_ref()
753 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
757 qp_num = iwqp->ibqp.qp_num; in irdma_qp_rem_ref()
758 iwdev->rf->qp_table[qp_num] = NULL; in irdma_qp_rem_ref()
759 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
760 complete(&iwqp->free_qp); in irdma_qp_rem_ref()
767 refcount_inc(&iwcq->refcnt); in irdma_cq_add_ref()
772 struct ib_device *ibdev = ibcq->device; in irdma_cq_rem_ref()
777 spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags); in irdma_cq_rem_ref()
778 if (!refcount_dec_and_test(&iwcq->refcnt)) { in irdma_cq_rem_ref()
779 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); in irdma_cq_rem_ref()
783 iwdev->rf->cq_table[iwcq->cq_num] = NULL; in irdma_cq_rem_ref()
784 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); in irdma_cq_rem_ref()
785 complete(&iwcq->free_cq); in irdma_cq_rem_ref()
790 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; in to_ibdev()
794 * irdma_get_qp - get qp address
802 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) in irdma_get_qp()
805 return &iwdev->rf->qp_table[qpn]->ibqp; in irdma_get_qp()
809 * irdma_remove_cqp_head - return head entry and remove
815 struct list_head *list = &dev->cqp_cmd_head; in irdma_remove_cqp_head()
820 entry = list->next; in irdma_remove_cqp_head()
827 * irdma_cqp_sds_cmd - create cqp command for sd
840 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_sds_cmd()
842 return -ENOMEM; in irdma_cqp_sds_cmd()
844 cqp_info = &cqp_request->info; in irdma_cqp_sds_cmd()
845 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, in irdma_cqp_sds_cmd()
846 sizeof(cqp_info->in.u.update_pe_sds.info)); in irdma_cqp_sds_cmd()
847 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; in irdma_cqp_sds_cmd()
848 cqp_info->post_sq = 1; in irdma_cqp_sds_cmd()
849 cqp_info->in.u.update_pe_sds.dev = dev; in irdma_cqp_sds_cmd()
850 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; in irdma_cqp_sds_cmd()
853 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_sds_cmd()
859 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
865 struct irdma_sc_dev *dev = qp->dev; in irdma_cqp_qp_suspend_resume()
867 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_cqp_qp_suspend_resume()
872 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_cqp_qp_suspend_resume()
874 return -ENOMEM; in irdma_cqp_qp_suspend_resume()
876 cqp_info = &cqp_request->info; in irdma_cqp_qp_suspend_resume()
877 cqp_info->cqp_cmd = op; in irdma_cqp_qp_suspend_resume()
878 cqp_info->in.u.suspend_resume.cqp = cqp; in irdma_cqp_qp_suspend_resume()
879 cqp_info->in.u.suspend_resume.qp = qp; in irdma_cqp_qp_suspend_resume()
880 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_suspend_resume()
883 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_qp_suspend_resume()
889 * irdma_term_modify_qp - modify qp for term message
900 iwqp = qp->qp_uk.back_qp; in irdma_term_modify_qp()
905 * irdma_terminate_done - after terminate is completed
916 iwqp = qp->qp_uk.back_qp; in irdma_terminate_done()
917 spin_lock_irqsave(&iwqp->lock, flags); in irdma_terminate_done()
918 if (iwqp->hte_added) { in irdma_terminate_done()
919 iwqp->hte_added = 0; in irdma_terminate_done()
922 first_time = !(qp->term_flags & IRDMA_TERM_DONE); in irdma_terminate_done()
923 qp->term_flags |= IRDMA_TERM_DONE; in irdma_terminate_done()
924 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_terminate_done()
937 struct irdma_sc_qp *qp = &iwqp->sc_qp; in irdma_terminate_timeout()
940 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_terminate_timeout()
944 * irdma_terminate_start_timer - start terminate timeout
951 iwqp = qp->qp_uk.back_qp; in irdma_terminate_start_timer()
952 irdma_qp_add_ref(&iwqp->ibqp); in irdma_terminate_start_timer()
953 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); in irdma_terminate_start_timer()
954 iwqp->terminate_timer.expires = jiffies + HZ; in irdma_terminate_start_timer()
956 add_timer(&iwqp->terminate_timer); in irdma_terminate_start_timer()
960 * irdma_terminate_del_timer - delete terminate timeout
968 iwqp = qp->qp_uk.back_qp; in irdma_terminate_del_timer()
969 ret = del_timer(&iwqp->terminate_timer); in irdma_terminate_del_timer()
971 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_terminate_del_timer()
975 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
988 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_query_fpm_val_cmd()
990 return -ENOMEM; in irdma_cqp_query_fpm_val_cmd()
992 cqp_info = &cqp_request->info; in irdma_cqp_query_fpm_val_cmd()
993 cqp_request->param = NULL; in irdma_cqp_query_fpm_val_cmd()
994 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; in irdma_cqp_query_fpm_val_cmd()
995 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; in irdma_cqp_query_fpm_val_cmd()
996 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; in irdma_cqp_query_fpm_val_cmd()
997 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; in irdma_cqp_query_fpm_val_cmd()
998 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; in irdma_cqp_query_fpm_val_cmd()
999 cqp_info->post_sq = 1; in irdma_cqp_query_fpm_val_cmd()
1000 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; in irdma_cqp_query_fpm_val_cmd()
1003 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_query_fpm_val_cmd()
1009 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
1022 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_commit_fpm_val_cmd()
1024 return -ENOMEM; in irdma_cqp_commit_fpm_val_cmd()
1026 cqp_info = &cqp_request->info; in irdma_cqp_commit_fpm_val_cmd()
1027 cqp_request->param = NULL; in irdma_cqp_commit_fpm_val_cmd()
1028 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; in irdma_cqp_commit_fpm_val_cmd()
1029 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; in irdma_cqp_commit_fpm_val_cmd()
1030 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; in irdma_cqp_commit_fpm_val_cmd()
1031 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; in irdma_cqp_commit_fpm_val_cmd()
1032 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; in irdma_cqp_commit_fpm_val_cmd()
1033 cqp_info->post_sq = 1; in irdma_cqp_commit_fpm_val_cmd()
1034 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; in irdma_cqp_commit_fpm_val_cmd()
1037 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_commit_fpm_val_cmd()
1043 * irdma_cqp_cq_create_cmd - create a cq for the cqp
1050 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_cq_create_cmd()
1057 return -ENOMEM; in irdma_cqp_cq_create_cmd()
1059 cqp_info = &cqp_request->info; in irdma_cqp_cq_create_cmd()
1060 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; in irdma_cqp_cq_create_cmd()
1061 cqp_info->post_sq = 1; in irdma_cqp_cq_create_cmd()
1062 cqp_info->in.u.cq_create.cq = cq; in irdma_cqp_cq_create_cmd()
1063 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_cq_create_cmd()
1072 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1079 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_qp_create_cmd()
1087 return -ENOMEM; in irdma_cqp_qp_create_cmd()
1089 cqp_info = &cqp_request->info; in irdma_cqp_qp_create_cmd()
1090 qp_info = &cqp_request->info.in.u.qp_create.info; in irdma_cqp_qp_create_cmd()
1092 qp_info->cq_num_valid = true; in irdma_cqp_qp_create_cmd()
1093 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; in irdma_cqp_qp_create_cmd()
1094 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; in irdma_cqp_qp_create_cmd()
1095 cqp_info->post_sq = 1; in irdma_cqp_qp_create_cmd()
1096 cqp_info->in.u.qp_create.qp = qp; in irdma_cqp_qp_create_cmd()
1097 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_create_cmd()
1106 * irdma_dealloc_push_page - free a push page for qp
1117 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) in irdma_dealloc_push_page()
1120 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_dealloc_push_page()
1124 cqp_info = &cqp_request->info; in irdma_dealloc_push_page()
1125 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; in irdma_dealloc_push_page()
1126 cqp_info->post_sq = 1; in irdma_dealloc_push_page()
1127 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; in irdma_dealloc_push_page()
1128 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; in irdma_dealloc_push_page()
1129 cqp_info->in.u.manage_push_page.info.free_page = 1; in irdma_dealloc_push_page()
1130 cqp_info->in.u.manage_push_page.info.push_page_type = 0; in irdma_dealloc_push_page()
1131 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; in irdma_dealloc_push_page()
1132 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; in irdma_dealloc_push_page()
1135 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; in irdma_dealloc_push_page()
1136 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_dealloc_push_page()
1140 * irdma_free_qp_rsrc - free up memory resources for qp
1145 struct irdma_device *iwdev = iwqp->iwdev; in irdma_free_qp_rsrc()
1146 struct irdma_pci_f *rf = iwdev->rf; in irdma_free_qp_rsrc()
1147 u32 qp_num = iwqp->ibqp.qp_num; in irdma_free_qp_rsrc()
1149 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); in irdma_free_qp_rsrc()
1150 irdma_dealloc_push_page(rf, &iwqp->sc_qp); in irdma_free_qp_rsrc()
1151 if (iwqp->sc_qp.vsi) { in irdma_free_qp_rsrc()
1152 irdma_qp_rem_qos(&iwqp->sc_qp); in irdma_free_qp_rsrc()
1153 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, in irdma_free_qp_rsrc()
1154 iwqp->sc_qp.user_pri); in irdma_free_qp_rsrc()
1158 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); in irdma_free_qp_rsrc()
1159 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, in irdma_free_qp_rsrc()
1160 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); in irdma_free_qp_rsrc()
1161 iwqp->q2_ctx_mem.va = NULL; in irdma_free_qp_rsrc()
1162 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, in irdma_free_qp_rsrc()
1163 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); in irdma_free_qp_rsrc()
1164 iwqp->kqp.dma_mem.va = NULL; in irdma_free_qp_rsrc()
1165 kfree(iwqp->kqp.sq_wrid_mem); in irdma_free_qp_rsrc()
1166 kfree(iwqp->kqp.rq_wrid_mem); in irdma_free_qp_rsrc()
1170 * irdma_cq_wq_destroy - send cq destroy cqp
1179 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cq_wq_destroy()
1183 cqp_info = &cqp_request->info; in irdma_cq_wq_destroy()
1184 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; in irdma_cq_wq_destroy()
1185 cqp_info->post_sq = 1; in irdma_cq_wq_destroy()
1186 cqp_info->in.u.cq_destroy.cq = cq; in irdma_cq_wq_destroy()
1187 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; in irdma_cq_wq_destroy()
1190 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cq_wq_destroy()
1194 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1202 cqp_info = &cqp_request->info; in irdma_hw_modify_qp_callback()
1203 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; in irdma_hw_modify_qp_callback()
1204 atomic_dec(&iwqp->hw_mod_qp_pend); in irdma_hw_modify_qp_callback()
1205 wake_up(&iwqp->mod_qp_waitq); in irdma_hw_modify_qp_callback()
1209 * irdma_hw_modify_qp - setup cqp for modify qp
1219 struct irdma_pci_f *rf = iwdev->rf; in irdma_hw_modify_qp()
1224 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_hw_modify_qp()
1226 return -ENOMEM; in irdma_hw_modify_qp()
1229 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; in irdma_hw_modify_qp()
1230 atomic_inc(&iwqp->hw_mod_qp_pend); in irdma_hw_modify_qp()
1232 cqp_info = &cqp_request->info; in irdma_hw_modify_qp()
1233 m_info = &cqp_info->in.u.qp_modify.info; in irdma_hw_modify_qp()
1235 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; in irdma_hw_modify_qp()
1236 cqp_info->post_sq = 1; in irdma_hw_modify_qp()
1237 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; in irdma_hw_modify_qp()
1238 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; in irdma_hw_modify_qp()
1240 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_modify_qp()
1242 if (rdma_protocol_roce(&iwdev->ibdev, 1)) in irdma_hw_modify_qp()
1245 switch (m_info->next_iwarp_state) { in irdma_hw_modify_qp()
1252 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) in irdma_hw_modify_qp()
1253 irdma_send_reset(iwqp->cm_node); in irdma_hw_modify_qp()
1255 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; in irdma_hw_modify_qp()
1259 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); in irdma_hw_modify_qp()
1261 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, in irdma_hw_modify_qp()
1264 return -ENOMEM; in irdma_hw_modify_qp()
1266 cqp_info = &cqp_request->info; in irdma_hw_modify_qp()
1267 m_info = &cqp_info->in.u.qp_modify.info; in irdma_hw_modify_qp()
1269 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; in irdma_hw_modify_qp()
1270 cqp_info->post_sq = 1; in irdma_hw_modify_qp()
1271 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; in irdma_hw_modify_qp()
1272 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; in irdma_hw_modify_qp()
1273 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; in irdma_hw_modify_qp()
1274 m_info->reset_tcp_conn = true; in irdma_hw_modify_qp()
1276 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_modify_qp()
1289 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1301 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1308 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_qp_destroy_cmd()
1315 return -ENOMEM; in irdma_cqp_qp_destroy_cmd()
1317 cqp_info = &cqp_request->info; in irdma_cqp_qp_destroy_cmd()
1319 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; in irdma_cqp_qp_destroy_cmd()
1320 cqp_info->post_sq = 1; in irdma_cqp_qp_destroy_cmd()
1321 cqp_info->in.u.qp_destroy.qp = qp; in irdma_cqp_qp_destroy_cmd()
1322 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_destroy_cmd()
1323 cqp_info->in.u.qp_destroy.remove_hash_idx = true; in irdma_cqp_qp_destroy_cmd()
1326 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_qp_destroy_cmd()
1332 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1341 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); in irdma_ieq_mpa_crc_ae()
1348 * irdma_init_hash_desc - initialize hash for crc calculation
1358 return -EINVAL; in irdma_init_hash_desc()
1364 return -EINVAL; in irdma_init_hash_desc()
1367 tdesc->tfm = tfm; in irdma_init_hash_desc()
1374 * irdma_free_hash_desc - free hash desc
1380 crypto_free_shash(desc->tfm); in irdma_free_hash_desc()
1386 * irdma_ieq_check_mpacrc - check if mpa crc is OK
1399 return -EINVAL; in irdma_ieq_check_mpacrc()
1405 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1414 struct irdma_device *iwdev = buf->vsi->back_vsi; in irdma_ieq_get_qp()
1419 struct iphdr *iph = (struct iphdr *)buf->iph; in irdma_ieq_get_qp()
1420 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; in irdma_ieq_get_qp()
1422 if (iph->version == 4) { in irdma_ieq_get_qp()
1423 loc_addr[0] = ntohl(iph->daddr); in irdma_ieq_get_qp()
1424 rem_addr[0] = ntohl(iph->saddr); in irdma_ieq_get_qp()
1426 ip6h = (struct ipv6hdr *)buf->iph; in irdma_ieq_get_qp()
1427 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); in irdma_ieq_get_qp()
1428 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); in irdma_ieq_get_qp()
1430 loc_port = ntohs(tcph->dest); in irdma_ieq_get_qp()
1431 rem_port = ntohs(tcph->source); in irdma_ieq_get_qp()
1432 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, in irdma_ieq_get_qp()
1433 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); in irdma_ieq_get_qp()
1437 iwqp = cm_node->iwqp; in irdma_ieq_get_qp()
1440 return &iwqp->sc_qp; in irdma_ieq_get_qp()
1444 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1449 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; in irdma_send_ieq_ack()
1450 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; in irdma_send_ieq_ack()
1451 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; in irdma_send_ieq_ack()
1453 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; in irdma_send_ieq_ack()
1454 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); in irdma_send_ieq_ack()
1460 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1467 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; in irdma_puda_ieq_get_ah_info()
1472 ah_info->do_lpbk = true; in irdma_puda_ieq_get_ah_info()
1473 ah_info->vlan_tag = buf->vlan_id; in irdma_puda_ieq_get_ah_info()
1474 ah_info->insert_vlan_tag = buf->vlan_valid; in irdma_puda_ieq_get_ah_info()
1475 ah_info->ipv4_valid = buf->ipv4; in irdma_puda_ieq_get_ah_info()
1476 ah_info->vsi = qp->vsi; in irdma_puda_ieq_get_ah_info()
1478 if (buf->smac_valid) in irdma_puda_ieq_get_ah_info()
1479 ether_addr_copy(ah_info->mac_addr, buf->smac); in irdma_puda_ieq_get_ah_info()
1481 if (buf->ipv4) { in irdma_puda_ieq_get_ah_info()
1482 ah_info->ipv4_valid = true; in irdma_puda_ieq_get_ah_info()
1483 iph = (struct iphdr *)buf->iph; in irdma_puda_ieq_get_ah_info()
1484 ah_info->hop_ttl = iph->ttl; in irdma_puda_ieq_get_ah_info()
1485 ah_info->tc_tos = iph->tos; in irdma_puda_ieq_get_ah_info()
1486 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); in irdma_puda_ieq_get_ah_info()
1487 ah_info->src_ip_addr[0] = ntohl(iph->saddr); in irdma_puda_ieq_get_ah_info()
1489 ip6h = (struct ipv6hdr *)buf->iph; in irdma_puda_ieq_get_ah_info()
1490 ah_info->hop_ttl = ip6h->hop_limit; in irdma_puda_ieq_get_ah_info()
1491 ah_info->tc_tos = ip6h->priority; in irdma_puda_ieq_get_ah_info()
1492 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, in irdma_puda_ieq_get_ah_info()
1493 ip6h->daddr.in6_u.u6_addr32); in irdma_puda_ieq_get_ah_info()
1494 irdma_copy_ip_ntohl(ah_info->src_ip_addr, in irdma_puda_ieq_get_ah_info()
1495 ip6h->saddr.in6_u.u6_addr32); in irdma_puda_ieq_get_ah_info()
1498 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), in irdma_puda_ieq_get_ah_info()
1499 ah_info->dest_ip_addr, in irdma_puda_ieq_get_ah_info()
1500 ah_info->ipv4_valid, in irdma_puda_ieq_get_ah_info()
1505 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1517 u8 *addr = buf->mem.va; in irdma_gen1_ieq_update_tcpip_info()
1519 iphlen = (buf->ipv4) ? 20 : 40; in irdma_gen1_ieq_update_tcpip_info()
1520 iph = (struct iphdr *)(addr + buf->maclen); in irdma_gen1_ieq_update_tcpip_info()
1521 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); in irdma_gen1_ieq_update_tcpip_info()
1522 pktsize = len + buf->tcphlen + iphlen; in irdma_gen1_ieq_update_tcpip_info()
1523 iph->tot_len = htons(pktsize); in irdma_gen1_ieq_update_tcpip_info()
1524 tcph->seq = htonl(seqnum); in irdma_gen1_ieq_update_tcpip_info()
1528 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1539 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_ieq_update_tcpip_info()
1542 addr = buf->mem.va; in irdma_ieq_update_tcpip_info()
1544 tcph->seq = htonl(seqnum); in irdma_ieq_update_tcpip_info()
1548 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1561 u8 *mem = buf->mem.va; in irdma_gen1_puda_get_tcpip_info()
1562 struct ethhdr *ethh = buf->mem.va; in irdma_gen1_puda_get_tcpip_info()
1564 if (ethh->h_proto == htons(0x8100)) { in irdma_gen1_puda_get_tcpip_info()
1565 info->vlan_valid = true; in irdma_gen1_puda_get_tcpip_info()
1566 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & in irdma_gen1_puda_get_tcpip_info()
1570 buf->maclen = (info->vlan_valid) ? 18 : 14; in irdma_gen1_puda_get_tcpip_info()
1571 iphlen = (info->l3proto) ? 40 : 20; in irdma_gen1_puda_get_tcpip_info()
1572 buf->ipv4 = (info->l3proto) ? false : true; in irdma_gen1_puda_get_tcpip_info()
1573 buf->iph = mem + buf->maclen; in irdma_gen1_puda_get_tcpip_info()
1574 iph = (struct iphdr *)buf->iph; in irdma_gen1_puda_get_tcpip_info()
1575 buf->tcph = buf->iph + iphlen; in irdma_gen1_puda_get_tcpip_info()
1576 tcph = (struct tcphdr *)buf->tcph; in irdma_gen1_puda_get_tcpip_info()
1578 if (buf->ipv4) { in irdma_gen1_puda_get_tcpip_info()
1579 pkt_len = ntohs(iph->tot_len); in irdma_gen1_puda_get_tcpip_info()
1581 ip6h = (struct ipv6hdr *)buf->iph; in irdma_gen1_puda_get_tcpip_info()
1582 pkt_len = ntohs(ip6h->payload_len) + iphlen; in irdma_gen1_puda_get_tcpip_info()
1585 buf->totallen = pkt_len + buf->maclen; in irdma_gen1_puda_get_tcpip_info()
1587 if (info->payload_len < buf->totallen) { in irdma_gen1_puda_get_tcpip_info()
1588 ibdev_dbg(to_ibdev(buf->vsi->dev), in irdma_gen1_puda_get_tcpip_info()
1590 info->payload_len, buf->totallen); in irdma_gen1_puda_get_tcpip_info()
1591 return -EINVAL; in irdma_gen1_puda_get_tcpip_info()
1594 buf->tcphlen = tcph->doff << 2; in irdma_gen1_puda_get_tcpip_info()
1595 buf->datalen = pkt_len - iphlen - buf->tcphlen; in irdma_gen1_puda_get_tcpip_info()
1596 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; in irdma_gen1_puda_get_tcpip_info()
1597 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; in irdma_gen1_puda_get_tcpip_info()
1598 buf->seqnum = ntohl(tcph->seq); in irdma_gen1_puda_get_tcpip_info()
1604 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1615 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_puda_get_tcpip_info()
1618 mem = buf->mem.va; in irdma_puda_get_tcpip_info()
1619 buf->vlan_valid = info->vlan_valid; in irdma_puda_get_tcpip_info()
1620 if (info->vlan_valid) in irdma_puda_get_tcpip_info()
1621 buf->vlan_id = info->vlan; in irdma_puda_get_tcpip_info()
1623 buf->ipv4 = info->ipv4; in irdma_puda_get_tcpip_info()
1624 if (buf->ipv4) in irdma_puda_get_tcpip_info()
1625 buf->iph = mem + IRDMA_IPV4_PAD; in irdma_puda_get_tcpip_info()
1627 buf->iph = mem; in irdma_puda_get_tcpip_info()
1629 buf->tcph = mem + IRDMA_TCP_OFFSET; in irdma_puda_get_tcpip_info()
1630 tcph = (struct tcphdr *)buf->tcph; in irdma_puda_get_tcpip_info()
1631 pkt_len = info->payload_len; in irdma_puda_get_tcpip_info()
1632 buf->totallen = pkt_len; in irdma_puda_get_tcpip_info()
1633 buf->tcphlen = tcph->doff << 2; in irdma_puda_get_tcpip_info()
1634 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; in irdma_puda_get_tcpip_info()
1635 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; in irdma_puda_get_tcpip_info()
1636 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; in irdma_puda_get_tcpip_info()
1637 buf->seqnum = ntohl(tcph->seq); in irdma_puda_get_tcpip_info()
1639 if (info->smac_valid) { in irdma_puda_get_tcpip_info()
1640 ether_addr_copy(buf->smac, info->smac); in irdma_puda_get_tcpip_info()
1641 buf->smac_valid = true; in irdma_puda_get_tcpip_info()
1648 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1655 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; in irdma_hw_stats_timeout()
1657 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) in irdma_hw_stats_timeout()
1658 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); in irdma_hw_stats_timeout()
1660 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); in irdma_hw_stats_timeout()
1662 mod_timer(&pf_devstat->stats_timer, in irdma_hw_stats_timeout()
1667 * irdma_hw_stats_start_timer - Start periodic stats timer
1672 struct irdma_vsi_pestat *devstat = vsi->pestat; in irdma_hw_stats_start_timer()
1674 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); in irdma_hw_stats_start_timer()
1675 mod_timer(&devstat->stats_timer, in irdma_hw_stats_start_timer()
1680 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1685 struct irdma_vsi_pestat *devstat = vsi->pestat; in irdma_hw_stats_stop_timer()
1687 del_timer_sync(&devstat->stats_timer); in irdma_hw_stats_stop_timer()
1691 * irdma_process_stats - Checking for wrap and update stats
1696 sc_vsi_update_stats(pestat->vsi); in irdma_process_stats()
1700 * irdma_cqp_gather_stats_gen1 - Gather stats
1708 pestat->gather_info.gather_stats_va; in irdma_cqp_gather_stats_gen1()
1709 const struct irdma_hw_stat_map *map = dev->hw_stats_map; in irdma_cqp_gather_stats_gen1()
1710 u16 max_stats_idx = dev->hw_attrs.max_stat_idx; in irdma_cqp_gather_stats_gen1()
1716 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? in irdma_cqp_gather_stats_gen1()
1717 pestat->gather_info.stats_inst_index : in irdma_cqp_gather_stats_gen1()
1718 pestat->hw->hmc.hmc_fn_id; in irdma_cqp_gather_stats_gen1()
1724 new_val = rd32(dev->hw, in irdma_cqp_gather_stats_gen1()
1725 dev->hw_stats_regs[i] + stats_inst_offset_32); in irdma_cqp_gather_stats_gen1()
1727 new_val = rd64(dev->hw, in irdma_cqp_gather_stats_gen1()
1728 dev->hw_stats_regs[i] + stats_inst_offset_64); in irdma_cqp_gather_stats_gen1()
1729 gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val; in irdma_cqp_gather_stats_gen1()
1736 * irdma_process_cqp_stats - Checking for wrap and update stats
1741 struct irdma_vsi_pestat *pestat = cqp_request->param; in irdma_process_cqp_stats()
1747 * irdma_cqp_gather_stats_cmd - Gather stats
1757 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_gather_stats_cmd()
1764 return -ENOMEM; in irdma_cqp_gather_stats_cmd()
1766 cqp_info = &cqp_request->info; in irdma_cqp_gather_stats_cmd()
1768 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; in irdma_cqp_gather_stats_cmd()
1769 cqp_info->post_sq = 1; in irdma_cqp_gather_stats_cmd()
1770 cqp_info->in.u.stats_gather.info = pestat->gather_info; in irdma_cqp_gather_stats_cmd()
1771 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; in irdma_cqp_gather_stats_cmd()
1772 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; in irdma_cqp_gather_stats_cmd()
1773 cqp_request->param = pestat; in irdma_cqp_gather_stats_cmd()
1775 cqp_request->callback_fcn = irdma_process_cqp_stats; in irdma_cqp_gather_stats_cmd()
1779 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_gather_stats_cmd()
1785 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1793 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); in irdma_cqp_stats_inst_cmd()
1794 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_stats_inst_cmd()
1804 return -ENOMEM; in irdma_cqp_stats_inst_cmd()
1806 cqp_info = &cqp_request->info; in irdma_cqp_stats_inst_cmd()
1808 cqp_info->cqp_cmd = cmd; in irdma_cqp_stats_inst_cmd()
1809 cqp_info->post_sq = 1; in irdma_cqp_stats_inst_cmd()
1810 cqp_info->in.u.stats_manage.info = *stats_info; in irdma_cqp_stats_inst_cmd()
1811 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; in irdma_cqp_stats_inst_cmd()
1812 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; in irdma_cqp_stats_inst_cmd()
1815 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; in irdma_cqp_stats_inst_cmd()
1822 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1835 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_ceq_cmd()
1837 return -ENOMEM; in irdma_cqp_ceq_cmd()
1839 cqp_info = &cqp_request->info; in irdma_cqp_ceq_cmd()
1840 cqp_info->post_sq = 1; in irdma_cqp_ceq_cmd()
1841 cqp_info->cqp_cmd = op; in irdma_cqp_ceq_cmd()
1842 cqp_info->in.u.ceq_create.ceq = sc_ceq; in irdma_cqp_ceq_cmd()
1843 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_ceq_cmd()
1846 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ceq_cmd()
1852 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1865 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_aeq_cmd()
1867 return -ENOMEM; in irdma_cqp_aeq_cmd()
1869 cqp_info = &cqp_request->info; in irdma_cqp_aeq_cmd()
1870 cqp_info->post_sq = 1; in irdma_cqp_aeq_cmd()
1871 cqp_info->cqp_cmd = op; in irdma_cqp_aeq_cmd()
1872 cqp_info->in.u.aeq_create.aeq = sc_aeq; in irdma_cqp_aeq_cmd()
1873 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_aeq_cmd()
1876 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_aeq_cmd()
1882 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1891 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_ws_node_cmd()
1892 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; in irdma_cqp_ws_node_cmd()
1896 bool poll; in irdma_cqp_ws_node_cmd() local
1898 if (!rf->sc_dev.ceq_valid) in irdma_cqp_ws_node_cmd()
1899 poll = true; in irdma_cqp_ws_node_cmd()
1901 poll = false; in irdma_cqp_ws_node_cmd()
1903 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); in irdma_cqp_ws_node_cmd()
1905 return -ENOMEM; in irdma_cqp_ws_node_cmd()
1907 cqp_info = &cqp_request->info; in irdma_cqp_ws_node_cmd()
1909 cqp_info->cqp_cmd = cmd; in irdma_cqp_ws_node_cmd()
1910 cqp_info->post_sq = 1; in irdma_cqp_ws_node_cmd()
1911 cqp_info->in.u.ws_node.info = *node_info; in irdma_cqp_ws_node_cmd()
1912 cqp_info->in.u.ws_node.cqp = cqp; in irdma_cqp_ws_node_cmd()
1913 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; in irdma_cqp_ws_node_cmd()
1918 if (poll) { in irdma_cqp_ws_node_cmd()
1923 node_info->qs_handle = compl_info.op_ret_val; in irdma_cqp_ws_node_cmd()
1924 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", in irdma_cqp_ws_node_cmd()
1927 node_info->qs_handle = cqp_request->compl_info.op_ret_val; in irdma_cqp_ws_node_cmd()
1931 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ws_node_cmd()
1937 * irdma_ah_cqp_op - perform an AH cqp operation
1957 return -EINVAL; in irdma_ah_cqp_op()
1959 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_ah_cqp_op()
1961 return -ENOMEM; in irdma_ah_cqp_op()
1963 cqp_info = &cqp_request->info; in irdma_ah_cqp_op()
1964 cqp_info->cqp_cmd = cmd; in irdma_ah_cqp_op()
1965 cqp_info->post_sq = 1; in irdma_ah_cqp_op()
1967 cqp_info->in.u.ah_create.info = sc_ah->ah_info; in irdma_ah_cqp_op()
1968 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; in irdma_ah_cqp_op()
1969 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; in irdma_ah_cqp_op()
1971 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; in irdma_ah_cqp_op()
1972 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; in irdma_ah_cqp_op()
1973 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; in irdma_ah_cqp_op()
1977 cqp_request->callback_fcn = callback_fcn; in irdma_ah_cqp_op()
1978 cqp_request->param = cb_param; in irdma_ah_cqp_op()
1981 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_ah_cqp_op()
1984 return -ENOMEM; in irdma_ah_cqp_op()
1987 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); in irdma_ah_cqp_op()
1993 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1998 struct irdma_sc_qp *qp = cqp_request->param; in irdma_ieq_ah_cb()
1999 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; in irdma_ieq_ah_cb()
2002 spin_lock_irqsave(&qp->pfpdu.lock, flags); in irdma_ieq_ah_cb()
2003 if (!cqp_request->compl_info.op_ret_val) { in irdma_ieq_ah_cb()
2004 sc_ah->ah_info.ah_valid = true; in irdma_ieq_ah_cb()
2005 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); in irdma_ieq_ah_cb()
2007 sc_ah->ah_info.ah_valid = false; in irdma_ieq_ah_cb()
2008 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); in irdma_ieq_ah_cb()
2010 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); in irdma_ieq_ah_cb()
2014 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
2019 struct irdma_cm_node *cm_node = cqp_request->param; in irdma_ilq_ah_cb()
2020 struct irdma_sc_ah *sc_ah = cm_node->ah; in irdma_ilq_ah_cb()
2022 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; in irdma_ilq_ah_cb()
2027 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2048 return -ENOMEM; in irdma_puda_create_ah()
2050 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, in irdma_puda_create_ah()
2051 &ah_info->ah_idx, &rf->next_ah); in irdma_puda_create_ah()
2055 ah->dev = dev; in irdma_puda_create_ah()
2056 ah->ah_info = *ah_info; in irdma_puda_create_ah()
2070 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); in irdma_puda_create_ah()
2074 return -ENOMEM; in irdma_puda_create_ah()
2078 * irdma_puda_free_ah - free a puda address handle
2089 if (ah->ah_info.ah_valid) { in irdma_puda_free_ah()
2091 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); in irdma_puda_free_ah()
2098 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2103 struct irdma_sc_ah *sc_ah = cqp_request->param; in irdma_gsi_ud_qp_ah_cb()
2105 if (!cqp_request->compl_info.op_ret_val) in irdma_gsi_ud_qp_ah_cb()
2106 sc_ah->ah_info.ah_valid = true; in irdma_gsi_ud_qp_ah_cb()
2108 sc_ah->ah_info.ah_valid = false; in irdma_gsi_ud_qp_ah_cb()
2112 * irdma_prm_add_pble_mem - add moemory to pble resources
2121 if (pchunk->size & 0xfff) in irdma_prm_add_pble_mem()
2122 return -EINVAL; in irdma_prm_add_pble_mem()
2124 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; in irdma_prm_add_pble_mem()
2126 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); in irdma_prm_add_pble_mem()
2127 if (!pchunk->bitmapbuf) in irdma_prm_add_pble_mem()
2128 return -ENOMEM; in irdma_prm_add_pble_mem()
2130 pchunk->sizeofbitmap = sizeofbitmap; in irdma_prm_add_pble_mem()
2132 pprm->total_pble_alloc += pchunk->size >> 3; in irdma_prm_add_pble_mem()
2133 pprm->free_pble_cnt += pchunk->size >> 3; in irdma_prm_add_pble_mem()
2139 * irdma_prm_get_pbles - get pble's from prm
2153 struct list_head *chunk_entry = pprm->clist.next; in irdma_prm_get_pbles()
2159 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); in irdma_prm_get_pbles()
2161 spin_lock_irqsave(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2162 while (chunk_entry != &pprm->clist) { in irdma_prm_get_pbles()
2164 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, in irdma_prm_get_pbles()
2165 pchunk->sizeofbitmap, 0, in irdma_prm_get_pbles()
2167 if (bit_idx < pchunk->sizeofbitmap) in irdma_prm_get_pbles()
2171 chunk_entry = pchunk->list.next; in irdma_prm_get_pbles()
2174 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { in irdma_prm_get_pbles()
2175 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2176 return -ENOMEM; in irdma_prm_get_pbles()
2179 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); in irdma_prm_get_pbles()
2180 offset = bit_idx << pprm->pble_shift; in irdma_prm_get_pbles()
2181 *vaddr = pchunk->vaddr + offset; in irdma_prm_get_pbles()
2182 *fpm_addr = pchunk->fpm_addr + offset; in irdma_prm_get_pbles()
2184 chunkinfo->pchunk = pchunk; in irdma_prm_get_pbles()
2185 chunkinfo->bit_idx = bit_idx; in irdma_prm_get_pbles()
2186 chunkinfo->bits_used = bits_needed; in irdma_prm_get_pbles()
2188 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); in irdma_prm_get_pbles()
2189 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2195 * irdma_prm_return_pbles - return pbles back to prm
2204 spin_lock_irqsave(&pprm->prm_lock, flags); in irdma_prm_return_pbles()
2205 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); in irdma_prm_return_pbles()
2206 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, in irdma_prm_return_pbles()
2207 chunkinfo->bits_used); in irdma_prm_return_pbles()
2208 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_return_pbles()
2224 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, in irdma_map_vm_page_list()
2226 if (dma_mapping_error(hw->device, pg_dma[i])) in irdma_map_vm_page_list()
2236 return -ENOMEM; in irdma_map_vm_page_list()
2244 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); in irdma_unmap_vm_page_list()
2248 * irdma_pble_free_paged_mem - free virtual paged memory
2253 if (!chunk->pg_cnt) in irdma_pble_free_paged_mem()
2256 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, in irdma_pble_free_paged_mem()
2257 chunk->pg_cnt); in irdma_pble_free_paged_mem()
2260 kfree(chunk->dmainfo.dmaaddrs); in irdma_pble_free_paged_mem()
2261 chunk->dmainfo.dmaaddrs = NULL; in irdma_pble_free_paged_mem()
2262 vfree(chunk->vaddr); in irdma_pble_free_paged_mem()
2263 chunk->vaddr = NULL; in irdma_pble_free_paged_mem()
2264 chunk->type = 0; in irdma_pble_free_paged_mem()
2268 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2277 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); in irdma_pble_get_paged_mem()
2278 if (!chunk->dmainfo.dmaaddrs) in irdma_pble_get_paged_mem()
2279 return -ENOMEM; in irdma_pble_get_paged_mem()
2286 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, in irdma_pble_get_paged_mem()
2291 chunk->vaddr = va; in irdma_pble_get_paged_mem()
2292 chunk->size = size; in irdma_pble_get_paged_mem()
2293 chunk->pg_cnt = pg_cnt; in irdma_pble_get_paged_mem()
2294 chunk->type = PBLE_SD_PAGED; in irdma_pble_get_paged_mem()
2298 kfree(chunk->dmainfo.dmaaddrs); in irdma_pble_get_paged_mem()
2299 chunk->dmainfo.dmaaddrs = NULL; in irdma_pble_get_paged_mem()
2301 return -ENOMEM; in irdma_pble_get_paged_mem()
2305 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2314 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, in irdma_alloc_ws_node_id()
2322 * irdma_free_ws_node_id - Free a tx scheduler node ID
2330 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); in irdma_free_ws_node_id()
2334 * irdma_modify_qp_to_err - Modify a QP to error
2339 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; in irdma_modify_qp_to_err()
2342 if (qp->iwdev->rf->reset) in irdma_modify_qp_to_err()
2346 if (rdma_protocol_roce(qp->ibqp.device, 1)) in irdma_modify_qp_to_err()
2347 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); in irdma_modify_qp_to_err()
2349 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); in irdma_modify_qp_to_err()
2356 if (!iwqp->ibqp.event_handler) in irdma_ib_qp_event()
2370 ibevent.device = iwqp->ibqp.device; in irdma_ib_qp_event()
2371 ibevent.element.qp = &iwqp->ibqp; in irdma_ib_qp_event()
2372 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); in irdma_ib_qp_event()
2382 ukcq = &iwcq->sc_cq.cq_uk; in irdma_cq_empty()
2387 return polarity != ukcq->polarity; in irdma_cq_empty()
2395 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) { in irdma_remove_cmpls_list()
2397 list_del(&cmpl_node->list); in irdma_remove_cmpls_list()
2406 if (list_empty(&iwcq->cmpl_generated)) in irdma_generated_cmpls()
2407 return -ENOENT; in irdma_generated_cmpls()
2408 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); in irdma_generated_cmpls()
2409 list_del(&cmpl->list); in irdma_generated_cmpls()
2410 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); in irdma_generated_cmpls()
2413 ibdev_dbg(iwcq->ibcq.device, in irdma_generated_cmpls()
2414 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n", in irdma_generated_cmpls()
2415 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, in irdma_generated_cmpls()
2416 cq_poll_info->wr_id); in irdma_generated_cmpls()
2422 * irdma_set_cpi_common_values - fill in values for polling info struct
2430 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; in irdma_set_cpi_common_values()
2431 cpi->error = true; in irdma_set_cpi_common_values()
2432 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; in irdma_set_cpi_common_values()
2433 cpi->minor_err = FLUSH_GENERAL_ERR; in irdma_set_cpi_common_values()
2434 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp; in irdma_set_cpi_common_values()
2435 cpi->qp_id = qp_num; in irdma_set_cpi_common_values()
2440 if (!cq->ibcq.comp_handler) in irdma_comp_handler()
2442 if (atomic_cmpxchg(&cq->armed, 1, 0)) in irdma_comp_handler()
2443 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in irdma_comp_handler()
2448 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; in irdma_generate_flush_completions()
2449 struct irdma_ring *sq_ring = &qp->sq_ring; in irdma_generate_flush_completions()
2450 struct irdma_ring *rq_ring = &qp->rq_ring; in irdma_generate_flush_completions()
2458 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2459 if (irdma_cq_empty(iwqp->iwscq)) { in irdma_generate_flush_completions()
2462 spin_lock_irqsave(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2466 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2467 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2471 wqe_idx = sq_ring->tail; in irdma_generate_flush_completions()
2472 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); in irdma_generate_flush_completions()
2474 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; in irdma_generate_flush_completions()
2475 sw_wqe = qp->sq_base[wqe_idx].elem; in irdma_generate_flush_completions()
2477 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); in irdma_generate_flush_completions()
2478 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; in irdma_generate_flush_completions()
2481 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); in irdma_generate_flush_completions()
2482 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { in irdma_generate_flush_completions()
2486 ibdev_dbg(iwqp->iwscq->ibcq.device, in irdma_generate_flush_completions()
2488 __func__, cmpl->cpi.wr_id, qp->qp_id); in irdma_generate_flush_completions()
2489 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); in irdma_generate_flush_completions()
2492 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2493 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2495 irdma_comp_handler(iwqp->iwscq); in irdma_generate_flush_completions()
2497 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2498 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, in irdma_generate_flush_completions()
2502 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2503 if (irdma_cq_empty(iwqp->iwrcq)) { in irdma_generate_flush_completions()
2506 spin_lock_irqsave(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2510 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2511 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2515 wqe_idx = rq_ring->tail; in irdma_generate_flush_completions()
2516 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); in irdma_generate_flush_completions()
2518 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; in irdma_generate_flush_completions()
2519 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; in irdma_generate_flush_completions()
2520 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; in irdma_generate_flush_completions()
2522 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); in irdma_generate_flush_completions()
2523 ibdev_dbg(iwqp->iwrcq->ibcq.device, in irdma_generate_flush_completions()
2525 __func__, cmpl->cpi.wr_id, qp->qp_id, in irdma_generate_flush_completions()
2527 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); in irdma_generate_flush_completions()
2531 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2532 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2534 irdma_comp_handler(iwqp->iwrcq); in irdma_generate_flush_completions()
2536 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2537 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, in irdma_generate_flush_completions()