Lines Matching +full:dcb +full:- +full:algorithm
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
13 * irdma_get_qp_from_list - get next qp from a list
27 entry = head->next; in irdma_get_qp_from_list()
29 lastentry = &qp->list; in irdma_get_qp_from_list()
30 entry = lastentry->next; in irdma_get_qp_from_list()
39 * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI
49 mutex_lock(&vsi->qos[i].qos_mutex); in irdma_sc_suspend_resume_qps()
50 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); in irdma_sc_suspend_resume_qps()
53 if (!qp->dev->ws_add(vsi, i)) { in irdma_sc_suspend_resume_qps()
54 qp->qs_handle = in irdma_sc_suspend_resume_qps()
55 vsi->qos[qp->user_pri].qs_handle; in irdma_sc_suspend_resume_qps()
64 atomic_inc(&vsi->qp_suspend_reqs); in irdma_sc_suspend_resume_qps()
66 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); in irdma_sc_suspend_resume_qps()
68 mutex_unlock(&vsi->qos[i].qos_mutex); in irdma_sc_suspend_resume_qps()
77 vsi->qos_rel_bw = l2p->vsi_rel_bw; in irdma_set_qos_info()
78 vsi->qos_prio_type = l2p->vsi_prio_type; in irdma_set_qos_info()
79 vsi->dscp_mode = l2p->dscp_mode; in irdma_set_qos_info()
80 if (l2p->dscp_mode) { in irdma_set_qos_info()
81 memcpy(vsi->dscp_map, l2p->dscp_map, sizeof(vsi->dscp_map)); in irdma_set_qos_info()
83 l2p->up2tc[i] = i; in irdma_set_qos_info()
86 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_set_qos_info()
87 vsi->qos[i].qs_handle = l2p->qs_handle_list[i]; in irdma_set_qos_info()
88 vsi->qos[i].traffic_class = l2p->up2tc[i]; in irdma_set_qos_info()
89 vsi->qos[i].rel_bw = in irdma_set_qos_info()
90 l2p->tc_info[vsi->qos[i].traffic_class].rel_bw; in irdma_set_qos_info()
91 vsi->qos[i].prio_type = in irdma_set_qos_info()
92 l2p->tc_info[vsi->qos[i].traffic_class].prio_type; in irdma_set_qos_info()
93 vsi->qos[i].valid = false; in irdma_set_qos_info()
98 * irdma_change_l2params - given the new l2 parameters, change all qp
105 if (l2params->mtu_changed) { in irdma_change_l2params()
106 vsi->mtu = l2params->mtu; in irdma_change_l2params()
107 if (vsi->ieq) in irdma_change_l2params()
111 if (!l2params->tc_changed) in irdma_change_l2params()
114 vsi->tc_change_pending = false; in irdma_change_l2params()
120 * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
125 struct irdma_sc_vsi *vsi = qp->vsi; in irdma_qp_rem_qos()
127 ibdev_dbg(to_ibdev(qp->dev), in irdma_qp_rem_qos()
128 "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", in irdma_qp_rem_qos()
129 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, in irdma_qp_rem_qos()
130 qp->on_qoslist); in irdma_qp_rem_qos()
131 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); in irdma_qp_rem_qos()
132 if (qp->on_qoslist) { in irdma_qp_rem_qos()
133 qp->on_qoslist = false; in irdma_qp_rem_qos()
134 list_del(&qp->list); in irdma_qp_rem_qos()
136 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex); in irdma_qp_rem_qos()
140 * irdma_qp_add_qos - called during setctx for qp to be added to qos
145 struct irdma_sc_vsi *vsi = qp->vsi; in irdma_qp_add_qos()
147 ibdev_dbg(to_ibdev(qp->dev), in irdma_qp_add_qos()
148 "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", in irdma_qp_add_qos()
149 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, in irdma_qp_add_qos()
150 qp->on_qoslist); in irdma_qp_add_qos()
151 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); in irdma_qp_add_qos()
152 if (!qp->on_qoslist) { in irdma_qp_add_qos()
153 list_add(&qp->list, &vsi->qos[qp->user_pri].qplist); in irdma_qp_add_qos()
154 qp->on_qoslist = true; in irdma_qp_add_qos()
155 qp->qs_handle = vsi->qos[qp->user_pri].qs_handle; in irdma_qp_add_qos()
157 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex); in irdma_qp_add_qos()
161 * irdma_sc_pd_init - initialize sc pd struct
170 pd->pd_id = pd_id; in irdma_sc_pd_init()
171 pd->abi_ver = abi_ver; in irdma_sc_pd_init()
172 pd->dev = dev; in irdma_sc_pd_init()
176 * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
191 return -ENOMEM; in irdma_sc_add_arp_cache_entry()
192 set_64bit_val(wqe, 8, info->reach_max); in irdma_sc_add_arp_cache_entry()
193 set_64bit_val(wqe, 16, ether_addr_to_u64(info->mac_addr)); in irdma_sc_add_arp_cache_entry()
195 hdr = info->arp_index | in irdma_sc_add_arp_cache_entry()
197 FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info->permanent ? 1 : 0)) | in irdma_sc_add_arp_cache_entry()
199 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_arp_cache_entry()
213 * irdma_sc_del_arp_cache_entry - dele arp cache entry
227 return -ENOMEM; in irdma_sc_del_arp_cache_entry()
231 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_del_arp_cache_entry()
246 * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
261 return -ENOMEM; in irdma_sc_manage_apbvt_entry()
263 set_64bit_val(wqe, 16, info->port); in irdma_sc_manage_apbvt_entry()
266 FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) | in irdma_sc_manage_apbvt_entry()
267 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_apbvt_entry()
281 * irdma_sc_manage_qhash_table_entry - manage quad hash entries
308 struct irdma_sc_vsi *vsi = info->vsi; in irdma_sc_manage_qhash_table_entry()
312 return -ENOMEM; in irdma_sc_manage_qhash_table_entry()
314 set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr)); in irdma_sc_manage_qhash_table_entry()
316 qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) | in irdma_sc_manage_qhash_table_entry()
317 FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port); in irdma_sc_manage_qhash_table_entry()
318 if (info->ipv4_valid) { in irdma_sc_manage_qhash_table_entry()
320 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0])); in irdma_sc_manage_qhash_table_entry()
323 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) | in irdma_sc_manage_qhash_table_entry()
324 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1])); in irdma_sc_manage_qhash_table_entry()
327 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) | in irdma_sc_manage_qhash_table_entry()
328 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3])); in irdma_sc_manage_qhash_table_entry()
331 vsi->qos[info->user_pri].qs_handle); in irdma_sc_manage_qhash_table_entry()
332 if (info->vlan_valid) in irdma_sc_manage_qhash_table_entry()
333 qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id); in irdma_sc_manage_qhash_table_entry()
335 if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) { in irdma_sc_manage_qhash_table_entry()
336 qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port); in irdma_sc_manage_qhash_table_entry()
337 if (!info->ipv4_valid) { in irdma_sc_manage_qhash_table_entry()
339 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) | in irdma_sc_manage_qhash_table_entry()
340 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1])); in irdma_sc_manage_qhash_table_entry()
342 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) | in irdma_sc_manage_qhash_table_entry()
343 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3])); in irdma_sc_manage_qhash_table_entry()
346 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0])); in irdma_sc_manage_qhash_table_entry()
351 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | in irdma_sc_manage_qhash_table_entry()
354 FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) | in irdma_sc_manage_qhash_table_entry()
355 FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) | in irdma_sc_manage_qhash_table_entry()
356 FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) | in irdma_sc_manage_qhash_table_entry()
357 FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type); in irdma_sc_manage_qhash_table_entry()
371 * irdma_sc_qp_init - initialize qp
381 if (info->qp_uk_init_info.max_sq_frag_cnt > in irdma_sc_qp_init()
382 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags || in irdma_sc_qp_init()
383 info->qp_uk_init_info.max_rq_frag_cnt > in irdma_sc_qp_init()
384 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags) in irdma_sc_qp_init()
385 return -EINVAL; in irdma_sc_qp_init()
387 qp->dev = info->pd->dev; in irdma_sc_qp_init()
388 qp->vsi = info->vsi; in irdma_sc_qp_init()
389 qp->ieq_qp = info->vsi->exception_lan_q; in irdma_sc_qp_init()
390 qp->sq_pa = info->sq_pa; in irdma_sc_qp_init()
391 qp->rq_pa = info->rq_pa; in irdma_sc_qp_init()
392 qp->hw_host_ctx_pa = info->host_ctx_pa; in irdma_sc_qp_init()
393 qp->q2_pa = info->q2_pa; in irdma_sc_qp_init()
394 qp->shadow_area_pa = info->shadow_area_pa; in irdma_sc_qp_init()
395 qp->q2_buf = info->q2; in irdma_sc_qp_init()
396 qp->pd = info->pd; in irdma_sc_qp_init()
397 qp->hw_host_ctx = info->host_ctx; in irdma_sc_qp_init()
398 info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db; in irdma_sc_qp_init()
399 ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info); in irdma_sc_qp_init()
403 qp->virtual_map = info->virtual_map; in irdma_sc_qp_init()
404 pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_qp_init()
406 if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) || in irdma_sc_qp_init()
407 (info->virtual_map && info->rq_pa >= pble_obj_cnt)) in irdma_sc_qp_init()
408 return -EINVAL; in irdma_sc_qp_init()
410 qp->llp_stream_handle = (void *)(-1); in irdma_sc_qp_init()
411 qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, in irdma_sc_qp_init()
413 ibdev_dbg(to_ibdev(qp->dev), in irdma_sc_qp_init()
415 qp->hw_sq_size, qp->qp_uk.sq_ring.size); in irdma_sc_qp_init()
416 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4) in irdma_sc_qp_init()
419 ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt, in irdma_sc_qp_init()
424 qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size * in irdma_sc_qp_init()
426 ibdev_dbg(to_ibdev(qp->dev), in irdma_sc_qp_init()
428 qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size); in irdma_sc_qp_init()
429 qp->sq_tph_val = info->sq_tph_val; in irdma_sc_qp_init()
430 qp->rq_tph_val = info->rq_tph_val; in irdma_sc_qp_init()
431 qp->sq_tph_en = info->sq_tph_en; in irdma_sc_qp_init()
432 qp->rq_tph_en = info->rq_tph_en; in irdma_sc_qp_init()
433 qp->rcv_tph_en = info->rcv_tph_en; in irdma_sc_qp_init()
434 qp->xmit_tph_en = info->xmit_tph_en; in irdma_sc_qp_init()
435 qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq; in irdma_sc_qp_init()
436 qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; in irdma_sc_qp_init()
442 * irdma_sc_qp_create - create qp
455 cqp = qp->dev->cqp; in irdma_sc_qp_create()
456 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || in irdma_sc_qp_create()
457 qp->qp_uk.qp_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt) in irdma_sc_qp_create()
458 return -EINVAL; in irdma_sc_qp_create()
462 return -ENOMEM; in irdma_sc_qp_create()
464 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); in irdma_sc_qp_create()
465 set_64bit_val(wqe, 40, qp->shadow_area_pa); in irdma_sc_qp_create()
467 hdr = qp->qp_uk.qp_id | in irdma_sc_qp_create()
469 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info->ord_valid ? 1 : 0)) | in irdma_sc_qp_create()
470 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | in irdma_sc_qp_create()
471 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | in irdma_sc_qp_create()
472 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | in irdma_sc_qp_create()
473 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | in irdma_sc_qp_create()
474 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | in irdma_sc_qp_create()
475 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | in irdma_sc_qp_create()
477 info->arp_cache_idx_valid) | in irdma_sc_qp_create()
478 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | in irdma_sc_qp_create()
479 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_create()
493 * irdma_sc_qp_modify - modify qp cqp wqe
508 cqp = qp->dev->cqp; in irdma_sc_qp_modify()
511 return -ENOMEM; in irdma_sc_qp_modify()
513 if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) { in irdma_sc_qp_modify()
514 if (info->dont_send_fin) in irdma_sc_qp_modify()
516 if (info->dont_send_term) in irdma_sc_qp_modify()
520 term_len = info->termlen; in irdma_sc_qp_modify()
524 FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) | in irdma_sc_qp_modify()
526 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); in irdma_sc_qp_modify()
527 set_64bit_val(wqe, 40, qp->shadow_area_pa); in irdma_sc_qp_modify()
529 hdr = qp->qp_uk.qp_id | in irdma_sc_qp_modify()
531 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) | in irdma_sc_qp_modify()
532 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | in irdma_sc_qp_modify()
534 info->cached_var_valid) | in irdma_sc_qp_modify()
535 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | in irdma_sc_qp_modify()
536 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | in irdma_sc_qp_modify()
537 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | in irdma_sc_qp_modify()
538 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | in irdma_sc_qp_modify()
539 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | in irdma_sc_qp_modify()
540 FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) | in irdma_sc_qp_modify()
542 info->remove_hash_idx) | in irdma_sc_qp_modify()
544 FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) | in irdma_sc_qp_modify()
546 info->arp_cache_idx_valid) | in irdma_sc_qp_modify()
547 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | in irdma_sc_qp_modify()
548 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_modify()
562 * irdma_sc_qp_destroy - cqp destroy qp
576 cqp = qp->dev->cqp; in irdma_sc_qp_destroy()
579 return -ENOMEM; in irdma_sc_qp_destroy()
581 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa); in irdma_sc_qp_destroy()
582 set_64bit_val(wqe, 40, qp->shadow_area_pa); in irdma_sc_qp_destroy()
584 hdr = qp->qp_uk.qp_id | in irdma_sc_qp_destroy()
586 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | in irdma_sc_qp_destroy()
589 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_destroy()
603 * irdma_sc_get_encoded_ird_size -
632 * irdma_sc_qp_setctx_roce - set qp's context
645 roce_info = info->roce_info; in irdma_sc_qp_setctx_roce()
646 udp = info->udp_info; in irdma_sc_qp_setctx_roce()
647 qp->user_pri = info->user_pri; in irdma_sc_qp_setctx_roce()
648 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { in irdma_sc_qp_setctx_roce()
653 push_idx = qp->push_idx; in irdma_sc_qp_setctx_roce()
656 FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | in irdma_sc_qp_setctx_roce()
657 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | in irdma_sc_qp_setctx_roce()
658 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | in irdma_sc_qp_setctx_roce()
659 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | in irdma_sc_qp_setctx_roce()
660 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | in irdma_sc_qp_setctx_roce()
663 FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) | in irdma_sc_qp_setctx_roce()
664 FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) | in irdma_sc_qp_setctx_roce()
665 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) | in irdma_sc_qp_setctx_roce()
666 FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) | in irdma_sc_qp_setctx_roce()
667 FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) | in irdma_sc_qp_setctx_roce()
668 FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) | in irdma_sc_qp_setctx_roce()
669 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag)); in irdma_sc_qp_setctx_roce()
670 set_64bit_val(qp_ctx, 8, qp->sq_pa); in irdma_sc_qp_setctx_roce()
671 set_64bit_val(qp_ctx, 16, qp->rq_pa); in irdma_sc_qp_setctx_roce()
672 if ((roce_info->dcqcn_en || roce_info->dctcp_en) && in irdma_sc_qp_setctx_roce()
673 !(udp->tos & 0x03)) in irdma_sc_qp_setctx_roce()
674 udp->tos |= ECN_CODE_PT_VAL; in irdma_sc_qp_setctx_roce()
676 FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | in irdma_sc_qp_setctx_roce()
677 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) | in irdma_sc_qp_setctx_roce()
678 FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) | in irdma_sc_qp_setctx_roce()
679 FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) | in irdma_sc_qp_setctx_roce()
680 FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port)); in irdma_sc_qp_setctx_roce()
682 FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) | in irdma_sc_qp_setctx_roce()
683 FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3])); in irdma_sc_qp_setctx_roce()
685 FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) | in irdma_sc_qp_setctx_roce()
686 FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1])); in irdma_sc_qp_setctx_roce()
688 FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) | in irdma_sc_qp_setctx_roce()
689 FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) | in irdma_sc_qp_setctx_roce()
690 FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx)); in irdma_sc_qp_setctx_roce()
692 FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) | in irdma_sc_qp_setctx_roce()
693 FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) | in irdma_sc_qp_setctx_roce()
694 FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) | in irdma_sc_qp_setctx_roce()
695 FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label)); in irdma_sc_qp_setctx_roce()
697 FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) | in irdma_sc_qp_setctx_roce()
698 FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp)); in irdma_sc_qp_setctx_roce()
700 FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) | in irdma_sc_qp_setctx_roce()
701 FIELD_PREP(IRDMAQPC_LSN, udp->lsn)); in irdma_sc_qp_setctx_roce()
703 FIELD_PREP(IRDMAQPC_EPSN, udp->epsn)); in irdma_sc_qp_setctx_roce()
705 FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) | in irdma_sc_qp_setctx_roce()
706 FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una)); in irdma_sc_qp_setctx_roce()
708 FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd)); in irdma_sc_qp_setctx_roce()
710 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) | in irdma_sc_qp_setctx_roce()
711 FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) | in irdma_sc_qp_setctx_roce()
712 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) | in irdma_sc_qp_setctx_roce()
713 FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin)); in irdma_sc_qp_setctx_roce()
715 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | in irdma_sc_qp_setctx_roce()
716 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); in irdma_sc_qp_setctx_roce()
718 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); in irdma_sc_qp_setctx_roce()
719 set_64bit_val(qp_ctx, 152, ether_addr_to_u64(roce_info->mac_addr) << 16); in irdma_sc_qp_setctx_roce()
721 FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) | in irdma_sc_qp_setctx_roce()
722 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) | in irdma_sc_qp_setctx_roce()
723 FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) | in irdma_sc_qp_setctx_roce()
724 FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) | in irdma_sc_qp_setctx_roce()
725 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | in irdma_sc_qp_setctx_roce()
726 FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) | in irdma_sc_qp_setctx_roce()
727 FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) | in irdma_sc_qp_setctx_roce()
728 FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) | in irdma_sc_qp_setctx_roce()
729 FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) | in irdma_sc_qp_setctx_roce()
730 FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) | in irdma_sc_qp_setctx_roce()
731 FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) | in irdma_sc_qp_setctx_roce()
732 FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) | in irdma_sc_qp_setctx_roce()
733 FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en)); in irdma_sc_qp_setctx_roce()
735 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); in irdma_sc_qp_setctx_roce()
737 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | in irdma_sc_qp_setctx_roce()
738 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | in irdma_sc_qp_setctx_roce()
739 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle)); in irdma_sc_qp_setctx_roce()
741 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) | in irdma_sc_qp_setctx_roce()
742 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2])); in irdma_sc_qp_setctx_roce()
744 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) | in irdma_sc_qp_setctx_roce()
745 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0])); in irdma_sc_qp_setctx_roce()
747 FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) | in irdma_sc_qp_setctx_roce()
748 FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low)); in irdma_sc_qp_setctx_roce()
750 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); in irdma_sc_qp_setctx_roce()
756 /* irdma_sc_alloc_local_mac_entry - allocate a mac entry
769 return -ENOMEM; in irdma_sc_alloc_local_mac_entry()
773 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_local_mac_entry()
789 * irdma_sc_add_local_mac_entry - add mac enry
804 return -ENOMEM; in irdma_sc_add_local_mac_entry()
806 set_64bit_val(wqe, 32, ether_addr_to_u64(info->mac_addr)); in irdma_sc_add_local_mac_entry()
808 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) | in irdma_sc_add_local_mac_entry()
811 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_local_mac_entry()
826 * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
842 return -ENOMEM; in irdma_sc_del_local_mac_entry()
847 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_del_local_mac_entry()
864 * irdma_sc_qp_setctx - set qp's context
880 iw = info->iwarp_info; in irdma_sc_qp_setctx()
881 tcp = info->tcp_info; in irdma_sc_qp_setctx()
882 dev = qp->dev; in irdma_sc_qp_setctx()
883 if (iw->rcv_mark_en) { in irdma_sc_qp_setctx()
884 qp->pfpdu.marker_len = 4; in irdma_sc_qp_setctx()
885 qp->pfpdu.rcv_start_seq = tcp->rcv_nxt; in irdma_sc_qp_setctx()
887 qp->user_pri = info->user_pri; in irdma_sc_qp_setctx()
888 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { in irdma_sc_qp_setctx()
893 push_idx = qp->push_idx; in irdma_sc_qp_setctx()
895 qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | in irdma_sc_qp_setctx()
896 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | in irdma_sc_qp_setctx()
897 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | in irdma_sc_qp_setctx()
898 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | in irdma_sc_qp_setctx()
899 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | in irdma_sc_qp_setctx()
903 set_64bit_val(qp_ctx, 8, qp->sq_pa); in irdma_sc_qp_setctx()
904 set_64bit_val(qp_ctx, 16, qp->rq_pa); in irdma_sc_qp_setctx()
906 qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | in irdma_sc_qp_setctx()
907 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size); in irdma_sc_qp_setctx()
908 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_sc_qp_setctx()
910 qp->src_mac_addr_idx); in irdma_sc_qp_setctx()
912 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | in irdma_sc_qp_setctx()
913 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); in irdma_sc_qp_setctx()
915 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); in irdma_sc_qp_setctx()
917 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | in irdma_sc_qp_setctx()
918 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | in irdma_sc_qp_setctx()
919 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) | in irdma_sc_qp_setctx()
920 FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp)); in irdma_sc_qp_setctx()
921 if (info->iwarp_info_valid) { in irdma_sc_qp_setctx()
922 qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) | in irdma_sc_qp_setctx()
923 FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) | in irdma_sc_qp_setctx()
924 FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) | in irdma_sc_qp_setctx()
925 FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) | in irdma_sc_qp_setctx()
926 FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) | in irdma_sc_qp_setctx()
927 FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) | in irdma_sc_qp_setctx()
929 iw->err_rq_idx_valid); in irdma_sc_qp_setctx()
930 qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id); in irdma_sc_qp_setctx()
931 qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) | in irdma_sc_qp_setctx()
932 FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin); in irdma_sc_qp_setctx()
934 FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) | in irdma_sc_qp_setctx()
935 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); in irdma_sc_qp_setctx()
937 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) in irdma_sc_qp_setctx()
938 mac = ether_addr_to_u64(iw->mac_addr); in irdma_sc_qp_setctx()
941 mac << 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent)); in irdma_sc_qp_setctx()
943 FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) | in irdma_sc_qp_setctx()
944 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) | in irdma_sc_qp_setctx()
945 FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) | in irdma_sc_qp_setctx()
946 FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) | in irdma_sc_qp_setctx()
947 FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) | in irdma_sc_qp_setctx()
948 FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) | in irdma_sc_qp_setctx()
949 FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) | in irdma_sc_qp_setctx()
950 FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) | in irdma_sc_qp_setctx()
951 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | in irdma_sc_qp_setctx()
953 FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) | in irdma_sc_qp_setctx()
954 FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) | in irdma_sc_qp_setctx()
955 FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) | in irdma_sc_qp_setctx()
956 …FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nx… in irdma_sc_qp_setctx()
957 …FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nx… in irdma_sc_qp_setctx()
958 FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en)); in irdma_sc_qp_setctx()
960 if (info->tcp_info_valid) { in irdma_sc_qp_setctx()
961 qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) | in irdma_sc_qp_setctx()
962 FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) | in irdma_sc_qp_setctx()
964 tcp->insert_vlan_tag) | in irdma_sc_qp_setctx()
965 FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) | in irdma_sc_qp_setctx()
966 FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) | in irdma_sc_qp_setctx()
967 FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) | in irdma_sc_qp_setctx()
968 FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh); in irdma_sc_qp_setctx()
970 if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03)) in irdma_sc_qp_setctx()
971 tcp->tos |= ECN_CODE_PT_VAL; in irdma_sc_qp_setctx()
973 qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) | in irdma_sc_qp_setctx()
974 FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) | in irdma_sc_qp_setctx()
975 FIELD_PREP(IRDMAQPC_TOS, tcp->tos) | in irdma_sc_qp_setctx()
976 FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) | in irdma_sc_qp_setctx()
977 FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port); in irdma_sc_qp_setctx()
978 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { in irdma_sc_qp_setctx()
979 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx); in irdma_sc_qp_setctx()
981 qp->src_mac_addr_idx = tcp->src_mac_addr_idx; in irdma_sc_qp_setctx()
984 FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) | in irdma_sc_qp_setctx()
985 FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3])); in irdma_sc_qp_setctx()
987 FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) | in irdma_sc_qp_setctx()
988 FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1])); in irdma_sc_qp_setctx()
990 FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) | in irdma_sc_qp_setctx()
991 FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) | in irdma_sc_qp_setctx()
992 FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) | in irdma_sc_qp_setctx()
993 FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx)); in irdma_sc_qp_setctx()
994 qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) | in irdma_sc_qp_setctx()
995 FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) | in irdma_sc_qp_setctx()
997 tcp->ignore_tcp_opt) | in irdma_sc_qp_setctx()
999 tcp->ignore_tcp_uns_opt) | in irdma_sc_qp_setctx()
1000 FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) | in irdma_sc_qp_setctx()
1001 FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) | in irdma_sc_qp_setctx()
1002 FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale); in irdma_sc_qp_setctx()
1004 FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) | in irdma_sc_qp_setctx()
1005 FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age)); in irdma_sc_qp_setctx()
1007 FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) | in irdma_sc_qp_setctx()
1008 FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd)); in irdma_sc_qp_setctx()
1010 FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) | in irdma_sc_qp_setctx()
1011 FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd)); in irdma_sc_qp_setctx()
1013 FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) | in irdma_sc_qp_setctx()
1014 FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una)); in irdma_sc_qp_setctx()
1016 FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) | in irdma_sc_qp_setctx()
1017 FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var)); in irdma_sc_qp_setctx()
1019 FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) | in irdma_sc_qp_setctx()
1020 FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd)); in irdma_sc_qp_setctx()
1022 FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) | in irdma_sc_qp_setctx()
1023 FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2)); in irdma_sc_qp_setctx()
1024 qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) | in irdma_sc_qp_setctx()
1025 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh); in irdma_sc_qp_setctx()
1027 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) | in irdma_sc_qp_setctx()
1028 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2])); in irdma_sc_qp_setctx()
1030 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) | in irdma_sc_qp_setctx()
1031 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0])); in irdma_sc_qp_setctx()
1033 FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) | in irdma_sc_qp_setctx()
1034 FIELD_PREP(IRDMAQPC_TLOW, iw->t_low)); in irdma_sc_qp_setctx()
1036 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); in irdma_sc_qp_setctx()
1049 * irdma_sc_alloc_stag - mr stag alloc
1064 if (!info->total_len && !info->all_memory) in irdma_sc_alloc_stag()
1065 return -EINVAL; in irdma_sc_alloc_stag()
1067 if (info->page_size == 0x40000000) in irdma_sc_alloc_stag()
1069 else if (info->page_size == 0x200000) in irdma_sc_alloc_stag()
1074 cqp = dev->cqp; in irdma_sc_alloc_stag()
1077 return -ENOMEM; in irdma_sc_alloc_stag()
1080 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) | in irdma_sc_alloc_stag()
1081 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len)); in irdma_sc_alloc_stag()
1083 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); in irdma_sc_alloc_stag()
1085 FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index)); in irdma_sc_alloc_stag()
1087 if (info->chunk_size) in irdma_sc_alloc_stag()
1089 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx)); in irdma_sc_alloc_stag()
1093 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | in irdma_sc_alloc_stag()
1094 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | in irdma_sc_alloc_stag()
1096 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) | in irdma_sc_alloc_stag()
1097 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | in irdma_sc_alloc_stag()
1098 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | in irdma_sc_alloc_stag()
1099 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_stag()
1113 * irdma_sc_mr_reg_non_shared - non-shared mr registration
1132 if (!info->total_len && !info->all_memory) in irdma_sc_mr_reg_non_shared()
1133 return -EINVAL; in irdma_sc_mr_reg_non_shared()
1135 if (info->page_size == 0x40000000) in irdma_sc_mr_reg_non_shared()
1137 else if (info->page_size == 0x200000) in irdma_sc_mr_reg_non_shared()
1139 else if (info->page_size == 0x1000) in irdma_sc_mr_reg_non_shared()
1142 return -EINVAL; in irdma_sc_mr_reg_non_shared()
1144 if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY | in irdma_sc_mr_reg_non_shared()
1150 pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_mr_reg_non_shared()
1151 if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt) in irdma_sc_mr_reg_non_shared()
1152 return -EINVAL; in irdma_sc_mr_reg_non_shared()
1154 cqp = dev->cqp; in irdma_sc_mr_reg_non_shared()
1157 return -ENOMEM; in irdma_sc_mr_reg_non_shared()
1158 fbo = info->va & (info->page_size - 1); in irdma_sc_mr_reg_non_shared()
1161 (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ? in irdma_sc_mr_reg_non_shared()
1162 info->va : fbo)); in irdma_sc_mr_reg_non_shared()
1164 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) | in irdma_sc_mr_reg_non_shared()
1165 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); in irdma_sc_mr_reg_non_shared()
1167 FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) | in irdma_sc_mr_reg_non_shared()
1168 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); in irdma_sc_mr_reg_non_shared()
1169 if (!info->chunk_size) { in irdma_sc_mr_reg_non_shared()
1170 set_64bit_val(wqe, 32, info->reg_addr_pa); in irdma_sc_mr_reg_non_shared()
1175 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index)); in irdma_sc_mr_reg_non_shared()
1177 set_64bit_val(wqe, 40, info->hmc_fcn_index); in irdma_sc_mr_reg_non_shared()
1180 addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0; in irdma_sc_mr_reg_non_shared()
1183 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | in irdma_sc_mr_reg_non_shared()
1185 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | in irdma_sc_mr_reg_non_shared()
1188 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | in irdma_sc_mr_reg_non_shared()
1189 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | in irdma_sc_mr_reg_non_shared()
1190 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mr_reg_non_shared()
1204 * irdma_sc_dealloc_stag - deallocate stag
1218 cqp = dev->cqp; in irdma_sc_dealloc_stag()
1221 return -ENOMEM; in irdma_sc_dealloc_stag()
1224 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); in irdma_sc_dealloc_stag()
1226 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); in irdma_sc_dealloc_stag()
1229 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) | in irdma_sc_dealloc_stag()
1230 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_dealloc_stag()
1244 * irdma_sc_mw_alloc - mw allocate
1258 cqp = dev->cqp; in irdma_sc_mw_alloc()
1261 return -ENOMEM; in irdma_sc_mw_alloc()
1264 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); in irdma_sc_mw_alloc()
1266 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index)); in irdma_sc_mw_alloc()
1269 FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) | in irdma_sc_mw_alloc()
1271 info->mw1_bind_dont_vldt_key) | in irdma_sc_mw_alloc()
1272 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mw_alloc()
1286 * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
1301 if (info->page_size == 0x40000000) in irdma_sc_mr_fast_register()
1303 else if (info->page_size == 0x200000) in irdma_sc_mr_fast_register()
1308 sq_info.wr_id = info->wr_id; in irdma_sc_mr_fast_register()
1309 sq_info.signaled = info->signaled; in irdma_sc_mr_fast_register()
1311 wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, in irdma_sc_mr_fast_register()
1314 return -ENOMEM; in irdma_sc_mr_fast_register()
1316 irdma_clr_wqes(&qp->qp_uk, wqe_idx); in irdma_sc_mr_fast_register()
1318 ibdev_dbg(to_ibdev(qp->dev), in irdma_sc_mr_fast_register()
1320 info->wr_id, wqe_idx, in irdma_sc_mr_fast_register()
1321 &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); in irdma_sc_mr_fast_register()
1323 temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? in irdma_sc_mr_fast_register()
1324 (uintptr_t)info->va : info->fbo; in irdma_sc_mr_fast_register()
1328 info->first_pm_pbl_index >> 16); in irdma_sc_mr_fast_register()
1331 FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa)); in irdma_sc_mr_fast_register()
1333 info->total_len | in irdma_sc_mr_fast_register()
1334 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index)); in irdma_sc_mr_fast_register()
1336 hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) | in irdma_sc_mr_fast_register()
1337 FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) | in irdma_sc_mr_fast_register()
1339 FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) | in irdma_sc_mr_fast_register()
1341 FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) | in irdma_sc_mr_fast_register()
1342 FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) | in irdma_sc_mr_fast_register()
1343 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | in irdma_sc_mr_fast_register()
1344 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | in irdma_sc_mr_fast_register()
1345 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | in irdma_sc_mr_fast_register()
1346 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); in irdma_sc_mr_fast_register()
1355 irdma_uk_qp_post_wr(&qp->qp_uk); in irdma_sc_mr_fast_register()
1361 * irdma_sc_gen_rts_ae - request AE generated after RTS
1370 qp_uk = &qp->qp_uk; in irdma_sc_gen_rts_ae()
1372 wqe = qp_uk->sq_base[1].elem; in irdma_sc_gen_rts_ae()
1376 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); in irdma_sc_gen_rts_ae()
1383 wqe = qp_uk->sq_base[2].elem; in irdma_sc_gen_rts_ae()
1385 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); in irdma_sc_gen_rts_ae()
1394 * irdma_sc_send_lsmm - send last streaming mode message
1407 qp_uk = &qp->qp_uk; in irdma_sc_send_lsmm()
1408 wqe = qp_uk->sq_base->elem; in irdma_sc_send_lsmm()
1411 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { in irdma_sc_send_lsmm()
1419 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); in irdma_sc_send_lsmm()
1426 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); in irdma_sc_send_lsmm()
1434 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) in irdma_sc_send_lsmm()
1439 * irdma_sc_send_rtt - send last read0 or write0
1449 qp_uk = &qp->qp_uk; in irdma_sc_send_rtt()
1450 wqe = qp_uk->sq_base->elem; in irdma_sc_send_rtt()
1455 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { in irdma_sc_send_rtt()
1460 (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); in irdma_sc_send_rtt()
1464 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); in irdma_sc_send_rtt()
1467 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { in irdma_sc_send_rtt()
1471 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); in irdma_sc_send_rtt()
1474 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); in irdma_sc_send_rtt()
1484 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) in irdma_sc_send_rtt()
1489 * irdma_iwarp_opcode - determine if incoming is rdma layer
1498 if (info->q2_data_written) { in irdma_iwarp_opcode()
1507 * irdma_locate_mpa - return pointer to mpa in the pkt
1523 * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
1535 qp->flush_code = opcode; in irdma_bld_termhdr_ctrl()
1536 hdr->layer_etype = layer_etype; in irdma_bld_termhdr_ctrl()
1537 hdr->error_code = err; in irdma_bld_termhdr_ctrl()
1541 * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
1555 hdr->hdrct = DDP_LEN_FLAG; in irdma_bld_termhdr_ddp_rdma()
1560 hdr->hdrct |= DDP_HDR_FLAG; in irdma_bld_termhdr_ddp_rdma()
1565 hdr->hdrct |= DDP_HDR_FLAG; in irdma_bld_termhdr_ddp_rdma()
1570 hdr->hdrct |= RDMA_HDR_FLAG; in irdma_bld_termhdr_ddp_rdma()
1577 * irdma_bld_terminate_hdr - build terminate message header
1584 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; in irdma_bld_terminate_hdr()
1590 termhdr = (struct irdma_terminate_hdr *)qp->q2_buf; in irdma_bld_terminate_hdr()
1593 if (info->q2_data_written) { in irdma_bld_terminate_hdr()
1599 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; in irdma_bld_terminate_hdr()
1600 qp->sq_flush_code = info->sq; in irdma_bld_terminate_hdr()
1601 qp->rq_flush_code = info->rq; in irdma_bld_terminate_hdr()
1603 switch (info->ae_id) { in irdma_bld_terminate_hdr()
1605 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1616 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1617 if (info->q2_data_written) in irdma_bld_terminate_hdr()
1646 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1658 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1675 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1681 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1707 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; in irdma_bld_terminate_hdr()
1756 * irdma_terminate_send_fin() - Send fin for terminate message
1766 * irdma_terminate_connection() - Bad AE and send terminate to remote QP
1775 if (qp->term_flags & IRDMA_TERM_SENT) in irdma_terminate_connection()
1780 qp->term_flags |= IRDMA_TERM_SENT; in irdma_terminate_connection()
1786 * irdma_terminate_received - handle terminate received AE
1793 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; in irdma_terminate_received()
1801 if (info->q2_data_written) { in irdma_terminate_received()
1802 /* did not validate the frame - do it now */ in irdma_terminate_received()
1818 info->ae_id = aeq_id; in irdma_terminate_received()
1819 if (info->ae_id) { in irdma_terminate_received()
1820 /* Bad terminate recvd - send back a terminate */ in irdma_terminate_received()
1826 qp->term_flags |= IRDMA_TERM_RCVD; in irdma_terminate_received()
1827 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; in irdma_terminate_received()
1829 if (termhdr->layer_etype == RDMAP_REMOTE_PROT || in irdma_terminate_received()
1830 termhdr->layer_etype == RDMAP_REMOTE_OP) { in irdma_terminate_received()
1854 * irdma_sc_vsi_init - Init the vsi structure
1863 vsi->dev = info->dev; in irdma_sc_vsi_init()
1864 vsi->back_vsi = info->back_vsi; in irdma_sc_vsi_init()
1865 vsi->register_qset = info->register_qset; in irdma_sc_vsi_init()
1866 vsi->unregister_qset = info->unregister_qset; in irdma_sc_vsi_init()
1867 vsi->mtu = info->params->mtu; in irdma_sc_vsi_init()
1868 vsi->exception_lan_q = info->exception_lan_q; in irdma_sc_vsi_init()
1869 vsi->vsi_idx = info->pf_data_vsi_num; in irdma_sc_vsi_init()
1871 irdma_set_qos_info(vsi, info->params); in irdma_sc_vsi_init()
1873 mutex_init(&vsi->qos[i].qos_mutex); in irdma_sc_vsi_init()
1874 INIT_LIST_HEAD(&vsi->qos[i].qplist); in irdma_sc_vsi_init()
1876 if (vsi->register_qset) { in irdma_sc_vsi_init()
1877 vsi->dev->ws_add = irdma_ws_add; in irdma_sc_vsi_init()
1878 vsi->dev->ws_remove = irdma_ws_remove; in irdma_sc_vsi_init()
1879 vsi->dev->ws_reset = irdma_ws_reset; in irdma_sc_vsi_init()
1881 vsi->dev->ws_add = irdma_null_ws_add; in irdma_sc_vsi_init()
1882 vsi->dev->ws_remove = irdma_null_ws_remove; in irdma_sc_vsi_init()
1883 vsi->dev->ws_reset = irdma_null_ws_reset; in irdma_sc_vsi_init()
1888 * irdma_get_stats_idx - Return stats index
1894 struct irdma_sc_dev *dev = vsi->dev; in irdma_get_stats_idx()
1897 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { in irdma_get_stats_idx()
1904 if (!dev->stats_idx_array[i]) { in irdma_get_stats_idx()
1905 dev->stats_idx_array[i] = true; in irdma_get_stats_idx()
1914 * irdma_hw_stats_init_gen1 - Initialize stat reg table used for gen1
1921 struct irdma_sc_dev *dev = vsi->dev; in irdma_hw_stats_init_gen1()
1923 u64 *stat_reg = vsi->hw_stats_regs; in irdma_hw_stats_init_gen1()
1924 u64 *regs = dev->hw_stats_regs; in irdma_hw_stats_init_gen1()
1925 u16 i, stats_reg_set = vsi->stats_idx; in irdma_hw_stats_init_gen1()
1927 map = dev->hw_stats_map; in irdma_hw_stats_init_gen1()
1930 stats_reg_set += vsi->stats_inst_alloc ? IRDMA_FIRST_NON_PF_STAT : 0; in irdma_hw_stats_init_gen1()
1932 for (i = 0; i < dev->hw_attrs.max_stat_idx; i++) { in irdma_hw_stats_init_gen1()
1941 * irdma_vsi_stats_init - Initialize the vsi statistics
1950 vsi->pestat = info->pestat; in irdma_vsi_stats_init()
1951 vsi->pestat->hw = vsi->dev->hw; in irdma_vsi_stats_init()
1952 vsi->pestat->vsi = vsi; in irdma_vsi_stats_init()
1953 stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem; in irdma_vsi_stats_init()
1954 stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1); in irdma_vsi_stats_init()
1955 stats_buff_mem->va = dma_alloc_coherent(vsi->pestat->hw->device, in irdma_vsi_stats_init()
1956 stats_buff_mem->size, in irdma_vsi_stats_init()
1957 &stats_buff_mem->pa, in irdma_vsi_stats_init()
1959 if (!stats_buff_mem->va) in irdma_vsi_stats_init()
1960 return -ENOMEM; in irdma_vsi_stats_init()
1962 vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va; in irdma_vsi_stats_init()
1963 vsi->pestat->gather_info.last_gather_stats_va = in irdma_vsi_stats_init()
1964 (void *)((uintptr_t)stats_buff_mem->va + in irdma_vsi_stats_init()
1970 vsi->stats_idx = info->fcn_id; in irdma_vsi_stats_init()
1971 if (info->alloc_stats_inst) { in irdma_vsi_stats_init()
1975 vsi->stats_inst_alloc = true; in irdma_vsi_stats_init()
1976 vsi->stats_idx = stats_idx; in irdma_vsi_stats_init()
1977 vsi->pestat->gather_info.use_stats_inst = true; in irdma_vsi_stats_init()
1978 vsi->pestat->gather_info.stats_inst_index = stats_idx; in irdma_vsi_stats_init()
1982 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_vsi_stats_init()
1989 * irdma_vsi_stats_free - Free the vsi stats
1995 struct irdma_sc_dev *dev = vsi->dev; in irdma_vsi_stats_free()
1996 u8 stats_idx = vsi->stats_idx; in irdma_vsi_stats_free()
1998 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { in irdma_vsi_stats_free()
1999 if (vsi->stats_inst_alloc) { in irdma_vsi_stats_free()
2000 stats_info.stats_idx = vsi->stats_idx; in irdma_vsi_stats_free()
2005 if (vsi->stats_inst_alloc && in irdma_vsi_stats_free()
2006 stats_idx < vsi->dev->hw_attrs.max_stat_inst) in irdma_vsi_stats_free()
2007 vsi->dev->stats_idx_array[stats_idx] = false; in irdma_vsi_stats_free()
2010 if (!vsi->pestat) in irdma_vsi_stats_free()
2013 dma_free_coherent(vsi->pestat->hw->device, in irdma_vsi_stats_free()
2014 vsi->pestat->gather_info.stats_buff_mem.size, in irdma_vsi_stats_free()
2015 vsi->pestat->gather_info.stats_buff_mem.va, in irdma_vsi_stats_free()
2016 vsi->pestat->gather_info.stats_buff_mem.pa); in irdma_vsi_stats_free()
2017 vsi->pestat->gather_info.stats_buff_mem.va = NULL; in irdma_vsi_stats_free()
2021 * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
2023 * @queue_type: queue type selected for the calculation algorithm
2042 * irdma_sc_gather_stats - collect the statistics
2054 if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE) in irdma_sc_gather_stats()
2055 return -ENOMEM; in irdma_sc_gather_stats()
2059 return -ENOMEM; in irdma_sc_gather_stats()
2062 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index)); in irdma_sc_gather_stats()
2063 set_64bit_val(wqe, 32, info->stats_buff_mem.pa); in irdma_sc_gather_stats()
2065 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_gather_stats()
2066 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) | in irdma_sc_gather_stats()
2068 info->stats_inst_index) | in irdma_sc_gather_stats()
2070 info->use_hmc_fcn_index) | in irdma_sc_gather_stats()
2080 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_gather_stats()
2082 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_gather_stats()
2088 * irdma_sc_manage_stats_inst - allocate or free stats instance
2103 return -ENOMEM; in irdma_sc_manage_stats_inst()
2106 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id)); in irdma_sc_manage_stats_inst()
2107 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_manage_stats_inst()
2110 info->use_hmc_fcn_index) | in irdma_sc_manage_stats_inst()
2111 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) | in irdma_sc_manage_stats_inst()
2126 * irdma_sc_set_up_map - set the up map table
2140 return -ENOMEM; in irdma_sc_set_up_map()
2143 temp |= (u64)info->map[i] << (i * 8); in irdma_sc_set_up_map()
2147 FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) | in irdma_sc_set_up_map()
2148 FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx)); in irdma_sc_set_up_map()
2150 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | in irdma_sc_set_up_map()
2151 FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) | in irdma_sc_set_up_map()
2153 info->use_cnp_up_override) | in irdma_sc_set_up_map()
2167 * irdma_sc_manage_ws_node - create/modify/destroy WS node
2182 return -ENOMEM; in irdma_sc_manage_ws_node()
2185 FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) | in irdma_sc_manage_ws_node()
2186 FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight)); in irdma_sc_manage_ws_node()
2188 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | in irdma_sc_manage_ws_node()
2190 FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) | in irdma_sc_manage_ws_node()
2191 FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) | in irdma_sc_manage_ws_node()
2192 FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) | in irdma_sc_manage_ws_node()
2193 FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) | in irdma_sc_manage_ws_node()
2195 FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) | in irdma_sc_manage_ws_node()
2196 FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id); in irdma_sc_manage_ws_node()
2209 * irdma_sc_qp_flush_wqes - flush qp's wqe
2225 if (info->rq && !qp->flush_rq) in irdma_sc_qp_flush_wqes()
2227 if (info->sq && !qp->flush_sq) in irdma_sc_qp_flush_wqes()
2229 qp->flush_sq |= flush_sq; in irdma_sc_qp_flush_wqes()
2230 qp->flush_rq |= flush_rq; in irdma_sc_qp_flush_wqes()
2233 ibdev_dbg(to_ibdev(qp->dev), in irdma_sc_qp_flush_wqes()
2235 qp->qp_uk.qp_id); in irdma_sc_qp_flush_wqes()
2236 return -EALREADY; in irdma_sc_qp_flush_wqes()
2239 cqp = qp->pd->dev->cqp; in irdma_sc_qp_flush_wqes()
2242 return -ENOMEM; in irdma_sc_qp_flush_wqes()
2244 if (info->userflushcode) { in irdma_sc_qp_flush_wqes()
2247 info->rq_minor_code) | in irdma_sc_qp_flush_wqes()
2249 info->rq_major_code); in irdma_sc_qp_flush_wqes()
2252 info->sq_minor_code) | in irdma_sc_qp_flush_wqes()
2254 info->sq_major_code); in irdma_sc_qp_flush_wqes()
2258 temp = (info->generate_ae) ? in irdma_sc_qp_flush_wqes()
2259 info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, in irdma_sc_qp_flush_wqes()
2260 info->ae_src) : 0; in irdma_sc_qp_flush_wqes()
2263 hdr = qp->qp_uk.qp_id | in irdma_sc_qp_flush_wqes()
2265 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) | in irdma_sc_qp_flush_wqes()
2266 FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) | in irdma_sc_qp_flush_wqes()
2269 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_flush_wqes()
2283 * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
2298 cqp = qp->pd->dev->cqp; in irdma_sc_gen_ae()
2301 return -ENOMEM; in irdma_sc_gen_ae()
2303 temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, in irdma_sc_gen_ae()
2304 info->ae_src); in irdma_sc_gen_ae()
2307 hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE, in irdma_sc_gen_ae()
2310 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_gen_ae()
2323 /*** irdma_sc_qp_upload_context - upload qp's context
2337 cqp = dev->cqp; in irdma_sc_qp_upload_context()
2340 return -ENOMEM; in irdma_sc_qp_upload_context()
2342 set_64bit_val(wqe, 16, info->buf_pa); in irdma_sc_qp_upload_context()
2344 hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) | in irdma_sc_qp_upload_context()
2346 FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) | in irdma_sc_qp_upload_context()
2347 FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) | in irdma_sc_qp_upload_context()
2348 FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) | in irdma_sc_qp_upload_context()
2349 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_upload_context()
2363 * irdma_sc_manage_push_page - Handle push page
2376 if (info->free_page && in irdma_sc_manage_push_page()
2377 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) in irdma_sc_manage_push_page()
2378 return -EINVAL; in irdma_sc_manage_push_page()
2382 return -ENOMEM; in irdma_sc_manage_push_page()
2384 set_64bit_val(wqe, 16, info->qs_handle); in irdma_sc_manage_push_page()
2385 hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) | in irdma_sc_manage_push_page()
2386 FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) | in irdma_sc_manage_push_page()
2388 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_manage_push_page()
2389 FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page); in irdma_sc_manage_push_page()
2403 * irdma_sc_suspend_qp - suspend qp for param change
2416 return -ENOMEM; in irdma_sc_suspend_qp()
2418 hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) | in irdma_sc_suspend_qp()
2420 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_suspend_qp()
2433 * irdma_sc_resume_qp - resume qp after suspend
2446 return -ENOMEM; in irdma_sc_resume_qp()
2449 FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle)); in irdma_sc_resume_qp()
2451 hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) | in irdma_sc_resume_qp()
2453 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_resume_qp()
2466 * irdma_sc_cq_ack - acknowledge completion q
2471 writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db); in irdma_sc_cq_ack()
2475 * irdma_sc_cq_init - initialize completion q
2483 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_cq_init()
2484 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) in irdma_sc_cq_init()
2485 return -EINVAL; in irdma_sc_cq_init()
2487 cq->cq_pa = info->cq_base_pa; in irdma_sc_cq_init()
2488 cq->dev = info->dev; in irdma_sc_cq_init()
2489 cq->ceq_id = info->ceq_id; in irdma_sc_cq_init()
2490 info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db; in irdma_sc_cq_init()
2491 info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db; in irdma_sc_cq_init()
2492 irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info); in irdma_sc_cq_init()
2494 cq->virtual_map = info->virtual_map; in irdma_sc_cq_init()
2495 cq->pbl_chunk_size = info->pbl_chunk_size; in irdma_sc_cq_init()
2496 cq->ceqe_mask = info->ceqe_mask; in irdma_sc_cq_init()
2497 cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP; in irdma_sc_cq_init()
2498 cq->shadow_area_pa = info->shadow_area_pa; in irdma_sc_cq_init()
2499 cq->shadow_read_threshold = info->shadow_read_threshold; in irdma_sc_cq_init()
2500 cq->ceq_id_valid = info->ceq_id_valid; in irdma_sc_cq_init()
2501 cq->tph_en = info->tph_en; in irdma_sc_cq_init()
2502 cq->tph_val = info->tph_val; in irdma_sc_cq_init()
2503 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; in irdma_sc_cq_init()
2504 cq->vsi = info->vsi; in irdma_sc_cq_init()
2510 * irdma_sc_cq_create - create completion q
2525 cqp = cq->dev->cqp; in irdma_sc_cq_create()
2526 if (cq->cq_uk.cq_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt) in irdma_sc_cq_create()
2527 return -EINVAL; in irdma_sc_cq_create()
2529 if (cq->ceq_id >= cq->dev->hmc_fpm_misc.max_ceqs) in irdma_sc_cq_create()
2530 return -EINVAL; in irdma_sc_cq_create()
2532 ceq = cq->dev->ceq[cq->ceq_id]; in irdma_sc_cq_create()
2533 if (ceq && ceq->reg_cq) in irdma_sc_cq_create()
2541 if (ceq && ceq->reg_cq) in irdma_sc_cq_create()
2543 return -ENOMEM; in irdma_sc_cq_create()
2546 set_64bit_val(wqe, 0, cq->cq_uk.cq_size); in irdma_sc_cq_create()
2549 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold)); in irdma_sc_cq_create()
2550 set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa)); in irdma_sc_cq_create()
2551 set_64bit_val(wqe, 40, cq->shadow_area_pa); in irdma_sc_cq_create()
2553 FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0))); in irdma_sc_cq_create()
2555 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | in irdma_sc_cq_create()
2556 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); in irdma_sc_cq_create()
2558 hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) | in irdma_sc_cq_create()
2559 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), in irdma_sc_cq_create()
2562 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | in irdma_sc_cq_create()
2564 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | in irdma_sc_cq_create()
2565 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | in irdma_sc_cq_create()
2566 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | in irdma_sc_cq_create()
2567 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | in irdma_sc_cq_create()
2569 cq->cq_uk.avoid_mem_cflct) | in irdma_sc_cq_create()
2570 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_create()
2584 * irdma_sc_cq_destroy - destroy completion q
2596 cqp = cq->dev->cqp; in irdma_sc_cq_destroy()
2599 return -ENOMEM; in irdma_sc_cq_destroy()
2601 ceq = cq->dev->ceq[cq->ceq_id]; in irdma_sc_cq_destroy()
2602 if (ceq && ceq->reg_cq) in irdma_sc_cq_destroy()
2605 set_64bit_val(wqe, 0, cq->cq_uk.cq_size); in irdma_sc_cq_destroy()
2607 set_64bit_val(wqe, 40, cq->shadow_area_pa); in irdma_sc_cq_destroy()
2609 (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); in irdma_sc_cq_destroy()
2611 hdr = cq->cq_uk.cq_id | in irdma_sc_cq_destroy()
2612 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), in irdma_sc_cq_destroy()
2615 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | in irdma_sc_cq_destroy()
2616 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | in irdma_sc_cq_destroy()
2617 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | in irdma_sc_cq_destroy()
2618 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | in irdma_sc_cq_destroy()
2619 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | in irdma_sc_cq_destroy()
2620 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) | in irdma_sc_cq_destroy()
2621 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_destroy()
2635 * irdma_sc_cq_resize - set resized cq buffer info
2641 cq->virtual_map = info->virtual_map; in irdma_sc_cq_resize()
2642 cq->cq_pa = info->cq_pa; in irdma_sc_cq_resize()
2643 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; in irdma_sc_cq_resize()
2644 cq->pbl_chunk_size = info->pbl_chunk_size; in irdma_sc_cq_resize()
2645 irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size); in irdma_sc_cq_resize()
2649 * irdma_sc_cq_modify - modify a Completion Queue
2664 pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_cq_modify()
2665 if (info->cq_resize && info->virtual_map && in irdma_sc_cq_modify()
2666 info->first_pm_pbl_idx >= pble_obj_cnt) in irdma_sc_cq_modify()
2667 return -EINVAL; in irdma_sc_cq_modify()
2669 cqp = cq->dev->cqp; in irdma_sc_cq_modify()
2672 return -ENOMEM; in irdma_sc_cq_modify()
2674 set_64bit_val(wqe, 0, info->cq_size); in irdma_sc_cq_modify()
2677 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold)); in irdma_sc_cq_modify()
2678 set_64bit_val(wqe, 32, info->cq_pa); in irdma_sc_cq_modify()
2679 set_64bit_val(wqe, 40, cq->shadow_area_pa); in irdma_sc_cq_modify()
2680 set_64bit_val(wqe, 48, info->first_pm_pbl_idx); in irdma_sc_cq_modify()
2682 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | in irdma_sc_cq_modify()
2683 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); in irdma_sc_cq_modify()
2685 hdr = cq->cq_uk.cq_id | in irdma_sc_cq_modify()
2687 FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) | in irdma_sc_cq_modify()
2688 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) | in irdma_sc_cq_modify()
2689 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) | in irdma_sc_cq_modify()
2690 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) | in irdma_sc_cq_modify()
2691 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | in irdma_sc_cq_modify()
2692 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | in irdma_sc_cq_modify()
2694 cq->cq_uk.avoid_mem_cflct) | in irdma_sc_cq_modify()
2695 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_modify()
2709 * irdma_check_cqp_progress - check cqp processing progress
2715 u64 completed_ops = atomic64_read(&dev->cqp->completed_ops); in irdma_check_cqp_progress()
2717 if (timeout->compl_cqp_cmds != completed_ops) { in irdma_check_cqp_progress()
2718 timeout->compl_cqp_cmds = completed_ops; in irdma_check_cqp_progress()
2719 timeout->count = 0; in irdma_check_cqp_progress()
2720 } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) { in irdma_check_cqp_progress()
2721 timeout->count++; in irdma_check_cqp_progress()
2726 * irdma_get_cqp_reg_info - get head and tail for cqp using registers
2735 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_get_cqp_reg_info()
2741 * irdma_cqp_poll_registers - poll cqp registers
2755 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_cqp_poll_registers()
2756 ibdev_dbg(to_ibdev(cqp->dev), in irdma_cqp_poll_registers()
2759 return -EIO; in irdma_cqp_poll_registers()
2763 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_cqp_poll_registers()
2764 atomic64_inc(&cqp->completed_ops); in irdma_cqp_poll_registers()
2767 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_cqp_poll_registers()
2770 return -ETIMEDOUT; in irdma_cqp_poll_registers()
2774 * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
2810 * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer
2854 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) { in irdma_sc_parse_fpm_commit_buf()
2892 * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
2916 * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer
2934 obj_info = hmc_info->hmc_obj; in irdma_sc_parse_fpm_query_buf()
2937 hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp); in irdma_sc_parse_fpm_query_buf()
2940 hmc_fpm_misc->max_sds = max_pe_sds; in irdma_sc_parse_fpm_query_buf()
2941 hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index; in irdma_sc_parse_fpm_query_buf()
2964 hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp); in irdma_sc_parse_fpm_query_buf()
2965 if (!hmc_fpm_misc->xf_block_size) in irdma_sc_parse_fpm_query_buf()
2966 return -EINVAL; in irdma_sc_parse_fpm_query_buf()
2973 hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp); in irdma_sc_parse_fpm_query_buf()
2974 if (!hmc_fpm_misc->q1_block_size) in irdma_sc_parse_fpm_query_buf()
2975 return -EINVAL; in irdma_sc_parse_fpm_query_buf()
2984 hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp); in irdma_sc_parse_fpm_query_buf()
2985 hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp); in irdma_sc_parse_fpm_query_buf()
2986 hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp); in irdma_sc_parse_fpm_query_buf()
2987 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_sc_parse_fpm_query_buf()
2996 hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp); in irdma_sc_parse_fpm_query_buf()
2997 if (!hmc_fpm_misc->rrf_block_size && in irdma_sc_parse_fpm_query_buf()
2999 return -EINVAL; in irdma_sc_parse_fpm_query_buf()
3008 hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp); in irdma_sc_parse_fpm_query_buf()
3009 if (!hmc_fpm_misc->ooiscf_block_size && in irdma_sc_parse_fpm_query_buf()
3011 return -EINVAL; in irdma_sc_parse_fpm_query_buf()
3017 * irdma_sc_find_reg_cq - find cq ctx index
3026 for (i = 0; i < ceq->reg_cq_size; i++) { in irdma_sc_find_reg_cq()
3027 if (cq == ceq->reg_cq[i]) in irdma_sc_find_reg_cq()
3035 * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq
3043 spin_lock_irqsave(&ceq->req_cq_lock, flags); in irdma_sc_add_cq_ctx()
3045 if (ceq->reg_cq_size == ceq->elem_cnt) { in irdma_sc_add_cq_ctx()
3046 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); in irdma_sc_add_cq_ctx()
3047 return -ENOMEM; in irdma_sc_add_cq_ctx()
3050 ceq->reg_cq[ceq->reg_cq_size++] = cq; in irdma_sc_add_cq_ctx()
3052 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); in irdma_sc_add_cq_ctx()
3058 * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq
3067 spin_lock_irqsave(&ceq->req_cq_lock, flags); in irdma_sc_remove_cq_ctx()
3072 ceq->reg_cq_size--; in irdma_sc_remove_cq_ctx()
3073 if (cq_ctx_idx != ceq->reg_cq_size) in irdma_sc_remove_cq_ctx()
3074 ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size]; in irdma_sc_remove_cq_ctx()
3075 ceq->reg_cq[ceq->reg_cq_size] = NULL; in irdma_sc_remove_cq_ctx()
3078 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); in irdma_sc_remove_cq_ctx()
3082 * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair
3093 if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 || in irdma_sc_cqp_init()
3094 info->sq_size < IRDMA_CQP_SW_SQSIZE_4 || in irdma_sc_cqp_init()
3095 ((info->sq_size & (info->sq_size - 1)))) in irdma_sc_cqp_init()
3096 return -EINVAL; in irdma_sc_cqp_init()
3098 hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size, in irdma_sc_cqp_init()
3100 cqp->size = sizeof(*cqp); in irdma_sc_cqp_init()
3101 cqp->sq_size = info->sq_size; in irdma_sc_cqp_init()
3102 cqp->hw_sq_size = hw_sq_size; in irdma_sc_cqp_init()
3103 cqp->sq_base = info->sq; in irdma_sc_cqp_init()
3104 cqp->host_ctx = info->host_ctx; in irdma_sc_cqp_init()
3105 cqp->sq_pa = info->sq_pa; in irdma_sc_cqp_init()
3106 cqp->host_ctx_pa = info->host_ctx_pa; in irdma_sc_cqp_init()
3107 cqp->dev = info->dev; in irdma_sc_cqp_init()
3108 cqp->struct_ver = info->struct_ver; in irdma_sc_cqp_init()
3109 cqp->hw_maj_ver = info->hw_maj_ver; in irdma_sc_cqp_init()
3110 cqp->hw_min_ver = info->hw_min_ver; in irdma_sc_cqp_init()
3111 cqp->scratch_array = info->scratch_array; in irdma_sc_cqp_init()
3112 cqp->polarity = 0; in irdma_sc_cqp_init()
3113 cqp->en_datacenter_tcp = info->en_datacenter_tcp; in irdma_sc_cqp_init()
3114 cqp->ena_vf_count = info->ena_vf_count; in irdma_sc_cqp_init()
3115 cqp->hmc_profile = info->hmc_profile; in irdma_sc_cqp_init()
3116 cqp->ceqs_per_vf = info->ceqs_per_vf; in irdma_sc_cqp_init()
3117 cqp->disable_packed = info->disable_packed; in irdma_sc_cqp_init()
3118 cqp->rocev2_rto_policy = info->rocev2_rto_policy; in irdma_sc_cqp_init()
3119 cqp->protocol_used = info->protocol_used; in irdma_sc_cqp_init()
3120 memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); in irdma_sc_cqp_init()
3121 info->dev->cqp = cqp; in irdma_sc_cqp_init()
3123 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); in irdma_sc_cqp_init()
3124 cqp->requested_ops = 0; in irdma_sc_cqp_init()
3125 atomic64_set(&cqp->completed_ops, 0); in irdma_sc_cqp_init()
3127 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); in irdma_sc_cqp_init()
3129 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_sc_cqp_init()
3130 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]); in irdma_sc_cqp_init()
3131 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_init()
3133 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_init()
3135 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, in irdma_sc_cqp_init()
3136 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity); in irdma_sc_cqp_init()
3141 * irdma_sc_cqp_create - create cqp during bringup
3153 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; in irdma_sc_cqp_create()
3154 cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size, in irdma_sc_cqp_create()
3156 cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device, in irdma_sc_cqp_create()
3157 cqp->sdbuf.size, &cqp->sdbuf.pa, in irdma_sc_cqp_create()
3159 if (!cqp->sdbuf.va) in irdma_sc_cqp_create()
3160 return -ENOMEM; in irdma_sc_cqp_create()
3162 spin_lock_init(&cqp->dev->cqp_lock); in irdma_sc_cqp_create()
3164 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | in irdma_sc_cqp_create()
3165 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | in irdma_sc_cqp_create()
3166 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | in irdma_sc_cqp_create()
3167 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); in irdma_sc_cqp_create()
3170 cqp->rocev2_rto_policy) | in irdma_sc_cqp_create()
3172 cqp->protocol_used); in irdma_sc_cqp_create()
3175 set_64bit_val(cqp->host_ctx, 0, temp); in irdma_sc_cqp_create()
3176 set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa); in irdma_sc_cqp_create()
3178 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | in irdma_sc_cqp_create()
3179 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); in irdma_sc_cqp_create()
3180 set_64bit_val(cqp->host_ctx, 16, temp); in irdma_sc_cqp_create()
3181 set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp); in irdma_sc_cqp_create()
3182 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | in irdma_sc_cqp_create()
3183 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); in irdma_sc_cqp_create()
3185 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | in irdma_sc_cqp_create()
3186 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); in irdma_sc_cqp_create()
3188 set_64bit_val(cqp->host_ctx, 32, temp); in irdma_sc_cqp_create()
3189 set_64bit_val(cqp->host_ctx, 40, 0); in irdma_sc_cqp_create()
3192 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | in irdma_sc_cqp_create()
3193 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | in irdma_sc_cqp_create()
3194 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); in irdma_sc_cqp_create()
3196 set_64bit_val(cqp->host_ctx, 48, temp); in irdma_sc_cqp_create()
3199 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | in irdma_sc_cqp_create()
3200 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | in irdma_sc_cqp_create()
3201 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | in irdma_sc_cqp_create()
3202 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); in irdma_sc_cqp_create()
3204 set_64bit_val(cqp->host_ctx, 56, temp); in irdma_sc_cqp_create()
3206 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false); in irdma_sc_cqp_create()
3207 p1 = cqp->host_ctx_pa >> 32; in irdma_sc_cqp_create()
3208 p2 = (u32)cqp->host_ctx_pa; in irdma_sc_cqp_create()
3210 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_create()
3211 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_create()
3214 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_create()
3215 ret_code = -ETIMEDOUT; in irdma_sc_cqp_create()
3218 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_create()
3219 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_create()
3222 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { in irdma_sc_cqp_create()
3223 ret_code = -EOPNOTSUPP; in irdma_sc_cqp_create()
3227 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cqp_create()
3231 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_create()
3232 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_create()
3233 cqp->sdbuf.va = NULL; in irdma_sc_cqp_create()
3234 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_cqp_create()
3241 * irdma_sc_cqp_post_sq - post of cqp's sq
3246 writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db); in irdma_sc_cqp_post_sq()
3248 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_post_sq()
3250 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_cqp_post_sq()
3254 * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq
3266 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { in irdma_sc_cqp_get_next_send_wqe_idx()
3267 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_get_next_send_wqe_idx()
3269 cqp->sq_ring.head, cqp->sq_ring.tail, in irdma_sc_cqp_get_next_send_wqe_idx()
3270 cqp->sq_ring.size); in irdma_sc_cqp_get_next_send_wqe_idx()
3273 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); in irdma_sc_cqp_get_next_send_wqe_idx()
3277 cqp->requested_ops++; in irdma_sc_cqp_get_next_send_wqe_idx()
3279 cqp->polarity = !cqp->polarity; in irdma_sc_cqp_get_next_send_wqe_idx()
3280 wqe = cqp->sq_base[*wqe_idx].elem; in irdma_sc_cqp_get_next_send_wqe_idx()
3281 cqp->scratch_array[*wqe_idx] = scratch; in irdma_sc_cqp_get_next_send_wqe_idx()
3288 * irdma_sc_cqp_destroy - destroy cqp during close
3296 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_destroy()
3297 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_destroy()
3299 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_destroy()
3300 ret_code = -ETIMEDOUT; in irdma_sc_cqp_destroy()
3303 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_destroy()
3304 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_destroy()
3305 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); in irdma_sc_cqp_destroy()
3307 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_destroy()
3308 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_destroy()
3309 cqp->sdbuf.va = NULL; in irdma_sc_cqp_destroy()
3314 * irdma_sc_ccq_arm - enable intr for control cq
3324 get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val); in irdma_sc_ccq_arm()
3333 set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val); in irdma_sc_ccq_arm()
3337 writel(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db); in irdma_sc_ccq_arm()
3341 * irdma_sc_ccq_get_cqe_info - get ccq's cq entry
3356 if (ccq->cq_uk.avoid_mem_cflct) in irdma_sc_ccq_get_cqe_info()
3357 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk); in irdma_sc_ccq_get_cqe_info()
3359 cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk); in irdma_sc_ccq_get_cqe_info()
3363 if (polarity != ccq->cq_uk.polarity) in irdma_sc_ccq_get_cqe_info()
3364 return -ENOENT; in irdma_sc_ccq_get_cqe_info()
3371 info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp); in irdma_sc_ccq_get_cqe_info()
3372 info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR; in irdma_sc_ccq_get_cqe_info()
3373 info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp); in irdma_sc_ccq_get_cqe_info()
3374 if (info->error) { in irdma_sc_ccq_get_cqe_info()
3375 info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp); in irdma_sc_ccq_get_cqe_info()
3376 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_ccq_get_cqe_info()
3377 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_ccq_get_cqe_info()
3382 info->scratch = cqp->scratch_array[wqe_idx]; in irdma_sc_ccq_get_cqe_info()
3385 info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1); in irdma_sc_ccq_get_cqe_info()
3386 get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1); in irdma_sc_ccq_get_cqe_info()
3387 info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1); in irdma_sc_ccq_get_cqe_info()
3388 info->cqp = cqp; in irdma_sc_ccq_get_cqe_info()
3391 IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code); in irdma_sc_ccq_get_cqe_info()
3392 if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)) in irdma_sc_ccq_get_cqe_info()
3393 ccq->cq_uk.polarity ^= 1; in irdma_sc_ccq_get_cqe_info()
3396 IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring); in irdma_sc_ccq_get_cqe_info()
3397 set_64bit_val(ccq->cq_uk.shadow_area, 0, in irdma_sc_ccq_get_cqe_info()
3398 IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)); in irdma_sc_ccq_get_cqe_info()
3402 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_sc_ccq_get_cqe_info()
3403 atomic64_inc(&cqp->completed_ops); in irdma_sc_ccq_get_cqe_info()
3409 * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
3422 ccq = cqp->dev->ccq; in irdma_sc_poll_for_cqp_op_done()
3424 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) in irdma_sc_poll_for_cqp_op_done()
3425 return -ETIMEDOUT; in irdma_sc_poll_for_cqp_op_done()
3428 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_poll_for_cqp_op_done()
3432 ret_code = -EIO; in irdma_sc_poll_for_cqp_op_done()
3438 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_poll_for_cqp_op_done()
3450 * irdma_sc_manage_hmc_pm_func_table - manage of function table
3465 return -ENOMEM; in irdma_sc_manage_hmc_pm_func_table()
3475 hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) | in irdma_sc_manage_hmc_pm_func_table()
3478 FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) | in irdma_sc_manage_hmc_pm_func_table()
3479 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_hmc_pm_func_table()
3494 * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion
3505 * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values
3525 return -ENOMEM; in irdma_sc_commit_fpm_val()
3528 set_64bit_val(wqe, 32, commit_fpm_mem->pa); in irdma_sc_commit_fpm_val()
3532 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_commit_fpm_val()
3546 cqp->dev->hw_attrs.max_done_count); in irdma_sc_commit_fpm_val()
3555 * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for
3566 * irdma_sc_query_fpm_val - cqp wqe query fpm values
3586 return -ENOMEM; in irdma_sc_query_fpm_val()
3589 set_64bit_val(wqe, 32, query_fpm_mem->pa); in irdma_sc_query_fpm_val()
3592 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_query_fpm_val()
3605 cqp->dev->hw_attrs.max_done_count); in irdma_sc_query_fpm_val()
3614 * irdma_sc_ceq_init - initialize ceq
3623 if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size || in irdma_sc_ceq_init()
3624 info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size) in irdma_sc_ceq_init()
3625 return -EINVAL; in irdma_sc_ceq_init()
3627 if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs) in irdma_sc_ceq_init()
3628 return -EINVAL; in irdma_sc_ceq_init()
3629 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_ceq_init()
3631 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) in irdma_sc_ceq_init()
3632 return -EINVAL; in irdma_sc_ceq_init()
3634 ceq->size = sizeof(*ceq); in irdma_sc_ceq_init()
3635 ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base; in irdma_sc_ceq_init()
3636 ceq->ceq_id = info->ceq_id; in irdma_sc_ceq_init()
3637 ceq->dev = info->dev; in irdma_sc_ceq_init()
3638 ceq->elem_cnt = info->elem_cnt; in irdma_sc_ceq_init()
3639 ceq->ceq_elem_pa = info->ceqe_pa; in irdma_sc_ceq_init()
3640 ceq->virtual_map = info->virtual_map; in irdma_sc_ceq_init()
3641 ceq->itr_no_expire = info->itr_no_expire; in irdma_sc_ceq_init()
3642 ceq->reg_cq = info->reg_cq; in irdma_sc_ceq_init()
3643 ceq->reg_cq_size = 0; in irdma_sc_ceq_init()
3644 spin_lock_init(&ceq->req_cq_lock); in irdma_sc_ceq_init()
3645 ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0); in irdma_sc_ceq_init()
3646 ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0); in irdma_sc_ceq_init()
3647 ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL); in irdma_sc_ceq_init()
3648 ceq->tph_en = info->tph_en; in irdma_sc_ceq_init()
3649 ceq->tph_val = info->tph_val; in irdma_sc_ceq_init()
3650 ceq->vsi = info->vsi; in irdma_sc_ceq_init()
3651 ceq->polarity = 1; in irdma_sc_ceq_init()
3652 IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt); in irdma_sc_ceq_init()
3653 ceq->dev->ceq[info->ceq_id] = ceq; in irdma_sc_ceq_init()
3659 * irdma_sc_ceq_create - create ceq wqe
3672 cqp = ceq->dev->cqp; in irdma_sc_ceq_create()
3675 return -ENOMEM; in irdma_sc_ceq_create()
3676 set_64bit_val(wqe, 16, ceq->elem_cnt); in irdma_sc_ceq_create()
3678 (ceq->virtual_map ? 0 : ceq->ceq_elem_pa)); in irdma_sc_ceq_create()
3680 (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); in irdma_sc_ceq_create()
3682 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) | in irdma_sc_ceq_create()
3683 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx)); in irdma_sc_ceq_create()
3684 hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) | in irdma_sc_ceq_create()
3686 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | in irdma_sc_ceq_create()
3687 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | in irdma_sc_ceq_create()
3688 FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) | in irdma_sc_ceq_create()
3689 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | in irdma_sc_ceq_create()
3690 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_create()
3704 * irdma_sc_cceq_create_done - poll for control ceq wqe to complete
3711 cqp = ceq->dev->cqp; in irdma_sc_cceq_create_done()
3717 * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete
3724 if (ceq->reg_cq) in irdma_sc_cceq_destroy_done()
3725 irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq); in irdma_sc_cceq_destroy_done()
3727 cqp = ceq->dev->cqp; in irdma_sc_cceq_destroy_done()
3728 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cceq_destroy_done()
3735 * irdma_sc_cceq_create - create cceq
3742 struct irdma_sc_dev *dev = ceq->dev; in irdma_sc_cceq_create()
3744 dev->ccq->vsi = ceq->vsi; in irdma_sc_cceq_create()
3745 if (ceq->reg_cq) { in irdma_sc_cceq_create()
3746 ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq); in irdma_sc_cceq_create()
3759 * irdma_sc_ceq_destroy - destroy ceq
3770 cqp = ceq->dev->cqp; in irdma_sc_ceq_destroy()
3773 return -ENOMEM; in irdma_sc_ceq_destroy()
3775 set_64bit_val(wqe, 16, ceq->elem_cnt); in irdma_sc_ceq_destroy()
3776 set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx); in irdma_sc_ceq_destroy()
3777 hdr = ceq->ceq_id | in irdma_sc_ceq_destroy()
3779 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | in irdma_sc_ceq_destroy()
3780 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | in irdma_sc_ceq_destroy()
3781 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | in irdma_sc_ceq_destroy()
3782 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_destroy()
3796 * irdma_sc_process_ceq - process ceq
3818 if (polarity != ceq->polarity) in irdma_sc_process_ceq()
3824 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); in irdma_sc_process_ceq()
3826 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) in irdma_sc_process_ceq()
3827 ceq->polarity ^= 1; in irdma_sc_process_ceq()
3832 if (ceq->reg_cq) { in irdma_sc_process_ceq()
3833 spin_lock_irqsave(&ceq->req_cq_lock, flags); in irdma_sc_process_ceq()
3835 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); in irdma_sc_process_ceq()
3838 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); in irdma_sc_process_ceq()
3839 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) in irdma_sc_process_ceq()
3840 ceq->polarity ^= 1; in irdma_sc_process_ceq()
3849 * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq
3860 u8 ceq_polarity = ceq->polarity; in irdma_sc_cleanup_ceqes()
3867 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0); in irdma_sc_cleanup_ceqes()
3881 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i); in irdma_sc_cleanup_ceqes()
3888 * irdma_sc_aeq_init - initialize aeq
3897 if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size || in irdma_sc_aeq_init()
3898 info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size) in irdma_sc_aeq_init()
3899 return -EINVAL; in irdma_sc_aeq_init()
3901 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_aeq_init()
3903 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) in irdma_sc_aeq_init()
3904 return -EINVAL; in irdma_sc_aeq_init()
3906 aeq->size = sizeof(*aeq); in irdma_sc_aeq_init()
3907 aeq->polarity = 1; in irdma_sc_aeq_init()
3908 aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base; in irdma_sc_aeq_init()
3909 aeq->dev = info->dev; in irdma_sc_aeq_init()
3910 aeq->elem_cnt = info->elem_cnt; in irdma_sc_aeq_init()
3911 aeq->aeq_elem_pa = info->aeq_elem_pa; in irdma_sc_aeq_init()
3912 IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt); in irdma_sc_aeq_init()
3913 aeq->virtual_map = info->virtual_map; in irdma_sc_aeq_init()
3914 aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL); in irdma_sc_aeq_init()
3915 aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0); in irdma_sc_aeq_init()
3916 aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0); in irdma_sc_aeq_init()
3917 aeq->msix_idx = info->msix_idx; in irdma_sc_aeq_init()
3918 info->dev->aeq = aeq; in irdma_sc_aeq_init()
3924 * irdma_sc_aeq_create - create aeq
3936 cqp = aeq->dev->cqp; in irdma_sc_aeq_create()
3939 return -ENOMEM; in irdma_sc_aeq_create()
3940 set_64bit_val(wqe, 16, aeq->elem_cnt); in irdma_sc_aeq_create()
3942 (aeq->virtual_map ? 0 : aeq->aeq_elem_pa)); in irdma_sc_aeq_create()
3944 (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0)); in irdma_sc_aeq_create()
3947 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | in irdma_sc_aeq_create()
3948 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | in irdma_sc_aeq_create()
3949 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_create()
3963 * irdma_sc_aeq_destroy - destroy aeq during close
3976 dev = aeq->dev; in irdma_sc_aeq_destroy()
3977 writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]); in irdma_sc_aeq_destroy()
3979 cqp = dev->cqp; in irdma_sc_aeq_destroy()
3982 return -ENOMEM; in irdma_sc_aeq_destroy()
3983 set_64bit_val(wqe, 16, aeq->elem_cnt); in irdma_sc_aeq_destroy()
3984 set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx); in irdma_sc_aeq_destroy()
3986 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | in irdma_sc_aeq_destroy()
3987 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | in irdma_sc_aeq_destroy()
3988 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_destroy()
4001 * irdma_sc_get_next_aeqe - get next aeq entry
4017 if (aeq->polarity != polarity) in irdma_sc_get_next_aeqe()
4018 return -ENOENT; in irdma_sc_get_next_aeqe()
4029 info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp); in irdma_sc_get_next_aeqe()
4030 info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) | in irdma_sc_get_next_aeqe()
4032 info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp); in irdma_sc_get_next_aeqe()
4033 info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp); in irdma_sc_get_next_aeqe()
4034 info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp); in irdma_sc_get_next_aeqe()
4035 info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp); in irdma_sc_get_next_aeqe()
4036 info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp); in irdma_sc_get_next_aeqe()
4038 info->ae_src = ae_src; in irdma_sc_get_next_aeqe()
4039 switch (info->ae_id) { in irdma_sc_get_next_aeqe()
4080 info->qp = true; in irdma_sc_get_next_aeqe()
4081 info->compl_ctx = compl_ctx; in irdma_sc_get_next_aeqe()
4084 info->cq = true; in irdma_sc_get_next_aeqe()
4085 info->compl_ctx = compl_ctx << 1; in irdma_sc_get_next_aeqe()
4100 info->qp = true; in irdma_sc_get_next_aeqe()
4101 info->compl_ctx = compl_ctx; in irdma_sc_get_next_aeqe()
4110 info->qp = true; in irdma_sc_get_next_aeqe()
4111 info->rq = true; in irdma_sc_get_next_aeqe()
4112 info->compl_ctx = compl_ctx; in irdma_sc_get_next_aeqe()
4118 info->cq = true; in irdma_sc_get_next_aeqe()
4119 info->compl_ctx = compl_ctx << 1; in irdma_sc_get_next_aeqe()
4123 info->qp = true; in irdma_sc_get_next_aeqe()
4124 info->sq = true; in irdma_sc_get_next_aeqe()
4125 info->compl_ctx = compl_ctx; in irdma_sc_get_next_aeqe()
4129 info->qp = true; in irdma_sc_get_next_aeqe()
4130 info->compl_ctx = compl_ctx; in irdma_sc_get_next_aeqe()
4131 info->in_rdrsp_wr = true; in irdma_sc_get_next_aeqe()
4135 info->qp = true; in irdma_sc_get_next_aeqe()
4136 info->compl_ctx = compl_ctx; in irdma_sc_get_next_aeqe()
4137 info->out_rdrsp = true; in irdma_sc_get_next_aeqe()
4144 IRDMA_RING_MOVE_TAIL(aeq->aeq_ring); in irdma_sc_get_next_aeqe()
4145 if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring)) in irdma_sc_get_next_aeqe()
4146 aeq->polarity ^= 1; in irdma_sc_get_next_aeqe()
4152 * irdma_sc_repost_aeq_entries - repost completed aeq entries
4158 writel(count, dev->hw_regs[IRDMA_AEQALLOC]); in irdma_sc_repost_aeq_entries()
4162 * irdma_sc_ccq_init - initialize control cq
4170 if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size || in irdma_sc_ccq_init()
4171 info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size) in irdma_sc_ccq_init()
4172 return -EINVAL; in irdma_sc_ccq_init()
4174 if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs) in irdma_sc_ccq_init()
4175 return -EINVAL; in irdma_sc_ccq_init()
4177 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; in irdma_sc_ccq_init()
4179 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) in irdma_sc_ccq_init()
4180 return -EINVAL; in irdma_sc_ccq_init()
4182 cq->cq_pa = info->cq_pa; in irdma_sc_ccq_init()
4183 cq->cq_uk.cq_base = info->cq_base; in irdma_sc_ccq_init()
4184 cq->shadow_area_pa = info->shadow_area_pa; in irdma_sc_ccq_init()
4185 cq->cq_uk.shadow_area = info->shadow_area; in irdma_sc_ccq_init()
4186 cq->shadow_read_threshold = info->shadow_read_threshold; in irdma_sc_ccq_init()
4187 cq->dev = info->dev; in irdma_sc_ccq_init()
4188 cq->ceq_id = info->ceq_id; in irdma_sc_ccq_init()
4189 cq->cq_uk.cq_size = info->num_elem; in irdma_sc_ccq_init()
4190 cq->cq_type = IRDMA_CQ_TYPE_CQP; in irdma_sc_ccq_init()
4191 cq->ceqe_mask = info->ceqe_mask; in irdma_sc_ccq_init()
4192 IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem); in irdma_sc_ccq_init()
4193 cq->cq_uk.cq_id = 0; /* control cq is id 0 always */ in irdma_sc_ccq_init()
4194 cq->ceq_id_valid = info->ceq_id_valid; in irdma_sc_ccq_init()
4195 cq->tph_en = info->tph_en; in irdma_sc_ccq_init()
4196 cq->tph_val = info->tph_val; in irdma_sc_ccq_init()
4197 cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct; in irdma_sc_ccq_init()
4198 cq->pbl_list = info->pbl_list; in irdma_sc_ccq_init()
4199 cq->virtual_map = info->virtual_map; in irdma_sc_ccq_init()
4200 cq->pbl_chunk_size = info->pbl_chunk_size; in irdma_sc_ccq_init()
4201 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; in irdma_sc_ccq_init()
4202 cq->cq_uk.polarity = true; in irdma_sc_ccq_init()
4203 cq->vsi = info->vsi; in irdma_sc_ccq_init()
4204 cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db; in irdma_sc_ccq_init()
4207 cq->cq_uk.cqe_alloc_db = NULL; in irdma_sc_ccq_init()
4209 info->dev->ccq = cq; in irdma_sc_ccq_init()
4214 * irdma_sc_ccq_create_done - poll cqp for ccq create
4221 cqp = ccq->dev->cqp; in irdma_sc_ccq_create_done()
4227 * irdma_sc_ccq_create - create control cq
4247 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; in irdma_sc_ccq_create()
4253 * irdma_sc_ccq_destroy - destroy ccq during close
4266 cqp = ccq->dev->cqp; in irdma_sc_ccq_destroy()
4269 return -ENOMEM; in irdma_sc_ccq_destroy()
4271 set_64bit_val(wqe, 0, ccq->cq_uk.cq_size); in irdma_sc_ccq_destroy()
4273 set_64bit_val(wqe, 40, ccq->shadow_area_pa); in irdma_sc_ccq_destroy()
4275 hdr = ccq->cq_uk.cq_id | in irdma_sc_ccq_destroy()
4276 FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0), in irdma_sc_ccq_destroy()
4279 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) | in irdma_sc_ccq_destroy()
4280 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) | in irdma_sc_ccq_destroy()
4281 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) | in irdma_sc_ccq_destroy()
4282 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) | in irdma_sc_ccq_destroy()
4283 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ccq_destroy()
4295 cqp->dev->hw_attrs.max_done_count); in irdma_sc_ccq_destroy()
4298 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_ccq_destroy()
4304 * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
4316 hmc_info = dev->hmc_info; in irdma_sc_init_iw_hmc()
4317 hmc_fpm_misc = &dev->hmc_fpm_misc; in irdma_sc_init_iw_hmc()
4318 query_fpm_mem.pa = dev->fpm_query_buf_pa; in irdma_sc_init_iw_hmc()
4319 query_fpm_mem.va = dev->fpm_query_buf; in irdma_sc_init_iw_hmc()
4320 hmc_info->hmc_fn_id = hmc_fn_id; in irdma_sc_init_iw_hmc()
4323 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_init_iw_hmc()
4339 * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp
4353 hmc_info = dev->hmc_info; in irdma_sc_cfg_iw_fpm()
4354 obj_info = hmc_info->hmc_obj; in irdma_sc_cfg_iw_fpm()
4355 buf = dev->fpm_commit_buf; in irdma_sc_cfg_iw_fpm()
4387 commit_fpm_mem.pa = dev->fpm_commit_buf_pa; in irdma_sc_cfg_iw_fpm()
4388 commit_fpm_mem.va = dev->fpm_commit_buf; in irdma_sc_cfg_iw_fpm()
4394 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_cfg_iw_fpm()
4397 irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf, in irdma_sc_cfg_iw_fpm()
4398 hmc_info->hmc_obj, in irdma_sc_cfg_iw_fpm()
4399 &hmc_info->sd_table.sd_cnt); in irdma_sc_cfg_iw_fpm()
4408 * cqp_sds_wqe_fill - fill cqp wqe doe sd
4420 struct irdma_dma_mem *sdbuf = &cqp->sdbuf; in cqp_sds_wqe_fill()
4426 return -ENOMEM; in cqp_sds_wqe_fill()
4428 wqe_entries = (info->cnt > 3) ? 3 : info->cnt; in cqp_sds_wqe_fill()
4429 mem_entries = info->cnt - wqe_entries; in cqp_sds_wqe_fill()
4433 memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4); in cqp_sds_wqe_fill()
4435 data = (u64)sdbuf->pa + offset; in cqp_sds_wqe_fill()
4439 data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info->hmc_fn_id); in cqp_sds_wqe_fill()
4445 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) | in cqp_sds_wqe_fill()
4448 set_64bit_val(wqe, 56, info->entry[2].data); in cqp_sds_wqe_fill()
4452 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) | in cqp_sds_wqe_fill()
4455 set_64bit_val(wqe, 40, info->entry[1].data); in cqp_sds_wqe_fill()
4459 FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd)); in cqp_sds_wqe_fill()
4461 set_64bit_val(wqe, 8, info->entry[0].data); in cqp_sds_wqe_fill()
4468 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in cqp_sds_wqe_fill()
4477 (char *)sdbuf->va + offset, in cqp_sds_wqe_fill()
4487 * irdma_update_pe_sds - cqp wqe for sd
4495 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_pe_sds()
4506 * irdma_update_sds_noccq - update sd before ccq created
4514 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_sds_noccq()
4525 cqp->dev->hw_attrs.max_done_count); in irdma_update_sds_noccq()
4529 * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
4546 return -ENOMEM; in irdma_sc_static_hmc_pages_allocated()
4553 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_static_hmc_pages_allocated()
4568 cqp->dev->hw_attrs.max_done_count); in irdma_sc_static_hmc_pages_allocated()
4579 * irdma_cqp_ring_full - check if cqp ring is full
4584 return IRDMA_RING_FULL_ERR(cqp->sq_ring); in irdma_cqp_ring_full()
4588 * irdma_est_sd - returns approximate number of SDs for HMC
4601 size += round_up(hmc_info->hmc_obj[i].cnt * in irdma_est_sd()
4602 hmc_info->hmc_obj[i].size, 512); in irdma_est_sd()
4603 size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt * in irdma_est_sd()
4604 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512); in irdma_est_sd()
4611 sd = 0xFFFFFFFF - 1; in irdma_est_sd()
4618 * irdma_sc_query_rdma_features_done - poll cqp for query features done
4629 * irdma_sc_query_rdma_features - query RDMA features and FW ver
4642 return -ENOMEM; in irdma_sc_query_rdma_features()
4644 temp = buf->pa; in irdma_sc_query_rdma_features()
4648 cqp->polarity) | in irdma_sc_query_rdma_features()
4649 FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) | in irdma_sc_query_rdma_features()
4663 * irdma_get_rdma_features - get RDMA features
4675 feat_buf.va = dma_alloc_coherent(dev->hw->device, feat_buf.size, in irdma_get_rdma_features()
4678 return -ENOMEM; in irdma_get_rdma_features()
4680 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4682 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4689 ret_code = -EINVAL; in irdma_get_rdma_features()
4694 dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va, in irdma_get_rdma_features()
4699 feat_buf.va = dma_alloc_coherent(dev->hw->device, in irdma_get_rdma_features()
4703 return -ENOMEM; in irdma_get_rdma_features()
4705 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4707 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4714 ret_code = -EINVAL; in irdma_get_rdma_features()
4732 dev->feature_info[feat_type] = temp; in irdma_get_rdma_features()
4735 dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va, in irdma_get_rdma_features()
4746 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { in irdma_q1_cnt()
4747 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted); in irdma_q1_cnt()
4749 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) in irdma_q1_cnt()
4750 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512); in irdma_q1_cnt()
4752 q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted; in irdma_q1_cnt()
4761 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes); in cfg_fpm_value_gen_1()
4767 struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc; in cfg_fpm_value_gen_2()
4769 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = in cfg_fpm_value_gen_2()
4770 4 * hmc_fpm_misc->xf_block_size * qpwanted; in cfg_fpm_value_gen_2()
4772 hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted; in cfg_fpm_value_gen_2()
4774 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt) in cfg_fpm_value_gen_2()
4775 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted; in cfg_fpm_value_gen_2()
4776 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt) in cfg_fpm_value_gen_2()
4777 hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt = in cfg_fpm_value_gen_2()
4778 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt / in cfg_fpm_value_gen_2()
4779 hmc_fpm_misc->rrf_block_size; in cfg_fpm_value_gen_2()
4780 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt) in cfg_fpm_value_gen_2()
4781 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted; in cfg_fpm_value_gen_2()
4782 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt) in cfg_fpm_value_gen_2()
4783 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt = in cfg_fpm_value_gen_2()
4784 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt / in cfg_fpm_value_gen_2()
4785 hmc_fpm_misc->ooiscf_block_size; in cfg_fpm_value_gen_2()
4789 * irdma_cfg_fpm_val - configure HMC objects
4806 hmc_info = dev->hmc_info; in irdma_cfg_fpm_val()
4807 hmc_fpm_misc = &dev->hmc_fpm_misc; in irdma_cfg_fpm_val()
4809 ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id); in irdma_cfg_fpm_val()
4818 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; in irdma_cfg_fpm_val()
4822 sd_needed, hmc_info->first_sd_index); in irdma_cfg_fpm_val()
4824 hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds); in irdma_cfg_fpm_val()
4826 qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt); in irdma_cfg_fpm_val()
4834 mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt; in irdma_cfg_fpm_val()
4835 pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt; in irdma_cfg_fpm_val()
4839 qp_count, hmc_fpm_misc->max_sds, in irdma_cfg_fpm_val()
4840 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt, in irdma_cfg_fpm_val()
4841 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt, in irdma_cfg_fpm_val()
4842 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt, in irdma_cfg_fpm_val()
4843 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt, in irdma_cfg_fpm_val()
4844 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt, in irdma_cfg_fpm_val()
4845 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt); in irdma_cfg_fpm_val()
4846 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt = in irdma_cfg_fpm_val()
4847 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt; in irdma_cfg_fpm_val()
4848 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt = in irdma_cfg_fpm_val()
4849 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt; in irdma_cfg_fpm_val()
4850 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt = in irdma_cfg_fpm_val()
4851 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt; in irdma_cfg_fpm_val()
4853 hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1; in irdma_cfg_fpm_val()
4855 while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt) in irdma_cfg_fpm_val()
4860 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted; in irdma_cfg_fpm_val()
4861 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt = in irdma_cfg_fpm_val()
4862 min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt); in irdma_cfg_fpm_val()
4863 hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */ in irdma_cfg_fpm_val()
4864 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted; in irdma_cfg_fpm_val()
4866 hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512); in irdma_cfg_fpm_val()
4870 hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt = in irdma_cfg_fpm_val()
4871 powerof2 * hmc_fpm_misc->ht_multiplier; in irdma_cfg_fpm_val()
4872 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_cfg_fpm_val()
4877 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted); in irdma_cfg_fpm_val()
4878 hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt = in irdma_cfg_fpm_val()
4879 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size; in irdma_cfg_fpm_val()
4880 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt = in irdma_cfg_fpm_val()
4881 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size; in irdma_cfg_fpm_val()
4882 hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt = in irdma_cfg_fpm_val()
4883 (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket; in irdma_cfg_fpm_val()
4885 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; in irdma_cfg_fpm_val()
4888 "HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n", in irdma_cfg_fpm_val()
4889 sd_needed, hmc_fpm_misc->max_sds, mrwanted, in irdma_cfg_fpm_val()
4893 if (sd_needed <= hmc_fpm_misc->max_sds) in irdma_cfg_fpm_val()
4896 sd_diff = sd_needed - hmc_fpm_misc->max_sds; in irdma_cfg_fpm_val()
4906 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && in irdma_cfg_fpm_val()
4908 pblewanted -= 256 * FPM_MULTIPLIER * sd_diff; in irdma_cfg_fpm_val()
4911 pblewanted -= 10 * FPM_MULTIPLIER; in irdma_cfg_fpm_val()
4913 pblewanted -= FPM_MULTIPLIER; in irdma_cfg_fpm_val()
4915 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256) in irdma_cfg_fpm_val()
4916 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2; in irdma_cfg_fpm_val()
4917 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) in irdma_cfg_fpm_val()
4918 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; in irdma_cfg_fpm_val()
4921 mrwanted -= FPM_MULTIPLIER; in irdma_cfg_fpm_val()
4924 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) in irdma_cfg_fpm_val()
4925 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; in irdma_cfg_fpm_val()
4929 if (sd_needed > hmc_fpm_misc->max_sds) { in irdma_cfg_fpm_val()
4932 loop_count, sd_needed, hmc_info->sd_table.sd_cnt); in irdma_cfg_fpm_val()
4933 return -EINVAL; in irdma_cfg_fpm_val()
4936 if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) { in irdma_cfg_fpm_val()
4937 pblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 * in irdma_cfg_fpm_val()
4939 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; in irdma_cfg_fpm_val()
4946 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt, in irdma_cfg_fpm_val()
4947 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt, in irdma_cfg_fpm_val()
4948 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt, in irdma_cfg_fpm_val()
4949 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt, in irdma_cfg_fpm_val()
4950 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, in irdma_cfg_fpm_val()
4951 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt, in irdma_cfg_fpm_val()
4952 hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index); in irdma_cfg_fpm_val()
4954 ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id); in irdma_cfg_fpm_val()
4958 readl(dev->hw_regs[IRDMA_CQPERRCODES])); in irdma_cfg_fpm_val()
4963 (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1); in irdma_cfg_fpm_val()
4969 return -ENOMEM; in irdma_cfg_fpm_val()
4971 hmc_info->sd_table.sd_entry = virt_mem.va; in irdma_cfg_fpm_val()
4977 * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available
4988 dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++; in irdma_exec_cqp_cmd()
4989 switch (pcmdinfo->cqp_cmd) { in irdma_exec_cqp_cmd()
4991 status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq, in irdma_exec_cqp_cmd()
4992 pcmdinfo->in.u.ceq_destroy.scratch, in irdma_exec_cqp_cmd()
4993 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
4996 status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq, in irdma_exec_cqp_cmd()
4997 pcmdinfo->in.u.aeq_destroy.scratch, in irdma_exec_cqp_cmd()
4998 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5002 status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq, in irdma_exec_cqp_cmd()
5003 pcmdinfo->in.u.ceq_create.scratch, in irdma_exec_cqp_cmd()
5004 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5007 status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq, in irdma_exec_cqp_cmd()
5008 pcmdinfo->in.u.aeq_create.scratch, in irdma_exec_cqp_cmd()
5009 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5012 status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev, in irdma_exec_cqp_cmd()
5013 &pcmdinfo->in.u.qp_upload_context.info, in irdma_exec_cqp_cmd()
5014 pcmdinfo->in.u.qp_upload_context.scratch, in irdma_exec_cqp_cmd()
5015 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5018 status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq, in irdma_exec_cqp_cmd()
5019 pcmdinfo->in.u.cq_create.scratch, in irdma_exec_cqp_cmd()
5020 pcmdinfo->in.u.cq_create.check_overflow, in irdma_exec_cqp_cmd()
5021 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5024 status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq, in irdma_exec_cqp_cmd()
5025 &pcmdinfo->in.u.cq_modify.info, in irdma_exec_cqp_cmd()
5026 pcmdinfo->in.u.cq_modify.scratch, in irdma_exec_cqp_cmd()
5027 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5030 status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq, in irdma_exec_cqp_cmd()
5031 pcmdinfo->in.u.cq_destroy.scratch, in irdma_exec_cqp_cmd()
5032 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5035 status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp, in irdma_exec_cqp_cmd()
5036 &pcmdinfo->in.u.qp_flush_wqes.info, in irdma_exec_cqp_cmd()
5037 pcmdinfo->in.u.qp_flush_wqes.scratch, in irdma_exec_cqp_cmd()
5038 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5041 status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp, in irdma_exec_cqp_cmd()
5042 &pcmdinfo->in.u.gen_ae.info, in irdma_exec_cqp_cmd()
5043 pcmdinfo->in.u.gen_ae.scratch, in irdma_exec_cqp_cmd()
5044 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5047 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp, in irdma_exec_cqp_cmd()
5048 &pcmdinfo->in.u.manage_push_page.info, in irdma_exec_cqp_cmd()
5049 pcmdinfo->in.u.manage_push_page.scratch, in irdma_exec_cqp_cmd()
5050 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5053 status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev, in irdma_exec_cqp_cmd()
5054 &pcmdinfo->in.u.update_pe_sds.info, in irdma_exec_cqp_cmd()
5055 pcmdinfo->in.u.update_pe_sds.scratch); in irdma_exec_cqp_cmd()
5060 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp, in irdma_exec_cqp_cmd()
5061 &pcmdinfo->in.u.manage_hmc_pm.info, in irdma_exec_cqp_cmd()
5062 pcmdinfo->in.u.manage_hmc_pm.scratch, in irdma_exec_cqp_cmd()
5066 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5067 pcmdinfo->in.u.suspend_resume.qp, in irdma_exec_cqp_cmd()
5068 pcmdinfo->in.u.suspend_resume.scratch); in irdma_exec_cqp_cmd()
5071 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5072 pcmdinfo->in.u.suspend_resume.qp, in irdma_exec_cqp_cmd()
5073 pcmdinfo->in.u.suspend_resume.scratch); in irdma_exec_cqp_cmd()
5076 val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa; in irdma_exec_cqp_cmd()
5077 val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va; in irdma_exec_cqp_cmd()
5078 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp, in irdma_exec_cqp_cmd()
5079 pcmdinfo->in.u.query_fpm_val.scratch, in irdma_exec_cqp_cmd()
5080 pcmdinfo->in.u.query_fpm_val.hmc_fn_id, in irdma_exec_cqp_cmd()
5084 val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa; in irdma_exec_cqp_cmd()
5085 val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va; in irdma_exec_cqp_cmd()
5086 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp, in irdma_exec_cqp_cmd()
5087 pcmdinfo->in.u.commit_fpm_val.scratch, in irdma_exec_cqp_cmd()
5088 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id, in irdma_exec_cqp_cmd()
5097 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp, in irdma_exec_cqp_cmd()
5098 &pcmdinfo->in.u.stats_manage.info, in irdma_exec_cqp_cmd()
5100 pcmdinfo->in.u.stats_manage.scratch); in irdma_exec_cqp_cmd()
5103 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp, in irdma_exec_cqp_cmd()
5104 &pcmdinfo->in.u.stats_gather.info, in irdma_exec_cqp_cmd()
5105 pcmdinfo->in.u.stats_gather.scratch); in irdma_exec_cqp_cmd()
5108 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5109 &pcmdinfo->in.u.ws_node.info, in irdma_exec_cqp_cmd()
5111 pcmdinfo->in.u.ws_node.scratch); in irdma_exec_cqp_cmd()
5114 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5115 &pcmdinfo->in.u.ws_node.info, in irdma_exec_cqp_cmd()
5117 pcmdinfo->in.u.ws_node.scratch); in irdma_exec_cqp_cmd()
5120 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5121 &pcmdinfo->in.u.ws_node.info, in irdma_exec_cqp_cmd()
5123 pcmdinfo->in.u.ws_node.scratch); in irdma_exec_cqp_cmd()
5126 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp, in irdma_exec_cqp_cmd()
5127 &pcmdinfo->in.u.up_map.info, in irdma_exec_cqp_cmd()
5128 pcmdinfo->in.u.up_map.scratch); in irdma_exec_cqp_cmd()
5131 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp, in irdma_exec_cqp_cmd()
5132 &pcmdinfo->in.u.query_rdma.query_buff_mem, in irdma_exec_cqp_cmd()
5133 pcmdinfo->in.u.query_rdma.scratch); in irdma_exec_cqp_cmd()
5136 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5137 pcmdinfo->in.u.del_arp_cache_entry.scratch, in irdma_exec_cqp_cmd()
5138 pcmdinfo->in.u.del_arp_cache_entry.arp_index, in irdma_exec_cqp_cmd()
5139 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5142 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp, in irdma_exec_cqp_cmd()
5143 &pcmdinfo->in.u.manage_apbvt_entry.info, in irdma_exec_cqp_cmd()
5144 pcmdinfo->in.u.manage_apbvt_entry.scratch, in irdma_exec_cqp_cmd()
5145 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5148 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp, in irdma_exec_cqp_cmd()
5149 &pcmdinfo->in.u.manage_qhash_table_entry.info, in irdma_exec_cqp_cmd()
5150 pcmdinfo->in.u.manage_qhash_table_entry.scratch, in irdma_exec_cqp_cmd()
5151 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5154 status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp, in irdma_exec_cqp_cmd()
5155 &pcmdinfo->in.u.qp_modify.info, in irdma_exec_cqp_cmd()
5156 pcmdinfo->in.u.qp_modify.scratch, in irdma_exec_cqp_cmd()
5157 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5160 status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp, in irdma_exec_cqp_cmd()
5161 &pcmdinfo->in.u.qp_create.info, in irdma_exec_cqp_cmd()
5162 pcmdinfo->in.u.qp_create.scratch, in irdma_exec_cqp_cmd()
5163 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5166 status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp, in irdma_exec_cqp_cmd()
5167 pcmdinfo->in.u.qp_destroy.scratch, in irdma_exec_cqp_cmd()
5168 pcmdinfo->in.u.qp_destroy.remove_hash_idx, in irdma_exec_cqp_cmd()
5169 pcmdinfo->in.u.qp_destroy.ignore_mw_bnd, in irdma_exec_cqp_cmd()
5170 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5173 status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev, in irdma_exec_cqp_cmd()
5174 &pcmdinfo->in.u.alloc_stag.info, in irdma_exec_cqp_cmd()
5175 pcmdinfo->in.u.alloc_stag.scratch, in irdma_exec_cqp_cmd()
5176 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5179 status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev, in irdma_exec_cqp_cmd()
5180 &pcmdinfo->in.u.mr_reg_non_shared.info, in irdma_exec_cqp_cmd()
5181 pcmdinfo->in.u.mr_reg_non_shared.scratch, in irdma_exec_cqp_cmd()
5182 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5185 status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev, in irdma_exec_cqp_cmd()
5186 &pcmdinfo->in.u.dealloc_stag.info, in irdma_exec_cqp_cmd()
5187 pcmdinfo->in.u.dealloc_stag.scratch, in irdma_exec_cqp_cmd()
5188 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5191 status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev, in irdma_exec_cqp_cmd()
5192 &pcmdinfo->in.u.mw_alloc.info, in irdma_exec_cqp_cmd()
5193 pcmdinfo->in.u.mw_alloc.scratch, in irdma_exec_cqp_cmd()
5194 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5197 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5198 &pcmdinfo->in.u.add_arp_cache_entry.info, in irdma_exec_cqp_cmd()
5199 pcmdinfo->in.u.add_arp_cache_entry.scratch, in irdma_exec_cqp_cmd()
5200 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5203 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5204 pcmdinfo->in.u.alloc_local_mac_entry.scratch, in irdma_exec_cqp_cmd()
5205 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5208 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5209 &pcmdinfo->in.u.add_local_mac_entry.info, in irdma_exec_cqp_cmd()
5210 pcmdinfo->in.u.add_local_mac_entry.scratch, in irdma_exec_cqp_cmd()
5211 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5214 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5215 pcmdinfo->in.u.del_local_mac_entry.scratch, in irdma_exec_cqp_cmd()
5216 pcmdinfo->in.u.del_local_mac_entry.entry_idx, in irdma_exec_cqp_cmd()
5217 pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count, in irdma_exec_cqp_cmd()
5218 pcmdinfo->post_sq); in irdma_exec_cqp_cmd()
5221 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp, in irdma_exec_cqp_cmd()
5222 &pcmdinfo->in.u.ah_create.info, in irdma_exec_cqp_cmd()
5223 pcmdinfo->in.u.ah_create.scratch); in irdma_exec_cqp_cmd()
5226 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp, in irdma_exec_cqp_cmd()
5227 &pcmdinfo->in.u.ah_destroy.info, in irdma_exec_cqp_cmd()
5228 pcmdinfo->in.u.ah_destroy.scratch); in irdma_exec_cqp_cmd()
5231 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp, in irdma_exec_cqp_cmd()
5232 &pcmdinfo->in.u.mc_create.info, in irdma_exec_cqp_cmd()
5233 pcmdinfo->in.u.mc_create.scratch); in irdma_exec_cqp_cmd()
5236 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp, in irdma_exec_cqp_cmd()
5237 &pcmdinfo->in.u.mc_destroy.info, in irdma_exec_cqp_cmd()
5238 pcmdinfo->in.u.mc_destroy.scratch); in irdma_exec_cqp_cmd()
5241 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp, in irdma_exec_cqp_cmd()
5242 &pcmdinfo->in.u.mc_modify.info, in irdma_exec_cqp_cmd()
5243 pcmdinfo->in.u.mc_modify.scratch); in irdma_exec_cqp_cmd()
5246 status = -EOPNOTSUPP; in irdma_exec_cqp_cmd()
5254 * irdma_process_cqp_cmd - process all cqp commands
5264 spin_lock_irqsave(&dev->cqp_lock, flags); in irdma_process_cqp_cmd()
5265 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp)) in irdma_process_cqp_cmd()
5268 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head); in irdma_process_cqp_cmd()
5269 spin_unlock_irqrestore(&dev->cqp_lock, flags); in irdma_process_cqp_cmd()
5274 * irdma_process_bh - called from tasklet for cqp list
5283 spin_lock_irqsave(&dev->cqp_lock, flags); in irdma_process_bh()
5284 while (!list_empty(&dev->cqp_cmd_head) && in irdma_process_bh()
5285 !irdma_cqp_ring_full(dev->cqp)) { in irdma_process_bh()
5291 spin_unlock_irqrestore(&dev->cqp_lock, flags); in irdma_process_bh()
5296 * irdma_cfg_aeq- Configure AEQ interrupt
5308 writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]); in irdma_cfg_aeq()
5312 * sc_vsi_update_stats - Update statistics
5320 gather_stats = vsi->pestat->gather_info.gather_stats_va; in sc_vsi_update_stats()
5321 last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va; in sc_vsi_update_stats()
5322 irdma_update_stats(&vsi->pestat->hw_stats, gather_stats, in sc_vsi_update_stats()
5323 last_gather_stats, vsi->dev->hw_stats_map, in sc_vsi_update_stats()
5324 vsi->dev->hw_attrs.max_stat_idx); in sc_vsi_update_stats()
5328 * irdma_wait_pe_ready - Check if firmware is ready
5339 statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]); in irdma_wait_pe_ready()
5340 statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]); in irdma_wait_pe_ready()
5341 statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]); in irdma_wait_pe_ready()
5346 } while (retrycount++ < dev->hw_attrs.max_pe_ready_count); in irdma_wait_pe_ready()
5347 return -1; in irdma_wait_pe_ready()
5352 switch (dev->hw_attrs.uk_attrs.hw_rev) { in irdma_sc_init_hw()
5363 * irdma_sc_dev_init - Initialize control part of device
5375 INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */ in irdma_sc_dev_init()
5376 mutex_init(&dev->ws_mutex); in irdma_sc_dev_init()
5377 dev->hmc_fn_id = info->hmc_fn_id; in irdma_sc_dev_init()
5378 dev->fpm_query_buf_pa = info->fpm_query_buf_pa; in irdma_sc_dev_init()
5379 dev->fpm_query_buf = info->fpm_query_buf; in irdma_sc_dev_init()
5380 dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa; in irdma_sc_dev_init()
5381 dev->fpm_commit_buf = info->fpm_commit_buf; in irdma_sc_dev_init()
5382 dev->hw = info->hw; in irdma_sc_dev_init()
5383 dev->hw->hw_addr = info->bar0; in irdma_sc_dev_init()
5385 dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID; in irdma_sc_dev_init()
5386 dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES; in irdma_sc_dev_init()
5387 dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES; in irdma_sc_dev_init()
5388 dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES; in irdma_sc_dev_init()
5389 dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES; in irdma_sc_dev_init()
5390 dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE; in irdma_sc_dev_init()
5391 dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE; in irdma_sc_dev_init()
5392 dev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT; in irdma_sc_dev_init()
5393 dev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD; in irdma_sc_dev_init()
5394 dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE; in irdma_sc_dev_init()
5395 dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE; in irdma_sc_dev_init()
5396 dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE; in irdma_sc_dev_init()
5397 dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT; in irdma_sc_dev_init()
5398 dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE; in irdma_sc_dev_init()
5399 dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES; in irdma_sc_dev_init()
5400 dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR); in irdma_sc_dev_init()
5402 dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA; in irdma_sc_dev_init()
5403 dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA; in irdma_sc_dev_init()
5404 dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS; in irdma_sc_dev_init()
5405 dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT; in irdma_sc_dev_init()
5407 dev->hw_attrs.max_pe_ready_count = 14; in irdma_sc_dev_init()
5408 dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT; in irdma_sc_dev_init()
5409 dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT; in irdma_sc_dev_init()
5410 dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS; in irdma_sc_dev_init()
5412 dev->hw_attrs.uk_attrs.hw_rev = ver; in irdma_sc_dev_init()
5416 return -ETIMEDOUT; in irdma_sc_dev_init()
5418 val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]); in irdma_sc_dev_init()
5424 return -ENODEV; in irdma_sc_dev_init()
5426 dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET]; in irdma_sc_dev_init()
5432 * irdma_stat_val - Extract HW counter value from statistics buffer
5434 * @byteoff: byte offset of counter value in the buffer (8B-aligned)
5436 * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter)
5447 * irdma_stat_delta - Calculate counter delta
5450 * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter)
5455 return new_val - old_val; in irdma_stat_delta()
5457 /* roll-over case */ in irdma_stat_delta()
5458 return max_val - old_val + new_val + 1; in irdma_stat_delta()
5462 * irdma_update_stats - Update statistics
5474 u64 *stats_val = hw_stats->stats_val; in irdma_update_stats()
5478 u64 new_val = irdma_stat_val(gather_stats->val, map[i].byteoff, in irdma_update_stats()
5480 u64 last_val = irdma_stat_val(last_gather_stats->val, in irdma_update_stats()