Lines Matching +full:com +full:- +full:seq
2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
87 struct c4iw_debugfs_data *d = file->private_data; in debugfs_read()
89 return simple_read_from_buffer(buf, count, ppos, d->buf, d->pos); in debugfs_read()
97 if (!wq->rdev->wr_log) in c4iw_log_wr_stats()
100 idx = (atomic_inc_return(&wq->rdev->wr_log_idx) - 1) & in c4iw_log_wr_stats()
101 (wq->rdev->wr_log_size - 1); in c4iw_log_wr_stats()
102 le.poll_sge_ts = cxgb4_read_sge_timestamp(wq->rdev->lldi.ports[0]); in c4iw_log_wr_stats()
107 le.qid = wq->sq.qid; in c4iw_log_wr_stats()
109 le.post_host_time = wq->sq.sw_sq[wq->sq.cidx].host_time; in c4iw_log_wr_stats()
110 le.post_sge_ts = wq->sq.sw_sq[wq->sq.cidx].sge_ts; in c4iw_log_wr_stats()
113 le.qid = wq->rq.qid; in c4iw_log_wr_stats()
115 le.post_host_time = wq->rq.sw_rq[wq->rq.cidx].host_time; in c4iw_log_wr_stats()
116 le.post_sge_ts = wq->rq.sw_rq[wq->rq.cidx].sge_ts; in c4iw_log_wr_stats()
119 wq->rdev->wr_log[idx] = le; in c4iw_log_wr_stats()
122 static int wr_log_show(struct seq_file *seq, void *v) in wr_log_show() argument
124 struct c4iw_dev *dev = seq->private; in wr_log_show()
130 #define ts2ns(ts) div64_u64((ts) * dev->rdev.lldi.cclk_ps, 1000) in wr_log_show()
132 idx = atomic_read(&dev->rdev.wr_log_idx) & in wr_log_show()
133 (dev->rdev.wr_log_size - 1); in wr_log_show()
134 end = idx - 1; in wr_log_show()
136 end = dev->rdev.wr_log_size - 1; in wr_log_show()
137 lep = &dev->rdev.wr_log[idx]; in wr_log_show()
139 if (lep->valid) { in wr_log_show()
142 prev_time = lep->poll_host_time; in wr_log_show()
144 seq_printf(seq, "%04u: nsec %llu qid %u opcode " in wr_log_show()
150 ktime_to_ns(ktime_sub(lep->poll_host_time, in wr_log_show()
152 lep->qid, lep->opcode, in wr_log_show()
153 lep->opcode == FW_RI_RECEIVE ? in wr_log_show()
155 lep->wr_id, in wr_log_show()
156 ktime_to_ns(ktime_sub(lep->poll_host_time, in wr_log_show()
157 lep->post_host_time)), in wr_log_show()
158 lep->post_sge_ts, lep->cqe_sge_ts, in wr_log_show()
159 lep->poll_sge_ts, in wr_log_show()
160 ts2ns(lep->poll_sge_ts - lep->post_sge_ts), in wr_log_show()
161 ts2ns(lep->poll_sge_ts - lep->cqe_sge_ts)); in wr_log_show()
162 prev_time = lep->poll_host_time; in wr_log_show()
165 if (idx > (dev->rdev.wr_log_size - 1)) in wr_log_show()
167 lep = &dev->rdev.wr_log[idx]; in wr_log_show()
175 return single_open(file, wr_log_show, inode->i_private); in wr_log_open()
181 struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; in wr_log_clear()
184 if (dev->rdev.wr_log) in wr_log_clear()
185 for (i = 0; i < dev->rdev.wr_log_size; i++) in wr_log_clear()
186 dev->rdev.wr_log[i].valid = 0; in wr_log_clear()
213 struct iw_cm_id *id = ep->com.cm_id; in set_ep_sin_addrs()
215 *m_lsin = (struct sockaddr_in *)&ep->com.local_addr; in set_ep_sin_addrs()
216 *m_rsin = (struct sockaddr_in *)&ep->com.remote_addr; in set_ep_sin_addrs()
218 *lsin = (struct sockaddr_in *)&id->local_addr; in set_ep_sin_addrs()
219 *rsin = (struct sockaddr_in *)&id->remote_addr; in set_ep_sin_addrs()
232 struct iw_cm_id *id = ep->com.cm_id; in set_ep_sin6_addrs()
234 *m_lsin6 = (struct sockaddr_in6 *)&ep->com.local_addr; in set_ep_sin6_addrs()
235 *m_rsin6 = (struct sockaddr_in6 *)&ep->com.remote_addr; in set_ep_sin6_addrs()
237 *lsin6 = (struct sockaddr_in6 *)&id->local_addr; in set_ep_sin6_addrs()
238 *rsin6 = (struct sockaddr_in6 *)&id->remote_addr; in set_ep_sin6_addrs()
250 if (id != qp->wq.sq.qid) in dump_qp()
253 space = qpd->bufsize - qpd->pos - 1; in dump_qp()
257 if (qp->ep) { in dump_qp()
258 struct c4iw_ep *ep = qp->ep; in dump_qp()
260 if (ep->com.local_addr.ss_family == AF_INET) { in dump_qp()
267 cc = snprintf(qpd->buf + qpd->pos, space, in dump_qp()
270 "%pI4:%u/%u->%pI4:%u/%u\n", in dump_qp()
271 qp->wq.sq.qid, qp->srq ? "srq" : "rq", in dump_qp()
272 qp->srq ? qp->srq->idx : qp->wq.rq.qid, in dump_qp()
273 (int)qp->attr.state, in dump_qp()
274 qp->wq.sq.flags & T4_SQ_ONCHIP, in dump_qp()
275 ep->hwtid, (int)ep->com.state, in dump_qp()
276 &lsin->sin_addr, ntohs(lsin->sin_port), in dump_qp()
277 ntohs(m_lsin->sin_port), in dump_qp()
278 &rsin->sin_addr, ntohs(rsin->sin_port), in dump_qp()
279 ntohs(m_rsin->sin_port)); in dump_qp()
288 cc = snprintf(qpd->buf + qpd->pos, space, in dump_qp()
291 "%pI6:%u/%u->%pI6:%u/%u\n", in dump_qp()
292 qp->wq.sq.qid, qp->wq.rq.qid, in dump_qp()
293 (int)qp->attr.state, in dump_qp()
294 qp->wq.sq.flags & T4_SQ_ONCHIP, in dump_qp()
295 ep->hwtid, (int)ep->com.state, in dump_qp()
296 &lsin6->sin6_addr, in dump_qp()
297 ntohs(lsin6->sin6_port), in dump_qp()
298 ntohs(m_lsin6->sin6_port), in dump_qp()
299 &rsin6->sin6_addr, in dump_qp()
300 ntohs(rsin6->sin6_port), in dump_qp()
301 ntohs(m_rsin6->sin6_port)); in dump_qp()
304 cc = snprintf(qpd->buf + qpd->pos, space, in dump_qp()
306 qp->wq.sq.qid, qp->wq.rq.qid, in dump_qp()
307 (int)qp->attr.state, in dump_qp()
308 qp->wq.sq.flags & T4_SQ_ONCHIP); in dump_qp()
310 qpd->pos += cc; in dump_qp()
316 struct c4iw_debugfs_data *qpd = file->private_data; in qp_release()
321 vfree(qpd->buf); in qp_release()
335 return -ENOMEM; in qp_open()
337 qpd->devp = inode->i_private; in qp_open()
338 qpd->pos = 0; in qp_open()
344 xa_for_each(&qpd->devp->qps, index, qp) in qp_open()
347 qpd->bufsize = count * 180; in qp_open()
348 qpd->buf = vmalloc(qpd->bufsize); in qp_open()
349 if (!qpd->buf) { in qp_open()
351 return -ENOMEM; in qp_open()
354 xa_lock_irq(&qpd->devp->qps); in qp_open()
355 xa_for_each(&qpd->devp->qps, index, qp) in qp_open()
357 xa_unlock_irq(&qpd->devp->qps); in qp_open()
359 qpd->buf[qpd->pos++] = 0; in qp_open()
360 file->private_data = qpd; in qp_open()
379 space = stagd->bufsize - stagd->pos - 1; in dump_stag()
383 ret = cxgb4_read_tpte(stagd->devp->rdev.lldi.ports[0], (u32)id<<8, in dump_stag()
386 dev_err(&stagd->devp->rdev.lldi.pdev->dev, in dump_stag()
390 cc = snprintf(stagd->buf + stagd->pos, space, in dump_stag()
403 stagd->pos += cc; in dump_stag()
409 struct c4iw_debugfs_data *stagd = file->private_data; in stag_release()
414 vfree(stagd->buf); in stag_release()
429 ret = -ENOMEM; in stag_open()
432 stagd->devp = inode->i_private; in stag_open()
433 stagd->pos = 0; in stag_open()
435 xa_for_each(&stagd->devp->mrs, index, p) in stag_open()
438 stagd->bufsize = count * 256; in stag_open()
439 stagd->buf = vmalloc(stagd->bufsize); in stag_open()
440 if (!stagd->buf) { in stag_open()
441 ret = -ENOMEM; in stag_open()
445 xa_lock_irq(&stagd->devp->mrs); in stag_open()
446 xa_for_each(&stagd->devp->mrs, index, p) in stag_open()
448 xa_unlock_irq(&stagd->devp->mrs); in stag_open()
450 stagd->buf[stagd->pos++] = 0; in stag_open()
451 file->private_data = stagd; in stag_open()
469 static int stats_show(struct seq_file *seq, void *v) in stats_show() argument
471 struct c4iw_dev *dev = seq->private; in stats_show()
473 seq_printf(seq, " Object: %10s %10s %10s %10s\n", "Total", "Current", in stats_show()
475 seq_printf(seq, " PDID: %10llu %10llu %10llu %10llu\n", in stats_show()
476 dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur, in stats_show()
477 dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail); in stats_show()
478 seq_printf(seq, " QID: %10llu %10llu %10llu %10llu\n", in stats_show()
479 dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur, in stats_show()
480 dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail); in stats_show()
481 seq_printf(seq, " SRQS: %10llu %10llu %10llu %10llu\n", in stats_show()
482 dev->rdev.stats.srqt.total, dev->rdev.stats.srqt.cur, in stats_show()
483 dev->rdev.stats.srqt.max, dev->rdev.stats.srqt.fail); in stats_show()
484 seq_printf(seq, " TPTMEM: %10llu %10llu %10llu %10llu\n", in stats_show()
485 dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur, in stats_show()
486 dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail); in stats_show()
487 seq_printf(seq, " PBLMEM: %10llu %10llu %10llu %10llu\n", in stats_show()
488 dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur, in stats_show()
489 dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail); in stats_show()
490 seq_printf(seq, " RQTMEM: %10llu %10llu %10llu %10llu\n", in stats_show()
491 dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur, in stats_show()
492 dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail); in stats_show()
493 seq_printf(seq, " OCQPMEM: %10llu %10llu %10llu %10llu\n", in stats_show()
494 dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur, in stats_show()
495 dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail); in stats_show()
496 seq_printf(seq, " DB FULL: %10llu\n", dev->rdev.stats.db_full); in stats_show()
497 seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty); in stats_show()
498 seq_printf(seq, " DB DROP: %10llu\n", dev->rdev.stats.db_drop); in stats_show()
499 seq_printf(seq, " DB State: %s Transitions %llu FC Interruptions %llu\n", in stats_show()
500 db_state_str[dev->db_state], in stats_show()
501 dev->rdev.stats.db_state_transitions, in stats_show()
502 dev->rdev.stats.db_fc_interruptions); in stats_show()
503 seq_printf(seq, "TCAM_FULL: %10llu\n", dev->rdev.stats.tcam_full); in stats_show()
504 seq_printf(seq, "ACT_OFLD_CONN_FAILS: %10llu\n", in stats_show()
505 dev->rdev.stats.act_ofld_conn_fails); in stats_show()
506 seq_printf(seq, "PAS_OFLD_CONN_FAILS: %10llu\n", in stats_show()
507 dev->rdev.stats.pas_ofld_conn_fails); in stats_show()
508 seq_printf(seq, "NEG_ADV_RCVD: %10llu\n", dev->rdev.stats.neg_adv); in stats_show()
509 seq_printf(seq, "AVAILABLE IRD: %10u\n", dev->avail_ird); in stats_show()
515 return single_open(file, stats_show, inode->i_private); in stats_open()
521 struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; in stats_clear()
523 mutex_lock(&dev->rdev.stats.lock); in stats_clear()
524 dev->rdev.stats.pd.max = 0; in stats_clear()
525 dev->rdev.stats.pd.fail = 0; in stats_clear()
526 dev->rdev.stats.qid.max = 0; in stats_clear()
527 dev->rdev.stats.qid.fail = 0; in stats_clear()
528 dev->rdev.stats.stag.max = 0; in stats_clear()
529 dev->rdev.stats.stag.fail = 0; in stats_clear()
530 dev->rdev.stats.pbl.max = 0; in stats_clear()
531 dev->rdev.stats.pbl.fail = 0; in stats_clear()
532 dev->rdev.stats.rqt.max = 0; in stats_clear()
533 dev->rdev.stats.rqt.fail = 0; in stats_clear()
534 dev->rdev.stats.rqt.max = 0; in stats_clear()
535 dev->rdev.stats.rqt.fail = 0; in stats_clear()
536 dev->rdev.stats.ocqp.max = 0; in stats_clear()
537 dev->rdev.stats.ocqp.fail = 0; in stats_clear()
538 dev->rdev.stats.db_full = 0; in stats_clear()
539 dev->rdev.stats.db_empty = 0; in stats_clear()
540 dev->rdev.stats.db_drop = 0; in stats_clear()
541 dev->rdev.stats.db_state_transitions = 0; in stats_clear()
542 dev->rdev.stats.tcam_full = 0; in stats_clear()
543 dev->rdev.stats.act_ofld_conn_fails = 0; in stats_clear()
544 dev->rdev.stats.pas_ofld_conn_fails = 0; in stats_clear()
545 mutex_unlock(&dev->rdev.stats.lock); in stats_clear()
563 space = epd->bufsize - epd->pos - 1; in dump_ep()
567 if (ep->com.local_addr.ss_family == AF_INET) { in dump_ep()
574 cc = snprintf(epd->buf + epd->pos, space, in dump_ep()
578 "%pI4:%d/%d <-> %pI4:%d/%d\n", in dump_ep()
579 ep, ep->com.cm_id, ep->com.qp, in dump_ep()
580 (int)ep->com.state, ep->com.flags, in dump_ep()
581 ep->com.history, ep->hwtid, ep->atid, in dump_ep()
582 ep->stats.connect_neg_adv, in dump_ep()
583 ep->stats.abort_neg_adv, in dump_ep()
584 &lsin->sin_addr, ntohs(lsin->sin_port), in dump_ep()
585 ntohs(m_lsin->sin_port), in dump_ep()
586 &rsin->sin_addr, ntohs(rsin->sin_port), in dump_ep()
587 ntohs(m_rsin->sin_port)); in dump_ep()
595 cc = snprintf(epd->buf + epd->pos, space, in dump_ep()
599 "%pI6:%d/%d <-> %pI6:%d/%d\n", in dump_ep()
600 ep, ep->com.cm_id, ep->com.qp, in dump_ep()
601 (int)ep->com.state, ep->com.flags, in dump_ep()
602 ep->com.history, ep->hwtid, ep->atid, in dump_ep()
603 ep->stats.connect_neg_adv, in dump_ep()
604 ep->stats.abort_neg_adv, in dump_ep()
605 &lsin6->sin6_addr, ntohs(lsin6->sin6_port), in dump_ep()
606 ntohs(m_lsin6->sin6_port), in dump_ep()
607 &rsin6->sin6_addr, ntohs(rsin6->sin6_port), in dump_ep()
608 ntohs(m_rsin6->sin6_port)); in dump_ep()
611 epd->pos += cc; in dump_ep()
621 space = epd->bufsize - epd->pos - 1; in dump_listen_ep()
625 if (ep->com.local_addr.ss_family == AF_INET) { in dump_listen_ep()
627 &ep->com.cm_id->local_addr; in dump_listen_ep()
629 &ep->com.cm_id->m_local_addr; in dump_listen_ep()
631 cc = snprintf(epd->buf + epd->pos, space, in dump_listen_ep()
634 ep, ep->com.cm_id, (int)ep->com.state, in dump_listen_ep()
635 ep->com.flags, ep->stid, ep->backlog, in dump_listen_ep()
636 &lsin->sin_addr, ntohs(lsin->sin_port), in dump_listen_ep()
637 ntohs(m_lsin->sin_port)); in dump_listen_ep()
640 &ep->com.cm_id->local_addr; in dump_listen_ep()
642 &ep->com.cm_id->m_local_addr; in dump_listen_ep()
644 cc = snprintf(epd->buf + epd->pos, space, in dump_listen_ep()
647 ep, ep->com.cm_id, (int)ep->com.state, in dump_listen_ep()
648 ep->com.flags, ep->stid, ep->backlog, in dump_listen_ep()
649 &lsin6->sin6_addr, ntohs(lsin6->sin6_port), in dump_listen_ep()
650 ntohs(m_lsin6->sin6_port)); in dump_listen_ep()
653 epd->pos += cc; in dump_listen_ep()
659 struct c4iw_debugfs_data *epd = file->private_data; in ep_release()
664 vfree(epd->buf); in ep_release()
680 ret = -ENOMEM; in ep_open()
683 epd->devp = inode->i_private; in ep_open()
684 epd->pos = 0; in ep_open()
686 xa_for_each(&epd->devp->hwtids, index, ep) in ep_open()
688 xa_for_each(&epd->devp->atids, index, ep) in ep_open()
690 xa_for_each(&epd->devp->stids, index, lep) in ep_open()
693 epd->bufsize = count * 240; in ep_open()
694 epd->buf = vmalloc(epd->bufsize); in ep_open()
695 if (!epd->buf) { in ep_open()
696 ret = -ENOMEM; in ep_open()
700 xa_lock_irq(&epd->devp->hwtids); in ep_open()
701 xa_for_each(&epd->devp->hwtids, index, ep) in ep_open()
703 xa_unlock_irq(&epd->devp->hwtids); in ep_open()
704 xa_lock_irq(&epd->devp->atids); in ep_open()
705 xa_for_each(&epd->devp->atids, index, ep) in ep_open()
707 xa_unlock_irq(&epd->devp->atids); in ep_open()
708 xa_lock_irq(&epd->devp->stids); in ep_open()
709 xa_for_each(&epd->devp->stids, index, lep) in ep_open()
711 xa_unlock_irq(&epd->devp->stids); in ep_open()
713 file->private_data = epd; in ep_open()
730 debugfs_create_file_size("qps", S_IWUSR, devp->debugfs_root, in setup_debugfs()
733 debugfs_create_file_size("stags", S_IWUSR, devp->debugfs_root, in setup_debugfs()
736 debugfs_create_file_size("stats", S_IWUSR, devp->debugfs_root, in setup_debugfs()
739 debugfs_create_file_size("eps", S_IWUSR, devp->debugfs_root, in setup_debugfs()
743 debugfs_create_file_size("wr_log", S_IWUSR, devp->debugfs_root, in setup_debugfs()
753 mutex_lock(&uctx->lock); in c4iw_release_dev_ucontext()
754 list_for_each_safe(pos, nxt, &uctx->qpids) { in c4iw_release_dev_ucontext()
756 list_del_init(&entry->entry); in c4iw_release_dev_ucontext()
757 if (!(entry->qid & rdev->qpmask)) { in c4iw_release_dev_ucontext()
758 c4iw_put_resource(&rdev->resource.qid_table, in c4iw_release_dev_ucontext()
759 entry->qid); in c4iw_release_dev_ucontext()
760 mutex_lock(&rdev->stats.lock); in c4iw_release_dev_ucontext()
761 rdev->stats.qid.cur -= rdev->qpmask + 1; in c4iw_release_dev_ucontext()
762 mutex_unlock(&rdev->stats.lock); in c4iw_release_dev_ucontext()
767 list_for_each_safe(pos, nxt, &uctx->cqids) { in c4iw_release_dev_ucontext()
769 list_del_init(&entry->entry); in c4iw_release_dev_ucontext()
772 mutex_unlock(&uctx->lock); in c4iw_release_dev_ucontext()
778 INIT_LIST_HEAD(&uctx->qpids); in c4iw_init_dev_ucontext()
779 INIT_LIST_HEAD(&uctx->cqids); in c4iw_init_dev_ucontext()
780 mutex_init(&uctx->lock); in c4iw_init_dev_ucontext()
789 c4iw_init_dev_ucontext(rdev, &rdev->uctx); in c4iw_rdev_open()
796 if (rdev->lldi.udb_density != rdev->lldi.ucq_density) { in c4iw_rdev_open()
798 pci_name(rdev->lldi.pdev), rdev->lldi.udb_density, in c4iw_rdev_open()
799 rdev->lldi.ucq_density); in c4iw_rdev_open()
800 return -EINVAL; in c4iw_rdev_open()
802 if (rdev->lldi.vr->qp.start != rdev->lldi.vr->cq.start || in c4iw_rdev_open()
803 rdev->lldi.vr->qp.size != rdev->lldi.vr->cq.size) { in c4iw_rdev_open()
805 pci_name(rdev->lldi.pdev), rdev->lldi.vr->qp.start, in c4iw_rdev_open()
806 rdev->lldi.vr->qp.size, rdev->lldi.vr->cq.size, in c4iw_rdev_open()
807 rdev->lldi.vr->cq.size); in c4iw_rdev_open()
808 return -EINVAL; in c4iw_rdev_open()
812 if (rdev->lldi.sge_host_page_size > PAGE_SIZE) { in c4iw_rdev_open()
814 pci_name(rdev->lldi.pdev), in c4iw_rdev_open()
815 rdev->lldi.sge_host_page_size); in c4iw_rdev_open()
816 return -EINVAL; in c4iw_rdev_open()
819 factor = PAGE_SIZE / rdev->lldi.sge_host_page_size; in c4iw_rdev_open()
820 rdev->qpmask = (rdev->lldi.udb_density * factor) - 1; in c4iw_rdev_open()
821 rdev->cqmask = (rdev->lldi.ucq_density * factor) - 1; in c4iw_rdev_open()
824 pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, in c4iw_rdev_open()
825 rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), in c4iw_rdev_open()
826 rdev->lldi.vr->pbl.start, in c4iw_rdev_open()
827 rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, in c4iw_rdev_open()
828 rdev->lldi.vr->rq.size, in c4iw_rdev_open()
829 rdev->lldi.vr->qp.start, in c4iw_rdev_open()
830 rdev->lldi.vr->qp.size, in c4iw_rdev_open()
831 rdev->lldi.vr->cq.start, in c4iw_rdev_open()
832 rdev->lldi.vr->cq.size, in c4iw_rdev_open()
833 rdev->lldi.vr->srq.size); in c4iw_rdev_open()
835 &rdev->lldi.pdev->resource[2], in c4iw_rdev_open()
836 rdev->lldi.db_reg, rdev->lldi.gts_reg, in c4iw_rdev_open()
837 rdev->qpmask, rdev->cqmask); in c4iw_rdev_open()
840 return -EINVAL; in c4iw_rdev_open()
842 rdev->stats.pd.total = T4_MAX_NUM_PD; in c4iw_rdev_open()
843 rdev->stats.stag.total = rdev->lldi.vr->stag.size; in c4iw_rdev_open()
844 rdev->stats.pbl.total = rdev->lldi.vr->pbl.size; in c4iw_rdev_open()
845 rdev->stats.rqt.total = rdev->lldi.vr->rq.size; in c4iw_rdev_open()
846 rdev->stats.srqt.total = rdev->lldi.vr->srq.size; in c4iw_rdev_open()
847 rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size; in c4iw_rdev_open()
848 rdev->stats.qid.total = rdev->lldi.vr->qp.size; in c4iw_rdev_open()
851 T4_MAX_NUM_PD, rdev->lldi.vr->srq.size); in c4iw_rdev_open()
871 rdev->status_page = (struct t4_dev_status_page *) in c4iw_rdev_open()
873 if (!rdev->status_page) { in c4iw_rdev_open()
874 err = -ENOMEM; in c4iw_rdev_open()
877 rdev->status_page->qp_start = rdev->lldi.vr->qp.start; in c4iw_rdev_open()
878 rdev->status_page->qp_size = rdev->lldi.vr->qp.size; in c4iw_rdev_open()
879 rdev->status_page->cq_start = rdev->lldi.vr->cq.start; in c4iw_rdev_open()
880 rdev->status_page->cq_size = rdev->lldi.vr->cq.size; in c4iw_rdev_open()
881 rdev->status_page->write_cmpl_supported = rdev->lldi.write_cmpl_support; in c4iw_rdev_open()
884 rdev->wr_log = kcalloc(1 << c4iw_wr_log_size_order, in c4iw_rdev_open()
885 sizeof(*rdev->wr_log), in c4iw_rdev_open()
887 if (rdev->wr_log) { in c4iw_rdev_open()
888 rdev->wr_log_size = 1 << c4iw_wr_log_size_order; in c4iw_rdev_open()
889 atomic_set(&rdev->wr_log_idx, 0); in c4iw_rdev_open()
893 rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free"); in c4iw_rdev_open()
894 if (!rdev->free_workq) { in c4iw_rdev_open()
895 err = -ENOMEM; in c4iw_rdev_open()
899 rdev->status_page->db_off = 0; in c4iw_rdev_open()
901 init_completion(&rdev->rqt_compl); in c4iw_rdev_open()
902 init_completion(&rdev->pbl_compl); in c4iw_rdev_open()
903 kref_init(&rdev->rqt_kref); in c4iw_rdev_open()
904 kref_init(&rdev->pbl_kref); in c4iw_rdev_open()
908 if (c4iw_wr_log && rdev->wr_log) in c4iw_rdev_open()
909 kfree(rdev->wr_log); in c4iw_rdev_open()
910 free_page((unsigned long)rdev->status_page); in c4iw_rdev_open()
918 c4iw_destroy_resource(&rdev->resource); in c4iw_rdev_open()
924 kfree(rdev->wr_log); in c4iw_rdev_close()
925 c4iw_release_dev_ucontext(rdev, &rdev->uctx); in c4iw_rdev_close()
926 free_page((unsigned long)rdev->status_page); in c4iw_rdev_close()
929 wait_for_completion(&rdev->pbl_compl); in c4iw_rdev_close()
930 wait_for_completion(&rdev->rqt_compl); in c4iw_rdev_close()
932 destroy_workqueue(rdev->free_workq); in c4iw_rdev_close()
933 c4iw_destroy_resource(&rdev->resource); in c4iw_rdev_close()
938 c4iw_rdev_close(&ctx->dev->rdev); in c4iw_dealloc()
939 WARN_ON(!xa_empty(&ctx->dev->cqs)); in c4iw_dealloc()
940 WARN_ON(!xa_empty(&ctx->dev->qps)); in c4iw_dealloc()
941 WARN_ON(!xa_empty(&ctx->dev->mrs)); in c4iw_dealloc()
942 wait_event(ctx->dev->wait, xa_empty(&ctx->dev->hwtids)); in c4iw_dealloc()
943 WARN_ON(!xa_empty(&ctx->dev->stids)); in c4iw_dealloc()
944 WARN_ON(!xa_empty(&ctx->dev->atids)); in c4iw_dealloc()
945 if (ctx->dev->rdev.bar2_kva) in c4iw_dealloc()
946 iounmap(ctx->dev->rdev.bar2_kva); in c4iw_dealloc()
947 if (ctx->dev->rdev.oc_mw_kva) in c4iw_dealloc()
948 iounmap(ctx->dev->rdev.oc_mw_kva); in c4iw_dealloc()
949 ib_dealloc_device(&ctx->dev->ibdev); in c4iw_dealloc()
950 ctx->dev = NULL; in c4iw_dealloc()
955 pr_debug("c4iw_dev %p\n", ctx->dev); in c4iw_remove()
956 debugfs_remove_recursive(ctx->dev->debugfs_root); in c4iw_remove()
957 c4iw_unregister_device(ctx->dev); in c4iw_remove()
963 return infop->vr->stag.size > 0 && infop->vr->pbl.size > 0 && in rdma_supported()
964 infop->vr->rq.size > 0 && infop->vr->qp.size > 0 && in rdma_supported()
965 infop->vr->cq.size > 0; in rdma_supported()
975 pci_name(infop->pdev)); in c4iw_alloc()
976 return ERR_PTR(-ENOSYS); in c4iw_alloc()
979 pr_info("%s: On-Chip Queues not supported on this device\n", in c4iw_alloc()
980 pci_name(infop->pdev)); in c4iw_alloc()
985 return ERR_PTR(-ENOMEM); in c4iw_alloc()
987 devp->rdev.lldi = *infop; in c4iw_alloc()
989 /* init various hw-queue params based on lld info */ in c4iw_alloc()
991 devp->rdev.lldi.sge_ingpadboundary, in c4iw_alloc()
992 devp->rdev.lldi.sge_egrstatuspagesize); in c4iw_alloc()
994 devp->rdev.hw_queue.t4_eq_status_entries = in c4iw_alloc()
995 devp->rdev.lldi.sge_egrstatuspagesize / 64; in c4iw_alloc()
996 devp->rdev.hw_queue.t4_max_eq_size = 65520; in c4iw_alloc()
997 devp->rdev.hw_queue.t4_max_iq_size = 65520; in c4iw_alloc()
998 devp->rdev.hw_queue.t4_max_rq_size = 8192 - in c4iw_alloc()
999 devp->rdev.hw_queue.t4_eq_status_entries - 1; in c4iw_alloc()
1000 devp->rdev.hw_queue.t4_max_sq_size = in c4iw_alloc()
1001 devp->rdev.hw_queue.t4_max_eq_size - in c4iw_alloc()
1002 devp->rdev.hw_queue.t4_eq_status_entries - 1; in c4iw_alloc()
1003 devp->rdev.hw_queue.t4_max_qp_depth = in c4iw_alloc()
1004 devp->rdev.hw_queue.t4_max_rq_size; in c4iw_alloc()
1005 devp->rdev.hw_queue.t4_max_cq_depth = in c4iw_alloc()
1006 devp->rdev.hw_queue.t4_max_iq_size - 2; in c4iw_alloc()
1007 devp->rdev.hw_queue.t4_stat_len = in c4iw_alloc()
1008 devp->rdev.lldi.sge_egrstatuspagesize; in c4iw_alloc()
1015 devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2); in c4iw_alloc()
1016 if (!is_t4(devp->rdev.lldi.adapter_type)) { in c4iw_alloc()
1017 devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa, in c4iw_alloc()
1018 pci_resource_len(devp->rdev.lldi.pdev, 2)); in c4iw_alloc()
1019 if (!devp->rdev.bar2_kva) { in c4iw_alloc()
1021 ib_dealloc_device(&devp->ibdev); in c4iw_alloc()
1022 return ERR_PTR(-EINVAL); in c4iw_alloc()
1025 devp->rdev.oc_mw_pa = in c4iw_alloc()
1026 pci_resource_start(devp->rdev.lldi.pdev, 2) + in c4iw_alloc()
1027 pci_resource_len(devp->rdev.lldi.pdev, 2) - in c4iw_alloc()
1028 roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size); in c4iw_alloc()
1029 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, in c4iw_alloc()
1030 devp->rdev.lldi.vr->ocq.size); in c4iw_alloc()
1031 if (!devp->rdev.oc_mw_kva) { in c4iw_alloc()
1033 ib_dealloc_device(&devp->ibdev); in c4iw_alloc()
1034 return ERR_PTR(-EINVAL); in c4iw_alloc()
1039 devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size, in c4iw_alloc()
1040 devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva); in c4iw_alloc()
1042 ret = c4iw_rdev_open(&devp->rdev); in c4iw_alloc()
1045 ib_dealloc_device(&devp->ibdev); in c4iw_alloc()
1049 xa_init_flags(&devp->cqs, XA_FLAGS_LOCK_IRQ); in c4iw_alloc()
1050 xa_init_flags(&devp->qps, XA_FLAGS_LOCK_IRQ); in c4iw_alloc()
1051 xa_init_flags(&devp->mrs, XA_FLAGS_LOCK_IRQ); in c4iw_alloc()
1052 xa_init_flags(&devp->hwtids, XA_FLAGS_LOCK_IRQ); in c4iw_alloc()
1053 xa_init_flags(&devp->atids, XA_FLAGS_LOCK_IRQ); in c4iw_alloc()
1054 xa_init_flags(&devp->stids, XA_FLAGS_LOCK_IRQ); in c4iw_alloc()
1055 mutex_init(&devp->rdev.stats.lock); in c4iw_alloc()
1056 mutex_init(&devp->db_mutex); in c4iw_alloc()
1057 INIT_LIST_HEAD(&devp->db_fc_list); in c4iw_alloc()
1058 init_waitqueue_head(&devp->wait); in c4iw_alloc()
1059 devp->avail_ird = devp->rdev.lldi.max_ird_adapter; in c4iw_alloc()
1062 devp->debugfs_root = debugfs_create_dir( in c4iw_alloc()
1063 pci_name(devp->rdev.lldi.pdev), in c4iw_alloc()
1079 pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", in c4iw_uld_add()
1084 ctx = ERR_PTR(-ENOMEM); in c4iw_uld_add()
1087 ctx->lldi = *infop; in c4iw_uld_add()
1090 pci_name(ctx->lldi.pdev), in c4iw_uld_add()
1091 ctx->lldi.nchan, ctx->lldi.nrxq, in c4iw_uld_add()
1092 ctx->lldi.ntxq, ctx->lldi.nports); in c4iw_uld_add()
1095 list_add_tail(&ctx->entry, &uld_ctx_list); in c4iw_uld_add()
1098 for (i = 0; i < ctx->lldi.nrxq; i++) in c4iw_uld_add()
1099 pr_debug("rxqid[%u] %u\n", i, ctx->lldi.rxq_ids[i]); in c4iw_uld_add()
1117 skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) + in copy_gl_to_skb_pkt()
1118 sizeof(struct rss_header) - pktshift, GFP_ATOMIC); in copy_gl_to_skb_pkt()
1122 __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) + in copy_gl_to_skb_pkt()
1123 sizeof(struct rss_header) - pktshift); in copy_gl_to_skb_pkt()
1137 gl->va + pktshift, in copy_gl_to_skb_pkt()
1138 gl->tot_len - pktshift); in copy_gl_to_skb_pkt()
1151 skb = copy_gl_to_skb_pkt(gl , rsp, dev->rdev.lldi.sge_pktshift); in recv_rx_pkt()
1170 struct c4iw_dev *dev = ctx->dev; in c4iw_uld_rx_handler()
1176 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; in c4iw_uld_rx_handler()
1186 u32 qid = be32_to_cpu(rc->pldbuflen_qid); in c4iw_uld_rx_handler()
1189 } else if (unlikely(*(u8 *)rsp != *(u8 *)gl->va)) { in c4iw_uld_rx_handler()
1194 pci_name(ctx->lldi.pdev), gl->va, in c4iw_uld_rx_handler()
1196 be64_to_cpu(*(__force __be64 *)gl->va), in c4iw_uld_rx_handler()
1197 gl->tot_len); in c4iw_uld_rx_handler()
1216 return -1; in c4iw_uld_rx_handler()
1226 pr_info("%s: Up\n", pci_name(ctx->lldi.pdev)); in c4iw_uld_state_change()
1227 if (!ctx->dev) { in c4iw_uld_state_change()
1228 ctx->dev = c4iw_alloc(&ctx->lldi); in c4iw_uld_state_change()
1229 if (IS_ERR(ctx->dev)) { in c4iw_uld_state_change()
1231 pci_name(ctx->lldi.pdev), in c4iw_uld_state_change()
1232 PTR_ERR(ctx->dev)); in c4iw_uld_state_change()
1233 ctx->dev = NULL; in c4iw_uld_state_change()
1237 INIT_WORK(&ctx->reg_work, c4iw_register_device); in c4iw_uld_state_change()
1238 queue_work(reg_workq, &ctx->reg_work); in c4iw_uld_state_change()
1242 pr_info("%s: Down\n", pci_name(ctx->lldi.pdev)); in c4iw_uld_state_change()
1243 if (ctx->dev) in c4iw_uld_state_change()
1248 pr_info("%s: Fatal Error\n", pci_name(ctx->lldi.pdev)); in c4iw_uld_state_change()
1249 if (ctx->dev) { in c4iw_uld_state_change()
1252 ctx->dev->rdev.flags |= T4_FATAL_ERROR; in c4iw_uld_state_change()
1254 event.device = &ctx->dev->ibdev; in c4iw_uld_state_change()
1260 pr_info("%s: Detach\n", pci_name(ctx->lldi.pdev)); in c4iw_uld_state_change()
1261 if (ctx->dev) in c4iw_uld_state_change()
1273 xa_lock_irqsave(&ctx->dev->qps, flags); in stop_queues()
1274 ctx->dev->rdev.stats.db_state_transitions++; in stop_queues()
1275 ctx->dev->db_state = STOPPED; in stop_queues()
1276 if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) { in stop_queues()
1277 xa_for_each(&ctx->dev->qps, index, qp) in stop_queues()
1278 t4_disable_wq_db(&qp->wq); in stop_queues()
1280 ctx->dev->rdev.status_page->db_off = 1; in stop_queues()
1282 xa_unlock_irqrestore(&ctx->dev->qps, flags); in stop_queues()
1287 spin_lock(&qp->lock); in resume_rc_qp()
1288 t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc, NULL); in resume_rc_qp()
1289 qp->wq.sq.wq_pidx_inc = 0; in resume_rc_qp()
1290 t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc, NULL); in resume_rc_qp()
1291 qp->wq.rq.wq_pidx_inc = 0; in resume_rc_qp()
1292 spin_unlock(&qp->lock); in resume_rc_qp()
1301 qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp, in resume_a_chunk()
1303 list_del_init(&qp->db_fc_entry); in resume_a_chunk()
1305 if (list_empty(&ctx->dev->db_fc_list)) in resume_a_chunk()
1312 xa_lock_irq(&ctx->dev->qps); in resume_queues()
1313 if (ctx->dev->db_state != STOPPED) in resume_queues()
1315 ctx->dev->db_state = FLOW_CONTROL; in resume_queues()
1317 if (list_empty(&ctx->dev->db_fc_list)) { in resume_queues()
1321 WARN_ON(ctx->dev->db_state != FLOW_CONTROL); in resume_queues()
1322 ctx->dev->db_state = NORMAL; in resume_queues()
1323 ctx->dev->rdev.stats.db_state_transitions++; in resume_queues()
1324 if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) { in resume_queues()
1325 xa_for_each(&ctx->dev->qps, index, qp) in resume_queues()
1326 t4_enable_wq_db(&qp->wq); in resume_queues()
1328 ctx->dev->rdev.status_page->db_off = 0; in resume_queues()
1332 if (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) in resume_queues()
1333 < (ctx->dev->rdev.lldi.dbfifo_int_thresh << in resume_queues()
1337 if (!list_empty(&ctx->dev->db_fc_list)) { in resume_queues()
1338 xa_unlock_irq(&ctx->dev->qps); in resume_queues()
1343 xa_lock_irq(&ctx->dev->qps); in resume_queues()
1344 if (ctx->dev->db_state != FLOW_CONTROL) in resume_queues()
1350 if (ctx->dev->db_state != NORMAL) in resume_queues()
1351 ctx->dev->rdev.stats.db_fc_interruptions++; in resume_queues()
1352 xa_unlock_irq(&ctx->dev->qps); in resume_queues()
1364 for (idx = 0; idx < qp_list->idx; idx++) in deref_qps()
1365 c4iw_qp_rem_ref(&qp_list->qps[idx]->ibqp); in deref_qps()
1373 for (idx = 0; idx < qp_list->idx; idx++) { in recover_lost_dbs()
1374 struct c4iw_qp *qp = qp_list->qps[idx]; in recover_lost_dbs()
1376 xa_lock_irq(&qp->rhp->qps); in recover_lost_dbs()
1377 spin_lock(&qp->lock); in recover_lost_dbs()
1378 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], in recover_lost_dbs()
1379 qp->wq.sq.qid, in recover_lost_dbs()
1380 t4_sq_host_wq_pidx(&qp->wq), in recover_lost_dbs()
1381 t4_sq_wq_size(&qp->wq)); in recover_lost_dbs()
1383 pr_err("%s: Fatal error - DB overflow recovery failed - error syncing SQ qid %u\n", in recover_lost_dbs()
1384 pci_name(ctx->lldi.pdev), qp->wq.sq.qid); in recover_lost_dbs()
1385 spin_unlock(&qp->lock); in recover_lost_dbs()
1386 xa_unlock_irq(&qp->rhp->qps); in recover_lost_dbs()
1389 qp->wq.sq.wq_pidx_inc = 0; in recover_lost_dbs()
1391 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], in recover_lost_dbs()
1392 qp->wq.rq.qid, in recover_lost_dbs()
1393 t4_rq_host_wq_pidx(&qp->wq), in recover_lost_dbs()
1394 t4_rq_wq_size(&qp->wq)); in recover_lost_dbs()
1397 pr_err("%s: Fatal error - DB overflow recovery failed - error syncing RQ qid %u\n", in recover_lost_dbs()
1398 pci_name(ctx->lldi.pdev), qp->wq.rq.qid); in recover_lost_dbs()
1399 spin_unlock(&qp->lock); in recover_lost_dbs()
1400 xa_unlock_irq(&qp->rhp->qps); in recover_lost_dbs()
1403 qp->wq.rq.wq_pidx_inc = 0; in recover_lost_dbs()
1404 spin_unlock(&qp->lock); in recover_lost_dbs()
1405 xa_unlock_irq(&qp->rhp->qps); in recover_lost_dbs()
1408 while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) { in recover_lost_dbs()
1428 ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]); in recover_queues()
1430 pr_err("%s: Fatal error - DB overflow recovery failed\n", in recover_queues()
1431 pci_name(ctx->lldi.pdev)); in recover_queues()
1436 xa_lock_irq(&ctx->dev->qps); in recover_queues()
1437 WARN_ON(ctx->dev->db_state != STOPPED); in recover_queues()
1438 ctx->dev->db_state = RECOVERY; in recover_queues()
1439 xa_for_each(&ctx->dev->qps, index, qp) in recover_queues()
1444 xa_unlock_irq(&ctx->dev->qps); in recover_queues()
1450 xa_for_each(&ctx->dev->qps, index, qp) { in recover_queues()
1451 c4iw_qp_add_ref(&qp->ibqp); in recover_queues()
1455 xa_unlock_irq(&ctx->dev->qps); in recover_queues()
1464 xa_lock_irq(&ctx->dev->qps); in recover_queues()
1465 WARN_ON(ctx->dev->db_state != RECOVERY); in recover_queues()
1466 ctx->dev->db_state = STOPPED; in recover_queues()
1467 xa_unlock_irq(&ctx->dev->qps); in recover_queues()
1477 ctx->dev->rdev.stats.db_full++; in c4iw_uld_control()
1481 mutex_lock(&ctx->dev->rdev.stats.lock); in c4iw_uld_control()
1482 ctx->dev->rdev.stats.db_empty++; in c4iw_uld_control()
1483 mutex_unlock(&ctx->dev->rdev.stats.lock); in c4iw_uld_control()
1487 mutex_lock(&ctx->dev->rdev.stats.lock); in c4iw_uld_control()
1488 ctx->dev->rdev.stats.db_drop++; in c4iw_uld_control()
1489 mutex_unlock(&ctx->dev->rdev.stats.lock); in c4iw_uld_control()
1493 pci_name(ctx->lldi.pdev), control); in c4iw_uld_control()
1527 kref_init(&wr_waitp->kref); in c4iw_alloc_wr_wait()
1546 return -ENOMEM; in c4iw_init_module()
1560 if (ctx->dev) in c4iw_exit_module()