2  * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
14  *      - Redistributions of source code must retain the above
17  *      - Redistributions in binary form must reproduce the above
25  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
43 #define T4_PAGESIZE_MASK 0xffff000  /* 4KB-128MB */
49 	__be32 rsvd1;	/* flit 0 - hw owns */
54 	u8 qp_err;	/* flit 1 - sw owns */
70 #define T4_MAX_SEND_SGE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \
72 #define T4_MAX_SEND_INLINE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \
74 #define T4_MAX_WRITE_INLINE ((T4_SQ_NUM_BYTES - \
75 			sizeof(struct fw_ri_rdma_write_wr) - \
77 #define T4_MAX_WRITE_SGE ((T4_SQ_NUM_BYTES - \
78 			sizeof(struct fw_ri_rdma_write_wr) - \
80 #define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \
122 	wqe->send.opcode = (u8)opcode;  in init_wr_hdr()
123 	wqe->send.flags = flags;  in init_wr_hdr()
124 	wqe->send.wrid = wrid;  in init_wr_hdr()
125 	wqe->send.r1[0] = 0;  in init_wr_hdr()
126 	wqe->send.r1[1] = 0;  in init_wr_hdr()
127 	wqe->send.r1[2] = 0;  in init_wr_hdr()
128 	wqe->send.len16 = len16;  in init_wr_hdr()
143 #define T4_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8	/* attempt to invalidate a  */
149 #define T4_ERR_PBL_ADDR_BOUND              0xB	/* pbl addr out of bounds:  */
252 #define SW_CQE(x)         (CQE_SWCQE_G(be32_to_cpu((x)->header)))
253 #define DRAIN_CQE(x)      (CQE_DRAIN_G(be32_to_cpu((x)->header)))
254 #define CQE_QPID(x)       (CQE_QPID_G(be32_to_cpu((x)->header)))
255 #define CQE_TYPE(x)       (CQE_TYPE_G(be32_to_cpu((x)->header)))
258 #define CQE_STATUS(x)     (CQE_STATUS_G(be32_to_cpu((x)->header)))
259 #define CQE_OPCODE(x)     (CQE_OPCODE_G(be32_to_cpu((x)->header)))
262 	(CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND) || \
263 	(CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE) || \
264 	(CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_INV) || \
265 	(CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE_INV))
267 #define CQE_LEN(x)        (be32_to_cpu((x)->len))
270 #define CQE_WRID_STAG(x)  (be32_to_cpu((x)->u.rcqe.stag))
271 #define CQE_WRID_MSN(x)   (be32_to_cpu((x)->u.rcqe.msn))
272 #define CQE_ABS_RQE_IDX(x) (be32_to_cpu((x)->u.srcqe.abs_rqe_idx))
274 	(x)->u.imm_data_rcqe.iw_imm_data.ib_imm_data.imm_data32)
277 #define CQE_WRID_SQ_IDX(x)	((x)->u.scqe.cidx)
278 #define CQE_WRID_FR_STAG(x)     (be32_to_cpu((x)->u.scqe.stag))
281 #define CQE_WRID_HI(x)		(be32_to_cpu((x)->u.gen.wrid_hi))
282 #define CQE_WRID_LOW(x)		(be32_to_cpu((x)->u.gen.wrid_low))
283 #define CQE_DRAIN_COOKIE(x)	((x)->u.drain_cookie)
302 #define CQE_OVFBIT(x)	((unsigned)CQE_OVFBIT_G(be64_to_cpu((x)->bits_type_ts)))
303 #define CQE_GENBIT(x)	((unsigned)CQE_GENBIT_G(be64_to_cpu((x)->bits_type_ts)))
304 #define CQE_TS(x)	(CQE_TS_G(be64_to_cpu((x)->bits_type_ts)))
427 	return srq->size - 1 - srq->in_use;  in t4_srq_avail()
432 	srq->in_use++;  in t4_srq_produce()
433 	if (++srq->pidx == srq->size)  in t4_srq_produce()
434 		srq->pidx = 0;  in t4_srq_produce()
435 	srq->wq_pidx += DIV_ROUND_UP(len16 * 16, T4_EQ_ENTRY_SIZE);  in t4_srq_produce()
436 	if (srq->wq_pidx >= srq->size * T4_RQ_NUM_SLOTS)  in t4_srq_produce()
437 		srq->wq_pidx %= srq->size * T4_RQ_NUM_SLOTS;  in t4_srq_produce()
438 	srq->queue[srq->size].status.host_pidx = srq->pidx;  in t4_srq_produce()
443 	srq->pending_in_use++;  in t4_srq_produce_pending_wr()
444 	srq->in_use++;  in t4_srq_produce_pending_wr()
445 	if (++srq->pending_pidx == srq->size)  in t4_srq_produce_pending_wr()
446 		srq->pending_pidx = 0;  in t4_srq_produce_pending_wr()
451 	srq->pending_in_use--;  in t4_srq_consume_pending_wr()
452 	srq->in_use--;  in t4_srq_consume_pending_wr()
453 	if (++srq->pending_cidx == srq->size)  in t4_srq_consume_pending_wr()
454 		srq->pending_cidx = 0;  in t4_srq_consume_pending_wr()
459 	srq->in_use--;  in t4_srq_produce_ooo()
460 	srq->ooo_count++;  in t4_srq_produce_ooo()
465 	srq->cidx++;  in t4_srq_consume_ooo()
466 	if (srq->cidx == srq->size)  in t4_srq_consume_ooo()
467 		srq->cidx  = 0;  in t4_srq_consume_ooo()
468 	srq->queue[srq->size].status.host_cidx = srq->cidx;  in t4_srq_consume_ooo()
469 	srq->ooo_count--;  in t4_srq_consume_ooo()
474 	srq->in_use--;  in t4_srq_consume()
475 	if (++srq->cidx == srq->size)  in t4_srq_consume()
476 		srq->cidx = 0;  in t4_srq_consume()
477 	srq->queue[srq->size].status.host_cidx = srq->cidx;  in t4_srq_consume()
482 	return wq->rq.in_use;  in t4_rqes_posted()
487 	return wq->rq.in_use == 0;  in t4_rq_empty()
492 	return wq->rq.size - 1 - wq->rq.in_use;  in t4_rq_avail()
497 	wq->rq.in_use++;  in t4_rq_produce()
498 	if (++wq->rq.pidx == wq->rq.size)  in t4_rq_produce()
499 		wq->rq.pidx = 0;  in t4_rq_produce()
500 	wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_rq_produce()
501 	if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS)  in t4_rq_produce()
502 		wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_produce()
507 	wq->rq.in_use--;  in t4_rq_consume()
508 	if (++wq->rq.cidx == wq->rq.size)  in t4_rq_consume()
509 		wq->rq.cidx = 0;  in t4_rq_consume()
514 	return wq->rq.queue[wq->rq.size].status.host_wq_pidx;  in t4_rq_host_wq_pidx()
519 		return wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_wq_size()
524 	return sq->flags & T4_SQ_ONCHIP;  in t4_sq_onchip()
529 	return wq->sq.in_use == 0;  in t4_sq_empty()
534 	return wq->sq.size - 1 - wq->sq.in_use;  in t4_sq_avail()
539 	wq->sq.in_use++;  in t4_sq_produce()
540 	if (++wq->sq.pidx == wq->sq.size)  in t4_sq_produce()
541 		wq->sq.pidx = 0;  in t4_sq_produce()
542 	wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_sq_produce()
543 	if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS)  in t4_sq_produce()
544 		wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_produce()
549 	if (wq->sq.cidx == wq->sq.flush_cidx)  in t4_sq_consume()
550 		wq->sq.flush_cidx = -1;  in t4_sq_consume()
551 	wq->sq.in_use--;  in t4_sq_consume()
552 	if (++wq->sq.cidx == wq->sq.size)  in t4_sq_consume()
553 		wq->sq.cidx = 0;  in t4_sq_consume()
558 	return wq->sq.queue[wq->sq.size].status.host_wq_pidx;  in t4_sq_host_wq_pidx()
563 		return wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_wq_size()
578 		count--;  in pio_copy()
587 	if (inc == 1 && srq->bar2_qid == 0 && wqe) {  in t4_ring_srq_db()
588 		pr_debug("%s : WC srq->pidx = %d; len16=%d\n",  in t4_ring_srq_db()
589 			 __func__, srq->pidx, len16);  in t4_ring_srq_db()
590 		pio_copy(srq->bar2_va + SGE_UDB_WCDOORBELL, (u64 *)wqe);  in t4_ring_srq_db()
592 		pr_debug("%s: DB srq->pidx = %d; len16=%d\n",  in t4_ring_srq_db()
593 			 __func__, srq->pidx, len16);  in t4_ring_srq_db()
594 		writel(PIDX_T5_V(inc) | QID_V(srq->bar2_qid),  in t4_ring_srq_db()
595 		       srq->bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_srq_db()
606 	if (wq->sq.bar2_va) {  in t4_ring_sq_db()
607 		if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) {  in t4_ring_sq_db()
608 			pr_debug("WC wq->sq.pidx = %d\n", wq->sq.pidx);  in t4_ring_sq_db()
610 				 (wq->sq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_sq_db()
613 			pr_debug("DB wq->sq.pidx = %d\n", wq->sq.pidx);  in t4_ring_sq_db()
614 			writel(PIDX_T5_V(inc) | QID_V(wq->sq.bar2_qid),  in t4_ring_sq_db()
615 			       wq->sq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_sq_db()
622 	writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db);  in t4_ring_sq_db()
631 	if (wq->rq.bar2_va) {  in t4_ring_rq_db()
632 		if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) {  in t4_ring_rq_db()
633 			pr_debug("WC wq->rq.pidx = %d\n", wq->rq.pidx);  in t4_ring_rq_db()
635 				 (wq->rq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_rq_db()
638 			pr_debug("DB wq->rq.pidx = %d\n", wq->rq.pidx);  in t4_ring_rq_db()
639 			writel(PIDX_T5_V(inc) | QID_V(wq->rq.bar2_qid),  in t4_ring_rq_db()
640 			       wq->rq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_rq_db()
647 	writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db);  in t4_ring_rq_db()
652 	return *wq->qp_errp;  in t4_wq_in_error()
658 		*wq->srqidxp = srqidx;  in t4_set_wq_in_error()
659 	*wq->qp_errp = 1;  in t4_set_wq_in_error()
664 	wq->rq.queue[wq->rq.size].status.db_off = 1;  in t4_disable_wq_db()
669 	wq->rq.queue[wq->rq.size].status.db_off = 0;  in t4_enable_wq_db()
705 	if (cq->bar2_va)  in write_gts()
706 		writel(val | INGRESSQID_V(cq->bar2_qid),  in write_gts()
707 		       cq->bar2_va + SGE_UDB_GTS);  in write_gts()
709 		writel(val | INGRESSQID_V(cq->cqid), cq->gts);  in write_gts()
714 	return test_and_clear_bit(CQ_ARMED, &cq->flags);  in t4_clear_cq_armed()
721 	set_bit(CQ_ARMED, &cq->flags);  in t4_arm_cq()
722 	while (cq->cidx_inc > CIDXINC_M) {  in t4_arm_cq()
725 		cq->cidx_inc -= CIDXINC_M;  in t4_arm_cq()
727 	val = SEINTARM_V(se) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(6);  in t4_arm_cq()
729 	cq->cidx_inc = 0;  in t4_arm_cq()
735 	cq->sw_in_use++;  in t4_swcq_produce()
736 	if (cq->sw_in_use == cq->size) {  in t4_swcq_produce()
738 			__func__, cq->cqid);  in t4_swcq_produce()
739 		cq->error = 1;  in t4_swcq_produce()
740 		cq->sw_in_use--;  in t4_swcq_produce()
743 	if (++cq->sw_pidx == cq->size)  in t4_swcq_produce()
744 		cq->sw_pidx = 0;  in t4_swcq_produce()
749 	cq->sw_in_use--;  in t4_swcq_consume()
750 	if (++cq->sw_cidx == cq->size)  in t4_swcq_consume()
751 		cq->sw_cidx = 0;  in t4_swcq_consume()
756 	cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts;  in t4_hwcq_consume()
757 	if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == CIDXINC_M) {  in t4_hwcq_consume()
760 		val = SEINTARM_V(0) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(7);  in t4_hwcq_consume()
762 		cq->cidx_inc = 0;  in t4_hwcq_consume()
764 	if (++cq->cidx == cq->size) {  in t4_hwcq_consume()
765 		cq->cidx = 0;  in t4_hwcq_consume()
766 		cq->gen ^= 1;  in t4_hwcq_consume()
772 	return (CQE_GENBIT(cqe) == cq->gen);  in t4_valid_cqe()
777 	return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]);  in t4_cq_notempty()
785 	if (cq->cidx == 0)  in t4_next_hw_cqe()
786 		prev_cidx = cq->size - 1;  in t4_next_hw_cqe()
788 		prev_cidx = cq->cidx - 1;  in t4_next_hw_cqe()
790 	if (cq->queue[prev_cidx].bits_type_ts != cq->bits_type_ts) {  in t4_next_hw_cqe()
791 		ret = -EOVERFLOW;  in t4_next_hw_cqe()
792 		cq->error = 1;  in t4_next_hw_cqe()
793 		pr_err("cq overflow cqid %u\n", cq->cqid);  in t4_next_hw_cqe()
794 	} else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {  in t4_next_hw_cqe()
798 		*cqe = &cq->queue[cq->cidx];  in t4_next_hw_cqe()
801 		ret = -ENODATA;  in t4_next_hw_cqe()
809 	if (cq->error)  in t4_next_cqe()
810 		ret = -ENODATA;  in t4_next_cqe()
811 	else if (cq->sw_in_use)  in t4_next_cqe()
812 		*cqe = &cq->sw_queue[cq->sw_cidx];  in t4_next_cqe()
820 	*cq->qp_errp = 1;  in t4_set_cq_in_error()