1 // SPDX-License-Identifier: GPL-2.0-only
18 #include <linux/dma-mapping.h>
46 	(HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \
50  * hinic_rxq_clean_stats - Clean the statistics of specific queue
51  * @rxq: Logical Rx Queue
55 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in hinic_rxq_clean_stats()
57 	u64_stats_update_begin(&rxq_stats->syncp);  in hinic_rxq_clean_stats()
58 	rxq_stats->pkts  = 0;  in hinic_rxq_clean_stats()
59 	rxq_stats->bytes = 0;  in hinic_rxq_clean_stats()
60 	rxq_stats->errors = 0;  in hinic_rxq_clean_stats()
61 	rxq_stats->csum_errors = 0;  in hinic_rxq_clean_stats()
62 	rxq_stats->other_errors = 0;  in hinic_rxq_clean_stats()
63 	u64_stats_update_end(&rxq_stats->syncp);  in hinic_rxq_clean_stats()
67  * hinic_rxq_get_stats - get statistics of Rx Queue
68  * @rxq: Logical Rx Queue
73 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in hinic_rxq_get_stats()
77 		start = u64_stats_fetch_begin(&rxq_stats->syncp);  in hinic_rxq_get_stats()
78 		stats->pkts = rxq_stats->pkts;  in hinic_rxq_get_stats()
79 		stats->bytes = rxq_stats->bytes;  in hinic_rxq_get_stats()
80 		stats->errors = rxq_stats->csum_errors +  in hinic_rxq_get_stats()
81 				rxq_stats->other_errors;  in hinic_rxq_get_stats()
82 		stats->csum_errors = rxq_stats->csum_errors;  in hinic_rxq_get_stats()
83 		stats->other_errors = rxq_stats->other_errors;  in hinic_rxq_get_stats()
84 	} while (u64_stats_fetch_retry(&rxq_stats->syncp, start));  in hinic_rxq_get_stats()
88  * rxq_stats_init - Initialize the statistics of specific queue
89  * @rxq: Logical Rx Queue
93 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;  in rxq_stats_init()
95 	u64_stats_init(&rxq_stats->syncp);  in rxq_stats_init()
102 	struct net_device *netdev = rxq->netdev;  in rx_csum()
107 	if (!(netdev->features & NETIF_F_RXCSUM))  in rx_csum()
111 		skb->ip_summed = CHECKSUM_UNNECESSARY;  in rx_csum()
115 			rxq->rxq_stats.csum_errors++;  in rx_csum()
116 		skb->ip_summed = CHECKSUM_NONE;  in rx_csum()
121  * rx_alloc_skb - allocate skb and map it to dma address
122  * @rxq: rx queue
130 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_alloc_skb()
131 	struct hinic_hwdev *hwdev = nic_dev->hwdev;  in rx_alloc_skb()
132 	struct hinic_hwif *hwif = hwdev->hwif;  in rx_alloc_skb()
133 	struct pci_dev *pdev = hwif->pdev;  in rx_alloc_skb()
138 	skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);  in rx_alloc_skb()
142 	addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,  in rx_alloc_skb()
144 	err = dma_mapping_error(&pdev->dev, addr);  in rx_alloc_skb()
146 		dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);  in rx_alloc_skb()
159  * rx_unmap_skb - unmap the dma address of the skb
160  * @rxq: rx queue
165 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_unmap_skb()
166 	struct hinic_hwdev *hwdev = nic_dev->hwdev;  in rx_unmap_skb()
167 	struct hinic_hwif *hwif = hwdev->hwif;  in rx_unmap_skb()
168 	struct pci_dev *pdev = hwif->pdev;  in rx_unmap_skb()
170 	dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,  in rx_unmap_skb()
175  * rx_free_skb - unmap and free skb
176  * @rxq: rx queue
188  * rx_alloc_pkts - allocate pkts in rx queue
189  * @rxq: rx queue
195 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_alloc_pkts()
204 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);  in rx_alloc_pkts()
207 	if (free_wqebbs > nic_dev->rx_weight)  in rx_alloc_pkts()
208 		free_wqebbs = nic_dev->rx_weight;  in rx_alloc_pkts()
215 		hinic_set_sge(&sge, dma_addr, skb->len);  in rx_alloc_pkts()
217 		rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,  in rx_alloc_pkts()
224 		hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);  in rx_alloc_pkts()
226 		hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);  in rx_alloc_pkts()
233 		hinic_rq_update(rxq->rq, prod_idx);  in rx_alloc_pkts()
240  * free_all_rx_skbs - free all skbs in rx queue
241  * @rxq: rx queue
245 	struct hinic_rq *rq = rxq->rq;  in free_all_rx_skbs()
250 	while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {  in free_all_rx_skbs()
254 		hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);  in free_all_rx_skbs()
256 		hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);  in free_all_rx_skbs()
258 		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));  in free_all_rx_skbs()
263  * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
264  * @rxq: rx queue
281 		rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,  in rx_recv_jumbo_pkt()
286 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);  in rx_recv_jumbo_pkt()
290 		prefetch(skb->data);  in rx_recv_jumbo_pkt()
295 		left_pkt_len -= curr_len;  in rx_recv_jumbo_pkt()
300 			skb_shinfo(head_skb)->frag_list = skb;  in rx_recv_jumbo_pkt()
302 			curr_skb->next = skb;  in rx_recv_jumbo_pkt()
304 		head_skb->len += skb->len;  in rx_recv_jumbo_pkt()
305 		head_skb->data_len += skb->len;  in rx_recv_jumbo_pkt()
306 		head_skb->truesize += skb->truesize;  in rx_recv_jumbo_pkt()
317 	struct net_device *netdev = nic_dev->netdev;  in hinic_copy_lp_data()
318 	u8 *lb_buf = nic_dev->lb_test_rx_buf;  in hinic_copy_lp_data()
319 	int lb_len = nic_dev->lb_pkt_len;  in hinic_copy_lp_data()
323 	if (nic_dev->lb_test_rx_idx == LP_PKT_CNT) {  in hinic_copy_lp_data()
324 		nic_dev->lb_test_rx_idx = 0;  in hinic_copy_lp_data()
328 	if (skb->len != nic_dev->lb_pkt_len) {  in hinic_copy_lp_data()
330 		nic_dev->lb_test_rx_idx++;  in hinic_copy_lp_data()
334 	pkt_offset = nic_dev->lb_test_rx_idx * lb_len;  in hinic_copy_lp_data()
336 	memcpy(lb_buf + pkt_offset, skb->data, frag_len);  in hinic_copy_lp_data()
338 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {  in hinic_copy_lp_data()
339 		frag_data = skb_frag_address(&skb_shinfo(skb)->frags[i]);  in hinic_copy_lp_data()
340 		frag_len = (int)skb_frag_size(&skb_shinfo(skb)->frags[i]);  in hinic_copy_lp_data()
344 	nic_dev->lb_test_rx_idx++;  in hinic_copy_lp_data()
348  * rxq_recv - Rx handler
349  * @rxq: rx queue
356 	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);  in rxq_recv()
357 	struct net_device *netdev = rxq->netdev;  in rxq_recv()
359 	struct hinic_rq *rq = rxq->rq;  in rxq_recv()
379 		rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,  in rxq_recv()
387 		cqe = rq->cqe[ci];  in rxq_recv()
388 		status =  be32_to_cpu(cqe->status);  in rxq_recv()
389 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);  in rxq_recv()
395 		prefetch(skb->data);  in rxq_recv()
403 			num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -  in rxq_recv()
410 		offload_type = be32_to_cpu(cqe->offload_type);  in rxq_recv()
411 		vlan_len = be32_to_cpu(cqe->len);  in rxq_recv()
412 		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&  in rxq_recv()
418 		if (unlikely(nic_dev->flags & HINIC_LP_TEST))  in rxq_recv()
421 		skb_record_rx_queue(skb, qp->q_id);  in rxq_recv()
422 		skb->protocol = eth_type_trans(skb, rxq->netdev);  in rxq_recv()
424 		napi_gro_receive(&rxq->napi, skb);  in rxq_recv()
431 			rx_bytes += ((num_lro - 1) *  in rxq_recv()
435 			(u16)(pkt_len >> rxq->rx_buff_shift) +  in rxq_recv()
436 			((pkt_len & (rxq->buf_len - 1)) ? 1 : 0);  in rxq_recv()
439 		cqe->status = 0;  in rxq_recv()
445 	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);  in rxq_recv()
449 	u64_stats_update_begin(&rxq->rxq_stats.syncp);  in rxq_recv()
450 	rxq->rxq_stats.pkts += pkts;  in rxq_recv()
451 	rxq->rxq_stats.bytes += rx_bytes;  in rxq_recv()
452 	u64_stats_update_end(&rxq->rxq_stats.syncp);  in rxq_recv()
460 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_poll()
461 	struct hinic_rq *rq = rxq->rq;  in rx_poll()
470 	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))  in rx_poll()
471 		hinic_hwdev_set_msix_state(nic_dev->hwdev,  in rx_poll()
472 					   rq->msix_entry,  in rx_poll()
480 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_add_napi()
482 	netif_napi_add_weight(rxq->netdev, &rxq->napi, rx_poll,  in rx_add_napi()
483 			      nic_dev->rx_weight);  in rx_add_napi()
484 	napi_enable(&rxq->napi);  in rx_add_napi()
489 	napi_disable(&rxq->napi);  in rx_del_napi()
490 	netif_napi_del(&rxq->napi);  in rx_del_napi()
496 	struct hinic_rq *rq = rxq->rq;  in rx_irq()
500 	nic_dev = netdev_priv(rxq->netdev);  in rx_irq()
501 	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))  in rx_irq()
502 		hinic_hwdev_set_msix_state(nic_dev->hwdev,  in rx_irq()
503 					   rq->msix_entry,  in rx_irq()
506 	nic_dev = netdev_priv(rxq->netdev);  in rx_irq()
507 	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);  in rx_irq()
509 	napi_schedule(&rxq->napi);  in rx_irq()
515 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);  in rx_request_irq()
518 	struct hinic_hwdev *hwdev = nic_dev->hwdev;  in rx_request_irq()
519 	struct hinic_rq *rq = rxq->rq;  in rx_request_irq()
527 	hinic_hwdev_msix_set(hwdev, rq->msix_entry,  in rx_request_irq()
532 	intr_coal = &nic_dev->rx_intr_coalesce[qp->q_id];  in rx_request_irq()
533 	interrupt_info.msix_index = rq->msix_entry;  in rx_request_irq()
534 	interrupt_info.coalesce_timer_cnt = intr_coal->coalesce_timer_cfg;  in rx_request_irq()
535 	interrupt_info.pending_cnt = intr_coal->pending_limt;  in rx_request_irq()
536 	interrupt_info.resend_timer_cnt = intr_coal->resend_timer_cfg;  in rx_request_irq()
540 		netif_err(nic_dev, drv, rxq->netdev,  in rx_request_irq()
541 			  "Failed to set RX interrupt coalescing attribute\n");  in rx_request_irq()
545 	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);  in rx_request_irq()
549 	cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask);  in rx_request_irq()
550 	err = irq_set_affinity_and_hint(rq->irq, &rq->affinity_mask);  in rx_request_irq()
557 	free_irq(rq->irq, rxq);  in rx_request_irq()
565 	struct hinic_rq *rq = rxq->rq;  in rx_free_irq()
567 	irq_update_affinity_hint(rq->irq, NULL);  in rx_free_irq()
568 	free_irq(rq->irq, rxq);  in rx_free_irq()
573  * hinic_init_rxq - Initialize the Rx Queue
574  * @rxq: Logical Rx Queue
575  * @rq: Hardware Rx Queue to connect the Logical queue with
576  * @netdev: network device to connect the Logical queue with
578  * Return 0 - Success, negative - Failure
586 	rxq->netdev = netdev;  in hinic_init_rxq()
587 	rxq->rq = rq;  in hinic_init_rxq()
588 	rxq->buf_len = HINIC_RX_BUF_SZ;  in hinic_init_rxq()
589 	rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ);  in hinic_init_rxq()
593 	rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL,  in hinic_init_rxq()
594 				       "%s_rxq%d", netdev->name, qp->q_id);  in hinic_init_rxq()
595 	if (!rxq->irq_name)  in hinic_init_rxq()
596 		return -ENOMEM;  in hinic_init_rxq()
600 		err = -ENOMEM;  in hinic_init_rxq()
606 		netdev_err(netdev, "Failed to request Rx irq\n");  in hinic_init_rxq()
615 	devm_kfree(&netdev->dev, rxq->irq_name);  in hinic_init_rxq()
620  * hinic_clean_rxq - Clean the Rx Queue
621  * @rxq: Logical Rx Queue
625 	struct net_device *netdev = rxq->netdev;  in hinic_clean_rxq()
630 	devm_kfree(&netdev->dev, rxq->irq_name);  in hinic_clean_rxq()