Lines Matching refs:rxr
376 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) in bnxt_sched_reset_rxr() argument
378 if (!rxr->bnapi->in_reset) { in bnxt_sched_reset_rxr()
379 rxr->bnapi->in_reset = true; in bnxt_sched_reset_rxr()
386 rxr->rx_next_cons = 0xffff; in bnxt_sched_reset_rxr()
868 struct bnxt_rx_ring_info *rxr, in __bnxt_alloc_rx_page() argument
875 page = page_pool_dev_alloc_frag(rxr->page_pool, offset, in __bnxt_alloc_rx_page()
878 page = page_pool_dev_alloc_pages(rxr->page_pool); in __bnxt_alloc_rx_page()
912 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, in bnxt_alloc_rx_data() argument
915 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; in bnxt_alloc_rx_data()
916 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; in bnxt_alloc_rx_data()
922 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); in bnxt_alloc_rx_data()
945 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) in bnxt_reuse_rx_data() argument
947 u16 prod = rxr->rx_prod; in bnxt_reuse_rx_data()
949 struct bnxt *bp = rxr->bnapi->bp; in bnxt_reuse_rx_data()
952 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; in bnxt_reuse_rx_data()
953 cons_rx_buf = &rxr->rx_buf_ring[cons]; in bnxt_reuse_rx_data()
960 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; in bnxt_reuse_rx_data()
961 cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)]; in bnxt_reuse_rx_data()
966 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) in bnxt_find_next_agg_idx() argument
968 u16 next, max = rxr->rx_agg_bmap_size; in bnxt_find_next_agg_idx()
970 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); in bnxt_find_next_agg_idx()
972 next = find_first_zero_bit(rxr->rx_agg_bmap, max); in bnxt_find_next_agg_idx()
977 struct bnxt_rx_ring_info *rxr, in bnxt_alloc_rx_page() argument
981 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)]; in bnxt_alloc_rx_page()
985 u16 sw_prod = rxr->rx_sw_agg_prod; in bnxt_alloc_rx_page()
988 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); in bnxt_alloc_rx_page()
993 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) in bnxt_alloc_rx_page()
994 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); in bnxt_alloc_rx_page()
996 __set_bit(sw_prod, rxr->rx_agg_bmap); in bnxt_alloc_rx_page()
997 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; in bnxt_alloc_rx_page()
998 rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod)); in bnxt_alloc_rx_page()
1021 struct bnxt_rx_ring_info *rxr, in bnxt_get_tpa_agg_p5() argument
1024 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; in bnxt_get_tpa_agg_p5()
1034 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in bnxt_reuse_rx_agg_bufs() local
1035 u16 prod = rxr->rx_agg_prod; in bnxt_reuse_rx_agg_bufs()
1036 u16 sw_prod = rxr->rx_sw_agg_prod; in bnxt_reuse_rx_agg_bufs()
1051 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); in bnxt_reuse_rx_agg_bufs()
1055 __clear_bit(cons, rxr->rx_agg_bmap); in bnxt_reuse_rx_agg_bufs()
1057 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) in bnxt_reuse_rx_agg_bufs()
1058 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); in bnxt_reuse_rx_agg_bufs()
1060 __set_bit(sw_prod, rxr->rx_agg_bmap); in bnxt_reuse_rx_agg_bufs()
1061 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; in bnxt_reuse_rx_agg_bufs()
1062 cons_rx_buf = &rxr->rx_agg_ring[cons]; in bnxt_reuse_rx_agg_bufs()
1074 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)]; in bnxt_reuse_rx_agg_bufs()
1082 rxr->rx_agg_prod = prod; in bnxt_reuse_rx_agg_bufs()
1083 rxr->rx_sw_agg_prod = sw_prod; in bnxt_reuse_rx_agg_bufs()
1087 struct bnxt_rx_ring_info *rxr, in bnxt_rx_multi_page_skb() argument
1094 u16 prod = rxr->rx_prod; in bnxt_rx_multi_page_skb()
1098 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); in bnxt_rx_multi_page_skb()
1100 bnxt_reuse_rx_data(rxr, cons, data); in bnxt_rx_multi_page_skb()
1108 page_pool_recycle_direct(rxr->page_pool, page); in bnxt_rx_multi_page_skb()
1119 struct bnxt_rx_ring_info *rxr, in bnxt_rx_page_skb() argument
1128 u16 prod = rxr->rx_prod; in bnxt_rx_page_skb()
1132 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); in bnxt_rx_page_skb()
1134 bnxt_reuse_rx_data(rxr, cons, data); in bnxt_rx_page_skb()
1144 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); in bnxt_rx_page_skb()
1146 page_pool_recycle_direct(rxr->page_pool, page); in bnxt_rx_page_skb()
1166 struct bnxt_rx_ring_info *rxr, u16 cons, in bnxt_rx_skb() argument
1171 u16 prod = rxr->rx_prod; in bnxt_rx_skb()
1175 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); in bnxt_rx_skb()
1177 bnxt_reuse_rx_data(rxr, cons, data); in bnxt_rx_skb()
1202 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in __bnxt_rx_agg_pages() local
1203 u16 prod = rxr->rx_agg_prod; in __bnxt_rx_agg_pages()
1219 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); in __bnxt_rx_agg_pages()
1226 cons_rx_buf = &rxr->rx_agg_ring[cons]; in __bnxt_rx_agg_pages()
1230 __clear_bit(cons, rxr->rx_agg_bmap); in __bnxt_rx_agg_pages()
1243 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { in __bnxt_rx_agg_pages()
1250 rxr->rx_agg_prod = prod; in __bnxt_rx_agg_pages()
1261 rxr->rx_agg_prod = prod; in __bnxt_rx_agg_pages()
1409 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) in bnxt_alloc_agg_idx() argument
1411 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; in bnxt_alloc_agg_idx()
1422 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) in bnxt_free_agg_idx() argument
1424 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; in bnxt_free_agg_idx()
1429 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) in bnxt_lookup_agg_idx() argument
1431 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; in bnxt_lookup_agg_idx()
1467 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, in bnxt_tpa_start() argument
1479 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); in bnxt_tpa_start()
1484 prod = rxr->rx_prod; in bnxt_tpa_start()
1485 cons_rx_buf = &rxr->rx_buf_ring[cons]; in bnxt_tpa_start()
1486 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; in bnxt_tpa_start()
1487 tpa_info = &rxr->rx_tpa[agg_id]; in bnxt_tpa_start()
1489 if (unlikely(cons != rxr->rx_next_cons || in bnxt_tpa_start()
1492 cons, rxr->rx_next_cons, in bnxt_tpa_start()
1494 bnxt_sched_reset_rxr(bp, rxr); in bnxt_tpa_start()
1503 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; in bnxt_tpa_start()
1539 rxr->rx_prod = NEXT_RX(prod); in bnxt_tpa_start()
1541 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons)); in bnxt_tpa_start()
1542 cons_rx_buf = &rxr->rx_buf_ring[cons]; in bnxt_tpa_start()
1544 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); in bnxt_tpa_start()
1545 rxr->rx_prod = NEXT_RX(rxr->rx_prod); in bnxt_tpa_start()
1770 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in bnxt_tpa_end() local
1791 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); in bnxt_tpa_end()
1793 tpa_info = &rxr->rx_tpa[agg_id]; in bnxt_tpa_end()
1801 bnxt_free_agg_idx(rxr, agg_id); in bnxt_tpa_end()
1807 tpa_info = &rxr->rx_tpa[agg_id]; in bnxt_tpa_end()
1912 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, in bnxt_tpa_agg() argument
1918 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); in bnxt_tpa_agg()
1919 tpa_info = &rxr->rx_tpa[agg_id]; in bnxt_tpa_agg()
2022 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in bnxt_rx_pkt() local
2046 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); in bnxt_rx_pkt()
2062 prod = rxr->rx_prod; in bnxt_rx_pkt()
2066 bnxt_tpa_start(bp, rxr, cmp_type, in bnxt_rx_pkt()
2091 if (unlikely(cons != rxr->rx_next_cons)) { in bnxt_rx_pkt()
2095 if (rxr->rx_next_cons != 0xffff) in bnxt_rx_pkt()
2097 cons, rxr->rx_next_cons); in bnxt_rx_pkt()
2098 bnxt_sched_reset_rxr(bp, rxr); in bnxt_rx_pkt()
2103 rx_buf = &rxr->rx_buf_ring[cons]; in bnxt_rx_pkt()
2124 bnxt_reuse_rx_data(rxr, cons, data); in bnxt_rx_pkt()
2136 bnxt_sched_reset_rxr(bp, rxr); in bnxt_rx_pkt()
2146 if (bnxt_xdp_attached(bp, rxr)) { in bnxt_rx_pkt()
2147 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); in bnxt_rx_pkt()
2159 if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) { in bnxt_rx_pkt()
2170 bnxt_reuse_rx_data(rxr, cons, data); in bnxt_rx_pkt()
2177 bnxt_xdp_buff_frags_free(rxr, &xdp); in bnxt_rx_pkt()
2188 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, in bnxt_rx_pkt()
2200 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1); in bnxt_rx_pkt()
2203 bnxt_xdp_buff_frags_free(rxr, &xdp); in bnxt_rx_pkt()
2276 rxr->rx_prod = NEXT_RX(prod); in bnxt_rx_pkt()
2277 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons)); in bnxt_rx_pkt()
2712 struct bnxt_rx_ring_info *rxr; in bnxt_async_event_process() local
2729 rxr = bp->bnapi[grp_idx]->rx_ring; in bnxt_async_event_process()
2730 bnxt_sched_reset_rxr(bp, rxr); in bnxt_async_event_process()
2965 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in __bnxt_poll_work_done() local
2967 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); in __bnxt_poll_work_done()
2971 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in __bnxt_poll_work_done() local
2973 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); in __bnxt_poll_work_done()
3001 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in bnxt_poll_nitroa0() local
3058 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); in bnxt_poll_nitroa0()
3061 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); in bnxt_poll_nitroa0()
3312 static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) in bnxt_free_one_rx_ring() argument
3320 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; in bnxt_free_one_rx_ring()
3329 page_pool_recycle_direct(rxr->page_pool, data); in bnxt_free_one_rx_ring()
3339 static void bnxt_free_one_rx_agg_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) in bnxt_free_one_rx_agg_ring() argument
3346 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; in bnxt_free_one_rx_agg_ring()
3353 __clear_bit(i, rxr->rx_agg_bmap); in bnxt_free_one_rx_agg_ring()
3355 page_pool_recycle_direct(rxr->page_pool, page); in bnxt_free_one_rx_agg_ring()
3361 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; in bnxt_free_one_rx_ring_skbs() local
3366 if (!rxr->rx_tpa) in bnxt_free_one_rx_ring_skbs()
3370 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; in bnxt_free_one_rx_ring_skbs()
3386 if (!rxr->rx_buf_ring) in bnxt_free_one_rx_ring_skbs()
3389 bnxt_free_one_rx_ring(bp, rxr); in bnxt_free_one_rx_ring_skbs()
3392 if (!rxr->rx_agg_ring) in bnxt_free_one_rx_ring_skbs()
3395 bnxt_free_one_rx_agg_ring(bp, rxr); in bnxt_free_one_rx_ring_skbs()
3398 map = rxr->rx_tpa_idx_map; in bnxt_free_one_rx_ring_skbs()
3528 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_free_tpa_info() local
3530 kfree(rxr->rx_tpa_idx_map); in bnxt_free_tpa_info()
3531 rxr->rx_tpa_idx_map = NULL; in bnxt_free_tpa_info()
3532 if (rxr->rx_tpa) { in bnxt_free_tpa_info()
3534 kfree(rxr->rx_tpa[j].agg_arr); in bnxt_free_tpa_info()
3535 rxr->rx_tpa[j].agg_arr = NULL; in bnxt_free_tpa_info()
3538 kfree(rxr->rx_tpa); in bnxt_free_tpa_info()
3539 rxr->rx_tpa = NULL; in bnxt_free_tpa_info()
3555 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_alloc_tpa_info() local
3558 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), in bnxt_alloc_tpa_info()
3560 if (!rxr->rx_tpa) in bnxt_alloc_tpa_info()
3569 rxr->rx_tpa[j].agg_arr = agg; in bnxt_alloc_tpa_info()
3571 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), in bnxt_alloc_tpa_info()
3573 if (!rxr->rx_tpa_idx_map) in bnxt_alloc_tpa_info()
3588 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_free_rx_rings() local
3591 if (rxr->xdp_prog) in bnxt_free_rx_rings()
3592 bpf_prog_put(rxr->xdp_prog); in bnxt_free_rx_rings()
3594 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) in bnxt_free_rx_rings()
3595 xdp_rxq_info_unreg(&rxr->xdp_rxq); in bnxt_free_rx_rings()
3597 page_pool_destroy(rxr->page_pool); in bnxt_free_rx_rings()
3598 rxr->page_pool = NULL; in bnxt_free_rx_rings()
3600 kfree(rxr->rx_agg_bmap); in bnxt_free_rx_rings()
3601 rxr->rx_agg_bmap = NULL; in bnxt_free_rx_rings()
3603 ring = &rxr->rx_ring_struct; in bnxt_free_rx_rings()
3606 ring = &rxr->rx_agg_ring_struct; in bnxt_free_rx_rings()
3612 struct bnxt_rx_ring_info *rxr, in bnxt_alloc_rx_page_pool() argument
3621 pp.napi = &rxr->bnapi->napi; in bnxt_alloc_rx_page_pool()
3628 rxr->page_pool = page_pool_create(&pp); in bnxt_alloc_rx_page_pool()
3629 if (IS_ERR(rxr->page_pool)) { in bnxt_alloc_rx_page_pool()
3630 int err = PTR_ERR(rxr->page_pool); in bnxt_alloc_rx_page_pool()
3632 rxr->page_pool = NULL; in bnxt_alloc_rx_page_pool()
3650 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_alloc_rx_rings() local
3654 ring = &rxr->rx_ring_struct; in bnxt_alloc_rx_rings()
3660 rc = bnxt_alloc_rx_page_pool(bp, rxr, cpu_node); in bnxt_alloc_rx_rings()
3664 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); in bnxt_alloc_rx_rings()
3668 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, in bnxt_alloc_rx_rings()
3670 rxr->page_pool); in bnxt_alloc_rx_rings()
3672 xdp_rxq_info_unreg(&rxr->xdp_rxq); in bnxt_alloc_rx_rings()
3684 ring = &rxr->rx_agg_ring_struct; in bnxt_alloc_rx_rings()
3690 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; in bnxt_alloc_rx_rings()
3691 mem_size = rxr->rx_agg_bmap_size / 8; in bnxt_alloc_rx_rings()
3692 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); in bnxt_alloc_rx_rings()
3693 if (!rxr->rx_agg_bmap) in bnxt_alloc_rx_rings()
3993 struct bnxt_rx_ring_info *rxr) in bnxt_init_rx_ring_struct() argument
3998 ring = &rxr->rx_ring_struct; in bnxt_init_rx_ring_struct()
4002 rmem->pg_arr = (void **)rxr->rx_desc_ring; in bnxt_init_rx_ring_struct()
4003 rmem->dma_arr = rxr->rx_desc_mapping; in bnxt_init_rx_ring_struct()
4005 rmem->vmem = (void **)&rxr->rx_buf_ring; in bnxt_init_rx_ring_struct()
4007 ring = &rxr->rx_agg_ring_struct; in bnxt_init_rx_ring_struct()
4011 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; in bnxt_init_rx_ring_struct()
4012 rmem->dma_arr = rxr->rx_agg_desc_mapping; in bnxt_init_rx_ring_struct()
4014 rmem->vmem = (void **)&rxr->rx_agg_ring; in bnxt_init_rx_ring_struct()
4018 struct bnxt_rx_ring_info *rxr) in bnxt_reset_rx_ring_struct() argument
4024 rxr->page_pool->p.napi = NULL; in bnxt_reset_rx_ring_struct()
4025 rxr->page_pool = NULL; in bnxt_reset_rx_ring_struct()
4026 memset(&rxr->xdp_rxq, 0, sizeof(struct xdp_rxq_info)); in bnxt_reset_rx_ring_struct()
4028 ring = &rxr->rx_ring_struct; in bnxt_reset_rx_ring_struct()
4038 ring = &rxr->rx_agg_ring_struct; in bnxt_reset_rx_ring_struct()
4057 struct bnxt_rx_ring_info *rxr; in bnxt_init_ring_struct() local
4073 rxr = bnapi->rx_ring; in bnxt_init_ring_struct()
4074 if (!rxr) in bnxt_init_ring_struct()
4077 ring = &rxr->rx_ring_struct; in bnxt_init_ring_struct()
4081 rmem->pg_arr = (void **)rxr->rx_desc_ring; in bnxt_init_ring_struct()
4082 rmem->dma_arr = rxr->rx_desc_mapping; in bnxt_init_ring_struct()
4084 rmem->vmem = (void **)&rxr->rx_buf_ring; in bnxt_init_ring_struct()
4086 ring = &rxr->rx_agg_ring_struct; in bnxt_init_ring_struct()
4090 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; in bnxt_init_ring_struct()
4091 rmem->dma_arr = rxr->rx_agg_desc_mapping; in bnxt_init_ring_struct()
4093 rmem->vmem = (void **)&rxr->rx_agg_ring; in bnxt_init_ring_struct()
4132 struct bnxt_rx_ring_info *rxr, in bnxt_alloc_one_rx_ring_skb() argument
4138 prod = rxr->rx_prod; in bnxt_alloc_one_rx_ring_skb()
4140 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { in bnxt_alloc_one_rx_ring_skb()
4147 rxr->rx_prod = prod; in bnxt_alloc_one_rx_ring_skb()
4151 struct bnxt_rx_ring_info *rxr, in bnxt_alloc_one_rx_ring_page() argument
4157 prod = rxr->rx_agg_prod; in bnxt_alloc_one_rx_ring_page()
4159 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) { in bnxt_alloc_one_rx_ring_page()
4166 rxr->rx_agg_prod = prod; in bnxt_alloc_one_rx_ring_page()
4171 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; in bnxt_alloc_one_rx_ring() local
4174 bnxt_alloc_one_rx_ring_skb(bp, rxr, ring_nr); in bnxt_alloc_one_rx_ring()
4179 bnxt_alloc_one_rx_ring_page(bp, rxr, ring_nr); in bnxt_alloc_one_rx_ring()
4181 if (rxr->rx_tpa) { in bnxt_alloc_one_rx_ring()
4190 rxr->rx_tpa[i].data = data; in bnxt_alloc_one_rx_ring()
4191 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; in bnxt_alloc_one_rx_ring()
4192 rxr->rx_tpa[i].mapping = mapping; in bnxt_alloc_one_rx_ring()
4199 struct bnxt_rx_ring_info *rxr) in bnxt_init_one_rx_ring_rxbd() argument
4210 ring = &rxr->rx_ring_struct; in bnxt_init_one_rx_ring_rxbd()
4216 struct bnxt_rx_ring_info *rxr) in bnxt_init_one_rx_agg_ring_rxbd() argument
4221 ring = &rxr->rx_agg_ring_struct; in bnxt_init_one_rx_agg_ring_rxbd()
4233 struct bnxt_rx_ring_info *rxr; in bnxt_init_one_rx_ring() local
4235 rxr = &bp->rx_ring[ring_nr]; in bnxt_init_one_rx_ring()
4236 bnxt_init_one_rx_ring_rxbd(bp, rxr); in bnxt_init_one_rx_ring()
4239 &rxr->bnapi->napi); in bnxt_init_one_rx_ring()
4243 rxr->xdp_prog = bp->xdp_prog; in bnxt_init_one_rx_ring()
4246 bnxt_init_one_rx_agg_ring_rxbd(bp, rxr); in bnxt_init_one_rx_ring()
4988 struct bnxt_rx_ring_info *rxr; in bnxt_clear_ring_indices() local
5003 rxr = bnapi->rx_ring; in bnxt_clear_ring_indices()
5004 if (rxr) { in bnxt_clear_ring_indices()
5005 rxr->rx_prod = 0; in bnxt_clear_ring_indices()
5006 rxr->rx_agg_prod = 0; in bnxt_clear_ring_indices()
5007 rxr->rx_sw_agg_prod = 0; in bnxt_clear_ring_indices()
5008 rxr->rx_next_cons = 0; in bnxt_clear_ring_indices()
5195 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_alloc_mem() local
5198 rxr->rx_ring_struct.ring_mem.flags = in bnxt_alloc_mem()
5200 rxr->rx_agg_ring_struct.ring_mem.flags = in bnxt_alloc_mem()
5203 rxr->rx_cpr = &bp->bnapi[i]->cp_ring; in bnxt_alloc_mem()
5205 rxr->bnapi = bp->bnapi[i]; in bnxt_alloc_mem()
6181 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) in bnxt_cp_ring_for_rx() argument
6184 return rxr->rx_cpr->cp_ring_struct.fw_ring_id; in bnxt_cp_ring_for_rx()
6186 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); in bnxt_cp_ring_for_rx()
6286 struct bnxt_rx_ring_info *rxr; in bnxt_fill_hw_rss_tbl_p5() local
6300 rxr = &bp->rx_ring[j]; in bnxt_fill_hw_rss_tbl_p5()
6302 ring_id = rxr->rx_ring_struct.fw_ring_id; in bnxt_fill_hw_rss_tbl_p5()
6304 ring_id = bnxt_cp_ring_for_rx(bp, rxr); in bnxt_fill_hw_rss_tbl_p5()
6508 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; in bnxt_hwrm_vnic_cfg() local
6511 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); in bnxt_hwrm_vnic_cfg()
6513 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); in bnxt_hwrm_vnic_cfg()
6977 struct bnxt_rx_ring_info *rxr) in bnxt_hwrm_rx_ring_alloc() argument
6979 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; in bnxt_hwrm_rx_ring_alloc()
6980 struct bnxt_napi *bnapi = rxr->bnapi; in bnxt_hwrm_rx_ring_alloc()
6989 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); in bnxt_hwrm_rx_ring_alloc()
6996 struct bnxt_rx_ring_info *rxr) in bnxt_hwrm_rx_agg_ring_alloc() argument
6998 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; in bnxt_hwrm_rx_agg_ring_alloc()
7009 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, in bnxt_hwrm_rx_agg_ring_alloc()
7011 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); in bnxt_hwrm_rx_agg_ring_alloc()
7012 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); in bnxt_hwrm_rx_agg_ring_alloc()
7084 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_hwrm_ring_alloc() local
7086 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); in bnxt_hwrm_ring_alloc()
7091 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); in bnxt_hwrm_ring_alloc()
7093 struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr; in bnxt_hwrm_ring_alloc()
7094 struct bnxt_napi *bnapi = rxr->bnapi; in bnxt_hwrm_ring_alloc()
7155 struct bnxt_rx_ring_info *rxr, in bnxt_hwrm_rx_ring_free() argument
7158 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; in bnxt_hwrm_rx_ring_free()
7159 u32 grp_idx = rxr->bnapi->index; in bnxt_hwrm_rx_ring_free()
7165 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); in bnxt_hwrm_rx_ring_free()
7175 struct bnxt_rx_ring_info *rxr, in bnxt_hwrm_rx_agg_ring_free() argument
7178 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; in bnxt_hwrm_rx_agg_ring_free()
7179 u32 grp_idx = rxr->bnapi->index; in bnxt_hwrm_rx_agg_ring_free()
7190 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); in bnxt_hwrm_rx_agg_ring_free()
13203 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; in bnxt_dump_rx_sw_state() local
13206 if (!rxr) in bnxt_dump_rx_sw_state()
13210 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, in bnxt_dump_rx_sw_state()
13211 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, in bnxt_dump_rx_sw_state()
13212 rxr->rx_sw_agg_prod); in bnxt_dump_rx_sw_state()
13241 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; in bnxt_hwrm_rx_ring_reset() local
13243 struct bnxt_napi *bnapi = rxr->bnapi; in bnxt_hwrm_rx_ring_reset()
13399 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; in bnxt_rx_ring_reset() local
13403 if (!rxr->bnapi->in_reset) in bnxt_rx_ring_reset()
13417 rxr->rx_prod = 0; in bnxt_rx_ring_reset()
13418 rxr->rx_agg_prod = 0; in bnxt_rx_ring_reset()
13419 rxr->rx_sw_agg_prod = 0; in bnxt_rx_ring_reset()
13420 rxr->rx_next_cons = 0; in bnxt_rx_ring_reset()
13421 rxr->bnapi->in_reset = false; in bnxt_rx_ring_reset()
13423 cpr = &rxr->bnapi->cp_ring; in bnxt_rx_ring_reset()
13426 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); in bnxt_rx_ring_reset()
13427 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); in bnxt_rx_ring_reset()
15036 static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) in bnxt_alloc_rx_agg_bmap() argument
15040 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; in bnxt_alloc_rx_agg_bmap()
15041 mem_size = rxr->rx_agg_bmap_size / 8; in bnxt_alloc_rx_agg_bmap()
15042 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); in bnxt_alloc_rx_agg_bmap()
15043 if (!rxr->rx_agg_bmap) in bnxt_alloc_rx_agg_bmap()
15051 struct bnxt_rx_ring_info *rxr, *clone; in bnxt_queue_mem_alloc() local
15056 rxr = &bp->rx_ring[idx]; in bnxt_queue_mem_alloc()
15058 memcpy(clone, rxr, sizeof(*rxr)); in bnxt_queue_mem_alloc()
15067 rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); in bnxt_queue_mem_alloc()
15121 struct bnxt_rx_ring_info *rxr = qmem; in bnxt_queue_mem_free() local
15125 bnxt_free_one_rx_ring(bp, rxr); in bnxt_queue_mem_free()
15126 bnxt_free_one_rx_agg_ring(bp, rxr); in bnxt_queue_mem_free()
15128 xdp_rxq_info_unreg(&rxr->xdp_rxq); in bnxt_queue_mem_free()
15130 page_pool_destroy(rxr->page_pool); in bnxt_queue_mem_free()
15131 rxr->page_pool = NULL; in bnxt_queue_mem_free()
15133 ring = &rxr->rx_ring_struct; in bnxt_queue_mem_free()
15136 ring = &rxr->rx_agg_ring_struct; in bnxt_queue_mem_free()
15139 kfree(rxr->rx_agg_bmap); in bnxt_queue_mem_free()
15140 rxr->rx_agg_bmap = NULL; in bnxt_queue_mem_free()
15201 struct bnxt_rx_ring_info *rxr, *clone; in bnxt_queue_start() local
15206 rxr = &bp->rx_ring[idx]; in bnxt_queue_start()
15209 rxr->rx_prod = clone->rx_prod; in bnxt_queue_start()
15210 rxr->rx_agg_prod = clone->rx_agg_prod; in bnxt_queue_start()
15211 rxr->rx_sw_agg_prod = clone->rx_sw_agg_prod; in bnxt_queue_start()
15212 rxr->rx_next_cons = clone->rx_next_cons; in bnxt_queue_start()
15213 rxr->page_pool = clone->page_pool; in bnxt_queue_start()
15214 rxr->xdp_rxq = clone->xdp_rxq; in bnxt_queue_start()
15216 bnxt_copy_rx_ring(bp, rxr, clone); in bnxt_queue_start()
15218 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); in bnxt_queue_start()
15221 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, rxr); in bnxt_queue_start()
15225 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); in bnxt_queue_start()
15227 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); in bnxt_queue_start()
15229 cpr = &rxr->bnapi->cp_ring; in bnxt_queue_start()
15242 bnxt_hwrm_rx_ring_free(bp, rxr, false); in bnxt_queue_start()
15249 struct bnxt_rx_ring_info *rxr; in bnxt_queue_stop() local
15260 rxr = &bp->rx_ring[idx]; in bnxt_queue_stop()
15261 bnxt_hwrm_rx_ring_free(bp, rxr, false); in bnxt_queue_stop()
15262 bnxt_hwrm_rx_agg_ring_free(bp, rxr, false); in bnxt_queue_stop()
15263 rxr->rx_next_cons = 0; in bnxt_queue_stop()
15264 page_pool_disable_direct_recycling(rxr->page_pool); in bnxt_queue_stop()
15266 memcpy(qmem, rxr, sizeof(*rxr)); in bnxt_queue_stop()