183 static struct dpaa_bp *dpaa_bp_array[BM_MAX_NUM_OF_POOLS];
499 static struct dpaa_bp *dpaa_bpid2pool(int bpid)  in dpaa_bpid2pool()
519 static void dpaa_bpid2pool_map(int bpid, struct dpaa_bp *dpaa_bp)  in dpaa_bpid2pool_map()  argument
521 	dpaa_bp_array[bpid] = dpaa_bp;  in dpaa_bpid2pool_map()
522 	refcount_set(&dpaa_bp->refs, 1);  in dpaa_bpid2pool_map()
525 static int dpaa_bp_alloc_pool(struct dpaa_bp *dpaa_bp)  in dpaa_bp_alloc_pool()  argument
529 	if (dpaa_bp->size == 0 || dpaa_bp->config_count == 0) {  in dpaa_bp_alloc_pool()
536 	if (dpaa_bp->bpid != FSL_DPAA_BPID_INV &&  in dpaa_bp_alloc_pool()
537 	    dpaa_bpid2pool_use(dpaa_bp->bpid))  in dpaa_bp_alloc_pool()
540 	if (dpaa_bp->bpid == FSL_DPAA_BPID_INV) {  in dpaa_bp_alloc_pool()
541 		dpaa_bp->pool = bman_new_pool();  in dpaa_bp_alloc_pool()
542 		if (!dpaa_bp->pool) {  in dpaa_bp_alloc_pool()
548 		dpaa_bp->bpid = (u8)bman_get_bpid(dpaa_bp->pool);  in dpaa_bp_alloc_pool()
551 	if (dpaa_bp->seed_cb) {  in dpaa_bp_alloc_pool()
552 		err = dpaa_bp->seed_cb(dpaa_bp);  in dpaa_bp_alloc_pool()
557 	dpaa_bpid2pool_map(dpaa_bp->bpid, dpaa_bp);  in dpaa_bp_alloc_pool()
563 	bman_free_pool(dpaa_bp->pool);  in dpaa_bp_alloc_pool()
569 static void dpaa_bp_drain(struct dpaa_bp *bp)  in dpaa_bp_drain()
599 static void dpaa_bp_free(struct dpaa_bp *dpaa_bp)  in dpaa_bp_free()  argument
601 	struct dpaa_bp *bp = dpaa_bpid2pool(dpaa_bp->bpid);  in dpaa_bp_free()
622 	dpaa_bp_free(priv->dpaa_bp);  in dpaa_bps_free()
1264 static int dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp *bp,  in dpaa_eth_init_rx_port()
1316 			       struct dpaa_bp *bp,  in dpaa_eth_init_ports()
1337 static int dpaa_bman_release(const struct dpaa_bp *dpaa_bp,  in dpaa_bman_release()  argument
1342 	err = bman_release(dpaa_bp->pool, bmb, cnt);  in dpaa_bman_release()
1344 	if (WARN_ON(err) && dpaa_bp->free_buf_cb)  in dpaa_bman_release()
1346 			dpaa_bp->free_buf_cb(dpaa_bp, &bmb[cnt]);  in dpaa_bman_release()
1354 	struct dpaa_bp *dpaa_bp;  in dpaa_release_sgt_members()  local
1360 		dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);  in dpaa_release_sgt_members()
1361 		if (!dpaa_bp)  in dpaa_release_sgt_members()
1375 		dpaa_bman_release(dpaa_bp, bmb, j);  in dpaa_release_sgt_members()
1383 	struct dpaa_bp *dpaa_bp;  in dpaa_fd_release()  local
1391 	dpaa_bp = dpaa_bpid2pool(fd->bpid);  in dpaa_fd_release()
1392 	if (!dpaa_bp)  in dpaa_fd_release()
1399 		dma_unmap_page(dpaa_bp->priv->rx_dma_dev, qm_fd_addr(fd),  in dpaa_fd_release()
1404 		addr = dma_map_page(dpaa_bp->priv->rx_dma_dev,  in dpaa_fd_release()
1407 		if (dma_mapping_error(dpaa_bp->priv->rx_dma_dev, addr)) {  in dpaa_fd_release()
1414 	dpaa_bman_release(dpaa_bp, &bmb, 1);  in dpaa_fd_release()
1546 static int dpaa_bp_add_8_bufs(const struct dpaa_bp *dpaa_bp)  in dpaa_bp_add_8_bufs()  argument
1548 	struct net_device *net_dev = dpaa_bp->priv->net_dev;  in dpaa_bp_add_8_bufs()
1561 		addr = dma_map_page(dpaa_bp->priv->rx_dma_dev, p, 0,  in dpaa_bp_add_8_bufs()
1563 		if (unlikely(dma_mapping_error(dpaa_bp->priv->rx_dma_dev,  in dpaa_bp_add_8_bufs()
1574 	return dpaa_bman_release(dpaa_bp, bmb, i);  in dpaa_bp_add_8_bufs()
1589 static int dpaa_bp_seed(struct dpaa_bp *dpaa_bp)  in dpaa_bp_seed()  argument
1595 		int *count_ptr = per_cpu_ptr(dpaa_bp->percpu_count, i);  in dpaa_bp_seed()
1601 		for (j = 0; j < dpaa_bp->config_count; j += 8)  in dpaa_bp_seed()
1602 			*count_ptr += dpaa_bp_add_8_bufs(dpaa_bp);  in dpaa_bp_seed()
1610 static int dpaa_eth_refill_bpool(struct dpaa_bp *dpaa_bp, int *countptr)  in dpaa_eth_refill_bpool()  argument
1617 			new_bufs = dpaa_bp_add_8_bufs(dpaa_bp);  in dpaa_eth_refill_bpool()
1638 	struct dpaa_bp *dpaa_bp;  in dpaa_eth_refill_bpools()  local
1641 	dpaa_bp = priv->dpaa_bp;  in dpaa_eth_refill_bpools()
1642 	if (!dpaa_bp)  in dpaa_eth_refill_bpools()
1644 	countptr = this_cpu_ptr(dpaa_bp->percpu_count);  in dpaa_eth_refill_bpools()
1646 	return dpaa_eth_refill_bpool(dpaa_bp, countptr);  in dpaa_eth_refill_bpools()
1766 	struct dpaa_bp *dpaa_bp;  in contig_fd_to_skb()  local
1773 	dpaa_bp = dpaa_bpid2pool(fd->bpid);  in contig_fd_to_skb()
1774 	if (!dpaa_bp)  in contig_fd_to_skb()
1777 	skb = build_skb(vaddr, dpaa_bp->size +  in contig_fd_to_skb()
1805 	struct dpaa_bp *dpaa_bp;  in sg_fd_to_skb()  local
1833 		dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);  in sg_fd_to_skb()
1834 		if (!dpaa_bp)  in sg_fd_to_skb()
1838 			sz = dpaa_bp->size +  in sg_fd_to_skb()
1877 					frag_len, dpaa_bp->size);  in sg_fd_to_skb()
1881 		count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);  in sg_fd_to_skb()
1906 			dpaa_bp = dpaa_bpid2pool(sgt[j].bpid);  in sg_fd_to_skb()
1907 			if (dpaa_bp) {  in sg_fd_to_skb()
1908 				count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);  in sg_fd_to_skb()
2477 	struct dpaa_bp *dpaa_bp;  in rx_error_dqrr()  local
2482 	dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);  in rx_error_dqrr()
2483 	if (!dpaa_bp)  in rx_error_dqrr()
2676 	struct dpaa_bp *dpaa_bp;  in rx_default_dqrr()  local
2691 	dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);  in rx_default_dqrr()
2692 	if (!dpaa_bp)  in rx_default_dqrr()
2725 	dma_unmap_page(dpaa_bp->priv->rx_dma_dev, addr, DPAA_BP_RAW_SIZE,  in rx_default_dqrr()
2738 	count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);  in rx_default_dqrr()
2979 	int max_contig_data = priv->dpaa_bp->size - priv->rx_headroom;  in xdp_validate_mtu()
3172 static inline void dpaa_bp_free_pf(const struct dpaa_bp *bp,  in dpaa_bp_free_pf()
3184 static struct dpaa_bp *dpaa_bp_alloc(struct device *dev)  in dpaa_bp_alloc()
3186 	struct dpaa_bp *dpaa_bp;  in dpaa_bp_alloc()  local
3188 	dpaa_bp = devm_kzalloc(dev, sizeof(*dpaa_bp), GFP_KERNEL);  in dpaa_bp_alloc()
3189 	if (!dpaa_bp)  in dpaa_bp_alloc()
3192 	dpaa_bp->bpid = FSL_DPAA_BPID_INV;  in dpaa_bp_alloc()
3193 	dpaa_bp->percpu_count = devm_alloc_percpu(dev, *dpaa_bp->percpu_count);  in dpaa_bp_alloc()
3194 	if (!dpaa_bp->percpu_count)  in dpaa_bp_alloc()
3197 	dpaa_bp->config_count = FSL_DPAA_ETH_MAX_BUF_COUNT;  in dpaa_bp_alloc()
3199 	dpaa_bp->seed_cb = dpaa_bp_seed;  in dpaa_bp_alloc()
3200 	dpaa_bp->free_buf_cb = dpaa_bp_free_pf;  in dpaa_bp_alloc()
3202 	return dpaa_bp;  in dpaa_bp_alloc()
3287 	struct dpaa_bp *dpaa_bp = NULL;  in dpaa_eth_probe()  local
3397 	dpaa_bp = dpaa_bp_alloc(dev);  in dpaa_eth_probe()
3398 	if (IS_ERR(dpaa_bp)) {  in dpaa_eth_probe()
3399 		err = PTR_ERR(dpaa_bp);  in dpaa_eth_probe()
3403 	dpaa_bp->raw_size = DPAA_BP_RAW_SIZE;  in dpaa_eth_probe()
3405 	dpaa_bp->size = dpaa_bp_size(dpaa_bp->raw_size);  in dpaa_eth_probe()
3406 	dpaa_bp->priv = priv;  in dpaa_eth_probe()
3408 	err = dpaa_bp_alloc_pool(dpaa_bp);  in dpaa_eth_probe()
3411 	priv->dpaa_bp = dpaa_bp;  in dpaa_eth_probe()
3471 	err = dpaa_eth_init_ports(mac_dev, dpaa_bp, &port_fqs,  in dpaa_eth_probe()