94 					       struct rbdr *rbdr, gfp_t gfp)  in nicvf_alloc_page()  argument
101 	pgcache = &rbdr->pgcache[rbdr->pgidx];  in nicvf_alloc_page()
111 		if (rbdr->is_xdp) {  in nicvf_alloc_page()
129 		if (rbdr->pgalloc >= rbdr->pgcnt) {  in nicvf_alloc_page()
139 		rbdr->pgalloc++;  in nicvf_alloc_page()
143 	if (rbdr->is_xdp) {  in nicvf_alloc_page()
163 	rbdr->pgidx++;  in nicvf_alloc_page()
164 	rbdr->pgidx &= (rbdr->pgcnt - 1);  in nicvf_alloc_page()
167 	next = &rbdr->pgcache[rbdr->pgidx];  in nicvf_alloc_page()
176 static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr,  in nicvf_alloc_rcv_buffer()  argument
184 	if (!rbdr->is_xdp && nic->rb_page &&  in nicvf_alloc_rcv_buffer()
194 	pgcache = nicvf_alloc_page(nic, rbdr, gfp);  in nicvf_alloc_rcv_buffer()
203 	if (rbdr->is_xdp)  in nicvf_alloc_rcv_buffer()
210 	if (rbdr->is_xdp && pgcache && pgcache->dma_addr) {  in nicvf_alloc_rcv_buffer()
253 static int  nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,  in nicvf_init_rbdr()  argument
261 	err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len,  in nicvf_init_rbdr()
267 	rbdr->desc = rbdr->dmem.base;  in nicvf_init_rbdr()
269 	rbdr->dma_size = buf_size;  in nicvf_init_rbdr()
270 	rbdr->enable = true;  in nicvf_init_rbdr()
271 	rbdr->thresh = RBDR_THRESH;  in nicvf_init_rbdr()
272 	rbdr->head = 0;  in nicvf_init_rbdr()
273 	rbdr->tail = 0;  in nicvf_init_rbdr()
285 		rbdr->pgcnt = ring_len / (PAGE_SIZE / buf_size);  in nicvf_init_rbdr()
286 		rbdr->is_xdp = false;  in nicvf_init_rbdr()
288 		rbdr->pgcnt = ring_len;  in nicvf_init_rbdr()
289 		rbdr->is_xdp = true;  in nicvf_init_rbdr()
291 	rbdr->pgcnt = roundup_pow_of_two(rbdr->pgcnt);  in nicvf_init_rbdr()
292 	rbdr->pgcache = kcalloc(rbdr->pgcnt, sizeof(*rbdr->pgcache),  in nicvf_init_rbdr()
294 	if (!rbdr->pgcache)  in nicvf_init_rbdr()
296 	rbdr->pgidx = 0;  in nicvf_init_rbdr()
297 	rbdr->pgalloc = 0;  in nicvf_init_rbdr()
301 		err = nicvf_alloc_rcv_buffer(nic, rbdr, GFP_KERNEL,  in nicvf_init_rbdr()
305 			rbdr->tail = idx - 1;  in nicvf_init_rbdr()
309 		desc = GET_RBDR_DESC(rbdr, idx);  in nicvf_init_rbdr()
319 static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)  in nicvf_free_rbdr()  argument
326 	if (!rbdr)  in nicvf_free_rbdr()
329 	rbdr->enable = false;  in nicvf_free_rbdr()
330 	if (!rbdr->dmem.base)  in nicvf_free_rbdr()
333 	head = rbdr->head;  in nicvf_free_rbdr()
334 	tail = rbdr->tail;  in nicvf_free_rbdr()
338 		desc = GET_RBDR_DESC(rbdr, head);  in nicvf_free_rbdr()
346 		head &= (rbdr->dmem.q_len - 1);  in nicvf_free_rbdr()
349 	desc = GET_RBDR_DESC(rbdr, tail);  in nicvf_free_rbdr()
362 	while (head < rbdr->pgcnt) {  in nicvf_free_rbdr()
363 		pgcache = &rbdr->pgcache[head];  in nicvf_free_rbdr()
365 			if (rbdr->is_xdp) {  in nicvf_free_rbdr()
375 	nicvf_free_q_desc_mem(nic, &rbdr->dmem);  in nicvf_free_rbdr()
386 	struct rbdr *rbdr;  in nicvf_refill_rbdr()  local
395 	rbdr = &qs->rbdr[rbdr_idx];  in nicvf_refill_rbdr()
397 	if (!rbdr->enable)  in nicvf_refill_rbdr()
416 		tail &= (rbdr->dmem.q_len - 1);  in nicvf_refill_rbdr()
418 		if (nicvf_alloc_rcv_buffer(nic, rbdr, gfp, RCV_FRAG_LEN, &rbuf))  in nicvf_refill_rbdr()
421 		desc = GET_RBDR_DESC(rbdr, tail);  in nicvf_refill_rbdr()
443 	if (!nic->rb_alloc_fail && rbdr->enable &&  in nicvf_refill_rbdr()
661 			       struct rbdr *rbdr, int qidx)  in nicvf_reclaim_rbdr()  argument
667 	rbdr->head = nicvf_queue_reg_read(nic,  in nicvf_reclaim_rbdr()
670 	rbdr->tail = nicvf_queue_reg_read(nic,  in nicvf_reclaim_rbdr()
920 	struct rbdr *rbdr;  in nicvf_rbdr_config()  local
923 	rbdr = &qs->rbdr[qidx];  in nicvf_rbdr_config()
924 	nicvf_reclaim_rbdr(nic, rbdr, qidx);  in nicvf_rbdr_config()
930 			      qidx, (u64)(rbdr->dmem.phys_base));  in nicvf_rbdr_config()
940 	rbdr_cfg.lines = rbdr->dma_size / 128;  in nicvf_rbdr_config()
950 			      qidx, rbdr->thresh - 1);  in nicvf_rbdr_config()
996 		nicvf_free_rbdr(nic, &qs->rbdr[qidx]);  in nicvf_free_resources()
1014 		if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len,  in nicvf_alloc_resources()