640 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_num_free_bufs()  local
641 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_num_free_bufs()
642 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_num_free_bufs()
643 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_num_free_bufs()
656 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_post_buf()  local
657 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_post_buf()
658 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_post_buf()
659 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_post_buf()
660 	struct ce_desc *base = dest_ring->base_addr_owner_space;  in __ath10k_ce_rx_post_buf()
673 	dest_ring->per_transfer_context[write_index] = ctx;  in __ath10k_ce_rx_post_buf()
676 	dest_ring->write_index = write_index;  in __ath10k_ce_rx_post_buf()
687 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in __ath10k_ce_rx_post_buf_64()  local
688 	unsigned int nentries_mask = dest_ring->nentries_mask;  in __ath10k_ce_rx_post_buf_64()
689 	unsigned int write_index = dest_ring->write_index;  in __ath10k_ce_rx_post_buf_64()
690 	unsigned int sw_index = dest_ring->sw_index;  in __ath10k_ce_rx_post_buf_64()
691 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in __ath10k_ce_rx_post_buf_64()
706 	dest_ring->per_transfer_context[write_index] = ctx;  in __ath10k_ce_rx_post_buf_64()
709 	dest_ring->write_index = write_index;  in __ath10k_ce_rx_post_buf_64()
717 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;  in ath10k_ce_rx_update_write_idx()  local
718 	unsigned int nentries_mask = dest_ring->nentries_mask;  in ath10k_ce_rx_update_write_idx()
719 	unsigned int write_index = dest_ring->write_index;  in ath10k_ce_rx_update_write_idx()
726 	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)  in ath10k_ce_rx_update_write_idx()
731 	dest_ring->write_index = write_index;  in ath10k_ce_rx_update_write_idx()
759 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock()  local
760 	unsigned int nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_completed_recv_next_nolock()
761 	unsigned int sw_index = dest_ring->sw_index;  in _ath10k_ce_completed_recv_next_nolock()
763 	struct ce_desc *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_completed_recv_next_nolock()
789 			dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_recv_next_nolock()
795 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_recv_next_nolock()
799 	dest_ring->sw_index = sw_index;  in _ath10k_ce_completed_recv_next_nolock()
809 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in _ath10k_ce_completed_recv_next_nolock_64()  local
810 	unsigned int nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_completed_recv_next_nolock_64()
811 	unsigned int sw_index = dest_ring->sw_index;  in _ath10k_ce_completed_recv_next_nolock_64()
812 	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_completed_recv_next_nolock_64()
838 			dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_completed_recv_next_nolock_64()
844 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_completed_recv_next_nolock_64()
848 	dest_ring->sw_index = sw_index;  in _ath10k_ce_completed_recv_next_nolock_64()
886 	struct ath10k_ce_ring *dest_ring;  in _ath10k_ce_revoke_recv_next()  local
894 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next()
896 	if (!dest_ring)  in _ath10k_ce_revoke_recv_next()
904 	nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_revoke_recv_next()
905 	sw_index = dest_ring->sw_index;  in _ath10k_ce_revoke_recv_next()
906 	write_index = dest_ring->write_index;  in _ath10k_ce_revoke_recv_next()
908 		struct ce_desc *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_revoke_recv_next()
916 				dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_revoke_recv_next()
919 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_revoke_recv_next()
924 		dest_ring->sw_index = sw_index;  in _ath10k_ce_revoke_recv_next()
939 	struct ath10k_ce_ring *dest_ring;  in _ath10k_ce_revoke_recv_next_64()  local
947 	dest_ring = ce_state->dest_ring;  in _ath10k_ce_revoke_recv_next_64()
949 	if (!dest_ring)  in _ath10k_ce_revoke_recv_next_64()
957 	nentries_mask = dest_ring->nentries_mask;  in _ath10k_ce_revoke_recv_next_64()
958 	sw_index = dest_ring->sw_index;  in _ath10k_ce_revoke_recv_next_64()
959 	write_index = dest_ring->write_index;  in _ath10k_ce_revoke_recv_next_64()
961 		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;  in _ath10k_ce_revoke_recv_next_64()
970 				dest_ring->per_transfer_context[sw_index];  in _ath10k_ce_revoke_recv_next_64()
973 		dest_ring->per_transfer_context[sw_index] = NULL;  in _ath10k_ce_revoke_recv_next_64()
978 		dest_ring->sw_index = sw_index;  in _ath10k_ce_revoke_recv_next_64()
1403 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;  in ath10k_ce_init_dest_ring()  local
1409 		memset(dest_ring->base_addr_owner_space, 0,  in ath10k_ce_init_dest_ring()
1412 		memset(dest_ring->base_addr_owner_space, 0,  in ath10k_ce_init_dest_ring()
1415 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);  in ath10k_ce_init_dest_ring()
1416 	dest_ring->sw_index &= dest_ring->nentries_mask;  in ath10k_ce_init_dest_ring()
1417 	dest_ring->write_index =  in ath10k_ce_init_dest_ring()
1419 	dest_ring->write_index &= dest_ring->nentries_mask;  in ath10k_ce_init_dest_ring()
1422 					  dest_ring->base_addr_ce_space);  in ath10k_ce_init_dest_ring()
1430 		   ce_id, nentries, dest_ring->base_addr_owner_space);  in ath10k_ce_init_dest_ring()
1570 	struct ath10k_ce_ring *dest_ring;  in ath10k_ce_alloc_dest_ring()  local
1576 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,  in ath10k_ce_alloc_dest_ring()
1578 	if (dest_ring == NULL)  in ath10k_ce_alloc_dest_ring()
1581 	dest_ring->nentries = nentries;  in ath10k_ce_alloc_dest_ring()
1582 	dest_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_dest_ring()
1588 	dest_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_dest_ring()
1593 	if (!dest_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_dest_ring()
1594 		kfree(dest_ring);  in ath10k_ce_alloc_dest_ring()
1598 	dest_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_dest_ring()
1600 	dest_ring->base_addr_owner_space =  in ath10k_ce_alloc_dest_ring()
1601 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_dest_ring()
1603 	dest_ring->base_addr_ce_space =  in ath10k_ce_alloc_dest_ring()
1604 				ALIGN(dest_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_dest_ring()
1607 	return dest_ring;  in ath10k_ce_alloc_dest_ring()
1614 	struct ath10k_ce_ring *dest_ring;  in ath10k_ce_alloc_dest_ring_64()  local
1620 	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,  in ath10k_ce_alloc_dest_ring_64()
1622 	if (!dest_ring)  in ath10k_ce_alloc_dest_ring_64()
1625 	dest_ring->nentries = nentries;  in ath10k_ce_alloc_dest_ring_64()
1626 	dest_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_dest_ring_64()
1631 	dest_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_dest_ring_64()
1636 	if (!dest_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_dest_ring_64()
1637 		kfree(dest_ring);  in ath10k_ce_alloc_dest_ring_64()
1641 	dest_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_dest_ring_64()
1646 	dest_ring->base_addr_owner_space =  in ath10k_ce_alloc_dest_ring_64()
1647 			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_dest_ring_64()
1649 	dest_ring->base_addr_ce_space =  in ath10k_ce_alloc_dest_ring_64()
1650 			ALIGN(dest_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_dest_ring_64()
1653 	return dest_ring;  in ath10k_ce_alloc_dest_ring_64()
1733 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe()
1735 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe()
1738 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe()
1739 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe()
1740 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe()
1744 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe()
1764 	if (ce_state->dest_ring) {  in _ath10k_ce_free_pipe_64()
1766 				  (ce_state->dest_ring->nentries *  in _ath10k_ce_free_pipe_64()
1769 				  ce_state->dest_ring->base_addr_owner_space,  in _ath10k_ce_free_pipe_64()
1770 				  ce_state->dest_ring->base_addr_ce_space);  in _ath10k_ce_free_pipe_64()
1771 		kfree(ce_state->dest_ring);  in _ath10k_ce_free_pipe_64()
1775 	ce_state->dest_ring = NULL;  in _ath10k_ce_free_pipe_64()
1914 		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,  in ath10k_ce_alloc_pipe()
1917 		if (IS_ERR(ce_state->dest_ring)) {  in ath10k_ce_alloc_pipe()
1918 			ret = PTR_ERR(ce_state->dest_ring);  in ath10k_ce_alloc_pipe()
1921 			ce_state->dest_ring = NULL;  in ath10k_ce_alloc_pipe()