259 void ath11k_ce_stop_shadow_timers(struct ath11k_base *ab)  in ath11k_ce_stop_shadow_timers()  argument
263 	if (!ab->hw_params.supports_shadow_regs)  in ath11k_ce_stop_shadow_timers()
266 	for (i = 0; i < ab->hw_params.ce_count; i++)  in ath11k_ce_stop_shadow_timers()
268 			ath11k_dp_shadow_stop_timer(ab, &ab->ce.hp_timer[i]);  in ath11k_ce_stop_shadow_timers()
274 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_rx_buf_enqueue_pipe()  local
282 	lockdep_assert_held(&ab->ce.ce_lock);  in ath11k_ce_rx_buf_enqueue_pipe()
286 	srng = &ab->hal.srng_list[ring->hal_ring_id];  in ath11k_ce_rx_buf_enqueue_pipe()
290 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_rx_buf_enqueue_pipe()
292 	if (unlikely(ath11k_hal_srng_src_num_free(ab, srng, false) < 1)) {  in ath11k_ce_rx_buf_enqueue_pipe()
297 	desc = ath11k_hal_srng_src_get_next_entry(ab, srng);  in ath11k_ce_rx_buf_enqueue_pipe()
313 	ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_rx_buf_enqueue_pipe()
322 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_rx_post_pipe()  local
330 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_rx_post_pipe()
340 		paddr = dma_map_single(ab->dev, skb->data,  in ath11k_ce_rx_post_pipe()
343 		if (unlikely(dma_mapping_error(ab->dev, paddr))) {  in ath11k_ce_rx_post_pipe()
344 			ath11k_warn(ab, "failed to dma map ce rx buf\n");  in ath11k_ce_rx_post_pipe()
355 			ath11k_warn(ab, "failed to enqueue rx buf: %d\n", ret);  in ath11k_ce_rx_post_pipe()
356 			dma_unmap_single(ab->dev, paddr,  in ath11k_ce_rx_post_pipe()
365 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_rx_post_pipe()
372 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_completed_recv_next()  local
379 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_recv_next()
384 	srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];  in ath11k_ce_completed_recv_next()
388 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_completed_recv_next()
390 	desc = ath11k_hal_srng_dst_get_next_entry(ab, srng);  in ath11k_ce_completed_recv_next()
410 	ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_completed_recv_next()
414 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_recv_next()
421 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_recv_process_cb()  local
430 		dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr,  in ath11k_ce_recv_process_cb()
434 			ath11k_warn(ab, "rxed more than expected (nbytes %d, max %d)",  in ath11k_ce_recv_process_cb()
445 		ath11k_dbg(ab, ATH11K_DBG_CE, "rx ce pipe %d len %d\n",  in ath11k_ce_recv_process_cb()
447 		pipe->recv_cb(ab, skb);  in ath11k_ce_recv_process_cb()
452 		ath11k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",  in ath11k_ce_recv_process_cb()
454 		mod_timer(&ab->rx_replenish_retry,  in ath11k_ce_recv_process_cb()
461 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_completed_send_next()  local
468 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_send_next()
473 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath11k_ce_completed_send_next()
477 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_completed_send_next()
479 	desc = ath11k_hal_srng_src_reap_next(ab, srng);  in ath11k_ce_completed_send_next()
495 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_completed_send_next()
502 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_tx_process_cb()  local
511 		dma_unmap_single(ab->dev, ATH11K_SKB_CB(skb)->paddr, skb->len,  in ath11k_ce_tx_process_cb()
514 		if ((!pipe->send_cb) || ab->hw_params.credit_flow) {  in ath11k_ce_tx_process_cb()
523 		ath11k_dbg(ab, ATH11K_DBG_CE, "tx ce pipe %d len %d\n",  in ath11k_ce_tx_process_cb()
525 		pipe->send_cb(ab, skb);  in ath11k_ce_tx_process_cb()
529 static void ath11k_ce_srng_msi_ring_params_setup(struct ath11k_base *ab, u32 ce_id,  in ath11k_ce_srng_msi_ring_params_setup()  argument
539 	ret = ath11k_get_user_msi_vector(ab, "CE",  in ath11k_ce_srng_msi_ring_params_setup()
546 	ath11k_get_msi_address(ab, &addr_lo, &addr_hi);  in ath11k_ce_srng_msi_ring_params_setup()
547 	ath11k_get_ce_msi_idx(ab, ce_id, &msi_data_idx);  in ath11k_ce_srng_msi_ring_params_setup()
555 static int ath11k_ce_init_ring(struct ath11k_base *ab,  in ath11k_ce_init_ring()  argument
566 	if (!(CE_ATTR_DIS_INTR & ab->hw_params.host_ce_config[ce_id].flags))  in ath11k_ce_init_ring()
567 		ath11k_ce_srng_msi_ring_params_setup(ab, ce_id, &params);  in ath11k_ce_init_ring()
571 		if (!(CE_ATTR_DIS_INTR & ab->hw_params.host_ce_config[ce_id].flags))  in ath11k_ce_init_ring()
575 		params.max_buffer_len = ab->hw_params.host_ce_config[ce_id].src_sz_max;  in ath11k_ce_init_ring()
576 		if (!(ab->hw_params.host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {  in ath11k_ce_init_ring()
583 		if (!(ab->hw_params.host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {  in ath11k_ce_init_ring()
589 		ath11k_warn(ab, "Invalid CE ring type %d\n", type);  in ath11k_ce_init_ring()
595 	ret = ath11k_hal_srng_setup(ab, type, ce_id, 0, &params);  in ath11k_ce_init_ring()
597 		ath11k_warn(ab, "failed to setup srng: %d ring_id %d\n",  in ath11k_ce_init_ring()
604 	if (ab->hw_params.supports_shadow_regs &&  in ath11k_ce_init_ring()
606 		ath11k_dp_shadow_init_timer(ab, &ab->ce.hp_timer[ce_id],  in ath11k_ce_init_ring()
614 ath11k_ce_alloc_ring(struct ath11k_base *ab, int nentries, int desc_sz)  in ath11k_ce_alloc_ring()  argument
630 		dma_alloc_coherent(ab->dev,  in ath11k_ce_alloc_ring()
650 static int ath11k_ce_alloc_pipe(struct ath11k_base *ab, int ce_id)  in ath11k_ce_alloc_pipe()  argument
652 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath11k_ce_alloc_pipe()
653 	const struct ce_attr *attr = &ab->hw_params.host_ce_config[ce_id];  in ath11k_ce_alloc_pipe()
664 		ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);  in ath11k_ce_alloc_pipe()
674 		ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);  in ath11k_ce_alloc_pipe()
680 		ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz);  in ath11k_ce_alloc_pipe()
689 void ath11k_ce_per_engine_service(struct ath11k_base *ab, u16 ce_id)  in ath11k_ce_per_engine_service()  argument
691 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath11k_ce_per_engine_service()
692 	const struct ce_attr *attr = &ab->hw_params.host_ce_config[ce_id];  in ath11k_ce_per_engine_service()
701 void ath11k_ce_poll_send_completed(struct ath11k_base *ab, u8 pipe_id)  in ath11k_ce_poll_send_completed()  argument
703 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath11k_ce_poll_send_completed()
704 	const struct ce_attr *attr =  &ab->hw_params.host_ce_config[pipe_id];  in ath11k_ce_poll_send_completed()
711 int ath11k_ce_send(struct ath11k_base *ab, struct sk_buff *skb, u8 pipe_id,  in ath11k_ce_send()  argument
714 	struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath11k_ce_send()
728 		spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
739 		spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
742 			ath11k_ce_poll_send_completed(ab, pipe->pipe_num);  in ath11k_ce_send()
745 	if (test_bit(ATH11K_FLAG_CRASH_FLUSH, &ab->dev_flags))  in ath11k_ce_send()
748 	spin_lock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
753 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath11k_ce_send()
757 	ath11k_hal_srng_access_begin(ab, srng);  in ath11k_ce_send()
759 	if (unlikely(ath11k_hal_srng_src_num_free(ab, srng, false) < 1)) {  in ath11k_ce_send()
760 		ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_send()
765 	desc = ath11k_hal_srng_src_get_next_reaped(ab, srng);  in ath11k_ce_send()
767 		ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_send()
782 	ath11k_hal_srng_access_end(ab, srng);  in ath11k_ce_send()
785 		ath11k_dp_shadow_start_timer(ab, srng, &ab->ce.hp_timer[pipe_id]);  in ath11k_ce_send()
789 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
796 	spin_unlock_bh(&ab->ce.ce_lock);  in ath11k_ce_send()
803 	struct ath11k_base *ab = pipe->ab;  in ath11k_ce_rx_pipe_cleanup()  local
817 		dma_unmap_single(ab->dev, ATH11K_SKB_RXCB(skb)->paddr,  in ath11k_ce_rx_pipe_cleanup()
823 static void ath11k_ce_shadow_config(struct ath11k_base *ab)  in ath11k_ce_shadow_config()  argument
827 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_shadow_config()
828 		if (ab->hw_params.host_ce_config[i].src_nentries)  in ath11k_ce_shadow_config()
829 			ath11k_hal_srng_update_shadow_config(ab,  in ath11k_ce_shadow_config()
832 		if (ab->hw_params.host_ce_config[i].dest_nentries) {  in ath11k_ce_shadow_config()
833 			ath11k_hal_srng_update_shadow_config(ab,  in ath11k_ce_shadow_config()
836 			ath11k_hal_srng_update_shadow_config(ab,  in ath11k_ce_shadow_config()
842 void ath11k_ce_get_shadow_config(struct ath11k_base *ab,  in ath11k_ce_get_shadow_config()  argument
845 	if (!ab->hw_params.supports_shadow_regs)  in ath11k_ce_get_shadow_config()
848 	ath11k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);  in ath11k_ce_get_shadow_config()
858 	ath11k_hal_srng_shadow_config(ab);  in ath11k_ce_get_shadow_config()
859 	ath11k_ce_shadow_config(ab);  in ath11k_ce_get_shadow_config()
862 	ath11k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);  in ath11k_ce_get_shadow_config()
866 void ath11k_ce_cleanup_pipes(struct ath11k_base *ab)  in ath11k_ce_cleanup_pipes()  argument
871 	ath11k_ce_stop_shadow_timers(ab);  in ath11k_ce_cleanup_pipes()
873 	for (pipe_num = 0; pipe_num < ab->hw_params.ce_count; pipe_num++) {  in ath11k_ce_cleanup_pipes()
874 		pipe = &ab->ce.ce_pipe[pipe_num];  in ath11k_ce_cleanup_pipes()
878 		ath11k_ce_poll_send_completed(ab, pipe_num);  in ath11k_ce_cleanup_pipes()
885 void ath11k_ce_rx_post_buf(struct ath11k_base *ab)  in ath11k_ce_rx_post_buf()  argument
891 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_rx_post_buf()
892 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_rx_post_buf()
898 			ath11k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",  in ath11k_ce_rx_post_buf()
900 			mod_timer(&ab->rx_replenish_retry,  in ath11k_ce_rx_post_buf()
911 	struct ath11k_base *ab = from_timer(ab, t, rx_replenish_retry);  in ath11k_ce_rx_replenish_retry()  local
913 	ath11k_ce_rx_post_buf(ab);  in ath11k_ce_rx_replenish_retry()
916 int ath11k_ce_init_pipes(struct ath11k_base *ab)  in ath11k_ce_init_pipes()  argument
922 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_init_pipes()
923 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_init_pipes()
926 			ret = ath11k_ce_init_ring(ab, pipe->src_ring, i,  in ath11k_ce_init_pipes()
929 				ath11k_warn(ab, "failed to init src ring: %d\n",  in ath11k_ce_init_pipes()
940 			ret = ath11k_ce_init_ring(ab, pipe->dest_ring, i,  in ath11k_ce_init_pipes()
943 				ath11k_warn(ab, "failed to init dest ring: %d\n",  in ath11k_ce_init_pipes()
957 			ret = ath11k_ce_init_ring(ab, pipe->status_ring, i,  in ath11k_ce_init_pipes()
960 				ath11k_warn(ab, "failed to init dest status ing: %d\n",  in ath11k_ce_init_pipes()
974 void ath11k_ce_free_pipes(struct ath11k_base *ab)  in ath11k_ce_free_pipes()  argument
981 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_free_pipes()
982 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_free_pipes()
985 			ath11k_dp_shadow_stop_timer(ab, &ab->ce.hp_timer[i]);  in ath11k_ce_free_pipes()
990 			dma_free_coherent(ab->dev,  in ath11k_ce_free_pipes()
1002 			dma_free_coherent(ab->dev,  in ath11k_ce_free_pipes()
1015 			dma_free_coherent(ab->dev,  in ath11k_ce_free_pipes()
1027 int ath11k_ce_alloc_pipes(struct ath11k_base *ab)  in ath11k_ce_alloc_pipes()  argument
1034 	spin_lock_init(&ab->ce.ce_lock);  in ath11k_ce_alloc_pipes()
1036 	for (i = 0; i < ab->hw_params.ce_count; i++) {  in ath11k_ce_alloc_pipes()
1037 		attr = &ab->hw_params.host_ce_config[i];  in ath11k_ce_alloc_pipes()
1038 		pipe = &ab->ce.ce_pipe[i];  in ath11k_ce_alloc_pipes()
1040 		pipe->ab = ab;  in ath11k_ce_alloc_pipes()
1043 		ret = ath11k_ce_alloc_pipe(ab, i);  in ath11k_ce_alloc_pipes()
1046 			ath11k_ce_free_pipes(ab);  in ath11k_ce_alloc_pipes()
1074 int ath11k_ce_get_attr_flags(struct ath11k_base *ab, int ce_id)  in ath11k_ce_get_attr_flags()  argument
1076 	if (ce_id >= ab->hw_params.ce_count)  in ath11k_ce_get_attr_flags()
1079 	return ab->hw_params.host_ce_config[ce_id].flags;  in ath11k_ce_get_attr_flags()