368 		struct cdnsp_ring *ep_ring;  in cdnsp_ring_doorbell_for_active_rings()  local
373 		ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);  in cdnsp_ring_doorbell_for_active_rings()
374 		if (!ep_ring)  in cdnsp_ring_doorbell_for_active_rings()
377 		if (!ep_ring->stream_active || ep_ring->stream_rejected)  in cdnsp_ring_doorbell_for_active_rings()
380 		list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,  in cdnsp_ring_doorbell_for_active_rings()
438 	struct cdnsp_ring *ep_ring;  in cdnsp_find_new_dequeue_state()  local
443 	ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);  in cdnsp_find_new_dequeue_state()
444 	if (!ep_ring)  in cdnsp_find_new_dequeue_state()
452 	new_seg = ep_ring->deq_seg;  in cdnsp_find_new_dequeue_state()
453 	new_deq = ep_ring->dequeue;  in cdnsp_find_new_dequeue_state()
479 		cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq);  in cdnsp_find_new_dequeue_state()
504 			     struct cdnsp_ring *ep_ring,  in cdnsp_td_to_noop()  argument
521 		cdnsp_next_trb(pdev, ep_ring, &seg, &trb);  in cdnsp_td_to_noop()
638 	struct cdnsp_ring *ep_ring;  in cdnsp_cmd_set_deq()  local
657 	ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id);  in cdnsp_cmd_set_deq()
659 	if (cdnsp_trb_is_link(ep_ring->dequeue)) {  in cdnsp_cmd_set_deq()
660 		ep_ring->deq_seg = ep_ring->deq_seg->next;  in cdnsp_cmd_set_deq()
661 		ep_ring->dequeue = ep_ring->deq_seg->trbs;  in cdnsp_cmd_set_deq()
664 	while (ep_ring->dequeue != deq_state->new_deq_ptr) {  in cdnsp_cmd_set_deq()
665 		ep_ring->num_trbs_free++;  in cdnsp_cmd_set_deq()
666 		ep_ring->dequeue++;  in cdnsp_cmd_set_deq()
668 		if (cdnsp_trb_is_link(ep_ring->dequeue)) {  in cdnsp_cmd_set_deq()
669 			if (ep_ring->dequeue == deq_state->new_deq_ptr)  in cdnsp_cmd_set_deq()
672 			ep_ring->deq_seg = ep_ring->deq_seg->next;  in cdnsp_cmd_set_deq()
673 			ep_ring->dequeue = ep_ring->deq_seg->trbs;  in cdnsp_cmd_set_deq()
696 	struct cdnsp_ring *ep_ring;  in cdnsp_remove_request()  local
708 	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_remove_request()
726 		cdnsp_td_to_noop(pdev, ep_ring, cur_td, false);  in cdnsp_remove_request()
733 	ep_ring->num_tds--;  in cdnsp_remove_request()
746 	cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td);  in cdnsp_remove_request()
893 			     struct cdnsp_ring *ep_ring,  in cdnsp_td_cleanup()  argument
899 	cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td);  in cdnsp_td_cleanup()
911 	ep_ring->num_tds--;  in cdnsp_td_cleanup()
923 	struct cdnsp_ring *ep_ring;  in cdnsp_finish_td()  local
926 	ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in cdnsp_finish_td()
941 	while (ep_ring->dequeue != td->last_trb)  in cdnsp_finish_td()
942 		cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_finish_td()
944 	cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_finish_td()
946 	cdnsp_td_cleanup(pdev, td, ep_ring, status);  in cdnsp_finish_td()
1001 	struct cdnsp_ring *ep_ring;  in cdnsp_process_ctrl_td()  local
1006 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_process_ctrl_td()
1025 		td = list_entry(ep_ring->td_list.next, struct cdnsp_td,  in cdnsp_process_ctrl_td()
1029 		cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state,  in cdnsp_process_ctrl_td()
1052 	struct cdnsp_ring *ep_ring;  in cdnsp_process_isoc_td()  local
1056 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_process_isoc_td()
1095 		td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb);  in cdnsp_process_isoc_td()
1112 	struct cdnsp_ring *ep_ring;  in cdnsp_skip_isoc_td()  local
1114 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_skip_isoc_td()
1119 	while (ep_ring->dequeue != td->last_trb)  in cdnsp_skip_isoc_td()
1120 		cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_skip_isoc_td()
1122 	cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_skip_isoc_td()
1124 	cdnsp_td_cleanup(pdev, td, ep_ring, &status);  in cdnsp_skip_isoc_td()
1138 	struct cdnsp_ring *ep_ring;  in cdnsp_process_bulk_intr_td()  local
1141 	ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in cdnsp_process_bulk_intr_td()
1165 		ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) +  in cdnsp_process_bulk_intr_td()
1179 	struct cdnsp_ring *ep_ring;  in cdnsp_handle_tx_nrdy()  local
1200 			ep_ring = pep->stream_info.stream_rings[cur_stream];  in cdnsp_handle_tx_nrdy()
1201 			ep_ring->stream_active = 1;  in cdnsp_handle_tx_nrdy()
1202 			ep_ring->stream_rejected = 0;  in cdnsp_handle_tx_nrdy()
1210 		ep_ring = pep->stream_info.stream_rings[dev_sid];  in cdnsp_handle_tx_nrdy()
1211 		ep_ring->stream_active = 0;  in cdnsp_handle_tx_nrdy()
1212 		ep_ring->stream_rejected = 1;  in cdnsp_handle_tx_nrdy()
1214 		list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,  in cdnsp_handle_tx_nrdy()
1233 	struct cdnsp_ring *ep_ring;  in cdnsp_handle_tx_event()  local
1249 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_handle_tx_event()
1265 	if (!ep_ring) {  in cdnsp_handle_tx_event()
1308 		if (list_empty(&ep_ring->td_list)) {  in cdnsp_handle_tx_event()
1318 			      ep_ring->last_td_was_short))  in cdnsp_handle_tx_event()
1319 				trace_cdnsp_trb_without_td(ep_ring,  in cdnsp_handle_tx_event()
1330 		td = list_entry(ep_ring->td_list.next, struct cdnsp_td,  in cdnsp_handle_tx_event()
1334 		ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg,  in cdnsp_handle_tx_event()
1335 					 ep_ring->dequeue, td->last_trb,  in cdnsp_handle_tx_event()
1344 			trace_cdnsp_handle_transfer(ep_ring,  in cdnsp_handle_tx_event()
1382 			ep_ring->last_td_was_short = true;  in cdnsp_handle_tx_event()
1384 			ep_ring->last_td_was_short = false;  in cdnsp_handle_tx_event()
1626 			      struct cdnsp_ring *ep_ring,  in cdnsp_prepare_ring()  argument
1645 		if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs))  in cdnsp_prepare_ring()
1650 		num_trbs_needed = num_trbs - ep_ring->num_trbs_free;  in cdnsp_prepare_ring()
1651 		if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed,  in cdnsp_prepare_ring()
1658 	while (cdnsp_trb_is_link(ep_ring->enqueue)) {  in cdnsp_prepare_ring()
1659 		ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN);  in cdnsp_prepare_ring()
1662 		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);  in cdnsp_prepare_ring()
1665 		if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue))  in cdnsp_prepare_ring()
1666 			ep_ring->cycle_state ^= 1;  in cdnsp_prepare_ring()
1667 		ep_ring->enq_seg = ep_ring->enq_seg->next;  in cdnsp_prepare_ring()
1668 		ep_ring->enqueue = ep_ring->enq_seg->trbs;  in cdnsp_prepare_ring()
1677 	struct cdnsp_ring *ep_ring;  in cdnsp_prepare_transfer()  local
1680 	ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep,  in cdnsp_prepare_transfer()
1682 	if (!ep_ring)  in cdnsp_prepare_transfer()
1685 	ret = cdnsp_prepare_ring(pdev, ep_ring,  in cdnsp_prepare_transfer()
1695 	list_add_tail(&preq->td.td_list, &ep_ring->td_list);  in cdnsp_prepare_transfer()
1696 	ep_ring->num_tds++;  in cdnsp_prepare_transfer()
1699 	preq->td.start_seg = ep_ring->enq_seg;  in cdnsp_prepare_transfer()
1700 	preq->td.first_trb = ep_ring->enqueue;  in cdnsp_prepare_transfer()
2046 	struct cdnsp_ring *ep_ring;  in cdnsp_queue_ctrl_tx()  local
2051 	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_queue_ctrl_tx()
2052 	if (!ep_ring)  in cdnsp_queue_ctrl_tx()
2085 		cdnsp_queue_trb(pdev, ep_ring, true,  in cdnsp_queue_ctrl_tx()
2088 				field | ep_ring->cycle_state |  in cdnsp_queue_ctrl_tx()
2098 			cdnsp_queue_trb(pdev, ep_ring, true,  in cdnsp_queue_ctrl_tx()
2101 					field | ep_ring->cycle_state |  in cdnsp_queue_ctrl_tx()
2110 	preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_ctrl_tx()
2114 		field = ep_ring->cycle_state;  in cdnsp_queue_ctrl_tx()
2116 		field = (ep_ring->cycle_state ^ 1);  in cdnsp_queue_ctrl_tx()
2128 	cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0),  in cdnsp_queue_ctrl_tx()
2226 	struct cdnsp_ring *ep_ring;  in cdnsp_queue_isoc_tx()  local
2235 	ep_ring = preq->pep->ring;  in cdnsp_queue_isoc_tx()
2255 	start_trb = &ep_ring->enqueue->generic;  in cdnsp_queue_isoc_tx()
2256 	start_cycle = ep_ring->cycle_state;  in cdnsp_queue_isoc_tx()
2302 			field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state;  in cdnsp_queue_isoc_tx()
2318 			preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_isoc_tx()
2322 		cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming,  in cdnsp_queue_isoc_tx()
2361 	ep_ring->num_tds--;  in cdnsp_queue_isoc_tx()
2370 	preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_isoc_tx()
2372 	cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true);  in cdnsp_queue_isoc_tx()
2375 	ep_ring->enqueue = preq->td.first_trb;  in cdnsp_queue_isoc_tx()
2376 	ep_ring->enq_seg = preq->td.start_seg;  in cdnsp_queue_isoc_tx()
2377 	ep_ring->cycle_state = start_cycle;  in cdnsp_queue_isoc_tx()