636 	struct xhci_ring *ep_ring;  in xhci_move_dequeue_past_td()  local
648 	ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,  in xhci_move_dequeue_past_td()
650 	if (!ep_ring) {  in xhci_move_dequeue_past_td()
657 	new_seg = ep_ring->deq_seg;  in xhci_move_dequeue_past_td()
658 	new_deq = ep_ring->dequeue;  in xhci_move_dequeue_past_td()
681 		next_trb(xhci, ep_ring, &new_seg, &new_deq);  in xhci_move_dequeue_past_td()
743 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,  in td_to_noop()  argument
759 		next_trb(xhci, ep_ring, &seg, &trb);  in td_to_noop()
818 			   struct xhci_ring *ep_ring, int status)  in xhci_td_cleanup()  argument
826 	xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);  in xhci_td_cleanup()
1290 		struct xhci_ring *ep_ring,  in update_ring_for_set_deq_completion()  argument
1295 	dequeue_temp = ep_ring->dequeue;  in update_ring_for_set_deq_completion()
1303 	if (trb_is_link(ep_ring->dequeue)) {  in update_ring_for_set_deq_completion()
1304 		ep_ring->deq_seg = ep_ring->deq_seg->next;  in update_ring_for_set_deq_completion()
1305 		ep_ring->dequeue = ep_ring->deq_seg->trbs;  in update_ring_for_set_deq_completion()
1308 	while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {  in update_ring_for_set_deq_completion()
1310 		ep_ring->dequeue++;  in update_ring_for_set_deq_completion()
1311 		if (trb_is_link(ep_ring->dequeue)) {  in update_ring_for_set_deq_completion()
1312 			if (ep_ring->dequeue ==  in update_ring_for_set_deq_completion()
1315 			ep_ring->deq_seg = ep_ring->deq_seg->next;  in update_ring_for_set_deq_completion()
1316 			ep_ring->dequeue = ep_ring->deq_seg->trbs;  in update_ring_for_set_deq_completion()
1318 		if (ep_ring->dequeue == dequeue_temp) {  in update_ring_for_set_deq_completion()
1337 	struct xhci_ring *ep_ring;  in xhci_handle_cmd_set_deq()  local
1350 	ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id);  in xhci_handle_cmd_set_deq()
1351 	if (!ep_ring) {  in xhci_handle_cmd_set_deq()
1427 				ep_ring, ep_index);  in xhci_handle_cmd_set_deq()
1437 		ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);  in xhci_handle_cmd_set_deq()
1442 			xhci_td_cleanup(ep->xhci, td, ep_ring, td->status);  in xhci_handle_cmd_set_deq()
2188 		     struct xhci_ring *ep_ring, struct xhci_td *td,  in finish_td()  argument
2261 	ep_ring->dequeue = td->last_trb;  in finish_td()
2262 	ep_ring->deq_seg = td->last_trb_seg;  in finish_td()
2263 	inc_deq(xhci, ep_ring);  in finish_td()
2265 	return xhci_td_cleanup(xhci, td, ep_ring, td->status);  in finish_td()
2287 		struct xhci_ring *ep_ring,  struct xhci_td *td,  in process_ctrl_td()  argument
2375 	return finish_td(xhci, ep, ep_ring, td, trb_comp_code);  in process_ctrl_td()
2382 		struct xhci_ring *ep_ring, struct xhci_td *td,  in process_isoc_td()  argument
2480 	return finish_td(xhci, ep, ep_ring, td, trb_comp_code);  in process_isoc_td()
2512 		struct xhci_ring *ep_ring, struct xhci_td *td,  in process_bulk_intr_td()  argument
2545 		td->urb->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb);  in process_bulk_intr_td()
2566 			sum_trb_lengths(xhci, ep_ring, ep_trb) +  in process_bulk_intr_td()
2575 	return finish_td(xhci, ep, ep_ring, td, trb_comp_code);  in process_bulk_intr_td()
2615 	struct xhci_ring *ep_ring;  in handle_tx_event()  local
2637 	ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);  in handle_tx_event()
2647 	if (!ep_ring)  in handle_tx_event()
2659 				 slot_id, ep_index, ep_ring->last_td_was_short);  in handle_tx_event()
2791 	td = list_first_entry_or_null(&ep_ring->td_list, struct xhci_td, td_list);  in handle_tx_event()
2795 		ep_ring->dequeue = td->last_trb;  in handle_tx_event()
2796 		ep_ring->deq_seg = td->last_trb_seg;  in handle_tx_event()
2797 		inc_deq(xhci, ep_ring);  in handle_tx_event()
2798 		xhci_td_cleanup(xhci, td, ep_ring, td->status);  in handle_tx_event()
2801 	if (list_empty(&ep_ring->td_list)) {  in handle_tx_event()
2810 		    !ep_ring->last_td_was_short) {  in handle_tx_event()
2820 		td = list_first_entry(&ep_ring->td_list, struct xhci_td,  in handle_tx_event()
2830 				if (!list_empty(&ep_ring->td_list))  in handle_tx_event()
2857 			    ep_ring->last_td_was_short) {  in handle_tx_event()
2858 				ep_ring->last_td_was_short = false;  in handle_tx_event()
2887 		ep_ring->last_td_was_short = true;  in handle_tx_event()
2889 		ep_ring->last_td_was_short = false;  in handle_tx_event()
2892 	trace_xhci_handle_transfer(ep_ring, (struct xhci_generic_trb *) ep_trb);  in handle_tx_event()
2908 		process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);  in handle_tx_event()
2910 		process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);  in handle_tx_event()
2912 		process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);  in handle_tx_event()
3174 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,  in prepare_ring()  argument
3209 	if (ep_ring != xhci->cmd_ring) {  in prepare_ring()
3210 		new_segs = xhci_ring_expansion_needed(xhci, ep_ring, num_trbs);  in prepare_ring()
3211 	} else if (xhci_num_trbs_free(ep_ring) <= num_trbs) {  in prepare_ring()
3219 		if (xhci_ring_expansion(xhci, ep_ring, new_segs, mem_flags)) {  in prepare_ring()
3225 	while (trb_is_link(ep_ring->enqueue)) {  in prepare_ring()
3229 		if (!xhci_link_chain_quirk(xhci, ep_ring->type))  in prepare_ring()
3230 			ep_ring->enqueue->link.control &=  in prepare_ring()
3233 			ep_ring->enqueue->link.control |=  in prepare_ring()
3237 		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);  in prepare_ring()
3240 		if (link_trb_toggles_cycle(ep_ring->enqueue))  in prepare_ring()
3241 			ep_ring->cycle_state ^= 1;  in prepare_ring()
3243 		ep_ring->enq_seg = ep_ring->enq_seg->next;  in prepare_ring()
3244 		ep_ring->enqueue = ep_ring->enq_seg->trbs;  in prepare_ring()
3247 		if (link_trb_count++ > ep_ring->num_segs) {  in prepare_ring()
3253 	if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) {  in prepare_ring()
3273 	struct xhci_ring *ep_ring;  in prepare_transfer()  local
3276 	ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index,  in prepare_transfer()
3278 	if (!ep_ring) {  in prepare_transfer()
3284 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),  in prepare_transfer()
3303 	list_add_tail(&td->td_list, &ep_ring->td_list);  in prepare_transfer()
3304 	td->start_seg = ep_ring->enq_seg;  in prepare_transfer()
3305 	td->first_trb = ep_ring->enqueue;  in prepare_transfer()
3709 	struct xhci_ring *ep_ring;  in xhci_queue_ctrl_tx()  local
3719 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in xhci_queue_ctrl_tx()
3720 	if (!ep_ring)  in xhci_queue_ctrl_tx()
3753 	start_trb = &ep_ring->enqueue->generic;  in xhci_queue_ctrl_tx()
3754 	start_cycle = ep_ring->cycle_state;  in xhci_queue_ctrl_tx()
3774 	queue_trb(xhci, ep_ring, true,  in xhci_queue_ctrl_tx()
3810 		queue_trb(xhci, ep_ring, true,  in xhci_queue_ctrl_tx()
3814 				field | ep_ring->cycle_state);  in xhci_queue_ctrl_tx()
3818 	td->last_trb = ep_ring->enqueue;  in xhci_queue_ctrl_tx()
3819 	td->last_trb_seg = ep_ring->enq_seg;  in xhci_queue_ctrl_tx()
3827 	queue_trb(xhci, ep_ring, false,  in xhci_queue_ctrl_tx()
3832 			field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);  in xhci_queue_ctrl_tx()
4003 	struct xhci_ring *ep_ring;  in xhci_queue_isoc_tx()  local
4019 	ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;  in xhci_queue_isoc_tx()
4028 	start_trb = &ep_ring->enqueue->generic;  in xhci_queue_isoc_tx()
4029 	start_cycle = ep_ring->cycle_state;  in xhci_queue_isoc_tx()
4079 			(i ? ep_ring->cycle_state : !start_cycle);  in xhci_queue_isoc_tx()
4092 					ep_ring->cycle_state;  in xhci_queue_isoc_tx()
4104 				td->last_trb = ep_ring->enqueue;  in xhci_queue_isoc_tx()
4105 				td->last_trb_seg = ep_ring->enq_seg;  in xhci_queue_isoc_tx()
4130 			queue_trb(xhci, ep_ring, more_trbs_coming,  in xhci_queue_isoc_tx()
4173 	urb_priv->td[0].last_trb = ep_ring->enqueue;  in xhci_queue_isoc_tx()
4175 	td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);  in xhci_queue_isoc_tx()
4178 	ep_ring->enqueue = urb_priv->td[0].first_trb;  in xhci_queue_isoc_tx()
4179 	ep_ring->enq_seg = urb_priv->td[0].start_seg;  in xhci_queue_isoc_tx()
4180 	ep_ring->cycle_state = start_cycle;  in xhci_queue_isoc_tx()
4196 	struct xhci_ring *ep_ring;  in xhci_queue_isoc_tx_prepare()  local
4206 	ep_ring = xdev->eps[ep_index].ring;  in xhci_queue_isoc_tx_prepare()
4217 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),  in xhci_queue_isoc_tx_prepare()
4229 	if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {  in xhci_queue_isoc_tx_prepare()