240 	struct iudma_ch			*iudma;  member
260 	struct iudma_ch			*iudma;  member
307 	struct iudma_ch			iudma[BCM63XX_NUM_IUDMA];  member
562 		udc->iudma[i].max_pkt = max_pkt;  in bcm63xx_ep_setup()
592 static void iudma_write(struct bcm63xx_udc *udc, struct iudma_ch *iudma,  in iudma_write()  argument
597 	const int max_bd_bytes = !irq_coalesce && !iudma->is_tx ?  in iudma_write()
598 		iudma->max_pkt : IUDMA_MAX_FRAGMENT;  in iudma_write()
600 	iudma->n_bds_used = 0;  in iudma_write()
602 	breq->iudma = iudma;  in iudma_write()
604 	if ((bytes_left % iudma->max_pkt == 0) && bytes_left && breq->req.zero)  in iudma_write()
608 		struct bcm_enet_desc *d = iudma->write_bd;  in iudma_write()
612 		if (d == iudma->end_bd) {  in iudma_write()
614 			iudma->write_bd = iudma->bd_ring;  in iudma_write()
616 			iudma->write_bd++;  in iudma_write()
618 		iudma->n_bds_used++;  in iudma_write()
640 		if (!iudma->is_tx || iudma->n_bds_used == iudma->n_bds ||  in iudma_write()
656 			ENETDMAC_CHANCFG_REG, iudma->ch_idx);  in iudma_write()
668 static int iudma_read(struct bcm63xx_udc *udc, struct iudma_ch *iudma)  in iudma_read()  argument
671 	struct bcm_enet_desc *d = iudma->read_bd;  in iudma_read()
673 	if (!iudma->n_bds_used)  in iudma_read()
676 	for (i = 0; i < iudma->n_bds_used; i++) {  in iudma_read()
686 		if (d == iudma->end_bd)  in iudma_read()
687 			d = iudma->bd_ring;  in iudma_read()
692 	iudma->read_bd = d;  in iudma_read()
693 	iudma->n_bds_used = 0;  in iudma_read()
702 static void iudma_reset_channel(struct bcm63xx_udc *udc, struct iudma_ch *iudma)  in iudma_reset_channel()  argument
706 	int ch_idx = iudma->ch_idx;  in iudma_reset_channel()
708 	if (!iudma->is_tx)  in iudma_reset_channel()
709 		bcm63xx_fifo_reset_ep(udc, max(0, iudma->ep_num));  in iudma_reset_channel()
719 		if (iudma->is_tx && iudma->ep_num >= 0)  in iudma_reset_channel()
720 			bcm63xx_fifo_reset_ep(udc, iudma->ep_num);  in iudma_reset_channel()
737 	for (d = iudma->bd_ring; d <= iudma->end_bd; d++)  in iudma_reset_channel()
741 	iudma->read_bd = iudma->write_bd = iudma->bd_ring;  in iudma_reset_channel()
742 	iudma->n_bds_used = 0;  in iudma_reset_channel()
749 	usb_dmas_writel(udc, iudma->bd_ring_dma, ENETDMAS_RSTART_REG, ch_idx);  in iudma_reset_channel()
760 	struct iudma_ch *iudma = &udc->iudma[ch_idx];  in iudma_init_channel()  local
765 	iudma->ep_num = cfg->ep_num;  in iudma_init_channel()
766 	iudma->ch_idx = ch_idx;  in iudma_init_channel()
767 	iudma->is_tx = !!(ch_idx & 0x01);  in iudma_init_channel()
768 	if (iudma->ep_num >= 0) {  in iudma_init_channel()
769 		bep = &udc->bep[iudma->ep_num];  in iudma_init_channel()
770 		bep->iudma = iudma;  in iudma_init_channel()
774 	iudma->bep = bep;  in iudma_init_channel()
775 	iudma->udc = udc;  in iudma_init_channel()
778 	if (iudma->ep_num <= 0)  in iudma_init_channel()
779 		iudma->enabled = true;  in iudma_init_channel()
781 	iudma->n_bds = n_bds;  in iudma_init_channel()
782 	iudma->bd_ring = dmam_alloc_coherent(udc->dev,  in iudma_init_channel()
784 		&iudma->bd_ring_dma, GFP_KERNEL);  in iudma_init_channel()
785 	if (!iudma->bd_ring)  in iudma_init_channel()
787 	iudma->end_bd = &iudma->bd_ring[n_bds - 1];  in iudma_init_channel()
808 		iudma_reset_channel(udc, &udc->iudma[i]);  in iudma_init()
828 		iudma_reset_channel(udc, &udc->iudma[i]);  in iudma_uninit()
1035 	struct iudma_ch *iudma = bep->iudma;  in bcm63xx_ep_enable()  local
1045 	if (iudma->enabled) {  in bcm63xx_ep_enable()
1050 	iudma->enabled = true;  in bcm63xx_ep_enable()
1053 	iudma_reset_channel(udc, iudma);  in bcm63xx_ep_enable()
1074 	struct iudma_ch *iudma = bep->iudma;  in bcm63xx_ep_disable()  local
1082 	if (!iudma->enabled) {  in bcm63xx_ep_disable()
1086 	iudma->enabled = false;  in bcm63xx_ep_disable()
1088 	iudma_reset_channel(udc, iudma);  in bcm63xx_ep_disable()
1093 						 iudma->is_tx);  in bcm63xx_ep_disable()
1098 			usb_gadget_giveback_request(&iudma->bep->ep, &breq->req);  in bcm63xx_ep_disable()
1177 	if (!bep->iudma->enabled) {  in bcm63xx_udc_queue()
1182 	rc = usb_gadget_map_request(&udc->gadget, req, bep->iudma->is_tx);  in bcm63xx_udc_queue()
1186 			iudma_write(udc, bep->iudma, breq);  in bcm63xx_udc_queue()
1218 	usb_gadget_unmap_request(&udc->gadget, &breq->req, bep->iudma->is_tx);  in bcm63xx_udc_dequeue()
1221 		iudma_reset_channel(udc, bep->iudma);  in bcm63xx_udc_dequeue()
1229 			iudma_write(udc, bep->iudma, next);  in bcm63xx_udc_dequeue()
1385 	struct iudma_ch *iudma = &udc->iudma[ch_idx];  in bcm63xx_ep0_map_write()  local
1392 	usb_gadget_map_request(&udc->gadget, req, iudma->is_tx);  in bcm63xx_ep0_map_write()
1393 	iudma_write(udc, iudma, breq);  in bcm63xx_ep0_map_write()
1550 		iudma_reset_channel(udc, &udc->iudma[IUDMA_EP0_RXCHAN]);  in bcm63xx_ep0_do_idle()
1627 			iudma_reset_channel(udc, &udc->iudma[IUDMA_EP0_TXCHAN]);  in bcm63xx_ep0_one_round()
1653 			iudma_reset_channel(udc, &udc->iudma[IUDMA_EP0_RXCHAN]);  in bcm63xx_ep0_one_round()
1672 			iudma_reset_channel(udc, &udc->iudma[IUDMA_EP0_TXCHAN]);  in bcm63xx_ep0_one_round()
2040 	struct iudma_ch *iudma = dev_id;  in bcm63xx_udc_data_isr()  local
2041 	struct bcm63xx_udc *udc = iudma->udc;  in bcm63xx_udc_data_isr()
2051 			ENETDMAC_IR_REG, iudma->ch_idx);  in bcm63xx_udc_data_isr()
2052 	bep = iudma->bep;  in bcm63xx_udc_data_isr()
2053 	rc = iudma_read(udc, iudma);  in bcm63xx_udc_data_isr()
2056 	if (iudma->ch_idx == IUDMA_EP0_RXCHAN ||  in bcm63xx_udc_data_isr()
2057 	    iudma->ch_idx == IUDMA_EP0_TXCHAN) {  in bcm63xx_udc_data_isr()
2074 				iudma_write(udc, iudma, breq);  in bcm63xx_udc_data_isr()
2095 					iudma_write(udc, iudma, next);  in bcm63xx_udc_data_isr()
2098 				iudma_write(udc, iudma, breq);  in bcm63xx_udc_data_isr()
2105 		usb_gadget_unmap_request(&udc->gadget, req, iudma->is_tx);  in bcm63xx_udc_data_isr()
2173 		struct iudma_ch *iudma = &udc->iudma[ch_idx];  in bcm63xx_iudma_dbg_show()  local
2203 		seq_printf(s, "  desc: %d/%d used", iudma->n_bds_used,  in bcm63xx_iudma_dbg_show()
2204 			   iudma->n_bds);  in bcm63xx_iudma_dbg_show()
2206 		if (iudma->bep)  in bcm63xx_iudma_dbg_show()
2207 			seq_printf(s, "; %zu queued\n", list_count_nodes(&iudma->bep->queue));  in bcm63xx_iudma_dbg_show()
2211 		for (i = 0; i < iudma->n_bds; i++) {  in bcm63xx_iudma_dbg_show()
2212 			struct bcm_enet_desc *d = &iudma->bd_ring[i];  in bcm63xx_iudma_dbg_show()
2218 			if (d == iudma->read_bd)  in bcm63xx_iudma_dbg_show()
2220 			if (d == iudma->write_bd)  in bcm63xx_iudma_dbg_show()
2334 				     dev_name(dev), &udc->iudma[i]) < 0)  in bcm63xx_udc_probe()