489 static inline bool atmel_spi_is_vmalloc_xfer(struct spi_transfer *xfer)  in atmel_spi_is_vmalloc_xfer()  argument
491 	return is_vmalloc_addr(xfer->tx_buf) || is_vmalloc_addr(xfer->rx_buf);  in atmel_spi_is_vmalloc_xfer()
495 				struct spi_transfer *xfer)  in atmel_spi_use_dma()  argument
497 	return as->use_dma && xfer->len >= DMA_MIN_BYTES;  in atmel_spi_use_dma()
502 			      struct spi_transfer *xfer)  in atmel_spi_can_dma()  argument
507 		return atmel_spi_use_dma(as, xfer) &&  in atmel_spi_can_dma()
508 			!atmel_spi_is_vmalloc_xfer(xfer);  in atmel_spi_can_dma()
510 		return atmel_spi_use_dma(as, xfer);  in atmel_spi_can_dma()
653 				       struct spi_transfer *xfer)  in atmel_spi_next_xfer_single()  argument
656 	unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;  in atmel_spi_next_xfer_single()
667 	if (xfer->bits_per_word > 8)  in atmel_spi_next_xfer_single()
668 		spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));  in atmel_spi_next_xfer_single()
670 		spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));  in atmel_spi_next_xfer_single()
674 		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,  in atmel_spi_next_xfer_single()
675 		xfer->bits_per_word);  in atmel_spi_next_xfer_single()
685 				     struct spi_transfer *xfer)  in atmel_spi_next_xfer_fifo()  argument
689 	u32 offset = xfer->len - as->current_remaining_bytes;  in atmel_spi_next_xfer_fifo()
690 	const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset);  in atmel_spi_next_xfer_fifo()
691 	const u8  *bytes = (const u8  *)((u8 *)xfer->tx_buf + offset);  in atmel_spi_next_xfer_fifo()
698 	current_remaining_data = ((xfer->bits_per_word > 8) ?  in atmel_spi_next_xfer_fifo()
717 		if (xfer->bits_per_word > 8) {  in atmel_spi_next_xfer_fifo()
730 		if (xfer->bits_per_word > 8)  in atmel_spi_next_xfer_fifo()
741 		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,  in atmel_spi_next_xfer_fifo()
742 		xfer->bits_per_word);  in atmel_spi_next_xfer_fifo()
755 				    struct spi_transfer *xfer)  in atmel_spi_next_xfer_pio()  argument
760 		atmel_spi_next_xfer_fifo(host, xfer);  in atmel_spi_next_xfer_pio()
762 		atmel_spi_next_xfer_single(host, xfer);  in atmel_spi_next_xfer_pio()
769 				struct spi_transfer *xfer,  in atmel_spi_next_xfer_dma_submit()  argument
786 	*plen = xfer->len;  in atmel_spi_next_xfer_dma_submit()
788 	if (atmel_spi_dma_slave_config(as, xfer->bits_per_word))  in atmel_spi_next_xfer_dma_submit()
792 	if (atmel_spi_is_vmalloc_xfer(xfer) &&  in atmel_spi_next_xfer_dma_submit()
796 						     xfer->len,  in atmel_spi_next_xfer_dma_submit()
802 						 xfer->rx_sg.sgl,  in atmel_spi_next_xfer_dma_submit()
803 						 xfer->rx_sg.nents,  in atmel_spi_next_xfer_dma_submit()
811 	if (atmel_spi_is_vmalloc_xfer(xfer) &&  in atmel_spi_next_xfer_dma_submit()
813 		memcpy(as->addr_tx_bbuf, xfer->tx_buf, xfer->len);  in atmel_spi_next_xfer_dma_submit()
816 						     xfer->len, DMA_MEM_TO_DEV,  in atmel_spi_next_xfer_dma_submit()
821 						 xfer->tx_sg.sgl,  in atmel_spi_next_xfer_dma_submit()
822 						 xfer->tx_sg.nents,  in atmel_spi_next_xfer_dma_submit()
832 		xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma,  in atmel_spi_next_xfer_dma_submit()
833 		xfer->rx_buf, (unsigned long long)xfer->rx_dma);  in atmel_spi_next_xfer_dma_submit()
862 				struct spi_transfer *xfer,  in atmel_spi_next_xfer_data()  argument
867 	*rx_dma = xfer->rx_dma + xfer->len - *plen;  in atmel_spi_next_xfer_data()
868 	*tx_dma = xfer->tx_dma + xfer->len - *plen;  in atmel_spi_next_xfer_data()
875 				    struct spi_transfer *xfer)  in atmel_spi_set_xfer_speed()  argument
895 	scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz);  in atmel_spi_set_xfer_speed()
904 			xfer->speed_hz, scbr, bus_hz/255);  in atmel_spi_set_xfer_speed()
910 			xfer->speed_hz, scbr, bus_hz);  in atmel_spi_set_xfer_speed()
916 	xfer->effective_speed_hz = bus_hz / scbr;  in atmel_spi_set_xfer_speed()
926 					struct spi_transfer *xfer)  in atmel_spi_pdc_next_xfer()  argument
935 	atmel_spi_next_xfer_data(host, xfer, &tx_dma, &rx_dma, &len);  in atmel_spi_pdc_next_xfer()
941 	if (xfer->bits_per_word > 8)  in atmel_spi_pdc_next_xfer()
948 		xfer, xfer->len, xfer->tx_buf,  in atmel_spi_pdc_next_xfer()
949 		(unsigned long long)xfer->tx_dma, xfer->rx_buf,  in atmel_spi_pdc_next_xfer()
950 		(unsigned long long)xfer->rx_dma);  in atmel_spi_pdc_next_xfer()
954 		atmel_spi_next_xfer_data(host, xfer, &tx_dma, &rx_dma, &len);  in atmel_spi_pdc_next_xfer()
960 		if (xfer->bits_per_word > 8)  in atmel_spi_pdc_next_xfer()
967 			xfer, xfer->len, xfer->tx_buf,  in atmel_spi_pdc_next_xfer()
968 			(unsigned long long)xfer->tx_dma, xfer->rx_buf,  in atmel_spi_pdc_next_xfer()
969 			(unsigned long long)xfer->rx_dma);  in atmel_spi_pdc_next_xfer()
992 atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_dma_map_xfer()  argument
996 	xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS;  in atmel_spi_dma_map_xfer()
997 	if (xfer->tx_buf) {  in atmel_spi_dma_map_xfer()
1000 		void *nonconst_tx = (void *)xfer->tx_buf;  in atmel_spi_dma_map_xfer()
1002 		xfer->tx_dma = dma_map_single(dev,  in atmel_spi_dma_map_xfer()
1003 				nonconst_tx, xfer->len,  in atmel_spi_dma_map_xfer()
1005 		if (dma_mapping_error(dev, xfer->tx_dma))  in atmel_spi_dma_map_xfer()
1008 	if (xfer->rx_buf) {  in atmel_spi_dma_map_xfer()
1009 		xfer->rx_dma = dma_map_single(dev,  in atmel_spi_dma_map_xfer()
1010 				xfer->rx_buf, xfer->len,  in atmel_spi_dma_map_xfer()
1012 		if (dma_mapping_error(dev, xfer->rx_dma)) {  in atmel_spi_dma_map_xfer()
1013 			if (xfer->tx_buf)  in atmel_spi_dma_map_xfer()
1015 						xfer->tx_dma, xfer->len,  in atmel_spi_dma_map_xfer()
1024 				     struct spi_transfer *xfer)  in atmel_spi_dma_unmap_xfer()  argument
1026 	if (xfer->tx_dma != INVALID_DMA_ADDRESS)  in atmel_spi_dma_unmap_xfer()
1027 		dma_unmap_single(host->dev.parent, xfer->tx_dma,  in atmel_spi_dma_unmap_xfer()
1028 				 xfer->len, DMA_TO_DEVICE);  in atmel_spi_dma_unmap_xfer()
1029 	if (xfer->rx_dma != INVALID_DMA_ADDRESS)  in atmel_spi_dma_unmap_xfer()
1030 		dma_unmap_single(host->dev.parent, xfer->rx_dma,  in atmel_spi_dma_unmap_xfer()
1031 				 xfer->len, DMA_FROM_DEVICE);  in atmel_spi_dma_unmap_xfer()
1040 atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_pump_single_data()  argument
1044 	unsigned long	xfer_pos = xfer->len - as->current_remaining_bytes;  in atmel_spi_pump_single_data()
1046 	if (xfer->bits_per_word > 8) {  in atmel_spi_pump_single_data()
1047 		rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);  in atmel_spi_pump_single_data()
1050 		rxp = ((u8 *)xfer->rx_buf) + xfer_pos;  in atmel_spi_pump_single_data()
1053 	if (xfer->bits_per_word > 8) {  in atmel_spi_pump_single_data()
1064 atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_pump_fifo_data()  argument
1068 	u32 offset = xfer->len - as->current_remaining_bytes;  in atmel_spi_pump_fifo_data()
1069 	u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset);  in atmel_spi_pump_fifo_data()
1070 	u8  *bytes = (u8  *)((u8 *)xfer->rx_buf + offset);  in atmel_spi_pump_fifo_data()
1074 	num_bytes = ((xfer->bits_per_word > 8) ?  in atmel_spi_pump_fifo_data()
1084 	if (xfer->bits_per_word > 8)  in atmel_spi_pump_fifo_data()
1090 		if (xfer->bits_per_word > 8)  in atmel_spi_pump_fifo_data()
1104 atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_pump_pio_data()  argument
1107 		atmel_spi_pump_fifo_data(as, xfer);  in atmel_spi_pump_pio_data()
1109 		atmel_spi_pump_single_data(as, xfer);  in atmel_spi_pump_pio_data()
1121 	struct spi_transfer	*xfer;  in atmel_spi_pio_interrupt()  local
1155 			xfer = as->current_transfer;  in atmel_spi_pio_interrupt()
1156 			atmel_spi_pump_pio_data(as, xfer);  in atmel_spi_pio_interrupt()
1350 					struct spi_transfer *xfer)  in atmel_spi_one_transfer()  argument
1365 	if (bits != xfer->bits_per_word - 8) {  in atmel_spi_one_transfer()
1376 		if (atmel_spi_dma_map_xfer(as, xfer) < 0)  in atmel_spi_one_transfer()
1380 	atmel_spi_set_xfer_speed(as, spi, xfer);  in atmel_spi_one_transfer()
1383 	as->current_transfer = xfer;  in atmel_spi_one_transfer()
1384 	as->current_remaining_bytes = xfer->len;  in atmel_spi_one_transfer()
1390 			atmel_spi_pdc_next_xfer(host, xfer);  in atmel_spi_one_transfer()
1392 		} else if (atmel_spi_use_dma(as, xfer)) {  in atmel_spi_one_transfer()
1395 								xfer, &len);  in atmel_spi_one_transfer()
1408 			atmel_spi_next_xfer_pio(host, xfer);  in atmel_spi_one_transfer()
1412 		dma_timeout = msecs_to_jiffies(spi_controller_xfer_timeout(host, xfer));  in atmel_spi_one_transfer()
1449 		} else if (atmel_spi_use_dma(as, xfer)) {  in atmel_spi_one_transfer()
1455 		atmel_spi_dma_unmap_xfer(host, xfer);  in atmel_spi_one_transfer()