122 	struct uart_port port;  member
134 static int sprd_start_dma_rx(struct uart_port *port);
135 static int sprd_tx_dma_config(struct uart_port *port);
137 static inline unsigned int serial_in(struct uart_port *port,  in serial_in()  argument
140 	return readl_relaxed(port->membase + offset);  in serial_in()
143 static inline void serial_out(struct uart_port *port, unsigned int offset,  in serial_out()  argument
146 	writel_relaxed(value, port->membase + offset);  in serial_out()
149 static unsigned int sprd_tx_empty(struct uart_port *port)  in sprd_tx_empty()  argument
151 	if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_tx_empty()
157 static unsigned int sprd_get_mctrl(struct uart_port *port)  in sprd_get_mctrl()  argument
162 static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)  in sprd_set_mctrl()  argument
164 	u32 val = serial_in(port, SPRD_CTL1);  in sprd_set_mctrl()
171 	serial_out(port, SPRD_CTL1, val);  in sprd_set_mctrl()
174 static void sprd_stop_rx(struct uart_port *port)  in sprd_stop_rx()  argument
177 		container_of(port, struct sprd_uart_port, port);  in sprd_stop_rx()
183 	iclr = serial_in(port, SPRD_ICLR);  in sprd_stop_rx()
184 	ien = serial_in(port, SPRD_IEN);  in sprd_stop_rx()
189 	serial_out(port, SPRD_IEN, ien);  in sprd_stop_rx()
190 	serial_out(port, SPRD_ICLR, iclr);  in sprd_stop_rx()
193 static void sprd_uart_dma_enable(struct uart_port *port, bool enable)  in sprd_uart_dma_enable()  argument
195 	u32 val = serial_in(port, SPRD_CTL1);  in sprd_uart_dma_enable()
202 	serial_out(port, SPRD_CTL1, val);  in sprd_uart_dma_enable()
205 static void sprd_stop_tx_dma(struct uart_port *port)  in sprd_stop_tx_dma()  argument
208 		container_of(port, struct sprd_uart_port, port);  in sprd_stop_tx_dma()
217 		uart_xmit_advance(port, trans_len);  in sprd_stop_tx_dma()
218 		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,  in sprd_stop_tx_dma()
226 static int sprd_tx_buf_remap(struct uart_port *port)  in sprd_tx_buf_remap()  argument
229 		container_of(port, struct sprd_uart_port, port);  in sprd_tx_buf_remap()
230 	struct tty_port *tport = &port->state->port;  in sprd_tx_buf_remap()
236 	sp->tx_dma.phys_addr = dma_map_single(port->dev, tail,  in sprd_tx_buf_remap()
239 	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);  in sprd_tx_buf_remap()
244 	struct uart_port *port = (struct uart_port *)data;  in sprd_complete_tx_dma()  local
246 		container_of(port, struct sprd_uart_port, port);  in sprd_complete_tx_dma()
247 	struct tty_port *tport = &port->state->port;  in sprd_complete_tx_dma()
250 	uart_port_lock_irqsave(port, &flags);  in sprd_complete_tx_dma()
251 	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,  in sprd_complete_tx_dma()
254 	uart_xmit_advance(port, sp->tx_dma.trans_len);  in sprd_complete_tx_dma()
257 		uart_write_wakeup(port);  in sprd_complete_tx_dma()
259 	if (kfifo_is_empty(&tport->xmit_fifo) || sprd_tx_buf_remap(port) ||  in sprd_complete_tx_dma()
260 	    sprd_tx_dma_config(port))  in sprd_complete_tx_dma()
263 	uart_port_unlock_irqrestore(port, flags);  in sprd_complete_tx_dma()
266 static int sprd_uart_dma_submit(struct uart_port *port,  in sprd_uart_dma_submit()  argument
285 	dma_des->callback_param = port;  in sprd_uart_dma_submit()
296 static int sprd_tx_dma_config(struct uart_port *port)  in sprd_tx_dma_config()  argument
299 		container_of(port, struct sprd_uart_port, port);  in sprd_tx_dma_config()
304 		.dst_addr = port->mapbase + SPRD_TXD,  in sprd_tx_dma_config()
314 	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,  in sprd_tx_dma_config()
318 static void sprd_start_tx_dma(struct uart_port *port)  in sprd_start_tx_dma()  argument
321 		container_of(port, struct sprd_uart_port, port);  in sprd_start_tx_dma()
322 	struct tty_port *tport = &port->state->port;  in sprd_start_tx_dma()
324 	if (port->x_char) {  in sprd_start_tx_dma()
325 		serial_out(port, SPRD_TXD, port->x_char);  in sprd_start_tx_dma()
326 		port->icount.tx++;  in sprd_start_tx_dma()
327 		port->x_char = 0;  in sprd_start_tx_dma()
331 	if (kfifo_is_empty(&tport->xmit_fifo) || uart_tx_stopped(port)) {  in sprd_start_tx_dma()
332 		sprd_stop_tx_dma(port);  in sprd_start_tx_dma()
339 	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))  in sprd_start_tx_dma()
343 static void sprd_rx_full_thld(struct uart_port *port, u32 thld)  in sprd_rx_full_thld()  argument
345 	u32 val = serial_in(port, SPRD_CTL2);  in sprd_rx_full_thld()
349 	serial_out(port, SPRD_CTL2, val);  in sprd_rx_full_thld()
354 	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,  in sprd_rx_alloc_buf()
365 		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,  in sprd_rx_free_buf()
370 static int sprd_rx_dma_config(struct uart_port *port, u32 burst)  in sprd_rx_dma_config()  argument
373 		container_of(port, struct sprd_uart_port, port);  in sprd_rx_dma_config()
375 		.src_addr = port->mapbase + SPRD_RXD,  in sprd_rx_dma_config()
384 static void sprd_uart_dma_rx(struct uart_port *port)  in sprd_uart_dma_rx()  argument
387 		container_of(port, struct sprd_uart_port, port);  in sprd_uart_dma_rx()
388 	struct tty_port *tty = &port->state->port;  in sprd_uart_dma_rx()
390 	port->icount.rx += sp->rx_dma.trans_len;  in sprd_uart_dma_rx()
395 static void sprd_uart_dma_irq(struct uart_port *port)  in sprd_uart_dma_irq()  argument
398 		container_of(port, struct sprd_uart_port, port);  in sprd_uart_dma_irq()
405 		sprd_stop_rx(port);  in sprd_uart_dma_irq()
419 	sprd_uart_dma_rx(port);  in sprd_uart_dma_irq()
425 	struct uart_port *port = (struct uart_port *)data;  in sprd_complete_rx_dma()  local
427 		container_of(port, struct sprd_uart_port, port);  in sprd_complete_rx_dma()
432 	uart_port_lock_irqsave(port, &flags);  in sprd_complete_rx_dma()
437 		sprd_stop_rx(port);  in sprd_complete_rx_dma()
438 		uart_port_unlock_irqrestore(port, flags);  in sprd_complete_rx_dma()
445 		sprd_uart_dma_rx(port);  in sprd_complete_rx_dma()
449 	if (sprd_start_dma_rx(port))  in sprd_complete_rx_dma()
450 		sprd_stop_rx(port);  in sprd_complete_rx_dma()
452 	uart_port_unlock_irqrestore(port, flags);  in sprd_complete_rx_dma()
455 static int sprd_start_dma_rx(struct uart_port *port)  in sprd_start_dma_rx()  argument
458 		container_of(port, struct sprd_uart_port, port);  in sprd_start_dma_rx()
466 	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);  in sprd_start_dma_rx()
467 	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);  in sprd_start_dma_rx()
471 	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,  in sprd_start_dma_rx()
475 static void sprd_release_dma(struct uart_port *port)  in sprd_release_dma()  argument
478 		container_of(port, struct sprd_uart_port, port);  in sprd_release_dma()
480 	sprd_uart_dma_enable(port, false);  in sprd_release_dma()
492 static void sprd_request_dma(struct uart_port *port)  in sprd_request_dma()  argument
495 		container_of(port, struct sprd_uart_port, port);  in sprd_request_dma()
500 	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");  in sprd_request_dma()
502 		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",  in sprd_request_dma()
507 	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");  in sprd_request_dma()
509 		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",  in sprd_request_dma()
515 static void sprd_stop_tx(struct uart_port *port)  in sprd_stop_tx()  argument
517 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_stop_tx()
518 						 port);  in sprd_stop_tx()
522 		sprd_stop_tx_dma(port);  in sprd_stop_tx()
526 	iclr = serial_in(port, SPRD_ICLR);  in sprd_stop_tx()
527 	ien = serial_in(port, SPRD_IEN);  in sprd_stop_tx()
532 	serial_out(port, SPRD_IEN, ien);  in sprd_stop_tx()
533 	serial_out(port, SPRD_ICLR, iclr);  in sprd_stop_tx()
536 static void sprd_start_tx(struct uart_port *port)  in sprd_start_tx()  argument
538 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_start_tx()
539 						 port);  in sprd_start_tx()
543 		sprd_start_tx_dma(port);  in sprd_start_tx()
547 	ien = serial_in(port, SPRD_IEN);  in sprd_start_tx()
550 		serial_out(port, SPRD_IEN, ien);  in sprd_start_tx()
555 static void sprd_break_ctl(struct uart_port *port, int break_state)  in sprd_break_ctl()  argument
560 static int handle_lsr_errors(struct uart_port *port,  in handle_lsr_errors()  argument
569 		port->icount.brk++;  in handle_lsr_errors()
570 		ret = uart_handle_break(port);  in handle_lsr_errors()
574 		port->icount.parity++;  in handle_lsr_errors()
576 		port->icount.frame++;  in handle_lsr_errors()
578 		port->icount.overrun++;  in handle_lsr_errors()
581 	*lsr &= port->read_status_mask;  in handle_lsr_errors()
592 static inline void sprd_rx(struct uart_port *port)  in sprd_rx()  argument
594 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_rx()
595 						 port);  in sprd_rx()
596 	struct tty_port *tty = &port->state->port;  in sprd_rx()
601 		sprd_uart_dma_irq(port);  in sprd_rx()
605 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&  in sprd_rx()
607 		lsr = serial_in(port, SPRD_LSR);  in sprd_rx()
608 		ch = serial_in(port, SPRD_RXD);  in sprd_rx()
610 		port->icount.rx++;  in sprd_rx()
614 			if (handle_lsr_errors(port, &flag, &lsr))  in sprd_rx()
616 		if (uart_handle_sysrq_char(port, ch))  in sprd_rx()
619 		uart_insert_char(port, lsr, SPRD_LSR_OE, ch, flag);  in sprd_rx()
625 static inline void sprd_tx(struct uart_port *port)  in sprd_tx()  argument
629 	uart_port_tx_limited(port, ch, THLD_TX_EMPTY,  in sprd_tx()
631 		serial_out(port, SPRD_TXD, ch),  in sprd_tx()
635 /* this handles the interrupt from one port */
638 	struct uart_port *port = dev_id;  in sprd_handle_irq()  local
641 	uart_port_lock(port);  in sprd_handle_irq()
643 	ims = serial_in(port, SPRD_IMSR);  in sprd_handle_irq()
646 		uart_port_unlock(port);  in sprd_handle_irq()
651 		serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);  in sprd_handle_irq()
654 		serial_out(port, SPRD_ICLR, SPRD_IMSR_BREAK_DETECT);  in sprd_handle_irq()
658 		sprd_rx(port);  in sprd_handle_irq()
661 		sprd_tx(port);  in sprd_handle_irq()
663 	uart_port_unlock(port);  in sprd_handle_irq()
668 static void sprd_uart_dma_startup(struct uart_port *port,  in sprd_uart_dma_startup()  argument
673 	sprd_request_dma(port);  in sprd_uart_dma_startup()
677 	ret = sprd_start_dma_rx(port);  in sprd_uart_dma_startup()
681 		dev_warn(port->dev, "fail to start RX dma mode\n");  in sprd_uart_dma_startup()
684 	sprd_uart_dma_enable(port, true);  in sprd_uart_dma_startup()
687 static int sprd_startup(struct uart_port *port)  in sprd_startup()  argument
695 	serial_out(port, SPRD_CTL2,  in sprd_startup()
700 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)  in sprd_startup()
701 		serial_in(port, SPRD_RXD);  in sprd_startup()
705 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_startup()
709 	serial_out(port, SPRD_IEN, 0);  in sprd_startup()
710 	serial_out(port, SPRD_ICLR, ~0);  in sprd_startup()
713 	sp = container_of(port, struct sprd_uart_port, port);  in sprd_startup()
714 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);  in sprd_startup()
716 	sprd_uart_dma_startup(port, sp);  in sprd_startup()
718 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,  in sprd_startup()
719 			       IRQF_SHARED, sp->name, port);  in sprd_startup()
721 		dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",  in sprd_startup()
722 			port->irq, ret);  in sprd_startup()
725 	fc = serial_in(port, SPRD_CTL1);  in sprd_startup()
727 	serial_out(port, SPRD_CTL1, fc);  in sprd_startup()
730 	uart_port_lock_irqsave(port, &flags);  in sprd_startup()
731 	ien = serial_in(port, SPRD_IEN);  in sprd_startup()
735 	serial_out(port, SPRD_IEN, ien);  in sprd_startup()
736 	uart_port_unlock_irqrestore(port, flags);  in sprd_startup()
741 static void sprd_shutdown(struct uart_port *port)  in sprd_shutdown()  argument
743 	sprd_release_dma(port);  in sprd_shutdown()
744 	serial_out(port, SPRD_IEN, 0);  in sprd_shutdown()
745 	serial_out(port, SPRD_ICLR, ~0);  in sprd_shutdown()
746 	devm_free_irq(port->dev, port->irq, port);  in sprd_shutdown()
749 static void sprd_set_termios(struct uart_port *port, struct ktermios *termios,  in sprd_set_termios()  argument
757 	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);  in sprd_set_termios()
759 	quot = port->uartclk / baud;  in sprd_set_termios()
796 	uart_port_lock_irqsave(port, &flags);  in sprd_set_termios()
798 	/* update the per-port timeout */  in sprd_set_termios()
799 	uart_update_timeout(port, termios->c_cflag, baud);  in sprd_set_termios()
801 	port->read_status_mask = SPRD_LSR_OE;  in sprd_set_termios()
803 		port->read_status_mask |= SPRD_LSR_FE | SPRD_LSR_PE;  in sprd_set_termios()
805 		port->read_status_mask |= SPRD_LSR_BI;  in sprd_set_termios()
808 	port->ignore_status_mask = 0;  in sprd_set_termios()
810 		port->ignore_status_mask |= SPRD_LSR_PE | SPRD_LSR_FE;  in sprd_set_termios()
812 		port->ignore_status_mask |= SPRD_LSR_BI;  in sprd_set_termios()
818 			port->ignore_status_mask |= SPRD_LSR_OE;  in sprd_set_termios()
822 	fc = serial_in(port, SPRD_CTL1);  in sprd_set_termios()
831 	serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);  in sprd_set_termios()
834 	serial_out(port, SPRD_CLKD1,  in sprd_set_termios()
836 	serial_out(port, SPRD_LCR, lcr);  in sprd_set_termios()
838 	serial_out(port, SPRD_CTL1, fc);  in sprd_set_termios()
840 	uart_port_unlock_irqrestore(port, flags);  in sprd_set_termios()
847 static const char *sprd_type(struct uart_port *port)  in sprd_type()  argument
852 static void sprd_release_port(struct uart_port *port)  in sprd_release_port()  argument
857 static int sprd_request_port(struct uart_port *port)  in sprd_request_port()  argument
862 static void sprd_config_port(struct uart_port *port, int flags)  in sprd_config_port()  argument
865 		port->type = PORT_SPRD;  in sprd_config_port()
868 static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)  in sprd_verify_port()  argument
872 	if (port->irq != ser->irq)  in sprd_verify_port()
874 	if (port->iotype != ser->io_type)  in sprd_verify_port()
879 static void sprd_pm(struct uart_port *port, unsigned int state,  in sprd_pm()  argument
883 		container_of(port, struct sprd_uart_port, port);  in sprd_pm()
896 static int sprd_poll_init(struct uart_port *port)  in sprd_poll_init()  argument
898 	if (port->state->pm_state != UART_PM_STATE_ON) {  in sprd_poll_init()
899 		sprd_pm(port, UART_PM_STATE_ON, 0);  in sprd_poll_init()
900 		port->state->pm_state = UART_PM_STATE_ON;  in sprd_poll_init()
906 static int sprd_poll_get_char(struct uart_port *port)  in sprd_poll_get_char()  argument
908 	while (!(serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK))  in sprd_poll_get_char()
911 	return serial_in(port, SPRD_RXD);  in sprd_poll_get_char()
914 static void sprd_poll_put_char(struct uart_port *port, unsigned char ch)  in sprd_poll_put_char()  argument
916 	while (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_poll_put_char()
919 	serial_out(port, SPRD_TXD, ch);  in sprd_poll_put_char()
948 static void wait_for_xmitr(struct uart_port *port)  in wait_for_xmitr()  argument
954 		status = serial_in(port, SPRD_STS1);  in wait_for_xmitr()
961 static void sprd_console_putchar(struct uart_port *port, unsigned char ch)  in sprd_console_putchar()  argument
963 	wait_for_xmitr(port);  in sprd_console_putchar()
964 	serial_out(port, SPRD_TXD, ch);  in sprd_console_putchar()
970 	struct uart_port *port = &sprd_port[co->index]->port;  in sprd_console_write()  local
974 	if (port->sysrq)  in sprd_console_write()
977 		locked = uart_port_trylock_irqsave(port, &flags);  in sprd_console_write()
979 		uart_port_lock_irqsave(port, &flags);  in sprd_console_write()
981 	uart_console_write(port, s, count, sprd_console_putchar);  in sprd_console_write()
984 	wait_for_xmitr(port);  in sprd_console_write()
987 		uart_port_unlock_irqrestore(port, flags);  in sprd_console_write()
1002 	if (!sprd_uart_port || !sprd_uart_port->port.membase) {  in sprd_console_setup()
1003 		pr_info("serial port %d not yet initialized\n", co->index);  in sprd_console_setup()
1010 	return uart_set_options(&sprd_uart_port->port, co, baud,  in sprd_console_setup()
1035 static void sprd_putc(struct uart_port *port, unsigned char c)  in sprd_putc()  argument
1040 	       !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))  in sprd_putc()
1043 	writeb(c, port->membase + SPRD_TXD);  in sprd_putc()
1050 	uart_console_write(&dev->port, s, n, sprd_putc);  in sprd_early_write()
1056 	if (!device->port.membase)  in sprd_early_console_setup()
1084 		uart_remove_one_port(&sprd_uart_driver, &sup->port);  in sprd_remove()
1085 		sprd_port[sup->port.line] = NULL;  in sprd_remove()
1108 	struct sprd_uart_port *u = container_of(uport, struct sprd_uart_port, port);  in sprd_clk_init()
1166 	up = &sport->port;  in sprd_probe()
1233 	uart_suspend_port(&sprd_uart_driver, &sup->port);  in sprd_suspend()
1242 	uart_resume_port(&sprd_uart_driver, &sup->port);  in sprd_resume()