628 	int mbox;  in tsi721_omsg_msix()  local
630 	mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;  in tsi721_omsg_msix()
631 	tsi721_omsg_handler(priv, mbox);  in tsi721_omsg_msix()
647 	int mbox;  in tsi721_imsg_msix()  local
649 	mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;  in tsi721_imsg_msix()
650 	tsi721_imsg_handler(priv, mbox + 4);  in tsi721_imsg_msix()
1694 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,  in tsi721_add_outb_message()  argument
1702 	if (!priv->omsg_init[mbox] ||  in tsi721_add_outb_message()
1706 	spin_lock_irqsave(&priv->omsg_ring[mbox].lock, flags);  in tsi721_add_outb_message()
1708 	tx_slot = priv->omsg_ring[mbox].tx_slot;  in tsi721_add_outb_message()
1711 	memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);  in tsi721_add_outb_message()
1717 	desc = priv->omsg_ring[mbox].omd_base;  in tsi721_add_outb_message()
1725 		cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |  in tsi721_add_outb_message()
1728 		cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &  in tsi721_add_outb_message()
1731 		cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);  in tsi721_add_outb_message()
1733 	priv->omsg_ring[mbox].wr_count++;  in tsi721_add_outb_message()
1736 	if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {  in tsi721_add_outb_message()
1737 		priv->omsg_ring[mbox].tx_slot = 0;  in tsi721_add_outb_message()
1739 		priv->omsg_ring[mbox].wr_count++;  in tsi721_add_outb_message()
1745 	iowrite32(priv->omsg_ring[mbox].wr_count,  in tsi721_add_outb_message()
1746 		priv->regs + TSI721_OBDMAC_DWRCNT(mbox));  in tsi721_add_outb_message()
1747 	ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));  in tsi721_add_outb_message()
1749 	spin_unlock_irqrestore(&priv->omsg_ring[mbox].lock, flags);  in tsi721_add_outb_message()
1903 				 int mbox, int entries)  in tsi721_open_outb_mbox()  argument
1911 	    (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {  in tsi721_open_outb_mbox()
1916 	if ((mbox_sel & (1 << mbox)) == 0) {  in tsi721_open_outb_mbox()
1921 	priv->omsg_ring[mbox].dev_id = dev_id;  in tsi721_open_outb_mbox()
1922 	priv->omsg_ring[mbox].size = entries;  in tsi721_open_outb_mbox()
1923 	priv->omsg_ring[mbox].sts_rdptr = 0;  in tsi721_open_outb_mbox()
1924 	spin_lock_init(&priv->omsg_ring[mbox].lock);  in tsi721_open_outb_mbox()
1929 		priv->omsg_ring[mbox].omq_base[i] =  in tsi721_open_outb_mbox()
1932 				&priv->omsg_ring[mbox].omq_phys[i],  in tsi721_open_outb_mbox()
1934 		if (priv->omsg_ring[mbox].omq_base[i] == NULL) {  in tsi721_open_outb_mbox()
1936 				  "ENOMEM for OB_MSG_%d data buffer", mbox);  in tsi721_open_outb_mbox()
1943 	priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(  in tsi721_open_outb_mbox()
1946 				&priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);  in tsi721_open_outb_mbox()
1947 	if (priv->omsg_ring[mbox].omd_base == NULL) {  in tsi721_open_outb_mbox()
1949 			"ENOMEM for OB_MSG_%d descriptor memory", mbox);  in tsi721_open_outb_mbox()
1954 	priv->omsg_ring[mbox].tx_slot = 0;  in tsi721_open_outb_mbox()
1957 	priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);  in tsi721_open_outb_mbox()
1958 	priv->omsg_ring[mbox].sts_base = dma_alloc_coherent(&priv->pdev->dev,  in tsi721_open_outb_mbox()
1959 							    priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),  in tsi721_open_outb_mbox()
1960 							    &priv->omsg_ring[mbox].sts_phys,  in tsi721_open_outb_mbox()
1962 	if (priv->omsg_ring[mbox].sts_base == NULL) {  in tsi721_open_outb_mbox()
1964 			"ENOMEM for OB_MSG_%d status FIFO", mbox);  in tsi721_open_outb_mbox()
1974 	iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),  in tsi721_open_outb_mbox()
1975 			priv->regs + TSI721_OBDMAC_DPTRH(mbox));  in tsi721_open_outb_mbox()
1976 	iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &  in tsi721_open_outb_mbox()
1978 			priv->regs + TSI721_OBDMAC_DPTRL(mbox));  in tsi721_open_outb_mbox()
1981 	iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),  in tsi721_open_outb_mbox()
1982 			priv->regs + TSI721_OBDMAC_DSBH(mbox));  in tsi721_open_outb_mbox()
1983 	iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &  in tsi721_open_outb_mbox()
1985 			priv->regs + TSI721_OBDMAC_DSBL(mbox));  in tsi721_open_outb_mbox()
1986 	iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),  in tsi721_open_outb_mbox()
1987 		priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));  in tsi721_open_outb_mbox()
1993 		int idx = TSI721_VECT_OMB0_DONE + mbox;  in tsi721_open_outb_mbox()
2002 				mbox);  in tsi721_open_outb_mbox()
2006 		idx = TSI721_VECT_OMB0_INT + mbox;  in tsi721_open_outb_mbox()
2012 				"Unable to get MSI-X IRQ for MBOX%d-INT", mbox);  in tsi721_open_outb_mbox()
2013 			idx = TSI721_VECT_OMB0_DONE + mbox;  in tsi721_open_outb_mbox()
2020 	tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);  in tsi721_open_outb_mbox()
2023 	bd_ptr = priv->omsg_ring[mbox].omd_base;  in tsi721_open_outb_mbox()
2027 		cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &  in tsi721_open_outb_mbox()
2030 		cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);  in tsi721_open_outb_mbox()
2031 	priv->omsg_ring[mbox].wr_count = 0;  in tsi721_open_outb_mbox()
2036 		  priv->regs + TSI721_OBDMAC_CTL(mbox));  in tsi721_open_outb_mbox()
2037 	ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));  in tsi721_open_outb_mbox()
2040 	priv->omsg_init[mbox] = 1;  in tsi721_open_outb_mbox()
2047 		priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),  in tsi721_open_outb_mbox()
2048 		priv->omsg_ring[mbox].sts_base,  in tsi721_open_outb_mbox()
2049 		priv->omsg_ring[mbox].sts_phys);  in tsi721_open_outb_mbox()
2051 	priv->omsg_ring[mbox].sts_base = NULL;  in tsi721_open_outb_mbox()
2057 		priv->omsg_ring[mbox].omd_base,  in tsi721_open_outb_mbox()
2058 		priv->omsg_ring[mbox].omd_phys);  in tsi721_open_outb_mbox()
2060 	priv->omsg_ring[mbox].omd_base = NULL;  in tsi721_open_outb_mbox()
2063 	for (i = 0; i < priv->omsg_ring[mbox].size; i++) {  in tsi721_open_outb_mbox()
2064 		if (priv->omsg_ring[mbox].omq_base[i]) {  in tsi721_open_outb_mbox()
2067 				priv->omsg_ring[mbox].omq_base[i],  in tsi721_open_outb_mbox()
2068 				priv->omsg_ring[mbox].omq_phys[i]);  in tsi721_open_outb_mbox()
2070 			priv->omsg_ring[mbox].omq_base[i] = NULL;  in tsi721_open_outb_mbox()
2083 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)  in tsi721_close_outb_mbox()  argument
2088 	if (!priv->omsg_init[mbox])  in tsi721_close_outb_mbox()
2090 	priv->omsg_init[mbox] = 0;  in tsi721_close_outb_mbox()
2094 	tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);  in tsi721_close_outb_mbox()
2098 		free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,  in tsi721_close_outb_mbox()
2100 		free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,  in tsi721_close_outb_mbox()
2107 		priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),  in tsi721_close_outb_mbox()
2108 		priv->omsg_ring[mbox].sts_base,  in tsi721_close_outb_mbox()
2109 		priv->omsg_ring[mbox].sts_phys);  in tsi721_close_outb_mbox()
2111 	priv->omsg_ring[mbox].sts_base = NULL;  in tsi721_close_outb_mbox()
2115 		(priv->omsg_ring[mbox].size + 1) *  in tsi721_close_outb_mbox()
2117 		priv->omsg_ring[mbox].omd_base,  in tsi721_close_outb_mbox()
2118 		priv->omsg_ring[mbox].omd_phys);  in tsi721_close_outb_mbox()
2120 	priv->omsg_ring[mbox].omd_base = NULL;  in tsi721_close_outb_mbox()
2123 	for (i = 0; i < priv->omsg_ring[mbox].size; i++) {  in tsi721_close_outb_mbox()
2124 		if (priv->omsg_ring[mbox].omq_base[i]) {  in tsi721_close_outb_mbox()
2127 				priv->omsg_ring[mbox].omq_base[i],  in tsi721_close_outb_mbox()
2128 				priv->omsg_ring[mbox].omq_phys[i]);  in tsi721_close_outb_mbox()
2130 			priv->omsg_ring[mbox].omq_base[i] = NULL;  in tsi721_close_outb_mbox()
2144 	u32 mbox = ch - 4;  in tsi721_imsg_handler()  local
2148 	spin_lock(&priv->imsg_ring[mbox].lock);  in tsi721_imsg_handler()
2153 		tsi_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout", mbox);  in tsi721_imsg_handler()
2156 		tsi_info(&priv->pdev->dev, "IB MBOX%d PCIe error", mbox);  in tsi721_imsg_handler()
2159 		tsi_info(&priv->pdev->dev, "IB MBOX%d IB free queue low", mbox);  in tsi721_imsg_handler()
2166 		mport->inb_msg[mbox].mcback)  in tsi721_imsg_handler()
2167 		mport->inb_msg[mbox].mcback(mport,  in tsi721_imsg_handler()
2168 				priv->imsg_ring[mbox].dev_id, mbox, -1);  in tsi721_imsg_handler()
2179 	spin_unlock(&priv->imsg_ring[mbox].lock);  in tsi721_imsg_handler()
2192 				int mbox, int entries)  in tsi721_open_inb_mbox()  argument
2195 	int ch = mbox + 4;  in tsi721_open_inb_mbox()
2202 	    (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {  in tsi721_open_inb_mbox()
2207 	if ((mbox_sel & (1 << mbox)) == 0) {  in tsi721_open_inb_mbox()
2213 	priv->imsg_ring[mbox].dev_id = dev_id;  in tsi721_open_inb_mbox()
2214 	priv->imsg_ring[mbox].size = entries;  in tsi721_open_inb_mbox()
2215 	priv->imsg_ring[mbox].rx_slot = 0;  in tsi721_open_inb_mbox()
2216 	priv->imsg_ring[mbox].desc_rdptr = 0;  in tsi721_open_inb_mbox()
2217 	priv->imsg_ring[mbox].fq_wrptr = 0;  in tsi721_open_inb_mbox()
2218 	for (i = 0; i < priv->imsg_ring[mbox].size; i++)  in tsi721_open_inb_mbox()
2219 		priv->imsg_ring[mbox].imq_base[i] = NULL;  in tsi721_open_inb_mbox()
2220 	spin_lock_init(&priv->imsg_ring[mbox].lock);  in tsi721_open_inb_mbox()
2223 	priv->imsg_ring[mbox].buf_base =  in tsi721_open_inb_mbox()
2226 				   &priv->imsg_ring[mbox].buf_phys,  in tsi721_open_inb_mbox()
2229 	if (priv->imsg_ring[mbox].buf_base == NULL) {  in tsi721_open_inb_mbox()
2231 			"Failed to allocate buffers for IB MBOX%d", mbox);  in tsi721_open_inb_mbox()
2237 	priv->imsg_ring[mbox].imfq_base =  in tsi721_open_inb_mbox()
2240 				   &priv->imsg_ring[mbox].imfq_phys,  in tsi721_open_inb_mbox()
2243 	if (priv->imsg_ring[mbox].imfq_base == NULL) {  in tsi721_open_inb_mbox()
2245 			"Failed to allocate free queue for IB MBOX%d", mbox);  in tsi721_open_inb_mbox()
2251 	priv->imsg_ring[mbox].imd_base =  in tsi721_open_inb_mbox()
2254 				   &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);  in tsi721_open_inb_mbox()
2256 	if (priv->imsg_ring[mbox].imd_base == NULL) {  in tsi721_open_inb_mbox()
2259 			mbox);  in tsi721_open_inb_mbox()
2265 	free_ptr = priv->imsg_ring[mbox].imfq_base;  in tsi721_open_inb_mbox()
2268 				(u64)(priv->imsg_ring[mbox].buf_phys) +  in tsi721_open_inb_mbox()
2289 	iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),  in tsi721_open_inb_mbox()
2291 	iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &  in tsi721_open_inb_mbox()
2298 	iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),  in tsi721_open_inb_mbox()
2300 	iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &  in tsi721_open_inb_mbox()
2310 		int idx = TSI721_VECT_IMB0_RCV + mbox;  in tsi721_open_inb_mbox()
2319 				mbox);  in tsi721_open_inb_mbox()
2323 		idx = TSI721_VECT_IMB0_INT + mbox;  in tsi721_open_inb_mbox()
2329 				"Unable to get MSI-X IRQ for IBOX%d-INT", mbox);  in tsi721_open_inb_mbox()
2331 				priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,  in tsi721_open_inb_mbox()
2344 	priv->imsg_ring[mbox].fq_wrptr = entries - 1;  in tsi721_open_inb_mbox()
2347 	priv->imsg_init[mbox] = 1;  in tsi721_open_inb_mbox()
2353 		priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),  in tsi721_open_inb_mbox()
2354 		priv->imsg_ring[mbox].imd_base,  in tsi721_open_inb_mbox()
2355 		priv->imsg_ring[mbox].imd_phys);  in tsi721_open_inb_mbox()
2357 	priv->imsg_ring[mbox].imd_base = NULL;  in tsi721_open_inb_mbox()
2362 		priv->imsg_ring[mbox].size * 8,  in tsi721_open_inb_mbox()
2363 		priv->imsg_ring[mbox].imfq_base,  in tsi721_open_inb_mbox()
2364 		priv->imsg_ring[mbox].imfq_phys);  in tsi721_open_inb_mbox()
2366 	priv->imsg_ring[mbox].imfq_base = NULL;  in tsi721_open_inb_mbox()
2370 		priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,  in tsi721_open_inb_mbox()
2371 		priv->imsg_ring[mbox].buf_base,  in tsi721_open_inb_mbox()
2372 		priv->imsg_ring[mbox].buf_phys);  in tsi721_open_inb_mbox()
2374 	priv->imsg_ring[mbox].buf_base = NULL;  in tsi721_open_inb_mbox()
2385 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)  in tsi721_close_inb_mbox()  argument
2389 	int ch = mbox + 4;  in tsi721_close_inb_mbox()
2391 	if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */  in tsi721_close_inb_mbox()
2393 	priv->imsg_init[mbox] = 0;  in tsi721_close_inb_mbox()
2402 		free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,  in tsi721_close_inb_mbox()
2404 		free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,  in tsi721_close_inb_mbox()
2410 	for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)  in tsi721_close_inb_mbox()
2411 		priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;  in tsi721_close_inb_mbox()
2415 		priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,  in tsi721_close_inb_mbox()
2416 		priv->imsg_ring[mbox].buf_base,  in tsi721_close_inb_mbox()
2417 		priv->imsg_ring[mbox].buf_phys);  in tsi721_close_inb_mbox()
2419 	priv->imsg_ring[mbox].buf_base = NULL;  in tsi721_close_inb_mbox()
2423 		priv->imsg_ring[mbox].size * 8,  in tsi721_close_inb_mbox()
2424 		priv->imsg_ring[mbox].imfq_base,  in tsi721_close_inb_mbox()
2425 		priv->imsg_ring[mbox].imfq_phys);  in tsi721_close_inb_mbox()
2427 	priv->imsg_ring[mbox].imfq_base = NULL;  in tsi721_close_inb_mbox()
2431 		priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),  in tsi721_close_inb_mbox()
2432 		priv->imsg_ring[mbox].imd_base,  in tsi721_close_inb_mbox()
2433 		priv->imsg_ring[mbox].imd_phys);  in tsi721_close_inb_mbox()
2435 	priv->imsg_ring[mbox].imd_base = NULL;  in tsi721_close_inb_mbox()
2446 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)  in tsi721_add_inb_buffer()  argument
2452 	rx_slot = priv->imsg_ring[mbox].rx_slot;  in tsi721_add_inb_buffer()
2453 	if (priv->imsg_ring[mbox].imq_base[rx_slot]) {  in tsi721_add_inb_buffer()
2461 	priv->imsg_ring[mbox].imq_base[rx_slot] = buf;  in tsi721_add_inb_buffer()
2463 	if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)  in tsi721_add_inb_buffer()
2464 		priv->imsg_ring[mbox].rx_slot = 0;  in tsi721_add_inb_buffer()
2477 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)  in tsi721_get_inb_message()  argument
2486 	int ch = mbox + 4;  in tsi721_get_inb_message()
2489 	if (!priv->imsg_init[mbox])  in tsi721_get_inb_message()
2492 	desc = priv->imsg_ring[mbox].imd_base;  in tsi721_get_inb_message()
2493 	desc += priv->imsg_ring[mbox].desc_rdptr;  in tsi721_get_inb_message()
2498 	rx_slot = priv->imsg_ring[mbox].rx_slot;  in tsi721_get_inb_message()
2499 	while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {  in tsi721_get_inb_message()
2500 		if (++rx_slot == priv->imsg_ring[mbox].size)  in tsi721_get_inb_message()
2507 	rx_virt = priv->imsg_ring[mbox].buf_base +  in tsi721_get_inb_message()
2508 		  (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);  in tsi721_get_inb_message()
2510 	buf = priv->imsg_ring[mbox].imq_base[rx_slot];  in tsi721_get_inb_message()
2516 	priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;  in tsi721_get_inb_message()
2519 	if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)  in tsi721_get_inb_message()
2520 		priv->imsg_ring[mbox].desc_rdptr = 0;  in tsi721_get_inb_message()
2522 	iowrite32(priv->imsg_ring[mbox].desc_rdptr,  in tsi721_get_inb_message()
2526 	free_ptr = priv->imsg_ring[mbox].imfq_base;  in tsi721_get_inb_message()
2527 	free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);  in tsi721_get_inb_message()
2529 	if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)  in tsi721_get_inb_message()
2530 		priv->imsg_ring[mbox].fq_wrptr = 0;  in tsi721_get_inb_message()
2532 	iowrite32(priv->imsg_ring[mbox].fq_wrptr,  in tsi721_get_inb_message()