26 /* Vring size. */
71  * @num: vring size (number of descriptors)
72  * @align: vring alignment size
73  * @index: vring index
74  * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
98 /* Check whether vring is in drop mode. */
186  * @vring: Tx/Rx ring
203 	struct mlxbf_tmfifo_vring *vring[2];  member
249 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_free_vrings()  local
253 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_free_vrings()
254 		if (vring->va) {  in mlxbf_tmfifo_free_vrings()
255 			size = vring_size(vring->num, vring->align);  in mlxbf_tmfifo_free_vrings()
257 					  vring->va, vring->dma);  in mlxbf_tmfifo_free_vrings()
258 			vring->va = NULL;  in mlxbf_tmfifo_free_vrings()
259 			if (vring->vq) {  in mlxbf_tmfifo_free_vrings()
260 				vring_del_virtqueue(vring->vq);  in mlxbf_tmfifo_free_vrings()
261 				vring->vq = NULL;  in mlxbf_tmfifo_free_vrings()
271 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_alloc_vrings()  local
278 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_alloc_vrings()
279 		vring->fifo = fifo;  in mlxbf_tmfifo_alloc_vrings()
280 		vring->num = MLXBF_TMFIFO_VRING_SIZE;  in mlxbf_tmfifo_alloc_vrings()
281 		vring->align = SMP_CACHE_BYTES;  in mlxbf_tmfifo_alloc_vrings()
282 		vring->index = i;  in mlxbf_tmfifo_alloc_vrings()
283 		vring->vdev_id = tm_vdev->vdev.id.device;  in mlxbf_tmfifo_alloc_vrings()
284 		vring->drop_desc.len = VRING_DROP_DESC_MAX_LEN;  in mlxbf_tmfifo_alloc_vrings()
287 		size = vring_size(vring->num, vring->align);  in mlxbf_tmfifo_alloc_vrings()
295 		vring->va = va;  in mlxbf_tmfifo_alloc_vrings()
296 		vring->dma = dma;  in mlxbf_tmfifo_alloc_vrings()
325 /* Get the next packet descriptor from the vring. */
327 mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_get_next_desc()  argument
329 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_get_next_desc()
330 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_get_next_desc()
333 	if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx))  in mlxbf_tmfifo_get_next_desc()
339 	idx = vring->next_avail % vr->num;  in mlxbf_tmfifo_get_next_desc()
344 	vring->next_avail++;  in mlxbf_tmfifo_get_next_desc()
350 static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_release_desc()  argument
353 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_release_desc()
354 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_release_desc()
372 static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_get_pkt_len()  argument
375 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_get_pkt_len()
376 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_get_pkt_len()
390 static void mlxbf_tmfifo_release_pkt(struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_release_pkt()  argument
395 	if (vring->desc_head) {  in mlxbf_tmfifo_release_pkt()
396 		desc_head = vring->desc_head;  in mlxbf_tmfifo_release_pkt()
397 		len = vring->pkt_len;  in mlxbf_tmfifo_release_pkt()
399 		desc_head = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_release_pkt()
400 		len = mlxbf_tmfifo_get_pkt_len(vring, desc_head);  in mlxbf_tmfifo_release_pkt()
404 		mlxbf_tmfifo_release_desc(vring, desc_head, len);  in mlxbf_tmfifo_release_pkt()
406 	vring->pkt_len = 0;  in mlxbf_tmfifo_release_pkt()
407 	vring->desc = NULL;  in mlxbf_tmfifo_release_pkt()
408 	vring->desc_head = NULL;  in mlxbf_tmfifo_release_pkt()
411 static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_init_net_desc()  argument
414 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_init_net_desc()
423 mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx)  in mlxbf_tmfifo_get_next_pkt()  argument
427 	desc = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_get_next_pkt()
428 	if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET)  in mlxbf_tmfifo_get_next_pkt()
429 		mlxbf_tmfifo_init_net_desc(vring, desc, is_rx);  in mlxbf_tmfifo_get_next_pkt()
431 	vring->desc_head = desc;  in mlxbf_tmfifo_get_next_pkt()
432 	vring->desc = desc;  in mlxbf_tmfifo_get_next_pkt()
454 					    struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_console_output_one()  argument
457 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_console_output_one()
487 					struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_console_output()  argument
492 	desc = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_console_output()
495 		len = mlxbf_tmfifo_get_pkt_len(vring, desc);  in mlxbf_tmfifo_console_output()
499 			mlxbf_tmfifo_release_desc(vring, desc, len);  in mlxbf_tmfifo_console_output()
503 		mlxbf_tmfifo_console_output_one(cons, vring, desc);  in mlxbf_tmfifo_console_output()
504 		mlxbf_tmfifo_release_desc(vring, desc, len);  in mlxbf_tmfifo_console_output()
505 		desc = mlxbf_tmfifo_get_next_desc(vring);  in mlxbf_tmfifo_console_output()
601 static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_rxtx_word()  argument
605 	struct virtio_device *vdev = vring->vq->vdev;  in mlxbf_tmfifo_rxtx_word()
606 	struct mlxbf_tmfifo *fifo = vring->fifo;  in mlxbf_tmfifo_rxtx_word()
617 	if (vring->cur_len + sizeof(u64) <= len) {  in mlxbf_tmfifo_rxtx_word()
620 			if (!IS_VRING_DROP(vring))  in mlxbf_tmfifo_rxtx_word()
621 				memcpy(addr + vring->cur_len, &data,  in mlxbf_tmfifo_rxtx_word()
624 			memcpy(&data, addr + vring->cur_len,  in mlxbf_tmfifo_rxtx_word()
627 		vring->cur_len += sizeof(u64);  in mlxbf_tmfifo_rxtx_word()
631 			if (!IS_VRING_DROP(vring))  in mlxbf_tmfifo_rxtx_word()
632 				memcpy(addr + vring->cur_len, &data,  in mlxbf_tmfifo_rxtx_word()
633 				       len - vring->cur_len);  in mlxbf_tmfifo_rxtx_word()
636 			memcpy(&data, addr + vring->cur_len,  in mlxbf_tmfifo_rxtx_word()
637 			       len - vring->cur_len);  in mlxbf_tmfifo_rxtx_word()
639 		vring->cur_len = len;  in mlxbf_tmfifo_rxtx_word()
650  * In Rx case, the packet might be found to belong to a different vring since
654 static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_rxtx_header()  argument
658 	struct mlxbf_tmfifo *fifo = vring->fifo;  in mlxbf_tmfifo_rxtx_header()
690 		 * Check whether the new packet still belongs to this vring.  in mlxbf_tmfifo_rxtx_header()
691 		 * If not, update the pkt_len of the new vring.  in mlxbf_tmfifo_rxtx_header()
693 		if (vdev_id != vring->vdev_id) {  in mlxbf_tmfifo_rxtx_header()
698 			vring->desc = *desc;  in mlxbf_tmfifo_rxtx_header()
699 			vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX];  in mlxbf_tmfifo_rxtx_header()
703 		if (drop_rx && !IS_VRING_DROP(vring)) {  in mlxbf_tmfifo_rxtx_header()
704 			if (vring->desc_head)  in mlxbf_tmfifo_rxtx_header()
705 				mlxbf_tmfifo_release_pkt(vring);  in mlxbf_tmfifo_rxtx_header()
706 			*desc = &vring->drop_desc;  in mlxbf_tmfifo_rxtx_header()
707 			vring->desc_head = *desc;  in mlxbf_tmfifo_rxtx_header()
708 			vring->desc = *desc;  in mlxbf_tmfifo_rxtx_header()
711 		vring->pkt_len = ntohs(hdr.len) + hdr_len;  in mlxbf_tmfifo_rxtx_header()
714 		hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ?  in mlxbf_tmfifo_rxtx_header()
716 		vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, *desc);  in mlxbf_tmfifo_rxtx_header()
717 		hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ?  in mlxbf_tmfifo_rxtx_header()
719 		hdr.len = htons(vring->pkt_len - hdr_len);  in mlxbf_tmfifo_rxtx_header()
723 	vring->cur_len = hdr_len;  in mlxbf_tmfifo_rxtx_header()
724 	vring->rem_len = vring->pkt_len;  in mlxbf_tmfifo_rxtx_header()
725 	fifo->vring[is_rx] = vring;  in mlxbf_tmfifo_rxtx_header()
733 static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring,  in mlxbf_tmfifo_rxtx_one_desc()  argument
736 	const struct vring *vr = virtqueue_get_vring(vring->vq);  in mlxbf_tmfifo_rxtx_one_desc()
737 	struct mlxbf_tmfifo *fifo = vring->fifo;  in mlxbf_tmfifo_rxtx_one_desc()
744 	vdev = &fifo->vdev[vring->vdev_id]->vdev;  in mlxbf_tmfifo_rxtx_one_desc()
747 	if (!vring->desc) {  in mlxbf_tmfifo_rxtx_one_desc()
748 		desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx);  in mlxbf_tmfifo_rxtx_one_desc()
752 				desc = &vring->drop_desc;  in mlxbf_tmfifo_rxtx_one_desc()
753 				vring->desc_head = desc;  in mlxbf_tmfifo_rxtx_one_desc()
754 				vring->desc = desc;  in mlxbf_tmfifo_rxtx_one_desc()
760 		desc = vring->desc;  in mlxbf_tmfifo_rxtx_one_desc()
764 	if (vring->pkt_len == 0) {  in mlxbf_tmfifo_rxtx_one_desc()
765 		mlxbf_tmfifo_rxtx_header(vring, &desc, is_rx, &vring_change);  in mlxbf_tmfifo_rxtx_one_desc()
776 	if (len > vring->rem_len)  in mlxbf_tmfifo_rxtx_one_desc()
777 		len = vring->rem_len;  in mlxbf_tmfifo_rxtx_one_desc()
780 	if (vring->cur_len < len) {  in mlxbf_tmfifo_rxtx_one_desc()
781 		mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len);  in mlxbf_tmfifo_rxtx_one_desc()
786 	if (vring->cur_len == len) {  in mlxbf_tmfifo_rxtx_one_desc()
787 		vring->cur_len = 0;  in mlxbf_tmfifo_rxtx_one_desc()
788 		vring->rem_len -= len;  in mlxbf_tmfifo_rxtx_one_desc()
791 		if (!IS_VRING_DROP(vring) && vring->rem_len > 0 &&  in mlxbf_tmfifo_rxtx_one_desc()
800 		fifo->vring[is_rx] = NULL;  in mlxbf_tmfifo_rxtx_one_desc()
801 		if (!IS_VRING_DROP(vring)) {  in mlxbf_tmfifo_rxtx_one_desc()
802 			mlxbf_tmfifo_release_pkt(vring);  in mlxbf_tmfifo_rxtx_one_desc()
804 			vring->pkt_len = 0;  in mlxbf_tmfifo_rxtx_one_desc()
805 			vring->desc_head = NULL;  in mlxbf_tmfifo_rxtx_one_desc()
806 			vring->desc = NULL;  in mlxbf_tmfifo_rxtx_one_desc()
818 		vring_interrupt(0, vring->vq);  in mlxbf_tmfifo_rxtx_one_desc()
824 	vring->desc = desc;  in mlxbf_tmfifo_rxtx_one_desc()
829 static void mlxbf_tmfifo_check_tx_timeout(struct mlxbf_tmfifo_vring *vring)  in mlxbf_tmfifo_check_tx_timeout()  argument
834 	if (vring->vdev_id != VIRTIO_ID_NET)  in mlxbf_tmfifo_check_tx_timeout()
838 	if (!vring->tx_timeout) {  in mlxbf_tmfifo_check_tx_timeout()
840 		vring->tx_timeout = jiffies +  in mlxbf_tmfifo_check_tx_timeout()
843 	} else if (time_before(jiffies, vring->tx_timeout)) {  in mlxbf_tmfifo_check_tx_timeout()
855 	vring->rem_padding = round_up(vring->rem_len, sizeof(u64));  in mlxbf_tmfifo_check_tx_timeout()
856 	mlxbf_tmfifo_release_pkt(vring);  in mlxbf_tmfifo_check_tx_timeout()
857 	vring->cur_len = 0;  in mlxbf_tmfifo_check_tx_timeout()
858 	vring->rem_len = 0;  in mlxbf_tmfifo_check_tx_timeout()
859 	vring->fifo->vring[0] = NULL;  in mlxbf_tmfifo_check_tx_timeout()
868 	spin_lock_irqsave(&vring->fifo->spin_lock[0], flags);  in mlxbf_tmfifo_check_tx_timeout()
869 	vring_interrupt(0, vring->vq);  in mlxbf_tmfifo_check_tx_timeout()
870 	spin_unlock_irqrestore(&vring->fifo->spin_lock[0], flags);  in mlxbf_tmfifo_check_tx_timeout()
874 static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx)  in mlxbf_tmfifo_rxtx()  argument
876 	int avail = 0, devid = vring->vdev_id;  in mlxbf_tmfifo_rxtx()
880 	fifo = vring->fifo;  in mlxbf_tmfifo_rxtx()
886 	/* Return if another vring is running. */  in mlxbf_tmfifo_rxtx()
887 	if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring)  in mlxbf_tmfifo_rxtx()
908 			vring->tx_timeout = 0;  in mlxbf_tmfifo_rxtx()
909 			while (vring->rem_padding >= sizeof(u64)) {  in mlxbf_tmfifo_rxtx()
910 				writeq(0, vring->fifo->tx.data);  in mlxbf_tmfifo_rxtx()
911 				vring->rem_padding -= sizeof(u64);  in mlxbf_tmfifo_rxtx()
924 		more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail);  in mlxbf_tmfifo_rxtx()
929 		mlxbf_tmfifo_check_tx_timeout(vring);  in mlxbf_tmfifo_rxtx()
937 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_work_rxtx()  local
947 			vring = &tm_vdev->vrings[queue_id];  in mlxbf_tmfifo_work_rxtx()
948 			if (vring->vq)  in mlxbf_tmfifo_work_rxtx()
949 				mlxbf_tmfifo_rxtx(vring, is_rx);  in mlxbf_tmfifo_work_rxtx()
979 	struct mlxbf_tmfifo_vring *vring = vq->priv;  in mlxbf_tmfifo_virtio_notify()  local
984 	fifo = vring->fifo;  in mlxbf_tmfifo_virtio_notify()
990 	if (vring->index & BIT(0)) {  in mlxbf_tmfifo_virtio_notify()
993 		 * In such case, the vring needs to be served right away. For  in mlxbf_tmfifo_virtio_notify()
997 		if (vring->vdev_id == VIRTIO_ID_CONSOLE) {  in mlxbf_tmfifo_virtio_notify()
1000 			mlxbf_tmfifo_console_output(tm_vdev, vring);  in mlxbf_tmfifo_virtio_notify()
1039 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_virtio_del_vqs()  local
1044 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_virtio_del_vqs()
1047 		if (vring->desc)  in mlxbf_tmfifo_virtio_del_vqs()
1048 			mlxbf_tmfifo_release_pkt(vring);  in mlxbf_tmfifo_virtio_del_vqs()
1049 		vq = vring->vq;  in mlxbf_tmfifo_virtio_del_vqs()
1051 			vring->vq = NULL;  in mlxbf_tmfifo_virtio_del_vqs()
1065 	struct mlxbf_tmfifo_vring *vring;  in mlxbf_tmfifo_virtio_find_vqs()  local
1079 		vring = &tm_vdev->vrings[i];  in mlxbf_tmfifo_virtio_find_vqs()
1081 		/* zero vring */  in mlxbf_tmfifo_virtio_find_vqs()
1082 		size = vring_size(vring->num, vring->align);  in mlxbf_tmfifo_virtio_find_vqs()
1083 		memset(vring->va, 0, size);  in mlxbf_tmfifo_virtio_find_vqs()
1084 		vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,  in mlxbf_tmfifo_virtio_find_vqs()
1085 					 false, false, vring->va,  in mlxbf_tmfifo_virtio_find_vqs()
1094 		vq->num_max = vring->num;  in mlxbf_tmfifo_virtio_find_vqs()
1096 		vq->priv = vring;  in mlxbf_tmfifo_virtio_find_vqs()
1102 		vring->vq = vq;  in mlxbf_tmfifo_virtio_find_vqs()
1220 		dev_err(dev, "unable to allocate vring\n");  in mlxbf_tmfifo_create_vdev()