91 	struct vring vring;  member
112 	 * vring.
125 	} vring;  member
357  * making all of the arch DMA ops work on the vring device itself
523 	struct vring_virtqueue *vring = to_vvq(vq);  in virtqueue_add_desc_split()  local
524 	struct vring_desc_extra *extra = vring->split.desc_extra;  in virtqueue_add_desc_split()
580 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
591 		desc = vq->split.vring.desc;  in virtqueue_add_split()
647 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=  in virtqueue_add_split()
662 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
687 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
688 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
694 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
754 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
757 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
777 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split()
820 			vq->split.vring.used->idx);  in more_used_split()
848 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
850 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
852 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
854 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
872 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
897 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
899 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
920 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
924 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
935 			vq->split.vring.used->idx);  in virtqueue_poll_split()
953 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
961 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
964 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
982 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
989 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
995 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
1015 			vring_split->vring.avail->flags = cpu_to_virtio16(vdev,  in virtqueue_vring_init_split()
1024 	num = vq->split.vring.num;  in virtqueue_reinit_split()
1026 	vq->split.vring.avail->flags = 0;  in virtqueue_reinit_split()
1027 	vq->split.vring.avail->idx = 0;  in virtqueue_reinit_split()
1030 	vq->split.vring.avail->ring[num] = 0;  in virtqueue_reinit_split()
1032 	vq->split.vring.used->flags = 0;  in virtqueue_reinit_split()
1033 	vq->split.vring.used->idx = 0;  in virtqueue_reinit_split()
1036 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;  in virtqueue_reinit_split()
1056 	u32 num = vring_split->vring.num;  in vring_alloc_state_extra_split()
1082 			 vring_split->vring.desc,  in vring_free_split()
1130 	vring_init(&vring_split->vring, num, queue, vring_align);  in vring_alloc_queue_split()
1197 	virtqueue_init(vq, vring_split.vring.num);  in virtqueue_resize_split()
1311 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_indirect_packed()
1338 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1339 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1341 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1357 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1365 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1446 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1448 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1460 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1493 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1524 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1545 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1578 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1594 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
1654 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
1701 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
1702 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
1704 	if (unlikely(id >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1718 	if (unlikely(last_used >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1719 		last_used -= vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
1733 				&vq->packed.vring.driver->off_wrap,  in virtqueue_get_buf_ctx_packed()
1756 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
1773 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
1786 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
1821 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
1826 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
1827 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
1831 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
1845 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
1875 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
1885 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
1913 	if (vring_packed->vring.desc)  in vring_free_packed()
1915 				 vring_packed->vring.desc,  in vring_free_packed()
1919 	if (vring_packed->vring.driver)  in vring_free_packed()
1921 				 vring_packed->vring.driver,  in vring_free_packed()
1925 	if (vring_packed->vring.device)  in vring_free_packed()
1927 				 vring_packed->vring.device,  in vring_free_packed()
1953 	vring_packed->vring.desc         = ring;  in vring_alloc_queue_packed()
1966 	vring_packed->vring.driver          = driver;  in vring_alloc_queue_packed()
1977 	vring_packed->vring.device          = device;  in vring_alloc_queue_packed()
1980 	vring_packed->vring.num = num;  in vring_alloc_queue_packed()
1993 	u32 num = vring_packed->vring.num;  in vring_alloc_state_extra_packed()
2027 		vring_packed->vring.driver->flags =  in virtqueue_vring_init_packed()
2043 	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
2044 	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
2047 	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);  in virtqueue_reinit_packed()
2049 	virtqueue_init(vq, vq->packed.vring.num);  in virtqueue_reinit_packed()
2143 	virtqueue_init(vq, vring_packed.vring.num);  in virtqueue_resize_packed()
2582 			      "virtio vring IRQ raised before DRIVER_OK");  in vring_interrupt()
2656 	virtqueue_init(vq, vring_split->vring.num);  in __vring_new_virtqueue()
2715  * virtqueue_resize - resize the vring of vq
2720  * When it is really necessary to create a new vring, it will set the current vq
2722  * that is no longer used. Only after the new vring is successfully created, the
2723  * old vring will be released.
2750 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)  in virtqueue_resize()
2767  * virtqueue_set_dma_premapped - set the vring premapped mode
2772  * The vring in premapped mode does not do dma internally, so the driver must
2775  * the vring, it has to unmap the dma address.
2794 	num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_set_dma_premapped()
2860 	vring_init(&vring_split.vring, num, pages, vring_align);  in vring_new_virtqueue()
2875 					 vq->packed.vring.desc,  in vring_free()
2881 					 vq->packed.vring.driver,  in vring_free()
2887 					 vq->packed.vring.device,  in vring_free()
2896 					 vq->split.vring.desc,  in vring_free()
2968  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2969  * @_vq: the struct virtqueue containing the vring of interest.
2971  * Returns the size of the vring.  This is mainly used for boasting to
2979 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
3079 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
3093 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
3098 const struct vring *virtqueue_get_vring(const struct virtqueue *vq)  in virtqueue_get_vring()
3100 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()