37 /* Returns vring->num if empty, -ve on error. */
46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
251 	*descs = vrh->vring.desc;  in return_from_indirect()
252 	*desc_max = vrh->vring.num;  in return_from_indirect()
310 	/* We start traversing vring's descriptor table. */  in __vringh_iov()
311 	descs = vrh->vring.desc;  in __vringh_iov()
312 	desc_max = vrh->vring.num;  in __vringh_iov()
367 		if (count > vrh->vring.num || indirect_count > desc_max) {  in __vringh_iov()
461 	used_ring = vrh->vring.used;  in __vringh_complete()
464 	off = used_idx % vrh->vring.num;  in __vringh_complete()
467 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
468 		u16 part = vrh->vring.num - off;  in __vringh_complete()
485 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
488 			   &vrh->vring.used->idx);  in __vringh_complete()
514 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
517 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
524 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
527 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
554 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
556 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
560 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
563 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
572 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
574 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
590 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
593 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
644  * vringh_init_user - initialize a vringh for a userspace vring.
674 	vrh->vring.num = num;  in vringh_init_user()
675 	/* vring expects kernel addresses, but only used via accessors. */  in vringh_init_user()
676 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
677 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
678 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
685  * @vrh: the userspace vring.
695  * *head will be vrh->vring.num.  You may be able to ignore an invalid
712 	*head = vrh->vring.num;  in vringh_getdesc_user()
718 	if (err == vrh->vring.num)  in vringh_getdesc_user()
785  * @vrh: the vring.
801  * @vrh: the vring.
820  * @vrh: the vring.
838  * @vrh: the vring.
841  * now more buffers available in the vring.
851  * @vrh: the vring.
864  * @vrh: the vring we've called vringh_complete_user() on.
919  * vringh_init_kern - initialize a vringh for a kernelspace vring.
948 	vrh->vring.num = num;  in vringh_init_kern()
949 	vrh->vring.desc = desc;  in vringh_init_kern()
950 	vrh->vring.avail = avail;  in vringh_init_kern()
951 	vrh->vring.used = used;  in vringh_init_kern()
958  * @vrh: the kernelspace vring.
968  * *head will be vrh->vring.num.  You may be able to ignore an invalid
989 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
1033  * @vrh: the vring.
1049  * @vrh: the vring.
1069  * @vrh: the vring.
1072  * now more buffers available in the vring.
1082  * @vrh: the vring.
1095  * @vrh: the vring we've called vringh_complete_kern() on.
1447  * @vrh: the vring
1448  * @iotlb: iotlb associated with this vring
1462  * @vrh: the kernelspace vring.
1472  * *head will be vrh->vring.num.  You may be able to ignore an invalid
1493 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()
1508  * @vrh: the vring.
1525  * @vrh: the vring.
1542  * @vrh: the vring.
1559  * @vrh: the vring.
1579  * @vrh: the vring.
1582  * now more buffers available in the vring.
1592  * @vrh: the vring.
1605  * @vrh: the vring we've called vringh_complete_iotlb() on.