1  // SPDX-License-Identifier: GPL-2.0-only
2  /* Copyright (C) 2020 Red Hat, Inc.
3   * Author: Jason Wang <jasowang@redhat.com>
4   *
5   * IOTLB implementation for vhost.
6   */
7  #include <linux/slab.h>
8  #include <linux/vhost_iotlb.h>
9  #include <linux/module.h>
10  
11  #define MOD_VERSION  "0.1"
12  #define MOD_DESC     "VHOST IOTLB"
13  #define MOD_AUTHOR   "Jason Wang <jasowang@redhat.com>"
14  #define MOD_LICENSE  "GPL v2"
15  
16  #define START(map) ((map)->start)
17  #define LAST(map) ((map)->last)
18  
19  INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
20  		     rb, __u64, __subtree_last,
21  		     START, LAST, static inline, vhost_iotlb_itree);
22  
23  /**
24   * vhost_iotlb_map_free - remove a map node and free it
25   * @iotlb: the IOTLB
26   * @map: the map that want to be remove and freed
27   */
vhost_iotlb_map_free(struct vhost_iotlb * iotlb,struct vhost_iotlb_map * map)28  void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
29  			  struct vhost_iotlb_map *map)
30  {
31  	vhost_iotlb_itree_remove(map, &iotlb->root);
32  	list_del(&map->link);
33  	kfree(map);
34  	iotlb->nmaps--;
35  }
36  EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
37  
38  /**
39   * vhost_iotlb_add_range_ctx - add a new range to vhost IOTLB
40   * @iotlb: the IOTLB
41   * @start: start of the IOVA range
42   * @last: last of IOVA range
43   * @addr: the address that is mapped to @start
44   * @perm: access permission of this range
45   * @opaque: the opaque pointer for the new mapping
46   *
47   * Returns an error last is smaller than start or memory allocation
48   * fails
49   */
vhost_iotlb_add_range_ctx(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm,void * opaque)50  int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
51  			      u64 start, u64 last,
52  			      u64 addr, unsigned int perm,
53  			      void *opaque)
54  {
55  	struct vhost_iotlb_map *map;
56  
57  	if (last < start)
58  		return -EFAULT;
59  
60  	/* If the range being mapped is [0, ULONG_MAX], split it into two entries
61  	 * otherwise its size would overflow u64.
62  	 */
63  	if (start == 0 && last == ULONG_MAX) {
64  		u64 mid = last / 2;
65  		int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
66  				perm, opaque);
67  
68  		if (err)
69  			return err;
70  
71  		addr += mid + 1;
72  		start = mid + 1;
73  	}
74  
75  	if (iotlb->limit &&
76  	    iotlb->nmaps == iotlb->limit &&
77  	    iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
78  		map = list_first_entry(&iotlb->list, typeof(*map), link);
79  		vhost_iotlb_map_free(iotlb, map);
80  	}
81  
82  	map = kmalloc(sizeof(*map), GFP_ATOMIC);
83  	if (!map)
84  		return -ENOMEM;
85  
86  	map->start = start;
87  	map->size = last - start + 1;
88  	map->last = last;
89  	map->addr = addr;
90  	map->perm = perm;
91  	map->opaque = opaque;
92  
93  	iotlb->nmaps++;
94  	vhost_iotlb_itree_insert(map, &iotlb->root);
95  
96  	INIT_LIST_HEAD(&map->link);
97  	list_add_tail(&map->link, &iotlb->list);
98  
99  	return 0;
100  }
101  EXPORT_SYMBOL_GPL(vhost_iotlb_add_range_ctx);
102  
vhost_iotlb_add_range(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm)103  int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
104  			  u64 start, u64 last,
105  			  u64 addr, unsigned int perm)
106  {
107  	return vhost_iotlb_add_range_ctx(iotlb, start, last,
108  					 addr, perm, NULL);
109  }
110  EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
111  
112  /**
113   * vhost_iotlb_del_range - delete overlapped ranges from vhost IOTLB
114   * @iotlb: the IOTLB
115   * @start: start of the IOVA range
116   * @last: last of IOVA range
117   */
vhost_iotlb_del_range(struct vhost_iotlb * iotlb,u64 start,u64 last)118  void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
119  {
120  	struct vhost_iotlb_map *map;
121  
122  	while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
123  						   start, last)))
124  		vhost_iotlb_map_free(iotlb, map);
125  }
126  EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
127  
128  /**
129   * vhost_iotlb_init - initialize a vhost IOTLB
130   * @iotlb: the IOTLB that needs to be initialized
131   * @limit: maximum number of IOTLB entries
132   * @flags: VHOST_IOTLB_FLAG_XXX
133   */
vhost_iotlb_init(struct vhost_iotlb * iotlb,unsigned int limit,unsigned int flags)134  void vhost_iotlb_init(struct vhost_iotlb *iotlb, unsigned int limit,
135  		      unsigned int flags)
136  {
137  	iotlb->root = RB_ROOT_CACHED;
138  	iotlb->limit = limit;
139  	iotlb->nmaps = 0;
140  	iotlb->flags = flags;
141  	INIT_LIST_HEAD(&iotlb->list);
142  }
143  EXPORT_SYMBOL_GPL(vhost_iotlb_init);
144  
145  /**
146   * vhost_iotlb_alloc - add a new vhost IOTLB
147   * @limit: maximum number of IOTLB entries
148   * @flags: VHOST_IOTLB_FLAG_XXX
149   *
150   * Returns an error is memory allocation fails
151   */
vhost_iotlb_alloc(unsigned int limit,unsigned int flags)152  struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
153  {
154  	struct vhost_iotlb *iotlb = kzalloc(sizeof(*iotlb), GFP_KERNEL);
155  
156  	if (!iotlb)
157  		return NULL;
158  
159  	vhost_iotlb_init(iotlb, limit, flags);
160  
161  	return iotlb;
162  }
163  EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
164  
165  /**
166   * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
167   * @iotlb: the IOTLB to be reset
168   */
vhost_iotlb_reset(struct vhost_iotlb * iotlb)169  void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
170  {
171  	vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
172  }
173  EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
174  
175  /**
176   * vhost_iotlb_free - reset and free vhost IOTLB
177   * @iotlb: the IOTLB to be freed
178   */
vhost_iotlb_free(struct vhost_iotlb * iotlb)179  void vhost_iotlb_free(struct vhost_iotlb *iotlb)
180  {
181  	if (iotlb) {
182  		vhost_iotlb_reset(iotlb);
183  		kfree(iotlb);
184  	}
185  }
186  EXPORT_SYMBOL_GPL(vhost_iotlb_free);
187  
188  /**
189   * vhost_iotlb_itree_first - return the first overlapped range
190   * @iotlb: the IOTLB
191   * @start: start of IOVA range
192   * @last: last byte in IOVA range
193   */
194  struct vhost_iotlb_map *
vhost_iotlb_itree_first(struct vhost_iotlb * iotlb,u64 start,u64 last)195  vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
196  {
197  	return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
198  }
199  EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
200  
201  /**
202   * vhost_iotlb_itree_next - return the next overlapped range
203   * @map: the starting map node
204   * @start: start of IOVA range
205   * @last: last byte IOVA range
206   */
207  struct vhost_iotlb_map *
vhost_iotlb_itree_next(struct vhost_iotlb_map * map,u64 start,u64 last)208  vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
209  {
210  	return vhost_iotlb_itree_iter_next(map, start, last);
211  }
212  EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
213  
214  MODULE_VERSION(MOD_VERSION);
215  MODULE_DESCRIPTION(MOD_DESC);
216  MODULE_AUTHOR(MOD_AUTHOR);
217  MODULE_LICENSE(MOD_LICENSE);
218