1  /*
2   * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3   * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
4   * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
5   * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.
6   *
7   * This software is available to you under a choice of one of two
8   * licenses.  You may choose to be licensed under the terms of the GNU
9   * General Public License (GPL) Version 2, available from the file
10   * COPYING in the main directory of this source tree, or the
11   * OpenIB.org BSD license below:
12   *
13   *     Redistribution and use in source and binary forms, with or
14   *     without modification, are permitted provided that the following
15   *     conditions are met:
16   *
17   *      - Redistributions of source code must retain the above
18   *        copyright notice, this list of conditions and the following
19   *        disclaimer.
20   *
21   *      - Redistributions in binary form must reproduce the above
22   *        copyright notice, this list of conditions and the following
23   *        disclaimer in the documentation and/or other materials
24   *        provided with the distribution.
25   *
26   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29   * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30   * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31   * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32   * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33   * SOFTWARE.
34   */
35  
36  #include <linux/file.h>
37  #include <linux/fs.h>
38  #include <linux/slab.h>
39  #include <linux/sched.h>
40  
41  #include <linux/uaccess.h>
42  
43  #include <rdma/uverbs_types.h>
44  #include <rdma/uverbs_std_types.h>
45  #include "rdma_core.h"
46  
47  #include "uverbs.h"
48  #include "core_priv.h"
49  
50  /*
51   * Copy a response to userspace. If the provided 'resp' is larger than the
52   * user buffer it is silently truncated. If the user provided a larger buffer
53   * then the trailing portion is zero filled.
54   *
55   * These semantics are intended to support future extension of the output
56   * structures.
57   */
uverbs_response(struct uverbs_attr_bundle * attrs,const void * resp,size_t resp_len)58  static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
59  			   size_t resp_len)
60  {
61  	int ret;
62  
63  	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
64  		return uverbs_copy_to_struct_or_zero(
65  			attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);
66  
67  	if (copy_to_user(attrs->ucore.outbuf, resp,
68  			 min(attrs->ucore.outlen, resp_len)))
69  		return -EFAULT;
70  
71  	if (resp_len < attrs->ucore.outlen) {
72  		/*
73  		 * Zero fill any extra memory that user
74  		 * space might have provided.
75  		 */
76  		ret = clear_user(attrs->ucore.outbuf + resp_len,
77  				 attrs->ucore.outlen - resp_len);
78  		if (ret)
79  			return -EFAULT;
80  	}
81  
82  	return 0;
83  }
84  
85  /*
86   * Copy a request from userspace. If the provided 'req' is larger than the
87   * user buffer then the user buffer is zero extended into the 'req'. If 'req'
88   * is smaller than the user buffer then the uncopied bytes in the user buffer
89   * must be zero.
90   */
uverbs_request(struct uverbs_attr_bundle * attrs,void * req,size_t req_len)91  static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
92  			  size_t req_len)
93  {
94  	if (copy_from_user(req, attrs->ucore.inbuf,
95  			   min(attrs->ucore.inlen, req_len)))
96  		return -EFAULT;
97  
98  	if (attrs->ucore.inlen < req_len) {
99  		memset(req + attrs->ucore.inlen, 0,
100  		       req_len - attrs->ucore.inlen);
101  	} else if (attrs->ucore.inlen > req_len) {
102  		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
103  					  attrs->ucore.inlen - req_len))
104  			return -EOPNOTSUPP;
105  	}
106  	return 0;
107  }
108  
109  /*
110   * Generate the value for the 'response_length' protocol used by write_ex.
111   * This is the number of bytes the kernel actually wrote. Userspace can use
112   * this to detect what structure members in the response the kernel
113   * understood.
114   */
uverbs_response_length(struct uverbs_attr_bundle * attrs,size_t resp_len)115  static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
116  				  size_t resp_len)
117  {
118  	return min_t(size_t, attrs->ucore.outlen, resp_len);
119  }
120  
121  /*
122   * The iterator version of the request interface is for handlers that need to
123   * step over a flex array at the end of a command header.
124   */
125  struct uverbs_req_iter {
126  	const void __user *cur;
127  	const void __user *end;
128  };
129  
uverbs_request_start(struct uverbs_attr_bundle * attrs,struct uverbs_req_iter * iter,void * req,size_t req_len)130  static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
131  				struct uverbs_req_iter *iter,
132  				void *req,
133  				size_t req_len)
134  {
135  	if (attrs->ucore.inlen < req_len)
136  		return -ENOSPC;
137  
138  	if (copy_from_user(req, attrs->ucore.inbuf, req_len))
139  		return -EFAULT;
140  
141  	iter->cur = attrs->ucore.inbuf + req_len;
142  	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
143  	return 0;
144  }
145  
uverbs_request_next(struct uverbs_req_iter * iter,void * val,size_t len)146  static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,
147  			       size_t len)
148  {
149  	if (iter->cur + len > iter->end)
150  		return -ENOSPC;
151  
152  	if (copy_from_user(val, iter->cur, len))
153  		return -EFAULT;
154  
155  	iter->cur += len;
156  	return 0;
157  }
158  
uverbs_request_next_ptr(struct uverbs_req_iter * iter,size_t len)159  static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
160  						  size_t len)
161  {
162  	const void __user *res = iter->cur;
163  
164  	if (iter->cur + len > iter->end)
165  		return (void __force __user *)ERR_PTR(-ENOSPC);
166  	iter->cur += len;
167  	return res;
168  }
169  
uverbs_request_finish(struct uverbs_req_iter * iter)170  static int uverbs_request_finish(struct uverbs_req_iter *iter)
171  {
172  	if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
173  		return -EOPNOTSUPP;
174  	return 0;
175  }
176  
177  /*
178   * When calling a destroy function during an error unwind we need to pass in
179   * the udata that is sanitized of all user arguments. Ie from the driver
180   * perspective it looks like no udata was passed.
181   */
uverbs_get_cleared_udata(struct uverbs_attr_bundle * attrs)182  struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
183  {
184  	attrs->driver_udata = (struct ib_udata){};
185  	return &attrs->driver_udata;
186  }
187  
188  static struct ib_uverbs_completion_event_file *
_ib_uverbs_lookup_comp_file(s32 fd,struct uverbs_attr_bundle * attrs)189  _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
190  {
191  	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
192  					       fd, attrs);
193  
194  	if (IS_ERR(uobj))
195  		return (void *)uobj;
196  
197  	uverbs_uobject_get(uobj);
198  	uobj_put_read(uobj);
199  
200  	return container_of(uobj, struct ib_uverbs_completion_event_file,
201  			    uobj);
202  }
203  #define ib_uverbs_lookup_comp_file(_fd, _ufile)                                \
204  	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
205  
ib_alloc_ucontext(struct uverbs_attr_bundle * attrs)206  int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)
207  {
208  	struct ib_uverbs_file *ufile = attrs->ufile;
209  	struct ib_ucontext *ucontext;
210  	struct ib_device *ib_dev;
211  
212  	ib_dev = srcu_dereference(ufile->device->ib_dev,
213  				  &ufile->device->disassociate_srcu);
214  	if (!ib_dev)
215  		return -EIO;
216  
217  	ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);
218  	if (!ucontext)
219  		return -ENOMEM;
220  
221  	ucontext->device = ib_dev;
222  	ucontext->ufile = ufile;
223  	xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC);
224  
225  	rdma_restrack_new(&ucontext->res, RDMA_RESTRACK_CTX);
226  	rdma_restrack_set_name(&ucontext->res, NULL);
227  	attrs->context = ucontext;
228  	return 0;
229  }
230  
ib_init_ucontext(struct uverbs_attr_bundle * attrs)231  int ib_init_ucontext(struct uverbs_attr_bundle *attrs)
232  {
233  	struct ib_ucontext *ucontext = attrs->context;
234  	struct ib_uverbs_file *file = attrs->ufile;
235  	int ret;
236  
237  	if (!down_read_trylock(&file->hw_destroy_rwsem))
238  		return -EIO;
239  	mutex_lock(&file->ucontext_lock);
240  	if (file->ucontext) {
241  		ret = -EINVAL;
242  		goto err;
243  	}
244  
245  	ret = ib_rdmacg_try_charge(&ucontext->cg_obj, ucontext->device,
246  				   RDMACG_RESOURCE_HCA_HANDLE);
247  	if (ret)
248  		goto err;
249  
250  	ret = ucontext->device->ops.alloc_ucontext(ucontext,
251  						   &attrs->driver_udata);
252  	if (ret)
253  		goto err_uncharge;
254  
255  	rdma_restrack_add(&ucontext->res);
256  
257  	/*
258  	 * Make sure that ib_uverbs_get_ucontext() sees the pointer update
259  	 * only after all writes to setup the ucontext have completed
260  	 */
261  	smp_store_release(&file->ucontext, ucontext);
262  
263  	mutex_unlock(&file->ucontext_lock);
264  	up_read(&file->hw_destroy_rwsem);
265  	return 0;
266  
267  err_uncharge:
268  	ib_rdmacg_uncharge(&ucontext->cg_obj, ucontext->device,
269  			   RDMACG_RESOURCE_HCA_HANDLE);
270  err:
271  	mutex_unlock(&file->ucontext_lock);
272  	up_read(&file->hw_destroy_rwsem);
273  	return ret;
274  }
275  
ib_uverbs_get_context(struct uverbs_attr_bundle * attrs)276  static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
277  {
278  	struct ib_uverbs_get_context_resp resp;
279  	struct ib_uverbs_get_context cmd;
280  	struct ib_device *ib_dev;
281  	struct ib_uobject *uobj;
282  	int ret;
283  
284  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
285  	if (ret)
286  		return ret;
287  
288  	ret = ib_alloc_ucontext(attrs);
289  	if (ret)
290  		return ret;
291  
292  	uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);
293  	if (IS_ERR(uobj)) {
294  		ret = PTR_ERR(uobj);
295  		goto err_ucontext;
296  	}
297  
298  	resp = (struct ib_uverbs_get_context_resp){
299  		.num_comp_vectors = attrs->ufile->device->num_comp_vectors,
300  		.async_fd = uobj->id,
301  	};
302  	ret = uverbs_response(attrs, &resp, sizeof(resp));
303  	if (ret)
304  		goto err_uobj;
305  
306  	ret = ib_init_ucontext(attrs);
307  	if (ret)
308  		goto err_uobj;
309  
310  	ib_uverbs_init_async_event_file(
311  		container_of(uobj, struct ib_uverbs_async_event_file, uobj));
312  	rdma_alloc_commit_uobject(uobj, attrs);
313  	return 0;
314  
315  err_uobj:
316  	rdma_alloc_abort_uobject(uobj, attrs, false);
317  err_ucontext:
318  	rdma_restrack_put(&attrs->context->res);
319  	kfree(attrs->context);
320  	attrs->context = NULL;
321  	return ret;
322  }
323  
copy_query_dev_fields(struct ib_ucontext * ucontext,struct ib_uverbs_query_device_resp * resp,struct ib_device_attr * attr)324  static void copy_query_dev_fields(struct ib_ucontext *ucontext,
325  				  struct ib_uverbs_query_device_resp *resp,
326  				  struct ib_device_attr *attr)
327  {
328  	struct ib_device *ib_dev = ucontext->device;
329  
330  	resp->fw_ver		= attr->fw_ver;
331  	resp->node_guid		= ib_dev->node_guid;
332  	resp->sys_image_guid	= attr->sys_image_guid;
333  	resp->max_mr_size	= attr->max_mr_size;
334  	resp->page_size_cap	= attr->page_size_cap;
335  	resp->vendor_id		= attr->vendor_id;
336  	resp->vendor_part_id	= attr->vendor_part_id;
337  	resp->hw_ver		= attr->hw_ver;
338  	resp->max_qp		= attr->max_qp;
339  	resp->max_qp_wr		= attr->max_qp_wr;
340  	resp->device_cap_flags  = lower_32_bits(attr->device_cap_flags);
341  	resp->max_sge		= min(attr->max_send_sge, attr->max_recv_sge);
342  	resp->max_sge_rd	= attr->max_sge_rd;
343  	resp->max_cq		= attr->max_cq;
344  	resp->max_cqe		= attr->max_cqe;
345  	resp->max_mr		= attr->max_mr;
346  	resp->max_pd		= attr->max_pd;
347  	resp->max_qp_rd_atom	= attr->max_qp_rd_atom;
348  	resp->max_ee_rd_atom	= attr->max_ee_rd_atom;
349  	resp->max_res_rd_atom	= attr->max_res_rd_atom;
350  	resp->max_qp_init_rd_atom	= attr->max_qp_init_rd_atom;
351  	resp->max_ee_init_rd_atom	= attr->max_ee_init_rd_atom;
352  	resp->atomic_cap		= attr->atomic_cap;
353  	resp->max_ee			= attr->max_ee;
354  	resp->max_rdd			= attr->max_rdd;
355  	resp->max_mw			= attr->max_mw;
356  	resp->max_raw_ipv6_qp		= attr->max_raw_ipv6_qp;
357  	resp->max_raw_ethy_qp		= attr->max_raw_ethy_qp;
358  	resp->max_mcast_grp		= attr->max_mcast_grp;
359  	resp->max_mcast_qp_attach	= attr->max_mcast_qp_attach;
360  	resp->max_total_mcast_qp_attach	= attr->max_total_mcast_qp_attach;
361  	resp->max_ah			= attr->max_ah;
362  	resp->max_srq			= attr->max_srq;
363  	resp->max_srq_wr		= attr->max_srq_wr;
364  	resp->max_srq_sge		= attr->max_srq_sge;
365  	resp->max_pkeys			= attr->max_pkeys;
366  	resp->local_ca_ack_delay	= attr->local_ca_ack_delay;
367  	resp->phys_port_cnt = min_t(u32, ib_dev->phys_port_cnt, U8_MAX);
368  }
369  
ib_uverbs_query_device(struct uverbs_attr_bundle * attrs)370  static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
371  {
372  	struct ib_uverbs_query_device      cmd;
373  	struct ib_uverbs_query_device_resp resp;
374  	struct ib_ucontext *ucontext;
375  	int ret;
376  
377  	ucontext = ib_uverbs_get_ucontext(attrs);
378  	if (IS_ERR(ucontext))
379  		return PTR_ERR(ucontext);
380  
381  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
382  	if (ret)
383  		return ret;
384  
385  	memset(&resp, 0, sizeof resp);
386  	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
387  
388  	return uverbs_response(attrs, &resp, sizeof(resp));
389  }
390  
ib_uverbs_query_port(struct uverbs_attr_bundle * attrs)391  static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
392  {
393  	struct ib_uverbs_query_port      cmd;
394  	struct ib_uverbs_query_port_resp resp;
395  	struct ib_port_attr              attr;
396  	int                              ret;
397  	struct ib_ucontext *ucontext;
398  	struct ib_device *ib_dev;
399  
400  	ucontext = ib_uverbs_get_ucontext(attrs);
401  	if (IS_ERR(ucontext))
402  		return PTR_ERR(ucontext);
403  	ib_dev = ucontext->device;
404  
405  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
406  	if (ret)
407  		return ret;
408  
409  	ret = ib_query_port(ib_dev, cmd.port_num, &attr);
410  	if (ret)
411  		return ret;
412  
413  	memset(&resp, 0, sizeof resp);
414  	copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);
415  
416  	return uverbs_response(attrs, &resp, sizeof(resp));
417  }
418  
ib_uverbs_alloc_pd(struct uverbs_attr_bundle * attrs)419  static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
420  {
421  	struct ib_uverbs_alloc_pd_resp resp = {};
422  	struct ib_uverbs_alloc_pd      cmd;
423  	struct ib_uobject             *uobj;
424  	struct ib_pd                  *pd;
425  	int                            ret;
426  	struct ib_device *ib_dev;
427  
428  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
429  	if (ret)
430  		return ret;
431  
432  	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
433  	if (IS_ERR(uobj))
434  		return PTR_ERR(uobj);
435  
436  	pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);
437  	if (!pd) {
438  		ret = -ENOMEM;
439  		goto err;
440  	}
441  
442  	pd->device  = ib_dev;
443  	pd->uobject = uobj;
444  	atomic_set(&pd->usecnt, 0);
445  
446  	rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
447  	rdma_restrack_set_name(&pd->res, NULL);
448  
449  	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
450  	if (ret)
451  		goto err_alloc;
452  	rdma_restrack_add(&pd->res);
453  
454  	uobj->object = pd;
455  	uobj_finalize_uobj_create(uobj, attrs);
456  
457  	resp.pd_handle = uobj->id;
458  	return uverbs_response(attrs, &resp, sizeof(resp));
459  
460  err_alloc:
461  	rdma_restrack_put(&pd->res);
462  	kfree(pd);
463  err:
464  	uobj_alloc_abort(uobj, attrs);
465  	return ret;
466  }
467  
ib_uverbs_dealloc_pd(struct uverbs_attr_bundle * attrs)468  static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
469  {
470  	struct ib_uverbs_dealloc_pd cmd;
471  	int ret;
472  
473  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
474  	if (ret)
475  		return ret;
476  
477  	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
478  }
479  
480  struct xrcd_table_entry {
481  	struct rb_node  node;
482  	struct ib_xrcd *xrcd;
483  	struct inode   *inode;
484  };
485  
xrcd_table_insert(struct ib_uverbs_device * dev,struct inode * inode,struct ib_xrcd * xrcd)486  static int xrcd_table_insert(struct ib_uverbs_device *dev,
487  			    struct inode *inode,
488  			    struct ib_xrcd *xrcd)
489  {
490  	struct xrcd_table_entry *entry, *scan;
491  	struct rb_node **p = &dev->xrcd_tree.rb_node;
492  	struct rb_node *parent = NULL;
493  
494  	entry = kmalloc(sizeof *entry, GFP_KERNEL);
495  	if (!entry)
496  		return -ENOMEM;
497  
498  	entry->xrcd  = xrcd;
499  	entry->inode = inode;
500  
501  	while (*p) {
502  		parent = *p;
503  		scan = rb_entry(parent, struct xrcd_table_entry, node);
504  
505  		if (inode < scan->inode) {
506  			p = &(*p)->rb_left;
507  		} else if (inode > scan->inode) {
508  			p = &(*p)->rb_right;
509  		} else {
510  			kfree(entry);
511  			return -EEXIST;
512  		}
513  	}
514  
515  	rb_link_node(&entry->node, parent, p);
516  	rb_insert_color(&entry->node, &dev->xrcd_tree);
517  	igrab(inode);
518  	return 0;
519  }
520  
xrcd_table_search(struct ib_uverbs_device * dev,struct inode * inode)521  static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
522  						  struct inode *inode)
523  {
524  	struct xrcd_table_entry *entry;
525  	struct rb_node *p = dev->xrcd_tree.rb_node;
526  
527  	while (p) {
528  		entry = rb_entry(p, struct xrcd_table_entry, node);
529  
530  		if (inode < entry->inode)
531  			p = p->rb_left;
532  		else if (inode > entry->inode)
533  			p = p->rb_right;
534  		else
535  			return entry;
536  	}
537  
538  	return NULL;
539  }
540  
find_xrcd(struct ib_uverbs_device * dev,struct inode * inode)541  static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
542  {
543  	struct xrcd_table_entry *entry;
544  
545  	entry = xrcd_table_search(dev, inode);
546  	if (!entry)
547  		return NULL;
548  
549  	return entry->xrcd;
550  }
551  
xrcd_table_delete(struct ib_uverbs_device * dev,struct inode * inode)552  static void xrcd_table_delete(struct ib_uverbs_device *dev,
553  			      struct inode *inode)
554  {
555  	struct xrcd_table_entry *entry;
556  
557  	entry = xrcd_table_search(dev, inode);
558  	if (entry) {
559  		iput(inode);
560  		rb_erase(&entry->node, &dev->xrcd_tree);
561  		kfree(entry);
562  	}
563  }
564  
ib_uverbs_open_xrcd(struct uverbs_attr_bundle * attrs)565  static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
566  {
567  	struct ib_uverbs_device *ibudev = attrs->ufile->device;
568  	struct ib_uverbs_open_xrcd_resp	resp = {};
569  	struct ib_uverbs_open_xrcd	cmd;
570  	struct ib_uxrcd_object         *obj;
571  	struct ib_xrcd                 *xrcd = NULL;
572  	struct inode                   *inode = NULL;
573  	int				new_xrcd = 0;
574  	struct ib_device *ib_dev;
575  	struct fd f = EMPTY_FD;
576  	int ret;
577  
578  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
579  	if (ret)
580  		return ret;
581  
582  	mutex_lock(&ibudev->xrcd_tree_mutex);
583  
584  	if (cmd.fd != -1) {
585  		/* search for file descriptor */
586  		f = fdget(cmd.fd);
587  		if (!fd_file(f)) {
588  			ret = -EBADF;
589  			goto err_tree_mutex_unlock;
590  		}
591  
592  		inode = file_inode(fd_file(f));
593  		xrcd = find_xrcd(ibudev, inode);
594  		if (!xrcd && !(cmd.oflags & O_CREAT)) {
595  			/* no file descriptor. Need CREATE flag */
596  			ret = -EAGAIN;
597  			goto err_tree_mutex_unlock;
598  		}
599  
600  		if (xrcd && cmd.oflags & O_EXCL) {
601  			ret = -EINVAL;
602  			goto err_tree_mutex_unlock;
603  		}
604  	}
605  
606  	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
607  						   &ib_dev);
608  	if (IS_ERR(obj)) {
609  		ret = PTR_ERR(obj);
610  		goto err_tree_mutex_unlock;
611  	}
612  
613  	if (!xrcd) {
614  		xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);
615  		if (IS_ERR(xrcd)) {
616  			ret = PTR_ERR(xrcd);
617  			goto err;
618  		}
619  		new_xrcd = 1;
620  	}
621  
622  	atomic_set(&obj->refcnt, 0);
623  	obj->uobject.object = xrcd;
624  
625  	if (inode) {
626  		if (new_xrcd) {
627  			/* create new inode/xrcd table entry */
628  			ret = xrcd_table_insert(ibudev, inode, xrcd);
629  			if (ret)
630  				goto err_dealloc_xrcd;
631  		}
632  		atomic_inc(&xrcd->usecnt);
633  	}
634  
635  	if (fd_file(f))
636  		fdput(f);
637  
638  	mutex_unlock(&ibudev->xrcd_tree_mutex);
639  	uobj_finalize_uobj_create(&obj->uobject, attrs);
640  
641  	resp.xrcd_handle = obj->uobject.id;
642  	return uverbs_response(attrs, &resp, sizeof(resp));
643  
644  err_dealloc_xrcd:
645  	ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));
646  
647  err:
648  	uobj_alloc_abort(&obj->uobject, attrs);
649  
650  err_tree_mutex_unlock:
651  	if (fd_file(f))
652  		fdput(f);
653  
654  	mutex_unlock(&ibudev->xrcd_tree_mutex);
655  
656  	return ret;
657  }
658  
ib_uverbs_close_xrcd(struct uverbs_attr_bundle * attrs)659  static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
660  {
661  	struct ib_uverbs_close_xrcd cmd;
662  	int ret;
663  
664  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
665  	if (ret)
666  		return ret;
667  
668  	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
669  }
670  
ib_uverbs_dealloc_xrcd(struct ib_uobject * uobject,struct ib_xrcd * xrcd,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)671  int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
672  			   enum rdma_remove_reason why,
673  			   struct uverbs_attr_bundle *attrs)
674  {
675  	struct inode *inode;
676  	int ret;
677  	struct ib_uverbs_device *dev = attrs->ufile->device;
678  
679  	inode = xrcd->inode;
680  	if (inode && !atomic_dec_and_test(&xrcd->usecnt))
681  		return 0;
682  
683  	ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);
684  	if (ret) {
685  		atomic_inc(&xrcd->usecnt);
686  		return ret;
687  	}
688  
689  	if (inode)
690  		xrcd_table_delete(dev, inode);
691  
692  	return 0;
693  }
694  
ib_uverbs_reg_mr(struct uverbs_attr_bundle * attrs)695  static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
696  {
697  	struct ib_uverbs_reg_mr_resp resp = {};
698  	struct ib_uverbs_reg_mr      cmd;
699  	struct ib_uobject           *uobj;
700  	struct ib_pd                *pd;
701  	struct ib_mr                *mr;
702  	int                          ret;
703  	struct ib_device *ib_dev;
704  
705  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
706  	if (ret)
707  		return ret;
708  
709  	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
710  		return -EINVAL;
711  
712  	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
713  	if (IS_ERR(uobj))
714  		return PTR_ERR(uobj);
715  
716  	ret = ib_check_mr_access(ib_dev, cmd.access_flags);
717  	if (ret)
718  		goto err_free;
719  
720  	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
721  	if (!pd) {
722  		ret = -EINVAL;
723  		goto err_free;
724  	}
725  
726  	mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
727  					 cmd.access_flags,
728  					 &attrs->driver_udata);
729  	if (IS_ERR(mr)) {
730  		ret = PTR_ERR(mr);
731  		goto err_put;
732  	}
733  
734  	mr->device  = pd->device;
735  	mr->pd      = pd;
736  	mr->type    = IB_MR_TYPE_USER;
737  	mr->dm	    = NULL;
738  	mr->sig_attrs = NULL;
739  	mr->uobject = uobj;
740  	atomic_inc(&pd->usecnt);
741  	mr->iova = cmd.hca_va;
742  	mr->length = cmd.length;
743  
744  	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
745  	rdma_restrack_set_name(&mr->res, NULL);
746  	rdma_restrack_add(&mr->res);
747  
748  	uobj->object = mr;
749  	uobj_put_obj_read(pd);
750  	uobj_finalize_uobj_create(uobj, attrs);
751  
752  	resp.lkey = mr->lkey;
753  	resp.rkey = mr->rkey;
754  	resp.mr_handle = uobj->id;
755  	return uverbs_response(attrs, &resp, sizeof(resp));
756  
757  err_put:
758  	uobj_put_obj_read(pd);
759  err_free:
760  	uobj_alloc_abort(uobj, attrs);
761  	return ret;
762  }
763  
ib_uverbs_rereg_mr(struct uverbs_attr_bundle * attrs)764  static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
765  {
766  	struct ib_uverbs_rereg_mr      cmd;
767  	struct ib_uverbs_rereg_mr_resp resp;
768  	struct ib_mr                *mr;
769  	int                          ret;
770  	struct ib_uobject	    *uobj;
771  	struct ib_uobject *new_uobj;
772  	struct ib_device *ib_dev;
773  	struct ib_pd *orig_pd;
774  	struct ib_pd *new_pd;
775  	struct ib_mr *new_mr;
776  
777  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
778  	if (ret)
779  		return ret;
780  
781  	if (!cmd.flags)
782  		return -EINVAL;
783  
784  	if (cmd.flags & ~IB_MR_REREG_SUPPORTED)
785  		return -EOPNOTSUPP;
786  
787  	if ((cmd.flags & IB_MR_REREG_TRANS) &&
788  	    (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
789  		return -EINVAL;
790  
791  	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
792  	if (IS_ERR(uobj))
793  		return PTR_ERR(uobj);
794  
795  	mr = uobj->object;
796  
797  	if (mr->dm) {
798  		ret = -EINVAL;
799  		goto put_uobjs;
800  	}
801  
802  	if (cmd.flags & IB_MR_REREG_ACCESS) {
803  		ret = ib_check_mr_access(mr->device, cmd.access_flags);
804  		if (ret)
805  			goto put_uobjs;
806  	}
807  
808  	orig_pd = mr->pd;
809  	if (cmd.flags & IB_MR_REREG_PD) {
810  		new_pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
811  					   attrs);
812  		if (!new_pd) {
813  			ret = -EINVAL;
814  			goto put_uobjs;
815  		}
816  	} else {
817  		new_pd = mr->pd;
818  	}
819  
820  	/*
821  	 * The driver might create a new HW object as part of the rereg, we need
822  	 * to have a uobject ready to hold it.
823  	 */
824  	new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
825  	if (IS_ERR(new_uobj)) {
826  		ret = PTR_ERR(new_uobj);
827  		goto put_uobj_pd;
828  	}
829  
830  	new_mr = ib_dev->ops.rereg_user_mr(mr, cmd.flags, cmd.start, cmd.length,
831  					   cmd.hca_va, cmd.access_flags, new_pd,
832  					   &attrs->driver_udata);
833  	if (IS_ERR(new_mr)) {
834  		ret = PTR_ERR(new_mr);
835  		goto put_new_uobj;
836  	}
837  	if (new_mr) {
838  		new_mr->device = new_pd->device;
839  		new_mr->pd = new_pd;
840  		new_mr->type = IB_MR_TYPE_USER;
841  		new_mr->uobject = uobj;
842  		atomic_inc(&new_pd->usecnt);
843  		new_uobj->object = new_mr;
844  
845  		rdma_restrack_new(&new_mr->res, RDMA_RESTRACK_MR);
846  		rdma_restrack_set_name(&new_mr->res, NULL);
847  		rdma_restrack_add(&new_mr->res);
848  
849  		/*
850  		 * The new uobj for the new HW object is put into the same spot
851  		 * in the IDR and the old uobj & HW object is deleted.
852  		 */
853  		rdma_assign_uobject(uobj, new_uobj, attrs);
854  		rdma_alloc_commit_uobject(new_uobj, attrs);
855  		uobj_put_destroy(uobj);
856  		new_uobj = NULL;
857  		uobj = NULL;
858  		mr = new_mr;
859  	} else {
860  		if (cmd.flags & IB_MR_REREG_PD) {
861  			atomic_dec(&orig_pd->usecnt);
862  			mr->pd = new_pd;
863  			atomic_inc(&new_pd->usecnt);
864  		}
865  		if (cmd.flags & IB_MR_REREG_TRANS) {
866  			mr->iova = cmd.hca_va;
867  			mr->length = cmd.length;
868  		}
869  	}
870  
871  	memset(&resp, 0, sizeof(resp));
872  	resp.lkey      = mr->lkey;
873  	resp.rkey      = mr->rkey;
874  
875  	ret = uverbs_response(attrs, &resp, sizeof(resp));
876  
877  put_new_uobj:
878  	if (new_uobj)
879  		uobj_alloc_abort(new_uobj, attrs);
880  put_uobj_pd:
881  	if (cmd.flags & IB_MR_REREG_PD)
882  		uobj_put_obj_read(new_pd);
883  
884  put_uobjs:
885  	if (uobj)
886  		uobj_put_write(uobj);
887  
888  	return ret;
889  }
890  
ib_uverbs_dereg_mr(struct uverbs_attr_bundle * attrs)891  static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
892  {
893  	struct ib_uverbs_dereg_mr cmd;
894  	int ret;
895  
896  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
897  	if (ret)
898  		return ret;
899  
900  	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
901  }
902  
ib_uverbs_alloc_mw(struct uverbs_attr_bundle * attrs)903  static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
904  {
905  	struct ib_uverbs_alloc_mw      cmd;
906  	struct ib_uverbs_alloc_mw_resp resp = {};
907  	struct ib_uobject             *uobj;
908  	struct ib_pd                  *pd;
909  	struct ib_mw                  *mw;
910  	int                            ret;
911  	struct ib_device *ib_dev;
912  
913  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
914  	if (ret)
915  		return ret;
916  
917  	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
918  	if (IS_ERR(uobj))
919  		return PTR_ERR(uobj);
920  
921  	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
922  	if (!pd) {
923  		ret = -EINVAL;
924  		goto err_free;
925  	}
926  
927  	if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {
928  		ret = -EINVAL;
929  		goto err_put;
930  	}
931  
932  	mw = rdma_zalloc_drv_obj(ib_dev, ib_mw);
933  	if (!mw) {
934  		ret = -ENOMEM;
935  		goto err_put;
936  	}
937  
938  	mw->device = ib_dev;
939  	mw->pd = pd;
940  	mw->uobject = uobj;
941  	mw->type = cmd.mw_type;
942  
943  	ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);
944  	if (ret)
945  		goto err_alloc;
946  
947  	atomic_inc(&pd->usecnt);
948  
949  	uobj->object = mw;
950  	uobj_put_obj_read(pd);
951  	uobj_finalize_uobj_create(uobj, attrs);
952  
953  	resp.rkey = mw->rkey;
954  	resp.mw_handle = uobj->id;
955  	return uverbs_response(attrs, &resp, sizeof(resp));
956  
957  err_alloc:
958  	kfree(mw);
959  err_put:
960  	uobj_put_obj_read(pd);
961  err_free:
962  	uobj_alloc_abort(uobj, attrs);
963  	return ret;
964  }
965  
ib_uverbs_dealloc_mw(struct uverbs_attr_bundle * attrs)966  static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
967  {
968  	struct ib_uverbs_dealloc_mw cmd;
969  	int ret;
970  
971  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
972  	if (ret)
973  		return ret;
974  
975  	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
976  }
977  
ib_uverbs_create_comp_channel(struct uverbs_attr_bundle * attrs)978  static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
979  {
980  	struct ib_uverbs_create_comp_channel	   cmd;
981  	struct ib_uverbs_create_comp_channel_resp  resp;
982  	struct ib_uobject			  *uobj;
983  	struct ib_uverbs_completion_event_file	  *ev_file;
984  	struct ib_device *ib_dev;
985  	int ret;
986  
987  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
988  	if (ret)
989  		return ret;
990  
991  	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
992  	if (IS_ERR(uobj))
993  		return PTR_ERR(uobj);
994  
995  	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
996  			       uobj);
997  	ib_uverbs_init_event_queue(&ev_file->ev_queue);
998  	uobj_finalize_uobj_create(uobj, attrs);
999  
1000  	resp.fd = uobj->id;
1001  	return uverbs_response(attrs, &resp, sizeof(resp));
1002  }
1003  
create_cq(struct uverbs_attr_bundle * attrs,struct ib_uverbs_ex_create_cq * cmd)1004  static int create_cq(struct uverbs_attr_bundle *attrs,
1005  		     struct ib_uverbs_ex_create_cq *cmd)
1006  {
1007  	struct ib_ucq_object           *obj;
1008  	struct ib_uverbs_completion_event_file    *ev_file = NULL;
1009  	struct ib_cq                   *cq;
1010  	int                             ret;
1011  	struct ib_uverbs_ex_create_cq_resp resp = {};
1012  	struct ib_cq_init_attr attr = {};
1013  	struct ib_device *ib_dev;
1014  
1015  	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
1016  		return -EINVAL;
1017  
1018  	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
1019  						 &ib_dev);
1020  	if (IS_ERR(obj))
1021  		return PTR_ERR(obj);
1022  
1023  	if (cmd->comp_channel >= 0) {
1024  		ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
1025  		if (IS_ERR(ev_file)) {
1026  			ret = PTR_ERR(ev_file);
1027  			goto err;
1028  		}
1029  	}
1030  
1031  	obj->uevent.uobject.user_handle = cmd->user_handle;
1032  	INIT_LIST_HEAD(&obj->comp_list);
1033  	INIT_LIST_HEAD(&obj->uevent.event_list);
1034  
1035  	attr.cqe = cmd->cqe;
1036  	attr.comp_vector = cmd->comp_vector;
1037  	attr.flags = cmd->flags;
1038  
1039  	cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
1040  	if (!cq) {
1041  		ret = -ENOMEM;
1042  		goto err_file;
1043  	}
1044  	cq->device        = ib_dev;
1045  	cq->uobject       = obj;
1046  	cq->comp_handler  = ib_uverbs_comp_handler;
1047  	cq->event_handler = ib_uverbs_cq_event_handler;
1048  	cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
1049  	atomic_set(&cq->usecnt, 0);
1050  
1051  	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
1052  	rdma_restrack_set_name(&cq->res, NULL);
1053  
1054  	ret = ib_dev->ops.create_cq(cq, &attr, attrs);
1055  	if (ret)
1056  		goto err_free;
1057  	rdma_restrack_add(&cq->res);
1058  
1059  	obj->uevent.uobject.object = cq;
1060  	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
1061  	if (obj->uevent.event_file)
1062  		uverbs_uobject_get(&obj->uevent.event_file->uobj);
1063  	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1064  
1065  	resp.base.cq_handle = obj->uevent.uobject.id;
1066  	resp.base.cqe = cq->cqe;
1067  	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1068  	return uverbs_response(attrs, &resp, sizeof(resp));
1069  
1070  err_free:
1071  	rdma_restrack_put(&cq->res);
1072  	kfree(cq);
1073  err_file:
1074  	if (ev_file)
1075  		ib_uverbs_release_ucq(ev_file, obj);
1076  err:
1077  	uobj_alloc_abort(&obj->uevent.uobject, attrs);
1078  	return ret;
1079  }
1080  
ib_uverbs_create_cq(struct uverbs_attr_bundle * attrs)1081  static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1082  {
1083  	struct ib_uverbs_create_cq      cmd;
1084  	struct ib_uverbs_ex_create_cq	cmd_ex;
1085  	int ret;
1086  
1087  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1088  	if (ret)
1089  		return ret;
1090  
1091  	memset(&cmd_ex, 0, sizeof(cmd_ex));
1092  	cmd_ex.user_handle = cmd.user_handle;
1093  	cmd_ex.cqe = cmd.cqe;
1094  	cmd_ex.comp_vector = cmd.comp_vector;
1095  	cmd_ex.comp_channel = cmd.comp_channel;
1096  
1097  	return create_cq(attrs, &cmd_ex);
1098  }
1099  
ib_uverbs_ex_create_cq(struct uverbs_attr_bundle * attrs)1100  static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1101  {
1102  	struct ib_uverbs_ex_create_cq  cmd;
1103  	int ret;
1104  
1105  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1106  	if (ret)
1107  		return ret;
1108  
1109  	if (cmd.comp_mask)
1110  		return -EINVAL;
1111  
1112  	if (cmd.reserved)
1113  		return -EINVAL;
1114  
1115  	return create_cq(attrs, &cmd);
1116  }
1117  
ib_uverbs_resize_cq(struct uverbs_attr_bundle * attrs)1118  static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1119  {
1120  	struct ib_uverbs_resize_cq	cmd;
1121  	struct ib_uverbs_resize_cq_resp	resp = {};
1122  	struct ib_cq			*cq;
1123  	int ret;
1124  
1125  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1126  	if (ret)
1127  		return ret;
1128  
1129  	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1130  	if (!cq)
1131  		return -EINVAL;
1132  
1133  	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1134  	if (ret)
1135  		goto out;
1136  
1137  	resp.cqe = cq->cqe;
1138  
1139  	ret = uverbs_response(attrs, &resp, sizeof(resp));
1140  out:
1141  	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
1142  				UVERBS_LOOKUP_READ);
1143  
1144  	return ret;
1145  }
1146  
copy_wc_to_user(struct ib_device * ib_dev,void __user * dest,struct ib_wc * wc)1147  static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1148  			   struct ib_wc *wc)
1149  {
1150  	struct ib_uverbs_wc tmp;
1151  
1152  	tmp.wr_id		= wc->wr_id;
1153  	tmp.status		= wc->status;
1154  	tmp.opcode		= wc->opcode;
1155  	tmp.vendor_err		= wc->vendor_err;
1156  	tmp.byte_len		= wc->byte_len;
1157  	tmp.ex.imm_data		= wc->ex.imm_data;
1158  	tmp.qp_num		= wc->qp->qp_num;
1159  	tmp.src_qp		= wc->src_qp;
1160  	tmp.wc_flags		= wc->wc_flags;
1161  	tmp.pkey_index		= wc->pkey_index;
1162  	if (rdma_cap_opa_ah(ib_dev, wc->port_num))
1163  		tmp.slid	= OPA_TO_IB_UCAST_LID(wc->slid);
1164  	else
1165  		tmp.slid	= ib_lid_cpu16(wc->slid);
1166  	tmp.sl			= wc->sl;
1167  	tmp.dlid_path_bits	= wc->dlid_path_bits;
1168  	tmp.port_num		= wc->port_num;
1169  	tmp.reserved		= 0;
1170  
1171  	if (copy_to_user(dest, &tmp, sizeof tmp))
1172  		return -EFAULT;
1173  
1174  	return 0;
1175  }
1176  
ib_uverbs_poll_cq(struct uverbs_attr_bundle * attrs)1177  static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1178  {
1179  	struct ib_uverbs_poll_cq       cmd;
1180  	struct ib_uverbs_poll_cq_resp  resp;
1181  	u8 __user                     *header_ptr;
1182  	u8 __user                     *data_ptr;
1183  	struct ib_cq                  *cq;
1184  	struct ib_wc                   wc;
1185  	int                            ret;
1186  
1187  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1188  	if (ret)
1189  		return ret;
1190  
1191  	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1192  	if (!cq)
1193  		return -EINVAL;
1194  
1195  	/* we copy a struct ib_uverbs_poll_cq_resp to user space */
1196  	header_ptr = attrs->ucore.outbuf;
1197  	data_ptr = header_ptr + sizeof resp;
1198  
1199  	memset(&resp, 0, sizeof resp);
1200  	while (resp.count < cmd.ne) {
1201  		ret = ib_poll_cq(cq, 1, &wc);
1202  		if (ret < 0)
1203  			goto out_put;
1204  		if (!ret)
1205  			break;
1206  
1207  		ret = copy_wc_to_user(cq->device, data_ptr, &wc);
1208  		if (ret)
1209  			goto out_put;
1210  
1211  		data_ptr += sizeof(struct ib_uverbs_wc);
1212  		++resp.count;
1213  	}
1214  
1215  	if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1216  		ret = -EFAULT;
1217  		goto out_put;
1218  	}
1219  	ret = 0;
1220  
1221  	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
1222  		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);
1223  
1224  out_put:
1225  	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
1226  				UVERBS_LOOKUP_READ);
1227  	return ret;
1228  }
1229  
ib_uverbs_req_notify_cq(struct uverbs_attr_bundle * attrs)1230  static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1231  {
1232  	struct ib_uverbs_req_notify_cq cmd;
1233  	struct ib_cq                  *cq;
1234  	int ret;
1235  
1236  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1237  	if (ret)
1238  		return ret;
1239  
1240  	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1241  	if (!cq)
1242  		return -EINVAL;
1243  
1244  	ib_req_notify_cq(cq, cmd.solicited_only ?
1245  			 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1246  
1247  	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
1248  				UVERBS_LOOKUP_READ);
1249  	return 0;
1250  }
1251  
ib_uverbs_destroy_cq(struct uverbs_attr_bundle * attrs)1252  static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1253  {
1254  	struct ib_uverbs_destroy_cq      cmd;
1255  	struct ib_uverbs_destroy_cq_resp resp;
1256  	struct ib_uobject		*uobj;
1257  	struct ib_ucq_object        	*obj;
1258  	int ret;
1259  
1260  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1261  	if (ret)
1262  		return ret;
1263  
1264  	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1265  	if (IS_ERR(uobj))
1266  		return PTR_ERR(uobj);
1267  
1268  	obj = container_of(uobj, struct ib_ucq_object, uevent.uobject);
1269  	memset(&resp, 0, sizeof(resp));
1270  	resp.comp_events_reported  = obj->comp_events_reported;
1271  	resp.async_events_reported = obj->uevent.events_reported;
1272  
1273  	uobj_put_destroy(uobj);
1274  
1275  	return uverbs_response(attrs, &resp, sizeof(resp));
1276  }
1277  
create_qp(struct uverbs_attr_bundle * attrs,struct ib_uverbs_ex_create_qp * cmd)1278  static int create_qp(struct uverbs_attr_bundle *attrs,
1279  		     struct ib_uverbs_ex_create_qp *cmd)
1280  {
1281  	struct ib_uqp_object		*obj;
1282  	struct ib_device		*device;
1283  	struct ib_pd			*pd = NULL;
1284  	struct ib_xrcd			*xrcd = NULL;
1285  	struct ib_uobject		*xrcd_uobj = ERR_PTR(-ENOENT);
1286  	struct ib_cq			*scq = NULL, *rcq = NULL;
1287  	struct ib_srq			*srq = NULL;
1288  	struct ib_qp			*qp;
1289  	struct ib_qp_init_attr		attr = {};
1290  	struct ib_uverbs_ex_create_qp_resp resp = {};
1291  	int				ret;
1292  	struct ib_rwq_ind_table *ind_tbl = NULL;
1293  	bool has_sq = true;
1294  	struct ib_device *ib_dev;
1295  
1296  	switch (cmd->qp_type) {
1297  	case IB_QPT_RAW_PACKET:
1298  		if (!capable(CAP_NET_RAW))
1299  			return -EPERM;
1300  		break;
1301  	case IB_QPT_RC:
1302  	case IB_QPT_UC:
1303  	case IB_QPT_UD:
1304  	case IB_QPT_XRC_INI:
1305  	case IB_QPT_XRC_TGT:
1306  	case IB_QPT_DRIVER:
1307  		break;
1308  	default:
1309  		return -EINVAL;
1310  	}
1311  
1312  	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1313  						 &ib_dev);
1314  	if (IS_ERR(obj))
1315  		return PTR_ERR(obj);
1316  	obj->uxrcd = NULL;
1317  	obj->uevent.uobject.user_handle = cmd->user_handle;
1318  	mutex_init(&obj->mcast_lock);
1319  
1320  	if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1321  		ind_tbl = uobj_get_obj_read(rwq_ind_table,
1322  					    UVERBS_OBJECT_RWQ_IND_TBL,
1323  					    cmd->rwq_ind_tbl_handle, attrs);
1324  		if (!ind_tbl) {
1325  			ret = -EINVAL;
1326  			goto err_put;
1327  		}
1328  
1329  		attr.rwq_ind_tbl = ind_tbl;
1330  	}
1331  
1332  	if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1333  		ret = -EINVAL;
1334  		goto err_put;
1335  	}
1336  
1337  	if (ind_tbl && !cmd->max_send_wr)
1338  		has_sq = false;
1339  
1340  	if (cmd->qp_type == IB_QPT_XRC_TGT) {
1341  		xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1342  					  attrs);
1343  
1344  		if (IS_ERR(xrcd_uobj)) {
1345  			ret = -EINVAL;
1346  			goto err_put;
1347  		}
1348  
1349  		xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1350  		if (!xrcd) {
1351  			ret = -EINVAL;
1352  			goto err_put;
1353  		}
1354  		device = xrcd->device;
1355  	} else {
1356  		if (cmd->qp_type == IB_QPT_XRC_INI) {
1357  			cmd->max_recv_wr = 0;
1358  			cmd->max_recv_sge = 0;
1359  		} else {
1360  			if (cmd->is_srq) {
1361  				srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1362  							cmd->srq_handle, attrs);
1363  				if (!srq || srq->srq_type == IB_SRQT_XRC) {
1364  					ret = -EINVAL;
1365  					goto err_put;
1366  				}
1367  			}
1368  
1369  			if (!ind_tbl) {
1370  				if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1371  					rcq = uobj_get_obj_read(
1372  						cq, UVERBS_OBJECT_CQ,
1373  						cmd->recv_cq_handle, attrs);
1374  					if (!rcq) {
1375  						ret = -EINVAL;
1376  						goto err_put;
1377  					}
1378  				}
1379  			}
1380  		}
1381  
1382  		if (has_sq)
1383  			scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1384  						cmd->send_cq_handle, attrs);
1385  		if (!ind_tbl && cmd->qp_type != IB_QPT_XRC_INI)
1386  			rcq = rcq ?: scq;
1387  		pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1388  				       attrs);
1389  		if (!pd || (!scq && has_sq)) {
1390  			ret = -EINVAL;
1391  			goto err_put;
1392  		}
1393  
1394  		device = pd->device;
1395  	}
1396  
1397  	attr.event_handler = ib_uverbs_qp_event_handler;
1398  	attr.send_cq       = scq;
1399  	attr.recv_cq       = rcq;
1400  	attr.srq           = srq;
1401  	attr.xrcd	   = xrcd;
1402  	attr.sq_sig_type   = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1403  					      IB_SIGNAL_REQ_WR;
1404  	attr.qp_type       = cmd->qp_type;
1405  
1406  	attr.cap.max_send_wr     = cmd->max_send_wr;
1407  	attr.cap.max_recv_wr     = cmd->max_recv_wr;
1408  	attr.cap.max_send_sge    = cmd->max_send_sge;
1409  	attr.cap.max_recv_sge    = cmd->max_recv_sge;
1410  	attr.cap.max_inline_data = cmd->max_inline_data;
1411  
1412  	INIT_LIST_HEAD(&obj->uevent.event_list);
1413  	INIT_LIST_HEAD(&obj->mcast_list);
1414  
1415  	attr.create_flags = cmd->create_flags;
1416  	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1417  				IB_QP_CREATE_CROSS_CHANNEL |
1418  				IB_QP_CREATE_MANAGED_SEND |
1419  				IB_QP_CREATE_MANAGED_RECV |
1420  				IB_QP_CREATE_SCATTER_FCS |
1421  				IB_QP_CREATE_CVLAN_STRIPPING |
1422  				IB_QP_CREATE_SOURCE_QPN |
1423  				IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
1424  		ret = -EINVAL;
1425  		goto err_put;
1426  	}
1427  
1428  	if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1429  		if (!capable(CAP_NET_RAW)) {
1430  			ret = -EPERM;
1431  			goto err_put;
1432  		}
1433  
1434  		attr.source_qpn = cmd->source_qpn;
1435  	}
1436  
1437  	qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj,
1438  			       KBUILD_MODNAME);
1439  	if (IS_ERR(qp)) {
1440  		ret = PTR_ERR(qp);
1441  		goto err_put;
1442  	}
1443  	ib_qp_usecnt_inc(qp);
1444  
1445  	obj->uevent.uobject.object = qp;
1446  	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
1447  	if (obj->uevent.event_file)
1448  		uverbs_uobject_get(&obj->uevent.event_file->uobj);
1449  
1450  	if (xrcd) {
1451  		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1452  					  uobject);
1453  		atomic_inc(&obj->uxrcd->refcnt);
1454  		uobj_put_read(xrcd_uobj);
1455  	}
1456  
1457  	if (pd)
1458  		uobj_put_obj_read(pd);
1459  	if (scq)
1460  		rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,
1461  					UVERBS_LOOKUP_READ);
1462  	if (rcq && rcq != scq)
1463  		rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,
1464  					UVERBS_LOOKUP_READ);
1465  	if (srq)
1466  		rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
1467  					UVERBS_LOOKUP_READ);
1468  	if (ind_tbl)
1469  		uobj_put_obj_read(ind_tbl);
1470  	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1471  
1472  	resp.base.qpn             = qp->qp_num;
1473  	resp.base.qp_handle       = obj->uevent.uobject.id;
1474  	resp.base.max_recv_sge    = attr.cap.max_recv_sge;
1475  	resp.base.max_send_sge    = attr.cap.max_send_sge;
1476  	resp.base.max_recv_wr     = attr.cap.max_recv_wr;
1477  	resp.base.max_send_wr     = attr.cap.max_send_wr;
1478  	resp.base.max_inline_data = attr.cap.max_inline_data;
1479  	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1480  	return uverbs_response(attrs, &resp, sizeof(resp));
1481  
1482  err_put:
1483  	if (!IS_ERR(xrcd_uobj))
1484  		uobj_put_read(xrcd_uobj);
1485  	if (pd)
1486  		uobj_put_obj_read(pd);
1487  	if (scq)
1488  		rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,
1489  					UVERBS_LOOKUP_READ);
1490  	if (rcq && rcq != scq)
1491  		rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,
1492  					UVERBS_LOOKUP_READ);
1493  	if (srq)
1494  		rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
1495  					UVERBS_LOOKUP_READ);
1496  	if (ind_tbl)
1497  		uobj_put_obj_read(ind_tbl);
1498  
1499  	uobj_alloc_abort(&obj->uevent.uobject, attrs);
1500  	return ret;
1501  }
1502  
ib_uverbs_create_qp(struct uverbs_attr_bundle * attrs)1503  static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1504  {
1505  	struct ib_uverbs_create_qp      cmd;
1506  	struct ib_uverbs_ex_create_qp	cmd_ex;
1507  	int ret;
1508  
1509  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1510  	if (ret)
1511  		return ret;
1512  
1513  	memset(&cmd_ex, 0, sizeof(cmd_ex));
1514  	cmd_ex.user_handle = cmd.user_handle;
1515  	cmd_ex.pd_handle = cmd.pd_handle;
1516  	cmd_ex.send_cq_handle = cmd.send_cq_handle;
1517  	cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1518  	cmd_ex.srq_handle = cmd.srq_handle;
1519  	cmd_ex.max_send_wr = cmd.max_send_wr;
1520  	cmd_ex.max_recv_wr = cmd.max_recv_wr;
1521  	cmd_ex.max_send_sge = cmd.max_send_sge;
1522  	cmd_ex.max_recv_sge = cmd.max_recv_sge;
1523  	cmd_ex.max_inline_data = cmd.max_inline_data;
1524  	cmd_ex.sq_sig_all = cmd.sq_sig_all;
1525  	cmd_ex.qp_type = cmd.qp_type;
1526  	cmd_ex.is_srq = cmd.is_srq;
1527  
1528  	return create_qp(attrs, &cmd_ex);
1529  }
1530  
ib_uverbs_ex_create_qp(struct uverbs_attr_bundle * attrs)1531  static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1532  {
1533  	struct ib_uverbs_ex_create_qp cmd;
1534  	int ret;
1535  
1536  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1537  	if (ret)
1538  		return ret;
1539  
1540  	if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1541  		return -EINVAL;
1542  
1543  	if (cmd.reserved)
1544  		return -EINVAL;
1545  
1546  	return create_qp(attrs, &cmd);
1547  }
1548  
ib_uverbs_open_qp(struct uverbs_attr_bundle * attrs)1549  static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1550  {
1551  	struct ib_uverbs_create_qp_resp resp = {};
1552  	struct ib_uverbs_open_qp        cmd;
1553  	struct ib_uqp_object           *obj;
1554  	struct ib_xrcd		       *xrcd;
1555  	struct ib_qp                   *qp;
1556  	struct ib_qp_open_attr          attr = {};
1557  	int ret;
1558  	struct ib_uobject *xrcd_uobj;
1559  	struct ib_device *ib_dev;
1560  
1561  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1562  	if (ret)
1563  		return ret;
1564  
1565  	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1566  						 &ib_dev);
1567  	if (IS_ERR(obj))
1568  		return PTR_ERR(obj);
1569  
1570  	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1571  	if (IS_ERR(xrcd_uobj)) {
1572  		ret = -EINVAL;
1573  		goto err_put;
1574  	}
1575  
1576  	xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1577  	if (!xrcd) {
1578  		ret = -EINVAL;
1579  		goto err_xrcd;
1580  	}
1581  
1582  	attr.event_handler = ib_uverbs_qp_event_handler;
1583  	attr.qp_num        = cmd.qpn;
1584  	attr.qp_type       = cmd.qp_type;
1585  
1586  	INIT_LIST_HEAD(&obj->uevent.event_list);
1587  	INIT_LIST_HEAD(&obj->mcast_list);
1588  
1589  	qp = ib_open_qp(xrcd, &attr);
1590  	if (IS_ERR(qp)) {
1591  		ret = PTR_ERR(qp);
1592  		goto err_xrcd;
1593  	}
1594  
1595  	obj->uevent.uobject.object = qp;
1596  	obj->uevent.uobject.user_handle = cmd.user_handle;
1597  
1598  	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1599  	atomic_inc(&obj->uxrcd->refcnt);
1600  	qp->uobject = obj;
1601  	uobj_put_read(xrcd_uobj);
1602  	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1603  
1604  	resp.qpn = qp->qp_num;
1605  	resp.qp_handle = obj->uevent.uobject.id;
1606  	return uverbs_response(attrs, &resp, sizeof(resp));
1607  
1608  err_xrcd:
1609  	uobj_put_read(xrcd_uobj);
1610  err_put:
1611  	uobj_alloc_abort(&obj->uevent.uobject, attrs);
1612  	return ret;
1613  }
1614  
copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest * uverb_attr,struct rdma_ah_attr * rdma_attr)1615  static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1616  				   struct rdma_ah_attr *rdma_attr)
1617  {
1618  	const struct ib_global_route   *grh;
1619  
1620  	uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);
1621  	uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);
1622  	uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);
1623  	uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);
1624  	uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &
1625  					 IB_AH_GRH);
1626  	if (uverb_attr->is_global) {
1627  		grh = rdma_ah_read_grh(rdma_attr);
1628  		memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1629  		uverb_attr->flow_label        = grh->flow_label;
1630  		uverb_attr->sgid_index        = grh->sgid_index;
1631  		uverb_attr->hop_limit         = grh->hop_limit;
1632  		uverb_attr->traffic_class     = grh->traffic_class;
1633  	}
1634  	uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);
1635  }
1636  
ib_uverbs_query_qp(struct uverbs_attr_bundle * attrs)1637  static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1638  {
1639  	struct ib_uverbs_query_qp      cmd;
1640  	struct ib_uverbs_query_qp_resp resp;
1641  	struct ib_qp                   *qp;
1642  	struct ib_qp_attr              *attr;
1643  	struct ib_qp_init_attr         *init_attr;
1644  	int                            ret;
1645  
1646  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1647  	if (ret)
1648  		return ret;
1649  
1650  	attr      = kmalloc(sizeof *attr, GFP_KERNEL);
1651  	init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1652  	if (!attr || !init_attr) {
1653  		ret = -ENOMEM;
1654  		goto out;
1655  	}
1656  
1657  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1658  	if (!qp) {
1659  		ret = -EINVAL;
1660  		goto out;
1661  	}
1662  
1663  	ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1664  
1665  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
1666  				UVERBS_LOOKUP_READ);
1667  
1668  	if (ret)
1669  		goto out;
1670  
1671  	memset(&resp, 0, sizeof resp);
1672  
1673  	resp.qp_state               = attr->qp_state;
1674  	resp.cur_qp_state           = attr->cur_qp_state;
1675  	resp.path_mtu               = attr->path_mtu;
1676  	resp.path_mig_state         = attr->path_mig_state;
1677  	resp.qkey                   = attr->qkey;
1678  	resp.rq_psn                 = attr->rq_psn;
1679  	resp.sq_psn                 = attr->sq_psn;
1680  	resp.dest_qp_num            = attr->dest_qp_num;
1681  	resp.qp_access_flags        = attr->qp_access_flags;
1682  	resp.pkey_index             = attr->pkey_index;
1683  	resp.alt_pkey_index         = attr->alt_pkey_index;
1684  	resp.sq_draining            = attr->sq_draining;
1685  	resp.max_rd_atomic          = attr->max_rd_atomic;
1686  	resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;
1687  	resp.min_rnr_timer          = attr->min_rnr_timer;
1688  	resp.port_num               = attr->port_num;
1689  	resp.timeout                = attr->timeout;
1690  	resp.retry_cnt              = attr->retry_cnt;
1691  	resp.rnr_retry              = attr->rnr_retry;
1692  	resp.alt_port_num           = attr->alt_port_num;
1693  	resp.alt_timeout            = attr->alt_timeout;
1694  
1695  	copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1696  	copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
1697  
1698  	resp.max_send_wr            = init_attr->cap.max_send_wr;
1699  	resp.max_recv_wr            = init_attr->cap.max_recv_wr;
1700  	resp.max_send_sge           = init_attr->cap.max_send_sge;
1701  	resp.max_recv_sge           = init_attr->cap.max_recv_sge;
1702  	resp.max_inline_data        = init_attr->cap.max_inline_data;
1703  	resp.sq_sig_all             = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1704  
1705  	ret = uverbs_response(attrs, &resp, sizeof(resp));
1706  
1707  out:
1708  	kfree(attr);
1709  	kfree(init_attr);
1710  
1711  	return ret;
1712  }
1713  
1714  /* Remove ignored fields set in the attribute mask */
modify_qp_mask(enum ib_qp_type qp_type,int mask)1715  static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1716  {
1717  	switch (qp_type) {
1718  	case IB_QPT_XRC_INI:
1719  		return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1720  	case IB_QPT_XRC_TGT:
1721  		return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1722  				IB_QP_RNR_RETRY);
1723  	default:
1724  		return mask;
1725  	}
1726  }
1727  
copy_ah_attr_from_uverbs(struct ib_device * dev,struct rdma_ah_attr * rdma_attr,struct ib_uverbs_qp_dest * uverb_attr)1728  static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1729  				     struct rdma_ah_attr *rdma_attr,
1730  				     struct ib_uverbs_qp_dest *uverb_attr)
1731  {
1732  	rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1733  	if (uverb_attr->is_global) {
1734  		rdma_ah_set_grh(rdma_attr, NULL,
1735  				uverb_attr->flow_label,
1736  				uverb_attr->sgid_index,
1737  				uverb_attr->hop_limit,
1738  				uverb_attr->traffic_class);
1739  		rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1740  	} else {
1741  		rdma_ah_set_ah_flags(rdma_attr, 0);
1742  	}
1743  	rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1744  	rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1745  	rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1746  	rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1747  	rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1748  	rdma_ah_set_make_grd(rdma_attr, false);
1749  }
1750  
modify_qp(struct uverbs_attr_bundle * attrs,struct ib_uverbs_ex_modify_qp * cmd)1751  static int modify_qp(struct uverbs_attr_bundle *attrs,
1752  		     struct ib_uverbs_ex_modify_qp *cmd)
1753  {
1754  	struct ib_qp_attr *attr;
1755  	struct ib_qp *qp;
1756  	int ret;
1757  
1758  	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1759  	if (!attr)
1760  		return -ENOMEM;
1761  
1762  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
1763  			       attrs);
1764  	if (!qp) {
1765  		ret = -EINVAL;
1766  		goto out;
1767  	}
1768  
1769  	if ((cmd->base.attr_mask & IB_QP_PORT) &&
1770  	    !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
1771  		ret = -EINVAL;
1772  		goto release_qp;
1773  	}
1774  
1775  	if ((cmd->base.attr_mask & IB_QP_AV)) {
1776  		if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1777  			ret = -EINVAL;
1778  			goto release_qp;
1779  		}
1780  
1781  		if (cmd->base.attr_mask & IB_QP_STATE &&
1782  		    cmd->base.qp_state == IB_QPS_RTR) {
1783  		/* We are in INIT->RTR TRANSITION (if we are not,
1784  		 * this transition will be rejected in subsequent checks).
1785  		 * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1786  		 * but the IB_QP_STATE flag is required.
1787  		 *
1788  		 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1789  		 * when IB_QP_AV is set, has required inclusion of a valid
1790  		 * port number in the primary AV. (AVs are created and handled
1791  		 * differently for infiniband and ethernet (RoCE) ports).
1792  		 *
1793  		 * Check the port number included in the primary AV against
1794  		 * the port number in the qp struct, which was set (and saved)
1795  		 * in the RST->INIT transition.
1796  		 */
1797  			if (cmd->base.dest.port_num != qp->real_qp->port) {
1798  				ret = -EINVAL;
1799  				goto release_qp;
1800  			}
1801  		} else {
1802  		/* We are in SQD->SQD. (If we are not, this transition will
1803  		 * be rejected later in the verbs layer checks).
1804  		 * Check for both IB_QP_PORT and IB_QP_AV, these can be set
1805  		 * together in the SQD->SQD transition.
1806  		 *
1807  		 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1808  		 * verbs layer driver does not track primary port changes
1809  		 * resulting from path migration. Thus, in SQD, if the primary
1810  		 * AV is modified, the primary port should also be modified).
1811  		 *
1812  		 * Note that in this transition, the IB_QP_STATE flag
1813  		 * is not allowed.
1814  		 */
1815  			if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1816  			     == (IB_QP_AV | IB_QP_PORT)) &&
1817  			    cmd->base.port_num != cmd->base.dest.port_num) {
1818  				ret = -EINVAL;
1819  				goto release_qp;
1820  			}
1821  			if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1822  			    == IB_QP_AV) {
1823  				cmd->base.attr_mask |= IB_QP_PORT;
1824  				cmd->base.port_num = cmd->base.dest.port_num;
1825  			}
1826  		}
1827  	}
1828  
1829  	if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
1830  	    (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
1831  	    !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||
1832  	    cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
1833  		ret = -EINVAL;
1834  		goto release_qp;
1835  	}
1836  
1837  	if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&
1838  	    cmd->base.cur_qp_state > IB_QPS_ERR) ||
1839  	    (cmd->base.attr_mask & IB_QP_STATE &&
1840  	    cmd->base.qp_state > IB_QPS_ERR)) {
1841  		ret = -EINVAL;
1842  		goto release_qp;
1843  	}
1844  
1845  	if (cmd->base.attr_mask & IB_QP_STATE)
1846  		attr->qp_state = cmd->base.qp_state;
1847  	if (cmd->base.attr_mask & IB_QP_CUR_STATE)
1848  		attr->cur_qp_state = cmd->base.cur_qp_state;
1849  	if (cmd->base.attr_mask & IB_QP_PATH_MTU)
1850  		attr->path_mtu = cmd->base.path_mtu;
1851  	if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
1852  		attr->path_mig_state = cmd->base.path_mig_state;
1853  	if (cmd->base.attr_mask & IB_QP_QKEY) {
1854  		if (cmd->base.qkey & IB_QP_SET_QKEY &&
1855  		    !rdma_nl_get_privileged_qkey()) {
1856  			ret = -EPERM;
1857  			goto release_qp;
1858  		}
1859  		attr->qkey = cmd->base.qkey;
1860  	}
1861  	if (cmd->base.attr_mask & IB_QP_RQ_PSN)
1862  		attr->rq_psn = cmd->base.rq_psn;
1863  	if (cmd->base.attr_mask & IB_QP_SQ_PSN)
1864  		attr->sq_psn = cmd->base.sq_psn;
1865  	if (cmd->base.attr_mask & IB_QP_DEST_QPN)
1866  		attr->dest_qp_num = cmd->base.dest_qp_num;
1867  	if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)
1868  		attr->qp_access_flags = cmd->base.qp_access_flags;
1869  	if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)
1870  		attr->pkey_index = cmd->base.pkey_index;
1871  	if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1872  		attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
1873  	if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
1874  		attr->max_rd_atomic = cmd->base.max_rd_atomic;
1875  	if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1876  		attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;
1877  	if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)
1878  		attr->min_rnr_timer = cmd->base.min_rnr_timer;
1879  	if (cmd->base.attr_mask & IB_QP_PORT)
1880  		attr->port_num = cmd->base.port_num;
1881  	if (cmd->base.attr_mask & IB_QP_TIMEOUT)
1882  		attr->timeout = cmd->base.timeout;
1883  	if (cmd->base.attr_mask & IB_QP_RETRY_CNT)
1884  		attr->retry_cnt = cmd->base.retry_cnt;
1885  	if (cmd->base.attr_mask & IB_QP_RNR_RETRY)
1886  		attr->rnr_retry = cmd->base.rnr_retry;
1887  	if (cmd->base.attr_mask & IB_QP_ALT_PATH) {
1888  		attr->alt_port_num = cmd->base.alt_port_num;
1889  		attr->alt_timeout = cmd->base.alt_timeout;
1890  		attr->alt_pkey_index = cmd->base.alt_pkey_index;
1891  	}
1892  	if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)
1893  		attr->rate_limit = cmd->rate_limit;
1894  
1895  	if (cmd->base.attr_mask & IB_QP_AV)
1896  		copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
1897  					 &cmd->base.dest);
1898  
1899  	if (cmd->base.attr_mask & IB_QP_ALT_PATH)
1900  		copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
1901  					 &cmd->base.alt_dest);
1902  
1903  	ret = ib_modify_qp_with_udata(qp, attr,
1904  				      modify_qp_mask(qp->qp_type,
1905  						     cmd->base.attr_mask),
1906  				      &attrs->driver_udata);
1907  
1908  release_qp:
1909  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
1910  				UVERBS_LOOKUP_READ);
1911  out:
1912  	kfree(attr);
1913  
1914  	return ret;
1915  }
1916  
ib_uverbs_modify_qp(struct uverbs_attr_bundle * attrs)1917  static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
1918  {
1919  	struct ib_uverbs_ex_modify_qp cmd;
1920  	int ret;
1921  
1922  	ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
1923  	if (ret)
1924  		return ret;
1925  
1926  	if (cmd.base.attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1927  		return -EOPNOTSUPP;
1928  
1929  	return modify_qp(attrs, &cmd);
1930  }
1931  
ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle * attrs)1932  static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
1933  {
1934  	struct ib_uverbs_ex_modify_qp cmd;
1935  	struct ib_uverbs_ex_modify_qp_resp resp = {
1936  		.response_length = uverbs_response_length(attrs, sizeof(resp))
1937  	};
1938  	int ret;
1939  
1940  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1941  	if (ret)
1942  		return ret;
1943  
1944  	/*
1945  	 * Last bit is reserved for extending the attr_mask by
1946  	 * using another field.
1947  	 */
1948  	if (cmd.base.attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
1949  		return -EOPNOTSUPP;
1950  
1951  	ret = modify_qp(attrs, &cmd);
1952  	if (ret)
1953  		return ret;
1954  
1955  	return uverbs_response(attrs, &resp, sizeof(resp));
1956  }
1957  
ib_uverbs_destroy_qp(struct uverbs_attr_bundle * attrs)1958  static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
1959  {
1960  	struct ib_uverbs_destroy_qp      cmd;
1961  	struct ib_uverbs_destroy_qp_resp resp;
1962  	struct ib_uobject		*uobj;
1963  	struct ib_uqp_object        	*obj;
1964  	int ret;
1965  
1966  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1967  	if (ret)
1968  		return ret;
1969  
1970  	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1971  	if (IS_ERR(uobj))
1972  		return PTR_ERR(uobj);
1973  
1974  	obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
1975  	memset(&resp, 0, sizeof(resp));
1976  	resp.events_reported = obj->uevent.events_reported;
1977  
1978  	uobj_put_destroy(uobj);
1979  
1980  	return uverbs_response(attrs, &resp, sizeof(resp));
1981  }
1982  
alloc_wr(size_t wr_size,__u32 num_sge)1983  static void *alloc_wr(size_t wr_size, __u32 num_sge)
1984  {
1985  	if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof(struct ib_sge))) /
1986  			       sizeof(struct ib_sge))
1987  		return NULL;
1988  
1989  	return kmalloc(ALIGN(wr_size, sizeof(struct ib_sge)) +
1990  			       num_sge * sizeof(struct ib_sge),
1991  		       GFP_KERNEL);
1992  }
1993  
ib_uverbs_post_send(struct uverbs_attr_bundle * attrs)1994  static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
1995  {
1996  	struct ib_uverbs_post_send      cmd;
1997  	struct ib_uverbs_post_send_resp resp;
1998  	struct ib_uverbs_send_wr       *user_wr;
1999  	struct ib_send_wr              *wr = NULL, *last, *next;
2000  	const struct ib_send_wr	       *bad_wr;
2001  	struct ib_qp                   *qp;
2002  	int                             i, sg_ind;
2003  	int				is_ud;
2004  	int ret, ret2;
2005  	size_t                          next_size;
2006  	const struct ib_sge __user *sgls;
2007  	const void __user *wqes;
2008  	struct uverbs_req_iter iter;
2009  
2010  	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2011  	if (ret)
2012  		return ret;
2013  	wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
2014  	if (IS_ERR(wqes))
2015  		return PTR_ERR(wqes);
2016  	sgls = uverbs_request_next_ptr(
2017  		&iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
2018  	if (IS_ERR(sgls))
2019  		return PTR_ERR(sgls);
2020  	ret = uverbs_request_finish(&iter);
2021  	if (ret)
2022  		return ret;
2023  
2024  	user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2025  	if (!user_wr)
2026  		return -ENOMEM;
2027  
2028  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2029  	if (!qp) {
2030  		ret = -EINVAL;
2031  		goto out;
2032  	}
2033  
2034  	is_ud = qp->qp_type == IB_QPT_UD;
2035  	sg_ind = 0;
2036  	last = NULL;
2037  	for (i = 0; i < cmd.wr_count; ++i) {
2038  		if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
2039  				   cmd.wqe_size)) {
2040  			ret = -EFAULT;
2041  			goto out_put;
2042  		}
2043  
2044  		if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2045  			ret = -EINVAL;
2046  			goto out_put;
2047  		}
2048  
2049  		if (is_ud) {
2050  			struct ib_ud_wr *ud;
2051  
2052  			if (user_wr->opcode != IB_WR_SEND &&
2053  			    user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2054  				ret = -EINVAL;
2055  				goto out_put;
2056  			}
2057  
2058  			next_size = sizeof(*ud);
2059  			ud = alloc_wr(next_size, user_wr->num_sge);
2060  			if (!ud) {
2061  				ret = -ENOMEM;
2062  				goto out_put;
2063  			}
2064  
2065  			ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2066  						   user_wr->wr.ud.ah, attrs);
2067  			if (!ud->ah) {
2068  				kfree(ud);
2069  				ret = -EINVAL;
2070  				goto out_put;
2071  			}
2072  			ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2073  			ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2074  
2075  			next = &ud->wr;
2076  		} else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2077  			   user_wr->opcode == IB_WR_RDMA_WRITE ||
2078  			   user_wr->opcode == IB_WR_RDMA_READ) {
2079  			struct ib_rdma_wr *rdma;
2080  
2081  			next_size = sizeof(*rdma);
2082  			rdma = alloc_wr(next_size, user_wr->num_sge);
2083  			if (!rdma) {
2084  				ret = -ENOMEM;
2085  				goto out_put;
2086  			}
2087  
2088  			rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2089  			rdma->rkey = user_wr->wr.rdma.rkey;
2090  
2091  			next = &rdma->wr;
2092  		} else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2093  			   user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2094  			struct ib_atomic_wr *atomic;
2095  
2096  			next_size = sizeof(*atomic);
2097  			atomic = alloc_wr(next_size, user_wr->num_sge);
2098  			if (!atomic) {
2099  				ret = -ENOMEM;
2100  				goto out_put;
2101  			}
2102  
2103  			atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2104  			atomic->compare_add = user_wr->wr.atomic.compare_add;
2105  			atomic->swap = user_wr->wr.atomic.swap;
2106  			atomic->rkey = user_wr->wr.atomic.rkey;
2107  
2108  			next = &atomic->wr;
2109  		} else if (user_wr->opcode == IB_WR_SEND ||
2110  			   user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2111  			   user_wr->opcode == IB_WR_SEND_WITH_INV) {
2112  			next_size = sizeof(*next);
2113  			next = alloc_wr(next_size, user_wr->num_sge);
2114  			if (!next) {
2115  				ret = -ENOMEM;
2116  				goto out_put;
2117  			}
2118  		} else {
2119  			ret = -EINVAL;
2120  			goto out_put;
2121  		}
2122  
2123  		if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2124  		    user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2125  			next->ex.imm_data =
2126  					(__be32 __force) user_wr->ex.imm_data;
2127  		} else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2128  			next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2129  		}
2130  
2131  		if (!last)
2132  			wr = next;
2133  		else
2134  			last->next = next;
2135  		last = next;
2136  
2137  		next->next       = NULL;
2138  		next->wr_id      = user_wr->wr_id;
2139  		next->num_sge    = user_wr->num_sge;
2140  		next->opcode     = user_wr->opcode;
2141  		next->send_flags = user_wr->send_flags;
2142  
2143  		if (next->num_sge) {
2144  			next->sg_list = (void *) next +
2145  				ALIGN(next_size, sizeof(struct ib_sge));
2146  			if (copy_from_user(next->sg_list, sgls + sg_ind,
2147  					   next->num_sge *
2148  						   sizeof(struct ib_sge))) {
2149  				ret = -EFAULT;
2150  				goto out_put;
2151  			}
2152  			sg_ind += next->num_sge;
2153  		} else
2154  			next->sg_list = NULL;
2155  	}
2156  
2157  	resp.bad_wr = 0;
2158  	ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2159  	if (ret)
2160  		for (next = wr; next; next = next->next) {
2161  			++resp.bad_wr;
2162  			if (next == bad_wr)
2163  				break;
2164  		}
2165  
2166  	ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2167  	if (ret2)
2168  		ret = ret2;
2169  
2170  out_put:
2171  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
2172  				UVERBS_LOOKUP_READ);
2173  
2174  	while (wr) {
2175  		if (is_ud && ud_wr(wr)->ah)
2176  			uobj_put_obj_read(ud_wr(wr)->ah);
2177  		next = wr->next;
2178  		kfree(wr);
2179  		wr = next;
2180  	}
2181  
2182  out:
2183  	kfree(user_wr);
2184  
2185  	return ret;
2186  }
2187  
2188  static struct ib_recv_wr *
ib_uverbs_unmarshall_recv(struct uverbs_req_iter * iter,u32 wr_count,u32 wqe_size,u32 sge_count)2189  ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
2190  			  u32 wqe_size, u32 sge_count)
2191  {
2192  	struct ib_uverbs_recv_wr *user_wr;
2193  	struct ib_recv_wr        *wr = NULL, *last, *next;
2194  	int                       sg_ind;
2195  	int                       i;
2196  	int                       ret;
2197  	const struct ib_sge __user *sgls;
2198  	const void __user *wqes;
2199  
2200  	if (wqe_size < sizeof(struct ib_uverbs_recv_wr))
2201  		return ERR_PTR(-EINVAL);
2202  
2203  	wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
2204  	if (IS_ERR(wqes))
2205  		return ERR_CAST(wqes);
2206  	sgls = uverbs_request_next_ptr(
2207  		iter, sge_count * sizeof(struct ib_uverbs_sge));
2208  	if (IS_ERR(sgls))
2209  		return ERR_CAST(sgls);
2210  	ret = uverbs_request_finish(iter);
2211  	if (ret)
2212  		return ERR_PTR(ret);
2213  
2214  	user_wr = kmalloc(wqe_size, GFP_KERNEL);
2215  	if (!user_wr)
2216  		return ERR_PTR(-ENOMEM);
2217  
2218  	sg_ind = 0;
2219  	last = NULL;
2220  	for (i = 0; i < wr_count; ++i) {
2221  		if (copy_from_user(user_wr, wqes + i * wqe_size,
2222  				   wqe_size)) {
2223  			ret = -EFAULT;
2224  			goto err;
2225  		}
2226  
2227  		if (user_wr->num_sge + sg_ind > sge_count) {
2228  			ret = -EINVAL;
2229  			goto err;
2230  		}
2231  
2232  		if (user_wr->num_sge >=
2233  		    (U32_MAX - ALIGN(sizeof(*next), sizeof(struct ib_sge))) /
2234  			    sizeof(struct ib_sge)) {
2235  			ret = -EINVAL;
2236  			goto err;
2237  		}
2238  
2239  		next = kmalloc(ALIGN(sizeof(*next), sizeof(struct ib_sge)) +
2240  				       user_wr->num_sge * sizeof(struct ib_sge),
2241  			       GFP_KERNEL);
2242  		if (!next) {
2243  			ret = -ENOMEM;
2244  			goto err;
2245  		}
2246  
2247  		if (!last)
2248  			wr = next;
2249  		else
2250  			last->next = next;
2251  		last = next;
2252  
2253  		next->next       = NULL;
2254  		next->wr_id      = user_wr->wr_id;
2255  		next->num_sge    = user_wr->num_sge;
2256  
2257  		if (next->num_sge) {
2258  			next->sg_list = (void *)next +
2259  				ALIGN(sizeof(*next), sizeof(struct ib_sge));
2260  			if (copy_from_user(next->sg_list, sgls + sg_ind,
2261  					   next->num_sge *
2262  						   sizeof(struct ib_sge))) {
2263  				ret = -EFAULT;
2264  				goto err;
2265  			}
2266  			sg_ind += next->num_sge;
2267  		} else
2268  			next->sg_list = NULL;
2269  	}
2270  
2271  	kfree(user_wr);
2272  	return wr;
2273  
2274  err:
2275  	kfree(user_wr);
2276  
2277  	while (wr) {
2278  		next = wr->next;
2279  		kfree(wr);
2280  		wr = next;
2281  	}
2282  
2283  	return ERR_PTR(ret);
2284  }
2285  
ib_uverbs_post_recv(struct uverbs_attr_bundle * attrs)2286  static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2287  {
2288  	struct ib_uverbs_post_recv      cmd;
2289  	struct ib_uverbs_post_recv_resp resp;
2290  	struct ib_recv_wr              *wr, *next;
2291  	const struct ib_recv_wr	       *bad_wr;
2292  	struct ib_qp                   *qp;
2293  	int ret, ret2;
2294  	struct uverbs_req_iter iter;
2295  
2296  	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2297  	if (ret)
2298  		return ret;
2299  
2300  	wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2301  				       cmd.sge_count);
2302  	if (IS_ERR(wr))
2303  		return PTR_ERR(wr);
2304  
2305  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2306  	if (!qp) {
2307  		ret = -EINVAL;
2308  		goto out;
2309  	}
2310  
2311  	resp.bad_wr = 0;
2312  	ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2313  
2314  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
2315  				UVERBS_LOOKUP_READ);
2316  	if (ret) {
2317  		for (next = wr; next; next = next->next) {
2318  			++resp.bad_wr;
2319  			if (next == bad_wr)
2320  				break;
2321  		}
2322  	}
2323  
2324  	ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2325  	if (ret2)
2326  		ret = ret2;
2327  out:
2328  	while (wr) {
2329  		next = wr->next;
2330  		kfree(wr);
2331  		wr = next;
2332  	}
2333  
2334  	return ret;
2335  }
2336  
ib_uverbs_post_srq_recv(struct uverbs_attr_bundle * attrs)2337  static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2338  {
2339  	struct ib_uverbs_post_srq_recv      cmd;
2340  	struct ib_uverbs_post_srq_recv_resp resp;
2341  	struct ib_recv_wr                  *wr, *next;
2342  	const struct ib_recv_wr		   *bad_wr;
2343  	struct ib_srq                      *srq;
2344  	int ret, ret2;
2345  	struct uverbs_req_iter iter;
2346  
2347  	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2348  	if (ret)
2349  		return ret;
2350  
2351  	wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2352  				       cmd.sge_count);
2353  	if (IS_ERR(wr))
2354  		return PTR_ERR(wr);
2355  
2356  	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
2357  	if (!srq) {
2358  		ret = -EINVAL;
2359  		goto out;
2360  	}
2361  
2362  	resp.bad_wr = 0;
2363  	ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2364  
2365  	rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
2366  				UVERBS_LOOKUP_READ);
2367  
2368  	if (ret)
2369  		for (next = wr; next; next = next->next) {
2370  			++resp.bad_wr;
2371  			if (next == bad_wr)
2372  				break;
2373  		}
2374  
2375  	ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2376  	if (ret2)
2377  		ret = ret2;
2378  
2379  out:
2380  	while (wr) {
2381  		next = wr->next;
2382  		kfree(wr);
2383  		wr = next;
2384  	}
2385  
2386  	return ret;
2387  }
2388  
ib_uverbs_create_ah(struct uverbs_attr_bundle * attrs)2389  static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2390  {
2391  	struct ib_uverbs_create_ah	 cmd;
2392  	struct ib_uverbs_create_ah_resp	 resp;
2393  	struct ib_uobject		*uobj;
2394  	struct ib_pd			*pd;
2395  	struct ib_ah			*ah;
2396  	struct rdma_ah_attr		attr = {};
2397  	int ret;
2398  	struct ib_device *ib_dev;
2399  
2400  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2401  	if (ret)
2402  		return ret;
2403  
2404  	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2405  	if (IS_ERR(uobj))
2406  		return PTR_ERR(uobj);
2407  
2408  	if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) {
2409  		ret = -EINVAL;
2410  		goto err;
2411  	}
2412  
2413  	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2414  	if (!pd) {
2415  		ret = -EINVAL;
2416  		goto err;
2417  	}
2418  
2419  	attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
2420  	rdma_ah_set_make_grd(&attr, false);
2421  	rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2422  	rdma_ah_set_sl(&attr, cmd.attr.sl);
2423  	rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2424  	rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2425  	rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2426  
2427  	if (cmd.attr.is_global) {
2428  		rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2429  				cmd.attr.grh.sgid_index,
2430  				cmd.attr.grh.hop_limit,
2431  				cmd.attr.grh.traffic_class);
2432  		rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
2433  	} else {
2434  		rdma_ah_set_ah_flags(&attr, 0);
2435  	}
2436  
2437  	ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
2438  	if (IS_ERR(ah)) {
2439  		ret = PTR_ERR(ah);
2440  		goto err_put;
2441  	}
2442  
2443  	ah->uobject  = uobj;
2444  	uobj->user_handle = cmd.user_handle;
2445  	uobj->object = ah;
2446  	uobj_put_obj_read(pd);
2447  	uobj_finalize_uobj_create(uobj, attrs);
2448  
2449  	resp.ah_handle = uobj->id;
2450  	return uverbs_response(attrs, &resp, sizeof(resp));
2451  
2452  err_put:
2453  	uobj_put_obj_read(pd);
2454  err:
2455  	uobj_alloc_abort(uobj, attrs);
2456  	return ret;
2457  }
2458  
ib_uverbs_destroy_ah(struct uverbs_attr_bundle * attrs)2459  static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2460  {
2461  	struct ib_uverbs_destroy_ah cmd;
2462  	int ret;
2463  
2464  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2465  	if (ret)
2466  		return ret;
2467  
2468  	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2469  }
2470  
ib_uverbs_attach_mcast(struct uverbs_attr_bundle * attrs)2471  static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2472  {
2473  	struct ib_uverbs_attach_mcast cmd;
2474  	struct ib_qp                 *qp;
2475  	struct ib_uqp_object         *obj;
2476  	struct ib_uverbs_mcast_entry *mcast;
2477  	int                           ret;
2478  
2479  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2480  	if (ret)
2481  		return ret;
2482  
2483  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2484  	if (!qp)
2485  		return -EINVAL;
2486  
2487  	obj = qp->uobject;
2488  
2489  	mutex_lock(&obj->mcast_lock);
2490  	list_for_each_entry(mcast, &obj->mcast_list, list)
2491  		if (cmd.mlid == mcast->lid &&
2492  		    !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2493  			ret = 0;
2494  			goto out_put;
2495  		}
2496  
2497  	mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2498  	if (!mcast) {
2499  		ret = -ENOMEM;
2500  		goto out_put;
2501  	}
2502  
2503  	mcast->lid = cmd.mlid;
2504  	memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2505  
2506  	ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2507  	if (!ret)
2508  		list_add_tail(&mcast->list, &obj->mcast_list);
2509  	else
2510  		kfree(mcast);
2511  
2512  out_put:
2513  	mutex_unlock(&obj->mcast_lock);
2514  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
2515  				UVERBS_LOOKUP_READ);
2516  
2517  	return ret;
2518  }
2519  
ib_uverbs_detach_mcast(struct uverbs_attr_bundle * attrs)2520  static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2521  {
2522  	struct ib_uverbs_detach_mcast cmd;
2523  	struct ib_uqp_object         *obj;
2524  	struct ib_qp                 *qp;
2525  	struct ib_uverbs_mcast_entry *mcast;
2526  	int                           ret;
2527  	bool                          found = false;
2528  
2529  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2530  	if (ret)
2531  		return ret;
2532  
2533  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2534  	if (!qp)
2535  		return -EINVAL;
2536  
2537  	obj = qp->uobject;
2538  	mutex_lock(&obj->mcast_lock);
2539  
2540  	list_for_each_entry(mcast, &obj->mcast_list, list)
2541  		if (cmd.mlid == mcast->lid &&
2542  		    !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2543  			list_del(&mcast->list);
2544  			kfree(mcast);
2545  			found = true;
2546  			break;
2547  		}
2548  
2549  	if (!found) {
2550  		ret = -EINVAL;
2551  		goto out_put;
2552  	}
2553  
2554  	ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2555  
2556  out_put:
2557  	mutex_unlock(&obj->mcast_lock);
2558  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
2559  				UVERBS_LOOKUP_READ);
2560  	return ret;
2561  }
2562  
flow_resources_alloc(size_t num_specs)2563  struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2564  {
2565  	struct ib_uflow_resources *resources;
2566  
2567  	resources = kzalloc(sizeof(*resources), GFP_KERNEL);
2568  
2569  	if (!resources)
2570  		return NULL;
2571  
2572  	if (!num_specs)
2573  		goto out;
2574  
2575  	resources->counters =
2576  		kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL);
2577  	resources->collection =
2578  		kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL);
2579  
2580  	if (!resources->counters || !resources->collection)
2581  		goto err;
2582  
2583  out:
2584  	resources->max = num_specs;
2585  	return resources;
2586  
2587  err:
2588  	kfree(resources->counters);
2589  	kfree(resources);
2590  
2591  	return NULL;
2592  }
2593  EXPORT_SYMBOL(flow_resources_alloc);
2594  
ib_uverbs_flow_resources_free(struct ib_uflow_resources * uflow_res)2595  void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2596  {
2597  	unsigned int i;
2598  
2599  	if (!uflow_res)
2600  		return;
2601  
2602  	for (i = 0; i < uflow_res->collection_num; i++)
2603  		atomic_dec(&uflow_res->collection[i]->usecnt);
2604  
2605  	for (i = 0; i < uflow_res->counters_num; i++)
2606  		atomic_dec(&uflow_res->counters[i]->usecnt);
2607  
2608  	kfree(uflow_res->collection);
2609  	kfree(uflow_res->counters);
2610  	kfree(uflow_res);
2611  }
2612  EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2613  
flow_resources_add(struct ib_uflow_resources * uflow_res,enum ib_flow_spec_type type,void * ibobj)2614  void flow_resources_add(struct ib_uflow_resources *uflow_res,
2615  			enum ib_flow_spec_type type,
2616  			void *ibobj)
2617  {
2618  	WARN_ON(uflow_res->num >= uflow_res->max);
2619  
2620  	switch (type) {
2621  	case IB_FLOW_SPEC_ACTION_HANDLE:
2622  		atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
2623  		uflow_res->collection[uflow_res->collection_num++] =
2624  			(struct ib_flow_action *)ibobj;
2625  		break;
2626  	case IB_FLOW_SPEC_ACTION_COUNT:
2627  		atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
2628  		uflow_res->counters[uflow_res->counters_num++] =
2629  			(struct ib_counters *)ibobj;
2630  		break;
2631  	default:
2632  		WARN_ON(1);
2633  	}
2634  
2635  	uflow_res->num++;
2636  }
2637  EXPORT_SYMBOL(flow_resources_add);
2638  
kern_spec_to_ib_spec_action(struct uverbs_attr_bundle * attrs,struct ib_uverbs_flow_spec * kern_spec,union ib_flow_spec * ib_spec,struct ib_uflow_resources * uflow_res)2639  static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
2640  				       struct ib_uverbs_flow_spec *kern_spec,
2641  				       union ib_flow_spec *ib_spec,
2642  				       struct ib_uflow_resources *uflow_res)
2643  {
2644  	ib_spec->type = kern_spec->type;
2645  	switch (ib_spec->type) {
2646  	case IB_FLOW_SPEC_ACTION_TAG:
2647  		if (kern_spec->flow_tag.size !=
2648  		    sizeof(struct ib_uverbs_flow_spec_action_tag))
2649  			return -EINVAL;
2650  
2651  		ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2652  		ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2653  		break;
2654  	case IB_FLOW_SPEC_ACTION_DROP:
2655  		if (kern_spec->drop.size !=
2656  		    sizeof(struct ib_uverbs_flow_spec_action_drop))
2657  			return -EINVAL;
2658  
2659  		ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2660  		break;
2661  	case IB_FLOW_SPEC_ACTION_HANDLE:
2662  		if (kern_spec->action.size !=
2663  		    sizeof(struct ib_uverbs_flow_spec_action_handle))
2664  			return -EOPNOTSUPP;
2665  		ib_spec->action.act = uobj_get_obj_read(flow_action,
2666  							UVERBS_OBJECT_FLOW_ACTION,
2667  							kern_spec->action.handle,
2668  							attrs);
2669  		if (!ib_spec->action.act)
2670  			return -EINVAL;
2671  		ib_spec->action.size =
2672  			sizeof(struct ib_flow_spec_action_handle);
2673  		flow_resources_add(uflow_res,
2674  				   IB_FLOW_SPEC_ACTION_HANDLE,
2675  				   ib_spec->action.act);
2676  		uobj_put_obj_read(ib_spec->action.act);
2677  		break;
2678  	case IB_FLOW_SPEC_ACTION_COUNT:
2679  		if (kern_spec->flow_count.size !=
2680  			sizeof(struct ib_uverbs_flow_spec_action_count))
2681  			return -EINVAL;
2682  		ib_spec->flow_count.counters =
2683  			uobj_get_obj_read(counters,
2684  					  UVERBS_OBJECT_COUNTERS,
2685  					  kern_spec->flow_count.handle,
2686  					  attrs);
2687  		if (!ib_spec->flow_count.counters)
2688  			return -EINVAL;
2689  		ib_spec->flow_count.size =
2690  				sizeof(struct ib_flow_spec_action_count);
2691  		flow_resources_add(uflow_res,
2692  				   IB_FLOW_SPEC_ACTION_COUNT,
2693  				   ib_spec->flow_count.counters);
2694  		uobj_put_obj_read(ib_spec->flow_count.counters);
2695  		break;
2696  	default:
2697  		return -EINVAL;
2698  	}
2699  	return 0;
2700  }
2701  
spec_filter_size(const void * kern_spec_filter,u16 kern_filter_size,u16 ib_real_filter_sz)2702  static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,
2703  				u16 ib_real_filter_sz)
2704  {
2705  	/*
2706  	 * User space filter structures must be 64 bit aligned, otherwise this
2707  	 * may pass, but we won't handle additional new attributes.
2708  	 */
2709  
2710  	if (kern_filter_size > ib_real_filter_sz) {
2711  		if (memchr_inv(kern_spec_filter +
2712  			       ib_real_filter_sz, 0,
2713  			       kern_filter_size - ib_real_filter_sz))
2714  			return -EINVAL;
2715  		return ib_real_filter_sz;
2716  	}
2717  	return kern_filter_size;
2718  }
2719  
ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,const void * kern_spec_mask,const void * kern_spec_val,size_t kern_filter_sz,union ib_flow_spec * ib_spec)2720  int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
2721  					  const void *kern_spec_mask,
2722  					  const void *kern_spec_val,
2723  					  size_t kern_filter_sz,
2724  					  union ib_flow_spec *ib_spec)
2725  {
2726  	ssize_t actual_filter_sz;
2727  	ssize_t ib_filter_sz;
2728  
2729  	/* User flow spec size must be aligned to 4 bytes */
2730  	if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2731  		return -EINVAL;
2732  
2733  	ib_spec->type = type;
2734  
2735  	if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2736  		return -EINVAL;
2737  
2738  	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2739  	case IB_FLOW_SPEC_ETH:
2740  		ib_filter_sz = sizeof(struct ib_flow_eth_filter);
2741  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2742  						    kern_filter_sz,
2743  						    ib_filter_sz);
2744  		if (actual_filter_sz <= 0)
2745  			return -EINVAL;
2746  		ib_spec->size = sizeof(struct ib_flow_spec_eth);
2747  		memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2748  		memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2749  		break;
2750  	case IB_FLOW_SPEC_IPV4:
2751  		ib_filter_sz = sizeof(struct ib_flow_ipv4_filter);
2752  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2753  						    kern_filter_sz,
2754  						    ib_filter_sz);
2755  		if (actual_filter_sz <= 0)
2756  			return -EINVAL;
2757  		ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2758  		memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2759  		memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2760  		break;
2761  	case IB_FLOW_SPEC_IPV6:
2762  		ib_filter_sz = sizeof(struct ib_flow_ipv6_filter);
2763  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2764  						    kern_filter_sz,
2765  						    ib_filter_sz);
2766  		if (actual_filter_sz <= 0)
2767  			return -EINVAL;
2768  		ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2769  		memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2770  		memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2771  
2772  		if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2773  		    (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2774  			return -EINVAL;
2775  		break;
2776  	case IB_FLOW_SPEC_TCP:
2777  	case IB_FLOW_SPEC_UDP:
2778  		ib_filter_sz = sizeof(struct ib_flow_tcp_udp_filter);
2779  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2780  						    kern_filter_sz,
2781  						    ib_filter_sz);
2782  		if (actual_filter_sz <= 0)
2783  			return -EINVAL;
2784  		ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2785  		memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2786  		memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2787  		break;
2788  	case IB_FLOW_SPEC_VXLAN_TUNNEL:
2789  		ib_filter_sz = sizeof(struct ib_flow_tunnel_filter);
2790  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2791  						    kern_filter_sz,
2792  						    ib_filter_sz);
2793  		if (actual_filter_sz <= 0)
2794  			return -EINVAL;
2795  		ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2796  		memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2797  		memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2798  
2799  		if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2800  		    (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2801  			return -EINVAL;
2802  		break;
2803  	case IB_FLOW_SPEC_ESP:
2804  		ib_filter_sz = sizeof(struct ib_flow_esp_filter);
2805  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2806  						    kern_filter_sz,
2807  						    ib_filter_sz);
2808  		if (actual_filter_sz <= 0)
2809  			return -EINVAL;
2810  		ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);
2811  		memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
2812  		memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
2813  		break;
2814  	case IB_FLOW_SPEC_GRE:
2815  		ib_filter_sz = sizeof(struct ib_flow_gre_filter);
2816  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2817  						    kern_filter_sz,
2818  						    ib_filter_sz);
2819  		if (actual_filter_sz <= 0)
2820  			return -EINVAL;
2821  		ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
2822  		memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
2823  		memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
2824  		break;
2825  	case IB_FLOW_SPEC_MPLS:
2826  		ib_filter_sz = sizeof(struct ib_flow_mpls_filter);
2827  		actual_filter_sz = spec_filter_size(kern_spec_mask,
2828  						    kern_filter_sz,
2829  						    ib_filter_sz);
2830  		if (actual_filter_sz <= 0)
2831  			return -EINVAL;
2832  		ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
2833  		memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
2834  		memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
2835  		break;
2836  	default:
2837  		return -EINVAL;
2838  	}
2839  	return 0;
2840  }
2841  
kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec * kern_spec,union ib_flow_spec * ib_spec)2842  static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2843  				       union ib_flow_spec *ib_spec)
2844  {
2845  	size_t kern_filter_sz;
2846  	void *kern_spec_mask;
2847  	void *kern_spec_val;
2848  
2849  	if (check_sub_overflow((size_t)kern_spec->hdr.size,
2850  			       sizeof(struct ib_uverbs_flow_spec_hdr),
2851  			       &kern_filter_sz))
2852  		return -EINVAL;
2853  
2854  	kern_filter_sz /= 2;
2855  
2856  	kern_spec_val = (void *)kern_spec +
2857  		sizeof(struct ib_uverbs_flow_spec_hdr);
2858  	kern_spec_mask = kern_spec_val + kern_filter_sz;
2859  
2860  	return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type,
2861  						     kern_spec_mask,
2862  						     kern_spec_val,
2863  						     kern_filter_sz, ib_spec);
2864  }
2865  
kern_spec_to_ib_spec(struct uverbs_attr_bundle * attrs,struct ib_uverbs_flow_spec * kern_spec,union ib_flow_spec * ib_spec,struct ib_uflow_resources * uflow_res)2866  static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
2867  				struct ib_uverbs_flow_spec *kern_spec,
2868  				union ib_flow_spec *ib_spec,
2869  				struct ib_uflow_resources *uflow_res)
2870  {
2871  	if (kern_spec->reserved)
2872  		return -EINVAL;
2873  
2874  	if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2875  		return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
2876  						   uflow_res);
2877  	else
2878  		return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2879  }
2880  
ib_uverbs_ex_create_wq(struct uverbs_attr_bundle * attrs)2881  static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
2882  {
2883  	struct ib_uverbs_ex_create_wq cmd;
2884  	struct ib_uverbs_ex_create_wq_resp resp = {};
2885  	struct ib_uwq_object           *obj;
2886  	int err = 0;
2887  	struct ib_cq *cq;
2888  	struct ib_pd *pd;
2889  	struct ib_wq *wq;
2890  	struct ib_wq_init_attr wq_init_attr = {};
2891  	struct ib_device *ib_dev;
2892  
2893  	err = uverbs_request(attrs, &cmd, sizeof(cmd));
2894  	if (err)
2895  		return err;
2896  
2897  	if (cmd.comp_mask)
2898  		return -EOPNOTSUPP;
2899  
2900  	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
2901  						 &ib_dev);
2902  	if (IS_ERR(obj))
2903  		return PTR_ERR(obj);
2904  
2905  	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2906  	if (!pd) {
2907  		err = -EINVAL;
2908  		goto err_uobj;
2909  	}
2910  
2911  	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
2912  	if (!cq) {
2913  		err = -EINVAL;
2914  		goto err_put_pd;
2915  	}
2916  
2917  	wq_init_attr.cq = cq;
2918  	wq_init_attr.max_sge = cmd.max_sge;
2919  	wq_init_attr.max_wr = cmd.max_wr;
2920  	wq_init_attr.wq_type = cmd.wq_type;
2921  	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
2922  	wq_init_attr.create_flags = cmd.create_flags;
2923  	INIT_LIST_HEAD(&obj->uevent.event_list);
2924  	obj->uevent.uobject.user_handle = cmd.user_handle;
2925  
2926  	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
2927  	if (IS_ERR(wq)) {
2928  		err = PTR_ERR(wq);
2929  		goto err_put_cq;
2930  	}
2931  
2932  	wq->uobject = obj;
2933  	obj->uevent.uobject.object = wq;
2934  	wq->wq_type = wq_init_attr.wq_type;
2935  	wq->cq = cq;
2936  	wq->pd = pd;
2937  	wq->device = pd->device;
2938  	atomic_set(&wq->usecnt, 0);
2939  	atomic_inc(&pd->usecnt);
2940  	atomic_inc(&cq->usecnt);
2941  	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
2942  	if (obj->uevent.event_file)
2943  		uverbs_uobject_get(&obj->uevent.event_file->uobj);
2944  
2945  	uobj_put_obj_read(pd);
2946  	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
2947  				UVERBS_LOOKUP_READ);
2948  	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
2949  
2950  	resp.wq_handle = obj->uevent.uobject.id;
2951  	resp.max_sge = wq_init_attr.max_sge;
2952  	resp.max_wr = wq_init_attr.max_wr;
2953  	resp.wqn = wq->wq_num;
2954  	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2955  	return uverbs_response(attrs, &resp, sizeof(resp));
2956  
2957  err_put_cq:
2958  	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
2959  				UVERBS_LOOKUP_READ);
2960  err_put_pd:
2961  	uobj_put_obj_read(pd);
2962  err_uobj:
2963  	uobj_alloc_abort(&obj->uevent.uobject, attrs);
2964  
2965  	return err;
2966  }
2967  
ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle * attrs)2968  static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
2969  {
2970  	struct ib_uverbs_ex_destroy_wq	cmd;
2971  	struct ib_uverbs_ex_destroy_wq_resp	resp = {};
2972  	struct ib_uobject		*uobj;
2973  	struct ib_uwq_object		*obj;
2974  	int				ret;
2975  
2976  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2977  	if (ret)
2978  		return ret;
2979  
2980  	if (cmd.comp_mask)
2981  		return -EOPNOTSUPP;
2982  
2983  	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2984  	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
2985  	if (IS_ERR(uobj))
2986  		return PTR_ERR(uobj);
2987  
2988  	obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
2989  	resp.events_reported = obj->uevent.events_reported;
2990  
2991  	uobj_put_destroy(uobj);
2992  
2993  	return uverbs_response(attrs, &resp, sizeof(resp));
2994  }
2995  
ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle * attrs)2996  static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
2997  {
2998  	struct ib_uverbs_ex_modify_wq cmd;
2999  	struct ib_wq *wq;
3000  	struct ib_wq_attr wq_attr = {};
3001  	int ret;
3002  
3003  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3004  	if (ret)
3005  		return ret;
3006  
3007  	if (!cmd.attr_mask)
3008  		return -EINVAL;
3009  
3010  	if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3011  		return -EINVAL;
3012  
3013  	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3014  	if (!wq)
3015  		return -EINVAL;
3016  
3017  	if (cmd.attr_mask & IB_WQ_FLAGS) {
3018  		wq_attr.flags = cmd.flags;
3019  		wq_attr.flags_mask = cmd.flags_mask;
3020  	}
3021  
3022  	if (cmd.attr_mask & IB_WQ_CUR_STATE) {
3023  		if (cmd.curr_wq_state > IB_WQS_ERR)
3024  			return -EINVAL;
3025  
3026  		wq_attr.curr_wq_state = cmd.curr_wq_state;
3027  	} else {
3028  		wq_attr.curr_wq_state = wq->state;
3029  	}
3030  
3031  	if (cmd.attr_mask & IB_WQ_STATE) {
3032  		if (cmd.wq_state > IB_WQS_ERR)
3033  			return -EINVAL;
3034  
3035  		wq_attr.wq_state = cmd.wq_state;
3036  	} else {
3037  		wq_attr.wq_state = wq_attr.curr_wq_state;
3038  	}
3039  
3040  	ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
3041  					&attrs->driver_udata);
3042  	rdma_lookup_put_uobject(&wq->uobject->uevent.uobject,
3043  				UVERBS_LOOKUP_READ);
3044  	return ret;
3045  }
3046  
ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle * attrs)3047  static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3048  {
3049  	struct ib_uverbs_ex_create_rwq_ind_table cmd;
3050  	struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};
3051  	struct ib_uobject *uobj;
3052  	int err;
3053  	struct ib_rwq_ind_table_init_attr init_attr = {};
3054  	struct ib_rwq_ind_table *rwq_ind_tbl;
3055  	struct ib_wq **wqs = NULL;
3056  	u32 *wqs_handles = NULL;
3057  	struct ib_wq	*wq = NULL;
3058  	int i, num_read_wqs;
3059  	u32 num_wq_handles;
3060  	struct uverbs_req_iter iter;
3061  	struct ib_device *ib_dev;
3062  
3063  	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3064  	if (err)
3065  		return err;
3066  
3067  	if (cmd.comp_mask)
3068  		return -EOPNOTSUPP;
3069  
3070  	if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3071  		return -EINVAL;
3072  
3073  	num_wq_handles = 1 << cmd.log_ind_tbl_size;
3074  	wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3075  			      GFP_KERNEL);
3076  	if (!wqs_handles)
3077  		return -ENOMEM;
3078  
3079  	err = uverbs_request_next(&iter, wqs_handles,
3080  				  num_wq_handles * sizeof(__u32));
3081  	if (err)
3082  		goto err_free;
3083  
3084  	err = uverbs_request_finish(&iter);
3085  	if (err)
3086  		goto err_free;
3087  
3088  	wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3089  	if (!wqs) {
3090  		err = -ENOMEM;
3091  		goto  err_free;
3092  	}
3093  
3094  	for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3095  			num_read_wqs++) {
3096  		wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3097  				       wqs_handles[num_read_wqs], attrs);
3098  		if (!wq) {
3099  			err = -EINVAL;
3100  			goto put_wqs;
3101  		}
3102  
3103  		wqs[num_read_wqs] = wq;
3104  		atomic_inc(&wqs[num_read_wqs]->usecnt);
3105  	}
3106  
3107  	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3108  	if (IS_ERR(uobj)) {
3109  		err = PTR_ERR(uobj);
3110  		goto put_wqs;
3111  	}
3112  
3113  	rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table);
3114  	if (!rwq_ind_tbl) {
3115  		err = -ENOMEM;
3116  		goto err_uobj;
3117  	}
3118  
3119  	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3120  	init_attr.ind_tbl = wqs;
3121  
3122  	rwq_ind_tbl->ind_tbl = wqs;
3123  	rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3124  	rwq_ind_tbl->uobject = uobj;
3125  	uobj->object = rwq_ind_tbl;
3126  	rwq_ind_tbl->device = ib_dev;
3127  	atomic_set(&rwq_ind_tbl->usecnt, 0);
3128  
3129  	err = ib_dev->ops.create_rwq_ind_table(rwq_ind_tbl, &init_attr,
3130  					       &attrs->driver_udata);
3131  	if (err)
3132  		goto err_create;
3133  
3134  	for (i = 0; i < num_wq_handles; i++)
3135  		rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
3136  					UVERBS_LOOKUP_READ);
3137  	kfree(wqs_handles);
3138  	uobj_finalize_uobj_create(uobj, attrs);
3139  
3140  	resp.ind_tbl_handle = uobj->id;
3141  	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3142  	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3143  	return uverbs_response(attrs, &resp, sizeof(resp));
3144  
3145  err_create:
3146  	kfree(rwq_ind_tbl);
3147  err_uobj:
3148  	uobj_alloc_abort(uobj, attrs);
3149  put_wqs:
3150  	for (i = 0; i < num_read_wqs; i++) {
3151  		rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
3152  					UVERBS_LOOKUP_READ);
3153  		atomic_dec(&wqs[i]->usecnt);
3154  	}
3155  err_free:
3156  	kfree(wqs_handles);
3157  	kfree(wqs);
3158  	return err;
3159  }
3160  
ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle * attrs)3161  static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3162  {
3163  	struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
3164  	int ret;
3165  
3166  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3167  	if (ret)
3168  		return ret;
3169  
3170  	if (cmd.comp_mask)
3171  		return -EOPNOTSUPP;
3172  
3173  	return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3174  				    cmd.ind_tbl_handle, attrs);
3175  }
3176  
ib_uverbs_ex_create_flow(struct uverbs_attr_bundle * attrs)3177  static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3178  {
3179  	struct ib_uverbs_create_flow	  cmd;
3180  	struct ib_uverbs_create_flow_resp resp = {};
3181  	struct ib_uobject		  *uobj;
3182  	struct ib_flow			  *flow_id;
3183  	struct ib_uverbs_flow_attr	  *kern_flow_attr;
3184  	struct ib_flow_attr		  *flow_attr;
3185  	struct ib_qp			  *qp;
3186  	struct ib_uflow_resources	  *uflow_res;
3187  	struct ib_uverbs_flow_spec_hdr	  *kern_spec;
3188  	struct uverbs_req_iter iter;
3189  	int err;
3190  	void *ib_spec;
3191  	int i;
3192  	struct ib_device *ib_dev;
3193  
3194  	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3195  	if (err)
3196  		return err;
3197  
3198  	if (cmd.comp_mask)
3199  		return -EINVAL;
3200  
3201  	if (!capable(CAP_NET_RAW))
3202  		return -EPERM;
3203  
3204  	if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3205  		return -EINVAL;
3206  
3207  	if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3208  	    ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3209  	     (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3210  		return -EINVAL;
3211  
3212  	if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3213  		return -EINVAL;
3214  
3215  	if (cmd.flow_attr.size >
3216  	    (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3217  		return -EINVAL;
3218  
3219  	if (cmd.flow_attr.reserved[0] ||
3220  	    cmd.flow_attr.reserved[1])
3221  		return -EINVAL;
3222  
3223  	if (cmd.flow_attr.num_of_specs) {
3224  		kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3225  					 GFP_KERNEL);
3226  		if (!kern_flow_attr)
3227  			return -ENOMEM;
3228  
3229  		*kern_flow_attr = cmd.flow_attr;
3230  		err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
3231  					  cmd.flow_attr.size);
3232  		if (err)
3233  			goto err_free_attr;
3234  	} else {
3235  		kern_flow_attr = &cmd.flow_attr;
3236  	}
3237  
3238  	err = uverbs_request_finish(&iter);
3239  	if (err)
3240  		goto err_free_attr;
3241  
3242  	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3243  	if (IS_ERR(uobj)) {
3244  		err = PTR_ERR(uobj);
3245  		goto err_free_attr;
3246  	}
3247  
3248  	if (!rdma_is_port_valid(uobj->context->device, cmd.flow_attr.port)) {
3249  		err = -EINVAL;
3250  		goto err_uobj;
3251  	}
3252  
3253  	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3254  	if (!qp) {
3255  		err = -EINVAL;
3256  		goto err_uobj;
3257  	}
3258  
3259  	if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3260  		err = -EINVAL;
3261  		goto err_put;
3262  	}
3263  
3264  	flow_attr = kzalloc(struct_size(flow_attr, flows,
3265  				cmd.flow_attr.num_of_specs), GFP_KERNEL);
3266  	if (!flow_attr) {
3267  		err = -ENOMEM;
3268  		goto err_put;
3269  	}
3270  	uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);
3271  	if (!uflow_res) {
3272  		err = -ENOMEM;
3273  		goto err_free_flow_attr;
3274  	}
3275  
3276  	flow_attr->type = kern_flow_attr->type;
3277  	flow_attr->priority = kern_flow_attr->priority;
3278  	flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3279  	flow_attr->port = kern_flow_attr->port;
3280  	flow_attr->flags = kern_flow_attr->flags;
3281  	flow_attr->size = sizeof(*flow_attr);
3282  
3283  	kern_spec = kern_flow_attr->flow_specs;
3284  	ib_spec = flow_attr + 1;
3285  	for (i = 0; i < flow_attr->num_of_specs &&
3286  			cmd.flow_attr.size >= sizeof(*kern_spec) &&
3287  			cmd.flow_attr.size >= kern_spec->size;
3288  	     i++) {
3289  		err = kern_spec_to_ib_spec(
3290  				attrs, (struct ib_uverbs_flow_spec *)kern_spec,
3291  				ib_spec, uflow_res);
3292  		if (err)
3293  			goto err_free;
3294  
3295  		flow_attr->size +=
3296  			((union ib_flow_spec *) ib_spec)->size;
3297  		cmd.flow_attr.size -= kern_spec->size;
3298  		kern_spec = ((void *)kern_spec) + kern_spec->size;
3299  		ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3300  	}
3301  	if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3302  		pr_warn("create flow failed, flow %d: %u bytes left from uverb cmd\n",
3303  			i, cmd.flow_attr.size);
3304  		err = -EINVAL;
3305  		goto err_free;
3306  	}
3307  
3308  	flow_id = qp->device->ops.create_flow(qp, flow_attr,
3309  					      &attrs->driver_udata);
3310  
3311  	if (IS_ERR(flow_id)) {
3312  		err = PTR_ERR(flow_id);
3313  		goto err_free;
3314  	}
3315  
3316  	ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3317  
3318  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
3319  				UVERBS_LOOKUP_READ);
3320  	kfree(flow_attr);
3321  
3322  	if (cmd.flow_attr.num_of_specs)
3323  		kfree(kern_flow_attr);
3324  	uobj_finalize_uobj_create(uobj, attrs);
3325  
3326  	resp.flow_handle = uobj->id;
3327  	return uverbs_response(attrs, &resp, sizeof(resp));
3328  
3329  err_free:
3330  	ib_uverbs_flow_resources_free(uflow_res);
3331  err_free_flow_attr:
3332  	kfree(flow_attr);
3333  err_put:
3334  	rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
3335  				UVERBS_LOOKUP_READ);
3336  err_uobj:
3337  	uobj_alloc_abort(uobj, attrs);
3338  err_free_attr:
3339  	if (cmd.flow_attr.num_of_specs)
3340  		kfree(kern_flow_attr);
3341  	return err;
3342  }
3343  
ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle * attrs)3344  static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3345  {
3346  	struct ib_uverbs_destroy_flow	cmd;
3347  	int				ret;
3348  
3349  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3350  	if (ret)
3351  		return ret;
3352  
3353  	if (cmd.comp_mask)
3354  		return -EINVAL;
3355  
3356  	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3357  }
3358  
__uverbs_create_xsrq(struct uverbs_attr_bundle * attrs,struct ib_uverbs_create_xsrq * cmd,struct ib_udata * udata)3359  static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3360  				struct ib_uverbs_create_xsrq *cmd,
3361  				struct ib_udata *udata)
3362  {
3363  	struct ib_uverbs_create_srq_resp resp = {};
3364  	struct ib_usrq_object           *obj;
3365  	struct ib_pd                    *pd;
3366  	struct ib_srq                   *srq;
3367  	struct ib_srq_init_attr          attr;
3368  	int ret;
3369  	struct ib_uobject *xrcd_uobj;
3370  	struct ib_device *ib_dev;
3371  
3372  	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3373  						  &ib_dev);
3374  	if (IS_ERR(obj))
3375  		return PTR_ERR(obj);
3376  
3377  	if (cmd->srq_type == IB_SRQT_TM)
3378  		attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3379  
3380  	if (cmd->srq_type == IB_SRQT_XRC) {
3381  		xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3382  					  attrs);
3383  		if (IS_ERR(xrcd_uobj)) {
3384  			ret = -EINVAL;
3385  			goto err;
3386  		}
3387  
3388  		attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3389  		if (!attr.ext.xrc.xrcd) {
3390  			ret = -EINVAL;
3391  			goto err_put_xrcd;
3392  		}
3393  
3394  		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3395  		atomic_inc(&obj->uxrcd->refcnt);
3396  	}
3397  
3398  	if (ib_srq_has_cq(cmd->srq_type)) {
3399  		attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3400  						cmd->cq_handle, attrs);
3401  		if (!attr.ext.cq) {
3402  			ret = -EINVAL;
3403  			goto err_put_xrcd;
3404  		}
3405  	}
3406  
3407  	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3408  	if (!pd) {
3409  		ret = -EINVAL;
3410  		goto err_put_cq;
3411  	}
3412  
3413  	attr.event_handler  = ib_uverbs_srq_event_handler;
3414  	attr.srq_type       = cmd->srq_type;
3415  	attr.attr.max_wr    = cmd->max_wr;
3416  	attr.attr.max_sge   = cmd->max_sge;
3417  	attr.attr.srq_limit = cmd->srq_limit;
3418  
3419  	INIT_LIST_HEAD(&obj->uevent.event_list);
3420  	obj->uevent.uobject.user_handle = cmd->user_handle;
3421  
3422  	srq = ib_create_srq_user(pd, &attr, obj, udata);
3423  	if (IS_ERR(srq)) {
3424  		ret = PTR_ERR(srq);
3425  		goto err_put_pd;
3426  	}
3427  
3428  	obj->uevent.uobject.object = srq;
3429  	obj->uevent.uobject.user_handle = cmd->user_handle;
3430  	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
3431  	if (obj->uevent.event_file)
3432  		uverbs_uobject_get(&obj->uevent.event_file->uobj);
3433  
3434  	if (cmd->srq_type == IB_SRQT_XRC)
3435  		resp.srqn = srq->ext.xrc.srq_num;
3436  
3437  	if (cmd->srq_type == IB_SRQT_XRC)
3438  		uobj_put_read(xrcd_uobj);
3439  
3440  	if (ib_srq_has_cq(cmd->srq_type))
3441  		rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
3442  					UVERBS_LOOKUP_READ);
3443  
3444  	uobj_put_obj_read(pd);
3445  	uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
3446  
3447  	resp.srq_handle = obj->uevent.uobject.id;
3448  	resp.max_wr = attr.attr.max_wr;
3449  	resp.max_sge = attr.attr.max_sge;
3450  	return uverbs_response(attrs, &resp, sizeof(resp));
3451  
3452  err_put_pd:
3453  	uobj_put_obj_read(pd);
3454  err_put_cq:
3455  	if (ib_srq_has_cq(cmd->srq_type))
3456  		rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
3457  					UVERBS_LOOKUP_READ);
3458  
3459  err_put_xrcd:
3460  	if (cmd->srq_type == IB_SRQT_XRC) {
3461  		atomic_dec(&obj->uxrcd->refcnt);
3462  		uobj_put_read(xrcd_uobj);
3463  	}
3464  
3465  err:
3466  	uobj_alloc_abort(&obj->uevent.uobject, attrs);
3467  	return ret;
3468  }
3469  
ib_uverbs_create_srq(struct uverbs_attr_bundle * attrs)3470  static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3471  {
3472  	struct ib_uverbs_create_srq      cmd;
3473  	struct ib_uverbs_create_xsrq     xcmd;
3474  	int ret;
3475  
3476  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3477  	if (ret)
3478  		return ret;
3479  
3480  	memset(&xcmd, 0, sizeof(xcmd));
3481  	xcmd.response	 = cmd.response;
3482  	xcmd.user_handle = cmd.user_handle;
3483  	xcmd.srq_type	 = IB_SRQT_BASIC;
3484  	xcmd.pd_handle	 = cmd.pd_handle;
3485  	xcmd.max_wr	 = cmd.max_wr;
3486  	xcmd.max_sge	 = cmd.max_sge;
3487  	xcmd.srq_limit	 = cmd.srq_limit;
3488  
3489  	return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
3490  }
3491  
ib_uverbs_create_xsrq(struct uverbs_attr_bundle * attrs)3492  static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3493  {
3494  	struct ib_uverbs_create_xsrq     cmd;
3495  	int ret;
3496  
3497  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3498  	if (ret)
3499  		return ret;
3500  
3501  	return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
3502  }
3503  
ib_uverbs_modify_srq(struct uverbs_attr_bundle * attrs)3504  static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3505  {
3506  	struct ib_uverbs_modify_srq cmd;
3507  	struct ib_srq              *srq;
3508  	struct ib_srq_attr          attr;
3509  	int                         ret;
3510  
3511  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3512  	if (ret)
3513  		return ret;
3514  
3515  	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3516  	if (!srq)
3517  		return -EINVAL;
3518  
3519  	attr.max_wr    = cmd.max_wr;
3520  	attr.srq_limit = cmd.srq_limit;
3521  
3522  	ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
3523  					  &attrs->driver_udata);
3524  
3525  	rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
3526  				UVERBS_LOOKUP_READ);
3527  
3528  	return ret;
3529  }
3530  
ib_uverbs_query_srq(struct uverbs_attr_bundle * attrs)3531  static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3532  {
3533  	struct ib_uverbs_query_srq      cmd;
3534  	struct ib_uverbs_query_srq_resp resp;
3535  	struct ib_srq_attr              attr;
3536  	struct ib_srq                   *srq;
3537  	int                             ret;
3538  
3539  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3540  	if (ret)
3541  		return ret;
3542  
3543  	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3544  	if (!srq)
3545  		return -EINVAL;
3546  
3547  	ret = ib_query_srq(srq, &attr);
3548  
3549  	rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
3550  				UVERBS_LOOKUP_READ);
3551  
3552  	if (ret)
3553  		return ret;
3554  
3555  	memset(&resp, 0, sizeof resp);
3556  
3557  	resp.max_wr    = attr.max_wr;
3558  	resp.max_sge   = attr.max_sge;
3559  	resp.srq_limit = attr.srq_limit;
3560  
3561  	return uverbs_response(attrs, &resp, sizeof(resp));
3562  }
3563  
ib_uverbs_destroy_srq(struct uverbs_attr_bundle * attrs)3564  static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3565  {
3566  	struct ib_uverbs_destroy_srq      cmd;
3567  	struct ib_uverbs_destroy_srq_resp resp;
3568  	struct ib_uobject		 *uobj;
3569  	struct ib_uevent_object        	 *obj;
3570  	int ret;
3571  
3572  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3573  	if (ret)
3574  		return ret;
3575  
3576  	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3577  	if (IS_ERR(uobj))
3578  		return PTR_ERR(uobj);
3579  
3580  	obj = container_of(uobj, struct ib_uevent_object, uobject);
3581  	memset(&resp, 0, sizeof(resp));
3582  	resp.events_reported = obj->events_reported;
3583  
3584  	uobj_put_destroy(uobj);
3585  
3586  	return uverbs_response(attrs, &resp, sizeof(resp));
3587  }
3588  
ib_uverbs_ex_query_device(struct uverbs_attr_bundle * attrs)3589  static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3590  {
3591  	struct ib_uverbs_ex_query_device_resp resp = {};
3592  	struct ib_uverbs_ex_query_device  cmd;
3593  	struct ib_device_attr attr = {0};
3594  	struct ib_ucontext *ucontext;
3595  	struct ib_device *ib_dev;
3596  	int err;
3597  
3598  	ucontext = ib_uverbs_get_ucontext(attrs);
3599  	if (IS_ERR(ucontext))
3600  		return PTR_ERR(ucontext);
3601  	ib_dev = ucontext->device;
3602  
3603  	err = uverbs_request(attrs, &cmd, sizeof(cmd));
3604  	if (err)
3605  		return err;
3606  
3607  	if (cmd.comp_mask)
3608  		return -EINVAL;
3609  
3610  	if (cmd.reserved)
3611  		return -EINVAL;
3612  
3613  	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
3614  	if (err)
3615  		return err;
3616  
3617  	copy_query_dev_fields(ucontext, &resp.base, &attr);
3618  
3619  	resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3620  	resp.odp_caps.per_transport_caps.rc_odp_caps =
3621  		attr.odp_caps.per_transport_caps.rc_odp_caps;
3622  	resp.odp_caps.per_transport_caps.uc_odp_caps =
3623  		attr.odp_caps.per_transport_caps.uc_odp_caps;
3624  	resp.odp_caps.per_transport_caps.ud_odp_caps =
3625  		attr.odp_caps.per_transport_caps.ud_odp_caps;
3626  	resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;
3627  
3628  	resp.timestamp_mask = attr.timestamp_mask;
3629  	resp.hca_core_clock = attr.hca_core_clock;
3630  	resp.device_cap_flags_ex = attr.device_cap_flags;
3631  	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3632  	resp.rss_caps.max_rwq_indirection_tables =
3633  		attr.rss_caps.max_rwq_indirection_tables;
3634  	resp.rss_caps.max_rwq_indirection_table_size =
3635  		attr.rss_caps.max_rwq_indirection_table_size;
3636  	resp.max_wq_type_rq = attr.max_wq_type_rq;
3637  	resp.raw_packet_caps = attr.raw_packet_caps;
3638  	resp.tm_caps.max_rndv_hdr_size	= attr.tm_caps.max_rndv_hdr_size;
3639  	resp.tm_caps.max_num_tags	= attr.tm_caps.max_num_tags;
3640  	resp.tm_caps.max_ops		= attr.tm_caps.max_ops;
3641  	resp.tm_caps.max_sge		= attr.tm_caps.max_sge;
3642  	resp.tm_caps.flags		= attr.tm_caps.flags;
3643  	resp.cq_moderation_caps.max_cq_moderation_count  =
3644  		attr.cq_caps.max_cq_moderation_count;
3645  	resp.cq_moderation_caps.max_cq_moderation_period =
3646  		attr.cq_caps.max_cq_moderation_period;
3647  	resp.max_dm_size = attr.max_dm_size;
3648  	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3649  
3650  	return uverbs_response(attrs, &resp, sizeof(resp));
3651  }
3652  
ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle * attrs)3653  static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
3654  {
3655  	struct ib_uverbs_ex_modify_cq cmd;
3656  	struct ib_cq *cq;
3657  	int ret;
3658  
3659  	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3660  	if (ret)
3661  		return ret;
3662  
3663  	if (!cmd.attr_mask || cmd.reserved)
3664  		return -EINVAL;
3665  
3666  	if (cmd.attr_mask > IB_CQ_MODERATE)
3667  		return -EOPNOTSUPP;
3668  
3669  	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3670  	if (!cq)
3671  		return -EINVAL;
3672  
3673  	ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
3674  
3675  	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
3676  				UVERBS_LOOKUP_READ);
3677  	return ret;
3678  }
3679  
3680  /*
3681   * Describe the input structs for write(). Some write methods have an input
3682   * only struct, most have an input and output. If the struct has an output then
3683   * the 'response' u64 must be the first field in the request structure.
3684   *
3685   * If udata is present then both the request and response structs have a
3686   * trailing driver_data flex array. In this case the size of the base struct
3687   * cannot be changed.
3688   */
3689  #define UAPI_DEF_WRITE_IO(req, resp)                                           \
3690  	.write.has_resp = 1 +                                                  \
3691  			  BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) +    \
3692  			  BUILD_BUG_ON_ZERO(sizeof_field(req, response) !=    \
3693  					    sizeof(u64)),                      \
3694  	.write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3695  
3696  #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3697  
3698  #define UAPI_DEF_WRITE_UDATA_IO(req, resp)                                     \
3699  	UAPI_DEF_WRITE_IO(req, resp),                                          \
3700  		.write.has_udata =                                             \
3701  			1 +                                                    \
3702  			BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=        \
3703  					  sizeof(req)) +                       \
3704  			BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) !=       \
3705  					  sizeof(resp))
3706  
3707  #define UAPI_DEF_WRITE_UDATA_I(req)                                            \
3708  	UAPI_DEF_WRITE_I(req),                                                 \
3709  		.write.has_udata =                                             \
3710  			1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=    \
3711  					      sizeof(req))
3712  
3713  /*
3714   * The _EX versions are for use with WRITE_EX and allow the last struct member
3715   * to be specified. Buffers that do not include that member will be rejected.
3716   */
3717  #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member)     \
3718  	.write.has_resp = 1,                                                   \
3719  	.write.req_size = offsetofend(req, req_last_member),                   \
3720  	.write.resp_size = offsetofend(resp, resp_last_member)
3721  
3722  #define UAPI_DEF_WRITE_I_EX(req, req_last_member)                              \
3723  	.write.req_size = offsetofend(req, req_last_member)
3724  
3725  const struct uapi_definition uverbs_def_write_intf[] = {
3726  	DECLARE_UVERBS_OBJECT(
3727  		UVERBS_OBJECT_AH,
3728  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
3729  				     ib_uverbs_create_ah,
3730  				     UAPI_DEF_WRITE_UDATA_IO(
3731  					     struct ib_uverbs_create_ah,
3732  					     struct ib_uverbs_create_ah_resp)),
3733  		DECLARE_UVERBS_WRITE(
3734  			IB_USER_VERBS_CMD_DESTROY_AH,
3735  			ib_uverbs_destroy_ah,
3736  			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah)),
3737  		UAPI_DEF_OBJ_NEEDS_FN(create_user_ah),
3738  		UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),
3739  
3740  	DECLARE_UVERBS_OBJECT(
3741  		UVERBS_OBJECT_COMP_CHANNEL,
3742  		DECLARE_UVERBS_WRITE(
3743  			IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
3744  			ib_uverbs_create_comp_channel,
3745  			UAPI_DEF_WRITE_IO(
3746  				struct ib_uverbs_create_comp_channel,
3747  				struct ib_uverbs_create_comp_channel_resp))),
3748  
3749  	DECLARE_UVERBS_OBJECT(
3750  		UVERBS_OBJECT_CQ,
3751  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
3752  				     ib_uverbs_create_cq,
3753  				     UAPI_DEF_WRITE_UDATA_IO(
3754  					     struct ib_uverbs_create_cq,
3755  					     struct ib_uverbs_create_cq_resp),
3756  				     UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3757  		DECLARE_UVERBS_WRITE(
3758  			IB_USER_VERBS_CMD_DESTROY_CQ,
3759  			ib_uverbs_destroy_cq,
3760  			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
3761  					  struct ib_uverbs_destroy_cq_resp),
3762  			UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
3763  		DECLARE_UVERBS_WRITE(
3764  			IB_USER_VERBS_CMD_POLL_CQ,
3765  			ib_uverbs_poll_cq,
3766  			UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
3767  					  struct ib_uverbs_poll_cq_resp),
3768  			UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
3769  		DECLARE_UVERBS_WRITE(
3770  			IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
3771  			ib_uverbs_req_notify_cq,
3772  			UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
3773  			UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
3774  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
3775  				     ib_uverbs_resize_cq,
3776  				     UAPI_DEF_WRITE_UDATA_IO(
3777  					     struct ib_uverbs_resize_cq,
3778  					     struct ib_uverbs_resize_cq_resp),
3779  				     UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
3780  		DECLARE_UVERBS_WRITE_EX(
3781  			IB_USER_VERBS_EX_CMD_CREATE_CQ,
3782  			ib_uverbs_ex_create_cq,
3783  			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
3784  					     reserved,
3785  					     struct ib_uverbs_ex_create_cq_resp,
3786  					     response_length),
3787  			UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3788  		DECLARE_UVERBS_WRITE_EX(
3789  			IB_USER_VERBS_EX_CMD_MODIFY_CQ,
3790  			ib_uverbs_ex_modify_cq,
3791  			UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3792  			UAPI_DEF_METHOD_NEEDS_FN(modify_cq))),
3793  
3794  	DECLARE_UVERBS_OBJECT(
3795  		UVERBS_OBJECT_DEVICE,
3796  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
3797  				     ib_uverbs_get_context,
3798  				     UAPI_DEF_WRITE_UDATA_IO(
3799  					     struct ib_uverbs_get_context,
3800  					     struct ib_uverbs_get_context_resp)),
3801  		DECLARE_UVERBS_WRITE(
3802  			IB_USER_VERBS_CMD_QUERY_DEVICE,
3803  			ib_uverbs_query_device,
3804  			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
3805  					  struct ib_uverbs_query_device_resp)),
3806  		DECLARE_UVERBS_WRITE(
3807  			IB_USER_VERBS_CMD_QUERY_PORT,
3808  			ib_uverbs_query_port,
3809  			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
3810  					  struct ib_uverbs_query_port_resp),
3811  			UAPI_DEF_METHOD_NEEDS_FN(query_port)),
3812  		DECLARE_UVERBS_WRITE_EX(
3813  			IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
3814  			ib_uverbs_ex_query_device,
3815  			UAPI_DEF_WRITE_IO_EX(
3816  				struct ib_uverbs_ex_query_device,
3817  				reserved,
3818  				struct ib_uverbs_ex_query_device_resp,
3819  				response_length),
3820  			UAPI_DEF_METHOD_NEEDS_FN(query_device)),
3821  		UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
3822  		UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
3823  
3824  	DECLARE_UVERBS_OBJECT(
3825  		UVERBS_OBJECT_FLOW,
3826  		DECLARE_UVERBS_WRITE_EX(
3827  			IB_USER_VERBS_EX_CMD_CREATE_FLOW,
3828  			ib_uverbs_ex_create_flow,
3829  			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
3830  					     flow_attr,
3831  					     struct ib_uverbs_create_flow_resp,
3832  					     flow_handle),
3833  			UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
3834  		DECLARE_UVERBS_WRITE_EX(
3835  			IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
3836  			ib_uverbs_ex_destroy_flow,
3837  			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
3838  			UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
3839  
3840  	DECLARE_UVERBS_OBJECT(
3841  		UVERBS_OBJECT_MR,
3842  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
3843  				     ib_uverbs_dereg_mr,
3844  				     UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
3845  				     UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
3846  		DECLARE_UVERBS_WRITE(
3847  			IB_USER_VERBS_CMD_REG_MR,
3848  			ib_uverbs_reg_mr,
3849  			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
3850  						struct ib_uverbs_reg_mr_resp),
3851  			UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
3852  		DECLARE_UVERBS_WRITE(
3853  			IB_USER_VERBS_CMD_REREG_MR,
3854  			ib_uverbs_rereg_mr,
3855  			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
3856  						struct ib_uverbs_rereg_mr_resp),
3857  			UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
3858  
3859  	DECLARE_UVERBS_OBJECT(
3860  		UVERBS_OBJECT_MW,
3861  		DECLARE_UVERBS_WRITE(
3862  			IB_USER_VERBS_CMD_ALLOC_MW,
3863  			ib_uverbs_alloc_mw,
3864  			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
3865  						struct ib_uverbs_alloc_mw_resp),
3866  			UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
3867  		DECLARE_UVERBS_WRITE(
3868  			IB_USER_VERBS_CMD_DEALLOC_MW,
3869  			ib_uverbs_dealloc_mw,
3870  			UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
3871  			UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
3872  
3873  	DECLARE_UVERBS_OBJECT(
3874  		UVERBS_OBJECT_PD,
3875  		DECLARE_UVERBS_WRITE(
3876  			IB_USER_VERBS_CMD_ALLOC_PD,
3877  			ib_uverbs_alloc_pd,
3878  			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
3879  						struct ib_uverbs_alloc_pd_resp),
3880  			UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
3881  		DECLARE_UVERBS_WRITE(
3882  			IB_USER_VERBS_CMD_DEALLOC_PD,
3883  			ib_uverbs_dealloc_pd,
3884  			UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
3885  			UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
3886  
3887  	DECLARE_UVERBS_OBJECT(
3888  		UVERBS_OBJECT_QP,
3889  		DECLARE_UVERBS_WRITE(
3890  			IB_USER_VERBS_CMD_ATTACH_MCAST,
3891  			ib_uverbs_attach_mcast,
3892  			UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
3893  			UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
3894  			UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3895  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
3896  				     ib_uverbs_create_qp,
3897  				     UAPI_DEF_WRITE_UDATA_IO(
3898  					     struct ib_uverbs_create_qp,
3899  					     struct ib_uverbs_create_qp_resp),
3900  				     UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3901  		DECLARE_UVERBS_WRITE(
3902  			IB_USER_VERBS_CMD_DESTROY_QP,
3903  			ib_uverbs_destroy_qp,
3904  			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
3905  					  struct ib_uverbs_destroy_qp_resp),
3906  			UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
3907  		DECLARE_UVERBS_WRITE(
3908  			IB_USER_VERBS_CMD_DETACH_MCAST,
3909  			ib_uverbs_detach_mcast,
3910  			UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
3911  			UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3912  		DECLARE_UVERBS_WRITE(
3913  			IB_USER_VERBS_CMD_MODIFY_QP,
3914  			ib_uverbs_modify_qp,
3915  			UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
3916  			UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
3917  		DECLARE_UVERBS_WRITE(
3918  			IB_USER_VERBS_CMD_POST_RECV,
3919  			ib_uverbs_post_recv,
3920  			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
3921  					  struct ib_uverbs_post_recv_resp),
3922  			UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
3923  		DECLARE_UVERBS_WRITE(
3924  			IB_USER_VERBS_CMD_POST_SEND,
3925  			ib_uverbs_post_send,
3926  			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
3927  					  struct ib_uverbs_post_send_resp),
3928  			UAPI_DEF_METHOD_NEEDS_FN(post_send)),
3929  		DECLARE_UVERBS_WRITE(
3930  			IB_USER_VERBS_CMD_QUERY_QP,
3931  			ib_uverbs_query_qp,
3932  			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
3933  					  struct ib_uverbs_query_qp_resp),
3934  			UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
3935  		DECLARE_UVERBS_WRITE_EX(
3936  			IB_USER_VERBS_EX_CMD_CREATE_QP,
3937  			ib_uverbs_ex_create_qp,
3938  			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
3939  					     comp_mask,
3940  					     struct ib_uverbs_ex_create_qp_resp,
3941  					     response_length),
3942  			UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3943  		DECLARE_UVERBS_WRITE_EX(
3944  			IB_USER_VERBS_EX_CMD_MODIFY_QP,
3945  			ib_uverbs_ex_modify_qp,
3946  			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
3947  					     base,
3948  					     struct ib_uverbs_ex_modify_qp_resp,
3949  					     response_length),
3950  			UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
3951  
3952  	DECLARE_UVERBS_OBJECT(
3953  		UVERBS_OBJECT_RWQ_IND_TBL,
3954  		DECLARE_UVERBS_WRITE_EX(
3955  			IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
3956  			ib_uverbs_ex_create_rwq_ind_table,
3957  			UAPI_DEF_WRITE_IO_EX(
3958  				struct ib_uverbs_ex_create_rwq_ind_table,
3959  				log_ind_tbl_size,
3960  				struct ib_uverbs_ex_create_rwq_ind_table_resp,
3961  				ind_tbl_num),
3962  			UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
3963  		DECLARE_UVERBS_WRITE_EX(
3964  			IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
3965  			ib_uverbs_ex_destroy_rwq_ind_table,
3966  			UAPI_DEF_WRITE_I(
3967  				struct ib_uverbs_ex_destroy_rwq_ind_table),
3968  			UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
3969  
3970  	DECLARE_UVERBS_OBJECT(
3971  		UVERBS_OBJECT_WQ,
3972  		DECLARE_UVERBS_WRITE_EX(
3973  			IB_USER_VERBS_EX_CMD_CREATE_WQ,
3974  			ib_uverbs_ex_create_wq,
3975  			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
3976  					     max_sge,
3977  					     struct ib_uverbs_ex_create_wq_resp,
3978  					     wqn),
3979  			UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
3980  		DECLARE_UVERBS_WRITE_EX(
3981  			IB_USER_VERBS_EX_CMD_DESTROY_WQ,
3982  			ib_uverbs_ex_destroy_wq,
3983  			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
3984  					     wq_handle,
3985  					     struct ib_uverbs_ex_destroy_wq_resp,
3986  					     reserved),
3987  			UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
3988  		DECLARE_UVERBS_WRITE_EX(
3989  			IB_USER_VERBS_EX_CMD_MODIFY_WQ,
3990  			ib_uverbs_ex_modify_wq,
3991  			UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
3992  					    curr_wq_state),
3993  			UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
3994  
3995  	DECLARE_UVERBS_OBJECT(
3996  		UVERBS_OBJECT_SRQ,
3997  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
3998  				     ib_uverbs_create_srq,
3999  				     UAPI_DEF_WRITE_UDATA_IO(
4000  					     struct ib_uverbs_create_srq,
4001  					     struct ib_uverbs_create_srq_resp),
4002  				     UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4003  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
4004  				     ib_uverbs_create_xsrq,
4005  				     UAPI_DEF_WRITE_UDATA_IO(
4006  					     struct ib_uverbs_create_xsrq,
4007  					     struct ib_uverbs_create_srq_resp),
4008  				     UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4009  		DECLARE_UVERBS_WRITE(
4010  			IB_USER_VERBS_CMD_DESTROY_SRQ,
4011  			ib_uverbs_destroy_srq,
4012  			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
4013  					  struct ib_uverbs_destroy_srq_resp),
4014  			UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
4015  		DECLARE_UVERBS_WRITE(
4016  			IB_USER_VERBS_CMD_MODIFY_SRQ,
4017  			ib_uverbs_modify_srq,
4018  			UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
4019  			UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
4020  		DECLARE_UVERBS_WRITE(
4021  			IB_USER_VERBS_CMD_POST_SRQ_RECV,
4022  			ib_uverbs_post_srq_recv,
4023  			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
4024  					  struct ib_uverbs_post_srq_recv_resp),
4025  			UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
4026  		DECLARE_UVERBS_WRITE(
4027  			IB_USER_VERBS_CMD_QUERY_SRQ,
4028  			ib_uverbs_query_srq,
4029  			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
4030  					  struct ib_uverbs_query_srq_resp),
4031  			UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
4032  
4033  	DECLARE_UVERBS_OBJECT(
4034  		UVERBS_OBJECT_XRCD,
4035  		DECLARE_UVERBS_WRITE(
4036  			IB_USER_VERBS_CMD_CLOSE_XRCD,
4037  			ib_uverbs_close_xrcd,
4038  			UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd)),
4039  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
4040  				     ib_uverbs_open_qp,
4041  				     UAPI_DEF_WRITE_UDATA_IO(
4042  					     struct ib_uverbs_open_qp,
4043  					     struct ib_uverbs_create_qp_resp)),
4044  		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
4045  				     ib_uverbs_open_xrcd,
4046  				     UAPI_DEF_WRITE_UDATA_IO(
4047  					     struct ib_uverbs_open_xrcd,
4048  					     struct ib_uverbs_open_xrcd_resp)),
4049  		UAPI_DEF_OBJ_NEEDS_FN(alloc_xrcd),
4050  		UAPI_DEF_OBJ_NEEDS_FN(dealloc_xrcd)),
4051  
4052  	{},
4053  };
4054