1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (c) 2022 Microsoft Corporation. All rights reserved.
4 */
5
6 #ifndef _MANA_IB_H_
7 #define _MANA_IB_H_
8
9 #include <rdma/ib_verbs.h>
10 #include <rdma/ib_mad.h>
11 #include <rdma/ib_umem.h>
12 #include <rdma/mana-abi.h>
13 #include <rdma/uverbs_ioctl.h>
14
15 #include <net/mana/mana.h>
16
17 #define PAGE_SZ_BM \
18 (SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K | \
19 SZ_512K | SZ_1M | SZ_2M)
20
21 /* MANA doesn't have any limit for MR size */
22 #define MANA_IB_MAX_MR_SIZE U64_MAX
23
24 /*
25 * The hardware limit of number of MRs is greater than maximum number of MRs
26 * that can possibly represent in 24 bits
27 */
28 #define MANA_IB_MAX_MR 0xFFFFFFu
29
30 /*
31 * The CA timeout is approx. 260ms (4us * 2^(DELAY))
32 */
33 #define MANA_CA_ACK_DELAY 16
34
35 struct mana_ib_adapter_caps {
36 u32 max_sq_id;
37 u32 max_rq_id;
38 u32 max_cq_id;
39 u32 max_qp_count;
40 u32 max_cq_count;
41 u32 max_mr_count;
42 u32 max_pd_count;
43 u32 max_inbound_read_limit;
44 u32 max_outbound_read_limit;
45 u32 mw_count;
46 u32 max_srq_count;
47 u32 max_qp_wr;
48 u32 max_send_sge_count;
49 u32 max_recv_sge_count;
50 u32 max_inline_data_size;
51 };
52
53 struct mana_ib_queue {
54 struct ib_umem *umem;
55 u64 gdma_region;
56 u64 id;
57 };
58
59 struct mana_ib_dev {
60 struct ib_device ib_dev;
61 struct gdma_dev *gdma_dev;
62 mana_handle_t adapter_handle;
63 struct gdma_queue *fatal_err_eq;
64 struct gdma_queue **eqs;
65 struct xarray qp_table_wq;
66 struct mana_ib_adapter_caps adapter_caps;
67 };
68
69 struct mana_ib_wq {
70 struct ib_wq ibwq;
71 struct mana_ib_queue queue;
72 int wqe;
73 u32 wq_buf_size;
74 mana_handle_t rx_object;
75 };
76
77 struct mana_ib_pd {
78 struct ib_pd ibpd;
79 u32 pdn;
80 mana_handle_t pd_handle;
81
82 /* Mutex for sharing access to vport_use_count */
83 struct mutex vport_mutex;
84 int vport_use_count;
85
86 bool tx_shortform_allowed;
87 u32 tx_vp_offset;
88 };
89
90 struct mana_ib_mr {
91 struct ib_mr ibmr;
92 struct ib_umem *umem;
93 mana_handle_t mr_handle;
94 };
95
96 struct mana_ib_cq {
97 struct ib_cq ibcq;
98 struct mana_ib_queue queue;
99 int cqe;
100 u32 comp_vector;
101 mana_handle_t cq_handle;
102 };
103
104 enum mana_rc_queue_type {
105 MANA_RC_SEND_QUEUE_REQUESTER = 0,
106 MANA_RC_SEND_QUEUE_RESPONDER,
107 MANA_RC_SEND_QUEUE_FMR,
108 MANA_RC_RECV_QUEUE_REQUESTER,
109 MANA_RC_RECV_QUEUE_RESPONDER,
110 MANA_RC_QUEUE_TYPE_MAX,
111 };
112
113 struct mana_ib_rc_qp {
114 struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
115 };
116
117 struct mana_ib_qp {
118 struct ib_qp ibqp;
119
120 mana_handle_t qp_handle;
121 union {
122 struct mana_ib_queue raw_sq;
123 struct mana_ib_rc_qp rc_qp;
124 };
125
126 /* The port on the IB device, starting with 1 */
127 u32 port;
128
129 refcount_t refcount;
130 struct completion free;
131 };
132
133 struct mana_ib_ucontext {
134 struct ib_ucontext ibucontext;
135 u32 doorbell;
136 };
137
138 struct mana_ib_rwq_ind_table {
139 struct ib_rwq_ind_table ib_ind_table;
140 };
141
142 enum mana_ib_command_code {
143 MANA_IB_GET_ADAPTER_CAP = 0x30001,
144 MANA_IB_CREATE_ADAPTER = 0x30002,
145 MANA_IB_DESTROY_ADAPTER = 0x30003,
146 MANA_IB_CONFIG_IP_ADDR = 0x30004,
147 MANA_IB_CONFIG_MAC_ADDR = 0x30005,
148 MANA_IB_CREATE_CQ = 0x30008,
149 MANA_IB_DESTROY_CQ = 0x30009,
150 MANA_IB_CREATE_RC_QP = 0x3000a,
151 MANA_IB_DESTROY_RC_QP = 0x3000b,
152 MANA_IB_SET_QP_STATE = 0x3000d,
153 };
154
155 struct mana_ib_query_adapter_caps_req {
156 struct gdma_req_hdr hdr;
157 }; /*HW Data */
158
159 struct mana_ib_query_adapter_caps_resp {
160 struct gdma_resp_hdr hdr;
161 u32 max_sq_id;
162 u32 max_rq_id;
163 u32 max_cq_id;
164 u32 max_qp_count;
165 u32 max_cq_count;
166 u32 max_mr_count;
167 u32 max_pd_count;
168 u32 max_inbound_read_limit;
169 u32 max_outbound_read_limit;
170 u32 mw_count;
171 u32 max_srq_count;
172 u32 max_requester_sq_size;
173 u32 max_responder_sq_size;
174 u32 max_requester_rq_size;
175 u32 max_responder_rq_size;
176 u32 max_send_sge_count;
177 u32 max_recv_sge_count;
178 u32 max_inline_data_size;
179 }; /* HW Data */
180
181 struct mana_rnic_create_adapter_req {
182 struct gdma_req_hdr hdr;
183 u32 notify_eq_id;
184 u32 reserved;
185 u64 feature_flags;
186 }; /*HW Data */
187
188 struct mana_rnic_create_adapter_resp {
189 struct gdma_resp_hdr hdr;
190 mana_handle_t adapter;
191 }; /* HW Data */
192
193 struct mana_rnic_destroy_adapter_req {
194 struct gdma_req_hdr hdr;
195 mana_handle_t adapter;
196 }; /*HW Data */
197
198 struct mana_rnic_destroy_adapter_resp {
199 struct gdma_resp_hdr hdr;
200 }; /* HW Data */
201
202 enum mana_ib_addr_op {
203 ADDR_OP_ADD = 1,
204 ADDR_OP_REMOVE = 2,
205 };
206
207 enum sgid_entry_type {
208 SGID_TYPE_IPV4 = 1,
209 SGID_TYPE_IPV6 = 2,
210 };
211
212 struct mana_rnic_config_addr_req {
213 struct gdma_req_hdr hdr;
214 mana_handle_t adapter;
215 enum mana_ib_addr_op op;
216 enum sgid_entry_type sgid_type;
217 u8 ip_addr[16];
218 }; /* HW Data */
219
220 struct mana_rnic_config_addr_resp {
221 struct gdma_resp_hdr hdr;
222 }; /* HW Data */
223
224 struct mana_rnic_config_mac_addr_req {
225 struct gdma_req_hdr hdr;
226 mana_handle_t adapter;
227 enum mana_ib_addr_op op;
228 u8 mac_addr[ETH_ALEN];
229 u8 reserved[6];
230 }; /* HW Data */
231
232 struct mana_rnic_config_mac_addr_resp {
233 struct gdma_resp_hdr hdr;
234 }; /* HW Data */
235
236 struct mana_rnic_create_cq_req {
237 struct gdma_req_hdr hdr;
238 mana_handle_t adapter;
239 u64 gdma_region;
240 u32 eq_id;
241 u32 doorbell_page;
242 }; /* HW Data */
243
244 struct mana_rnic_create_cq_resp {
245 struct gdma_resp_hdr hdr;
246 mana_handle_t cq_handle;
247 u32 cq_id;
248 u32 reserved;
249 }; /* HW Data */
250
251 struct mana_rnic_destroy_cq_req {
252 struct gdma_req_hdr hdr;
253 mana_handle_t adapter;
254 mana_handle_t cq_handle;
255 }; /* HW Data */
256
257 struct mana_rnic_destroy_cq_resp {
258 struct gdma_resp_hdr hdr;
259 }; /* HW Data */
260
261 enum mana_rnic_create_rc_flags {
262 MANA_RC_FLAG_NO_FMR = 2,
263 };
264
265 struct mana_rnic_create_qp_req {
266 struct gdma_req_hdr hdr;
267 mana_handle_t adapter;
268 mana_handle_t pd_handle;
269 mana_handle_t send_cq_handle;
270 mana_handle_t recv_cq_handle;
271 u64 dma_region[MANA_RC_QUEUE_TYPE_MAX];
272 u64 deprecated[2];
273 u64 flags;
274 u32 doorbell_page;
275 u32 max_send_wr;
276 u32 max_recv_wr;
277 u32 max_send_sge;
278 u32 max_recv_sge;
279 u32 reserved;
280 }; /* HW Data */
281
282 struct mana_rnic_create_qp_resp {
283 struct gdma_resp_hdr hdr;
284 mana_handle_t rc_qp_handle;
285 u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX];
286 u32 reserved;
287 }; /* HW Data*/
288
289 struct mana_rnic_destroy_rc_qp_req {
290 struct gdma_req_hdr hdr;
291 mana_handle_t adapter;
292 mana_handle_t rc_qp_handle;
293 }; /* HW Data */
294
295 struct mana_rnic_destroy_rc_qp_resp {
296 struct gdma_resp_hdr hdr;
297 }; /* HW Data */
298
299 struct mana_ib_ah_attr {
300 u8 src_addr[16];
301 u8 dest_addr[16];
302 u8 src_mac[ETH_ALEN];
303 u8 dest_mac[ETH_ALEN];
304 u8 src_addr_type;
305 u8 dest_addr_type;
306 u8 hop_limit;
307 u8 traffic_class;
308 u16 src_port;
309 u16 dest_port;
310 u32 reserved;
311 };
312
313 struct mana_rnic_set_qp_state_req {
314 struct gdma_req_hdr hdr;
315 mana_handle_t adapter;
316 mana_handle_t qp_handle;
317 u64 attr_mask;
318 u32 qp_state;
319 u32 path_mtu;
320 u32 rq_psn;
321 u32 sq_psn;
322 u32 dest_qpn;
323 u32 max_dest_rd_atomic;
324 u32 retry_cnt;
325 u32 rnr_retry;
326 u32 min_rnr_timer;
327 u32 reserved;
328 struct mana_ib_ah_attr ah_attr;
329 }; /* HW Data */
330
331 struct mana_rnic_set_qp_state_resp {
332 struct gdma_resp_hdr hdr;
333 }; /* HW Data */
334
mdev_to_gc(struct mana_ib_dev * mdev)335 static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
336 {
337 return mdev->gdma_dev->gdma_context;
338 }
339
mana_get_qp_ref(struct mana_ib_dev * mdev,uint32_t qid)340 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,
341 uint32_t qid)
342 {
343 struct mana_ib_qp *qp;
344 unsigned long flag;
345
346 xa_lock_irqsave(&mdev->qp_table_wq, flag);
347 qp = xa_load(&mdev->qp_table_wq, qid);
348 if (qp)
349 refcount_inc(&qp->refcount);
350 xa_unlock_irqrestore(&mdev->qp_table_wq, flag);
351 return qp;
352 }
353
mana_put_qp_ref(struct mana_ib_qp * qp)354 static inline void mana_put_qp_ref(struct mana_ib_qp *qp)
355 {
356 if (refcount_dec_and_test(&qp->refcount))
357 complete(&qp->free);
358 }
359
mana_ib_get_netdev(struct ib_device * ibdev,u32 port)360 static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port)
361 {
362 struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
363 struct gdma_context *gc = mdev_to_gc(mdev);
364 struct mana_context *mc = gc->mana.driver_data;
365
366 if (port < 1 || port > mc->num_ports)
367 return NULL;
368 return mc->ports[port - 1];
369 }
370
copy_in_reverse(u8 * dst,const u8 * src,u32 size)371 static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size)
372 {
373 u32 i;
374
375 for (i = 0; i < size; i++)
376 dst[size - 1 - i] = src[i];
377 }
378
379 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
380 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
381
382 int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
383 mana_handle_t *gdma_region);
384
385 int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
386 mana_handle_t *gdma_region, u64 virt);
387
388 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
389 mana_handle_t gdma_region);
390
391 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
392 struct mana_ib_queue *queue);
393 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
394
395 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
396 struct ib_wq_init_attr *init_attr,
397 struct ib_udata *udata);
398
399 int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
400 u32 wq_attr_mask, struct ib_udata *udata);
401
402 int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata);
403
404 int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
405 struct ib_rwq_ind_table_init_attr *init_attr,
406 struct ib_udata *udata);
407
408 int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl);
409
410 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);
411
412 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
413 u64 iova, int access_flags,
414 struct ib_udata *udata);
415
416 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
417
418 int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
419 struct ib_udata *udata);
420
421 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
422 int attr_mask, struct ib_udata *udata);
423
424 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
425
426 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id,
427 struct mana_ib_pd *pd, u32 doorbell_id);
428 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
429 u32 port);
430
431 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
432 struct uverbs_attr_bundle *attrs);
433
434 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
435
436 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
437 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
438
439 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
440 struct ib_udata *udata);
441 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
442
443 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma);
444
445 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
446 struct ib_port_immutable *immutable);
447 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
448 struct ib_udata *uhw);
449 int mana_ib_query_port(struct ib_device *ibdev, u32 port,
450 struct ib_port_attr *props);
451 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
452 union ib_gid *gid);
453
454 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
455
456 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);
457
458 int mana_ib_create_eqs(struct mana_ib_dev *mdev);
459
460 void mana_ib_destroy_eqs(struct mana_ib_dev *mdev);
461
462 int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev);
463
464 int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev);
465
466 int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
467
468 enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num);
469
470 int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context);
471
472 int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context);
473
474 int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac);
475
476 int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell);
477
478 int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
479
480 int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
481 struct ib_qp_init_attr *attr, u32 doorbell, u64 flags);
482 int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);
483 #endif
484