Lines Matching +full:in +full:-
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved.
12 static int get_pas_size(struct mlx5_srq_attr *in) in get_pas_size() argument
14 u32 log_page_size = in->log_page_size + 12; in get_pas_size()
15 u32 log_srq_size = in->log_size; in get_pas_size()
16 u32 log_rq_stride = in->wqe_shift; in get_pas_size()
17 u32 page_offset = in->page_offset; in get_pas_size()
18 u32 po_quanta = 1 << (log_page_size - 6); in get_pas_size()
27 static void set_wq(void *wq, struct mlx5_srq_attr *in) in set_wq() argument
29 MLX5_SET(wq, wq, wq_signature, !!(in->flags in set_wq()
31 MLX5_SET(wq, wq, log_wq_pg_sz, in->log_page_size); in set_wq()
32 MLX5_SET(wq, wq, log_wq_stride, in->wqe_shift + 4); in set_wq()
33 MLX5_SET(wq, wq, log_wq_sz, in->log_size); in set_wq()
34 MLX5_SET(wq, wq, page_offset, in->page_offset); in set_wq()
35 MLX5_SET(wq, wq, lwm, in->lwm); in set_wq()
36 MLX5_SET(wq, wq, pd, in->pd); in set_wq()
37 MLX5_SET64(wq, wq, dbr_addr, in->db_record); in set_wq()
40 static void set_srqc(void *srqc, struct mlx5_srq_attr *in) in set_srqc() argument
42 MLX5_SET(srqc, srqc, wq_signature, !!(in->flags in set_srqc()
44 MLX5_SET(srqc, srqc, log_page_size, in->log_page_size); in set_srqc()
45 MLX5_SET(srqc, srqc, log_rq_stride, in->wqe_shift); in set_srqc()
46 MLX5_SET(srqc, srqc, log_srq_size, in->log_size); in set_srqc()
47 MLX5_SET(srqc, srqc, page_offset, in->page_offset); in set_srqc()
48 MLX5_SET(srqc, srqc, lwm, in->lwm); in set_srqc()
49 MLX5_SET(srqc, srqc, pd, in->pd); in set_srqc()
50 MLX5_SET64(srqc, srqc, dbr_addr, in->db_record); in set_srqc()
51 MLX5_SET(srqc, srqc, xrcd, in->xrcd); in set_srqc()
52 MLX5_SET(srqc, srqc, cqn, in->cqn); in set_srqc()
55 static void get_wq(void *wq, struct mlx5_srq_attr *in) in get_wq() argument
58 in->flags &= MLX5_SRQ_FLAG_WQ_SIG; in get_wq()
59 in->log_page_size = MLX5_GET(wq, wq, log_wq_pg_sz); in get_wq()
60 in->wqe_shift = MLX5_GET(wq, wq, log_wq_stride) - 4; in get_wq()
61 in->log_size = MLX5_GET(wq, wq, log_wq_sz); in get_wq()
62 in->page_offset = MLX5_GET(wq, wq, page_offset); in get_wq()
63 in->lwm = MLX5_GET(wq, wq, lwm); in get_wq()
64 in->pd = MLX5_GET(wq, wq, pd); in get_wq()
65 in->db_record = MLX5_GET64(wq, wq, dbr_addr); in get_wq()
68 static void get_srqc(void *srqc, struct mlx5_srq_attr *in) in get_srqc() argument
71 in->flags &= MLX5_SRQ_FLAG_WQ_SIG; in get_srqc()
72 in->log_page_size = MLX5_GET(srqc, srqc, log_page_size); in get_srqc()
73 in->wqe_shift = MLX5_GET(srqc, srqc, log_rq_stride); in get_srqc()
74 in->log_size = MLX5_GET(srqc, srqc, log_srq_size); in get_srqc()
75 in->page_offset = MLX5_GET(srqc, srqc, page_offset); in get_srqc()
76 in->lwm = MLX5_GET(srqc, srqc, lwm); in get_srqc()
77 in->pd = MLX5_GET(srqc, srqc, pd); in get_srqc()
78 in->db_record = MLX5_GET64(srqc, srqc, dbr_addr); in get_srqc()
83 struct mlx5_srq_table *table = &dev->srq_table; in mlx5_cmd_get_srq()
86 xa_lock_irq(&table->array); in mlx5_cmd_get_srq()
87 srq = xa_load(&table->array, srqn); in mlx5_cmd_get_srq()
89 refcount_inc(&srq->common.refcount); in mlx5_cmd_get_srq()
90 xa_unlock_irq(&table->array); in mlx5_cmd_get_srq()
95 static int __set_srq_page_size(struct mlx5_srq_attr *in, in __set_srq_page_size() argument
99 return -EINVAL; in __set_srq_page_size()
100 in->log_page_size = order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT; in __set_srq_page_size()
102 if (WARN_ON(get_pas_size(in) != in __set_srq_page_size()
103 ib_umem_num_dma_blocks(in->umem, page_size) * sizeof(u64))) in __set_srq_page_size()
104 return -EINVAL; in __set_srq_page_size()
108 #define set_srq_page_size(in, typ, log_pgsz_fld) \ argument
109 __set_srq_page_size(in, mlx5_umem_find_best_quantized_pgoff( \
110 (in)->umem, typ, log_pgsz_fld, \
112 64, &(in)->page_offset))
115 struct mlx5_srq_attr *in) in create_srq_cmd() argument
125 if (in->umem) { in create_srq_cmd()
126 err = set_srq_page_size(in, srqc, log_page_size); in create_srq_cmd()
131 pas_size = get_pas_size(in); in create_srq_cmd()
135 return -ENOMEM; in create_srq_cmd()
137 MLX5_SET(create_srq_in, create_in, uid, in->uid); in create_srq_cmd()
141 set_srqc(srqc, in); in create_srq_cmd()
142 if (in->umem) in create_srq_cmd()
144 in->umem, in create_srq_cmd()
145 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT), in create_srq_cmd()
148 memcpy(pas, in->pas, pas_size); in create_srq_cmd()
153 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, in create_srq_cmd()
157 srq->srqn = MLX5_GET(create_srq_out, create_out, srqn); in create_srq_cmd()
158 srq->uid = in->uid; in create_srq_cmd()
166 u32 in[MLX5_ST_SZ_DW(destroy_srq_in)] = {}; in destroy_srq_cmd() local
168 MLX5_SET(destroy_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_SRQ); in destroy_srq_cmd()
169 MLX5_SET(destroy_srq_in, in, srqn, srq->srqn); in destroy_srq_cmd()
170 MLX5_SET(destroy_srq_in, in, uid, srq->uid); in destroy_srq_cmd()
172 return mlx5_cmd_exec_in(dev->mdev, destroy_srq, in); in destroy_srq_cmd()
178 u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {}; in arm_srq_cmd() local
180 MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ); in arm_srq_cmd()
181 MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ); in arm_srq_cmd()
182 MLX5_SET(arm_rq_in, in, srq_number, srq->srqn); in arm_srq_cmd()
183 MLX5_SET(arm_rq_in, in, lwm, lwm); in arm_srq_cmd()
184 MLX5_SET(arm_rq_in, in, uid, srq->uid); in arm_srq_cmd()
186 return mlx5_cmd_exec_in(dev->mdev, arm_rq, in); in arm_srq_cmd()
192 u32 in[MLX5_ST_SZ_DW(query_srq_in)] = {}; in query_srq_cmd() local
199 return -ENOMEM; in query_srq_cmd()
201 MLX5_SET(query_srq_in, in, opcode, MLX5_CMD_OP_QUERY_SRQ); in query_srq_cmd()
202 MLX5_SET(query_srq_in, in, srqn, srq->srqn); in query_srq_cmd()
203 err = mlx5_cmd_exec_inout(dev->mdev, query_srq, in, srq_out); in query_srq_cmd()
210 out->flags |= MLX5_SRQ_FLAG_ERR; in query_srq_cmd()
218 struct mlx5_srq_attr *in) in create_xrc_srq_cmd() argument
228 if (in->umem) { in create_xrc_srq_cmd()
229 err = set_srq_page_size(in, xrc_srqc, log_page_size); in create_xrc_srq_cmd()
234 pas_size = get_pas_size(in); in create_xrc_srq_cmd()
238 return -ENOMEM; in create_xrc_srq_cmd()
240 MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid); in create_xrc_srq_cmd()
245 set_srqc(xrc_srqc, in); in create_xrc_srq_cmd()
246 MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index); in create_xrc_srq_cmd()
247 if (in->umem) in create_xrc_srq_cmd()
249 in->umem, in create_xrc_srq_cmd()
250 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT), in create_xrc_srq_cmd()
253 memcpy(pas, in->pas, pas_size); in create_xrc_srq_cmd()
258 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, in create_xrc_srq_cmd()
263 srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn); in create_xrc_srq_cmd()
264 srq->uid = in->uid; in create_xrc_srq_cmd()
273 u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {}; in destroy_xrc_srq_cmd() local
275 MLX5_SET(destroy_xrc_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRC_SRQ); in destroy_xrc_srq_cmd()
276 MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn); in destroy_xrc_srq_cmd()
277 MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid); in destroy_xrc_srq_cmd()
279 return mlx5_cmd_exec_in(dev->mdev, destroy_xrc_srq, in); in destroy_xrc_srq_cmd()
285 u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {}; in arm_xrc_srq_cmd() local
287 MLX5_SET(arm_xrc_srq_in, in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ); in arm_xrc_srq_cmd()
288 MLX5_SET(arm_xrc_srq_in, in, op_mod, in arm_xrc_srq_cmd()
290 MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn); in arm_xrc_srq_cmd()
291 MLX5_SET(arm_xrc_srq_in, in, lwm, lwm); in arm_xrc_srq_cmd()
292 MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid); in arm_xrc_srq_cmd()
294 return mlx5_cmd_exec_in(dev->mdev, arm_xrc_srq, in); in arm_xrc_srq_cmd()
301 u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {}; in query_xrc_srq_cmd() local
308 return -ENOMEM; in query_xrc_srq_cmd()
310 MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ); in query_xrc_srq_cmd()
311 MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn); in query_xrc_srq_cmd()
313 err = mlx5_cmd_exec_inout(dev->mdev, query_xrc_srq, in, xrcsrq_out); in query_xrc_srq_cmd()
321 out->flags |= MLX5_SRQ_FLAG_ERR; in query_xrc_srq_cmd()
329 struct mlx5_srq_attr *in) in create_rmp_cmd() argument
341 if (in->umem) { in create_rmp_cmd()
342 err = set_srq_page_size(in, wq, log_wq_pg_sz); in create_rmp_cmd()
347 pas_size = get_pas_size(in); in create_rmp_cmd()
353 err = -ENOMEM; in create_rmp_cmd()
361 MLX5_SET(create_rmp_in, create_in, uid, in->uid); in create_rmp_cmd()
364 set_wq(wq, in); in create_rmp_cmd()
365 if (in->umem) in create_rmp_cmd()
367 in->umem, in create_rmp_cmd()
368 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT), in create_rmp_cmd()
371 memcpy(pas, in->pas, pas_size); in create_rmp_cmd()
374 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen); in create_rmp_cmd()
376 srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn); in create_rmp_cmd()
377 srq->uid = in->uid; in create_rmp_cmd()
388 u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {}; in destroy_rmp_cmd() local
390 MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP); in destroy_rmp_cmd()
391 MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn); in destroy_rmp_cmd()
392 MLX5_SET(destroy_rmp_in, in, uid, srq->uid); in destroy_rmp_cmd()
393 return mlx5_cmd_exec_in(dev->mdev, destroy_rmp, in); in destroy_rmp_cmd()
400 void *in = NULL; in arm_rmp_cmd() local
411 in = kvzalloc(inlen, GFP_KERNEL); in arm_rmp_cmd()
413 if (!in || !out) { in arm_rmp_cmd()
414 err = -ENOMEM; in arm_rmp_cmd()
418 rmpc = MLX5_ADDR_OF(modify_rmp_in, in, ctx); in arm_rmp_cmd()
419 bitmask = MLX5_ADDR_OF(modify_rmp_in, in, bitmask); in arm_rmp_cmd()
422 MLX5_SET(modify_rmp_in, in, rmp_state, MLX5_RMPC_STATE_RDY); in arm_rmp_cmd()
423 MLX5_SET(modify_rmp_in, in, rmpn, srq->srqn); in arm_rmp_cmd()
424 MLX5_SET(modify_rmp_in, in, uid, srq->uid); in arm_rmp_cmd()
428 MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP); in arm_rmp_cmd()
430 err = mlx5_cmd_exec_inout(dev->mdev, modify_rmp, in, out); in arm_rmp_cmd()
433 kvfree(in); in arm_rmp_cmd()
454 err = -ENOMEM; in query_rmp_cmd()
459 MLX5_SET(query_rmp_in, rmp_in, rmpn, srq->srqn); in query_rmp_cmd()
460 err = mlx5_cmd_exec_inout(dev->mdev, query_rmp, rmp_in, rmp_out); in query_rmp_cmd()
467 out->flags |= MLX5_SRQ_FLAG_ERR; in query_rmp_cmd()
476 struct mlx5_srq_attr *in) in create_xrq_cmd() argument
487 if (in->umem) { in create_xrq_cmd()
488 err = set_srq_page_size(in, wq, log_wq_pg_sz); in create_xrq_cmd()
493 pas_size = get_pas_size(in); in create_xrq_cmd()
497 return -ENOMEM; in create_xrq_cmd()
503 set_wq(wq, in); in create_xrq_cmd()
504 if (in->umem) in create_xrq_cmd()
506 in->umem, in create_xrq_cmd()
507 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT), in create_xrq_cmd()
510 memcpy(pas, in->pas, pas_size); in create_xrq_cmd()
512 if (in->type == IB_SRQT_TM) { in create_xrq_cmd()
514 if (in->flags & MLX5_SRQ_FLAG_RNDV) in create_xrq_cmd()
518 in->tm_log_list_size); in create_xrq_cmd()
520 MLX5_SET(xrqc, xrqc, user_index, in->user_index); in create_xrq_cmd()
521 MLX5_SET(xrqc, xrqc, cqn, in->cqn); in create_xrq_cmd()
523 MLX5_SET(create_xrq_in, create_in, uid, in->uid); in create_xrq_cmd()
524 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, in create_xrq_cmd()
528 srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn); in create_xrq_cmd()
529 srq->uid = in->uid; in create_xrq_cmd()
537 u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {}; in destroy_xrq_cmd() local
539 MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ); in destroy_xrq_cmd()
540 MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn); in destroy_xrq_cmd()
541 MLX5_SET(destroy_xrq_in, in, uid, srq->uid); in destroy_xrq_cmd()
543 return mlx5_cmd_exec_in(dev->mdev, destroy_xrq, in); in destroy_xrq_cmd()
550 u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {}; in arm_xrq_cmd() local
552 MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ); in arm_xrq_cmd()
553 MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ); in arm_xrq_cmd()
554 MLX5_SET(arm_rq_in, in, srq_number, srq->srqn); in arm_xrq_cmd()
555 MLX5_SET(arm_rq_in, in, lwm, lwm); in arm_xrq_cmd()
556 MLX5_SET(arm_rq_in, in, uid, srq->uid); in arm_xrq_cmd()
558 return mlx5_cmd_exec_in(dev->mdev, arm_rq, in); in arm_xrq_cmd()
564 u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {}; in query_xrq_cmd() local
572 return -ENOMEM; in query_xrq_cmd()
574 MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ); in query_xrq_cmd()
575 MLX5_SET(query_xrq_in, in, xrqn, srq->srqn); in query_xrq_cmd()
577 err = mlx5_cmd_exec_inout(dev->mdev, query_xrq, in, xrq_out); in query_xrq_cmd()
584 out->flags |= MLX5_SRQ_FLAG_ERR; in query_xrq_cmd()
585 out->tm_next_tag = in query_xrq_cmd()
588 out->tm_hw_phase_cnt = in query_xrq_cmd()
591 out->tm_sw_phase_cnt = in query_xrq_cmd()
601 struct mlx5_srq_attr *in) in create_srq_split() argument
603 if (!dev->mdev->issi) in create_srq_split()
604 return create_srq_cmd(dev, srq, in); in create_srq_split()
605 switch (srq->common.res) { in create_srq_split()
607 return create_xrc_srq_cmd(dev, srq, in); in create_srq_split()
609 return create_xrq_cmd(dev, srq, in); in create_srq_split()
611 return create_rmp_cmd(dev, srq, in); in create_srq_split()
617 if (!dev->mdev->issi) in destroy_srq_split()
619 switch (srq->common.res) { in destroy_srq_split()
630 struct mlx5_srq_attr *in) in mlx5_cmd_create_srq() argument
632 struct mlx5_srq_table *table = &dev->srq_table; in mlx5_cmd_create_srq()
635 switch (in->type) { in mlx5_cmd_create_srq()
637 srq->common.res = MLX5_RES_XSRQ; in mlx5_cmd_create_srq()
640 srq->common.res = MLX5_RES_XRQ; in mlx5_cmd_create_srq()
643 srq->common.res = MLX5_RES_SRQ; in mlx5_cmd_create_srq()
646 err = create_srq_split(dev, srq, in); in mlx5_cmd_create_srq()
650 refcount_set(&srq->common.refcount, 1); in mlx5_cmd_create_srq()
651 init_completion(&srq->common.free); in mlx5_cmd_create_srq()
653 err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL)); in mlx5_cmd_create_srq()
667 struct mlx5_srq_table *table = &dev->srq_table; in mlx5_cmd_destroy_srq()
672 tmp = xa_cmpxchg_irq(&table->array, srq->srqn, srq, XA_ZERO_ENTRY, 0); in mlx5_cmd_destroy_srq()
674 return xa_err(tmp) ?: -EINVAL; in mlx5_cmd_destroy_srq()
680 * because we are storing in pre-allocated space xarray in mlx5_cmd_destroy_srq()
683 xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0); in mlx5_cmd_destroy_srq()
686 xa_erase_irq(&table->array, srq->srqn); in mlx5_cmd_destroy_srq()
688 mlx5_core_res_put(&srq->common); in mlx5_cmd_destroy_srq()
689 wait_for_completion(&srq->common.free); in mlx5_cmd_destroy_srq()
696 if (!dev->mdev->issi) in mlx5_cmd_query_srq()
698 switch (srq->common.res) { in mlx5_cmd_query_srq()
711 if (!dev->mdev->issi) in mlx5_cmd_arm_srq()
713 switch (srq->common.res) { in mlx5_cmd_arm_srq()
738 srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; in srq_event_notifier()
740 xa_lock(&table->array); in srq_event_notifier()
741 srq = xa_load(&table->array, srqn); in srq_event_notifier()
743 refcount_inc(&srq->common.refcount); in srq_event_notifier()
744 xa_unlock(&table->array); in srq_event_notifier()
749 srq->event(srq, eqe->type); in srq_event_notifier()
751 mlx5_core_res_put(&srq->common); in srq_event_notifier()
758 struct mlx5_srq_table *table = &dev->srq_table; in mlx5_init_srq_table()
761 xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ); in mlx5_init_srq_table()
763 table->nb.notifier_call = srq_event_notifier; in mlx5_init_srq_table()
764 mlx5_notifier_register(dev->mdev, &table->nb); in mlx5_init_srq_table()
771 struct mlx5_srq_table *table = &dev->srq_table; in mlx5_cleanup_srq_table()
773 mlx5_notifier_unregister(dev->mdev, &table->nb); in mlx5_cleanup_srq_table()