Lines Matching full:ndev
131 struct mlx5_vdpa_net *ndev; member
159 static void free_fixed_resources(struct mlx5_vdpa_net *ndev);
160 static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev);
161 static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled);
162 static void teardown_vq_resources(struct mlx5_vdpa_net *ndev);
163 static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq);
281 static int create_tis(struct mlx5_vdpa_net *ndev) in create_tis() argument
283 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in create_tis()
289 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); in create_tis()
290 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); in create_tis()
297 static void destroy_tis(struct mlx5_vdpa_net *ndev) in destroy_tis() argument
299 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); in destroy_tis()
305 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) in cq_frag_buf_alloc() argument
312 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, in cq_frag_buf_alloc()
313 ndev->mvdev.mdev->priv.numa_node); in cq_frag_buf_alloc()
325 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) in umem_frag_buf_alloc() argument
329 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, in umem_frag_buf_alloc()
330 ndev->mvdev.mdev->priv.numa_node); in umem_frag_buf_alloc()
333 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) in cq_frag_buf_free() argument
335 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); in cq_frag_buf_free()
373 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, in qp_prepare() argument
381 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); in qp_prepare()
395 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); in qp_prepare()
397 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); in qp_prepare()
407 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) in rq_buf_alloc() argument
409 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, in rq_buf_alloc()
411 ndev->mvdev.mdev->priv.numa_node); in rq_buf_alloc()
414 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) in rq_buf_free() argument
416 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); in rq_buf_free()
419 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, in qp_create() argument
422 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in qp_create()
431 err = rq_buf_alloc(ndev, vqp, mvq->num_ent); in qp_create()
435 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); in qp_create()
447 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); in qp_create()
451 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); in qp_create()
461 vqp->mqp.uid = ndev->mvdev.res.uid; in qp_create()
471 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); in qp_create()
474 rq_buf_free(ndev, vqp); in qp_create()
479 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) in qp_destroy() argument
485 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); in qp_destroy()
486 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) in qp_destroy()
487 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); in qp_destroy()
489 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); in qp_destroy()
490 rq_buf_free(ndev, vqp); in qp_destroy()
513 struct mlx5_vdpa_net *ndev = mvq->ndev; in mlx5_vdpa_handle_completions() local
516 event_cb = &ndev->event_cbs[mvq->index]; in mlx5_vdpa_handle_completions()
531 struct mlx5_vdpa_net *ndev = mvq->ndev; in mlx5_vdpa_cq_comp() local
532 void __iomem *uar_page = ndev->mvdev.res.uar->map; in mlx5_vdpa_cq_comp()
555 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) in cq_create() argument
557 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in cq_create()
558 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in cq_create()
559 void __iomem *uar_page = ndev->mvdev.res.uar->map; in cq_create()
577 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); in cq_create()
591 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); in cq_create()
607 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); in cq_create()
626 cq_frag_buf_free(ndev, &vcq->buf); in cq_create()
628 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); in cq_create()
632 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) in cq_destroy() argument
634 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; in cq_destroy()
635 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in cq_destroy()
639 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); in cq_destroy()
642 cq_frag_buf_free(ndev, &vcq->buf); in cq_destroy()
643 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); in cq_destroy()
646 static int read_umem_params(struct mlx5_vdpa_net *ndev) in read_umem_params() argument
650 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; in read_umem_params()
665 mlx5_vdpa_warn(&ndev->mvdev, in read_umem_params()
672 ndev->umem_1_buffer_param_a = MLX5_GET(virtio_emulation_cap, caps, umem_1_buffer_param_a); in read_umem_params()
673 ndev->umem_1_buffer_param_b = MLX5_GET(virtio_emulation_cap, caps, umem_1_buffer_param_b); in read_umem_params()
675 ndev->umem_2_buffer_param_a = MLX5_GET(virtio_emulation_cap, caps, umem_2_buffer_param_a); in read_umem_params()
676 ndev->umem_2_buffer_param_b = MLX5_GET(virtio_emulation_cap, caps, umem_2_buffer_param_b); in read_umem_params()
678 ndev->umem_3_buffer_param_a = MLX5_GET(virtio_emulation_cap, caps, umem_3_buffer_param_a); in read_umem_params()
679 ndev->umem_3_buffer_param_b = MLX5_GET(virtio_emulation_cap, caps, umem_3_buffer_param_b); in read_umem_params()
686 static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, in set_umem_size() argument
694 p_a = ndev->umem_1_buffer_param_a; in set_umem_size()
695 p_b = ndev->umem_1_buffer_param_b; in set_umem_size()
699 p_a = ndev->umem_2_buffer_param_a; in set_umem_size()
700 p_b = ndev->umem_2_buffer_param_b; in set_umem_size()
704 p_a = ndev->umem_3_buffer_param_a; in set_umem_size()
705 p_b = ndev->umem_3_buffer_param_b; in set_umem_size()
713 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) in umem_frag_buf_free() argument
715 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); in umem_frag_buf_free()
718 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) in create_umem() argument
728 set_umem_size(ndev, mvq, num, &umem); in create_umem()
729 err = umem_frag_buf_alloc(ndev, umem, umem->size); in create_umem()
742 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); in create_umem()
750 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); in create_umem()
752 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); in create_umem()
764 umem_frag_buf_free(ndev, umem); in create_umem()
768 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) in umem_destroy() argument
788 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) in umem_destroy()
791 umem_frag_buf_free(ndev, umem); in umem_destroy()
794 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in umems_create() argument
800 err = create_umem(ndev, mvq, num); in umems_create()
808 umem_destroy(ndev, mvq, num); in umems_create()
813 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in umems_destroy() argument
818 umem_destroy(ndev, mvq, num); in umems_destroy()
821 static int get_queue_type(struct mlx5_vdpa_net *ndev) in get_queue_type() argument
825 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); in get_queue_type()
878 static int create_virtqueue(struct mlx5_vdpa_net *ndev, in create_virtqueue() argument
884 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in create_virtqueue()
895 err = umems_create(ndev, mvq); in create_virtqueue()
910 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in create_virtqueue()
918 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); in create_virtqueue()
921 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); in create_virtqueue()
971 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); in create_virtqueue()
972 if (counters_supported(&ndev->mvdev)) in create_virtqueue()
975 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); in create_virtqueue()
999 umems_destroy(ndev, mvq); in create_virtqueue()
1003 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in destroy_virtqueue() argument
1011 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); in destroy_virtqueue()
1014 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { in destroy_virtqueue()
1015 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); in destroy_virtqueue()
1019 umems_destroy(ndev, mvq); in destroy_virtqueue()
1021 mlx5_vdpa_put_mr(&ndev->mvdev, mvq->vq_mr); in destroy_virtqueue()
1024 mlx5_vdpa_put_mr(&ndev->mvdev, mvq->desc_mr); in destroy_virtqueue()
1038 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, in alloc_inout() argument
1054 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
1066 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
1083 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
1101 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); in alloc_inout()
1133 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) in modify_qp() argument
1141 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); in modify_qp()
1145 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); in modify_qp()
1150 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in connect_qps() argument
1154 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); in connect_qps()
1158 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); in connect_qps()
1162 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); in connect_qps()
1166 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); in connect_qps()
1170 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); in connect_qps()
1174 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); in connect_qps()
1178 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); in connect_qps()
1197 static void fill_query_virtqueue_cmd(struct mlx5_vdpa_net *ndev, in fill_query_virtqueue_cmd() argument
1206 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in fill_query_virtqueue_cmd()
1209 static void query_virtqueue_end(struct mlx5_vdpa_net *ndev, in query_virtqueue_end() argument
1221 static int query_virtqueues(struct mlx5_vdpa_net *ndev, in query_virtqueues() argument
1226 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in query_virtqueues()
1246 fill_query_virtqueue_cmd(ndev, &ndev->vqs[start_vq + i], &cmd_mem[i]); in query_virtqueues()
1249 err = mlx5_vdpa_exec_async_cmds(&ndev->mvdev, cmds, num_vqs); in query_virtqueues()
1267 query_virtqueue_end(ndev, &cmd_mem[i], &attrs[i]); in query_virtqueues()
1276 static bool is_resumable(struct mlx5_vdpa_net *ndev) in is_resumable() argument
1278 return ndev->mvdev.vdev.config->resume; in is_resumable()
1306 static void fill_modify_virtqueue_cmd(struct mlx5_vdpa_net *ndev, in fill_modify_virtqueue_cmd() argument
1311 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in fill_modify_virtqueue_cmd()
1323 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in fill_modify_virtqueue_cmd()
1345 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); in fill_modify_virtqueue_cmd()
1348 u16 mlx_features = get_features(ndev->mvdev.actual_features); in fill_modify_virtqueue_cmd()
1377 static void modify_virtqueue_end(struct mlx5_vdpa_net *ndev, in modify_virtqueue_end() argument
1381 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in modify_virtqueue_end()
1407 static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in counter_set_alloc() argument
1414 if (!counters_supported(&ndev->mvdev)) in counter_set_alloc()
1421 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in counter_set_alloc()
1423 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); in counter_set_alloc()
1432 static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in counter_set_dealloc() argument
1437 if (!counters_supported(&ndev->mvdev)) in counter_set_dealloc()
1442 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); in counter_set_dealloc()
1444 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) in counter_set_dealloc()
1445 mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); in counter_set_dealloc()
1458 static void alloc_vector(struct mlx5_vdpa_net *ndev, in alloc_vector() argument
1461 struct mlx5_vdpa_irq_pool *irqp = &ndev->irqp; in alloc_vector()
1470 dev_name(&ndev->mvdev.vdev.dev), mvq->index); in alloc_vector()
1471 ent->dev_id = &ndev->event_cbs[mvq->index]; in alloc_vector()
1484 static void dealloc_vector(struct mlx5_vdpa_net *ndev, in dealloc_vector() argument
1487 struct mlx5_vdpa_irq_pool *irqp = &ndev->irqp; in dealloc_vector()
1498 static int setup_vq(struct mlx5_vdpa_net *ndev, in setup_vq() argument
1508 err = cq_create(ndev, idx, mvq->num_ent); in setup_vq()
1512 err = qp_create(ndev, mvq, &mvq->fwqp); in setup_vq()
1516 err = qp_create(ndev, mvq, &mvq->vqqp); in setup_vq()
1520 err = connect_qps(ndev, mvq); in setup_vq()
1524 err = counter_set_alloc(ndev, mvq); in setup_vq()
1528 alloc_vector(ndev, mvq); in setup_vq()
1529 err = create_virtqueue(ndev, mvq, filled); in setup_vq()
1536 err = resume_vq(ndev, mvq); in setup_vq()
1544 destroy_virtqueue(ndev, mvq); in setup_vq()
1546 dealloc_vector(ndev, mvq); in setup_vq()
1547 counter_set_dealloc(ndev, mvq); in setup_vq()
1549 qp_destroy(ndev, &mvq->vqqp); in setup_vq()
1551 qp_destroy(ndev, &mvq->fwqp); in setup_vq()
1553 cq_destroy(ndev, idx); in setup_vq()
1557 static int modify_virtqueues(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs, int state) in modify_virtqueues() argument
1559 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in modify_virtqueues()
1579 mvq = &ndev->vqs[vq_idx]; in modify_virtqueues()
1587 if (!is_valid_state_change(mvq->fw_state, state, is_resumable(ndev))) { in modify_virtqueues()
1599 fill_modify_virtqueue_cmd(ndev, mvq, state, &cmd_mem[i]); in modify_virtqueues()
1602 err = mlx5_vdpa_exec_async_cmds(&ndev->mvdev, cmds, num_vqs); in modify_virtqueues()
1614 mvq = &ndev->vqs[vq_idx]; in modify_virtqueues()
1624 modify_virtqueue_end(ndev, mvq, state); in modify_virtqueues()
1633 static int suspend_vqs(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs) in suspend_vqs() argument
1640 if (start_vq >= ndev->cur_num_vqs) in suspend_vqs()
1643 mvq = &ndev->vqs[start_vq]; in suspend_vqs()
1650 err = modify_virtqueues(ndev, start_vq, num_vqs, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND); in suspend_vqs()
1658 err = query_virtqueues(ndev, start_vq, num_vqs, attrs); in suspend_vqs()
1663 mvq = &ndev->vqs[vq_idx]; in suspend_vqs()
1673 static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in suspend_vq() argument
1675 return suspend_vqs(ndev, mvq->index, 1); in suspend_vq()
1678 static int resume_vqs(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs) in resume_vqs() argument
1683 if (start_vq >= ndev->mvdev.max_vqs) in resume_vqs()
1686 mvq = &ndev->vqs[start_vq]; in resume_vqs()
1690 if (mvq->index >= ndev->cur_num_vqs) in resume_vqs()
1698 err = modify_virtqueues(ndev, start_vq, num_vqs, mvq->fw_state); in resume_vqs()
1703 if (!is_resumable(ndev)) { in resume_vqs()
1704 mlx5_vdpa_warn(&ndev->mvdev, "vq %d is not resumable\n", mvq->index); in resume_vqs()
1711 mlx5_vdpa_err(&ndev->mvdev, "resume vq %u called from bad state %d\n", in resume_vqs()
1716 return modify_virtqueues(ndev, start_vq, num_vqs, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); in resume_vqs()
1719 static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in resume_vq() argument
1721 return resume_vqs(ndev, mvq->index, 1); in resume_vq()
1724 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in teardown_vq() argument
1729 suspend_vq(ndev, mvq); in teardown_vq()
1731 destroy_virtqueue(ndev, mvq); in teardown_vq()
1732 dealloc_vector(ndev, mvq); in teardown_vq()
1733 counter_set_dealloc(ndev, mvq); in teardown_vq()
1734 qp_destroy(ndev, &mvq->vqqp); in teardown_vq()
1735 qp_destroy(ndev, &mvq->fwqp); in teardown_vq()
1736 cq_destroy(ndev, mvq->index); in teardown_vq()
1740 static int create_rqt(struct mlx5_vdpa_net *ndev) in create_rqt() argument
1742 int rqt_table_size = roundup_pow_of_two(ndev->rqt_size); in create_rqt()
1743 int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2); in create_rqt()
1756 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); in create_rqt()
1763 list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); in create_rqt()
1766 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); in create_rqt()
1776 static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) in modify_rqt() argument
1791 MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); in modify_rqt()
1798 list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); in modify_rqt()
1801 err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); in modify_rqt()
1809 static void destroy_rqt(struct mlx5_vdpa_net *ndev) in destroy_rqt() argument
1811 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); in destroy_rqt()
1814 static int create_tir(struct mlx5_vdpa_net *ndev) in create_tir() argument
1834 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); in create_tir()
1848 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); in create_tir()
1849 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); in create_tir()
1851 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); in create_tir()
1856 mlx5_vdpa_add_tirn(ndev); in create_tir()
1860 static void destroy_tir(struct mlx5_vdpa_net *ndev) in destroy_tir() argument
1862 mlx5_vdpa_remove_tirn(ndev); in destroy_tir()
1863 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); in destroy_tir()
1875 static int add_steering_counters(struct mlx5_vdpa_net *ndev, in add_steering_counters() argument
1883 node->ucast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); in add_steering_counters()
1887 node->mcast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); in add_steering_counters()
1898 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); in add_steering_counters()
1905 static void remove_steering_counters(struct mlx5_vdpa_net *ndev, in remove_steering_counters() argument
1909 mlx5_fc_destroy(ndev->mvdev.mdev, node->mcast_counter.counter); in remove_steering_counters()
1910 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); in remove_steering_counters()
1914 static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, in mlx5_vdpa_add_mac_vlan_rules() argument
1939 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN)) { in mlx5_vdpa_add_mac_vlan_rules()
1949 dests[0].tir_num = ndev->res.tirn; in mlx5_vdpa_add_mac_vlan_rules()
1950 err = add_steering_counters(ndev, node, &flow_act, dests); in mlx5_vdpa_add_mac_vlan_rules()
1957 node->ucast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); in mlx5_vdpa_add_mac_vlan_rules()
1971 node->mcast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); in mlx5_vdpa_add_mac_vlan_rules()
1977 mlx5_vdpa_add_rx_counters(ndev, node); in mlx5_vdpa_add_mac_vlan_rules()
1983 remove_steering_counters(ndev, node); in mlx5_vdpa_add_mac_vlan_rules()
1989 static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, in mlx5_vdpa_del_mac_vlan_rules() argument
1992 mlx5_vdpa_remove_rx_counters(ndev, node); in mlx5_vdpa_del_mac_vlan_rules()
2015 static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) in mac_vlan_lookup() argument
2021 hlist_for_each_entry(pos, &ndev->macvlan_hash[idx], hlist) { in mac_vlan_lookup()
2028 static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vid, bool tagged) in mac_vlan_add() argument
2036 if (mac_vlan_lookup(ndev, val)) in mac_vlan_add()
2045 ptr->ndev = ndev; in mac_vlan_add()
2046 err = mlx5_vdpa_add_mac_vlan_rules(ndev, ndev->config.mac, ptr); in mac_vlan_add()
2051 hlist_add_head(&ptr->hlist, &ndev->macvlan_hash[idx]); in mac_vlan_add()
2059 static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) in mac_vlan_del() argument
2063 ptr = mac_vlan_lookup(ndev, search_val(mac, vlan, tagged)); in mac_vlan_del()
2068 mlx5_vdpa_del_mac_vlan_rules(ndev, ptr); in mac_vlan_del()
2069 remove_steering_counters(ndev, ptr); in mac_vlan_del()
2073 static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) in clear_mac_vlan_table() argument
2080 hlist_for_each_entry_safe(pos, n, &ndev->macvlan_hash[i], hlist) { in clear_mac_vlan_table()
2082 mlx5_vdpa_del_mac_vlan_rules(ndev, pos); in clear_mac_vlan_table()
2083 remove_steering_counters(ndev, pos); in clear_mac_vlan_table()
2089 static int setup_steering(struct mlx5_vdpa_net *ndev) in setup_steering() argument
2098 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); in setup_steering()
2100 mlx5_vdpa_err(&ndev->mvdev, "failed to get flow namespace\n"); in setup_steering()
2104 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); in setup_steering()
2105 if (IS_ERR(ndev->rxft)) { in setup_steering()
2106 mlx5_vdpa_err(&ndev->mvdev, "failed to create flow table\n"); in setup_steering()
2107 return PTR_ERR(ndev->rxft); in setup_steering()
2109 mlx5_vdpa_add_rx_flow_table(ndev); in setup_steering()
2111 err = mac_vlan_add(ndev, ndev->config.mac, 0, false); in setup_steering()
2118 mlx5_vdpa_remove_rx_flow_table(ndev); in setup_steering()
2119 mlx5_destroy_flow_table(ndev->rxft); in setup_steering()
2123 static void teardown_steering(struct mlx5_vdpa_net *ndev) in teardown_steering() argument
2125 clear_mac_vlan_table(ndev); in teardown_steering()
2126 mlx5_vdpa_remove_rx_flow_table(ndev); in teardown_steering()
2127 mlx5_destroy_flow_table(ndev->rxft); in teardown_steering()
2132 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in handle_ctrl_mac() local
2146 if (!memcmp(ndev->config.mac, mac, 6)) { in handle_ctrl_mac()
2154 if (!is_zero_ether_addr(ndev->config.mac)) { in handle_ctrl_mac()
2155 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { in handle_ctrl_mac()
2157 ndev->config.mac); in handle_ctrl_mac()
2171 memcpy(mac_back, ndev->config.mac, ETH_ALEN); in handle_ctrl_mac()
2173 memcpy(ndev->config.mac, mac, ETH_ALEN); in handle_ctrl_mac()
2177 mac_vlan_del(ndev, mac_back, 0, false); in handle_ctrl_mac()
2179 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) { in handle_ctrl_mac()
2191 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { in handle_ctrl_mac()
2193 ndev->config.mac); in handle_ctrl_mac()
2201 memcpy(ndev->config.mac, mac_back, ETH_ALEN); in handle_ctrl_mac()
2203 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) in handle_ctrl_mac()
2221 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in change_num_qps() local
2222 int cur_vqs = ndev->cur_num_vqs; in change_num_qps()
2228 err = modify_rqt(ndev, new_vqs); in change_num_qps()
2232 if (is_resumable(ndev)) { in change_num_qps()
2233 suspend_vqs(ndev, new_vqs, cur_vqs - new_vqs); in change_num_qps()
2236 teardown_vq(ndev, &ndev->vqs[i]); in change_num_qps()
2239 ndev->cur_num_vqs = new_vqs; in change_num_qps()
2241 ndev->cur_num_vqs = new_vqs; in change_num_qps()
2244 err = setup_vq(ndev, &ndev->vqs[i], false); in change_num_qps()
2249 err = resume_vqs(ndev, cur_vqs, new_vqs - cur_vqs); in change_num_qps()
2253 err = modify_rqt(ndev, new_vqs); in change_num_qps()
2261 teardown_vq(ndev, &ndev->vqs[i]); in change_num_qps()
2263 ndev->cur_num_vqs = cur_vqs; in change_num_qps()
2270 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in handle_ctrl_mq() local
2297 newqps > ndev->rqt_size) in handle_ctrl_mq()
2300 if (ndev->cur_num_vqs == 2 * newqps) { in handle_ctrl_mq()
2318 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in handle_ctrl_vlan() local
2325 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN))) in handle_ctrl_vlan()
2335 if (mac_vlan_add(ndev, ndev->config.mac, id, true)) in handle_ctrl_vlan()
2346 mac_vlan_del(ndev, ndev->config.mac, id, true); in handle_ctrl_vlan()
2363 struct mlx5_vdpa_net *ndev; in mlx5_cvq_kick_handler() local
2369 ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_cvq_kick_handler()
2372 down_write(&ndev->reslock); in mlx5_cvq_kick_handler()
2377 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) in mlx5_cvq_kick_handler()
2425 up_write(&ndev->reslock); in mlx5_cvq_kick_handler()
2431 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_kick_vq() local
2441 queue_work(mvdev->wq, &ndev->cvq_ent.work); in mlx5_vdpa_kick_vq()
2445 mvq = &ndev->vqs[idx]; in mlx5_vdpa_kick_vq()
2449 iowrite16(idx, ndev->mvdev.res.kick_addr); in mlx5_vdpa_kick_vq()
2456 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_address() local
2469 mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_address()
2480 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_num() local
2493 mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_num()
2494 ndev->needs_teardown = num != mvq->num_ent; in mlx5_vdpa_set_vq_num()
2501 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_cb() local
2503 ndev->event_cbs[idx] = *cb; in mlx5_vdpa_set_vq_cb()
2532 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_ready() local
2546 mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_ready()
2548 suspend_vq(ndev, mvq); in mlx5_vdpa_set_vq_ready()
2550 if (resume_vq(ndev, mvq)) in mlx5_vdpa_set_vq_ready()
2560 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_vq_ready() local
2568 return ndev->vqs[idx].ready; in mlx5_vdpa_get_vq_ready()
2575 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_vq_state() local
2586 mvq = &ndev->vqs[idx]; in mlx5_vdpa_set_vq_state()
2602 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_vq_state() local
2615 mvq = &ndev->vqs[idx]; in mlx5_vdpa_get_vq_state()
2629 err = query_virtqueues(ndev, mvq->index, 1, &attr); in mlx5_vdpa_get_vq_state()
2713 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_device_features() local
2715 print_features(mvdev, ndev->mvdev.mlx_features, false); in mlx5_vdpa_get_device_features()
2716 return ndev->mvdev.mlx_features; in mlx5_vdpa_get_device_features()
2743 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in setup_virtqueues() local
2748 err = setup_vq(ndev, &ndev->vqs[i], filled); in setup_virtqueues()
2757 teardown_vq(ndev, &ndev->vqs[i]); in setup_virtqueues()
2762 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) in teardown_virtqueues() argument
2766 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) in teardown_virtqueues()
2767 teardown_vq(ndev, &ndev->vqs[i]); in teardown_virtqueues()
2820 struct mlx5_vdpa_net *ndev; in update_carrier() local
2824 ndev = to_mlx5_vdpa_ndev(mvdev); in update_carrier()
2826 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); in update_carrier()
2828 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); in update_carrier()
2830 if (ndev->config_cb.callback) in update_carrier()
2831 ndev->config_cb.callback(ndev->config_cb.private); in update_carrier()
2836 static int queue_link_work(struct mlx5_vdpa_net *ndev) in queue_link_work() argument
2844 wqent->mvdev = &ndev->mvdev; in queue_link_work()
2846 queue_work(ndev->mvdev.wq, &wqent->work); in queue_link_work()
2852 struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); in event_handler() local
2856 if (ndev->mvdev.suspended) in event_handler()
2863 if (queue_link_work(ndev)) in event_handler()
2876 static void register_link_notifier(struct mlx5_vdpa_net *ndev) in register_link_notifier() argument
2878 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS))) in register_link_notifier()
2881 ndev->nb.notifier_call = event_handler; in register_link_notifier()
2882 mlx5_notifier_register(ndev->mvdev.mdev, &ndev->nb); in register_link_notifier()
2883 ndev->nb_registered = true; in register_link_notifier()
2884 queue_link_work(ndev); in register_link_notifier()
2887 static void unregister_link_notifier(struct mlx5_vdpa_net *ndev) in unregister_link_notifier() argument
2889 if (!ndev->nb_registered) in unregister_link_notifier()
2892 ndev->nb_registered = false; in unregister_link_notifier()
2893 mlx5_notifier_unregister(ndev->mvdev.mdev, &ndev->nb); in unregister_link_notifier()
2894 if (ndev->mvdev.wq) in unregister_link_notifier()
2895 flush_workqueue(ndev->mvdev.wq); in unregister_link_notifier()
2906 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_driver_features() local
2917 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; in mlx5_vdpa_set_driver_features()
2922 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i]; in mlx5_vdpa_set_driver_features()
2937 ndev->needs_teardown = !!(diff_features & NEEDS_TEARDOWN_MASK); in mlx5_vdpa_set_driver_features()
2946 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_config_cb() local
2948 ndev->config_cb = *cb; in mlx5_vdpa_set_config_cb()
2970 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_status() local
2972 print_status(mvdev, ndev->mvdev.status, false); in mlx5_vdpa_get_status()
2973 return ndev->mvdev.status; in mlx5_vdpa_get_status()
2976 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) in save_channel_info() argument
2983 err = query_virtqueues(ndev, mvq->index, 1, &attr); in save_channel_info()
3000 static int save_channels_info(struct mlx5_vdpa_net *ndev) in save_channels_info() argument
3004 for (i = 0; i < ndev->mvdev.max_vqs; i++) { in save_channels_info()
3005 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); in save_channels_info()
3006 save_channel_info(ndev, &ndev->vqs[i]); in save_channels_info()
3011 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) in mlx5_clear_vqs() argument
3015 for (i = 0; i < ndev->mvdev.max_vqs; i++) in mlx5_clear_vqs()
3016 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); in mlx5_clear_vqs()
3019 static void restore_channels_info(struct mlx5_vdpa_net *ndev) in restore_channels_info() argument
3025 mlx5_clear_vqs(ndev); in restore_channels_info()
3026 mvqs_set_defaults(ndev); in restore_channels_info()
3027 for (i = 0; i < ndev->mvdev.max_vqs; i++) { in restore_channels_info()
3028 mvq = &ndev->vqs[i]; in restore_channels_info()
3048 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_change_map() local
3049 bool teardown = !is_resumable(ndev); in mlx5_vdpa_change_map()
3052 suspend_vqs(ndev, 0, ndev->cur_num_vqs); in mlx5_vdpa_change_map()
3054 err = save_channels_info(ndev); in mlx5_vdpa_change_map()
3058 teardown_vq_resources(ndev); in mlx5_vdpa_change_map()
3064 ndev->vqs[i].modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY | in mlx5_vdpa_change_map()
3071 restore_channels_info(ndev); in mlx5_vdpa_change_map()
3072 err = setup_vq_resources(ndev, true); in mlx5_vdpa_change_map()
3077 resume_vqs(ndev, 0, ndev->cur_num_vqs); in mlx5_vdpa_change_map()
3083 static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled) in setup_vq_resources() argument
3085 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; in setup_vq_resources()
3088 WARN_ON(!rwsem_is_locked(&ndev->reslock)); in setup_vq_resources()
3090 if (ndev->setup) { in setup_vq_resources()
3095 mlx5_vdpa_add_debugfs(ndev); in setup_vq_resources()
3097 err = read_umem_params(ndev); in setup_vq_resources()
3107 err = create_rqt(ndev); in setup_vq_resources()
3113 err = create_tir(ndev); in setup_vq_resources()
3119 err = setup_steering(ndev); in setup_vq_resources()
3124 ndev->setup = true; in setup_vq_resources()
3129 destroy_tir(ndev); in setup_vq_resources()
3131 destroy_rqt(ndev); in setup_vq_resources()
3133 teardown_virtqueues(ndev); in setup_vq_resources()
3135 mlx5_vdpa_remove_debugfs(ndev); in setup_vq_resources()
3141 static void teardown_vq_resources(struct mlx5_vdpa_net *ndev) in teardown_vq_resources() argument
3144 WARN_ON(!rwsem_is_locked(&ndev->reslock)); in teardown_vq_resources()
3146 if (!ndev->setup) in teardown_vq_resources()
3149 mlx5_vdpa_remove_debugfs(ndev); in teardown_vq_resources()
3150 teardown_steering(ndev); in teardown_vq_resources()
3151 destroy_tir(ndev); in teardown_vq_resources()
3152 destroy_rqt(ndev); in teardown_vq_resources()
3153 teardown_virtqueues(ndev); in teardown_vq_resources()
3154 ndev->setup = false; in teardown_vq_resources()
3155 ndev->needs_teardown = false; in teardown_vq_resources()
3181 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_status() local
3186 down_write(&ndev->reslock); in mlx5_vdpa_set_status()
3188 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { in mlx5_vdpa_set_status()
3195 register_link_notifier(ndev); in mlx5_vdpa_set_status()
3197 if (ndev->needs_teardown) in mlx5_vdpa_set_status()
3198 teardown_vq_resources(ndev); in mlx5_vdpa_set_status()
3200 if (ndev->setup) { in mlx5_vdpa_set_status()
3201 err = resume_vqs(ndev, 0, ndev->cur_num_vqs); in mlx5_vdpa_set_status()
3207 err = setup_vq_resources(ndev, true); in mlx5_vdpa_set_status()
3219 ndev->mvdev.status = status; in mlx5_vdpa_set_status()
3220 up_write(&ndev->reslock); in mlx5_vdpa_set_status()
3224 unregister_link_notifier(ndev); in mlx5_vdpa_set_status()
3226 mlx5_vdpa_clean_mrs(&ndev->mvdev); in mlx5_vdpa_set_status()
3227 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; in mlx5_vdpa_set_status()
3229 up_write(&ndev->reslock); in mlx5_vdpa_set_status()
3243 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in needs_vqs_reset() local
3244 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[0]; in needs_vqs_reset()
3263 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_compat_reset() local
3269 down_write(&ndev->reslock); in mlx5_vdpa_compat_reset()
3270 unregister_link_notifier(ndev); in mlx5_vdpa_compat_reset()
3273 teardown_vq_resources(ndev); in mlx5_vdpa_compat_reset()
3274 mvqs_set_defaults(ndev); in mlx5_vdpa_compat_reset()
3278 mlx5_vdpa_clean_mrs(&ndev->mvdev); in mlx5_vdpa_compat_reset()
3279 ndev->mvdev.status = 0; in mlx5_vdpa_compat_reset()
3280 ndev->mvdev.suspended = false; in mlx5_vdpa_compat_reset()
3281 ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT; in mlx5_vdpa_compat_reset()
3282 ndev->mvdev.cvq.ready = false; in mlx5_vdpa_compat_reset()
3283 ndev->mvdev.cvq.received_desc = 0; in mlx5_vdpa_compat_reset()
3284 ndev->mvdev.cvq.completed_desc = 0; in mlx5_vdpa_compat_reset()
3285 memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); in mlx5_vdpa_compat_reset()
3286 ndev->mvdev.actual_features = 0; in mlx5_vdpa_compat_reset()
3296 setup_vq_resources(ndev, false); in mlx5_vdpa_compat_reset()
3297 up_write(&ndev->reslock); in mlx5_vdpa_compat_reset()
3316 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_config() local
3319 memcpy(buf, (u8 *)&ndev->config + offset, len); in mlx5_vdpa_get_config()
3377 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_map() local
3380 down_write(&ndev->reslock); in mlx5_vdpa_set_map()
3382 up_write(&ndev->reslock); in mlx5_vdpa_set_map()
3389 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_reset_map() local
3392 down_write(&ndev->reslock); in mlx5_vdpa_reset_map()
3394 up_write(&ndev->reslock); in mlx5_vdpa_reset_map()
3408 static void free_irqs(struct mlx5_vdpa_net *ndev) in free_irqs() argument
3413 if (!msix_mode_supported(&ndev->mvdev)) in free_irqs()
3416 if (!ndev->irqp.entries) in free_irqs()
3419 for (i = ndev->irqp.num_ent - 1; i >= 0; i--) { in free_irqs()
3420 ent = ndev->irqp.entries + i; in free_irqs()
3422 pci_msix_free_irq(ndev->mvdev.mdev->pdev, ent->map); in free_irqs()
3424 kfree(ndev->irqp.entries); in free_irqs()
3431 struct mlx5_vdpa_net *ndev; in mlx5_vdpa_free() local
3433 ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_free()
3435 free_fixed_resources(ndev); in mlx5_vdpa_free()
3437 mlx5_vdpa_destroy_mr_resources(&ndev->mvdev); in mlx5_vdpa_free()
3440 if (!is_zero_ether_addr(ndev->config.mac)) { in mlx5_vdpa_free()
3442 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); in mlx5_vdpa_free()
3444 mlx5_vdpa_free_resources(&ndev->mvdev); in mlx5_vdpa_free()
3445 free_irqs(ndev); in mlx5_vdpa_free()
3446 kfree(ndev->event_cbs); in mlx5_vdpa_free()
3447 kfree(ndev->vqs); in mlx5_vdpa_free()
3454 struct mlx5_vdpa_net *ndev; in mlx5_get_vq_notification() local
3467 ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_get_vq_notification()
3468 addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; in mlx5_get_vq_notification()
3477 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_get_vq_irq() local
3486 mvq = &ndev->vqs[idx]; in mlx5_get_vq_irq()
3500 static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, in counter_set_query() argument
3509 if (!counters_supported(&ndev->mvdev)) in counter_set_query()
3519 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); in counter_set_query()
3522 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); in counter_set_query()
3537 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_get_vendor_vq_stats() local
3544 down_read(&ndev->reslock); in mlx5_vdpa_get_vendor_vq_stats()
3558 mvq = &ndev->vqs[idx]; in mlx5_vdpa_get_vendor_vq_stats()
3559 err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); in mlx5_vdpa_get_vendor_vq_stats()
3583 up_read(&ndev->reslock); in mlx5_vdpa_get_vendor_vq_stats()
3601 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_suspend() local
3606 down_write(&ndev->reslock); in mlx5_vdpa_suspend()
3607 err = suspend_vqs(ndev, 0, ndev->cur_num_vqs); in mlx5_vdpa_suspend()
3610 up_write(&ndev->reslock); in mlx5_vdpa_suspend()
3618 struct mlx5_vdpa_net *ndev; in mlx5_vdpa_resume() local
3621 ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_resume()
3625 down_write(&ndev->reslock); in mlx5_vdpa_resume()
3627 err = resume_vqs(ndev, 0, ndev->cur_num_vqs); in mlx5_vdpa_resume()
3628 queue_link_work(ndev); in mlx5_vdpa_resume()
3629 up_write(&ndev->reslock); in mlx5_vdpa_resume()
3706 static int alloc_fixed_resources(struct mlx5_vdpa_net *ndev) in alloc_fixed_resources() argument
3708 struct mlx5_vdpa_net_resources *res = &ndev->res; in alloc_fixed_resources()
3712 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); in alloc_fixed_resources()
3716 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); in alloc_fixed_resources()
3720 err = create_tis(ndev); in alloc_fixed_resources()
3729 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); in alloc_fixed_resources()
3733 static void free_fixed_resources(struct mlx5_vdpa_net *ndev) in free_fixed_resources() argument
3735 struct mlx5_vdpa_net_resources *res = &ndev->res; in free_fixed_resources()
3740 destroy_tis(ndev); in free_fixed_resources()
3741 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); in free_fixed_resources()
3745 static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev) in mvqs_set_defaults() argument
3750 for (i = 0; i < ndev->mvdev.max_vqs; ++i) { in mvqs_set_defaults()
3751 mvq = &ndev->vqs[i]; in mvqs_set_defaults()
3754 mvq->ndev = ndev; in mvqs_set_defaults()
3764 struct mlx5_vdpa_net *ndev; member
3790 static void allocate_irqs(struct mlx5_vdpa_net *ndev) in allocate_irqs() argument
3795 if (!msix_mode_supported(&ndev->mvdev)) in allocate_irqs()
3798 if (!ndev->mvdev.mdev->pdev) in allocate_irqs()
3801 ndev->irqp.entries = kcalloc(ndev->mvdev.max_vqs, sizeof(*ndev->irqp.entries), GFP_KERNEL); in allocate_irqs()
3802 if (!ndev->irqp.entries) in allocate_irqs()
3806 for (i = 0; i < ndev->mvdev.max_vqs; i++) { in allocate_irqs()
3807 ent = ndev->irqp.entries + i; in allocate_irqs()
3809 dev_name(&ndev->mvdev.vdev.dev), i); in allocate_irqs()
3810 ent->map = pci_msix_alloc_irq_at(ndev->mvdev.mdev->pdev, MSI_ANY_INDEX, NULL); in allocate_irqs()
3814 ndev->irqp.num_ent++; in allocate_irqs()
3825 struct mlx5_vdpa_net *ndev; in mlx5_vdpa_dev_add() local
3832 if (mgtdev->ndev) in mlx5_vdpa_dev_add()
3879 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mgtdev->vdpa_ops, in mlx5_vdpa_dev_add()
3881 if (IS_ERR(ndev)) in mlx5_vdpa_dev_add()
3882 return PTR_ERR(ndev); in mlx5_vdpa_dev_add()
3884 ndev->mvdev.max_vqs = max_vqs; in mlx5_vdpa_dev_add()
3885 mvdev = &ndev->mvdev; in mlx5_vdpa_dev_add()
3888 ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); in mlx5_vdpa_dev_add()
3889 ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); in mlx5_vdpa_dev_add()
3890 if (!ndev->vqs || !ndev->event_cbs) { in mlx5_vdpa_dev_add()
3894 ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT; in mlx5_vdpa_dev_add()
3896 mvqs_set_defaults(ndev); in mlx5_vdpa_dev_add()
3897 allocate_irqs(ndev); in mlx5_vdpa_dev_add()
3898 init_rwsem(&ndev->reslock); in mlx5_vdpa_dev_add()
3899 config = &ndev->config; in mlx5_vdpa_dev_add()
3912 ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); in mlx5_vdpa_dev_add()
3917 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); in mlx5_vdpa_dev_add()
3919 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); in mlx5_vdpa_dev_add()
3923 memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); in mlx5_vdpa_dev_add()
3947 mlx5_vdpa_warn(&ndev->mvdev, in mlx5_vdpa_dev_add()
3955 ndev->rqt_size = max_vqs / 2; in mlx5_vdpa_dev_add()
3957 ndev->rqt_size = 1; in mlx5_vdpa_dev_add()
3962 ndev->mvdev.mlx_features = device_features; in mlx5_vdpa_dev_add()
3964 err = mlx5_vdpa_alloc_resources(&ndev->mvdev); in mlx5_vdpa_dev_add()
3978 err = alloc_fixed_resources(ndev); in mlx5_vdpa_dev_add()
3982 ndev->cvq_ent.mvdev = mvdev; in mlx5_vdpa_dev_add()
3983 INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); in mlx5_vdpa_dev_add()
3995 mgtdev->ndev = ndev; in mlx5_vdpa_dev_add()
3998 if (ndev->setup) in mlx5_vdpa_dev_add()
4001 down_write(&ndev->reslock); in mlx5_vdpa_dev_add()
4002 err = setup_vq_resources(ndev, false); in mlx5_vdpa_dev_add()
4003 up_write(&ndev->reslock); in mlx5_vdpa_dev_add()
4022 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_dev_del() local
4025 unregister_link_notifier(ndev); in mlx5_vdpa_dev_del()
4028 down_write(&ndev->reslock); in mlx5_vdpa_dev_del()
4029 teardown_vq_resources(ndev); in mlx5_vdpa_dev_del()
4030 up_write(&ndev->reslock); in mlx5_vdpa_dev_del()
4035 mgtdev->ndev = NULL; in mlx5_vdpa_dev_del()
4044 struct mlx5_vdpa_net *ndev; in mlx5_vdpa_set_attr() local
4049 ndev = to_mlx5_vdpa_ndev(mvdev); in mlx5_vdpa_set_attr()
4051 config = &ndev->config; in mlx5_vdpa_set_attr()
4053 down_write(&ndev->reslock); in mlx5_vdpa_set_attr()
4061 up_write(&ndev->reslock); in mlx5_vdpa_set_attr()