52 static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in alloc_mr_key()  argument
59 	/* Allocate a key for mr from mr_table */  in alloc_mr_key()
63 		ibdev_err(ibdev, "failed to alloc id for MR key, id(%d)\n", id);  in alloc_mr_key()
67 	mr->key = hw_index_to_key(id); /* MR key */  in alloc_mr_key()
82 static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in free_mr_key()  argument
84 	unsigned long obj = key_to_hw_index(mr->key);  in free_mr_key()
90 static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,  in alloc_mr_pbl()  argument
94 	bool is_fast = mr->type == MR_TYPE_FRMR;  in alloc_mr_pbl()
98 	mr->pbl_hop_num = is_fast ? 1 : hr_dev->caps.pbl_hop_num;  in alloc_mr_pbl()
101 	buf_attr.region[0].size = mr->size;  in alloc_mr_pbl()
102 	buf_attr.region[0].hopnum = mr->pbl_hop_num;  in alloc_mr_pbl()
104 	buf_attr.user_access = mr->access;  in alloc_mr_pbl()
105 	/* fast MR's buffer is alloced before mapping, not at creation */  in alloc_mr_pbl()
107 	buf_attr.iova = mr->iova;  in alloc_mr_pbl()
108 	/* pagesize and hopnum is fixed for fast MR */  in alloc_mr_pbl()
112 	err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,  in alloc_mr_pbl()
120 	mr->npages = mr->pbl_mtr.hem_cfg.buf_pg_count;  in alloc_mr_pbl()
121 	mr->pbl_hop_num = buf_attr.region[0].hopnum;  in alloc_mr_pbl()
126 static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in free_mr_pbl()  argument
128 	hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);  in free_mr_pbl()
131 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in hns_roce_mr_free()  argument
136 	if (mr->enabled) {  in hns_roce_mr_free()
138 					      key_to_hw_index(mr->key) &  in hns_roce_mr_free()
145 	free_mr_pbl(hr_dev, mr);  in hns_roce_mr_free()
146 	free_mr_key(hr_dev, mr);  in hns_roce_mr_free()
150 			      struct hns_roce_mr *mr)  in hns_roce_mr_enable()  argument
152 	unsigned long mtpt_idx = key_to_hw_index(mr->key);  in hns_roce_mr_enable()
162 	if (mr->type != MR_TYPE_FRMR)  in hns_roce_mr_enable()
163 		ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr);  in hns_roce_mr_enable()
165 		ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr);  in hns_roce_mr_enable()
178 	mr->enabled = 1;  in hns_roce_mr_enable()
198 	struct hns_roce_mr *mr;  in hns_roce_get_dma_mr()  local
201 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in hns_roce_get_dma_mr()
202 	if (!mr)  in hns_roce_get_dma_mr()
205 	mr->type = MR_TYPE_DMA;  in hns_roce_get_dma_mr()
206 	mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_get_dma_mr()
207 	mr->access = acc;  in hns_roce_get_dma_mr()
210 	hns_roce_hem_list_init(&mr->pbl_mtr.hem_list);  in hns_roce_get_dma_mr()
211 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_get_dma_mr()
215 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_get_dma_mr()
219 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;  in hns_roce_get_dma_mr()
221 	return &mr->ibmr;  in hns_roce_get_dma_mr()
223 	free_mr_key(hr_dev, mr);  in hns_roce_get_dma_mr()
226 	kfree(mr);  in hns_roce_get_dma_mr()
235 	struct hns_roce_mr *mr;  in hns_roce_reg_user_mr()  local
238 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in hns_roce_reg_user_mr()
239 	if (!mr) {  in hns_roce_reg_user_mr()
244 	mr->iova = virt_addr;  in hns_roce_reg_user_mr()
245 	mr->size = length;  in hns_roce_reg_user_mr()
246 	mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_reg_user_mr()
247 	mr->access = access_flags;  in hns_roce_reg_user_mr()
248 	mr->type = MR_TYPE_MR;  in hns_roce_reg_user_mr()
250 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_reg_user_mr()
254 	ret = alloc_mr_pbl(hr_dev, mr, udata, start);  in hns_roce_reg_user_mr()
258 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_reg_user_mr()
262 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;  in hns_roce_reg_user_mr()
264 	return &mr->ibmr;  in hns_roce_reg_user_mr()
267 	free_mr_pbl(hr_dev, mr);  in hns_roce_reg_user_mr()
269 	free_mr_key(hr_dev, mr);  in hns_roce_reg_user_mr()
271 	kfree(mr);  in hns_roce_reg_user_mr()
285 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_rereg_user_mr()  local
290 	if (!mr->enabled) {  in hns_roce_rereg_user_mr()
300 	mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);  in hns_roce_rereg_user_mr()
312 	mr->enabled = 0;  in hns_roce_rereg_user_mr()
313 	mr->iova = virt_addr;  in hns_roce_rereg_user_mr()
314 	mr->size = length;  in hns_roce_rereg_user_mr()
317 		mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_rereg_user_mr()
320 		mr->access = mr_access_flags;  in hns_roce_rereg_user_mr()
323 		free_mr_pbl(hr_dev, mr);  in hns_roce_rereg_user_mr()
324 		ret = alloc_mr_pbl(hr_dev, mr, udata, start);  in hns_roce_rereg_user_mr()
326 			ibdev_err(ib_dev, "failed to alloc mr PBL, ret = %d.\n",  in hns_roce_rereg_user_mr()
332 	ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, mailbox->buf);  in hns_roce_rereg_user_mr()
345 	mr->enabled = 1;  in hns_roce_rereg_user_mr()
362 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_dereg_mr()  local
367 	hns_roce_mr_free(hr_dev, mr);  in hns_roce_dereg_mr()
368 	kfree(mr);  in hns_roce_dereg_mr()
378 	struct hns_roce_mr *mr;  in hns_roce_alloc_mr()  local
390 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in hns_roce_alloc_mr()
391 	if (!mr)  in hns_roce_alloc_mr()
394 	mr->type = MR_TYPE_FRMR;  in hns_roce_alloc_mr()
395 	mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_alloc_mr()
396 	mr->size = max_num_sg * (1 << PAGE_SHIFT);  in hns_roce_alloc_mr()
399 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_alloc_mr()
403 	ret = alloc_mr_pbl(hr_dev, mr, NULL, 0);  in hns_roce_alloc_mr()
407 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_alloc_mr()
411 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;  in hns_roce_alloc_mr()
412 	mr->ibmr.length = mr->size;  in hns_roce_alloc_mr()
414 	return &mr->ibmr;  in hns_roce_alloc_mr()
417 	free_mr_pbl(hr_dev, mr);  in hns_roce_alloc_mr()
419 	free_mr_key(hr_dev, mr);  in hns_roce_alloc_mr()
421 	kfree(mr);  in hns_roce_alloc_mr()
427 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_set_page()  local
429 	if (likely(mr->npages < mr->pbl_mtr.hem_cfg.buf_pg_count)) {  in hns_roce_set_page()
430 		mr->page_list[mr->npages++] = addr;  in hns_roce_set_page()
442 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_map_mr_sg()  local
443 	struct hns_roce_mtr *mtr = &mr->pbl_mtr;  in hns_roce_map_mr_sg()
451 	mr->npages = 0;  in hns_roce_map_mr_sg()
452 	mr->page_list = kvcalloc(mr->pbl_mtr.hem_cfg.buf_pg_count,  in hns_roce_map_mr_sg()
454 	if (!mr->page_list)  in hns_roce_map_mr_sg()
460 			  mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, sg_num);  in hns_roce_map_mr_sg()
465 	mtr->hem_cfg.region[0].count = mr->npages;  in hns_roce_map_mr_sg()
466 	mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num;  in hns_roce_map_mr_sg()
468 	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);  in hns_roce_map_mr_sg()
473 		mr->pbl_mtr.hem_cfg.buf_pg_shift = (u32)ilog2(ibmr->page_size);  in hns_roce_map_mr_sg()
477 	kvfree(mr->page_list);  in hns_roce_map_mr_sg()
478 	mr->page_list = NULL;  in hns_roce_map_mr_sg()