443 	struct hns_roce_mtr *mtr = &mr->pbl_mtr;  in hns_roce_map_mr_sg()  local
464 	mtr->hem_cfg.region[0].offset = 0;  in hns_roce_map_mr_sg()
465 	mtr->hem_cfg.region[0].count = mr->npages;  in hns_roce_map_mr_sg()
466 	mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num;  in hns_roce_map_mr_sg()
467 	mtr->hem_cfg.region_count = 1;  in hns_roce_map_mr_sg()
468 	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);  in hns_roce_map_mr_sg()
597 static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_map_region()  argument
612 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in mtr_map_region()
673 static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_bufs()  argument
676 	if (mtr->umem) {  in mtr_free_bufs()
677 		ib_umem_release(mtr->umem);  in mtr_free_bufs()
678 		mtr->umem = NULL;  in mtr_free_bufs()
682 	if (mtr->kmem) {  in mtr_free_bufs()
683 		hns_roce_buf_free(hr_dev, mtr->kmem);  in mtr_free_bufs()
684 		mtr->kmem = NULL;  in mtr_free_bufs()
688 static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_bufs()  argument
698 		mtr->kmem = NULL;  in mtr_alloc_bufs()
699 		mtr->umem = ib_umem_get(ibdev, user_addr, total_size,  in mtr_alloc_bufs()
701 		if (IS_ERR(mtr->umem)) {  in mtr_alloc_bufs()
703 				  PTR_ERR(mtr->umem));  in mtr_alloc_bufs()
707 		mtr->umem = NULL;  in mtr_alloc_bufs()
708 		mtr->kmem = hns_roce_buf_alloc(hr_dev, total_size,  in mtr_alloc_bufs()
712 		if (IS_ERR(mtr->kmem)) {  in mtr_alloc_bufs()
714 				  PTR_ERR(mtr->kmem));  in mtr_alloc_bufs()
715 			return PTR_ERR(mtr->kmem);  in mtr_alloc_bufs()
722 static int cal_mtr_pg_cnt(struct hns_roce_mtr *mtr)  in cal_mtr_pg_cnt()  argument
728 	for (i = 0; i < mtr->hem_cfg.region_count; i++) {  in cal_mtr_pg_cnt()
729 		region = &mtr->hem_cfg.region[i];  in cal_mtr_pg_cnt()
736 static bool need_split_huge_page(struct hns_roce_mtr *mtr)  in need_split_huge_page()  argument
743 	return mtr->hem_cfg.is_direct && mtr->hem_cfg.region_count > 1;  in need_split_huge_page()
746 static int mtr_map_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_map_bufs()  argument
749 	int page_count = cal_mtr_pg_cnt(mtr);  in mtr_map_bufs()
755 	page_shift = need_split_huge_page(mtr) ? HNS_HW_PAGE_SHIFT :  in mtr_map_bufs()
756 						 mtr->hem_cfg.buf_pg_shift;  in mtr_map_bufs()
762 	if (mtr->umem)  in mtr_map_bufs()
764 					       mtr->umem, page_shift);  in mtr_map_bufs()
767 					       mtr->kmem, page_shift);  in mtr_map_bufs()
776 	if (need_split_huge_page(mtr) && npage > 1) {  in mtr_map_bufs()
780 				  mtr->umem ? "umtr" : "kmtr", ret, npage);  in mtr_map_bufs()
786 	ret = hns_roce_mtr_map(hr_dev, mtr, pages, page_count);  in mtr_map_bufs()
796 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_map()  argument
808 	if (mtr->hem_cfg.is_direct) {  in hns_roce_mtr_map()
809 		mtr->hem_cfg.root_ba = pages[0];  in hns_roce_mtr_map()
813 	for (i = 0, mapped_cnt = 0; i < mtr->hem_cfg.region_count &&  in hns_roce_mtr_map()
815 		r = &mtr->hem_cfg.region[i];  in hns_roce_mtr_map()
830 		ret = mtr_map_region(hr_dev, mtr, r, &pages[r->offset],  in hns_roce_mtr_map()
882 				 struct hns_roce_mtr *mtr, u32 start_index,  in hns_roce_get_mhop_mtt()  argument
893 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in hns_roce_get_mhop_mtt()
911 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_find()  argument
914 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in hns_roce_mtr_find()
928 		ret = hns_roce_get_mhop_mtt(hr_dev, mtr, start_index,  in hns_roce_mtr_find()
935 			       struct hns_roce_mtr *mtr,  in get_best_page_shift()  argument
940 	if (!buf_attr->adaptive || buf_attr->type != MTR_PBL || !mtr->umem)  in get_best_page_shift()
943 	page_sz = ib_umem_find_best_pgsz(mtr->umem,  in get_best_page_shift()
954 			    struct hns_roce_mtr *mtr,  in get_best_hop_num()  argument
972 	if (mtr->umem)  in get_best_hop_num()
973 		ba_cnt = ib_umem_num_dma_blocks(mtr->umem, buf_pg_sz);  in get_best_hop_num()
1014 			    struct hns_roce_mtr *mtr,  in mtr_init_buf_cfg()  argument
1017 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in mtr_init_buf_cfg()
1031 	if (need_split_huge_page(mtr)) {  in mtr_init_buf_cfg()
1039 		cfg->buf_pg_count = mtr->umem ?  in mtr_init_buf_cfg()
1040 			ib_umem_num_dma_blocks(mtr->umem, buf_pg_sz) :  in mtr_init_buf_cfg()
1043 		pgoff = mtr->umem ? mtr->umem->address & ~PAGE_MASK : 0;  in mtr_init_buf_cfg()
1053 		if (attr->type == MTR_PBL && mtr->umem)  in mtr_init_buf_cfg()
1054 			r->count = ib_umem_num_dma_blocks(mtr->umem, buf_pg_sz);  in mtr_init_buf_cfg()
1072 				      struct hns_roce_mtr *mtr,  in cal_best_bt_pg_sz()  argument
1088 		for (i = 0; i < mtr->hem_cfg.region_count; i++) {  in cal_best_bt_pg_sz()
1089 			re = &mtr->hem_cfg.region[i];  in cal_best_bt_pg_sz()
1104 static int mtr_alloc_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_mtt()  argument
1107 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in mtr_alloc_mtt()
1110 	hns_roce_hem_list_init(&mtr->hem_list);  in mtr_alloc_mtt()
1112 		ba_page_shift = cal_best_bt_pg_sz(hr_dev, mtr, ba_page_shift);  in mtr_alloc_mtt()
1116 		ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,  in mtr_alloc_mtt()
1121 		cfg->root_ba = mtr->hem_list.root_ba;  in mtr_alloc_mtt()
1130 static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_mtt()  argument
1132 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in mtr_free_mtt()
1145 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_create()  argument
1157 		mtr->umem = NULL;  in hns_roce_mtr_create()
1158 		mtr->kmem = NULL;  in hns_roce_mtr_create()
1160 		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);  in hns_roce_mtr_create()
1167 		ret = get_best_page_shift(hr_dev, mtr, buf_attr);  in hns_roce_mtr_create()
1171 		ret = get_best_hop_num(hr_dev, mtr, buf_attr, ba_page_shift);  in hns_roce_mtr_create()
1176 	ret = mtr_init_buf_cfg(hr_dev, mtr, buf_attr);  in hns_roce_mtr_create()
1180 	ret = mtr_alloc_mtt(hr_dev, mtr, ba_page_shift);  in hns_roce_mtr_create()
1190 	ret = mtr_map_bufs(hr_dev, mtr);  in hns_roce_mtr_create()
1199 	mtr_free_mtt(hr_dev, mtr);  in hns_roce_mtr_create()
1201 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_create()
1206 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in hns_roce_mtr_destroy()  argument
1209 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in hns_roce_mtr_destroy()
1212 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_destroy()