14  * if this is the first key for an MR or MW
27 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)  in mr_check_range()  argument
29 	switch (mr->ibmr.type) {  in mr_check_range()
35 		if (iova < mr->ibmr.iova ||  in mr_check_range()
36 		    iova + length > mr->ibmr.iova + mr->ibmr.length) {  in mr_check_range()
37 			rxe_dbg_mr(mr, "iova/length out of range\n");  in mr_check_range()
43 		rxe_dbg_mr(mr, "mr type not supported\n");  in mr_check_range()
48 static void rxe_mr_init(int access, struct rxe_mr *mr)  in rxe_mr_init()  argument
50 	u32 key = mr->elem.index << 8 | rxe_get_next_key(-1);  in rxe_mr_init()
57 	mr->lkey = mr->ibmr.lkey = key;  in rxe_mr_init()
58 	mr->rkey = mr->ibmr.rkey = key;  in rxe_mr_init()
60 	mr->access = access;  in rxe_mr_init()
61 	mr->ibmr.page_size = PAGE_SIZE;  in rxe_mr_init()
62 	mr->page_mask = PAGE_MASK;  in rxe_mr_init()
63 	mr->page_shift = PAGE_SHIFT;  in rxe_mr_init()
64 	mr->state = RXE_MR_STATE_INVALID;  in rxe_mr_init()
67 void rxe_mr_init_dma(int access, struct rxe_mr *mr)  in rxe_mr_init_dma()  argument
69 	rxe_mr_init(access, mr);  in rxe_mr_init_dma()
71 	mr->state = RXE_MR_STATE_VALID;  in rxe_mr_init_dma()
72 	mr->ibmr.type = IB_MR_TYPE_DMA;  in rxe_mr_init_dma()
75 static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)  in rxe_mr_iova_to_index()  argument
77 	return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);  in rxe_mr_iova_to_index()
80 static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)  in rxe_mr_iova_to_page_offset()  argument
82 	return iova & (mr_page_size(mr) - 1);  in rxe_mr_iova_to_page_offset()
94 static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)  in rxe_mr_fill_pages_from_sgt()  argument
96 	XA_STATE(xas, &mr->page_list, 0);  in rxe_mr_fill_pages_from_sgt()
99 	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);  in rxe_mr_fill_pages_from_sgt()
111 				rxe_dbg_mr(mr, "Page can't be persistent\n");  in rxe_mr_fill_pages_from_sgt()
130 		     int access, struct rxe_mr *mr)  in rxe_mr_init_user()  argument
135 	rxe_mr_init(access, mr);  in rxe_mr_init_user()
137 	xa_init(&mr->page_list);  in rxe_mr_init_user()
141 		rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",  in rxe_mr_init_user()
146 	err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);  in rxe_mr_init_user()
152 	mr->umem = umem;  in rxe_mr_init_user()
153 	mr->ibmr.type = IB_MR_TYPE_USER;  in rxe_mr_init_user()
154 	mr->state = RXE_MR_STATE_VALID;  in rxe_mr_init_user()
159 static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)  in rxe_mr_alloc()  argument
161 	XA_STATE(xas, &mr->page_list, 0);  in rxe_mr_alloc()
165 	xa_init(&mr->page_list);  in rxe_mr_alloc()
183 	mr->num_buf = num_buf;  in rxe_mr_alloc()
188 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)  in rxe_mr_init_fast()  argument
193 	rxe_mr_init(RXE_ACCESS_REMOTE, mr);  in rxe_mr_init_fast()
195 	err = rxe_mr_alloc(mr, max_pages);  in rxe_mr_init_fast()
199 	mr->state = RXE_MR_STATE_FREE;  in rxe_mr_init_fast()
200 	mr->ibmr.type = IB_MR_TYPE_MEM_REG;  in rxe_mr_init_fast()
210 	struct rxe_mr *mr = to_rmr(ibmr);  in rxe_set_page()  local
212 	bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);  in rxe_set_page()
216 		rxe_dbg_mr(mr, "Page cannot be persistent\n");  in rxe_set_page()
220 	if (unlikely(mr->nbuf == mr->num_buf))  in rxe_set_page()
223 	err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));  in rxe_set_page()
227 	mr->nbuf++;  in rxe_set_page()
234 	struct rxe_mr *mr = to_rmr(ibmr);  in rxe_map_mr_sg()  local
235 	unsigned int page_size = mr_page_size(mr);  in rxe_map_mr_sg()
237 	mr->nbuf = 0;  in rxe_map_mr_sg()
238 	mr->page_shift = ilog2(page_size);  in rxe_map_mr_sg()
239 	mr->page_mask = ~((u64)page_size - 1);  in rxe_map_mr_sg()
240 	mr->page_offset = mr->ibmr.iova & (page_size - 1);  in rxe_map_mr_sg()
245 static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,  in rxe_mr_copy_xarray()  argument
248 	unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_mr_copy_xarray()
249 	unsigned long index = rxe_mr_iova_to_index(mr, iova);  in rxe_mr_copy_xarray()
255 		page = xa_load(&mr->page_list, index);  in rxe_mr_copy_xarray()
260 				mr_page_size(mr) - page_offset);  in rxe_mr_copy_xarray()
277 static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 dma_addr, void *addr,  in rxe_mr_copy_dma()  argument
304 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,  in rxe_mr_copy()  argument
312 	if (WARN_ON(!mr))  in rxe_mr_copy()
315 	if (mr->ibmr.type == IB_MR_TYPE_DMA) {  in rxe_mr_copy()
316 		rxe_mr_copy_dma(mr, iova, addr, length, dir);  in rxe_mr_copy()
320 	err = mr_check_range(mr, iova, length);  in rxe_mr_copy()
322 		rxe_dbg_mr(mr, "iova out of range\n");  in rxe_mr_copy()
326 	return rxe_mr_copy_xarray(mr, iova, addr, length, dir);  in rxe_mr_copy()
344 	struct rxe_mr		*mr	= NULL;  in copy_data()  local
357 		mr = lookup_mr(pd, access, sge->lkey, RXE_LOOKUP_LOCAL);  in copy_data()
358 		if (!mr) {  in copy_data()
368 			if (mr) {  in copy_data()
369 				rxe_put(mr);  in copy_data()
370 				mr = NULL;  in copy_data()
382 				mr = lookup_mr(pd, access, sge->lkey,  in copy_data()
384 				if (!mr) {  in copy_data()
398 			err = rxe_mr_copy(mr, iova, addr, bytes, dir);  in copy_data()
412 	if (mr)  in copy_data()
413 		rxe_put(mr);  in copy_data()
418 	if (mr)  in copy_data()
419 		rxe_put(mr);  in copy_data()
424 int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)  in rxe_flush_pmem_iova()  argument
433 	/* mr must be valid even if length is zero */  in rxe_flush_pmem_iova()
434 	if (WARN_ON(!mr))  in rxe_flush_pmem_iova()
440 	if (mr->ibmr.type == IB_MR_TYPE_DMA)  in rxe_flush_pmem_iova()
443 	err = mr_check_range(mr, iova, length);  in rxe_flush_pmem_iova()
448 		index = rxe_mr_iova_to_index(mr, iova);  in rxe_flush_pmem_iova()
449 		page = xa_load(&mr->page_list, index);  in rxe_flush_pmem_iova()
450 		page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_flush_pmem_iova()
454 				mr_page_size(mr) - page_offset);  in rxe_flush_pmem_iova()
471 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,  in rxe_mr_do_atomic_op()  argument
479 	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {  in rxe_mr_do_atomic_op()
480 		rxe_dbg_mr(mr, "mr not in valid state\n");  in rxe_mr_do_atomic_op()
484 	if (mr->ibmr.type == IB_MR_TYPE_DMA) {  in rxe_mr_do_atomic_op()
491 		err = mr_check_range(mr, iova, sizeof(value));  in rxe_mr_do_atomic_op()
493 			rxe_dbg_mr(mr, "iova out of range\n");  in rxe_mr_do_atomic_op()
496 		page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_mr_do_atomic_op()
497 		index = rxe_mr_iova_to_index(mr, iova);  in rxe_mr_do_atomic_op()
498 		page = xa_load(&mr->page_list, index);  in rxe_mr_do_atomic_op()
504 		rxe_dbg_mr(mr, "iova not aligned\n");  in rxe_mr_do_atomic_op()
529 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)  in rxe_mr_do_atomic_write()  argument
536 	if (unlikely(mr->state != RXE_MR_STATE_VALID)) {  in rxe_mr_do_atomic_write()
537 		rxe_dbg_mr(mr, "mr not in valid state\n");  in rxe_mr_do_atomic_write()
541 	if (mr->ibmr.type == IB_MR_TYPE_DMA) {  in rxe_mr_do_atomic_write()
549 		err = mr_check_range(mr, iova, sizeof(value));  in rxe_mr_do_atomic_write()
551 			rxe_dbg_mr(mr, "iova out of range\n");  in rxe_mr_do_atomic_write()
554 		page_offset = rxe_mr_iova_to_page_offset(mr, iova);  in rxe_mr_do_atomic_write()
555 		index = rxe_mr_iova_to_index(mr, iova);  in rxe_mr_do_atomic_write()
556 		page = xa_load(&mr->page_list, index);  in rxe_mr_do_atomic_write()
563 		rxe_dbg_mr(mr, "misaligned address\n");  in rxe_mr_do_atomic_write()
577 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)  in rxe_mr_do_atomic_write()  argument
619 	struct rxe_mr *mr;  in lookup_mr()  local
623 	mr = rxe_pool_get_index(&rxe->mr_pool, index);  in lookup_mr()
624 	if (!mr)  in lookup_mr()
627 	if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||  in lookup_mr()
628 		     (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||  in lookup_mr()
629 		     mr_pd(mr) != pd || ((access & mr->access) != access) ||  in lookup_mr()
630 		     mr->state != RXE_MR_STATE_VALID)) {  in lookup_mr()
631 		rxe_put(mr);  in lookup_mr()
632 		mr = NULL;  in lookup_mr()
635 	return mr;  in lookup_mr()
641 	struct rxe_mr *mr;  in rxe_invalidate_mr()  local
645 	mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);  in rxe_invalidate_mr()
646 	if (!mr) {  in rxe_invalidate_mr()
647 		rxe_dbg_qp(qp, "No MR for key %#x\n", key);  in rxe_invalidate_mr()
652 	remote = mr->access & RXE_ACCESS_REMOTE;  in rxe_invalidate_mr()
653 	if (remote ? (key != mr->rkey) : (key != mr->lkey)) {  in rxe_invalidate_mr()
654 		rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n",  in rxe_invalidate_mr()
655 			key, (remote ? mr->rkey : mr->lkey));  in rxe_invalidate_mr()
660 	if (atomic_read(&mr->num_mw) > 0) {  in rxe_invalidate_mr()
661 		rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n");  in rxe_invalidate_mr()
666 	if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) {  in rxe_invalidate_mr()
667 		rxe_dbg_mr(mr, "Type (%d) is wrong\n", mr->ibmr.type);  in rxe_invalidate_mr()
672 	mr->state = RXE_MR_STATE_FREE;  in rxe_invalidate_mr()
676 	rxe_put(mr);  in rxe_invalidate_mr()
681 /* user can (re)register fast MR by executing a REG_MR WQE.
682  * user is expected to hold a reference on the ib mr until the
684  * Once a fast MR is created this is the only way to change the
686  * the ib mr keys in sync with rxe mr keys.
690 	struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);  in rxe_reg_fast_mr()  local
694 	/* user can only register MR in free state */  in rxe_reg_fast_mr()
695 	if (unlikely(mr->state != RXE_MR_STATE_FREE)) {  in rxe_reg_fast_mr()
696 		rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey);  in rxe_reg_fast_mr()
700 	/* user can only register mr with qp in same protection domain */  in rxe_reg_fast_mr()
701 	if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {  in rxe_reg_fast_mr()
702 		rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n");  in rxe_reg_fast_mr()
707 	if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {  in rxe_reg_fast_mr()
708 		rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n",  in rxe_reg_fast_mr()
709 			key, mr->lkey);  in rxe_reg_fast_mr()
713 	mr->access = access;  in rxe_reg_fast_mr()
714 	mr->lkey = key;  in rxe_reg_fast_mr()
715 	mr->rkey = key;  in rxe_reg_fast_mr()
716 	mr->ibmr.iova = wqe->wr.wr.reg.mr->iova;  in rxe_reg_fast_mr()
717 	mr->state = RXE_MR_STATE_VALID;  in rxe_reg_fast_mr()
724 	struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);  in rxe_mr_cleanup()  local
726 	rxe_put(mr_pd(mr));  in rxe_mr_cleanup()
727 	ib_umem_release(mr->umem);  in rxe_mr_cleanup()
729 	if (mr->ibmr.type != IB_MR_TYPE_DMA)  in rxe_mr_cleanup()
730 		xa_destroy(&mr->page_list);  in rxe_mr_cleanup()