70 	struct rds_mr *mr;  in rds_mr_tree_walk()  local
74 		mr = rb_entry(parent, struct rds_mr, r_rb_node);  in rds_mr_tree_walk()
76 		if (key < mr->r_key)  in rds_mr_tree_walk()
78 		else if (key > mr->r_key)  in rds_mr_tree_walk()
81 			return mr;  in rds_mr_tree_walk()
93  * Destroy the transport-specific part of a MR.
95 static void rds_destroy_mr(struct rds_mr *mr)  in rds_destroy_mr()  argument
97 	struct rds_sock *rs = mr->r_sock;  in rds_destroy_mr()
101 	rdsdebug("RDS: destroy mr key is %x refcnt %u\n",  in rds_destroy_mr()
102 		 mr->r_key, kref_read(&mr->r_kref));  in rds_destroy_mr()
105 	if (!RB_EMPTY_NODE(&mr->r_rb_node))  in rds_destroy_mr()
106 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_destroy_mr()
107 	trans_private = mr->r_trans_private;  in rds_destroy_mr()
108 	mr->r_trans_private = NULL;  in rds_destroy_mr()
112 		mr->r_trans->free_mr(trans_private, mr->r_invalidate);  in rds_destroy_mr()
117 	struct rds_mr *mr = container_of(kref, struct rds_mr, r_kref);  in __rds_put_mr_final()  local
119 	rds_destroy_mr(mr);  in __rds_put_mr_final()
120 	kfree(mr);  in __rds_put_mr_final()
129 	struct rds_mr *mr;  in rds_rdma_drop_keys()  local
136 		mr = rb_entry(node, struct rds_mr, r_rb_node);  in rds_rdma_drop_keys()
137 		if (mr->r_trans == rs->rs_transport)  in rds_rdma_drop_keys()
138 			mr->r_invalidate = 0;  in rds_rdma_drop_keys()
139 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_drop_keys()
140 		RB_CLEAR_NODE(&mr->r_rb_node);  in rds_rdma_drop_keys()
142 		kref_put(&mr->r_kref, __rds_put_mr_final);  in rds_rdma_drop_keys()
176 	struct rds_mr *mr = NULL, *found;  in __rds_rdma_map()  local
219 	/* Restrict the size of mr irrespective of underlying transport  in __rds_rdma_map()
220 	 * To account for unaligned mr regions, subtract one from nr_pages  in __rds_rdma_map()
237 	mr = kzalloc(sizeof(struct rds_mr), GFP_KERNEL);  in __rds_rdma_map()
238 	if (!mr) {  in __rds_rdma_map()
243 	kref_init(&mr->r_kref);  in __rds_rdma_map()
244 	RB_CLEAR_NODE(&mr->r_rb_node);  in __rds_rdma_map()
245 	mr->r_trans = rs->rs_transport;  in __rds_rdma_map()
246 	mr->r_sock = rs;  in __rds_rdma_map()
249 		mr->r_use_once = 1;  in __rds_rdma_map()
251 		mr->r_invalidate = 1;  in __rds_rdma_map()
253 		mr->r_write = 1;  in __rds_rdma_map()
257 	 * pointers to the mr's sg array.  We check to see if we've mapped  in __rds_rdma_map()
286 	/* Obtain a transport specific MR. If this succeeds, the  in __rds_rdma_map()
287 	 * s/g list is now owned by the MR.  in __rds_rdma_map()
291 		sg, nents, rs, &mr->r_key, cp ? cp->cp_conn : NULL,  in __rds_rdma_map()
310 	mr->r_trans_private = trans_private;  in __rds_rdma_map()
313 	       mr->r_key, (void *)(unsigned long) args->cookie_addr);  in __rds_rdma_map()
320 		cookie = rds_rdma_make_cookie(mr->r_key, 0);  in __rds_rdma_map()
322 		cookie = rds_rdma_make_cookie(mr->r_key,  in __rds_rdma_map()
337 	/* Inserting the new MR into the rbtree bumps its  in __rds_rdma_map()
340 	found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);  in __rds_rdma_map()
343 	BUG_ON(found && found != mr);  in __rds_rdma_map()
345 	rdsdebug("RDS: get_mr key is %x\n", mr->r_key);  in __rds_rdma_map()
347 		kref_get(&mr->r_kref);  in __rds_rdma_map()
348 		*mr_ret = mr;  in __rds_rdma_map()
354 	if (mr)  in __rds_rdma_map()
355 		kref_put(&mr->r_kref, __rds_put_mr_final);  in __rds_rdma_map()
397  * Free the MR indicated by the given R_Key
402 	struct rds_mr *mr;  in rds_free_mr()  local
419 	/* Look up the MR given its R_key and remove it from the rbtree  in rds_free_mr()
424 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
425 	if (mr) {  in rds_free_mr()
426 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_free_mr()
427 		RB_CLEAR_NODE(&mr->r_rb_node);  in rds_free_mr()
429 			mr->r_invalidate = 1;  in rds_free_mr()
433 	if (!mr)  in rds_free_mr()
436 	kref_put(&mr->r_kref, __rds_put_mr_final);  in rds_free_mr()
442  * tells us this MR was used. It allows us to implement
447 	struct rds_mr *mr;  in rds_rdma_unuse()  local
452 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_rdma_unuse()
453 	if (!mr) {  in rds_rdma_unuse()
454 		pr_debug("rds: trying to unuse MR with unknown r_key %u!\n",  in rds_rdma_unuse()
460 	/* Get a reference so that the MR won't go away before calling  in rds_rdma_unuse()
463 	kref_get(&mr->r_kref);  in rds_rdma_unuse()
468 	if (mr->r_use_once || force) {  in rds_rdma_unuse()
469 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_unuse()
470 		RB_CLEAR_NODE(&mr->r_rb_node);  in rds_rdma_unuse()
478 	if (mr->r_trans->sync_mr)  in rds_rdma_unuse()
479 		mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE);  in rds_rdma_unuse()
482 	kref_put(&mr->r_kref, __rds_put_mr_final);  in rds_rdma_unuse()
484 	/* If the MR was marked as invalidate, this will  in rds_rdma_unuse()
487 		kref_put(&mr->r_kref, __rds_put_mr_final);  in rds_rdma_unuse()
645 	/* odp-mr is not supported for multiple requests within one message */  in rds_cmsg_rdma_args()
697 	 * destination address (which is really an offset into the MR)  in rds_cmsg_rdma_args()
808  * The application wants us to pass an RDMA destination (aka MR)
815 	struct rds_mr *mr;  in rds_cmsg_rdma_dest()  local
825 	/* We are reusing a previously mapped MR here. Most likely, the  in rds_cmsg_rdma_dest()
833 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_cmsg_rdma_dest()
834 	if (!mr)  in rds_cmsg_rdma_dest()
837 		kref_get(&mr->r_kref);  in rds_cmsg_rdma_dest()
840 	if (mr) {  in rds_cmsg_rdma_dest()
841 		mr->r_trans->sync_mr(mr->r_trans_private,  in rds_cmsg_rdma_dest()
843 		rm->rdma.op_rdma_mr = mr;  in rds_cmsg_rdma_dest()