49  * be read. "Data Ready" is signaled when an RDMA Receive completes,
50  * or when a set of RDMA Reads complete.
61  * svc_rdma_recvfrom must post RDMA Reads to pull the RPC Call's
63  * RDMA Reads using pages in svc_rqst::rq_pages, which are
76  * of the RPC Call message, using the RDMA Read sink pages kept in
86  * The first svc_rqst supplies pages for RDMA Reads. These are moved
91  * During the second svc_rdma_recvfrom call, RDMA Read sink pages
98 #include <rdma/ib_verbs.h>
99 #include <rdma/rdma_cm.h>
119 svc_rdma_recv_ctxt_alloc(struct svcxprt_rdma *rdma)  in svc_rdma_recv_ctxt_alloc()  argument
121 	int node = ibdev_to_node(rdma->sc_cm_id->device);  in svc_rdma_recv_ctxt_alloc()
129 	buffer = kmalloc_node(rdma->sc_max_req_size, GFP_KERNEL, node);  in svc_rdma_recv_ctxt_alloc()
132 	addr = ib_dma_map_single(rdma->sc_pd->device, buffer,  in svc_rdma_recv_ctxt_alloc()
133 				 rdma->sc_max_req_size, DMA_FROM_DEVICE);  in svc_rdma_recv_ctxt_alloc()
134 	if (ib_dma_mapping_error(rdma->sc_pd->device, addr))  in svc_rdma_recv_ctxt_alloc()
137 	svc_rdma_recv_cid_init(rdma, &ctxt->rc_cid);  in svc_rdma_recv_ctxt_alloc()
149 	ctxt->rc_recv_sge.length = rdma->sc_max_req_size;  in svc_rdma_recv_ctxt_alloc()
150 	ctxt->rc_recv_sge.lkey = rdma->sc_pd->local_dma_lkey;  in svc_rdma_recv_ctxt_alloc()
152 	svc_rdma_cc_init(rdma, &ctxt->rc_cc);  in svc_rdma_recv_ctxt_alloc()
163 static void svc_rdma_recv_ctxt_destroy(struct svcxprt_rdma *rdma,  in svc_rdma_recv_ctxt_destroy()  argument
166 	ib_dma_unmap_single(rdma->sc_pd->device, ctxt->rc_recv_sge.addr,  in svc_rdma_recv_ctxt_destroy()
174  * @rdma: svcxprt_rdma being torn down
177 void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma)  in svc_rdma_recv_ctxts_destroy()  argument
182 	while ((node = llist_del_first(&rdma->sc_recv_ctxts))) {  in svc_rdma_recv_ctxts_destroy()
184 		svc_rdma_recv_ctxt_destroy(rdma, ctxt);  in svc_rdma_recv_ctxts_destroy()
190  * @rdma: controlling svcxprt_rdma
194 struct svc_rdma_recv_ctxt *svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma)  in svc_rdma_recv_ctxt_get()  argument
199 	node = llist_del_first(&rdma->sc_recv_ctxts);  in svc_rdma_recv_ctxt_get()
210  * @rdma: controlling svcxprt_rdma
214 void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,  in svc_rdma_recv_ctxt_put()  argument
217 	svc_rdma_cc_release(rdma, &ctxt->rc_cc, DMA_FROM_DEVICE);  in svc_rdma_recv_ctxt_put()
229 	llist_add(&ctxt->rc_node, &rdma->sc_recv_ctxts);  in svc_rdma_recv_ctxt_put()
244 	struct svcxprt_rdma *rdma =  in svc_rdma_release_ctxt()  local
248 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_release_ctxt()
251 static bool svc_rdma_refresh_recvs(struct svcxprt_rdma *rdma,  in svc_rdma_refresh_recvs()  argument
259 	if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags))  in svc_rdma_refresh_recvs()
264 		ctxt = svc_rdma_recv_ctxt_get(rdma);  in svc_rdma_refresh_recvs()
271 		rdma->sc_pending_recvs++;  in svc_rdma_refresh_recvs()
276 	ret = ib_post_recv(rdma->sc_qp, recv_chain, &bad_wr);  in svc_rdma_refresh_recvs()
282 	trace_svcrdma_rq_post_err(rdma, ret);  in svc_rdma_refresh_recvs()
287 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_refresh_recvs()
296  * @rdma: fresh svcxprt_rdma
302 bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma)  in svc_rdma_post_recvs()  argument
309 	total = (rdma->sc_max_requests * 2) + rdma->sc_recv_batch;  in svc_rdma_post_recvs()
313 		ctxt = svc_rdma_recv_ctxt_alloc(rdma);  in svc_rdma_post_recvs()
316 		llist_add(&ctxt->rc_node, &rdma->sc_recv_ctxts);  in svc_rdma_post_recvs()
319 	return svc_rdma_refresh_recvs(rdma, rdma->sc_max_requests);  in svc_rdma_post_recvs()
323  * svc_rdma_wc_receive - Invoked by RDMA provider for each polled Receive WC
330 	struct svcxprt_rdma *rdma = cq->cq_context;  in svc_rdma_wc_receive()  local
334 	rdma->sc_pending_recvs--;  in svc_rdma_wc_receive()
352 	if (rdma->sc_pending_recvs < rdma->sc_max_requests)  in svc_rdma_wc_receive()
353 		if (!svc_rdma_refresh_recvs(rdma, rdma->sc_recv_batch))  in svc_rdma_wc_receive()
359 	spin_lock(&rdma->sc_rq_dto_lock);  in svc_rdma_wc_receive()
360 	list_add_tail(&ctxt->rc_list, &rdma->sc_rq_dto_q);  in svc_rdma_wc_receive()
362 	set_bit(XPT_DATA, &rdma->sc_xprt.xpt_flags);  in svc_rdma_wc_receive()
363 	spin_unlock(&rdma->sc_rq_dto_lock);  in svc_rdma_wc_receive()
364 	if (!test_bit(RDMAXPRT_CONN_PENDING, &rdma->sc_flags))  in svc_rdma_wc_receive()
365 		svc_xprt_enqueue(&rdma->sc_xprt);  in svc_rdma_wc_receive()
374 	svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_wc_receive()
375 	svc_xprt_deferred_close(&rdma->sc_xprt);  in svc_rdma_wc_receive()
380  * @rdma: svcxprt_rdma being shut down
383 void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma)  in svc_rdma_flush_recv_queues()  argument
387 	while ((ctxt = svc_rdma_next_recv_ctxt(&rdma->sc_read_complete_q))) {  in svc_rdma_flush_recv_queues()
389 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_flush_recv_queues()
391 	while ((ctxt = svc_rdma_next_recv_ctxt(&rdma->sc_rq_dto_q))) {  in svc_rdma_flush_recv_queues()
393 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_flush_recv_queues()
591 /* RPC-over-RDMA Version One private extension: Remote Invalidation.
598 static void svc_rdma_get_inv_rkey(struct svcxprt_rdma *rdma,  in svc_rdma_get_inv_rkey()  argument
607 	if (!rdma->sc_snd_w_inv)  in svc_rdma_get_inv_rkey()
648  * @rq_arg: xdr_buf containing ingress RPC/RDMA message
652  * RPC-over-RDMA transport header.
655  * RPC-over-RDMA header. For RDMA_MSG, this is the RPC message.
657  * The length of the RPC-over-RDMA header is returned.
728 static void svc_rdma_send_error(struct svcxprt_rdma *rdma,  in svc_rdma_send_error()  argument
734 	sctxt = svc_rdma_send_ctxt_get(rdma);  in svc_rdma_send_error()
737 	svc_rdma_send_error_msg(rdma, sctxt, rctxt, status);  in svc_rdma_send_error()
742  * the RPC/RDMA header small and fixed in size, so it is
774  * The incoming RPC/RDMA message is an RDMA_MSG type message
776  * was conveyed via RDMA Read).
812  * The incoming RPC/RDMA message is an RDMA_MSG type message
831  * The incoming RPC/RDMA message is an RDMA_NOMSG type message
832  * (the RPC message body was conveyed via RDMA Read).
861 	 * last page in the RDMA Read payload.  in svc_rdma_read_complete()
872 	 * procedure for that depends on what kind of RPC/RDMA  in svc_rdma_read_complete()
990 	 * so it can be restored when the RDMA Reads have  in svc_rdma_recvfrom()