Lines Matching refs:rdma
116 static void svc_rdma_send_cid_init(struct svcxprt_rdma *rdma, in svc_rdma_send_cid_init() argument
119 cid->ci_queue_id = rdma->sc_sq_cq->res.id; in svc_rdma_send_cid_init()
120 cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids); in svc_rdma_send_cid_init()
124 svc_rdma_send_ctxt_alloc(struct svcxprt_rdma *rdma) in svc_rdma_send_ctxt_alloc() argument
133 size += rdma->sc_max_send_sges * sizeof(struct ib_sge); in svc_rdma_send_ctxt_alloc()
137 buffer = kmalloc(rdma->sc_max_req_size, GFP_KERNEL); in svc_rdma_send_ctxt_alloc()
140 addr = ib_dma_map_single(rdma->sc_pd->device, buffer, in svc_rdma_send_ctxt_alloc()
141 rdma->sc_max_req_size, DMA_TO_DEVICE); in svc_rdma_send_ctxt_alloc()
142 if (ib_dma_mapping_error(rdma->sc_pd->device, addr)) in svc_rdma_send_ctxt_alloc()
145 svc_rdma_send_cid_init(rdma, &ctxt->sc_cid); in svc_rdma_send_ctxt_alloc()
155 rdma->sc_max_req_size); in svc_rdma_send_ctxt_alloc()
158 for (i = 0; i < rdma->sc_max_send_sges; i++) in svc_rdma_send_ctxt_alloc()
159 ctxt->sc_sges[i].lkey = rdma->sc_pd->local_dma_lkey; in svc_rdma_send_ctxt_alloc()
175 void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma) in svc_rdma_send_ctxts_destroy() argument
180 while ((node = llist_del_first(&rdma->sc_send_ctxts)) != NULL) { in svc_rdma_send_ctxts_destroy()
182 ib_dma_unmap_single(rdma->sc_pd->device, in svc_rdma_send_ctxts_destroy()
184 rdma->sc_max_req_size, in svc_rdma_send_ctxts_destroy()
198 struct svc_rdma_send_ctxt *svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma) in svc_rdma_send_ctxt_get() argument
203 spin_lock(&rdma->sc_send_lock); in svc_rdma_send_ctxt_get()
204 node = llist_del_first(&rdma->sc_send_ctxts); in svc_rdma_send_ctxt_get()
208 spin_unlock(&rdma->sc_send_lock); in svc_rdma_send_ctxt_get()
220 spin_unlock(&rdma->sc_send_lock); in svc_rdma_send_ctxt_get()
221 ctxt = svc_rdma_send_ctxt_alloc(rdma); in svc_rdma_send_ctxt_get()
232 void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma, in svc_rdma_send_ctxt_put() argument
235 struct ib_device *device = rdma->sc_cm_id->device; in svc_rdma_send_ctxt_put()
246 trace_svcrdma_dma_unmap_page(rdma, in svc_rdma_send_ctxt_put()
251 llist_add(&ctxt->sc_node, &rdma->sc_send_ctxts); in svc_rdma_send_ctxt_put()
260 void svc_rdma_wake_send_waiters(struct svcxprt_rdma *rdma, int avail) in svc_rdma_wake_send_waiters() argument
262 atomic_add(avail, &rdma->sc_sq_avail); in svc_rdma_wake_send_waiters()
264 if (unlikely(waitqueue_active(&rdma->sc_send_wait))) in svc_rdma_wake_send_waiters()
265 wake_up(&rdma->sc_send_wait); in svc_rdma_wake_send_waiters()
278 struct svcxprt_rdma *rdma = cq->cq_context; in svc_rdma_wc_send() local
283 svc_rdma_wake_send_waiters(rdma, 1); in svc_rdma_wc_send()
297 svc_xprt_deferred_close(&rdma->sc_xprt); in svc_rdma_wc_send()
308 int svc_rdma_send(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt) in svc_rdma_send() argument
316 ib_dma_sync_single_for_device(rdma->sc_pd->device, in svc_rdma_send()
323 if ((atomic_dec_return(&rdma->sc_sq_avail) < 0)) { in svc_rdma_send()
325 trace_svcrdma_sq_full(rdma); in svc_rdma_send()
326 atomic_inc(&rdma->sc_sq_avail); in svc_rdma_send()
327 wait_event(rdma->sc_send_wait, in svc_rdma_send()
328 atomic_read(&rdma->sc_sq_avail) > 1); in svc_rdma_send()
329 if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags)) in svc_rdma_send()
331 trace_svcrdma_sq_retry(rdma); in svc_rdma_send()
336 ret = ib_post_send(rdma->sc_qp, wr, NULL); in svc_rdma_send()
342 trace_svcrdma_sq_post_err(rdma, ret); in svc_rdma_send()
343 svc_xprt_deferred_close(&rdma->sc_xprt); in svc_rdma_send()
344 wake_up(&rdma->sc_send_wait); in svc_rdma_send()
521 struct svcxprt_rdma *rdma = args->md_rdma; in svc_rdma_page_dma_map() local
523 struct ib_device *dev = rdma->sc_cm_id->device; in svc_rdma_page_dma_map()
532 trace_svcrdma_dma_map_page(rdma, dma_addr, len); in svc_rdma_page_dma_map()
539 trace_svcrdma_dma_map_err(rdma, dma_addr, len); in svc_rdma_page_dma_map()
658 static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, in svc_rdma_pull_up_needed() argument
677 return args.pd_num_sges >= rdma->sc_max_send_sges; in svc_rdma_pull_up_needed()
739 static int svc_rdma_pull_up_reply_msg(const struct svcxprt_rdma *rdma, in svc_rdma_pull_up_reply_msg() argument
772 int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, in svc_rdma_map_reply_msg() argument
778 .md_rdma = rdma, in svc_rdma_map_reply_msg()
795 if (svc_rdma_pull_up_needed(rdma, sctxt, rctxt, xdr)) in svc_rdma_map_reply_msg()
796 return svc_rdma_pull_up_reply_msg(rdma, sctxt, rctxt, xdr); in svc_rdma_map_reply_msg()
820 static int svc_rdma_send_reply_msg(struct svcxprt_rdma *rdma, in svc_rdma_send_reply_msg() argument
827 ret = svc_rdma_map_reply_msg(rdma, sctxt, rctxt, &rqstp->rq_res); in svc_rdma_send_reply_msg()
838 ret = svc_rdma_send(rdma, sctxt); in svc_rdma_send_reply_msg()
843 svc_rdma_send_ctxt_put(rdma, sctxt); in svc_rdma_send_reply_msg()
862 void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma, in svc_rdma_send_error_msg() argument
881 *p++ = rdma->sc_fc_credits; in svc_rdma_send_error_msg()
908 if (svc_rdma_send(rdma, sctxt)) in svc_rdma_send_error_msg()
914 svc_rdma_send_ctxt_put(rdma, sctxt); in svc_rdma_send_error_msg()
932 struct svcxprt_rdma *rdma = in svc_rdma_sendto() local
946 sctxt = svc_rdma_send_ctxt_get(rdma); in svc_rdma_sendto()
956 ret = svc_rdma_send_reply_chunk(rdma, rctxt, &rqstp->rq_res); in svc_rdma_sendto()
963 *p++ = rdma->sc_fc_credits; in svc_rdma_sendto()
976 ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp); in svc_rdma_sendto()
990 svc_rdma_send_error_msg(rdma, sctxt, rctxt, ret); in svc_rdma_sendto()
994 svc_rdma_send_ctxt_put(rdma, sctxt); in svc_rdma_sendto()
997 svc_xprt_deferred_close(&rdma->sc_xprt); in svc_rdma_sendto()
1021 struct svcxprt_rdma *rdma; in svc_rdma_result_payload() local
1039 rdma = container_of(rqstp->rq_xprt, struct svcxprt_rdma, sc_xprt); in svc_rdma_result_payload()
1040 ret = svc_rdma_send_write_chunk(rdma, chunk, &subbuf); in svc_rdma_result_payload()