Lines Matching refs:r_xprt
126 static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt, in rpcrdma_args_inline() argument
130 struct rpcrdma_ep *ep = r_xprt->rx_ep; in rpcrdma_args_inline()
158 static bool rpcrdma_results_inline(struct rpcrdma_xprt *r_xprt, in rpcrdma_results_inline() argument
161 return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_results_inline()
169 rpcrdma_nonpayload_inline(const struct rpcrdma_xprt *r_xprt, in rpcrdma_nonpayload_inline() argument
175 r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_nonpayload_inline()
228 rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf, in rpcrdma_convert_iovs() argument
295 static struct rpcrdma_mr_seg *rpcrdma_mr_prepare(struct rpcrdma_xprt *r_xprt, in rpcrdma_mr_prepare() argument
303 *mr = rpcrdma_mr_get(r_xprt); in rpcrdma_mr_prepare()
310 return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr); in rpcrdma_mr_prepare()
313 trace_xprtrdma_nomrs_err(r_xprt, req); in rpcrdma_mr_prepare()
314 xprt_wait_for_buffer_space(&r_xprt->rx_xprt); in rpcrdma_mr_prepare()
315 rpcrdma_mrs_refresh(r_xprt); in rpcrdma_mr_prepare()
333 static int rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, in rpcrdma_encode_read_list() argument
351 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_snd_buf, pos, in rpcrdma_encode_read_list()
357 seg = rpcrdma_mr_prepare(r_xprt, req, seg, nsegs, false, &mr); in rpcrdma_encode_read_list()
365 r_xprt->rx_stats.read_chunk_count++; in rpcrdma_encode_read_list()
390 static int rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, in rpcrdma_encode_write_list() argument
396 struct rpcrdma_ep *ep = r_xprt->rx_ep; in rpcrdma_encode_write_list()
406 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, in rpcrdma_encode_write_list()
421 seg = rpcrdma_mr_prepare(r_xprt, req, seg, nsegs, true, &mr); in rpcrdma_encode_write_list()
429 r_xprt->rx_stats.write_chunk_count++; in rpcrdma_encode_write_list()
430 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_write_list()
441 r_xprt->rx_stats.write_chunk_count++; in rpcrdma_encode_write_list()
442 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_write_list()
468 static int rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, in rpcrdma_encode_reply_chunk() argument
486 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg); in rpcrdma_encode_reply_chunk()
499 seg = rpcrdma_mr_prepare(r_xprt, req, seg, nsegs, true, &mr); in rpcrdma_encode_reply_chunk()
507 r_xprt->rx_stats.reply_chunk_count++; in rpcrdma_encode_reply_chunk()
508 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_reply_chunk()
556 static void rpcrdma_prepare_hdr_sge(struct rpcrdma_xprt *r_xprt, in rpcrdma_prepare_hdr_sge() argument
574 static bool rpcrdma_prepare_head_iov(struct rpcrdma_xprt *r_xprt, in rpcrdma_prepare_head_iov() argument
581 if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) in rpcrdma_prepare_head_iov()
662 static void rpcrdma_pullup_tail_iov(struct rpcrdma_xprt *r_xprt, in rpcrdma_pullup_tail_iov() argument
671 r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len; in rpcrdma_pullup_tail_iov()
676 static void rpcrdma_pullup_pagelist(struct rpcrdma_xprt *r_xprt, in rpcrdma_pullup_pagelist() argument
694 r_xprt->rx_stats.pullup_copy_count += len; in rpcrdma_pullup_pagelist()
712 static bool rpcrdma_prepare_noch_pullup(struct rpcrdma_xprt *r_xprt, in rpcrdma_prepare_noch_pullup() argument
717 rpcrdma_pullup_tail_iov(r_xprt, req, xdr); in rpcrdma_prepare_noch_pullup()
720 rpcrdma_pullup_pagelist(r_xprt, req, xdr); in rpcrdma_prepare_noch_pullup()
723 return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len); in rpcrdma_prepare_noch_pullup()
726 static bool rpcrdma_prepare_noch_mapped(struct rpcrdma_xprt *r_xprt, in rpcrdma_prepare_noch_mapped() argument
732 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_noch_mapped()
748 static bool rpcrdma_prepare_readch(struct rpcrdma_xprt *r_xprt, in rpcrdma_prepare_readch() argument
752 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_readch()
790 inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, in rpcrdma_prepare_send_sges() argument
798 req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt); in rpcrdma_prepare_send_sges()
809 rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen); in rpcrdma_prepare_send_sges()
814 if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
818 if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
822 if (!rpcrdma_prepare_readch(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
860 rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) in rpcrdma_marshal_req() argument
887 *p++ = r_xprt->rx_buf.rb_max_requests; in rpcrdma_marshal_req()
905 if (rpcrdma_results_inline(r_xprt, rqst)) in rpcrdma_marshal_req()
908 rpcrdma_nonpayload_inline(r_xprt, rqst)) in rpcrdma_marshal_req()
927 if (rpcrdma_args_inline(r_xprt, rqst)) { in rpcrdma_marshal_req()
935 r_xprt->rx_stats.nomsg_call_count++; in rpcrdma_marshal_req()
962 ret = rpcrdma_encode_read_list(r_xprt, req, rqst, rtype); in rpcrdma_marshal_req()
965 ret = rpcrdma_encode_write_list(r_xprt, req, rqst, wtype); in rpcrdma_marshal_req()
968 ret = rpcrdma_encode_reply_chunk(r_xprt, req, rqst, wtype); in rpcrdma_marshal_req()
972 ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len, in rpcrdma_marshal_req()
982 r_xprt->rx_stats.failed_marshal_count++; in rpcrdma_marshal_req()
995 static void rpcrdma_update_cwnd(struct rpcrdma_xprt *r_xprt, u32 grant) in rpcrdma_update_cwnd() argument
997 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_update_cwnd()
1000 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, grant); in rpcrdma_update_cwnd()
1011 void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt) in rpcrdma_reset_cwnd() argument
1013 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reset_cwnd()
1017 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, 1); in rpcrdma_reset_cwnd()
1121 rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) in rpcrdma_is_bcall() argument
1124 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_is_bcall()
1159 rpcrdma_bc_receive_call(r_xprt, rep); in rpcrdma_is_bcall()
1262 rpcrdma_decode_msg(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep, in rpcrdma_decode_msg() argument
1284 r_xprt->rx_stats.fixup_copy_count += in rpcrdma_decode_msg()
1287 r_xprt->rx_stats.total_rdma_reply += writelist; in rpcrdma_decode_msg()
1292 rpcrdma_decode_nomsg(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) in rpcrdma_decode_nomsg() argument
1312 r_xprt->rx_stats.total_rdma_reply += replychunk; in rpcrdma_decode_nomsg()
1317 rpcrdma_decode_error(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep, in rpcrdma_decode_error() argument
1376 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_complete_rqst() local
1377 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_complete_rqst()
1383 status = rpcrdma_decode_msg(r_xprt, rep, rqst); in rpcrdma_complete_rqst()
1386 status = rpcrdma_decode_nomsg(r_xprt, rep); in rpcrdma_complete_rqst()
1389 status = rpcrdma_decode_error(r_xprt, rep, rqst); in rpcrdma_complete_rqst()
1406 r_xprt->rx_stats.bad_reply_count++; in rpcrdma_complete_rqst()
1429 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_reply_handler() local
1430 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reply_handler()
1431 struct rpcrdma_buffer *buf = &r_xprt->rx_buf; in rpcrdma_reply_handler()
1457 if (rpcrdma_is_bcall(r_xprt, rep)) in rpcrdma_reply_handler()
1472 else if (credits > r_xprt->rx_ep->re_max_requests) in rpcrdma_reply_handler()
1473 credits = r_xprt->rx_ep->re_max_requests; in rpcrdma_reply_handler()
1474 rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1), in rpcrdma_reply_handler()
1477 rpcrdma_update_cwnd(r_xprt, credits); in rpcrdma_reply_handler()
1490 frwr_unmap_async(r_xprt, req); in rpcrdma_reply_handler()