Lines Matching refs:qp
14 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
17 static inline void retry_first_write_send(struct rxe_qp *qp, in retry_first_write_send() argument
24 int to_send = (wqe->dma.resid > qp->mtu) ? in retry_first_write_send()
25 qp->mtu : wqe->dma.resid; in retry_first_write_send()
27 qp->req.opcode = next_opcode(qp, wqe, in retry_first_write_send()
37 wqe->iova += qp->mtu; in retry_first_write_send()
41 static void req_retry(struct rxe_qp *qp) in req_retry() argument
48 struct rxe_queue *q = qp->sq.queue; in req_retry()
55 qp->req.wqe_index = cons; in req_retry()
56 qp->req.psn = qp->comp.psn; in req_retry()
57 qp->req.opcode = -1; in req_retry()
61 wqe = queue_addr_from_index(qp->sq.queue, wqe_index); in req_retry()
62 mask = wr_opcode_mask(wqe->wr.opcode, qp); in req_retry()
86 npsn = (qp->comp.psn - wqe->first_psn) & in req_retry()
88 retry_first_write_send(qp, wqe, mask, npsn); in req_retry()
93 qp->mtu; in req_retry()
94 wqe->iova += npsn * qp->mtu; in req_retry()
104 struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer); in rnr_nak_timer() local
106 pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp)); in rnr_nak_timer()
107 rxe_run_task(&qp->req.task, 1); in rnr_nak_timer()
110 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) in req_next_wqe() argument
114 struct rxe_queue *q = qp->sq.queue; in req_next_wqe()
115 unsigned int index = qp->req.wqe_index; in req_next_wqe()
123 if (unlikely(qp->req.state == QP_STATE_DRAIN)) { in req_next_wqe()
127 spin_lock_irqsave(&qp->state_lock, flags); in req_next_wqe()
129 if (qp->req.state != QP_STATE_DRAIN) { in req_next_wqe()
131 spin_unlock_irqrestore(&qp->state_lock, in req_next_wqe()
139 spin_unlock_irqrestore(&qp->state_lock, in req_next_wqe()
144 qp->req.state = QP_STATE_DRAINED; in req_next_wqe()
145 spin_unlock_irqrestore(&qp->state_lock, flags); in req_next_wqe()
147 if (qp->ibqp.event_handler) { in req_next_wqe()
150 ev.device = qp->ibqp.device; in req_next_wqe()
151 ev.element.qp = &qp->ibqp; in req_next_wqe()
153 qp->ibqp.event_handler(&ev, in req_next_wqe()
154 qp->ibqp.qp_context); in req_next_wqe()
164 if (unlikely((qp->req.state == QP_STATE_DRAIN || in req_next_wqe()
165 qp->req.state == QP_STATE_DRAINED) && in req_next_wqe()
171 qp->req.wait_fence = 1; in req_next_wqe()
175 wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp); in req_next_wqe()
179 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits) in next_opcode_rc() argument
183 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || in next_opcode_rc()
184 qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) in next_opcode_rc()
194 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || in next_opcode_rc()
195 qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) in next_opcode_rc()
205 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
206 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
216 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
217 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
236 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
237 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
251 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits) in next_opcode_uc() argument
255 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || in next_opcode_uc()
256 qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) in next_opcode_uc()
266 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || in next_opcode_uc()
267 qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) in next_opcode_uc()
277 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || in next_opcode_uc()
278 qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) in next_opcode_uc()
288 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || in next_opcode_uc()
289 qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) in next_opcode_uc()
302 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in next_opcode() argument
305 int fits = (wqe->dma.resid <= qp->mtu); in next_opcode()
307 switch (qp_type(qp)) { in next_opcode()
309 return next_opcode_rc(qp, opcode, fits); in next_opcode()
312 return next_opcode_uc(qp, opcode, fits); in next_opcode()
333 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe) in check_init_depth() argument
340 qp->req.need_rd_atomic = 1; in check_init_depth()
341 depth = atomic_dec_return(&qp->req.rd_atomic); in check_init_depth()
344 qp->req.need_rd_atomic = 0; in check_init_depth()
349 atomic_inc(&qp->req.rd_atomic); in check_init_depth()
353 static inline int get_mtu(struct rxe_qp *qp) in get_mtu() argument
355 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in get_mtu()
357 if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC)) in get_mtu()
358 return qp->mtu; in get_mtu()
363 static struct sk_buff *init_req_packet(struct rxe_qp *qp, in init_req_packet() argument
368 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in init_req_packet()
385 pkt->qp = qp; in init_req_packet()
386 pkt->psn = qp->req.psn; in init_req_packet()
410 qp->attr.dest_qp_num; in init_req_packet()
413 (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK)); in init_req_packet()
415 qp->req.noack_pkts = 0; in init_req_packet()
446 if (qp->ibqp.qp_num == 1) in init_req_packet()
450 deth_set_sqp(pkt, qp->ibqp.qp_num); in init_req_packet()
456 static int finish_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in finish_packet() argument
475 err = copy_data(qp->pd, 0, &wqe->dma, in finish_packet()
491 static void update_wqe_state(struct rxe_qp *qp, in update_wqe_state() argument
496 if (qp_type(qp) == IB_QPT_RC) in update_wqe_state()
503 static void update_wqe_psn(struct rxe_qp *qp, in update_wqe_psn() argument
509 int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu; in update_wqe_psn()
516 wqe->first_psn = qp->req.psn; in update_wqe_psn()
517 wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK; in update_wqe_psn()
521 qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK; in update_wqe_psn()
523 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; in update_wqe_psn()
527 struct rxe_qp *qp, in save_state() argument
534 *rollback_psn = qp->req.psn; in save_state()
538 struct rxe_qp *qp, in rollback_state() argument
545 qp->req.psn = rollback_psn; in rollback_state()
548 static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in update_state() argument
551 qp->req.opcode = pkt->opcode; in update_state()
554 qp->req.wqe_index = queue_next_index(qp->sq.queue, in update_state()
555 qp->req.wqe_index); in update_state()
557 qp->need_req_skb = 0; in update_state()
559 if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer)) in update_state()
560 mod_timer(&qp->retrans_timer, in update_state()
561 jiffies + qp->qp_timeout_jiffies); in update_state()
564 static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) in rxe_do_local_ops() argument
574 ret = rxe_invalidate_mw(qp, rkey); in rxe_do_local_ops()
576 ret = rxe_invalidate_mr(qp, rkey); in rxe_do_local_ops()
584 ret = rxe_reg_fast_mr(qp, wqe); in rxe_do_local_ops()
591 ret = rxe_bind_mw(qp, wqe); in rxe_do_local_ops()
605 qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index); in rxe_do_local_ops()
608 qp->sq_sig_type == IB_SIGNAL_ALL_WR) in rxe_do_local_ops()
609 rxe_run_task(&qp->comp.task, 1); in rxe_do_local_ops()
616 struct rxe_qp *qp = (struct rxe_qp *)arg; in rxe_requester() local
627 struct rxe_queue *q = qp->sq.queue; in rxe_requester()
629 rxe_add_ref(qp); in rxe_requester()
632 if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR)) in rxe_requester()
635 if (unlikely(qp->req.state == QP_STATE_RESET)) { in rxe_requester()
636 qp->req.wqe_index = queue_get_consumer(q, in rxe_requester()
638 qp->req.opcode = -1; in rxe_requester()
639 qp->req.need_rd_atomic = 0; in rxe_requester()
640 qp->req.wait_psn = 0; in rxe_requester()
641 qp->req.need_retry = 0; in rxe_requester()
645 if (unlikely(qp->req.need_retry)) { in rxe_requester()
646 req_retry(qp); in rxe_requester()
647 qp->req.need_retry = 0; in rxe_requester()
650 wqe = req_next_wqe(qp); in rxe_requester()
655 ret = rxe_do_local_ops(qp, wqe); in rxe_requester()
662 if (unlikely(qp_type(qp) == IB_QPT_RC && in rxe_requester()
663 psn_compare(qp->req.psn, (qp->comp.psn + in rxe_requester()
665 qp->req.wait_psn = 1; in rxe_requester()
670 if (unlikely(atomic_read(&qp->skb_out) > in rxe_requester()
672 qp->need_req_skb = 1; in rxe_requester()
676 opcode = next_opcode(qp, wqe, wqe->wr.opcode); in rxe_requester()
684 if (check_init_depth(qp, wqe)) in rxe_requester()
688 mtu = get_mtu(qp); in rxe_requester()
691 if (qp_type(qp) == IB_QPT_UD) { in rxe_requester()
699 wqe->first_psn = qp->req.psn; in rxe_requester()
700 wqe->last_psn = qp->req.psn; in rxe_requester()
701 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; in rxe_requester()
702 qp->req.opcode = IB_OPCODE_UD_SEND_ONLY; in rxe_requester()
703 qp->req.wqe_index = queue_next_index(qp->sq.queue, in rxe_requester()
704 qp->req.wqe_index); in rxe_requester()
707 __rxe_do_task(&qp->comp.task); in rxe_requester()
708 rxe_drop_ref(qp); in rxe_requester()
714 skb = init_req_packet(qp, wqe, opcode, payload, &pkt); in rxe_requester()
716 pr_err("qp#%d Failed allocating skb\n", qp_num(qp)); in rxe_requester()
721 ret = finish_packet(qp, wqe, &pkt, skb, payload); in rxe_requester()
723 pr_debug("qp#%d Error during finish packet\n", qp_num(qp)); in rxe_requester()
738 save_state(wqe, qp, &rollback_wqe, &rollback_psn); in rxe_requester()
739 update_wqe_state(qp, wqe, &pkt); in rxe_requester()
740 update_wqe_psn(qp, wqe, &pkt, payload); in rxe_requester()
741 ret = rxe_xmit_packet(qp, &pkt, skb); in rxe_requester()
743 qp->need_req_skb = 1; in rxe_requester()
745 rollback_state(wqe, qp, &rollback_wqe, rollback_psn); in rxe_requester()
748 rxe_run_task(&qp->req.task, 1); in rxe_requester()
756 update_state(qp, wqe, &pkt, payload); in rxe_requester()
762 __rxe_do_task(&qp->comp.task); in rxe_requester()
765 rxe_drop_ref(qp); in rxe_requester()