Lines Matching refs:qp
95 struct siw_qp *qp; in siw_qp_llp_data_ready() local
102 qp = sk_to_qp(sk); in siw_qp_llp_data_ready()
104 if (likely(!qp->rx_stream.rx_suspend && in siw_qp_llp_data_ready()
105 down_read_trylock(&qp->state_lock))) { in siw_qp_llp_data_ready()
106 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 }; in siw_qp_llp_data_ready()
108 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) in siw_qp_llp_data_ready()
117 up_read(&qp->state_lock); in siw_qp_llp_data_ready()
119 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n", in siw_qp_llp_data_ready()
120 qp->rx_stream.rx_suspend); in siw_qp_llp_data_ready()
126 void siw_qp_llp_close(struct siw_qp *qp) in siw_qp_llp_close() argument
128 siw_dbg_qp(qp, "enter llp close, state = %s\n", in siw_qp_llp_close()
129 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
131 down_write(&qp->state_lock); in siw_qp_llp_close()
133 qp->rx_stream.rx_suspend = 1; in siw_qp_llp_close()
134 qp->tx_ctx.tx_suspend = 1; in siw_qp_llp_close()
135 qp->attrs.sk = NULL; in siw_qp_llp_close()
137 switch (qp->attrs.state) { in siw_qp_llp_close()
142 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
151 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) in siw_qp_llp_close()
152 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
154 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_llp_close()
158 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n", in siw_qp_llp_close()
159 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
162 siw_sq_flush(qp); in siw_qp_llp_close()
163 siw_rq_flush(qp); in siw_qp_llp_close()
168 if (qp->cep) { in siw_qp_llp_close()
169 siw_cep_put(qp->cep); in siw_qp_llp_close()
170 qp->cep = NULL; in siw_qp_llp_close()
173 up_write(&qp->state_lock); in siw_qp_llp_close()
175 siw_dbg_qp(qp, "llp close exit: state %s\n", in siw_qp_llp_close()
176 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
194 (void)siw_sq_start(cep->qp); in siw_qp_llp_write_space()
200 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size) in siw_qp_readq_init() argument
204 qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe)); in siw_qp_readq_init()
205 if (!qp->irq) { in siw_qp_readq_init()
206 qp->attrs.irq_size = 0; in siw_qp_readq_init()
212 qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe)); in siw_qp_readq_init()
213 if (!qp->orq) { in siw_qp_readq_init()
214 qp->attrs.orq_size = 0; in siw_qp_readq_init()
215 qp->attrs.irq_size = 0; in siw_qp_readq_init()
216 vfree(qp->irq); in siw_qp_readq_init()
220 qp->attrs.irq_size = irq_size; in siw_qp_readq_init()
221 qp->attrs.orq_size = orq_size; in siw_qp_readq_init()
222 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size); in siw_qp_readq_init()
226 static int siw_qp_enable_crc(struct siw_qp *qp) in siw_qp_enable_crc() argument
228 struct siw_rx_stream *c_rx = &qp->rx_stream; in siw_qp_enable_crc()
229 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_qp_enable_crc()
260 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl) in siw_qp_mpa_rts() argument
262 struct siw_wqe *wqe = tx_wqe(qp); in siw_qp_mpa_rts()
266 spin_lock_irqsave(&qp->sq_lock, flags); in siw_qp_mpa_rts()
269 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
295 spin_lock(&qp->orq_lock); in siw_qp_mpa_rts()
297 if (qp->attrs.orq_size) in siw_qp_mpa_rts()
298 rreq = orq_get_free(qp); in siw_qp_mpa_rts()
301 qp->orq_put++; in siw_qp_mpa_rts()
305 spin_unlock(&qp->orq_lock); in siw_qp_mpa_rts()
312 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
315 rv = siw_sq_start(qp); in siw_qp_mpa_rts()
364 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype, in siw_init_terminate() argument
367 if (!qp->term_info.valid) { in siw_init_terminate()
368 memset(&qp->term_info, 0, sizeof(qp->term_info)); in siw_init_terminate()
369 qp->term_info.layer = layer; in siw_init_terminate()
370 qp->term_info.etype = etype; in siw_init_terminate()
371 qp->term_info.ecode = ecode; in siw_init_terminate()
372 qp->term_info.in_tx = in_tx; in siw_init_terminate()
373 qp->term_info.valid = 1; in siw_init_terminate()
375 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", in siw_init_terminate()
387 void siw_send_terminate(struct siw_qp *qp) in siw_send_terminate() argument
393 struct socket *s = qp->attrs.sk; in siw_send_terminate()
394 struct siw_rx_stream *srx = &qp->rx_stream; in siw_send_terminate()
399 if (!qp->term_info.valid) in siw_send_terminate()
402 qp->term_info.valid = 0; in siw_send_terminate()
404 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { in siw_send_terminate()
405 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", in siw_send_terminate()
406 tx_type(tx_wqe(qp))); in siw_send_terminate()
409 if (!s && qp->cep) in siw_send_terminate()
411 s = qp->cep->sock; in siw_send_terminate()
414 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); in siw_send_terminate()
429 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || in siw_send_terminate()
430 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && in siw_send_terminate()
431 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { in siw_send_terminate()
441 __rdmap_term_set_layer(term, qp->term_info.layer); in siw_send_terminate()
442 __rdmap_term_set_etype(term, qp->term_info.etype); in siw_send_terminate()
443 __rdmap_term_set_ecode(term, qp->term_info.ecode); in siw_send_terminate()
445 switch (qp->term_info.layer) { in siw_send_terminate()
447 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) in siw_send_terminate()
451 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { in siw_send_terminate()
460 if (qp->term_info.in_tx) { in siw_send_terminate()
462 struct siw_wqe *wqe = tx_wqe(qp); in siw_send_terminate()
512 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || in siw_send_terminate()
513 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) in siw_send_terminate()
540 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && in siw_send_terminate()
541 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || in siw_send_terminate()
542 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && in siw_send_terminate()
543 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) in siw_send_terminate()
583 if (qp->tx_ctx.mpa_crc_hd) { in siw_send_terminate()
584 crypto_shash_init(qp->tx_ctx.mpa_crc_hd); in siw_send_terminate()
585 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, in siw_send_terminate()
591 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, in siw_send_terminate()
596 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); in siw_send_terminate()
600 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", in siw_send_terminate()
612 static void siw_qp_modify_nonstate(struct siw_qp *qp, in siw_qp_modify_nonstate() argument
618 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
620 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
623 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
625 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
628 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
630 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
634 static int siw_qp_nextstate_from_idle(struct siw_qp *qp, in siw_qp_nextstate_from_idle() argument
643 rv = siw_qp_enable_crc(qp); in siw_qp_nextstate_from_idle()
648 siw_dbg_qp(qp, "no socket\n"); in siw_qp_nextstate_from_idle()
653 siw_dbg_qp(qp, "no MPA\n"); in siw_qp_nextstate_from_idle()
660 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; in siw_qp_nextstate_from_idle()
661 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; in siw_qp_nextstate_from_idle()
662 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; in siw_qp_nextstate_from_idle()
667 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; in siw_qp_nextstate_from_idle()
668 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; in siw_qp_nextstate_from_idle()
669 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; in siw_qp_nextstate_from_idle()
675 rv = siw_qp_readq_init(qp, attrs->irq_size, in siw_qp_nextstate_from_idle()
680 qp->attrs.sk = attrs->sk; in siw_qp_nextstate_from_idle()
681 qp->attrs.state = SIW_QP_STATE_RTS; in siw_qp_nextstate_from_idle()
683 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n", in siw_qp_nextstate_from_idle()
685 qp->attrs.orq_size, qp->attrs.irq_size); in siw_qp_nextstate_from_idle()
689 siw_rq_flush(qp); in siw_qp_nextstate_from_idle()
690 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_idle()
691 if (qp->cep) { in siw_qp_nextstate_from_idle()
692 siw_cep_put(qp->cep); in siw_qp_nextstate_from_idle()
693 qp->cep = NULL; in siw_qp_nextstate_from_idle()
703 static int siw_qp_nextstate_from_rts(struct siw_qp *qp, in siw_qp_nextstate_from_rts() argument
718 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { in siw_qp_nextstate_from_rts()
719 qp->attrs.state = SIW_QP_STATE_CLOSING; in siw_qp_nextstate_from_rts()
721 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
722 siw_sq_flush(qp); in siw_qp_nextstate_from_rts()
724 siw_rq_flush(qp); in siw_qp_nextstate_from_rts()
730 qp->attrs.state = SIW_QP_STATE_TERMINATE; in siw_qp_nextstate_from_rts()
732 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, in siw_qp_nextstate_from_rts()
752 siw_sq_flush(qp); in siw_qp_nextstate_from_rts()
753 siw_rq_flush(qp); in siw_qp_nextstate_from_rts()
754 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
764 static void siw_qp_nextstate_from_term(struct siw_qp *qp, in siw_qp_nextstate_from_term() argument
769 siw_rq_flush(qp); in siw_qp_nextstate_from_term()
770 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_term()
772 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_term()
773 siw_sq_flush(qp); in siw_qp_nextstate_from_term()
781 static int siw_qp_nextstate_from_close(struct siw_qp *qp, in siw_qp_nextstate_from_close() argument
788 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); in siw_qp_nextstate_from_close()
789 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_nextstate_from_close()
804 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_close()
806 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_close()
807 siw_sq_flush(qp); in siw_qp_nextstate_from_close()
809 siw_rq_flush(qp); in siw_qp_nextstate_from_close()
813 siw_dbg_qp(qp, "state transition undefined: %s => %s\n", in siw_qp_nextstate_from_close()
814 siw_qp_state_to_string[qp->attrs.state], in siw_qp_nextstate_from_close()
825 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs, in siw_qp_modify() argument
833 siw_dbg_qp(qp, "state: %s => %s\n", in siw_qp_modify()
834 siw_qp_state_to_string[qp->attrs.state], in siw_qp_modify()
838 siw_qp_modify_nonstate(qp, attrs, mask); in siw_qp_modify()
843 switch (qp->attrs.state) { in siw_qp_modify()
846 rv = siw_qp_nextstate_from_idle(qp, attrs, mask); in siw_qp_modify()
850 drop_conn = siw_qp_nextstate_from_rts(qp, attrs); in siw_qp_modify()
854 siw_qp_nextstate_from_term(qp, attrs); in siw_qp_modify()
858 siw_qp_nextstate_from_close(qp, attrs); in siw_qp_modify()
864 siw_qp_cm_drop(qp, 0); in siw_qp_modify()
881 static int siw_activate_tx_from_sq(struct siw_qp *qp) in siw_activate_tx_from_sq() argument
884 struct siw_wqe *wqe = tx_wqe(qp); in siw_activate_tx_from_sq()
887 sqe = sq_get_next(qp); in siw_activate_tx_from_sq()
920 siw_dbg_qp(qp, "cannot fence read\n"); in siw_activate_tx_from_sq()
924 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
926 if (qp->attrs.orq_size && !siw_orq_empty(qp)) { in siw_activate_tx_from_sq()
927 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
930 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
936 if (unlikely(!qp->attrs.orq_size)) { in siw_activate_tx_from_sq()
943 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
945 rreq = orq_get_free(qp); in siw_activate_tx_from_sq()
952 qp->orq_put++; in siw_activate_tx_from_sq()
954 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
957 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
962 qp->sq_get++; in siw_activate_tx_from_sq()
965 siw_dbg_qp(qp, "error %d\n", rv); in siw_activate_tx_from_sq()
977 int siw_activate_tx(struct siw_qp *qp) in siw_activate_tx() argument
980 struct siw_wqe *wqe = tx_wqe(qp); in siw_activate_tx()
982 if (!qp->attrs.irq_size) in siw_activate_tx()
983 return siw_activate_tx_from_sq(qp); in siw_activate_tx()
985 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; in siw_activate_tx()
988 return siw_activate_tx_from_sq(qp); in siw_activate_tx()
994 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { in siw_activate_tx()
995 qp->irq_burst = 0; in siw_activate_tx()
996 return siw_activate_tx_from_sq(qp); in siw_activate_tx()
1022 qp->irq_get++; in siw_activate_tx()
1060 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, in siw_sqe_complete() argument
1063 struct siw_cq *cq = qp->scq; in siw_sqe_complete()
1087 cqe->base_qp = &qp->base_qp; in siw_sqe_complete()
1089 cqe->qp_id = qp_id(qp); in siw_sqe_complete()
1118 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, in siw_rqe_complete() argument
1121 struct siw_cq *cq = qp->rcq; in siw_rqe_complete()
1145 cqe->base_qp = &qp->base_qp; in siw_rqe_complete()
1151 cqe->qp_id = qp_id(qp); in siw_rqe_complete()
1188 void siw_sq_flush(struct siw_qp *qp) in siw_sq_flush() argument
1191 struct siw_wqe *wqe = tx_wqe(qp); in siw_sq_flush()
1197 while (qp->attrs.orq_size) { in siw_sq_flush()
1198 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; in siw_sq_flush()
1202 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) in siw_sq_flush()
1206 qp->orq_get++; in siw_sq_flush()
1212 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", in siw_sq_flush()
1225 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, in siw_sq_flush()
1233 while (qp->attrs.sq_size) { in siw_sq_flush()
1234 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; in siw_sq_flush()
1239 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) in siw_sq_flush()
1247 qp->sq_get++; in siw_sq_flush()
1250 siw_qp_event(qp, IB_EVENT_SQ_DRAINED); in siw_sq_flush()
1264 void siw_rq_flush(struct siw_qp *qp) in siw_rq_flush() argument
1266 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; in siw_rq_flush()
1272 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", in siw_rq_flush()
1278 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, in siw_rq_flush()
1283 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); in siw_rq_flush()
1287 wqe = &qp->rx_tagged.wqe_active; in siw_rq_flush()
1296 while (qp->attrs.rq_size) { in siw_rq_flush()
1298 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; in siw_rq_flush()
1303 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0) in siw_rq_flush()
1307 qp->rq_get++; in siw_rq_flush()
1311 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) in siw_qp_add() argument
1313 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, in siw_qp_add()
1317 kref_init(&qp->ref); in siw_qp_add()
1318 qp->sdev = sdev; in siw_qp_add()
1319 siw_dbg_qp(qp, "new QP\n"); in siw_qp_add()
1326 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); in siw_free_qp() local
1327 struct siw_device *sdev = qp->sdev; in siw_free_qp()
1330 if (qp->cep) in siw_free_qp()
1331 siw_cep_put(qp->cep); in siw_free_qp()
1333 found = xa_erase(&sdev->qp_xa, qp_id(qp)); in siw_free_qp()
1334 WARN_ON(found != qp); in siw_free_qp()
1336 list_del(&qp->devq); in siw_free_qp()
1339 vfree(qp->sendq); in siw_free_qp()
1340 vfree(qp->recvq); in siw_free_qp()
1341 vfree(qp->irq); in siw_free_qp()
1342 vfree(qp->orq); in siw_free_qp()
1344 siw_put_tx_cpu(qp->tx_cpu); in siw_free_qp()