Lines Matching refs:iwqp
229 static void irdma_alloc_push_page(struct irdma_qp *iwqp) in irdma_alloc_push_page() argument
233 struct irdma_device *iwdev = iwqp->iwdev; in irdma_alloc_push_page()
234 struct irdma_sc_qp *qp = &iwqp->sc_qp; in irdma_alloc_push_page()
452 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) in irdma_clean_cqes() argument
458 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); in irdma_clean_cqes()
462 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) in irdma_remove_push_mmap_entries() argument
464 if (iwqp->push_db_mmap_entry) { in irdma_remove_push_mmap_entries()
465 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); in irdma_remove_push_mmap_entries()
466 iwqp->push_db_mmap_entry = NULL; in irdma_remove_push_mmap_entries()
468 if (iwqp->push_wqe_mmap_entry) { in irdma_remove_push_mmap_entries()
469 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); in irdma_remove_push_mmap_entries()
470 iwqp->push_wqe_mmap_entry = NULL; in irdma_remove_push_mmap_entries()
475 struct irdma_qp *iwqp, in irdma_setup_push_mmap_entries() argument
487 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; in irdma_setup_push_mmap_entries()
488 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, in irdma_setup_push_mmap_entries()
491 if (!iwqp->push_wqe_mmap_entry) in irdma_setup_push_mmap_entries()
496 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, in irdma_setup_push_mmap_entries()
499 if (!iwqp->push_db_mmap_entry) { in irdma_setup_push_mmap_entries()
500 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); in irdma_setup_push_mmap_entries()
514 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_destroy_qp() local
515 struct irdma_device *iwdev = iwqp->iwdev; in irdma_destroy_qp()
517 iwqp->sc_qp.qp_uk.destroy_pending = true; in irdma_destroy_qp()
519 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) in irdma_destroy_qp()
520 irdma_modify_qp_to_err(&iwqp->sc_qp); in irdma_destroy_qp()
522 if (!iwqp->user_mode) in irdma_destroy_qp()
523 cancel_delayed_work_sync(&iwqp->dwork_flush); in irdma_destroy_qp()
525 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_destroy_qp()
526 wait_for_completion(&iwqp->free_qp); in irdma_destroy_qp()
527 irdma_free_lsmm_rsrc(iwqp); in irdma_destroy_qp()
528 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); in irdma_destroy_qp()
530 if (!iwqp->user_mode) { in irdma_destroy_qp()
531 if (iwqp->iwscq) { in irdma_destroy_qp()
532 irdma_clean_cqes(iwqp, iwqp->iwscq); in irdma_destroy_qp()
533 if (iwqp->iwrcq != iwqp->iwscq) in irdma_destroy_qp()
534 irdma_clean_cqes(iwqp, iwqp->iwrcq); in irdma_destroy_qp()
537 irdma_remove_push_mmap_entries(iwqp); in irdma_destroy_qp()
538 irdma_free_qp_rsrc(iwqp); in irdma_destroy_qp()
550 struct irdma_qp *iwqp, in irdma_setup_virt_qp() argument
553 struct irdma_pbl *iwpbl = iwqp->iwpbl; in irdma_setup_virt_qp()
556 iwqp->page = qpmr->sq_page; in irdma_setup_virt_qp()
576 struct irdma_qp *iwqp, in irdma_setup_kmode_qp() argument
580 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; in irdma_setup_kmode_qp()
608 iwqp->kqp.sq_wrid_mem = in irdma_setup_kmode_qp()
609 kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); in irdma_setup_kmode_qp()
610 if (!iwqp->kqp.sq_wrid_mem) in irdma_setup_kmode_qp()
613 iwqp->kqp.rq_wrid_mem = in irdma_setup_kmode_qp()
614 kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); in irdma_setup_kmode_qp()
615 if (!iwqp->kqp.rq_wrid_mem) { in irdma_setup_kmode_qp()
616 kfree(iwqp->kqp.sq_wrid_mem); in irdma_setup_kmode_qp()
617 iwqp->kqp.sq_wrid_mem = NULL; in irdma_setup_kmode_qp()
621 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; in irdma_setup_kmode_qp()
622 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; in irdma_setup_kmode_qp()
631 kfree(iwqp->kqp.sq_wrid_mem); in irdma_setup_kmode_qp()
632 iwqp->kqp.sq_wrid_mem = NULL; in irdma_setup_kmode_qp()
633 kfree(iwqp->kqp.rq_wrid_mem); in irdma_setup_kmode_qp()
634 iwqp->kqp.rq_wrid_mem = NULL; in irdma_setup_kmode_qp()
646 ukinfo->qp_id = iwqp->ibqp.qp_num; in irdma_setup_kmode_qp()
654 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) in irdma_cqp_create_qp_cmd() argument
656 struct irdma_pci_f *rf = iwqp->iwdev->rf; in irdma_cqp_create_qp_cmd()
675 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; in irdma_cqp_create_qp_cmd()
683 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, in irdma_roce_fill_and_set_qpctx_info() argument
686 struct irdma_device *iwdev = iwqp->iwdev; in irdma_roce_fill_and_set_qpctx_info()
691 udp_info = &iwqp->udp_info; in irdma_roce_fill_and_set_qpctx_info()
698 roce_info = &iwqp->roce_info; in irdma_roce_fill_and_set_qpctx_info()
711 if (!iwqp->user_mode) { in irdma_roce_fill_and_set_qpctx_info()
718 ctx_info->roce_info = &iwqp->roce_info; in irdma_roce_fill_and_set_qpctx_info()
719 ctx_info->udp_info = &iwqp->udp_info; in irdma_roce_fill_and_set_qpctx_info()
720 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); in irdma_roce_fill_and_set_qpctx_info()
723 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, in irdma_iw_fill_and_set_qpctx_info() argument
726 struct irdma_device *iwdev = iwqp->iwdev; in irdma_iw_fill_and_set_qpctx_info()
730 iwarp_info = &iwqp->iwarp_info; in irdma_iw_fill_and_set_qpctx_info()
740 if (!iwqp->user_mode) { in irdma_iw_fill_and_set_qpctx_info()
747 ctx_info->iwarp_info = &iwqp->iwarp_info; in irdma_iw_fill_and_set_qpctx_info()
749 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); in irdma_iw_fill_and_set_qpctx_info()
783 struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); in irdma_flush_worker() local
785 irdma_generate_flush_completions(iwqp); in irdma_flush_worker()
804 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_create_qp() local
837 qp = &iwqp->sc_qp; in irdma_create_qp()
838 qp->qp_uk.back_qp = iwqp; in irdma_create_qp()
841 iwqp->iwdev = iwdev; in irdma_create_qp()
842 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, in irdma_create_qp()
844 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, in irdma_create_qp()
845 iwqp->q2_ctx_mem.size, in irdma_create_qp()
846 &iwqp->q2_ctx_mem.pa, in irdma_create_qp()
848 if (!iwqp->q2_ctx_mem.va) in irdma_create_qp()
851 init_info.q2 = iwqp->q2_ctx_mem.va; in irdma_create_qp()
852 init_info.q2_pa = iwqp->q2_ctx_mem.pa; in irdma_create_qp()
864 iwqp->iwpd = iwpd; in irdma_create_qp()
865 iwqp->ibqp.qp_num = qp_num; in irdma_create_qp()
866 qp = &iwqp->sc_qp; in irdma_create_qp()
867 iwqp->iwscq = to_iwcq(init_attr->send_cq); in irdma_create_qp()
868 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); in irdma_create_qp()
869 iwqp->host_ctx.va = init_info.host_ctx; in irdma_create_qp()
870 iwqp->host_ctx.pa = init_info.host_ctx_pa; in irdma_create_qp()
871 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; in irdma_create_qp()
874 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; in irdma_create_qp()
877 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; in irdma_create_qp()
878 init_waitqueue_head(&iwqp->waitq); in irdma_create_qp()
879 init_waitqueue_head(&iwqp->mod_qp_waitq); in irdma_create_qp()
890 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; in irdma_create_qp()
891 iwqp->user_mode = 1; in irdma_create_qp()
900 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, in irdma_create_qp()
904 if (!iwqp->iwpbl) { in irdma_create_qp()
911 irdma_setup_virt_qp(iwdev, iwqp, &init_info); in irdma_create_qp()
913 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); in irdma_create_qp()
915 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); in irdma_create_qp()
948 ctx_info = &iwqp->ctx_info; in irdma_create_qp()
949 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; in irdma_create_qp()
950 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; in irdma_create_qp()
953 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); in irdma_create_qp()
955 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); in irdma_create_qp()
957 err_code = irdma_cqp_create_qp_cmd(iwqp); in irdma_create_qp()
961 refcount_set(&iwqp->refcnt, 1); in irdma_create_qp()
962 spin_lock_init(&iwqp->lock); in irdma_create_qp()
963 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); in irdma_create_qp()
964 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; in irdma_create_qp()
965 rf->qp_table[qp_num] = iwqp; in irdma_create_qp()
966 iwqp->max_send_wr = sq_size; in irdma_create_qp()
967 iwqp->max_recv_wr = rq_size; in irdma_create_qp()
971 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); in irdma_create_qp()
976 irdma_qp_add_qos(&iwqp->sc_qp); in irdma_create_qp()
997 irdma_destroy_qp(&iwqp->ibqp, udata); in irdma_create_qp()
1002 init_completion(&iwqp->free_qp); in irdma_create_qp()
1006 irdma_free_qp_rsrc(iwqp); in irdma_create_qp()
1010 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) in irdma_get_ib_acc_flags() argument
1014 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { in irdma_get_ib_acc_flags()
1015 if (iwqp->roce_info.wr_rdresp_en) { in irdma_get_ib_acc_flags()
1019 if (iwqp->roce_info.rd_en) in irdma_get_ib_acc_flags()
1021 if (iwqp->roce_info.bind_en) in irdma_get_ib_acc_flags()
1024 if (iwqp->iwarp_info.wr_rdresp_en) { in irdma_get_ib_acc_flags()
1028 if (iwqp->iwarp_info.rd_en) in irdma_get_ib_acc_flags()
1030 if (iwqp->iwarp_info.bind_en) in irdma_get_ib_acc_flags()
1046 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_query_qp() local
1047 struct irdma_sc_qp *qp = &iwqp->sc_qp; in irdma_query_qp()
1052 attr->qp_state = iwqp->ibqp_state; in irdma_query_qp()
1053 attr->cur_qp_state = iwqp->ibqp_state; in irdma_query_qp()
1054 attr->cap.max_send_wr = iwqp->max_send_wr; in irdma_query_qp()
1055 attr->cap.max_recv_wr = iwqp->max_recv_wr; in irdma_query_qp()
1059 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); in irdma_query_qp()
1062 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); in irdma_query_qp()
1063 attr->qkey = iwqp->roce_info.qkey; in irdma_query_qp()
1064 attr->rq_psn = iwqp->udp_info.epsn; in irdma_query_qp()
1065 attr->sq_psn = iwqp->udp_info.psn_nxt; in irdma_query_qp()
1066 attr->dest_qp_num = iwqp->roce_info.dest_qp; in irdma_query_qp()
1067 attr->pkey_index = iwqp->roce_info.p_key; in irdma_query_qp()
1068 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; in irdma_query_qp()
1069 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; in irdma_query_qp()
1070 attr->max_rd_atomic = iwqp->roce_info.ord_size; in irdma_query_qp()
1071 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; in irdma_query_qp()
1074 init_attr->event_handler = iwqp->ibqp.event_handler; in irdma_query_qp()
1075 init_attr->qp_context = iwqp->ibqp.qp_context; in irdma_query_qp()
1076 init_attr->send_cq = iwqp->ibqp.send_cq; in irdma_query_qp()
1077 init_attr->recv_cq = iwqp->ibqp.recv_cq; in irdma_query_qp()
1113 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_modify_qp_roce() local
1114 struct irdma_device *iwdev = iwqp->iwdev; in irdma_modify_qp_roce()
1126 ctx_info = &iwqp->ctx_info; in irdma_modify_qp_roce()
1127 roce_info = &iwqp->roce_info; in irdma_modify_qp_roce()
1128 udp_info = &iwqp->udp_info; in irdma_modify_qp_roce()
1175 struct irdma_av *av = &iwqp->roce_ah.av; in irdma_modify_qp_roce()
1180 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); in irdma_modify_qp_roce()
1189 irdma_qp_rem_qos(&iwqp->sc_qp); in irdma_modify_qp_roce()
1190 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); in irdma_modify_qp_roce()
1192 iwqp->sc_qp.user_pri = ctx_info->user_pri; in irdma_modify_qp_roce()
1193 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) in irdma_modify_qp_roce()
1195 irdma_qp_add_qos(&iwqp->sc_qp); in irdma_modify_qp_roce()
1289 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); in irdma_modify_qp_roce()
1294 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); in irdma_modify_qp_roce()
1296 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp_roce()
1298 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, in irdma_modify_qp_roce()
1299 iwqp->ibqp.qp_type, attr_mask)) { in irdma_modify_qp_roce()
1301 iwqp->ibqp.qp_num, iwqp->ibqp_state, in irdma_modify_qp_roce()
1306 info.curr_iwarp_state = iwqp->iwarp_state; in irdma_modify_qp_roce()
1310 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { in irdma_modify_qp_roce()
1315 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { in irdma_modify_qp_roce()
1321 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { in irdma_modify_qp_roce()
1331 if (iwqp->ibqp_state < IB_QPS_RTR || in irdma_modify_qp_roce()
1332 iwqp->ibqp_state == IB_QPS_ERR) { in irdma_modify_qp_roce()
1343 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && in irdma_modify_qp_roce()
1345 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1346 irdma_alloc_push_page(iwqp); in irdma_modify_qp_roce()
1347 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp_roce()
1351 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) in irdma_modify_qp_roce()
1354 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { in irdma_modify_qp_roce()
1365 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { in irdma_modify_qp_roce()
1366 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1368 irdma_hw_modify_qp(iwdev, iwqp, &info, true); in irdma_modify_qp_roce()
1369 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp_roce()
1372 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { in irdma_modify_qp_roce()
1373 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1379 irdma_flush_wqes(iwqp, in irdma_modify_qp_roce()
1395 iwqp->ibqp_state = attr->qp_state; in irdma_modify_qp_roce()
1398 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; in irdma_modify_qp_roce()
1399 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; in irdma_modify_qp_roce()
1400 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); in irdma_modify_qp_roce()
1401 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1406 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) in irdma_modify_qp_roce()
1408 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp_roce()
1409 if (iwqp->iwarp_state == info.curr_iwarp_state) { in irdma_modify_qp_roce()
1410 iwqp->iwarp_state = info.next_iwarp_state; in irdma_modify_qp_roce()
1411 iwqp->ibqp_state = attr->qp_state; in irdma_modify_qp_roce()
1413 if (iwqp->ibqp_state > IB_QPS_RTS && in irdma_modify_qp_roce()
1414 !iwqp->flush_issued) { in irdma_modify_qp_roce()
1415 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1416 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | in irdma_modify_qp_roce()
1419 iwqp->flush_issued = 1; in irdma_modify_qp_roce()
1421 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1424 iwqp->ibqp_state = attr->qp_state; in irdma_modify_qp_roce()
1431 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && in irdma_modify_qp_roce()
1432 !iwqp->push_wqe_mmap_entry && in irdma_modify_qp_roce()
1433 !irdma_setup_push_mmap_entries(ucontext, iwqp, in irdma_modify_qp_roce()
1436 uresp.push_offset = iwqp->sc_qp.push_offset; in irdma_modify_qp_roce()
1441 irdma_remove_push_mmap_entries(iwqp); in irdma_modify_qp_roce()
1451 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp_roce()
1468 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_modify_qp() local
1469 struct irdma_device *iwdev = iwqp->iwdev; in irdma_modify_qp()
1492 ctx_info = &iwqp->ctx_info; in irdma_modify_qp()
1493 offload_info = &iwqp->iwarp_info; in irdma_modify_qp()
1494 tcp_info = &iwqp->tcp_info; in irdma_modify_qp()
1495 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); in irdma_modify_qp()
1499 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, in irdma_modify_qp()
1500 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); in irdma_modify_qp()
1502 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp()
1504 info.curr_iwarp_state = iwqp->iwarp_state; in irdma_modify_qp()
1508 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { in irdma_modify_qp()
1513 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { in irdma_modify_qp()
1518 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && in irdma_modify_qp()
1520 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
1521 irdma_alloc_push_page(iwqp); in irdma_modify_qp()
1522 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp()
1526 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || in irdma_modify_qp()
1527 !iwqp->cm_id) { in irdma_modify_qp()
1533 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; in irdma_modify_qp()
1534 iwqp->hte_added = 1; in irdma_modify_qp()
1542 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { in irdma_modify_qp()
1547 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || in irdma_modify_qp()
1548 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { in irdma_modify_qp()
1553 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { in irdma_modify_qp()
1562 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { in irdma_modify_qp()
1572 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { in irdma_modify_qp()
1573 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
1579 irdma_flush_wqes(iwqp, in irdma_modify_qp()
1587 if (iwqp->sc_qp.term_flags) { in irdma_modify_qp()
1588 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
1589 irdma_terminate_del_timer(&iwqp->sc_qp); in irdma_modify_qp()
1590 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp()
1593 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && in irdma_modify_qp()
1595 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) in irdma_modify_qp()
1608 iwqp->ibqp_state = attr->qp_state; in irdma_modify_qp()
1621 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; in irdma_modify_qp()
1622 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; in irdma_modify_qp()
1623 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); in irdma_modify_qp()
1625 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
1630 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) in irdma_modify_qp()
1634 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp()
1635 if (iwqp->iwarp_state == info.curr_iwarp_state) { in irdma_modify_qp()
1636 iwqp->iwarp_state = info.next_iwarp_state; in irdma_modify_qp()
1637 iwqp->ibqp_state = attr->qp_state; in irdma_modify_qp()
1639 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
1642 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { in irdma_modify_qp()
1644 if (iwqp->hw_tcp_state) { in irdma_modify_qp()
1645 spin_lock_irqsave(&iwqp->lock, flags); in irdma_modify_qp()
1646 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; in irdma_modify_qp()
1647 iwqp->last_aeq = IRDMA_AE_RESET_SENT; in irdma_modify_qp()
1648 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
1650 irdma_cm_disconn(iwqp); in irdma_modify_qp()
1656 if (iwqp->cm_node) { in irdma_modify_qp()
1657 refcount_inc(&iwqp->cm_node->refcnt); in irdma_modify_qp()
1659 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); in irdma_modify_qp()
1660 if (iwqp->cm_id && close_timer_started == 1) in irdma_modify_qp()
1661 irdma_schedule_cm_timer(iwqp->cm_node, in irdma_modify_qp()
1662 (struct irdma_puda_buf *)iwqp, in irdma_modify_qp()
1665 irdma_rem_ref_cm_node(iwqp->cm_node); in irdma_modify_qp()
1677 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && in irdma_modify_qp()
1678 !iwqp->push_wqe_mmap_entry && in irdma_modify_qp()
1679 !irdma_setup_push_mmap_entries(ucontext, iwqp, in irdma_modify_qp()
1682 uresp.push_offset = iwqp->sc_qp.push_offset; in irdma_modify_qp()
1688 irdma_remove_push_mmap_entries(iwqp); in irdma_modify_qp()
1697 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_modify_qp()
3193 struct irdma_qp *iwqp; in irdma_post_send() local
3202 iwqp = to_iwqp(ibqp); in irdma_post_send()
3203 ukqp = &iwqp->sc_qp.qp_uk; in irdma_post_send()
3204 dev = &iwqp->iwdev->rf->sc_dev; in irdma_post_send()
3206 spin_lock_irqsave(&iwqp->lock, flags); in irdma_post_send()
3211 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) in irdma_post_send()
3243 if (iwqp->ibqp.qp_type == IB_QPT_UD || in irdma_post_send()
3244 iwqp->ibqp.qp_type == IB_QPT_GSI) { in irdma_post_send()
3322 err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, in irdma_post_send()
3328 ibdev_dbg(&iwqp->iwdev->ibdev, in irdma_post_send()
3339 if (!iwqp->flush_issued) { in irdma_post_send()
3340 if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) in irdma_post_send()
3342 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_post_send()
3344 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_post_send()
3345 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, in irdma_post_send()
3364 struct irdma_qp *iwqp; in irdma_post_recv() local
3370 iwqp = to_iwqp(ibqp); in irdma_post_recv()
3371 ukqp = &iwqp->sc_qp.qp_uk; in irdma_post_recv()
3373 spin_lock_irqsave(&iwqp->lock, flags); in irdma_post_recv()
3380 ibdev_dbg(&iwqp->iwdev->ibdev, in irdma_post_recv()
3389 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_post_recv()
3390 if (iwqp->flush_issued) in irdma_post_recv()
3391 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, in irdma_post_recv()
3980 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_attach_mcast() local
3981 struct irdma_device *iwdev = iwqp->iwdev; in irdma_attach_mcast()
4059 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; in irdma_attach_mcast()
4072 mcg_info.qp_id = iwqp->ibqp.qp_num; in irdma_attach_mcast()
4120 struct irdma_qp *iwqp = to_iwqp(ibqp); in irdma_detach_mcast() local
4121 struct irdma_device *iwdev = iwqp->iwdev; in irdma_detach_mcast()
4146 mcg_info.qp_id = iwqp->ibqp.qp_num; in irdma_detach_mcast()