Lines Matching refs:hwq
76 dev_dbg(&scq->hwq.pdev->dev, in __bnxt_qplib_add_flush_qp()
84 dev_dbg(&rcq->hwq.pdev->dev, in __bnxt_qplib_add_flush_qp()
143 qp->sq.hwq.prod = 0; in bnxt_qplib_clean_qp()
144 qp->sq.hwq.cons = 0; in bnxt_qplib_clean_qp()
146 qp->rq.hwq.prod = 0; in bnxt_qplib_clean_qp()
147 qp->rq.hwq.cons = 0; in bnxt_qplib_clean_qp()
237 struct bnxt_qplib_hwq *hwq = &nq->hwq; in clean_nq() local
244 spin_lock_bh(&hwq->lock); in clean_nq()
246 raw_cons = hwq->cons; in clean_nq()
248 sw_cons = HWQ_CMP(raw_cons, hwq); in clean_nq()
249 nq_ptr = (struct nq_base **)hwq->pbl_ptr; in clean_nq()
251 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) in clean_nq()
281 spin_unlock_bh(&hwq->lock); in clean_nq()
302 struct bnxt_qplib_hwq *hwq = &nq->hwq; in bnxt_qplib_service_nq() local
312 spin_lock_bh(&hwq->lock); in bnxt_qplib_service_nq()
314 raw_cons = hwq->cons; in bnxt_qplib_service_nq()
316 sw_cons = HWQ_CMP(raw_cons, hwq); in bnxt_qplib_service_nq()
317 nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL); in bnxt_qplib_service_nq()
318 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) in bnxt_qplib_service_nq()
383 if (hwq->cons != raw_cons) { in bnxt_qplib_service_nq()
384 hwq->cons = raw_cons; in bnxt_qplib_service_nq()
387 spin_unlock_bh(&hwq->lock); in bnxt_qplib_service_nq()
393 struct bnxt_qplib_hwq *hwq = &nq->hwq; in bnxt_qplib_nq_irq() local
397 sw_cons = HWQ_CMP(hwq->cons, hwq); in bnxt_qplib_nq_irq()
398 prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL)); in bnxt_qplib_nq_irq()
506 nq_db->dbinfo.hwq = &nq->hwq; in bnxt_qplib_map_nq_db()
547 if (nq->hwq.max_elements) { in bnxt_qplib_free_nq()
548 bnxt_qplib_free_hwq(nq->res, &nq->hwq); in bnxt_qplib_free_nq()
549 nq->hwq.max_elements = 0; in bnxt_qplib_free_nq()
560 if (!nq->hwq.max_elements || in bnxt_qplib_alloc_nq()
561 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) in bnxt_qplib_alloc_nq()
562 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; in bnxt_qplib_alloc_nq()
568 hwq_attr.depth = nq->hwq.max_elements; in bnxt_qplib_alloc_nq()
571 if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { in bnxt_qplib_alloc_nq()
599 bnxt_qplib_free_hwq(res, &srq->hwq); in bnxt_qplib_destroy_srq()
619 rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); in bnxt_qplib_create_srq()
623 srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq), in bnxt_qplib_create_srq()
636 req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements); in bnxt_qplib_create_srq()
637 pbl = &srq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_srq()
638 pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) << in bnxt_qplib_create_srq()
640 pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) << in bnxt_qplib_create_srq()
654 srq->last_idx = srq->hwq.max_elements - 1; in bnxt_qplib_create_srq()
655 for (idx = 0; idx < srq->hwq.max_elements; idx++) in bnxt_qplib_create_srq()
660 srq->dbinfo.hwq = &srq->hwq; in bnxt_qplib_create_srq()
671 bnxt_qplib_free_hwq(res, &srq->hwq); in bnxt_qplib_create_srq()
680 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; in bnxt_qplib_modify_srq()
730 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; in bnxt_qplib_post_srq_recv()
838 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); in bnxt_qplib_create_qp1()
847 pbl = &sq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp1()
849 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << in bnxt_qplib_create_qp1()
851 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK); in bnxt_qplib_create_qp1()
865 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); in bnxt_qplib_create_qp1()
872 pbl = &rq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp1()
874 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << in bnxt_qplib_create_qp1()
876 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK); in bnxt_qplib_create_qp1()
902 sq->dbinfo.hwq = &sq->hwq; in bnxt_qplib_create_qp1()
907 rq->dbinfo.hwq = &rq->hwq; in bnxt_qplib_create_qp1()
923 bnxt_qplib_free_hwq(res, &rq->hwq); in bnxt_qplib_create_qp1()
927 bnxt_qplib_free_hwq(res, &sq->hwq); in bnxt_qplib_create_qp1()
934 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_init_psn_ptr() local
940 hwq = &sq->hwq; in bnxt_qplib_init_psn_ptr()
942 fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg); in bnxt_qplib_init_psn_ptr()
945 hwq->pad_pgofft = indx_pad; in bnxt_qplib_init_psn_ptr()
946 hwq->pad_pg = (u64 *)psn_pg; in bnxt_qplib_init_psn_ptr()
947 hwq->pad_stride = size; in bnxt_qplib_init_psn_ptr()
989 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); in bnxt_qplib_create_qp()
1001 pbl = &sq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1003 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << in bnxt_qplib_create_qp()
1005 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK); in bnxt_qplib_create_qp()
1021 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); in bnxt_qplib_create_qp()
1029 pbl = &rq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1031 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << in bnxt_qplib_create_qp()
1033 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK); in bnxt_qplib_create_qp()
1111 sq->dbinfo.hwq = &sq->hwq; in bnxt_qplib_create_qp()
1116 rq->dbinfo.hwq = &rq->hwq; in bnxt_qplib_create_qp()
1133 bnxt_qplib_free_hwq(res, &rq->hwq); in bnxt_qplib_create_qp()
1137 bnxt_qplib_free_hwq(res, &sq->hwq); in bnxt_qplib_create_qp()
1317 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); in bnxt_qplib_modify_qp()
1318 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); in bnxt_qplib_modify_qp()
1405 qp->sq.max_wqe = qp->sq.hwq.max_elements; in bnxt_qplib_query_qp()
1406 qp->rq.max_wqe = qp->rq.hwq.max_elements; in bnxt_qplib_query_qp()
1420 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; in __clean_cq()
1491 bnxt_qplib_free_hwq(res, &qp->sq.hwq); in bnxt_qplib_free_qp_res()
1494 bnxt_qplib_free_hwq(res, &qp->rq.hwq); in bnxt_qplib_free_qp_res()
1589 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_put_inline() local
1597 hwq = &qp->sq.hwq; in bnxt_qplib_put_inline()
1608 il_dst = bnxt_qplib_get_prod_qe(hwq, *idx); in bnxt_qplib_put_inline()
1632 static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq, in bnxt_qplib_put_sges() argument
1640 dsge = bnxt_qplib_get_prod_qe(hwq, *idx); in bnxt_qplib_put_sges()
1678 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_pull_psn_buff() local
1683 hwq = &sq->hwq; in bnxt_qplib_pull_psn_buff()
1684 if (!hwq->pad_pg) in bnxt_qplib_pull_psn_buff()
1687 pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1688 pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1689 buff = (void *)(hwq->pad_pg[pg_num] + pg_indx * hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1707 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_post_send() local
1718 hwq = &sq->hwq; in bnxt_qplib_post_send()
1721 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1730 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1732 hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta); in bnxt_qplib_post_send()
1741 swq->slot_idx = hwq->prod; in bnxt_qplib_post_send()
1752 dev_dbg(&hwq->pdev->dev, in bnxt_qplib_post_send()
1757 base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_send()
1758 ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_send()
1766 data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, in bnxt_qplib_post_send()
1927 bnxt_qplib_hwq_incr_prod(hwq, swq->slots); in bnxt_qplib_post_send()
1938 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1960 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_post_recv() local
1967 hwq = &rq->hwq; in bnxt_qplib_post_recv()
1969 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
1977 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
1989 dev_dbg(&hwq->pdev->dev, in bnxt_qplib_post_recv()
1995 base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_recv()
1996 ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_recv()
2001 bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx); in bnxt_qplib_post_recv()
2005 sge = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_recv()
2015 bnxt_qplib_hwq_incr_prod(hwq, swq->slots); in bnxt_qplib_post_recv()
2025 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
2051 rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); in bnxt_qplib_create_cq()
2064 req.cq_size = cpu_to_le32(cq->hwq.max_elements); in bnxt_qplib_create_cq()
2065 pbl = &cq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_cq()
2066 pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) << in bnxt_qplib_create_cq()
2068 pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK); in bnxt_qplib_create_cq()
2088 cq->dbinfo.hwq = &cq->hwq; in bnxt_qplib_create_cq()
2098 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_create_cq()
2121 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_destroy_cq()
2154 bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots); in __flush_sq()
2201 bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots); in __flush_rq()
2248 dev_dbg(&cq->hwq.pdev->dev, in do_wa9060()
2261 peek_raw_cq_cons = cq->hwq.cons; in do_wa9060()
2263 i = cq->hwq.max_elements; in do_wa9060()
2265 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); in do_wa9060()
2266 peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, in do_wa9060()
2270 cq->hwq.max_elements)) { in do_wa9060()
2299 dev_dbg(&cq->hwq.pdev->dev, in do_wa9060()
2316 dev_err(&cq->hwq.pdev->dev, in do_wa9060()
2340 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2348 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2379 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2401 bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots); in bnxt_qplib_cq_process_req()
2424 spin_lock(&srq->hwq.lock); in bnxt_qplib_release_srqe()
2428 srq->hwq.cons++; /* Support for SRQE counter */ in bnxt_qplib_release_srqe()
2429 spin_unlock(&srq->hwq.lock); in bnxt_qplib_release_srqe()
2447 dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n"); in bnxt_qplib_cq_process_res_rc()
2451 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2471 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_rc()
2472 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2474 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_rc()
2487 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2498 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); in bnxt_qplib_cq_process_res_rc()
2528 dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n"); in bnxt_qplib_cq_process_res_ud()
2532 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2558 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_ud()
2559 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2561 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_ud()
2574 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2586 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); in bnxt_qplib_cq_process_res_ud()
2606 raw_cons = cq->hwq.cons; in bnxt_qplib_is_cq_empty()
2607 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); in bnxt_qplib_is_cq_empty()
2608 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL); in bnxt_qplib_is_cq_empty()
2610 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements); in bnxt_qplib_is_cq_empty()
2629 dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n"); in bnxt_qplib_cq_process_res_raweth_qp1()
2633 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2662 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2666 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_raweth_qp1()
2667 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2669 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_raweth_qp1()
2682 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2693 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); in bnxt_qplib_cq_process_res_raweth_qp1()
2721 dev_warn(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2728 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2745 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2770 bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots); in bnxt_qplib_cq_process_terminal()
2787 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2795 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2817 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_cutoff()
2838 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp); in bnxt_qplib_process_flush_list()
2843 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp); in bnxt_qplib_process_flush_list()
2859 raw_cons = cq->hwq.cons; in bnxt_qplib_poll_cq()
2863 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); in bnxt_qplib_poll_cq()
2864 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL); in bnxt_qplib_poll_cq()
2867 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) in bnxt_qplib_poll_cq()
2911 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_poll_cq()
2925 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_poll_cq()
2930 if (cq->hwq.cons != raw_cons) { in bnxt_qplib_poll_cq()
2931 cq->hwq.cons = raw_cons; in bnxt_qplib_poll_cq()