Lines Matching refs:rf
74 static void irdma_puda_ce_handler(struct irdma_pci_f *rf, in irdma_puda_ce_handler() argument
77 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_puda_ce_handler()
104 static void irdma_process_ceq(struct irdma_pci_f *rf, struct irdma_ceq *ceq) in irdma_process_ceq() argument
106 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_process_ceq()
126 queue_work(rf->cqp_cmpl_wq, &rf->cqp_cmpl_work); in irdma_process_ceq()
129 irdma_puda_ce_handler(rf, cq); in irdma_process_ceq()
211 static void irdma_process_aeq(struct irdma_pci_f *rf) in irdma_process_aeq() argument
213 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_process_aeq()
214 struct irdma_aeq *aeq = &rf->aeq; in irdma_process_aeq()
224 struct irdma_device *iwdev = rf->iwdev; in irdma_process_aeq()
245 spin_lock_irqsave(&rf->qptable_lock, flags); in irdma_process_aeq()
246 iwqp = rf->qp_table[info->qp_cq_id]; in irdma_process_aeq()
248 spin_unlock_irqrestore(&rf->qptable_lock, in irdma_process_aeq()
260 spin_unlock_irqrestore(&rf->qptable_lock, flags); in irdma_process_aeq()
433 struct irdma_pci_f *rf = from_tasklet(rf, t, dpc_tasklet); in irdma_dpc() local
435 if (rf->msix_shared) in irdma_dpc()
436 irdma_process_ceq(rf, rf->ceqlist); in irdma_dpc()
437 irdma_process_aeq(rf); in irdma_dpc()
438 irdma_ena_intr(&rf->sc_dev, rf->iw_msixtbl[0].idx); in irdma_dpc()
448 struct irdma_pci_f *rf = iwceq->rf; in irdma_ceq_dpc() local
450 irdma_process_ceq(rf, iwceq); in irdma_ceq_dpc()
451 irdma_ena_intr(&rf->sc_dev, iwceq->msix_idx); in irdma_ceq_dpc()
461 static int irdma_save_msix_info(struct irdma_pci_f *rf) in irdma_save_msix_info() argument
470 if (!rf->msix_count) in irdma_save_msix_info()
473 size = sizeof(struct irdma_msix_vector) * rf->msix_count; in irdma_save_msix_info()
474 size += struct_size(iw_qvlist, qv_info, rf->msix_count); in irdma_save_msix_info()
475 rf->iw_msixtbl = kzalloc(size, GFP_KERNEL); in irdma_save_msix_info()
476 if (!rf->iw_msixtbl) in irdma_save_msix_info()
479 rf->iw_qvlist = (struct irdma_qvlist_info *) in irdma_save_msix_info()
480 (&rf->iw_msixtbl[rf->msix_count]); in irdma_save_msix_info()
481 iw_qvlist = rf->iw_qvlist; in irdma_save_msix_info()
483 iw_qvlist->num_vectors = rf->msix_count; in irdma_save_msix_info()
484 if (rf->msix_count <= num_online_cpus()) in irdma_save_msix_info()
485 rf->msix_shared = true; in irdma_save_msix_info()
486 else if (rf->msix_count > num_online_cpus() + 1) in irdma_save_msix_info()
487 rf->msix_count = num_online_cpus() + 1; in irdma_save_msix_info()
489 pmsix = rf->msix_entries; in irdma_save_msix_info()
490 for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) { in irdma_save_msix_info()
491 rf->iw_msixtbl[i].idx = pmsix->entry; in irdma_save_msix_info()
492 rf->iw_msixtbl[i].irq = pmsix->vector; in irdma_save_msix_info()
493 rf->iw_msixtbl[i].cpu_affinity = ceq_idx; in irdma_save_msix_info()
496 if (rf->msix_shared) in irdma_save_msix_info()
505 iw_qvinfo->v_idx = rf->iw_msixtbl[i].idx; in irdma_save_msix_info()
519 struct irdma_pci_f *rf = data; in irdma_irq_handler() local
521 tasklet_schedule(&rf->dpc_tasklet); in irdma_irq_handler()
536 ibdev_err(to_ibdev(&iwceq->rf->sc_dev), "expected irq = %d received irq = %d\n", in irdma_ceq_handler()
551 static void irdma_destroy_irq(struct irdma_pci_f *rf, in irdma_destroy_irq() argument
554 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_irq()
569 static void irdma_destroy_cqp(struct irdma_pci_f *rf, bool free_hwcqp) in irdma_destroy_cqp() argument
571 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_cqp()
572 struct irdma_cqp *cqp = &rf->cqp; in irdma_destroy_cqp()
575 if (rf->cqp_cmpl_wq) in irdma_destroy_cqp()
576 destroy_workqueue(rf->cqp_cmpl_wq); in irdma_destroy_cqp()
582 irdma_cleanup_pending_cqp_op(rf); in irdma_destroy_cqp()
592 static void irdma_destroy_virt_aeq(struct irdma_pci_f *rf) in irdma_destroy_virt_aeq() argument
594 struct irdma_aeq *aeq = &rf->aeq; in irdma_destroy_virt_aeq()
598 irdma_unmap_vm_page_list(&rf->hw, pg_arr, pg_cnt); in irdma_destroy_virt_aeq()
599 irdma_free_pble(rf->pble_rsrc, &aeq->palloc); in irdma_destroy_virt_aeq()
611 static void irdma_destroy_aeq(struct irdma_pci_f *rf) in irdma_destroy_aeq() argument
613 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_aeq()
614 struct irdma_aeq *aeq = &rf->aeq; in irdma_destroy_aeq()
617 if (!rf->msix_shared) { in irdma_destroy_aeq()
618 rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, rf->iw_msixtbl->idx, false); in irdma_destroy_aeq()
619 irdma_destroy_irq(rf, rf->iw_msixtbl, rf); in irdma_destroy_aeq()
621 if (rf->reset) in irdma_destroy_aeq()
631 irdma_destroy_virt_aeq(rf); in irdma_destroy_aeq()
647 static void irdma_destroy_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq) in irdma_destroy_ceq() argument
649 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_ceq()
652 if (rf->reset) in irdma_destroy_ceq()
677 static void irdma_del_ceq_0(struct irdma_pci_f *rf) in irdma_del_ceq_0() argument
679 struct irdma_ceq *iwceq = rf->ceqlist; in irdma_del_ceq_0()
682 if (rf->msix_shared) { in irdma_del_ceq_0()
683 msix_vec = &rf->iw_msixtbl[0]; in irdma_del_ceq_0()
684 rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, in irdma_del_ceq_0()
687 irdma_destroy_irq(rf, msix_vec, rf); in irdma_del_ceq_0()
689 msix_vec = &rf->iw_msixtbl[1]; in irdma_del_ceq_0()
690 irdma_destroy_irq(rf, msix_vec, iwceq); in irdma_del_ceq_0()
693 irdma_destroy_ceq(rf, iwceq); in irdma_del_ceq_0()
694 rf->sc_dev.ceq_valid = false; in irdma_del_ceq_0()
695 rf->ceqs_count = 0; in irdma_del_ceq_0()
705 static void irdma_del_ceqs(struct irdma_pci_f *rf) in irdma_del_ceqs() argument
707 struct irdma_ceq *iwceq = &rf->ceqlist[1]; in irdma_del_ceqs()
711 if (rf->msix_shared) in irdma_del_ceqs()
712 msix_vec = &rf->iw_msixtbl[1]; in irdma_del_ceqs()
714 msix_vec = &rf->iw_msixtbl[2]; in irdma_del_ceqs()
716 for (i = 1; i < rf->ceqs_count; i++, msix_vec++, iwceq++) { in irdma_del_ceqs()
717 rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, msix_vec->ceq_id, in irdma_del_ceqs()
719 irdma_destroy_irq(rf, msix_vec, iwceq); in irdma_del_ceqs()
720 irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq, in irdma_del_ceqs()
722 dma_free_coherent(rf->sc_dev.hw->device, iwceq->mem.size, in irdma_del_ceqs()
726 rf->ceqs_count = 1; in irdma_del_ceqs()
736 static void irdma_destroy_ccq(struct irdma_pci_f *rf) in irdma_destroy_ccq() argument
738 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_ccq()
739 struct irdma_ccq *ccq = &rf->ccq; in irdma_destroy_ccq()
742 if (!rf->reset) in irdma_destroy_ccq()
818 static int irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged, in irdma_create_hmc_objs() argument
821 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_hmc_objs()
827 info.entry_type = rf->sd_type; in irdma_create_hmc_objs()
873 static int irdma_obj_aligned_mem(struct irdma_pci_f *rf, in irdma_obj_aligned_mem() argument
880 va = (unsigned long)rf->obj_next.va; in irdma_obj_aligned_mem()
886 memptr->pa = rf->obj_next.pa + extra; in irdma_obj_aligned_mem()
888 if (((u8 *)memptr->va + size) > ((u8 *)rf->obj_mem.va + rf->obj_mem.size)) in irdma_obj_aligned_mem()
891 rf->obj_next.va = (u8 *)memptr->va + size; in irdma_obj_aligned_mem()
892 rf->obj_next.pa = memptr->pa + size; in irdma_obj_aligned_mem()
904 static int irdma_create_cqp(struct irdma_pci_f *rf) in irdma_create_cqp() argument
908 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_cqp()
910 struct irdma_cqp *cqp = &rf->cqp; in irdma_create_cqp()
936 status = irdma_obj_aligned_mem(rf, &mem, sizeof(struct irdma_cqp_ctx), in irdma_create_cqp()
950 cqp_init_info.hmc_profile = rf->rsrc_profile; in irdma_create_cqp()
952 cqp_init_info.protocol_used = rf->protocol_used; in irdma_create_cqp()
954 switch (rf->rdma_ver) { in irdma_create_cqp()
991 irdma_destroy_cqp(rf, false); in irdma_create_cqp()
1003 static int irdma_create_ccq(struct irdma_pci_f *rf) in irdma_create_ccq() argument
1005 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_ccq()
1007 struct irdma_ccq *ccq = &rf->ccq; in irdma_create_ccq()
1021 status = irdma_obj_aligned_mem(rf, &ccq->shadow_area, in irdma_create_ccq()
1037 info.vsi = &rf->default_vsi; in irdma_create_ccq()
1062 status = irdma_alloc_local_mac_entry(iwdev->rf, in irdma_alloc_set_mac()
1065 status = irdma_add_local_mac_entry(iwdev->rf, in irdma_alloc_set_mac()
1069 irdma_del_local_mac_entry(iwdev->rf, in irdma_alloc_set_mac()
1086 static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, in irdma_cfg_ceq_vector() argument
1091 if (rf->msix_shared && !ceq_id) { in irdma_cfg_ceq_vector()
1092 tasklet_setup(&rf->dpc_tasklet, irdma_dpc); in irdma_cfg_ceq_vector()
1094 "AEQCEQ", rf); in irdma_cfg_ceq_vector()
1105 ibdev_dbg(&rf->iwdev->ibdev, "ERR: ceq irq config fail\n"); in irdma_cfg_ceq_vector()
1110 rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true); in irdma_cfg_ceq_vector()
1122 static int irdma_cfg_aeq_vector(struct irdma_pci_f *rf) in irdma_cfg_aeq_vector() argument
1124 struct irdma_msix_vector *msix_vec = rf->iw_msixtbl; in irdma_cfg_aeq_vector()
1127 if (!rf->msix_shared) { in irdma_cfg_aeq_vector()
1128 tasklet_setup(&rf->dpc_tasklet, irdma_dpc); in irdma_cfg_aeq_vector()
1130 "irdma", rf); in irdma_cfg_aeq_vector()
1133 ibdev_dbg(&rf->iwdev->ibdev, "ERR: aeq irq config fail\n"); in irdma_cfg_aeq_vector()
1137 rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, true); in irdma_cfg_aeq_vector()
1152 static int irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, in irdma_create_ceq() argument
1157 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_ceq()
1162 iwceq->rf = rf; in irdma_create_ceq()
1163 ceq_size = min(rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt, in irdma_create_ceq()
1179 scratch = (uintptr_t)&rf->cqp.sc_cqp; in irdma_create_ceq()
1183 status = irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq, in irdma_create_ceq()
1206 static int irdma_setup_ceq_0(struct irdma_pci_f *rf) in irdma_setup_ceq_0() argument
1214 num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs); in irdma_setup_ceq_0()
1215 rf->ceqlist = kcalloc(num_ceqs, sizeof(*rf->ceqlist), GFP_KERNEL); in irdma_setup_ceq_0()
1216 if (!rf->ceqlist) { in irdma_setup_ceq_0()
1221 iwceq = &rf->ceqlist[0]; in irdma_setup_ceq_0()
1222 status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi); in irdma_setup_ceq_0()
1224 ibdev_dbg(&rf->iwdev->ibdev, "ERR: create ceq status = %d\n", in irdma_setup_ceq_0()
1230 i = rf->msix_shared ? 0 : 1; in irdma_setup_ceq_0()
1231 msix_vec = &rf->iw_msixtbl[i]; in irdma_setup_ceq_0()
1234 status = irdma_cfg_ceq_vector(rf, iwceq, 0, msix_vec); in irdma_setup_ceq_0()
1236 irdma_destroy_ceq(rf, iwceq); in irdma_setup_ceq_0()
1240 irdma_ena_intr(&rf->sc_dev, msix_vec->idx); in irdma_setup_ceq_0()
1241 rf->ceqs_count++; in irdma_setup_ceq_0()
1244 if (status && !rf->ceqs_count) { in irdma_setup_ceq_0()
1245 kfree(rf->ceqlist); in irdma_setup_ceq_0()
1246 rf->ceqlist = NULL; in irdma_setup_ceq_0()
1249 rf->sc_dev.ceq_valid = true; in irdma_setup_ceq_0()
1263 static int irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi) in irdma_setup_ceqs() argument
1272 num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs); in irdma_setup_ceqs()
1273 i = (rf->msix_shared) ? 1 : 2; in irdma_setup_ceqs()
1275 iwceq = &rf->ceqlist[ceq_id]; in irdma_setup_ceqs()
1276 status = irdma_create_ceq(rf, iwceq, ceq_id, vsi); in irdma_setup_ceqs()
1278 ibdev_dbg(&rf->iwdev->ibdev, in irdma_setup_ceqs()
1283 msix_vec = &rf->iw_msixtbl[i]; in irdma_setup_ceqs()
1286 status = irdma_cfg_ceq_vector(rf, iwceq, ceq_id, msix_vec); in irdma_setup_ceqs()
1288 irdma_destroy_ceq(rf, iwceq); in irdma_setup_ceqs()
1291 irdma_ena_intr(&rf->sc_dev, msix_vec->idx); in irdma_setup_ceqs()
1292 rf->ceqs_count++; in irdma_setup_ceqs()
1298 irdma_del_ceqs(rf); in irdma_setup_ceqs()
1303 static int irdma_create_virt_aeq(struct irdma_pci_f *rf, u32 size) in irdma_create_virt_aeq() argument
1305 struct irdma_aeq *aeq = &rf->aeq; in irdma_create_virt_aeq()
1310 if (rf->rdma_ver < IRDMA_GEN_2) in irdma_create_virt_aeq()
1320 status = irdma_get_pble(rf->pble_rsrc, &aeq->palloc, pg_cnt, true); in irdma_create_virt_aeq()
1327 status = irdma_map_vm_page_list(&rf->hw, aeq->mem.va, pg_arr, pg_cnt); in irdma_create_virt_aeq()
1329 irdma_free_pble(rf->pble_rsrc, &aeq->palloc); in irdma_create_virt_aeq()
1344 static int irdma_create_aeq(struct irdma_pci_f *rf) in irdma_create_aeq() argument
1347 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_aeq()
1348 struct irdma_aeq *aeq = &rf->aeq; in irdma_create_aeq()
1349 struct irdma_hmc_info *hmc_info = rf->sc_dev.hmc_info; in irdma_create_aeq()
1351 u8 multiplier = (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? 2 : 1; in irdma_create_aeq()
1367 status = irdma_create_virt_aeq(rf, aeq_size); in irdma_create_aeq()
1381 info.msix_idx = rf->iw_msixtbl->idx; in irdma_create_aeq()
1394 irdma_destroy_virt_aeq(rf); in irdma_create_aeq()
1411 static int irdma_setup_aeq(struct irdma_pci_f *rf) in irdma_setup_aeq() argument
1413 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_setup_aeq()
1416 status = irdma_create_aeq(rf); in irdma_setup_aeq()
1420 status = irdma_cfg_aeq_vector(rf); in irdma_setup_aeq()
1422 irdma_destroy_aeq(rf); in irdma_setup_aeq()
1426 if (!rf->msix_shared) in irdma_setup_aeq()
1427 irdma_ena_intr(dev, rf->iw_msixtbl[0].idx); in irdma_setup_aeq()
1449 info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768); in irdma_initialize_ilq()
1479 info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768); in irdma_initialize_ieq()
1497 struct irdma_pci_f *rf = iwdev->rf; in irdma_reinitialize_ieq() local
1501 iwdev->rf->reset = true; in irdma_reinitialize_ieq()
1502 rf->gen_ops.request_reset(rf); in irdma_reinitialize_ieq()
1514 static int irdma_hmc_setup(struct irdma_pci_f *rf) in irdma_hmc_setup() argument
1519 qpcnt = rsrc_limits_table[rf->limits_sel].qplimit; in irdma_hmc_setup()
1521 rf->sd_type = IRDMA_SD_TYPE_DIRECT; in irdma_hmc_setup()
1522 status = irdma_cfg_fpm_val(&rf->sc_dev, qpcnt); in irdma_hmc_setup()
1526 status = irdma_create_hmc_objs(rf, true, rf->rdma_ver); in irdma_hmc_setup()
1535 static void irdma_del_init_mem(struct irdma_pci_f *rf) in irdma_del_init_mem() argument
1537 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_del_init_mem()
1541 kfree(rf->mem_rsrc); in irdma_del_init_mem()
1542 rf->mem_rsrc = NULL; in irdma_del_init_mem()
1543 dma_free_coherent(rf->hw.device, rf->obj_mem.size, rf->obj_mem.va, in irdma_del_init_mem()
1544 rf->obj_mem.pa); in irdma_del_init_mem()
1545 rf->obj_mem.va = NULL; in irdma_del_init_mem()
1546 if (rf->rdma_ver != IRDMA_GEN_1) { in irdma_del_init_mem()
1547 bitmap_free(rf->allocated_ws_nodes); in irdma_del_init_mem()
1548 rf->allocated_ws_nodes = NULL; in irdma_del_init_mem()
1550 kfree(rf->ceqlist); in irdma_del_init_mem()
1551 rf->ceqlist = NULL; in irdma_del_init_mem()
1552 kfree(rf->iw_msixtbl); in irdma_del_init_mem()
1553 rf->iw_msixtbl = NULL; in irdma_del_init_mem()
1554 kfree(rf->hmc_info_mem); in irdma_del_init_mem()
1555 rf->hmc_info_mem = NULL; in irdma_del_init_mem()
1566 static int irdma_initialize_dev(struct irdma_pci_f *rf) in irdma_initialize_dev() argument
1569 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_initialize_dev()
1578 rf->hmc_info_mem = kzalloc(size, GFP_KERNEL); in irdma_initialize_dev()
1579 if (!rf->hmc_info_mem) in irdma_initialize_dev()
1582 rf->pble_rsrc = (struct irdma_hmc_pble_rsrc *)rf->hmc_info_mem; in irdma_initialize_dev()
1583 dev->hmc_info = &rf->hw.hmc; in irdma_initialize_dev()
1585 (rf->pble_rsrc + 1); in irdma_initialize_dev()
1587 status = irdma_obj_aligned_mem(rf, &mem, IRDMA_QUERY_FPM_BUF_SIZE, in irdma_initialize_dev()
1595 status = irdma_obj_aligned_mem(rf, &mem, IRDMA_COMMIT_FPM_BUF_SIZE, in irdma_initialize_dev()
1603 info.bar0 = rf->hw.hw_addr; in irdma_initialize_dev()
1604 info.hmc_fn_id = rf->pf_id; in irdma_initialize_dev()
1605 info.hw = &rf->hw; in irdma_initialize_dev()
1606 status = irdma_sc_dev_init(rf->rdma_ver, &rf->sc_dev, &info); in irdma_initialize_dev()
1612 kfree(rf->hmc_info_mem); in irdma_initialize_dev()
1613 rf->hmc_info_mem = NULL; in irdma_initialize_dev()
1631 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_rt_deinit_hw()
1632 irdma_del_local_mac_entry(iwdev->rf, in irdma_rt_deinit_hw()
1641 iwdev->rf->reset); in irdma_rt_deinit_hw()
1647 iwdev->rf->reset); in irdma_rt_deinit_hw()
1663 static int irdma_setup_init_state(struct irdma_pci_f *rf) in irdma_setup_init_state() argument
1667 status = irdma_save_msix_info(rf); in irdma_setup_init_state()
1671 rf->hw.device = &rf->pcidev->dev; in irdma_setup_init_state()
1672 rf->obj_mem.size = ALIGN(8192, IRDMA_HW_PAGE_SIZE); in irdma_setup_init_state()
1673 rf->obj_mem.va = dma_alloc_coherent(rf->hw.device, rf->obj_mem.size, in irdma_setup_init_state()
1674 &rf->obj_mem.pa, GFP_KERNEL); in irdma_setup_init_state()
1675 if (!rf->obj_mem.va) { in irdma_setup_init_state()
1680 rf->obj_next = rf->obj_mem; in irdma_setup_init_state()
1681 status = irdma_initialize_dev(rf); in irdma_setup_init_state()
1688 dma_free_coherent(rf->hw.device, rf->obj_mem.size, rf->obj_mem.va, in irdma_setup_init_state()
1689 rf->obj_mem.pa); in irdma_setup_init_state()
1690 rf->obj_mem.va = NULL; in irdma_setup_init_state()
1692 kfree(rf->iw_msixtbl); in irdma_setup_init_state()
1693 rf->iw_msixtbl = NULL; in irdma_setup_init_state()
1705 iwdev->rf->used_pds = find_first_zero_bit(iwdev->rf->allocated_pds, in irdma_get_used_rsrc()
1706 iwdev->rf->max_pd); in irdma_get_used_rsrc()
1707 iwdev->rf->used_qps = find_first_zero_bit(iwdev->rf->allocated_qps, in irdma_get_used_rsrc()
1708 iwdev->rf->max_qp); in irdma_get_used_rsrc()
1709 iwdev->rf->used_cqs = find_first_zero_bit(iwdev->rf->allocated_cqs, in irdma_get_used_rsrc()
1710 iwdev->rf->max_cq); in irdma_get_used_rsrc()
1711 iwdev->rf->used_mrs = find_first_zero_bit(iwdev->rf->allocated_mrs, in irdma_get_used_rsrc()
1712 iwdev->rf->max_mr); in irdma_get_used_rsrc()
1715 void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf) in irdma_ctrl_deinit_hw() argument
1717 enum init_completion_state state = rf->init_state; in irdma_ctrl_deinit_hw()
1719 rf->init_state = INVALID_STATE; in irdma_ctrl_deinit_hw()
1720 if (rf->rsrc_created) { in irdma_ctrl_deinit_hw()
1721 irdma_destroy_aeq(rf); in irdma_ctrl_deinit_hw()
1722 irdma_destroy_pble_prm(rf->pble_rsrc); in irdma_ctrl_deinit_hw()
1723 irdma_del_ceqs(rf); in irdma_ctrl_deinit_hw()
1724 rf->rsrc_created = false; in irdma_ctrl_deinit_hw()
1728 irdma_del_ceq_0(rf); in irdma_ctrl_deinit_hw()
1731 irdma_destroy_ccq(rf); in irdma_ctrl_deinit_hw()
1735 irdma_del_hmc_objects(&rf->sc_dev, rf->sc_dev.hmc_info, true, in irdma_ctrl_deinit_hw()
1736 rf->reset, rf->rdma_ver); in irdma_ctrl_deinit_hw()
1739 irdma_destroy_cqp(rf, true); in irdma_ctrl_deinit_hw()
1742 irdma_del_init_mem(rf); in irdma_ctrl_deinit_hw()
1746 ibdev_warn(&rf->iwdev->ibdev, "bad init_state = %d\n", rf->init_state); in irdma_ctrl_deinit_hw()
1762 struct irdma_pci_f *rf = iwdev->rf; in irdma_rt_init_hw() local
1763 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_rt_init_hw()
1772 vsi_info.register_qset = rf->gen_ops.register_qset; in irdma_rt_init_hw()
1773 vsi_info.unregister_qset = rf->gen_ops.unregister_qset; in irdma_rt_init_hw()
1777 status = irdma_setup_cm_core(iwdev, rf->rdma_ver); in irdma_rt_init_hw()
1805 if (!rf->rsrc_created) { in irdma_rt_init_hw()
1806 status = irdma_setup_ceqs(rf, &iwdev->vsi); in irdma_rt_init_hw()
1812 status = irdma_hmc_init_pble(&rf->sc_dev, in irdma_rt_init_hw()
1813 rf->pble_rsrc); in irdma_rt_init_hw()
1815 irdma_del_ceqs(rf); in irdma_rt_init_hw()
1821 status = irdma_setup_aeq(rf); in irdma_rt_init_hw()
1823 irdma_destroy_pble_prm(rf->pble_rsrc); in irdma_rt_init_hw()
1824 irdma_del_ceqs(rf); in irdma_rt_init_hw()
1828 rf->rsrc_created = true; in irdma_rt_init_hw()
1831 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_rt_init_hw()
1849 dev_err(&rf->pcidev->dev, "HW runtime init FAIL status = %d last cmpl = %d\n", in irdma_rt_init_hw()
1862 int irdma_ctrl_init_hw(struct irdma_pci_f *rf) in irdma_ctrl_init_hw() argument
1864 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_ctrl_init_hw()
1867 status = irdma_setup_init_state(rf); in irdma_ctrl_init_hw()
1870 rf->init_state = INITIAL_STATE; in irdma_ctrl_init_hw()
1872 status = irdma_create_cqp(rf); in irdma_ctrl_init_hw()
1875 rf->init_state = CQP_CREATED; in irdma_ctrl_init_hw()
1877 status = irdma_hmc_setup(rf); in irdma_ctrl_init_hw()
1880 rf->init_state = HMC_OBJS_CREATED; in irdma_ctrl_init_hw()
1882 status = irdma_initialize_hw_rsrc(rf); in irdma_ctrl_init_hw()
1885 rf->init_state = HW_RSRC_INITIALIZED; in irdma_ctrl_init_hw()
1887 status = irdma_create_ccq(rf); in irdma_ctrl_init_hw()
1890 rf->init_state = CCQ_CREATED; in irdma_ctrl_init_hw()
1893 if (rf->rdma_ver != IRDMA_GEN_1) { in irdma_ctrl_init_hw()
1899 status = irdma_setup_ceq_0(rf); in irdma_ctrl_init_hw()
1902 rf->init_state = CEQ0_CREATED; in irdma_ctrl_init_hw()
1904 rf->cqp_cmpl_wq = alloc_ordered_workqueue("cqp_cmpl_wq", in irdma_ctrl_init_hw()
1906 if (!rf->cqp_cmpl_wq) { in irdma_ctrl_init_hw()
1910 INIT_WORK(&rf->cqp_cmpl_work, cqp_compl_worker); in irdma_ctrl_init_hw()
1915 dev_err(&rf->pcidev->dev, "IRDMA hardware initialization FAILED init_state=%d status=%d\n", in irdma_ctrl_init_hw()
1916 rf->init_state, status); in irdma_ctrl_init_hw()
1917 irdma_ctrl_deinit_hw(rf); in irdma_ctrl_init_hw()
1925 static void irdma_set_hw_rsrc(struct irdma_pci_f *rf) in irdma_set_hw_rsrc() argument
1927 rf->allocated_qps = (void *)(rf->mem_rsrc + in irdma_set_hw_rsrc()
1928 (sizeof(struct irdma_arp_entry) * rf->arp_table_size)); in irdma_set_hw_rsrc()
1929 rf->allocated_cqs = &rf->allocated_qps[BITS_TO_LONGS(rf->max_qp)]; in irdma_set_hw_rsrc()
1930 rf->allocated_mrs = &rf->allocated_cqs[BITS_TO_LONGS(rf->max_cq)]; in irdma_set_hw_rsrc()
1931 rf->allocated_pds = &rf->allocated_mrs[BITS_TO_LONGS(rf->max_mr)]; in irdma_set_hw_rsrc()
1932 rf->allocated_ahs = &rf->allocated_pds[BITS_TO_LONGS(rf->max_pd)]; in irdma_set_hw_rsrc()
1933 rf->allocated_mcgs = &rf->allocated_ahs[BITS_TO_LONGS(rf->max_ah)]; in irdma_set_hw_rsrc()
1934 rf->allocated_arps = &rf->allocated_mcgs[BITS_TO_LONGS(rf->max_mcg)]; in irdma_set_hw_rsrc()
1935 rf->qp_table = (struct irdma_qp **) in irdma_set_hw_rsrc()
1936 (&rf->allocated_arps[BITS_TO_LONGS(rf->arp_table_size)]); in irdma_set_hw_rsrc()
1938 spin_lock_init(&rf->rsrc_lock); in irdma_set_hw_rsrc()
1939 spin_lock_init(&rf->arp_lock); in irdma_set_hw_rsrc()
1940 spin_lock_init(&rf->qptable_lock); in irdma_set_hw_rsrc()
1941 spin_lock_init(&rf->qh_list_lock); in irdma_set_hw_rsrc()
1948 static u32 irdma_calc_mem_rsrc_size(struct irdma_pci_f *rf) in irdma_calc_mem_rsrc_size() argument
1952 rsrc_size = sizeof(struct irdma_arp_entry) * rf->arp_table_size; in irdma_calc_mem_rsrc_size()
1953 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_qp); in irdma_calc_mem_rsrc_size()
1954 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_mr); in irdma_calc_mem_rsrc_size()
1955 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_cq); in irdma_calc_mem_rsrc_size()
1956 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_pd); in irdma_calc_mem_rsrc_size()
1957 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->arp_table_size); in irdma_calc_mem_rsrc_size()
1958 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_ah); in irdma_calc_mem_rsrc_size()
1959 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_mcg); in irdma_calc_mem_rsrc_size()
1960 rsrc_size += sizeof(struct irdma_qp **) * rf->max_qp; in irdma_calc_mem_rsrc_size()
1969 u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf) in irdma_initialize_hw_rsrc() argument
1975 if (rf->rdma_ver != IRDMA_GEN_1) { in irdma_initialize_hw_rsrc()
1976 rf->allocated_ws_nodes = bitmap_zalloc(IRDMA_MAX_WS_NODES, in irdma_initialize_hw_rsrc()
1978 if (!rf->allocated_ws_nodes) in irdma_initialize_hw_rsrc()
1981 set_bit(0, rf->allocated_ws_nodes); in irdma_initialize_hw_rsrc()
1982 rf->max_ws_node_id = IRDMA_MAX_WS_NODES; in irdma_initialize_hw_rsrc()
1984 rf->max_cqe = rf->sc_dev.hw_attrs.uk_attrs.max_hw_cq_size; in irdma_initialize_hw_rsrc()
1985 rf->max_qp = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt; in irdma_initialize_hw_rsrc()
1986 rf->max_mr = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt; in irdma_initialize_hw_rsrc()
1987 rf->max_cq = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt; in irdma_initialize_hw_rsrc()
1988 rf->max_pd = rf->sc_dev.hw_attrs.max_hw_pds; in irdma_initialize_hw_rsrc()
1989 rf->arp_table_size = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt; in irdma_initialize_hw_rsrc()
1990 rf->max_ah = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt; in irdma_initialize_hw_rsrc()
1991 rf->max_mcg = rf->max_qp; in irdma_initialize_hw_rsrc()
1993 rsrc_size = irdma_calc_mem_rsrc_size(rf); in irdma_initialize_hw_rsrc()
1994 rf->mem_rsrc = kzalloc(rsrc_size, GFP_KERNEL); in irdma_initialize_hw_rsrc()
1995 if (!rf->mem_rsrc) { in irdma_initialize_hw_rsrc()
2000 rf->arp_table = (struct irdma_arp_entry *)rf->mem_rsrc; in irdma_initialize_hw_rsrc()
2002 irdma_set_hw_rsrc(rf); in irdma_initialize_hw_rsrc()
2004 set_bit(0, rf->allocated_mrs); in irdma_initialize_hw_rsrc()
2005 set_bit(0, rf->allocated_qps); in irdma_initialize_hw_rsrc()
2006 set_bit(0, rf->allocated_cqs); in irdma_initialize_hw_rsrc()
2007 set_bit(0, rf->allocated_pds); in irdma_initialize_hw_rsrc()
2008 set_bit(0, rf->allocated_arps); in irdma_initialize_hw_rsrc()
2009 set_bit(0, rf->allocated_ahs); in irdma_initialize_hw_rsrc()
2010 set_bit(0, rf->allocated_mcgs); in irdma_initialize_hw_rsrc()
2011 set_bit(2, rf->allocated_qps); /* qp 2 IEQ */ in irdma_initialize_hw_rsrc()
2012 set_bit(1, rf->allocated_qps); /* qp 1 ILQ */ in irdma_initialize_hw_rsrc()
2013 set_bit(1, rf->allocated_cqs); in irdma_initialize_hw_rsrc()
2014 set_bit(1, rf->allocated_pds); in irdma_initialize_hw_rsrc()
2015 set_bit(2, rf->allocated_cqs); in irdma_initialize_hw_rsrc()
2016 set_bit(2, rf->allocated_pds); in irdma_initialize_hw_rsrc()
2018 INIT_LIST_HEAD(&rf->mc_qht_list.list); in irdma_initialize_hw_rsrc()
2020 mrdrvbits = 24 - max(get_count_order(rf->max_mr), 14); in irdma_initialize_hw_rsrc()
2021 rf->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits)); in irdma_initialize_hw_rsrc()
2026 bitmap_free(rf->allocated_ws_nodes); in irdma_initialize_hw_rsrc()
2027 rf->allocated_ws_nodes = NULL; in irdma_initialize_hw_rsrc()
2037 void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) in irdma_cqp_ce_handler() argument
2040 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_cqp_ce_handler()
2048 spin_lock_irqsave(&rf->cqp.compl_lock, flags); in irdma_cqp_ce_handler()
2050 spin_unlock_irqrestore(&rf->cqp.compl_lock, flags); in irdma_cqp_ce_handler()
2059 ibdev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", in irdma_cqp_ce_handler()
2070 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ce_handler()
2074 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ce_handler()
2093 struct irdma_pci_f *rf = container_of(work, struct irdma_pci_f, in cqp_compl_worker() local
2095 struct irdma_sc_cq *cq = &rf->ccq.sc_cq; in cqp_compl_worker()
2097 irdma_cqp_ce_handler(rf, cq); in cqp_compl_worker()
2158 void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx) in irdma_del_local_mac_entry() argument
2160 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_del_local_mac_entry()
2176 irdma_handle_cqp_op(rf, cqp_request); in irdma_del_local_mac_entry()
2187 int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx) in irdma_add_local_mac_entry() argument
2190 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_add_local_mac_entry()
2209 status = irdma_handle_cqp_op(rf, cqp_request); in irdma_add_local_mac_entry()
2224 int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx) in irdma_alloc_local_mac_entry() argument
2226 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_alloc_local_mac_entry()
2240 status = irdma_handle_cqp_op(rf, cqp_request); in irdma_alloc_local_mac_entry()
2263 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, add_port); in irdma_cqp_manage_apbvt_cmd()
2274 cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp; in irdma_cqp_manage_apbvt_cmd()
2279 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); in irdma_cqp_manage_apbvt_cmd()
2280 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); in irdma_cqp_manage_apbvt_cmd()
2357 void irdma_manage_arp_cache(struct irdma_pci_f *rf, in irdma_manage_arp_cache() argument
2366 arp_index = irdma_arp_table(rf, ip_addr, ipv4, mac_addr, action); in irdma_manage_arp_cache()
2370 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_manage_arp_cache()
2384 cqp_info->in.u.add_arp_cache_entry.cqp = &rf->cqp.sc_cqp; in irdma_manage_arp_cache()
2389 cqp_info->in.u.del_arp_cache_entry.cqp = &rf->cqp.sc_cqp; in irdma_manage_arp_cache()
2394 irdma_handle_cqp_op(rf, cqp_request); in irdma_manage_arp_cache()
2395 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_manage_arp_cache()
2425 struct irdma_cqp *iwcqp = &iwdev->rf->cqp; in irdma_manage_qhash()
2490 cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp; in irdma_manage_qhash()
2494 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); in irdma_manage_qhash()
2547 int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, in irdma_hw_flush_wqes() argument
2556 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_hw_flush_wqes()
2569 status = irdma_handle_cqp_op(rf, cqp_request); in irdma_hw_flush_wqes()
2573 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_flush_wqes()
2604 new_req = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_hw_flush_wqes()
2617 status = irdma_handle_cqp_op(rf, new_req); in irdma_hw_flush_wqes()
2626 irdma_put_cqp_request(&rf->cqp, new_req); in irdma_hw_flush_wqes()
2637 ibdev_dbg(&rf->iwdev->ibdev, in irdma_hw_flush_wqes()
2639 iwqp->ibqp.qp_num, rf->protocol_used, iwqp->iwarp_state, in irdma_hw_flush_wqes()
2644 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_flush_wqes()
2656 void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, in irdma_gen_ae() argument
2663 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_gen_ae()
2675 irdma_handle_cqp_op(rf, cqp_request); in irdma_gen_ae()
2676 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_gen_ae()
2682 struct irdma_pci_f *rf = iwqp->iwdev->rf; in irdma_flush_wqes() local
2718 (void)irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &info, in irdma_flush_wqes()