Lines Matching refs:rvu
27 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
29 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
31 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);
33 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
61 static void rvu_setup_hw_capabilities(struct rvu *rvu) in rvu_setup_hw_capabilities() argument
63 struct rvu_hwinfo *hw = rvu->hw; in rvu_setup_hw_capabilities()
73 hw->rvu = rvu; in rvu_setup_hw_capabilities()
75 if (is_rvu_pre_96xx_C0(rvu)) { in rvu_setup_hw_capabilities()
82 if (is_rvu_96xx_A0(rvu) || is_rvu_95xx_A0(rvu)) in rvu_setup_hw_capabilities()
85 if (!is_rvu_pre_96xx_C0(rvu)) in rvu_setup_hw_capabilities()
88 if (!is_rvu_otx2(rvu)) in rvu_setup_hw_capabilities()
91 if (is_rvu_npc_hash_extract_en(rvu)) in rvu_setup_hw_capabilities()
98 int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero) in rvu_poll_reg() argument
105 reg = rvu->afreg_base + ((block << 28) | offset); in rvu_poll_reg()
225 int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot) in rvu_get_lf() argument
230 mutex_lock(&rvu->rsrc_lock); in rvu_get_lf()
234 mutex_unlock(&rvu->rsrc_lock); in rvu_get_lf()
240 mutex_unlock(&rvu->rsrc_lock); in rvu_get_lf()
253 int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc) in rvu_get_blkaddr() argument
294 devnum = rvu_get_hwvf(rvu, pcifunc); in rvu_get_blkaddr()
306 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16)); in rvu_get_blkaddr()
314 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16)); in rvu_get_blkaddr()
322 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16)); in rvu_get_blkaddr()
330 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16)); in rvu_get_blkaddr()
336 if (is_block_implemented(rvu->hw, blkaddr)) in rvu_get_blkaddr()
341 static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf, in rvu_update_rsrc_map() argument
350 dev_err(&rvu->pdev->dev, in rvu_update_rsrc_map()
359 devnum = rvu_get_hwvf(rvu, pcifunc); in rvu_update_rsrc_map()
400 rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs); in rvu_update_rsrc_map()
408 void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf) in rvu_get_pf_numvfs() argument
413 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); in rvu_get_pf_numvfs()
420 int rvu_get_hwvf(struct rvu *rvu, int pcifunc) in rvu_get_hwvf() argument
429 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); in rvu_get_hwvf()
434 struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc) in rvu_get_pfvf() argument
438 return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)]; in rvu_get_pfvf()
440 return &rvu->pf[rvu_get_pf(pcifunc)]; in rvu_get_pfvf()
443 static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc) in is_pf_func_valid() argument
449 if (pf >= rvu->hw->total_pfs) in is_pf_func_valid()
457 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); in is_pf_func_valid()
476 static void rvu_check_block_implemented(struct rvu *rvu) in rvu_check_block_implemented() argument
478 struct rvu_hwinfo *hw = rvu->hw; in rvu_check_block_implemented()
486 cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid)); in rvu_check_block_implemented()
492 static void rvu_setup_rvum_blk_revid(struct rvu *rvu) in rvu_setup_rvum_blk_revid() argument
494 rvu_write64(rvu, BLKADDR_RVUM, in rvu_setup_rvum_blk_revid()
499 static void rvu_clear_rvum_blk_revid(struct rvu *rvu) in rvu_clear_rvum_blk_revid() argument
501 rvu_write64(rvu, BLKADDR_RVUM, in rvu_clear_rvum_blk_revid()
505 int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf) in rvu_lf_reset() argument
512 rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12)); in rvu_lf_reset()
513 err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12), in rvu_lf_reset()
518 static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg) in rvu_block_reset() argument
520 struct rvu_block *block = &rvu->hw->block[blkaddr]; in rvu_block_reset()
526 rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0)); in rvu_block_reset()
527 err = rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true); in rvu_block_reset()
529 dev_err(rvu->dev, "HW block:%d reset timeout retrying again\n", blkaddr); in rvu_block_reset()
530 while (rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true) == -EBUSY) in rvu_block_reset()
535 static void rvu_reset_all_blocks(struct rvu *rvu) in rvu_reset_all_blocks() argument
538 rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST); in rvu_reset_all_blocks()
539 rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST); in rvu_reset_all_blocks()
540 rvu_block_reset(rvu, BLKADDR_NIX1, NIX_AF_BLK_RST); in rvu_reset_all_blocks()
541 rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST); in rvu_reset_all_blocks()
542 rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST); in rvu_reset_all_blocks()
543 rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST); in rvu_reset_all_blocks()
544 rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST); in rvu_reset_all_blocks()
545 rvu_block_reset(rvu, BLKADDR_CPT1, CPT_AF_BLK_RST); in rvu_reset_all_blocks()
546 rvu_block_reset(rvu, BLKADDR_NDC_NIX0_RX, NDC_AF_BLK_RST); in rvu_reset_all_blocks()
547 rvu_block_reset(rvu, BLKADDR_NDC_NIX0_TX, NDC_AF_BLK_RST); in rvu_reset_all_blocks()
548 rvu_block_reset(rvu, BLKADDR_NDC_NIX1_RX, NDC_AF_BLK_RST); in rvu_reset_all_blocks()
549 rvu_block_reset(rvu, BLKADDR_NDC_NIX1_TX, NDC_AF_BLK_RST); in rvu_reset_all_blocks()
550 rvu_block_reset(rvu, BLKADDR_NDC_NPA0, NDC_AF_BLK_RST); in rvu_reset_all_blocks()
553 static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block) in rvu_scan_block() argument
560 cfg = rvu_read64(rvu, block->addr, in rvu_scan_block()
569 pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF); in rvu_scan_block()
570 rvu_update_rsrc_map(rvu, pfvf, block, in rvu_scan_block()
574 rvu_set_msix_offset(rvu, pfvf, block, lf); in rvu_scan_block()
578 static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf) in rvu_check_min_msix_vec() argument
586 dev_warn(rvu->dev, in rvu_check_min_msix_vec()
600 dev_warn(rvu->dev, in rvu_check_min_msix_vec()
605 static int rvu_setup_msix_resources(struct rvu *rvu) in rvu_setup_msix_resources() argument
607 struct rvu_hwinfo *hw = rvu->hw; in rvu_setup_msix_resources()
615 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); in rvu_setup_msix_resources()
620 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf); in rvu_setup_msix_resources()
622 pfvf = &rvu->pf[pf]; in rvu_setup_msix_resources()
624 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf)); in rvu_setup_msix_resources()
626 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0); in rvu_setup_msix_resources()
634 pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max, in rvu_setup_msix_resources()
649 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf)); in rvu_setup_msix_resources()
653 rvu_write64(rvu, BLKADDR_RVUM, in rvu_setup_msix_resources()
658 pfvf = &rvu->hwvf[hwvf + vf]; in rvu_setup_msix_resources()
660 cfg = rvu_read64(rvu, BLKADDR_RVUM, in rvu_setup_msix_resources()
663 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1); in rvu_setup_msix_resources()
671 devm_kcalloc(rvu->dev, pfvf->msix.max, in rvu_setup_msix_resources()
680 cfg = rvu_read64(rvu, BLKADDR_RVUM, in rvu_setup_msix_resources()
685 rvu_write64(rvu, BLKADDR_RVUM, in rvu_setup_msix_resources()
695 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST); in rvu_setup_msix_resources()
697 if (rvu->fwdata && rvu->fwdata->msixtr_base) in rvu_setup_msix_resources()
698 phy_addr = rvu->fwdata->msixtr_base; in rvu_setup_msix_resources()
700 phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE); in rvu_setup_msix_resources()
702 iova = dma_map_resource(rvu->dev, phy_addr, in rvu_setup_msix_resources()
706 if (dma_mapping_error(rvu->dev, iova)) in rvu_setup_msix_resources()
709 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova); in rvu_setup_msix_resources()
710 rvu->msix_base_iova = iova; in rvu_setup_msix_resources()
711 rvu->msixtr_base_phy = phy_addr; in rvu_setup_msix_resources()
716 static void rvu_reset_msix(struct rvu *rvu) in rvu_reset_msix() argument
719 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, in rvu_reset_msix()
720 rvu->msixtr_base_phy); in rvu_reset_msix()
723 static void rvu_free_hw_resources(struct rvu *rvu) in rvu_free_hw_resources() argument
725 struct rvu_hwinfo *hw = rvu->hw; in rvu_free_hw_resources()
731 rvu_npa_freemem(rvu); in rvu_free_hw_resources()
732 rvu_npc_freemem(rvu); in rvu_free_hw_resources()
733 rvu_nix_freemem(rvu); in rvu_free_hw_resources()
743 pfvf = &rvu->pf[id]; in rvu_free_hw_resources()
748 pfvf = &rvu->hwvf[id]; in rvu_free_hw_resources()
753 if (!rvu->msix_base_iova) in rvu_free_hw_resources()
755 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST); in rvu_free_hw_resources()
757 dma_unmap_resource(rvu->dev, rvu->msix_base_iova, in rvu_free_hw_resources()
761 rvu_reset_msix(rvu); in rvu_free_hw_resources()
762 mutex_destroy(&rvu->rsrc_lock); in rvu_free_hw_resources()
765 static void rvu_setup_pfvf_macaddress(struct rvu *rvu) in rvu_setup_pfvf_macaddress() argument
767 struct rvu_hwinfo *hw = rvu->hw; in rvu_setup_pfvf_macaddress()
777 if (!is_pf_cgxmapped(rvu, pf)) in rvu_setup_pfvf_macaddress()
780 pfvf = &rvu->pf[pf]; in rvu_setup_pfvf_macaddress()
781 if (rvu->fwdata && pf < PF_MACNUM_MAX) { in rvu_setup_pfvf_macaddress()
782 mac = &rvu->fwdata->pf_macs[pf]; in rvu_setup_pfvf_macaddress()
794 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf); in rvu_setup_pfvf_macaddress()
796 pfvf = &rvu->hwvf[hwvf]; in rvu_setup_pfvf_macaddress()
797 if (rvu->fwdata && hwvf < VF_MACNUM_MAX) { in rvu_setup_pfvf_macaddress()
798 mac = &rvu->fwdata->vf_macs[hwvf]; in rvu_setup_pfvf_macaddress()
811 static int rvu_fwdata_init(struct rvu *rvu) in rvu_fwdata_init() argument
820 rvu->fwdata = ioremap_wc(fwdbase, sizeof(struct rvu_fwdata)); in rvu_fwdata_init()
821 if (!rvu->fwdata) in rvu_fwdata_init()
823 if (!is_rvu_fwdata_valid(rvu)) { in rvu_fwdata_init()
824 dev_err(rvu->dev, in rvu_fwdata_init()
826 iounmap(rvu->fwdata); in rvu_fwdata_init()
827 rvu->fwdata = NULL; in rvu_fwdata_init()
832 dev_info(rvu->dev, "Unable to fetch 'fwdata' from firmware\n"); in rvu_fwdata_init()
836 static void rvu_fwdata_exit(struct rvu *rvu) in rvu_fwdata_exit() argument
838 if (rvu->fwdata) in rvu_fwdata_exit()
839 iounmap(rvu->fwdata); in rvu_fwdata_exit()
842 static int rvu_setup_nix_hw_resource(struct rvu *rvu, int blkaddr) in rvu_setup_nix_hw_resource() argument
844 struct rvu_hwinfo *hw = rvu->hw; in rvu_setup_nix_hw_resource()
854 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); in rvu_setup_nix_hw_resource()
865 block->rvu = rvu; in rvu_setup_nix_hw_resource()
867 rvu->nix_blkaddr[blkid] = blkaddr; in rvu_setup_nix_hw_resource()
871 static int rvu_setup_cpt_hw_resource(struct rvu *rvu, int blkaddr) in rvu_setup_cpt_hw_resource() argument
873 struct rvu_hwinfo *hw = rvu->hw; in rvu_setup_cpt_hw_resource()
883 cfg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0); in rvu_setup_cpt_hw_resource()
895 block->rvu = rvu; in rvu_setup_cpt_hw_resource()
900 static void rvu_get_lbk_bufsize(struct rvu *rvu) in rvu_get_lbk_bufsize() argument
918 rvu->hw->lbk_bufsize = FIELD_GET(LBK_CONST_BUF_SIZE, lbk_const); in rvu_get_lbk_bufsize()
925 static int rvu_setup_hw_resources(struct rvu *rvu) in rvu_setup_hw_resources() argument
927 struct rvu_hwinfo *hw = rvu->hw; in rvu_setup_hw_resources()
933 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST); in rvu_setup_hw_resources()
942 cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST); in rvu_setup_hw_resources()
953 block->rvu = rvu; in rvu_setup_hw_resources()
957 dev_err(rvu->dev, in rvu_setup_hw_resources()
963 err = rvu_setup_nix_hw_resource(rvu, BLKADDR_NIX0); in rvu_setup_hw_resources()
965 dev_err(rvu->dev, in rvu_setup_hw_resources()
970 err = rvu_setup_nix_hw_resource(rvu, BLKADDR_NIX1); in rvu_setup_hw_resources()
972 dev_err(rvu->dev, in rvu_setup_hw_resources()
981 cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST); in rvu_setup_hw_resources()
993 block->rvu = rvu; in rvu_setup_hw_resources()
997 dev_err(rvu->dev, in rvu_setup_hw_resources()
1018 block->rvu = rvu; in rvu_setup_hw_resources()
1022 dev_err(rvu->dev, in rvu_setup_hw_resources()
1032 cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST); in rvu_setup_hw_resources()
1044 block->rvu = rvu; in rvu_setup_hw_resources()
1048 dev_err(rvu->dev, in rvu_setup_hw_resources()
1054 err = rvu_setup_cpt_hw_resource(rvu, BLKADDR_CPT0); in rvu_setup_hw_resources()
1056 dev_err(rvu->dev, in rvu_setup_hw_resources()
1060 err = rvu_setup_cpt_hw_resource(rvu, BLKADDR_CPT1); in rvu_setup_hw_resources()
1062 dev_err(rvu->dev, in rvu_setup_hw_resources()
1068 rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs, in rvu_setup_hw_resources()
1070 if (!rvu->pf) { in rvu_setup_hw_resources()
1071 dev_err(rvu->dev, in rvu_setup_hw_resources()
1076 rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs, in rvu_setup_hw_resources()
1078 if (!rvu->hwvf) { in rvu_setup_hw_resources()
1079 dev_err(rvu->dev, in rvu_setup_hw_resources()
1084 mutex_init(&rvu->rsrc_lock); in rvu_setup_hw_resources()
1086 rvu_fwdata_init(rvu); in rvu_setup_hw_resources()
1088 err = rvu_setup_msix_resources(rvu); in rvu_setup_hw_resources()
1090 dev_err(rvu->dev, in rvu_setup_hw_resources()
1101 block->fn_map = devm_kcalloc(rvu->dev, block->lf.max, in rvu_setup_hw_resources()
1111 rvu_scan_block(rvu, block); in rvu_setup_hw_resources()
1114 err = rvu_set_channels_base(rvu); in rvu_setup_hw_resources()
1118 err = rvu_npc_init(rvu); in rvu_setup_hw_resources()
1120 dev_err(rvu->dev, "%s: Failed to initialize npc\n", __func__); in rvu_setup_hw_resources()
1124 err = rvu_cgx_init(rvu); in rvu_setup_hw_resources()
1126 dev_err(rvu->dev, "%s: Failed to initialize cgx\n", __func__); in rvu_setup_hw_resources()
1130 err = rvu_npc_exact_init(rvu); in rvu_setup_hw_resources()
1132 dev_err(rvu->dev, "failed to initialize exact match table\n"); in rvu_setup_hw_resources()
1137 rvu_setup_pfvf_macaddress(rvu); in rvu_setup_hw_resources()
1139 err = rvu_npa_init(rvu); in rvu_setup_hw_resources()
1141 dev_err(rvu->dev, "%s: Failed to initialize npa\n", __func__); in rvu_setup_hw_resources()
1145 rvu_get_lbk_bufsize(rvu); in rvu_setup_hw_resources()
1147 err = rvu_nix_init(rvu); in rvu_setup_hw_resources()
1149 dev_err(rvu->dev, "%s: Failed to initialize nix\n", __func__); in rvu_setup_hw_resources()
1153 err = rvu_sdp_init(rvu); in rvu_setup_hw_resources()
1155 dev_err(rvu->dev, "%s: Failed to initialize sdp\n", __func__); in rvu_setup_hw_resources()
1159 rvu_program_channels(rvu); in rvu_setup_hw_resources()
1161 err = rvu_mcs_init(rvu); in rvu_setup_hw_resources()
1163 dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__); in rvu_setup_hw_resources()
1167 err = rvu_cpt_init(rvu); in rvu_setup_hw_resources()
1169 dev_err(rvu->dev, "%s: Failed to initialize cpt\n", __func__); in rvu_setup_hw_resources()
1176 rvu_mcs_exit(rvu); in rvu_setup_hw_resources()
1178 rvu_nix_freemem(rvu); in rvu_setup_hw_resources()
1180 rvu_npa_freemem(rvu); in rvu_setup_hw_resources()
1182 rvu_cgx_exit(rvu); in rvu_setup_hw_resources()
1184 rvu_npc_freemem(rvu); in rvu_setup_hw_resources()
1185 rvu_fwdata_exit(rvu); in rvu_setup_hw_resources()
1187 rvu_reset_msix(rvu); in rvu_setup_hw_resources()
1192 void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq) in rvu_aq_free() argument
1197 qmem_free(rvu->dev, aq->inst); in rvu_aq_free()
1198 qmem_free(rvu->dev, aq->res); in rvu_aq_free()
1199 devm_kfree(rvu->dev, aq); in rvu_aq_free()
1202 int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue, in rvu_aq_alloc() argument
1208 *ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL); in rvu_aq_alloc()
1214 err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size); in rvu_aq_alloc()
1216 devm_kfree(rvu->dev, aq); in rvu_aq_alloc()
1221 err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size); in rvu_aq_alloc()
1223 rvu_aq_free(rvu, aq); in rvu_aq_alloc()
1231 int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req, in rvu_mbox_handler_ready() argument
1234 if (rvu->fwdata) { in rvu_mbox_handler_ready()
1235 rsp->rclk_freq = rvu->fwdata->rclk; in rvu_mbox_handler_ready()
1236 rsp->sclk_freq = rvu->fwdata->sclk; in rvu_mbox_handler_ready()
1287 bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype) in is_pffunc_map_valid() argument
1291 if (!is_pf_func_valid(rvu, pcifunc)) in is_pffunc_map_valid()
1294 pfvf = rvu_get_pfvf(rvu, pcifunc); in is_pffunc_map_valid()
1303 static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block, in rvu_lookup_rsrc() argument
1309 rvu_write64(rvu, block->addr, block->lookup_reg, val); in rvu_lookup_rsrc()
1312 while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13)) in rvu_lookup_rsrc()
1315 val = rvu_read64(rvu, block->addr, block->lookup_reg); in rvu_lookup_rsrc()
1324 int rvu_get_blkaddr_from_slot(struct rvu *rvu, int blktype, u16 pcifunc, in rvu_get_blkaddr_from_slot() argument
1327 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); in rvu_get_blkaddr_from_slot()
1341 block = &rvu->hw->block[blkaddr]; in rvu_get_blkaddr_from_slot()
1344 if (!is_block_implemented(rvu->hw, blkaddr)) in rvu_get_blkaddr_from_slot()
1378 static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype) in rvu_detach_block() argument
1380 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); in rvu_detach_block()
1381 struct rvu_hwinfo *hw = rvu->hw; in rvu_detach_block()
1386 blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc); in rvu_detach_block()
1400 lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot); in rvu_detach_block()
1405 rvu_write64(rvu, blkaddr, block->lfcfg_reg | in rvu_detach_block()
1409 rvu_update_rsrc_map(rvu, pfvf, block, in rvu_detach_block()
1416 rvu_clear_msix_offset(rvu, pfvf, block, lf); in rvu_detach_block()
1420 static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach, in rvu_detach_rsrcs() argument
1423 struct rvu_hwinfo *hw = rvu->hw; in rvu_detach_rsrcs()
1428 mutex_lock(&rvu->rsrc_lock); in rvu_detach_rsrcs()
1459 rvu_detach_block(rvu, pcifunc, block->type); in rvu_detach_rsrcs()
1462 mutex_unlock(&rvu->rsrc_lock); in rvu_detach_rsrcs()
1466 int rvu_mbox_handler_detach_resources(struct rvu *rvu, in rvu_mbox_handler_detach_resources() argument
1470 return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc); in rvu_mbox_handler_detach_resources()
1473 int rvu_get_nix_blkaddr(struct rvu *rvu, u16 pcifunc) in rvu_get_nix_blkaddr() argument
1475 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); in rvu_get_nix_blkaddr()
1479 pf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); in rvu_get_nix_blkaddr()
1482 if (is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { in rvu_get_nix_blkaddr()
1491 if (!is_block_implemented(rvu->hw, BLKADDR_NIX1)) in rvu_get_nix_blkaddr()
1516 static int rvu_get_attach_blkaddr(struct rvu *rvu, int blktype, in rvu_get_attach_blkaddr() argument
1523 blkaddr = rvu_get_nix_blkaddr(rvu, pcifunc); in rvu_get_attach_blkaddr()
1527 return rvu_get_blkaddr(rvu, blktype, 0); in rvu_get_attach_blkaddr()
1534 return rvu_get_blkaddr(rvu, blktype, 0); in rvu_get_attach_blkaddr()
1537 if (is_block_implemented(rvu->hw, blkaddr)) in rvu_get_attach_blkaddr()
1543 static void rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype, in rvu_attach_block() argument
1546 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); in rvu_attach_block()
1547 struct rvu_hwinfo *hw = rvu->hw; in rvu_attach_block()
1556 blkaddr = rvu_get_attach_blkaddr(rvu, blktype, pcifunc, attach); in rvu_attach_block()
1571 rvu_write64(rvu, blkaddr, block->lfcfg_reg | in rvu_attach_block()
1573 rvu_update_rsrc_map(rvu, pfvf, block, in rvu_attach_block()
1577 rvu_set_msix_offset(rvu, pfvf, block, lf); in rvu_attach_block()
1581 static int rvu_check_rsrc_availability(struct rvu *rvu, in rvu_check_rsrc_availability() argument
1584 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); in rvu_check_rsrc_availability()
1586 struct rvu_hwinfo *hw = rvu->hw; in rvu_check_rsrc_availability()
1596 dev_err(&rvu->pdev->dev, in rvu_check_rsrc_availability()
1604 blkaddr = rvu_get_attach_blkaddr(rvu, BLKTYPE_NIX, in rvu_check_rsrc_availability()
1613 dev_err(&rvu->pdev->dev, in rvu_check_rsrc_availability()
1623 dev_err(&rvu->pdev->dev, in rvu_check_rsrc_availability()
1639 dev_err(&rvu->pdev->dev, in rvu_check_rsrc_availability()
1654 dev_err(&rvu->pdev->dev, in rvu_check_rsrc_availability()
1667 blkaddr = rvu_get_attach_blkaddr(rvu, BLKTYPE_CPT, in rvu_check_rsrc_availability()
1673 dev_err(&rvu->pdev->dev, in rvu_check_rsrc_availability()
1688 dev_info(rvu->dev, "Request for %s failed\n", block->name); in rvu_check_rsrc_availability()
1692 static bool rvu_attach_from_same_block(struct rvu *rvu, int blktype, in rvu_attach_from_same_block() argument
1697 blkaddr = rvu_get_attach_blkaddr(rvu, blktype, in rvu_attach_from_same_block()
1702 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, attach->hdr.pcifunc), in rvu_attach_from_same_block()
1708 int rvu_mbox_handler_attach_resources(struct rvu *rvu, in rvu_mbox_handler_attach_resources() argument
1717 rvu_detach_rsrcs(rvu, NULL, pcifunc); in rvu_mbox_handler_attach_resources()
1719 mutex_lock(&rvu->rsrc_lock); in rvu_mbox_handler_attach_resources()
1722 err = rvu_check_rsrc_availability(rvu, attach, pcifunc); in rvu_mbox_handler_attach_resources()
1728 rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1, attach); in rvu_mbox_handler_attach_resources()
1731 rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach); in rvu_mbox_handler_attach_resources()
1740 rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO); in rvu_mbox_handler_attach_resources()
1741 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, in rvu_mbox_handler_attach_resources()
1747 rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW); in rvu_mbox_handler_attach_resources()
1748 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, in rvu_mbox_handler_attach_resources()
1754 rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM); in rvu_mbox_handler_attach_resources()
1755 rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, in rvu_mbox_handler_attach_resources()
1761 rvu_attach_from_same_block(rvu, BLKTYPE_CPT, attach)) in rvu_mbox_handler_attach_resources()
1762 rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT); in rvu_mbox_handler_attach_resources()
1763 rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, in rvu_mbox_handler_attach_resources()
1768 mutex_unlock(&rvu->rsrc_lock); in rvu_mbox_handler_attach_resources()
1772 static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, in rvu_get_msix_offset() argument
1787 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, in rvu_set_msix_offset() argument
1793 cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg | in rvu_set_msix_offset()
1804 rvu_write64(rvu, block->addr, block->msixcfg_reg | in rvu_set_msix_offset()
1812 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, in rvu_clear_msix_offset() argument
1818 cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg | in rvu_clear_msix_offset()
1823 rvu_write64(rvu, block->addr, block->msixcfg_reg | in rvu_clear_msix_offset()
1826 offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf); in rvu_clear_msix_offset()
1836 int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req, in rvu_mbox_handler_msix_offset() argument
1839 struct rvu_hwinfo *hw = rvu->hw; in rvu_mbox_handler_msix_offset()
1844 pfvf = rvu_get_pfvf(rvu, pcifunc); in rvu_mbox_handler_msix_offset()
1849 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0); in rvu_mbox_handler_msix_offset()
1850 rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf); in rvu_mbox_handler_msix_offset()
1853 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); in rvu_mbox_handler_msix_offset()
1857 lf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); in rvu_mbox_handler_msix_offset()
1858 rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, blkaddr, lf); in rvu_mbox_handler_msix_offset()
1863 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot); in rvu_mbox_handler_msix_offset()
1865 rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf); in rvu_mbox_handler_msix_offset()
1870 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot); in rvu_mbox_handler_msix_offset()
1872 rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf); in rvu_mbox_handler_msix_offset()
1877 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot); in rvu_mbox_handler_msix_offset()
1879 rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf); in rvu_mbox_handler_msix_offset()
1884 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot); in rvu_mbox_handler_msix_offset()
1886 rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf); in rvu_mbox_handler_msix_offset()
1891 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT1], pcifunc, slot); in rvu_mbox_handler_msix_offset()
1893 rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT1, lf); in rvu_mbox_handler_msix_offset()
1899 int rvu_mbox_handler_free_rsrc_cnt(struct rvu *rvu, struct msg_req *req, in rvu_mbox_handler_free_rsrc_cnt() argument
1902 struct rvu_hwinfo *hw = rvu->hw; in rvu_mbox_handler_free_rsrc_cnt()
1907 mutex_lock(&rvu->rsrc_lock); in rvu_mbox_handler_free_rsrc_cnt()
1933 if (rvu->hw->cap.nix_fixed_txschq_mapping) { in rvu_mbox_handler_free_rsrc_cnt()
1939 if (!is_block_implemented(rvu->hw, BLKADDR_NIX1)) in rvu_mbox_handler_free_rsrc_cnt()
1963 if (!is_block_implemented(rvu->hw, BLKADDR_NIX1)) in rvu_mbox_handler_free_rsrc_cnt()
1987 mutex_unlock(&rvu->rsrc_lock); in rvu_mbox_handler_free_rsrc_cnt()
1992 int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req, in rvu_mbox_handler_vf_flr() argument
2000 cfg = rvu_read64(rvu, BLKADDR_RVUM, in rvu_mbox_handler_vf_flr()
2005 __rvu_flr_handler(rvu, pcifunc); in rvu_mbox_handler_vf_flr()
2012 int rvu_mbox_handler_get_hw_cap(struct rvu *rvu, struct msg_req *req, in rvu_mbox_handler_get_hw_cap() argument
2015 struct rvu_hwinfo *hw = rvu->hw; in rvu_mbox_handler_get_hw_cap()
2024 int rvu_mbox_handler_set_vf_perm(struct rvu *rvu, struct set_vf_perm *req, in rvu_mbox_handler_set_vf_perm() argument
2027 struct rvu_hwinfo *hw = rvu->hw; in rvu_mbox_handler_set_vf_perm()
2038 pfvf = rvu_get_pfvf(rvu, target); in rvu_mbox_handler_set_vf_perm()
2047 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, target); in rvu_mbox_handler_set_vf_perm()
2050 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], in rvu_mbox_handler_set_vf_perm()
2054 npc_enadis_default_mce_entry(rvu, target, nixlf, in rvu_mbox_handler_set_vf_perm()
2057 npc_enadis_default_mce_entry(rvu, target, nixlf, in rvu_mbox_handler_set_vf_perm()
2069 struct rvu *rvu = pci_get_drvdata(mbox->pdev); in rvu_process_mbox_msg() local
2098 err = rvu_mbox_handler_ ## _fn_name(rvu, \ in rvu_process_mbox_msg()
2119 struct rvu *rvu = mwork->rvu; in __rvu_mbox_handler() local
2129 mw = &rvu->afpf_wq_info; in __rvu_mbox_handler()
2132 mw = &rvu->afvf_wq_info; in __rvu_mbox_handler()
2173 dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n", in __rvu_mbox_handler()
2178 dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n", in __rvu_mbox_handler()
2204 struct rvu *rvu = mwork->rvu; in __rvu_mbox_up_handler() local
2214 mw = &rvu->afpf_wq_info; in __rvu_mbox_up_handler()
2217 mw = &rvu->afvf_wq_info; in __rvu_mbox_up_handler()
2229 dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n"); in __rvu_mbox_up_handler()
2239 dev_err(rvu->dev, in __rvu_mbox_up_handler()
2245 dev_err(rvu->dev, in __rvu_mbox_up_handler()
2256 dev_err(rvu->dev, in __rvu_mbox_up_handler()
2284 static int rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr, in rvu_get_mbox_regions() argument
2287 struct rvu_hwinfo *hw = rvu->hw; in rvu_get_mbox_regions()
2298 bar4 = rvu_read64(rvu, BLKADDR_RVUM, in rvu_get_mbox_regions()
2303 bar4 = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR); in rvu_get_mbox_regions()
2319 bar4 = rvu_read64(rvu, BLKADDR_RVUM, in rvu_get_mbox_regions()
2322 bar4 = rvu_read64(rvu, BLKADDR_RVUM, in rvu_get_mbox_regions()
2338 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, in rvu_mbox_init() argument
2358 reg_base = rvu->afreg_base; in rvu_mbox_init()
2359 err = rvu_get_mbox_regions(rvu, mbox_regions, num, TYPE_AFPF); in rvu_mbox_init()
2367 reg_base = rvu->pfreg_base; in rvu_mbox_init()
2368 err = rvu_get_mbox_regions(rvu, mbox_regions, num, TYPE_AFVF); in rvu_mbox_init()
2384 mw->mbox_wrk = devm_kcalloc(rvu->dev, num, in rvu_mbox_init()
2391 mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num, in rvu_mbox_init()
2398 err = otx2_mbox_regions_init(&mw->mbox, mbox_regions, rvu->pdev, in rvu_mbox_init()
2403 err = otx2_mbox_regions_init(&mw->mbox_up, mbox_regions, rvu->pdev, in rvu_mbox_init()
2410 mwork->rvu = rvu; in rvu_mbox_init()
2414 mwork->rvu = rvu; in rvu_mbox_init()
2494 struct rvu *rvu = (struct rvu *)rvu_irq; in rvu_mbox_intr_handler() local
2495 int vfs = rvu->vfs; in rvu_mbox_intr_handler()
2498 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT); in rvu_mbox_intr_handler()
2500 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr); in rvu_mbox_intr_handler()
2502 trace_otx2_msg_interrupt(rvu->pdev, "PF(s) to AF", intr); in rvu_mbox_intr_handler()
2507 rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr); in rvu_mbox_intr_handler()
2511 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1)); in rvu_mbox_intr_handler()
2512 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr); in rvu_mbox_intr_handler()
2514 rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr); in rvu_mbox_intr_handler()
2518 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0)); in rvu_mbox_intr_handler()
2519 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr); in rvu_mbox_intr_handler()
2521 trace_otx2_msg_interrupt(rvu->pdev, "VF(s) to AF", intr); in rvu_mbox_intr_handler()
2523 rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr); in rvu_mbox_intr_handler()
2528 static void rvu_enable_mbox_intr(struct rvu *rvu) in rvu_enable_mbox_intr() argument
2530 struct rvu_hwinfo *hw = rvu->hw; in rvu_enable_mbox_intr()
2533 rvu_write64(rvu, BLKADDR_RVUM, in rvu_enable_mbox_intr()
2537 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S, in rvu_enable_mbox_intr()
2541 static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr) in rvu_blklf_teardown() argument
2547 block = &rvu->hw->block[blkaddr]; in rvu_blklf_teardown()
2548 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), in rvu_blklf_teardown()
2553 lf = rvu_get_lf(rvu, block, pcifunc, slot); in rvu_blklf_teardown()
2559 rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf); in rvu_blklf_teardown()
2561 rvu_npa_lf_teardown(rvu, pcifunc, lf); in rvu_blklf_teardown()
2564 rvu_cpt_lf_teardown(rvu, pcifunc, block->addr, lf, in rvu_blklf_teardown()
2567 err = rvu_lf_reset(rvu, block, lf); in rvu_blklf_teardown()
2569 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n", in rvu_blklf_teardown()
2575 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc) in __rvu_flr_handler() argument
2577 if (rvu_npc_exact_has_match_table(rvu)) in __rvu_flr_handler()
2578 rvu_npc_exact_reset(rvu, pcifunc); in __rvu_flr_handler()
2580 mutex_lock(&rvu->flr_lock); in __rvu_flr_handler()
2586 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0); in __rvu_flr_handler()
2587 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX1); in __rvu_flr_handler()
2588 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0); in __rvu_flr_handler()
2589 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT1); in __rvu_flr_handler()
2590 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM); in __rvu_flr_handler()
2591 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW); in __rvu_flr_handler()
2592 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO); in __rvu_flr_handler()
2593 rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA); in __rvu_flr_handler()
2594 rvu_reset_lmt_map_tbl(rvu, pcifunc); in __rvu_flr_handler()
2595 rvu_detach_rsrcs(rvu, NULL, pcifunc); in __rvu_flr_handler()
2600 rvu_npc_free_mcam_entries(rvu, pcifunc, -1); in __rvu_flr_handler()
2602 mutex_unlock(&rvu->flr_lock); in __rvu_flr_handler()
2605 static void rvu_afvf_flr_handler(struct rvu *rvu, int vf) in rvu_afvf_flr_handler() argument
2610 __rvu_flr_handler(rvu, vf + 1); in rvu_afvf_flr_handler()
2618 rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); in rvu_afvf_flr_handler()
2619 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf)); in rvu_afvf_flr_handler()
2625 struct rvu *rvu = flrwork->rvu; in rvu_flr_handler() local
2630 pf = flrwork - rvu->flr_wrk; in rvu_flr_handler()
2631 if (pf >= rvu->hw->total_pfs) { in rvu_flr_handler()
2632 rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs); in rvu_flr_handler()
2636 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); in rvu_flr_handler()
2641 __rvu_flr_handler(rvu, (pcifunc | (vf + 1))); in rvu_flr_handler()
2643 __rvu_flr_handler(rvu, pcifunc); in rvu_flr_handler()
2646 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf)); in rvu_flr_handler()
2649 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S, BIT_ULL(pf)); in rvu_flr_handler()
2652 static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs) in rvu_afvf_queue_flr_work() argument
2660 intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg)); in rvu_afvf_queue_flr_work()
2668 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf)); in rvu_afvf_queue_flr_work()
2669 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf)); in rvu_afvf_queue_flr_work()
2671 dev = vf + start_vf + rvu->hw->total_pfs; in rvu_afvf_queue_flr_work()
2672 queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work); in rvu_afvf_queue_flr_work()
2678 struct rvu *rvu = (struct rvu *)rvu_irq; in rvu_flr_intr_handler() local
2682 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT); in rvu_flr_intr_handler()
2686 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { in rvu_flr_intr_handler()
2689 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT, in rvu_flr_intr_handler()
2692 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C, in rvu_flr_intr_handler()
2695 queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work); in rvu_flr_intr_handler()
2700 rvu_afvf_queue_flr_work(rvu, 0, 64); in rvu_flr_intr_handler()
2701 if (rvu->vfs > 64) in rvu_flr_intr_handler()
2702 rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64); in rvu_flr_intr_handler()
2707 static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr) in rvu_me_handle_vfset() argument
2717 rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf)); in rvu_me_handle_vfset()
2719 rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf)); in rvu_me_handle_vfset()
2727 struct rvu *rvu = (struct rvu *)rvu_irq; in rvu_me_vf_intr_handler() local
2731 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT); in rvu_me_vf_intr_handler()
2734 intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset)); in rvu_me_vf_intr_handler()
2736 rvu_me_handle_vfset(rvu, vfset, intr); in rvu_me_vf_intr_handler()
2745 struct rvu *rvu = (struct rvu *)rvu_irq; in rvu_me_pf_intr_handler() local
2749 intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT); in rvu_me_pf_intr_handler()
2754 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { in rvu_me_pf_intr_handler()
2757 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, in rvu_me_pf_intr_handler()
2760 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT, in rvu_me_pf_intr_handler()
2768 static void rvu_unregister_interrupts(struct rvu *rvu) in rvu_unregister_interrupts() argument
2772 rvu_cpt_unregister_interrupts(rvu); in rvu_unregister_interrupts()
2775 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C, in rvu_unregister_interrupts()
2776 INTR_MASK(rvu->hw->total_pfs) & ~1ULL); in rvu_unregister_interrupts()
2779 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C, in rvu_unregister_interrupts()
2780 INTR_MASK(rvu->hw->total_pfs) & ~1ULL); in rvu_unregister_interrupts()
2783 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C, in rvu_unregister_interrupts()
2784 INTR_MASK(rvu->hw->total_pfs) & ~1ULL); in rvu_unregister_interrupts()
2786 for (irq = 0; irq < rvu->num_vec; irq++) { in rvu_unregister_interrupts()
2787 if (rvu->irq_allocated[irq]) { in rvu_unregister_interrupts()
2788 free_irq(pci_irq_vector(rvu->pdev, irq), rvu); in rvu_unregister_interrupts()
2789 rvu->irq_allocated[irq] = false; in rvu_unregister_interrupts()
2793 pci_free_irq_vectors(rvu->pdev); in rvu_unregister_interrupts()
2794 rvu->num_vec = 0; in rvu_unregister_interrupts()
2797 static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu) in rvu_afvf_msix_vectors_num_ok() argument
2799 struct rvu_pfvf *pfvf = &rvu->pf[0]; in rvu_afvf_msix_vectors_num_ok()
2802 pfvf = &rvu->pf[0]; in rvu_afvf_msix_vectors_num_ok()
2803 offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff; in rvu_afvf_msix_vectors_num_ok()
2813 static int rvu_register_interrupts(struct rvu *rvu) in rvu_register_interrupts() argument
2817 rvu->num_vec = pci_msix_vec_count(rvu->pdev); in rvu_register_interrupts()
2819 rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec, in rvu_register_interrupts()
2821 if (!rvu->irq_name) in rvu_register_interrupts()
2824 rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec, in rvu_register_interrupts()
2826 if (!rvu->irq_allocated) in rvu_register_interrupts()
2830 ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec, in rvu_register_interrupts()
2831 rvu->num_vec, PCI_IRQ_MSIX); in rvu_register_interrupts()
2833 dev_err(rvu->dev, in rvu_register_interrupts()
2835 rvu->num_vec, ret); in rvu_register_interrupts()
2840 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox"); in rvu_register_interrupts()
2841 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX), in rvu_register_interrupts()
2843 &rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu); in rvu_register_interrupts()
2845 dev_err(rvu->dev, in rvu_register_interrupts()
2850 rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true; in rvu_register_interrupts()
2853 rvu_enable_mbox_intr(rvu); in rvu_register_interrupts()
2856 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE], in rvu_register_interrupts()
2858 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR), in rvu_register_interrupts()
2860 &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE], in rvu_register_interrupts()
2861 rvu); in rvu_register_interrupts()
2863 dev_err(rvu->dev, in rvu_register_interrupts()
2867 rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true; in rvu_register_interrupts()
2870 rvu_write64(rvu, BLKADDR_RVUM, in rvu_register_interrupts()
2871 RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs)); in rvu_register_interrupts()
2873 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S, in rvu_register_interrupts()
2874 INTR_MASK(rvu->hw->total_pfs) & ~1ULL); in rvu_register_interrupts()
2877 sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE], in rvu_register_interrupts()
2879 ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME), in rvu_register_interrupts()
2881 &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE], in rvu_register_interrupts()
2882 rvu); in rvu_register_interrupts()
2884 dev_err(rvu->dev, in rvu_register_interrupts()
2887 rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true; in rvu_register_interrupts()
2890 rvu_write64(rvu, BLKADDR_RVUM, in rvu_register_interrupts()
2891 RVU_AF_PFTRPEND, INTR_MASK(rvu->hw->total_pfs)); in rvu_register_interrupts()
2893 rvu_write64(rvu, BLKADDR_RVUM, in rvu_register_interrupts()
2894 RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs)); in rvu_register_interrupts()
2896 rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S, in rvu_register_interrupts()
2897 INTR_MASK(rvu->hw->total_pfs) & ~1ULL); in rvu_register_interrupts()
2899 if (!rvu_afvf_msix_vectors_num_ok(rvu)) in rvu_register_interrupts()
2903 pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM, in rvu_register_interrupts()
2908 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0"); in rvu_register_interrupts()
2909 ret = request_irq(pci_irq_vector(rvu->pdev, offset), in rvu_register_interrupts()
2911 &rvu->irq_name[offset * NAME_SIZE], in rvu_register_interrupts()
2912 rvu); in rvu_register_interrupts()
2914 dev_err(rvu->dev, in rvu_register_interrupts()
2917 rvu->irq_allocated[offset] = true; in rvu_register_interrupts()
2923 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1"); in rvu_register_interrupts()
2924 ret = request_irq(pci_irq_vector(rvu->pdev, offset), in rvu_register_interrupts()
2926 &rvu->irq_name[offset * NAME_SIZE], in rvu_register_interrupts()
2927 rvu); in rvu_register_interrupts()
2929 dev_err(rvu->dev, in rvu_register_interrupts()
2932 rvu->irq_allocated[offset] = true; in rvu_register_interrupts()
2936 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0"); in rvu_register_interrupts()
2937 ret = request_irq(pci_irq_vector(rvu->pdev, offset), in rvu_register_interrupts()
2939 &rvu->irq_name[offset * NAME_SIZE], rvu); in rvu_register_interrupts()
2941 dev_err(rvu->dev, in rvu_register_interrupts()
2945 rvu->irq_allocated[offset] = true; in rvu_register_interrupts()
2948 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1"); in rvu_register_interrupts()
2949 ret = request_irq(pci_irq_vector(rvu->pdev, offset), in rvu_register_interrupts()
2951 &rvu->irq_name[offset * NAME_SIZE], rvu); in rvu_register_interrupts()
2953 dev_err(rvu->dev, in rvu_register_interrupts()
2957 rvu->irq_allocated[offset] = true; in rvu_register_interrupts()
2961 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0"); in rvu_register_interrupts()
2962 ret = request_irq(pci_irq_vector(rvu->pdev, offset), in rvu_register_interrupts()
2964 &rvu->irq_name[offset * NAME_SIZE], rvu); in rvu_register_interrupts()
2966 dev_err(rvu->dev, in rvu_register_interrupts()
2970 rvu->irq_allocated[offset] = true; in rvu_register_interrupts()
2973 sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1"); in rvu_register_interrupts()
2974 ret = request_irq(pci_irq_vector(rvu->pdev, offset), in rvu_register_interrupts()
2976 &rvu->irq_name[offset * NAME_SIZE], rvu); in rvu_register_interrupts()
2978 dev_err(rvu->dev, in rvu_register_interrupts()
2982 rvu->irq_allocated[offset] = true; in rvu_register_interrupts()
2984 ret = rvu_cpt_register_interrupts(rvu); in rvu_register_interrupts()
2991 rvu_unregister_interrupts(rvu); in rvu_register_interrupts()
2995 static void rvu_flr_wq_destroy(struct rvu *rvu) in rvu_flr_wq_destroy() argument
2997 if (rvu->flr_wq) { in rvu_flr_wq_destroy()
2998 destroy_workqueue(rvu->flr_wq); in rvu_flr_wq_destroy()
2999 rvu->flr_wq = NULL; in rvu_flr_wq_destroy()
3003 static int rvu_flr_init(struct rvu *rvu) in rvu_flr_init() argument
3010 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { in rvu_flr_init()
3011 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); in rvu_flr_init()
3012 rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf), in rvu_flr_init()
3016 rvu->flr_wq = alloc_workqueue("rvu_afpf_flr", in rvu_flr_init()
3019 if (!rvu->flr_wq) in rvu_flr_init()
3022 num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev); in rvu_flr_init()
3023 rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs, in rvu_flr_init()
3025 if (!rvu->flr_wrk) { in rvu_flr_init()
3026 destroy_workqueue(rvu->flr_wq); in rvu_flr_init()
3031 rvu->flr_wrk[dev].rvu = rvu; in rvu_flr_init()
3032 INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler); in rvu_flr_init()
3035 mutex_init(&rvu->flr_lock); in rvu_flr_init()
3040 static void rvu_disable_afvf_intr(struct rvu *rvu) in rvu_disable_afvf_intr() argument
3042 int vfs = rvu->vfs; in rvu_disable_afvf_intr()
3044 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs)); in rvu_disable_afvf_intr()
3045 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs)); in rvu_disable_afvf_intr()
3046 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs)); in rvu_disable_afvf_intr()
3050 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), in rvu_disable_afvf_intr()
3052 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64)); in rvu_disable_afvf_intr()
3053 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64)); in rvu_disable_afvf_intr()
3056 static void rvu_enable_afvf_intr(struct rvu *rvu) in rvu_enable_afvf_intr() argument
3058 int vfs = rvu->vfs; in rvu_enable_afvf_intr()
3064 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs)); in rvu_enable_afvf_intr()
3065 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs)); in rvu_enable_afvf_intr()
3068 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs)); in rvu_enable_afvf_intr()
3069 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs)); in rvu_enable_afvf_intr()
3070 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs)); in rvu_enable_afvf_intr()
3076 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64)); in rvu_enable_afvf_intr()
3077 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1), in rvu_enable_afvf_intr()
3080 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64)); in rvu_enable_afvf_intr()
3081 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64)); in rvu_enable_afvf_intr()
3082 rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64)); in rvu_enable_afvf_intr()
3109 static int rvu_enable_sriov(struct rvu *rvu) in rvu_enable_sriov() argument
3111 struct pci_dev *pdev = rvu->pdev; in rvu_enable_sriov()
3114 if (!rvu_afvf_msix_vectors_num_ok(rvu)) { in rvu_enable_sriov()
3143 rvu->vfs = vfs; in rvu_enable_sriov()
3145 err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs, in rvu_enable_sriov()
3150 rvu_enable_afvf_intr(rvu); in rvu_enable_sriov()
3156 rvu_disable_afvf_intr(rvu); in rvu_enable_sriov()
3157 rvu_mbox_destroy(&rvu->afvf_wq_info); in rvu_enable_sriov()
3164 static void rvu_disable_sriov(struct rvu *rvu) in rvu_disable_sriov() argument
3166 rvu_disable_afvf_intr(rvu); in rvu_disable_sriov()
3167 rvu_mbox_destroy(&rvu->afvf_wq_info); in rvu_disable_sriov()
3168 pci_disable_sriov(rvu->pdev); in rvu_disable_sriov()
3171 static void rvu_update_module_params(struct rvu *rvu) in rvu_update_module_params() argument
3175 strscpy(rvu->mkex_pfl_name, in rvu_update_module_params()
3177 strscpy(rvu->kpu_pfl_name, in rvu_update_module_params()
3184 struct rvu *rvu; in rvu_probe() local
3187 rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL); in rvu_probe()
3188 if (!rvu) in rvu_probe()
3191 rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL); in rvu_probe()
3192 if (!rvu->hw) { in rvu_probe()
3193 devm_kfree(dev, rvu); in rvu_probe()
3197 pci_set_drvdata(pdev, rvu); in rvu_probe()
3198 rvu->pdev = pdev; in rvu_probe()
3199 rvu->dev = &pdev->dev; in rvu_probe()
3221 rvu->ptp = ptp_get(); in rvu_probe()
3222 if (IS_ERR(rvu->ptp)) { in rvu_probe()
3223 err = PTR_ERR(rvu->ptp); in rvu_probe()
3226 rvu->ptp = NULL; in rvu_probe()
3230 rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0); in rvu_probe()
3231 rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0); in rvu_probe()
3232 if (!rvu->afreg_base || !rvu->pfreg_base) { in rvu_probe()
3239 rvu_update_module_params(rvu); in rvu_probe()
3242 rvu_check_block_implemented(rvu); in rvu_probe()
3244 rvu_reset_all_blocks(rvu); in rvu_probe()
3246 rvu_setup_hw_capabilities(rvu); in rvu_probe()
3248 err = rvu_setup_hw_resources(rvu); in rvu_probe()
3253 err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF, in rvu_probe()
3254 rvu->hw->total_pfs, rvu_afpf_mbox_handler, in rvu_probe()
3261 err = rvu_flr_init(rvu); in rvu_probe()
3267 err = rvu_register_interrupts(rvu); in rvu_probe()
3273 err = rvu_register_dl(rvu); in rvu_probe()
3279 rvu_setup_rvum_blk_revid(rvu); in rvu_probe()
3282 err = rvu_enable_sriov(rvu); in rvu_probe()
3289 rvu_dbg_init(rvu); in rvu_probe()
3291 mutex_init(&rvu->rswitch.switch_lock); in rvu_probe()
3293 if (rvu->fwdata) in rvu_probe()
3294 ptp_start(rvu->ptp, rvu->fwdata->sclk, rvu->fwdata->ptp_ext_clk_rate, in rvu_probe()
3295 rvu->fwdata->ptp_ext_tstamp); in rvu_probe()
3299 rvu_unregister_dl(rvu); in rvu_probe()
3301 rvu_unregister_interrupts(rvu); in rvu_probe()
3303 rvu_flr_wq_destroy(rvu); in rvu_probe()
3305 rvu_mbox_destroy(&rvu->afpf_wq_info); in rvu_probe()
3307 rvu_cgx_exit(rvu); in rvu_probe()
3308 rvu_fwdata_exit(rvu); in rvu_probe()
3309 rvu_mcs_exit(rvu); in rvu_probe()
3310 rvu_reset_all_blocks(rvu); in rvu_probe()
3311 rvu_free_hw_resources(rvu); in rvu_probe()
3312 rvu_clear_rvum_blk_revid(rvu); in rvu_probe()
3314 ptp_put(rvu->ptp); in rvu_probe()
3321 devm_kfree(&pdev->dev, rvu->hw); in rvu_probe()
3322 devm_kfree(dev, rvu); in rvu_probe()
3328 struct rvu *rvu = pci_get_drvdata(pdev); in rvu_remove() local
3330 rvu_dbg_exit(rvu); in rvu_remove()
3331 rvu_unregister_dl(rvu); in rvu_remove()
3332 rvu_unregister_interrupts(rvu); in rvu_remove()
3333 rvu_flr_wq_destroy(rvu); in rvu_remove()
3334 rvu_cgx_exit(rvu); in rvu_remove()
3335 rvu_fwdata_exit(rvu); in rvu_remove()
3336 rvu_mcs_exit(rvu); in rvu_remove()
3337 rvu_mbox_destroy(&rvu->afpf_wq_info); in rvu_remove()
3338 rvu_disable_sriov(rvu); in rvu_remove()
3339 rvu_reset_all_blocks(rvu); in rvu_remove()
3340 rvu_free_hw_resources(rvu); in rvu_remove()
3341 rvu_clear_rvum_blk_revid(rvu); in rvu_remove()
3342 ptp_put(rvu->ptp); in rvu_remove()
3347 devm_kfree(&pdev->dev, rvu->hw); in rvu_remove()
3348 devm_kfree(&pdev->dev, rvu); in rvu_remove()