Lines Matching refs:cptvf
16 void *cptvf; member
29 vq_post_process(cwqe->cptvf, cwqe->qno); in vq_work_handler()
32 static int init_worker_threads(struct cpt_vf *cptvf) in init_worker_threads() argument
34 struct pci_dev *pdev = cptvf->pdev; in init_worker_threads()
42 if (cptvf->nr_queues) { in init_worker_threads()
44 cptvf->nr_queues); in init_worker_threads()
47 for (i = 0; i < cptvf->nr_queues; i++) { in init_worker_threads()
51 cwqe_info->vq_wqe[i].cptvf = cptvf; in init_worker_threads()
54 cptvf->wqe_info = cwqe_info; in init_worker_threads()
59 static void cleanup_worker_threads(struct cpt_vf *cptvf) in cleanup_worker_threads() argument
62 struct pci_dev *pdev = cptvf->pdev; in cleanup_worker_threads()
65 cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; in cleanup_worker_threads()
69 if (cptvf->nr_queues) { in cleanup_worker_threads()
71 cptvf->nr_queues); in cleanup_worker_threads()
74 for (i = 0; i < cptvf->nr_queues; i++) in cleanup_worker_threads()
78 cptvf->wqe_info = NULL; in cleanup_worker_threads()
136 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) in init_pending_queues() argument
138 struct pci_dev *pdev = cptvf->pdev; in init_pending_queues()
144 ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues); in init_pending_queues()
154 static void cleanup_pending_queues(struct cpt_vf *cptvf) in cleanup_pending_queues() argument
156 struct pci_dev *pdev = cptvf->pdev; in cleanup_pending_queues()
158 if (!cptvf->nr_queues) in cleanup_pending_queues()
162 cptvf->nr_queues); in cleanup_pending_queues()
163 free_pending_queues(&cptvf->pqinfo); in cleanup_pending_queues()
166 static void free_command_queues(struct cpt_vf *cptvf, in free_command_queues() argument
172 struct pci_dev *pdev = cptvf->pdev; in free_command_queues()
176 for (i = 0; i < cptvf->nr_queues; i++) { in free_command_queues()
200 static int alloc_command_queues(struct cpt_vf *cptvf, in alloc_command_queues() argument
207 struct pci_dev *pdev = cptvf->pdev; in alloc_command_queues()
212 cptvf->qsize = min(qlen, cqinfo->qchunksize) * in alloc_command_queues()
218 for (i = 0; i < cptvf->nr_queues; i++) { in alloc_command_queues()
272 free_command_queues(cptvf, cqinfo); in alloc_command_queues()
276 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) in init_command_queues() argument
278 struct pci_dev *pdev = cptvf->pdev; in init_command_queues()
282 ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE, in init_command_queues()
286 cptvf->nr_queues); in init_command_queues()
293 static void cleanup_command_queues(struct cpt_vf *cptvf) in cleanup_command_queues() argument
295 struct pci_dev *pdev = cptvf->pdev; in cleanup_command_queues()
297 if (!cptvf->nr_queues) in cleanup_command_queues()
301 cptvf->nr_queues); in cleanup_command_queues()
302 free_command_queues(cptvf, &cptvf->cqinfo); in cleanup_command_queues()
305 static void cptvf_sw_cleanup(struct cpt_vf *cptvf) in cptvf_sw_cleanup() argument
307 cleanup_worker_threads(cptvf); in cptvf_sw_cleanup()
308 cleanup_pending_queues(cptvf); in cptvf_sw_cleanup()
309 cleanup_command_queues(cptvf); in cptvf_sw_cleanup()
312 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) in cptvf_sw_init() argument
314 struct pci_dev *pdev = cptvf->pdev; in cptvf_sw_init()
321 cptvf->nr_queues = nr_queues; in cptvf_sw_init()
323 ret = init_command_queues(cptvf, qlen); in cptvf_sw_init()
330 ret = init_pending_queues(cptvf, qlen, nr_queues); in cptvf_sw_init()
338 ret = init_worker_threads(cptvf); in cptvf_sw_init()
347 cleanup_worker_threads(cptvf); in cptvf_sw_init()
348 cleanup_pending_queues(cptvf); in cptvf_sw_init()
351 cleanup_command_queues(cptvf); in cptvf_sw_init()
356 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) in cptvf_free_irq_affinity() argument
358 irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); in cptvf_free_irq_affinity()
359 free_cpumask_var(cptvf->affinity_mask[vec]); in cptvf_free_irq_affinity()
362 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) in cptvf_write_vq_ctl() argument
366 vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0)); in cptvf_write_vq_ctl()
368 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u); in cptvf_write_vq_ctl()
371 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) in cptvf_write_vq_doorbell() argument
375 vqx_dbell.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_doorbell()
378 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), in cptvf_write_vq_doorbell()
382 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) in cptvf_write_vq_inprog() argument
386 vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); in cptvf_write_vq_inprog()
388 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u); in cptvf_write_vq_inprog()
391 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) in cptvf_write_vq_done_numwait() argument
395 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_done_numwait()
398 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), in cptvf_write_vq_done_numwait()
402 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) in cptvf_write_vq_done_timewait() argument
406 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_done_timewait()
409 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), in cptvf_write_vq_done_timewait()
413 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) in cptvf_enable_swerr_interrupts() argument
417 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, in cptvf_enable_swerr_interrupts()
421 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), in cptvf_enable_swerr_interrupts()
425 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) in cptvf_enable_mbox_interrupts() argument
429 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, in cptvf_enable_mbox_interrupts()
433 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), in cptvf_enable_mbox_interrupts()
437 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) in cptvf_enable_done_interrupts() argument
441 vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base, in cptvf_enable_done_interrupts()
445 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), in cptvf_enable_done_interrupts()
449 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) in cptvf_clear_dovf_intr() argument
453 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_dovf_intr()
457 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_dovf_intr()
461 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) in cptvf_clear_irde_intr() argument
465 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_irde_intr()
469 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_irde_intr()
473 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) in cptvf_clear_nwrp_intr() argument
477 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_nwrp_intr()
481 cpt_write_csr64(cptvf->reg_base, in cptvf_clear_nwrp_intr()
485 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) in cptvf_clear_mbox_intr() argument
489 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_mbox_intr()
493 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_mbox_intr()
497 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) in cptvf_clear_swerr_intr() argument
501 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_swerr_intr()
505 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_swerr_intr()
509 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) in cptvf_read_vf_misc_intr_status() argument
511 return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); in cptvf_read_vf_misc_intr_status()
516 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; in cptvf_misc_intr_handler() local
517 struct pci_dev *pdev = cptvf->pdev; in cptvf_misc_intr_handler()
520 intr = cptvf_read_vf_misc_intr_status(cptvf); in cptvf_misc_intr_handler()
524 intr, cptvf->vfid); in cptvf_misc_intr_handler()
525 cptvf_handle_mbox_intr(cptvf); in cptvf_misc_intr_handler()
526 cptvf_clear_mbox_intr(cptvf); in cptvf_misc_intr_handler()
528 cptvf_clear_dovf_intr(cptvf); in cptvf_misc_intr_handler()
530 cptvf_write_vq_doorbell(cptvf, 0); in cptvf_misc_intr_handler()
532 intr, cptvf->vfid); in cptvf_misc_intr_handler()
534 cptvf_clear_irde_intr(cptvf); in cptvf_misc_intr_handler()
536 intr, cptvf->vfid); in cptvf_misc_intr_handler()
538 cptvf_clear_nwrp_intr(cptvf); in cptvf_misc_intr_handler()
540 intr, cptvf->vfid); in cptvf_misc_intr_handler()
542 cptvf_clear_swerr_intr(cptvf); in cptvf_misc_intr_handler()
544 intr, cptvf->vfid); in cptvf_misc_intr_handler()
547 cptvf->vfid); in cptvf_misc_intr_handler()
553 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, in get_cptvf_vq_wqe() argument
558 if (unlikely(qno >= cptvf->nr_queues)) in get_cptvf_vq_wqe()
560 nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; in get_cptvf_vq_wqe()
565 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) in cptvf_read_vq_done_count() argument
569 vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0)); in cptvf_read_vq_done_count()
573 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, in cptvf_write_vq_done_ack() argument
578 vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_done_ack()
581 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), in cptvf_write_vq_done_ack()
587 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; in cptvf_done_intr_handler() local
588 struct pci_dev *pdev = cptvf->pdev; in cptvf_done_intr_handler()
590 u32 intr = cptvf_read_vq_done_count(cptvf); in cptvf_done_intr_handler()
598 cptvf_write_vq_done_ack(cptvf, intr); in cptvf_done_intr_handler()
599 wqe = get_cptvf_vq_wqe(cptvf, 0); in cptvf_done_intr_handler()
602 cptvf->vfid); in cptvf_done_intr_handler()
611 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) in cptvf_set_irq_affinity() argument
613 struct pci_dev *pdev = cptvf->pdev; in cptvf_set_irq_affinity()
616 if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], in cptvf_set_irq_affinity()
619 cptvf->vfid); in cptvf_set_irq_affinity()
623 cpu = cptvf->vfid % num_online_cpus(); in cptvf_set_irq_affinity()
624 cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), in cptvf_set_irq_affinity()
625 cptvf->affinity_mask[vec]); in cptvf_set_irq_affinity()
627 cptvf->affinity_mask[vec]); in cptvf_set_irq_affinity()
630 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) in cptvf_write_vq_saddr() argument
635 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); in cptvf_write_vq_saddr()
638 static void cptvf_device_init(struct cpt_vf *cptvf) in cptvf_device_init() argument
643 cptvf_write_vq_ctl(cptvf, 0); in cptvf_device_init()
645 cptvf_write_vq_doorbell(cptvf, 0); in cptvf_device_init()
647 cptvf_write_vq_inprog(cptvf, 0); in cptvf_device_init()
650 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); in cptvf_device_init()
651 cptvf_write_vq_saddr(cptvf, base_addr); in cptvf_device_init()
653 cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); in cptvf_device_init()
654 cptvf_write_vq_done_numwait(cptvf, 1); in cptvf_device_init()
656 cptvf_write_vq_ctl(cptvf, 1); in cptvf_device_init()
658 cptvf->flags |= CPT_FLAG_DEVICE_READY; in cptvf_device_init()
664 struct cpt_vf *cptvf; in cptvf_probe() local
667 cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); in cptvf_probe()
668 if (!cptvf) in cptvf_probe()
671 pci_set_drvdata(pdev, cptvf); in cptvf_probe()
672 cptvf->pdev = pdev; in cptvf_probe()
686 cptvf->flags |= CPT_FLAG_VF_DRIVER; in cptvf_probe()
694 cptvf->reg_base = pcim_iomap(pdev, 0, 0); in cptvf_probe()
695 if (!cptvf->reg_base) { in cptvf_probe()
701 cptvf->node = dev_to_node(&pdev->dev); in cptvf_probe()
712 cptvf); in cptvf_probe()
719 cptvf_enable_mbox_interrupts(cptvf); in cptvf_probe()
720 cptvf_enable_swerr_interrupts(cptvf); in cptvf_probe()
724 err = cptvf_check_pf_ready(cptvf); in cptvf_probe()
731 cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; in cptvf_probe()
732 err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); in cptvf_probe()
738 err = cptvf_send_vq_size_msg(cptvf); in cptvf_probe()
745 cptvf_device_init(cptvf); in cptvf_probe()
747 cptvf->vfgrp = 1; in cptvf_probe()
748 err = cptvf_send_vf_to_grp_msg(cptvf); in cptvf_probe()
754 cptvf->priority = 1; in cptvf_probe()
755 err = cptvf_send_vf_priority_msg(cptvf); in cptvf_probe()
763 cptvf); in cptvf_probe()
770 cptvf_enable_done_interrupts(cptvf); in cptvf_probe()
773 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in cptvf_probe()
774 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in cptvf_probe()
776 err = cptvf_send_vf_up(cptvf); in cptvf_probe()
781 err = cvm_crypto_init(cptvf); in cptvf_probe()
789 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in cptvf_probe()
790 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in cptvf_probe()
792 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); in cptvf_probe()
794 pci_free_irq_vectors(cptvf->pdev); in cptvf_probe()
806 struct cpt_vf *cptvf = pci_get_drvdata(pdev); in cptvf_remove() local
808 if (!cptvf) { in cptvf_remove()
814 if (cptvf_send_vf_down(cptvf)) { in cptvf_remove()
817 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in cptvf_remove()
818 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in cptvf_remove()
819 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); in cptvf_remove()
820 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); in cptvf_remove()
821 pci_free_irq_vectors(cptvf->pdev); in cptvf_remove()
822 cptvf_sw_cleanup(cptvf); in cptvf_remove()