Lines Matching refs:zdev
24 static int zpci_refresh_global(struct zpci_dev *zdev) in zpci_refresh_global() argument
26 return zpci_refresh_trans((u64) zdev->fh << 32, zdev->start_dma, in zpci_refresh_global()
27 zdev->iommu_pages * PAGE_SIZE); in zpci_refresh_global()
135 static int __dma_update_trans(struct zpci_dev *zdev, unsigned long pa, in __dma_update_trans() argument
147 spin_lock_irqsave(&zdev->dma_table_lock, irq_flags); in __dma_update_trans()
148 if (!zdev->dma_table) { in __dma_update_trans()
154 entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr); in __dma_update_trans()
170 entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr); in __dma_update_trans()
177 spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags); in __dma_update_trans()
181 static int __dma_purge_tlb(struct zpci_dev *zdev, dma_addr_t dma_addr, in __dma_purge_tlb() argument
195 if (!zdev->tlb_refresh) in __dma_purge_tlb()
202 ret = zpci_refresh_trans((u64) zdev->fh << 32, dma_addr, in __dma_purge_tlb()
206 if (zpci_refresh_global(zdev)) in __dma_purge_tlb()
209 spin_lock_irqsave(&zdev->iommu_bitmap_lock, irqflags); in __dma_purge_tlb()
210 bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap, in __dma_purge_tlb()
211 zdev->lazy_bitmap, zdev->iommu_pages); in __dma_purge_tlb()
212 bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages); in __dma_purge_tlb()
213 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, irqflags); in __dma_purge_tlb()
220 static int dma_update_trans(struct zpci_dev *zdev, unsigned long pa, in dma_update_trans() argument
225 rc = __dma_update_trans(zdev, pa, dma_addr, size, flags); in dma_update_trans()
229 rc = __dma_purge_tlb(zdev, dma_addr, size, flags); in dma_update_trans()
231 __dma_update_trans(zdev, pa, dma_addr, size, ZPCI_PTE_INVALID); in dma_update_trans()
265 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in __dma_alloc_iommu() local
267 return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages, in __dma_alloc_iommu()
268 start, size, zdev->start_dma >> PAGE_SHIFT, in __dma_alloc_iommu()
275 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in dma_alloc_address() local
278 spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags); in dma_alloc_address()
279 offset = __dma_alloc_iommu(dev, zdev->next_bit, size); in dma_alloc_address()
283 if (zpci_refresh_global(zdev)) in dma_alloc_address()
286 bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap, in dma_alloc_address()
287 zdev->lazy_bitmap, zdev->iommu_pages); in dma_alloc_address()
288 bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages); in dma_alloc_address()
295 zdev->next_bit = offset + size; in dma_alloc_address()
296 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); in dma_alloc_address()
298 return zdev->start_dma + offset * PAGE_SIZE; in dma_alloc_address()
301 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); in dma_alloc_address()
307 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in dma_free_address() local
310 offset = (dma_addr - zdev->start_dma) >> PAGE_SHIFT; in dma_free_address()
312 spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags); in dma_free_address()
313 if (!zdev->iommu_bitmap) in dma_free_address()
317 bitmap_clear(zdev->iommu_bitmap, offset, size); in dma_free_address()
319 bitmap_set(zdev->lazy_bitmap, offset, size); in dma_free_address()
322 spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); in dma_free_address()
340 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_map_pages() local
361 ret = dma_update_trans(zdev, pa, dma_addr, size, flags); in s390_dma_map_pages()
365 atomic64_add(nr_pages, &zdev->mapped_pages); in s390_dma_map_pages()
380 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_unmap_pages() local
385 ret = dma_update_trans(zdev, 0, dma_addr, npages * PAGE_SIZE, in s390_dma_unmap_pages()
393 atomic64_add(npages, &zdev->unmapped_pages); in s390_dma_unmap_pages()
401 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_alloc() local
418 atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages); in s390_dma_alloc()
428 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in s390_dma_free() local
431 atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages); in s390_dma_free()
442 struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); in __s390_dma_map_sg() local
459 ret = __dma_update_trans(zdev, pa, dma_addr, in __s390_dma_map_sg()
466 ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags); in __s390_dma_map_sg()
471 atomic64_add(nr_pages, &zdev->mapped_pages); in __s390_dma_map_sg()
476 dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base, in __s390_dma_map_sg()
548 int zpci_dma_init_device(struct zpci_dev *zdev) in zpci_dma_init_device() argument
557 WARN_ON(zdev->s390_domain); in zpci_dma_init_device()
559 spin_lock_init(&zdev->iommu_bitmap_lock); in zpci_dma_init_device()
560 spin_lock_init(&zdev->dma_table_lock); in zpci_dma_init_device()
562 zdev->dma_table = dma_alloc_cpu_table(); in zpci_dma_init_device()
563 if (!zdev->dma_table) { in zpci_dma_init_device()
581 zdev->start_dma = PAGE_ALIGN(zdev->start_dma); in zpci_dma_init_device()
582 zdev->iommu_size = min3(s390_iommu_aperture, in zpci_dma_init_device()
583 ZPCI_TABLE_SIZE_RT - zdev->start_dma, in zpci_dma_init_device()
584 zdev->end_dma - zdev->start_dma + 1); in zpci_dma_init_device()
585 zdev->end_dma = zdev->start_dma + zdev->iommu_size - 1; in zpci_dma_init_device()
586 zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT; in zpci_dma_init_device()
587 zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8); in zpci_dma_init_device()
588 if (!zdev->iommu_bitmap) { in zpci_dma_init_device()
593 zdev->lazy_bitmap = vzalloc(zdev->iommu_pages / 8); in zpci_dma_init_device()
594 if (!zdev->lazy_bitmap) { in zpci_dma_init_device()
600 if (zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, in zpci_dma_init_device()
601 (u64)zdev->dma_table)) { in zpci_dma_init_device()
608 vfree(zdev->iommu_bitmap); in zpci_dma_init_device()
609 zdev->iommu_bitmap = NULL; in zpci_dma_init_device()
610 vfree(zdev->lazy_bitmap); in zpci_dma_init_device()
611 zdev->lazy_bitmap = NULL; in zpci_dma_init_device()
613 dma_free_cpu_table(zdev->dma_table); in zpci_dma_init_device()
614 zdev->dma_table = NULL; in zpci_dma_init_device()
619 int zpci_dma_exit_device(struct zpci_dev *zdev) in zpci_dma_exit_device() argument
628 WARN_ON(zdev->s390_domain); in zpci_dma_exit_device()
629 if (zdev_enabled(zdev)) in zpci_dma_exit_device()
630 cc = zpci_unregister_ioat(zdev, 0); in zpci_dma_exit_device()
639 dma_cleanup_tables(zdev->dma_table); in zpci_dma_exit_device()
640 zdev->dma_table = NULL; in zpci_dma_exit_device()
641 vfree(zdev->iommu_bitmap); in zpci_dma_exit_device()
642 zdev->iommu_bitmap = NULL; in zpci_dma_exit_device()
643 vfree(zdev->lazy_bitmap); in zpci_dma_exit_device()
644 zdev->lazy_bitmap = NULL; in zpci_dma_exit_device()
645 zdev->next_bit = 0; in zpci_dma_exit_device()