Lines Matching refs:dev
612 static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, in swiotlb_alloc_tlb() argument
621 if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) { in swiotlb_alloc_tlb()
627 return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, in swiotlb_alloc_tlb()
683 static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, in swiotlb_alloc_pool() argument
705 while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) { in swiotlb_alloc_pool()
777 struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr) in __swiotlb_find_pool() argument
779 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in __swiotlb_find_pool()
788 list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) { in __swiotlb_find_pool()
803 static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool) in swiotlb_del_pool() argument
807 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); in swiotlb_del_pool()
809 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); in swiotlb_del_pool()
820 void swiotlb_dev_init(struct device *dev) in swiotlb_dev_init() argument
822 dev->dma_io_tlb_mem = &io_tlb_default_mem; in swiotlb_dev_init()
824 INIT_LIST_HEAD(&dev->dma_io_tlb_pools); in swiotlb_dev_init()
825 spin_lock_init(&dev->dma_io_tlb_lock); in swiotlb_dev_init()
826 dev->dma_uses_io_tlb = false; in swiotlb_dev_init()
849 static unsigned int swiotlb_align_offset(struct device *dev, in swiotlb_align_offset() argument
852 return addr & dma_get_min_align_mask(dev) & in swiotlb_align_offset()
859 static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size, in swiotlb_bounce() argument
883 swiotlb_align_offset(dev, 0, orig_addr); in swiotlb_bounce()
889 dev_WARN_ONCE(dev, 1, in swiotlb_bounce()
1016 static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool, in swiotlb_search_pool_area() argument
1021 unsigned long boundary_mask = dma_get_seg_boundary(dev); in swiotlb_search_pool_area()
1023 phys_to_dma_unencrypted(dev, pool->start) & boundary_mask; in swiotlb_search_pool_area()
1025 unsigned int iotlb_align_mask = dma_get_min_align_mask(dev); in swiotlb_search_pool_area()
1027 unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr); in swiotlb_search_pool_area()
1119 inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots); in swiotlb_search_pool_area()
1141 static int swiotlb_search_area(struct device *dev, int start_cpu, in swiotlb_search_area() argument
1145 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_search_area()
1155 index = swiotlb_search_pool_area(dev, pool, area_index, in swiotlb_search_area()
1181 static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, in swiotlb_find_slots() argument
1185 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_find_slots()
1198 index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size, in swiotlb_find_slots()
1210 phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit); in swiotlb_find_slots()
1211 pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit, in swiotlb_find_slots()
1216 index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, in swiotlb_find_slots()
1224 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); in swiotlb_find_slots()
1225 list_add_rcu(&pool->node, &dev->dma_io_tlb_pools); in swiotlb_find_slots()
1226 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); in swiotlb_find_slots()
1230 WRITE_ONCE(dev->dma_uses_io_tlb, true); in swiotlb_find_slots()
1257 static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, in swiotlb_find_slots() argument
1265 *retpool = pool = &dev->dma_io_tlb_mem->defpool; in swiotlb_find_slots()
1268 index = swiotlb_search_pool_area(dev, pool, i, orig_addr, in swiotlb_find_slots()
1369 phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, in swiotlb_tbl_map_single() argument
1373 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_tbl_map_single()
1383 dev_warn_ratelimited(dev, in swiotlb_tbl_map_single()
1398 dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK, in swiotlb_tbl_map_single()
1401 offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr); in swiotlb_tbl_map_single()
1403 index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool); in swiotlb_tbl_map_single()
1406 dev_warn_ratelimited(dev, in swiotlb_tbl_map_single()
1416 dma_reset_need_sync(dev); in swiotlb_tbl_map_single()
1439 swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool); in swiotlb_tbl_map_single()
1443 static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr, in swiotlb_release_slots() argument
1447 unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr); in swiotlb_release_slots()
1494 dec_used(dev->dma_io_tlb_mem, nslots); in swiotlb_release_slots()
1510 static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr, in swiotlb_del_transient() argument
1516 dec_used(dev->dma_io_tlb_mem, pool->nslabs); in swiotlb_del_transient()
1517 swiotlb_del_pool(dev, pool); in swiotlb_del_transient()
1518 dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs); in swiotlb_del_transient()
1524 static inline bool swiotlb_del_transient(struct device *dev, in swiotlb_del_transient() argument
1535 void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr, in __swiotlb_tbl_unmap_single() argument
1544 swiotlb_bounce(dev, tlb_addr, mapping_size, in __swiotlb_tbl_unmap_single()
1547 if (swiotlb_del_transient(dev, tlb_addr, pool)) in __swiotlb_tbl_unmap_single()
1549 swiotlb_release_slots(dev, tlb_addr, pool); in __swiotlb_tbl_unmap_single()
1552 void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr, in __swiotlb_sync_single_for_device() argument
1557 swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool); in __swiotlb_sync_single_for_device()
1562 void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr, in __swiotlb_sync_single_for_cpu() argument
1567 swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool); in __swiotlb_sync_single_for_cpu()
1576 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size, in swiotlb_map() argument
1582 trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size); in swiotlb_map()
1584 swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs); in swiotlb_map()
1589 dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr); in swiotlb_map()
1590 if (unlikely(!dma_capable(dev, dma_addr, size, true))) { in swiotlb_map()
1591 __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir, in swiotlb_map()
1593 swiotlb_find_pool(dev, swiotlb_addr)); in swiotlb_map()
1594 dev_WARN_ONCE(dev, 1, in swiotlb_map()
1596 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); in swiotlb_map()
1600 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) in swiotlb_map()
1605 size_t swiotlb_max_mapping_size(struct device *dev) in swiotlb_max_mapping_size() argument
1607 int min_align_mask = dma_get_min_align_mask(dev); in swiotlb_max_mapping_size()
1629 bool is_swiotlb_active(struct device *dev) in is_swiotlb_active() argument
1631 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in is_swiotlb_active()
1751 struct page *swiotlb_alloc(struct device *dev, size_t size) in swiotlb_alloc() argument
1753 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_alloc()
1763 index = swiotlb_find_slots(dev, 0, size, align, &pool); in swiotlb_alloc()
1769 dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n", in swiotlb_alloc()
1771 swiotlb_release_slots(dev, tlb_addr, pool); in swiotlb_alloc()
1778 bool swiotlb_free(struct device *dev, struct page *page, size_t size) in swiotlb_free() argument
1783 pool = swiotlb_find_pool(dev, tlb_addr); in swiotlb_free()
1787 swiotlb_release_slots(dev, tlb_addr, pool); in swiotlb_free()
1793 struct device *dev) in rmem_swiotlb_device_init() argument
1802 dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping."); in rmem_swiotlb_device_init()
1850 dev->dma_io_tlb_mem = mem; in rmem_swiotlb_device_init()
1856 struct device *dev) in rmem_swiotlb_device_release() argument
1858 dev->dma_io_tlb_mem = &io_tlb_default_mem; in rmem_swiotlb_device_release()