Lines Matching refs:dir
156 size_t offset, size_t size, enum dma_data_direction dir, in dma_map_page_attrs() argument
162 BUG_ON(!valid_dma_direction(dir)); in dma_map_page_attrs()
169 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
171 addr = iommu_dma_map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
173 addr = ops->map_page(dev, page, offset, size, dir, attrs); in dma_map_page_attrs()
174 kmsan_handle_dma(page, offset, size, dir); in dma_map_page_attrs()
175 trace_dma_map_page(dev, page_to_phys(page) + offset, addr, size, dir, in dma_map_page_attrs()
177 debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); in dma_map_page_attrs()
184 enum dma_data_direction dir, unsigned long attrs) in dma_unmap_page_attrs() argument
188 BUG_ON(!valid_dma_direction(dir)); in dma_unmap_page_attrs()
191 dma_direct_unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
193 iommu_dma_unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
195 ops->unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
196 trace_dma_unmap_page(dev, addr, size, dir, attrs); in dma_unmap_page_attrs()
197 debug_dma_unmap_page(dev, addr, size, dir); in dma_unmap_page_attrs()
202 int nents, enum dma_data_direction dir, unsigned long attrs) in __dma_map_sg_attrs() argument
207 BUG_ON(!valid_dma_direction(dir)); in __dma_map_sg_attrs()
214 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); in __dma_map_sg_attrs()
216 ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); in __dma_map_sg_attrs()
218 ents = ops->map_sg(dev, sg, nents, dir, attrs); in __dma_map_sg_attrs()
221 kmsan_handle_dma_sg(sg, nents, dir); in __dma_map_sg_attrs()
222 trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); in __dma_map_sg_attrs()
223 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); in __dma_map_sg_attrs()
226 trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs); in __dma_map_sg_attrs()
251 int nents, enum dma_data_direction dir, unsigned long attrs) in dma_map_sg_attrs() argument
255 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); in dma_map_sg_attrs()
290 enum dma_data_direction dir, unsigned long attrs) in dma_map_sgtable() argument
294 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); in dma_map_sgtable()
303 int nents, enum dma_data_direction dir, in dma_unmap_sg_attrs() argument
308 BUG_ON(!valid_dma_direction(dir)); in dma_unmap_sg_attrs()
309 trace_dma_unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
310 debug_dma_unmap_sg(dev, sg, nents, dir); in dma_unmap_sg_attrs()
313 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
315 iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
317 ops->unmap_sg(dev, sg, nents, dir, attrs); in dma_unmap_sg_attrs()
322 size_t size, enum dma_data_direction dir, unsigned long attrs) in dma_map_resource() argument
327 BUG_ON(!valid_dma_direction(dir)); in dma_map_resource()
333 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); in dma_map_resource()
335 addr = iommu_dma_map_resource(dev, phys_addr, size, dir, attrs); in dma_map_resource()
337 addr = ops->map_resource(dev, phys_addr, size, dir, attrs); in dma_map_resource()
339 trace_dma_map_resource(dev, phys_addr, addr, size, dir, attrs); in dma_map_resource()
340 debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs); in dma_map_resource()
346 enum dma_data_direction dir, unsigned long attrs) in dma_unmap_resource() argument
350 BUG_ON(!valid_dma_direction(dir)); in dma_unmap_resource()
354 iommu_dma_unmap_resource(dev, addr, size, dir, attrs); in dma_unmap_resource()
356 ops->unmap_resource(dev, addr, size, dir, attrs); in dma_unmap_resource()
357 trace_dma_unmap_resource(dev, addr, size, dir, attrs); in dma_unmap_resource()
358 debug_dma_unmap_resource(dev, addr, size, dir); in dma_unmap_resource()
364 enum dma_data_direction dir) in __dma_sync_single_for_cpu() argument
368 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_single_for_cpu()
370 dma_direct_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
372 iommu_dma_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
374 ops->sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
375 trace_dma_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
376 debug_dma_sync_single_for_cpu(dev, addr, size, dir); in __dma_sync_single_for_cpu()
381 size_t size, enum dma_data_direction dir) in __dma_sync_single_for_device() argument
385 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_single_for_device()
387 dma_direct_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
389 iommu_dma_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
391 ops->sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
392 trace_dma_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
393 debug_dma_sync_single_for_device(dev, addr, size, dir); in __dma_sync_single_for_device()
398 int nelems, enum dma_data_direction dir) in __dma_sync_sg_for_cpu() argument
402 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_sg_for_cpu()
404 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
406 iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
408 ops->sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
409 trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
410 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); in __dma_sync_sg_for_cpu()
415 int nelems, enum dma_data_direction dir) in __dma_sync_sg_for_device() argument
419 BUG_ON(!valid_dma_direction(dir)); in __dma_sync_sg_for_device()
421 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
423 iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
425 ops->sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
426 trace_dma_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
427 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); in __dma_sync_sg_for_device()
686 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) in __dma_alloc_pages() argument
699 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); in __dma_alloc_pages()
701 return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp); in __dma_alloc_pages()
704 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); in __dma_alloc_pages()
708 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) in dma_alloc_pages() argument
710 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); in dma_alloc_pages()
714 size, dir, gfp, 0); in dma_alloc_pages()
715 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0); in dma_alloc_pages()
717 trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); in dma_alloc_pages()
724 dma_addr_t dma_handle, enum dma_data_direction dir) in __dma_free_pages() argument
730 dma_direct_free_pages(dev, size, page, dma_handle, dir); in __dma_free_pages()
732 dma_common_free_pages(dev, size, page, dma_handle, dir); in __dma_free_pages()
734 ops->free_pages(dev, size, page, dma_handle, dir); in __dma_free_pages()
738 dma_addr_t dma_handle, enum dma_data_direction dir) in dma_free_pages() argument
740 trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0); in dma_free_pages()
741 debug_dma_unmap_page(dev, dma_handle, size, dir); in dma_free_pages()
742 __dma_free_pages(dev, size, page, dma_handle, dir); in dma_free_pages()
760 enum dma_data_direction dir, gfp_t gfp) in alloc_single_sgt() argument
770 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); in alloc_single_sgt()
784 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) in dma_alloc_noncontiguous() argument
794 sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs); in dma_alloc_noncontiguous()
796 sgt = alloc_single_sgt(dev, size, dir, gfp); in dma_alloc_noncontiguous()
800 trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs); in dma_alloc_noncontiguous()
801 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); in dma_alloc_noncontiguous()
803 trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs); in dma_alloc_noncontiguous()
810 struct sg_table *sgt, enum dma_data_direction dir) in free_single_sgt() argument
813 dir); in free_single_sgt()
819 struct sg_table *sgt, enum dma_data_direction dir) in dma_free_noncontiguous() argument
821 trace_dma_free_sgt(dev, sgt, size, dir); in dma_free_noncontiguous()
822 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); in dma_free_noncontiguous()
825 iommu_dma_free_noncontiguous(dev, size, sgt, dir); in dma_free_noncontiguous()
827 free_single_sgt(dev, size, sgt, dir); in dma_free_noncontiguous()