Lines Matching refs:cookie
127 static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq) in fq_ring_free() argument
129 u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt); in fq_ring_free()
140 free_iova_fast(&cookie->iovad, in fq_ring_free()
148 static void fq_flush_iotlb(struct iommu_dma_cookie *cookie) in fq_flush_iotlb() argument
150 atomic64_inc(&cookie->fq_flush_start_cnt); in fq_flush_iotlb()
151 cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain); in fq_flush_iotlb()
152 atomic64_inc(&cookie->fq_flush_finish_cnt); in fq_flush_iotlb()
157 struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer); in fq_flush_timeout() local
160 atomic_set(&cookie->fq_timer_on, 0); in fq_flush_timeout()
161 fq_flush_iotlb(cookie); in fq_flush_timeout()
167 fq = per_cpu_ptr(cookie->fq, cpu); in fq_flush_timeout()
169 fq_ring_free(cookie, fq); in fq_flush_timeout()
174 static void queue_iova(struct iommu_dma_cookie *cookie, in queue_iova() argument
191 fq = raw_cpu_ptr(cookie->fq); in queue_iova()
199 fq_ring_free(cookie, fq); in queue_iova()
202 fq_flush_iotlb(cookie); in queue_iova()
203 fq_ring_free(cookie, fq); in queue_iova()
210 fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt); in queue_iova()
216 if (!atomic_read(&cookie->fq_timer_on) && in queue_iova()
217 !atomic_xchg(&cookie->fq_timer_on, 1)) in queue_iova()
218 mod_timer(&cookie->fq_timer, in queue_iova()
222 static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie) in iommu_dma_free_fq() argument
226 if (!cookie->fq) in iommu_dma_free_fq()
229 del_timer_sync(&cookie->fq_timer); in iommu_dma_free_fq()
232 struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu); in iommu_dma_free_fq()
238 free_percpu(cookie->fq); in iommu_dma_free_fq()
244 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_dma_init_fq() local
248 if (cookie->fq_domain) in iommu_dma_init_fq()
251 atomic64_set(&cookie->fq_flush_start_cnt, 0); in iommu_dma_init_fq()
252 atomic64_set(&cookie->fq_flush_finish_cnt, 0); in iommu_dma_init_fq()
272 cookie->fq = queue; in iommu_dma_init_fq()
274 timer_setup(&cookie->fq_timer, fq_flush_timeout, 0); in iommu_dma_init_fq()
275 atomic_set(&cookie->fq_timer_on, 0); in iommu_dma_init_fq()
281 WRITE_ONCE(cookie->fq_domain, domain); in iommu_dma_init_fq()
285 static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) in cookie_msi_granule() argument
287 if (cookie->type == IOMMU_DMA_IOVA_COOKIE) in cookie_msi_granule()
288 return cookie->iovad.granule; in cookie_msi_granule()
294 struct iommu_dma_cookie *cookie; in cookie_alloc() local
296 cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); in cookie_alloc()
297 if (cookie) { in cookie_alloc()
298 INIT_LIST_HEAD(&cookie->msi_page_list); in cookie_alloc()
299 cookie->type = type; in cookie_alloc()
301 return cookie; in cookie_alloc()
335 struct iommu_dma_cookie *cookie; in iommu_get_msi_cookie() local
343 cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE); in iommu_get_msi_cookie()
344 if (!cookie) in iommu_get_msi_cookie()
347 cookie->msi_iova = base; in iommu_get_msi_cookie()
348 domain->iova_cookie = cookie; in iommu_get_msi_cookie()
360 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_put_dma_cookie() local
363 if (!cookie) in iommu_put_dma_cookie()
366 if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { in iommu_put_dma_cookie()
367 iommu_dma_free_fq(cookie); in iommu_put_dma_cookie()
368 put_iova_domain(&cookie->iovad); in iommu_put_dma_cookie()
371 list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { in iommu_put_dma_cookie()
375 kfree(cookie); in iommu_put_dma_cookie()
400 static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie, in cookie_init_hw_msi_region() argument
403 struct iova_domain *iovad = &cookie->iovad; in cookie_init_hw_msi_region()
418 list_add(&msi_page->list, &cookie->msi_page_list); in cookie_init_hw_msi_region()
483 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iova_reserve_iommu_regions() local
484 struct iova_domain *iovad = &cookie->iovad; in iova_reserve_iommu_regions()
508 ret = cookie_init_hw_msi_region(cookie, region->start, in iova_reserve_iommu_regions()
543 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_dma_init_domain() local
548 if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) in iommu_dma_init_domain()
551 iovad = &cookie->iovad; in iommu_dma_init_domain()
570 mutex_lock(&cookie->mutex); in iommu_dma_init_domain()
595 mutex_unlock(&cookie->mutex); in iommu_dma_init_domain()
631 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_dma_alloc_iova() local
632 struct iova_domain *iovad = &cookie->iovad; in iommu_dma_alloc_iova()
635 if (cookie->type == IOMMU_DMA_MSI_COOKIE) { in iommu_dma_alloc_iova()
636 cookie->msi_iova += size; in iommu_dma_alloc_iova()
637 return cookie->msi_iova - size; in iommu_dma_alloc_iova()
660 static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, in iommu_dma_free_iova() argument
663 struct iova_domain *iovad = &cookie->iovad; in iommu_dma_free_iova()
666 if (cookie->type == IOMMU_DMA_MSI_COOKIE) in iommu_dma_free_iova()
667 cookie->msi_iova -= size; in iommu_dma_free_iova()
669 queue_iova(cookie, iova_pfn(iovad, iova), in iommu_dma_free_iova()
681 struct iommu_dma_cookie *cookie = domain->iova_cookie; in __iommu_dma_unmap() local
682 struct iova_domain *iovad = &cookie->iovad; in __iommu_dma_unmap()
690 iotlb_gather.queued = READ_ONCE(cookie->fq_domain); in __iommu_dma_unmap()
697 iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather); in __iommu_dma_unmap()
704 struct iommu_dma_cookie *cookie = domain->iova_cookie; in __iommu_dma_map() local
705 struct iova_domain *iovad = &cookie->iovad; in __iommu_dma_map()
720 iommu_dma_free_iova(cookie, iova, size, NULL); in __iommu_dma_map()
794 struct iommu_dma_cookie *cookie = domain->iova_cookie; in __iommu_dma_alloc_noncontiguous() local
795 struct iova_domain *iovad = &cookie->iovad; in __iommu_dma_alloc_noncontiguous()
858 iommu_dma_free_iova(cookie, iova, size, NULL); in __iommu_dma_alloc_noncontiguous()
993 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_dma_map_page() local
994 struct iova_domain *iovad = &cookie->iovad; in iommu_dma_map_page()
1195 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_dma_map_sg() local
1196 struct iova_domain *iovad = &cookie->iovad; in iommu_dma_map_sg()
1302 iommu_dma_free_iova(cookie, iova, iova_len, NULL); in iommu_dma_map_sg()
1610 struct iommu_dma_cookie *cookie = domain->iova_cookie; in iommu_dma_get_msi_page() local
1614 size_t size = cookie_msi_granule(cookie); in iommu_dma_get_msi_page()
1617 list_for_each_entry(msi_page, &cookie->msi_page_list, list) in iommu_dma_get_msi_page()
1635 list_add(&msi_page->list, &cookie->msi_page_list); in iommu_dma_get_msi_page()
1639 iommu_dma_free_iova(cookie, iova, size, NULL); in iommu_dma_get_msi_page()