Lines Matching refs:pgmap
48 static unsigned long pfn_first(struct dev_pagemap *pgmap, int range_id) in pfn_first() argument
50 struct range *range = &pgmap->ranges[range_id]; in pfn_first()
55 return pfn + vmem_altmap_offset(pgmap_altmap(pgmap)); in pfn_first()
58 bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn) in pgmap_pfn_valid() argument
62 for (i = 0; i < pgmap->nr_range; i++) { in pgmap_pfn_valid()
63 struct range *range = &pgmap->ranges[i]; in pgmap_pfn_valid()
67 return pfn >= pfn_first(pgmap, i); in pgmap_pfn_valid()
73 static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id) in pfn_end() argument
75 const struct range *range = &pgmap->ranges[range_id]; in pfn_end()
80 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id) in pfn_len() argument
82 return (pfn_end(pgmap, range_id) - in pfn_len()
83 pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift; in pfn_len()
86 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id) in pageunmap_range() argument
88 struct range *range = &pgmap->ranges[range_id]; in pageunmap_range()
92 first_page = pfn_to_page(pfn_first(pgmap, range_id)); in pageunmap_range()
98 if (pgmap->type == MEMORY_DEVICE_PRIVATE) { in pageunmap_range()
103 pgmap_altmap(pgmap)); in pageunmap_range()
112 void memunmap_pages(struct dev_pagemap *pgmap) in memunmap_pages() argument
116 percpu_ref_kill(&pgmap->ref); in memunmap_pages()
117 if (pgmap->type != MEMORY_DEVICE_PRIVATE && in memunmap_pages()
118 pgmap->type != MEMORY_DEVICE_COHERENT) in memunmap_pages()
119 for (i = 0; i < pgmap->nr_range; i++) in memunmap_pages()
120 percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i)); in memunmap_pages()
122 wait_for_completion(&pgmap->done); in memunmap_pages()
124 for (i = 0; i < pgmap->nr_range; i++) in memunmap_pages()
125 pageunmap_range(pgmap, i); in memunmap_pages()
126 percpu_ref_exit(&pgmap->ref); in memunmap_pages()
128 WARN_ONCE(pgmap->altmap.alloc, "failed to free all reserved pages\n"); in memunmap_pages()
139 struct dev_pagemap *pgmap = container_of(ref, struct dev_pagemap, ref); in dev_pagemap_percpu_release() local
141 complete(&pgmap->done); in dev_pagemap_percpu_release()
144 static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params, in pagemap_range() argument
147 const bool is_private = pgmap->type == MEMORY_DEVICE_PRIVATE; in pagemap_range()
148 struct range *range = &pgmap->ranges[range_id]; in pagemap_range()
152 if (WARN_ONCE(pgmap_altmap(pgmap) && range_id > 0, in pagemap_range()
181 PHYS_PFN(range->end), pgmap, GFP_KERNEL)); in pagemap_range()
244 PHYS_PFN(range_len(range)), pgmap); in pagemap_range()
245 if (pgmap->type != MEMORY_DEVICE_PRIVATE && in pagemap_range()
246 pgmap->type != MEMORY_DEVICE_COHERENT) in pagemap_range()
247 percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id)); in pagemap_range()
266 void *memremap_pages(struct dev_pagemap *pgmap, int nid) in memremap_pages() argument
269 .altmap = pgmap_altmap(pgmap), in memremap_pages()
270 .pgmap = pgmap, in memremap_pages()
273 const int nr_range = pgmap->nr_range; in memremap_pages()
279 switch (pgmap->type) { in memremap_pages()
285 if (!pgmap->ops || !pgmap->ops->migrate_to_ram) { in memremap_pages()
289 if (!pgmap->ops->page_free) { in memremap_pages()
293 if (!pgmap->owner) { in memremap_pages()
299 if (!pgmap->ops->page_free) { in memremap_pages()
303 if (!pgmap->owner) { in memremap_pages()
317 WARN(1, "Invalid pgmap type %d\n", pgmap->type); in memremap_pages()
321 init_completion(&pgmap->done); in memremap_pages()
322 error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0, in memremap_pages()
332 pgmap->nr_range = 0; in memremap_pages()
335 error = pagemap_range(pgmap, ¶ms, i, nid); in memremap_pages()
338 pgmap->nr_range++; in memremap_pages()
342 memunmap_pages(pgmap); in memremap_pages()
343 pgmap->nr_range = nr_range; in memremap_pages()
347 return __va(pgmap->ranges[0].start); in memremap_pages()
371 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap) in devm_memremap_pages() argument
376 ret = memremap_pages(pgmap, dev_to_node(dev)); in devm_memremap_pages()
381 pgmap); in devm_memremap_pages()
388 void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap) in devm_memunmap_pages() argument
390 devm_release_action(dev, devm_memremap_pages_release, pgmap); in devm_memunmap_pages()
403 struct dev_pagemap *pgmap) in get_dev_pagemap() argument
410 if (pgmap) { in get_dev_pagemap()
411 if (phys >= pgmap->range.start && phys <= pgmap->range.end) in get_dev_pagemap()
412 return pgmap; in get_dev_pagemap()
413 put_dev_pagemap(pgmap); in get_dev_pagemap()
418 pgmap = xa_load(&pgmap_array, PHYS_PFN(phys)); in get_dev_pagemap()
419 if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref)) in get_dev_pagemap()
420 pgmap = NULL; in get_dev_pagemap()
423 return pgmap; in get_dev_pagemap()
429 struct dev_pagemap *pgmap = folio->pgmap; in free_zone_device_folio() local
431 if (WARN_ON_ONCE(!pgmap)) in free_zone_device_folio()
461 if (pgmap->type != MEMORY_DEVICE_FS_DAX && in free_zone_device_folio()
462 pgmap->type != MEMORY_DEVICE_GENERIC) in free_zone_device_folio()
465 switch (pgmap->type) { in free_zone_device_folio()
468 if (WARN_ON_ONCE(!pgmap->ops || !pgmap->ops->page_free)) in free_zone_device_folio()
470 pgmap->ops->page_free(folio_page(folio, 0)); in free_zone_device_folio()
471 put_dev_pagemap(pgmap); in free_zone_device_folio()
487 if (WARN_ON_ONCE(!pgmap->ops || !pgmap->ops->page_free)) in free_zone_device_folio()
489 pgmap->ops->page_free(folio_page(folio, 0)); in free_zone_device_folio()