Lines Matching refs:mm
47 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, in ptep_ipte_local() argument
54 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_local()
58 asce = asce ? : mm->context.asce; in ptep_ipte_local()
67 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, in ptep_ipte_global() argument
74 asce = READ_ONCE(mm->context.gmap_asce); in ptep_ipte_global()
78 asce = asce ? : mm->context.asce; in ptep_ipte_global()
87 static inline pte_t ptep_flush_direct(struct mm_struct *mm, in ptep_flush_direct() argument
96 atomic_inc(&mm->context.flush_count); in ptep_flush_direct()
98 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in ptep_flush_direct()
99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
102 atomic_dec(&mm->context.flush_count); in ptep_flush_direct()
106 static inline pte_t ptep_flush_lazy(struct mm_struct *mm, in ptep_flush_lazy() argument
115 atomic_inc(&mm->context.flush_count); in ptep_flush_lazy()
116 if (cpumask_equal(&mm->context.cpu_attach_mask, in ptep_flush_lazy()
119 mm->context.flush_mm = 1; in ptep_flush_lazy()
121 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_lazy()
122 atomic_dec(&mm->context.flush_count); in ptep_flush_lazy()
174 struct mm_struct *mm) in pgste_update_all() argument
179 if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID) in pgste_update_all()
195 struct mm_struct *mm) in pgste_set_key() argument
201 if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID) in pgste_set_key()
239 static inline pgste_t pgste_pte_notify(struct mm_struct *mm, in pgste_pte_notify() argument
249 ptep_notify(mm, addr, ptep, bits); in pgste_pte_notify()
255 static inline pgste_t ptep_xchg_start(struct mm_struct *mm, in ptep_xchg_start() argument
260 if (mm_has_pgste(mm)) { in ptep_xchg_start()
262 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_xchg_start()
267 static inline pte_t ptep_xchg_commit(struct mm_struct *mm, in ptep_xchg_commit() argument
271 if (mm_has_pgste(mm)) { in ptep_xchg_commit()
273 pgste_set_key(ptep, pgste, new, mm); in ptep_xchg_commit()
275 pgste = pgste_update_all(old, pgste, mm); in ptep_xchg_commit()
288 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, in ptep_xchg_direct() argument
296 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_direct()
298 old = ptep_flush_direct(mm, addr, ptep, nodat); in ptep_xchg_direct()
299 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_direct()
305 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, in ptep_xchg_lazy() argument
313 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_lazy()
315 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_xchg_lazy()
316 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_lazy()
328 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_start() local
331 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_modify_prot_start()
333 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_modify_prot_start()
334 if (mm_has_pgste(mm)) { in ptep_modify_prot_start()
335 pgste = pgste_update_all(old, pgste, mm); in ptep_modify_prot_start()
345 struct mm_struct *mm = vma->vm_mm; in ptep_modify_prot_commit() local
349 if (mm_has_pgste(mm)) { in ptep_modify_prot_commit()
351 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
360 static inline void pmdp_idte_local(struct mm_struct *mm, in pmdp_idte_local() argument
365 mm->context.asce, IDTE_LOCAL); in pmdp_idte_local()
368 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_local()
369 gmap_pmdp_idte_local(mm, addr); in pmdp_idte_local()
372 static inline void pmdp_idte_global(struct mm_struct *mm, in pmdp_idte_global() argument
377 mm->context.asce, IDTE_GLOBAL); in pmdp_idte_global()
378 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
379 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
382 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
383 gmap_pmdp_idte_global(mm, addr); in pmdp_idte_global()
386 if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) in pmdp_idte_global()
387 gmap_pmdp_csp(mm, addr); in pmdp_idte_global()
391 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, in pmdp_flush_direct() argument
399 atomic_inc(&mm->context.flush_count); in pmdp_flush_direct()
401 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pmdp_flush_direct()
402 pmdp_idte_local(mm, addr, pmdp); in pmdp_flush_direct()
404 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_direct()
405 atomic_dec(&mm->context.flush_count); in pmdp_flush_direct()
409 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, in pmdp_flush_lazy() argument
417 atomic_inc(&mm->context.flush_count); in pmdp_flush_lazy()
418 if (cpumask_equal(&mm->context.cpu_attach_mask, in pmdp_flush_lazy()
421 mm->context.flush_mm = 1; in pmdp_flush_lazy()
422 if (mm_has_pgste(mm)) in pmdp_flush_lazy()
423 gmap_pmdp_invalidate(mm, addr); in pmdp_flush_lazy()
425 pmdp_idte_global(mm, addr, pmdp); in pmdp_flush_lazy()
427 atomic_dec(&mm->context.flush_count); in pmdp_flush_lazy()
432 static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp) in pmd_lookup() argument
440 vma = vma_lookup(mm, addr); in pmd_lookup()
444 pgd = pgd_offset(mm, addr); in pmd_lookup()
465 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_direct() argument
471 old = pmdp_flush_direct(mm, addr, pmdp); in pmdp_xchg_direct()
478 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, in pmdp_xchg_lazy() argument
484 old = pmdp_flush_lazy(mm, addr, pmdp); in pmdp_xchg_lazy()
491 static inline void pudp_idte_local(struct mm_struct *mm, in pudp_idte_local() argument
496 mm->context.asce, IDTE_LOCAL); in pudp_idte_local()
501 static inline void pudp_idte_global(struct mm_struct *mm, in pudp_idte_global() argument
506 mm->context.asce, IDTE_GLOBAL); in pudp_idte_global()
517 static inline pud_t pudp_flush_direct(struct mm_struct *mm, in pudp_flush_direct() argument
525 atomic_inc(&mm->context.flush_count); in pudp_flush_direct()
527 cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) in pudp_flush_direct()
528 pudp_idte_local(mm, addr, pudp); in pudp_flush_direct()
530 pudp_idte_global(mm, addr, pudp); in pudp_flush_direct()
531 atomic_dec(&mm->context.flush_count); in pudp_flush_direct()
535 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, in pudp_xchg_direct() argument
541 old = pudp_flush_direct(mm, addr, pudp); in pudp_xchg_direct()
549 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
554 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_deposit()
557 if (!pmd_huge_pte(mm, pmdp)) in pgtable_trans_huge_deposit()
560 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); in pgtable_trans_huge_deposit()
561 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
564 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) in pgtable_trans_huge_withdraw() argument
570 assert_spin_locked(pmd_lockptr(mm, pmdp)); in pgtable_trans_huge_withdraw()
573 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
576 pmd_huge_pte(mm, pmdp) = NULL; in pgtable_trans_huge_withdraw()
578 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; in pgtable_trans_huge_withdraw()
590 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, in ptep_set_pte_at() argument
599 pgste_set_key(ptep, pgste, entry, mm); in ptep_set_pte_at()
605 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_set_notify() argument
627 int ptep_force_prot(struct mm_struct *mm, unsigned long addr, in ptep_force_prot() argument
647 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
648 pgste = pgste_update_all(entry, pgste, mm); in ptep_force_prot()
652 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
662 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, in ptep_shadow_pte() argument
689 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) in ptep_unshadow_pte() argument
697 ptep_flush_direct(mm, saddr, ptep, nodat); in ptep_unshadow_pte()
703 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) in ptep_zap_swap_entry() argument
706 dec_mm_counter(mm, MM_SWAPENTS); in ptep_zap_swap_entry()
710 dec_mm_counter(mm, mm_counter(page)); in ptep_zap_swap_entry()
715 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, in ptep_zap_unused() argument
730 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); in ptep_zap_unused()
731 pte_clear(mm, addr, ptep); in ptep_zap_unused()
739 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_zap_key() argument
759 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, in ptep_test_and_clear_uc() argument
772 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_uc()
774 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_test_and_clear_uc()
786 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in set_guest_storage_key() argument
799 switch (pmd_lookup(mm, addr, &pmdp)) { in set_guest_storage_key()
808 ptl = pmd_lock(mm, pmdp); in set_guest_storage_key()
827 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in set_guest_storage_key()
864 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, in cond_set_guest_storage_key() argument
873 rc = get_guest_storage_key(current->mm, addr, &tmp); in cond_set_guest_storage_key()
885 rc = set_guest_storage_key(current->mm, addr, key, nq); in cond_set_guest_storage_key()
895 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) in reset_guest_reference_bit() argument
908 switch (pmd_lookup(mm, addr, &pmdp)) { in reset_guest_reference_bit()
917 ptl = pmd_lock(mm, pmdp); in reset_guest_reference_bit()
932 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in reset_guest_reference_bit()
955 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, in get_guest_storage_key() argument
970 switch (pmd_lookup(mm, addr, &pmdp)) { in get_guest_storage_key()
979 ptl = pmd_lock(mm, pmdp); in get_guest_storage_key()
994 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in get_guest_storage_key()
1020 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, in pgste_perform_essa() argument
1034 vma = vma_lookup(mm, hva); in pgste_perform_essa()
1037 ptep = get_locked_pte(mm, hva, &ptl); in pgste_perform_essa()
1126 int set_pgste_bits(struct mm_struct *mm, unsigned long hva, in set_pgste_bits() argument
1134 vma = vma_lookup(mm, hva); in set_pgste_bits()
1137 ptep = get_locked_pte(mm, hva, &ptl); in set_pgste_bits()
1159 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) in get_pgste() argument
1165 vma = vma_lookup(mm, hva); in get_pgste()
1168 ptep = get_locked_pte(mm, hva, &ptl); in get_pgste()