Lines Matching refs:ptep
48 pte_t *ptep, int nodat) in ptep_ipte_local() argument
61 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); in ptep_ipte_local()
63 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local()
68 pte_t *ptep, int nodat) in ptep_ipte_global() argument
81 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); in ptep_ipte_global()
83 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global()
88 unsigned long addr, pte_t *ptep, in ptep_flush_direct() argument
93 old = *ptep; in ptep_flush_direct()
99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
107 unsigned long addr, pte_t *ptep, in ptep_flush_lazy() argument
112 old = *ptep; in ptep_flush_lazy()
118 pte_val(*ptep) |= _PAGE_INVALID; in ptep_flush_lazy()
121 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_lazy()
126 static inline pgste_t pgste_get_lock(pte_t *ptep) in pgste_get_lock() argument
139 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) in pgste_get_lock()
140 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); in pgste_get_lock()
145 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) in pgste_set_unlock() argument
151 : "=Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
152 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) in pgste_set_unlock()
157 static inline pgste_t pgste_get(pte_t *ptep) in pgste_get() argument
161 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); in pgste_get()
166 static inline void pgste_set(pte_t *ptep, pgste_t pgste) in pgste_set() argument
169 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; in pgste_set()
194 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, in pgste_set_key() argument
203 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); in pgste_set_key()
216 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) in pgste_set_pte() argument
235 *ptep = entry; in pgste_set_pte()
241 pte_t *ptep, pgste_t pgste) in pgste_pte_notify() argument
249 ptep_notify(mm, addr, ptep, bits); in pgste_pte_notify()
256 unsigned long addr, pte_t *ptep) in ptep_xchg_start() argument
261 pgste = pgste_get_lock(ptep); in ptep_xchg_start()
262 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_xchg_start()
268 unsigned long addr, pte_t *ptep, in ptep_xchg_commit() argument
273 pgste_set_key(ptep, pgste, new, mm); in ptep_xchg_commit()
280 pgste = pgste_set_pte(ptep, pgste, new); in ptep_xchg_commit()
281 pgste_set_unlock(ptep, pgste); in ptep_xchg_commit()
283 *ptep = new; in ptep_xchg_commit()
289 pte_t *ptep, pte_t new) in ptep_xchg_direct() argument
296 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_direct()
298 old = ptep_flush_direct(mm, addr, ptep, nodat); in ptep_xchg_direct()
299 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_direct()
306 pte_t *ptep, pte_t new) in ptep_xchg_lazy() argument
313 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_xchg_lazy()
315 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_xchg_lazy()
316 old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); in ptep_xchg_lazy()
323 pte_t *ptep) in ptep_modify_prot_start() argument
331 pgste = ptep_xchg_start(mm, addr, ptep); in ptep_modify_prot_start()
333 old = ptep_flush_lazy(mm, addr, ptep, nodat); in ptep_modify_prot_start()
336 pgste_set(ptep, pgste); in ptep_modify_prot_start()
342 pte_t *ptep, pte_t old_pte, pte_t pte) in ptep_modify_prot_commit() argument
350 pgste = pgste_get(ptep); in ptep_modify_prot_commit()
351 pgste_set_key(ptep, pgste, pte, mm); in ptep_modify_prot_commit()
352 pgste = pgste_set_pte(ptep, pgste, pte); in ptep_modify_prot_commit()
353 pgste_set_unlock(ptep, pgste); in ptep_modify_prot_commit()
355 *ptep = pte; in ptep_modify_prot_commit()
568 pte_t *ptep; in pgtable_trans_huge_withdraw() local
581 ptep = (pte_t *) pgtable; in pgtable_trans_huge_withdraw()
582 pte_val(*ptep) = _PAGE_INVALID; in pgtable_trans_huge_withdraw()
583 ptep++; in pgtable_trans_huge_withdraw()
584 pte_val(*ptep) = _PAGE_INVALID; in pgtable_trans_huge_withdraw()
591 pte_t *ptep, pte_t entry) in ptep_set_pte_at() argument
597 pgste = pgste_get_lock(ptep); in ptep_set_pte_at()
599 pgste_set_key(ptep, pgste, entry, mm); in ptep_set_pte_at()
600 pgste = pgste_set_pte(ptep, pgste, entry); in ptep_set_pte_at()
601 pgste_set_unlock(ptep, pgste); in ptep_set_pte_at()
605 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_set_notify() argument
610 pgste = pgste_get_lock(ptep); in ptep_set_notify()
612 pgste_set_unlock(ptep, pgste); in ptep_set_notify()
628 pte_t *ptep, int prot, unsigned long bit) in ptep_force_prot() argument
634 pgste = pgste_get_lock(ptep); in ptep_force_prot()
635 entry = *ptep; in ptep_force_prot()
641 pgste_set_unlock(ptep, pgste); in ptep_force_prot()
647 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
652 ptep_flush_direct(mm, addr, ptep, nodat); in ptep_force_prot()
657 pgste = pgste_set_pte(ptep, pgste, entry); in ptep_force_prot()
658 pgste_set_unlock(ptep, pgste); in ptep_force_prot()
689 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) in ptep_unshadow_pte() argument
694 pgste = pgste_get_lock(ptep); in ptep_unshadow_pte()
697 ptep_flush_direct(mm, saddr, ptep, nodat); in ptep_unshadow_pte()
699 pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); in ptep_unshadow_pte()
700 pgste_set_unlock(ptep, pgste); in ptep_unshadow_pte()
716 pte_t *ptep, int reset) in ptep_zap_unused() argument
724 pgste = pgste_get_lock(ptep); in ptep_zap_unused()
726 pte = *ptep; in ptep_zap_unused()
731 pte_clear(mm, addr, ptep); in ptep_zap_unused()
735 pgste_set_unlock(ptep, pgste); in ptep_zap_unused()
739 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) in ptep_zap_key() argument
746 pgste = pgste_get_lock(ptep); in ptep_zap_key()
749 ptev = pte_val(*ptep); in ptep_zap_key()
752 pgste_set_unlock(ptep, pgste); in ptep_zap_key()
760 pte_t *ptep) in ptep_test_and_clear_uc() argument
767 pgste = pgste_get_lock(ptep); in ptep_test_and_clear_uc()
770 pte = *ptep; in ptep_test_and_clear_uc()
772 pgste = pgste_pte_notify(mm, addr, ptep, pgste); in ptep_test_and_clear_uc()
774 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_test_and_clear_uc()
779 *ptep = pte; in ptep_test_and_clear_uc()
781 pgste_set_unlock(ptep, pgste); in ptep_test_and_clear_uc()
793 pte_t *ptep; in set_guest_storage_key() local
827 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in set_guest_storage_key()
828 new = old = pgste_get_lock(ptep); in set_guest_storage_key()
834 if (!(pte_val(*ptep) & _PAGE_INVALID)) { in set_guest_storage_key()
837 paddr = pte_val(*ptep) & PAGE_MASK; in set_guest_storage_key()
851 pgste_set_unlock(ptep, new); in set_guest_storage_key()
852 pte_unmap_unlock(ptep, ptl); in set_guest_storage_key()
901 pte_t *ptep; in reset_guest_reference_bit() local
932 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in reset_guest_reference_bit()
933 new = old = pgste_get_lock(ptep); in reset_guest_reference_bit()
937 if (!(pte_val(*ptep) & _PAGE_INVALID)) { in reset_guest_reference_bit()
938 paddr = pte_val(*ptep) & PAGE_MASK; in reset_guest_reference_bit()
949 pgste_set_unlock(ptep, new); in reset_guest_reference_bit()
950 pte_unmap_unlock(ptep, ptl); in reset_guest_reference_bit()
962 pte_t *ptep; in get_guest_storage_key() local
994 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl); in get_guest_storage_key()
995 pgste = pgste_get_lock(ptep); in get_guest_storage_key()
997 paddr = pte_val(*ptep) & PAGE_MASK; in get_guest_storage_key()
998 if (!(pte_val(*ptep) & _PAGE_INVALID)) in get_guest_storage_key()
1002 pgste_set_unlock(ptep, pgste); in get_guest_storage_key()
1003 pte_unmap_unlock(ptep, ptl); in get_guest_storage_key()
1027 pte_t *ptep; in pgste_perform_essa() local
1037 ptep = get_locked_pte(mm, hva, &ptl); in pgste_perform_essa()
1038 if (unlikely(!ptep)) in pgste_perform_essa()
1040 pgste = pgste_get_lock(ptep); in pgste_perform_essa()
1043 *oldpte = pte_val(*ptep); in pgste_perform_essa()
1057 if (pte_val(*ptep) & _PAGE_INVALID) in pgste_perform_essa()
1063 if (pte_val(*ptep) & _PAGE_INVALID) in pgste_perform_essa()
1068 if (!(pte_val(*ptep) & _PAGE_INVALID)) { in pgste_perform_essa()
1092 if (!(pte_val(*ptep) & _PAGE_INVALID)) { in pgste_perform_essa()
1110 pgste_set_unlock(ptep, pgste); in pgste_perform_essa()
1111 pte_unmap_unlock(ptep, ptl); in pgste_perform_essa()
1132 pte_t *ptep; in set_pgste_bits() local
1137 ptep = get_locked_pte(mm, hva, &ptl); in set_pgste_bits()
1138 if (unlikely(!ptep)) in set_pgste_bits()
1140 new = pgste_get_lock(ptep); in set_pgste_bits()
1145 pgste_set_unlock(ptep, new); in set_pgste_bits()
1146 pte_unmap_unlock(ptep, ptl); in set_pgste_bits()
1163 pte_t *ptep; in get_pgste() local
1168 ptep = get_locked_pte(mm, hva, &ptl); in get_pgste()
1169 if (unlikely(!ptep)) in get_pgste()
1171 *pgstep = pgste_val(pgste_get(ptep)); in get_pgste()
1172 pte_unmap_unlock(ptep, ptl); in get_pgste()