1  /* SPDX-License-Identifier: GPL-2.0 */
2  #ifndef _ASM_POWERPC_MMU_H_
3  #define _ASM_POWERPC_MMU_H_
4  #ifdef __KERNEL__
5  
6  #include <linux/types.h>
7  
8  #include <asm/asm-const.h>
9  
10  /*
11   * MMU features bit definitions
12   */
13  
14  /*
15   * MMU families
16   */
17  #define MMU_FTR_HPTE_TABLE		ASM_CONST(0x00000001)
18  #define MMU_FTR_TYPE_8xx		ASM_CONST(0x00000002)
19  #define MMU_FTR_TYPE_40x		ASM_CONST(0x00000004)
20  #define MMU_FTR_TYPE_44x		ASM_CONST(0x00000008)
21  #define MMU_FTR_TYPE_FSL_E		ASM_CONST(0x00000010)
22  #define MMU_FTR_TYPE_47x		ASM_CONST(0x00000020)
23  
24  /* Radix page table supported and enabled */
25  #define MMU_FTR_TYPE_RADIX		ASM_CONST(0x00000040)
26  
27  /*
28   * Individual features below.
29   */
30  
31  /*
32   * Supports KUAP feature
33   * key 0 controlling userspace addresses on radix
34   * Key 3 on hash
35   */
36  #define MMU_FTR_BOOK3S_KUAP		ASM_CONST(0x00000200)
37  
38  /*
39   * Supports KUEP feature
40   * key 0 controlling userspace addresses on radix
41   * Key 3 on hash
42   */
43  #define MMU_FTR_BOOK3S_KUEP		ASM_CONST(0x00000400)
44  
45  /*
46   * Support for memory protection keys.
47   */
48  #define MMU_FTR_PKEY			ASM_CONST(0x00000800)
49  
50  /* Guest Translation Shootdown Enable */
51  #define MMU_FTR_GTSE			ASM_CONST(0x00001000)
52  
53  /*
54   * Support for 68 bit VA space. We added that from ISA 2.05
55   */
56  #define MMU_FTR_68_BIT_VA		ASM_CONST(0x00002000)
57  /*
58   * Kernel read only support.
59   * We added the ppp value 0b110 in ISA 2.04.
60   */
61  #define MMU_FTR_KERNEL_RO		ASM_CONST(0x00004000)
62  
63  /*
64   * We need to clear top 16bits of va (from the remaining 64 bits )in
65   * tlbie* instructions
66   */
67  #define MMU_FTR_TLBIE_CROP_VA		ASM_CONST(0x00008000)
68  
69  /* Enable use of high BAT registers */
70  #define MMU_FTR_USE_HIGH_BATS		ASM_CONST(0x00010000)
71  
72  /* Enable >32-bit physical addresses on 32-bit processor, only used
73   * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1
74   */
75  #define MMU_FTR_BIG_PHYS		ASM_CONST(0x00020000)
76  
77  /* Enable use of broadcast TLB invalidations. We don't always set it
78   * on processors that support it due to other constraints with the
79   * use of such invalidations
80   */
81  #define MMU_FTR_USE_TLBIVAX_BCAST	ASM_CONST(0x00040000)
82  
83  /* Enable use of tlbilx invalidate instructions.
84   */
85  #define MMU_FTR_USE_TLBILX		ASM_CONST(0x00080000)
86  
87  /* This indicates that the processor cannot handle multiple outstanding
88   * broadcast tlbivax or tlbsync. This makes the code use a spinlock
89   * around such invalidate forms.
90   */
91  #define MMU_FTR_LOCK_BCAST_INVAL	ASM_CONST(0x00100000)
92  
93  /* This indicates that the processor doesn't handle way selection
94   * properly and needs SW to track and update the LRU state.  This
95   * is specific to an errata on e300c2/c3/c4 class parts
96   */
97  #define MMU_FTR_NEED_DTLB_SW_LRU	ASM_CONST(0x00200000)
98  
99  /* Doesn't support the B bit (1T segment) in SLBIE
100   */
101  #define MMU_FTR_NO_SLBIE_B		ASM_CONST(0x02000000)
102  
103  /* Support 16M large pages
104   */
105  #define MMU_FTR_16M_PAGE		ASM_CONST(0x04000000)
106  
107  /* Supports TLBIEL variant
108   */
109  #define MMU_FTR_TLBIEL			ASM_CONST(0x08000000)
110  
111  /* Supports tlbies w/o locking
112   */
113  #define MMU_FTR_LOCKLESS_TLBIE		ASM_CONST(0x10000000)
114  
115  /* Large pages can be marked CI
116   */
117  #define MMU_FTR_CI_LARGE_PAGE		ASM_CONST(0x20000000)
118  
119  /* 1T segments available
120   */
121  #define MMU_FTR_1T_SEGMENT		ASM_CONST(0x40000000)
122  
123  // NX paste RMA reject in DSI
124  #define MMU_FTR_NX_DSI			ASM_CONST(0x80000000)
125  
126  /* MMU feature bit sets for various CPUs */
127  #define MMU_FTRS_DEFAULT_HPTE_ARCH_V2	(MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE)
128  #define MMU_FTRS_POWER		MMU_FTRS_DEFAULT_HPTE_ARCH_V2
129  #define MMU_FTRS_PPC970		MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
130  #define MMU_FTRS_POWER5		MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
131  #define MMU_FTRS_POWER6		MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
132  #define MMU_FTRS_POWER7		MMU_FTRS_POWER6
133  #define MMU_FTRS_POWER8		MMU_FTRS_POWER6
134  #define MMU_FTRS_POWER9		MMU_FTRS_POWER6
135  #define MMU_FTRS_POWER10	MMU_FTRS_POWER6
136  #define MMU_FTRS_CELL		MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
137  				MMU_FTR_CI_LARGE_PAGE
138  #define MMU_FTRS_PA6T		MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
139  				MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
140  #ifndef __ASSEMBLY__
141  #include <linux/bug.h>
142  #include <asm/cputable.h>
143  #include <asm/page.h>
144  
145  typedef pte_t *pgtable_t;
146  
147  #ifdef CONFIG_PPC_E500
148  #include <asm/percpu.h>
149  DECLARE_PER_CPU(int, next_tlbcam_idx);
150  #endif
151  
152  enum {
153  	MMU_FTRS_POSSIBLE =
154  #if defined(CONFIG_PPC_BOOK3S_604)
155  		MMU_FTR_HPTE_TABLE |
156  #endif
157  #ifdef CONFIG_PPC_8xx
158  		MMU_FTR_TYPE_8xx |
159  #endif
160  #ifdef CONFIG_40x
161  		MMU_FTR_TYPE_40x |
162  #endif
163  #ifdef CONFIG_PPC_47x
164  		MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL |
165  #elif defined(CONFIG_44x)
166  		MMU_FTR_TYPE_44x |
167  #endif
168  #ifdef CONFIG_PPC_E500
169  		MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX |
170  #endif
171  #ifdef CONFIG_PPC_BOOK3S_32
172  		MMU_FTR_USE_HIGH_BATS |
173  #endif
174  #ifdef CONFIG_PPC_83xx
175  		MMU_FTR_NEED_DTLB_SW_LRU |
176  #endif
177  #ifdef CONFIG_PPC_BOOK3S_64
178  		MMU_FTR_KERNEL_RO |
179  #ifdef CONFIG_PPC_64S_HASH_MMU
180  		MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
181  		MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
182  		MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
183  		MMU_FTR_68_BIT_VA | MMU_FTR_HPTE_TABLE |
184  #endif
185  #ifdef CONFIG_PPC_RADIX_MMU
186  		MMU_FTR_TYPE_RADIX |
187  		MMU_FTR_GTSE | MMU_FTR_NX_DSI |
188  #endif /* CONFIG_PPC_RADIX_MMU */
189  #endif
190  #ifdef CONFIG_PPC_KUAP
191  	MMU_FTR_BOOK3S_KUAP |
192  #endif /* CONFIG_PPC_KUAP */
193  #ifdef CONFIG_PPC_MEM_KEYS
194  	MMU_FTR_PKEY |
195  #endif
196  #ifdef CONFIG_PPC_KUEP
197  	MMU_FTR_BOOK3S_KUEP |
198  #endif /* CONFIG_PPC_KUAP */
199  
200  		0,
201  };
202  
203  #if defined(CONFIG_PPC_BOOK3S_604) && !defined(CONFIG_PPC_BOOK3S_603)
204  #define MMU_FTRS_ALWAYS		MMU_FTR_HPTE_TABLE
205  #endif
206  #ifdef CONFIG_PPC_8xx
207  #define MMU_FTRS_ALWAYS		MMU_FTR_TYPE_8xx
208  #endif
209  #ifdef CONFIG_40x
210  #define MMU_FTRS_ALWAYS		MMU_FTR_TYPE_40x
211  #endif
212  #ifdef CONFIG_PPC_47x
213  #define MMU_FTRS_ALWAYS		MMU_FTR_TYPE_47x
214  #elif defined(CONFIG_44x)
215  #define MMU_FTRS_ALWAYS		MMU_FTR_TYPE_44x
216  #endif
217  #ifdef CONFIG_PPC_E500
218  #define MMU_FTRS_ALWAYS		MMU_FTR_TYPE_FSL_E
219  #endif
220  
221  /* BOOK3S_64 options */
222  #if defined(CONFIG_PPC_RADIX_MMU) && !defined(CONFIG_PPC_64S_HASH_MMU)
223  #define MMU_FTRS_ALWAYS		MMU_FTR_TYPE_RADIX
224  #elif !defined(CONFIG_PPC_RADIX_MMU) && defined(CONFIG_PPC_64S_HASH_MMU)
225  #define MMU_FTRS_ALWAYS		MMU_FTR_HPTE_TABLE
226  #endif
227  
228  #ifndef MMU_FTRS_ALWAYS
229  #define MMU_FTRS_ALWAYS		0
230  #endif
231  
early_mmu_has_feature(unsigned long feature)232  static __always_inline bool early_mmu_has_feature(unsigned long feature)
233  {
234  	if (MMU_FTRS_ALWAYS & feature)
235  		return true;
236  
237  	return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
238  }
239  
240  #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
241  #include <linux/jump_label.h>
242  
243  #define NUM_MMU_FTR_KEYS	32
244  
245  extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
246  
247  extern void mmu_feature_keys_init(void);
248  
mmu_has_feature(unsigned long feature)249  static __always_inline bool mmu_has_feature(unsigned long feature)
250  {
251  	int i;
252  
253  #ifndef __clang__ /* clang can't cope with this */
254  	BUILD_BUG_ON(!__builtin_constant_p(feature));
255  #endif
256  
257  #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
258  	if (!static_key_initialized) {
259  		printk("Warning! mmu_has_feature() used prior to jump label init!\n");
260  		dump_stack();
261  		return early_mmu_has_feature(feature);
262  	}
263  #endif
264  
265  	if (MMU_FTRS_ALWAYS & feature)
266  		return true;
267  
268  	if (!(MMU_FTRS_POSSIBLE & feature))
269  		return false;
270  
271  	i = __builtin_ctzl(feature);
272  	return static_branch_likely(&mmu_feature_keys[i]);
273  }
274  
mmu_clear_feature(unsigned long feature)275  static inline void mmu_clear_feature(unsigned long feature)
276  {
277  	int i;
278  
279  	i = __builtin_ctzl(feature);
280  	cur_cpu_spec->mmu_features &= ~feature;
281  	static_branch_disable(&mmu_feature_keys[i]);
282  }
283  #else
284  
mmu_feature_keys_init(void)285  static inline void mmu_feature_keys_init(void)
286  {
287  
288  }
289  
mmu_has_feature(unsigned long feature)290  static __always_inline bool mmu_has_feature(unsigned long feature)
291  {
292  	return early_mmu_has_feature(feature);
293  }
294  
mmu_clear_feature(unsigned long feature)295  static inline void mmu_clear_feature(unsigned long feature)
296  {
297  	cur_cpu_spec->mmu_features &= ~feature;
298  }
299  #endif /* CONFIG_JUMP_LABEL */
300  
301  extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
302  
303  #ifdef CONFIG_PPC64
304  /* This is our real memory area size on ppc64 server, on embedded, we
305   * make it match the size our of bolted TLB area
306   */
307  extern u64 ppc64_rma_size;
308  
309  /* Cleanup function used by kexec */
310  extern void mmu_cleanup_all(void);
311  extern void radix__mmu_cleanup_all(void);
312  
313  /* Functions for creating and updating partition table on POWER9 */
314  extern void mmu_partition_table_init(void);
315  extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
316  					  unsigned long dw1, bool flush);
317  #endif /* CONFIG_PPC64 */
318  
319  struct mm_struct;
320  #ifdef CONFIG_DEBUG_VM
321  extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
322  #else /* CONFIG_DEBUG_VM */
assert_pte_locked(struct mm_struct * mm,unsigned long addr)323  static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
324  {
325  }
326  #endif /* !CONFIG_DEBUG_VM */
327  
radix_enabled(void)328  static __always_inline bool radix_enabled(void)
329  {
330  	return mmu_has_feature(MMU_FTR_TYPE_RADIX);
331  }
332  
early_radix_enabled(void)333  static __always_inline bool early_radix_enabled(void)
334  {
335  	return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
336  }
337  
338  #ifdef CONFIG_STRICT_KERNEL_RWX
strict_kernel_rwx_enabled(void)339  static inline bool strict_kernel_rwx_enabled(void)
340  {
341  	return rodata_enabled;
342  }
343  #else
strict_kernel_rwx_enabled(void)344  static inline bool strict_kernel_rwx_enabled(void)
345  {
346  	return false;
347  }
348  #endif
349  
strict_module_rwx_enabled(void)350  static inline bool strict_module_rwx_enabled(void)
351  {
352  	return IS_ENABLED(CONFIG_STRICT_MODULE_RWX) && strict_kernel_rwx_enabled();
353  }
354  #endif /* !__ASSEMBLY__ */
355  
356  /* The kernel use the constants below to index in the page sizes array.
357   * The use of fixed constants for this purpose is better for performances
358   * of the low level hash refill handlers.
359   *
360   * A non supported page size has a "shift" field set to 0
361   *
362   * Any new page size being implemented can get a new entry in here. Whether
363   * the kernel will use it or not is a different matter though. The actual page
364   * size used by hugetlbfs is not defined here and may be made variable
365   *
366   * Note: This array ended up being a false good idea as it's growing to the
367   * point where I wonder if we should replace it with something different,
368   * to think about, feedback welcome. --BenH.
369   */
370  
371  /* These are #defines as they have to be used in assembly */
372  #define MMU_PAGE_4K	0
373  #define MMU_PAGE_16K	1
374  #define MMU_PAGE_64K	2
375  #define MMU_PAGE_64K_AP	3	/* "Admixed pages" (hash64 only) */
376  #define MMU_PAGE_256K	4
377  #define MMU_PAGE_512K	5
378  #define MMU_PAGE_1M	6
379  #define MMU_PAGE_2M	7
380  #define MMU_PAGE_4M	8
381  #define MMU_PAGE_8M	9
382  #define MMU_PAGE_16M	10
383  #define MMU_PAGE_64M	11
384  #define MMU_PAGE_256M	12
385  #define MMU_PAGE_1G	13
386  #define MMU_PAGE_16G	14
387  #define MMU_PAGE_64G	15
388  
389  /*
390   * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16
391   * Also we need to change he type of mm_context.low/high_slices_psize.
392   */
393  #define MMU_PAGE_COUNT	16
394  
395  #ifdef CONFIG_PPC_BOOK3S_64
396  #include <asm/book3s/64/mmu.h>
397  #else /* CONFIG_PPC_BOOK3S_64 */
398  
399  #ifndef __ASSEMBLY__
400  /* MMU initialization */
401  extern void early_init_mmu(void);
402  extern void early_init_mmu_secondary(void);
403  extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
404  				       phys_addr_t first_memblock_size);
mmu_early_init_devtree(void)405  static inline void mmu_early_init_devtree(void) { }
406  
pkey_early_init_devtree(void)407  static inline void pkey_early_init_devtree(void) {}
408  
409  extern void *abatron_pteptrs[2];
410  #endif /* __ASSEMBLY__ */
411  #endif
412  
413  #if defined(CONFIG_PPC_BOOK3S_32)
414  /* 32-bit classic hash table MMU */
415  #include <asm/book3s/32/mmu-hash.h>
416  #elif defined(CONFIG_PPC_MMU_NOHASH)
417  #include <asm/nohash/mmu.h>
418  #endif
419  
420  #endif /* __KERNEL__ */
421  #endif /* _ASM_POWERPC_MMU_H_ */
422