1 /******************************************************************************
2 * Original code extracted from arch/x86/x86_64/mm.c
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #include <xen/init.h>
19 #include <xen/mm.h>
20 #include <xen/bitops.h>
21 #include <xen/nospec.h>
22
23 /**
24 * Maximum (non-inclusive) usable pdx. Must be
25 * modifiable after init due to memory hotplug
26 */
27 unsigned long __read_mostly max_pdx;
28
29 unsigned long __read_mostly pdx_group_valid[BITS_TO_LONGS(
30 (FRAMETABLE_NR + PDX_GROUP_COUNT - 1) / PDX_GROUP_COUNT)] = { [0] = 1 };
31
__mfn_valid(unsigned long mfn)32 bool __mfn_valid(unsigned long mfn)
33 {
34 bool invalid = mfn >= max_page;
35
36 #ifdef CONFIG_PDX_COMPRESSION
37 invalid |= mfn & pfn_hole_mask;
38 #endif
39
40 if ( unlikely(evaluate_nospec(invalid)) )
41 return false;
42
43 return test_bit(pfn_to_pdx(mfn) / PDX_GROUP_COUNT, pdx_group_valid);
44 }
45
set_pdx_range(unsigned long smfn,unsigned long emfn)46 void set_pdx_range(unsigned long smfn, unsigned long emfn)
47 {
48 unsigned long idx, eidx;
49
50 idx = pfn_to_pdx(smfn) / PDX_GROUP_COUNT;
51 eidx = (pfn_to_pdx(emfn - 1) + PDX_GROUP_COUNT) / PDX_GROUP_COUNT;
52
53 for ( ; idx < eidx; ++idx )
54 __set_bit(idx, pdx_group_valid);
55 }
56
57 #ifdef CONFIG_PDX_COMPRESSION
58
59 /*
60 * Diagram to make sense of the following variables. The masks and shifts
61 * are done on mfn values in order to convert to/from pdx:
62 *
63 * pfn_hole_mask
64 * pfn_pdx_hole_shift (mask bitsize)
65 * |
66 * |---------|
67 * | |
68 * V V
69 * --------------------------
70 * |HHHHHHH|000000000|LLLLLL| <--- mfn
71 * --------------------------
72 * ^ ^ ^ ^
73 * | | |------|
74 * | | |
75 * | | pfn_pdx_bottom_mask
76 * | |
77 * |-------|
78 * |
79 * pfn_top_mask
80 *
81 * ma_{top,va_bottom}_mask is simply a shifted pfn_{top,pdx_bottom}_mask,
82 * where ma_top_mask has zeroes shifted in while ma_va_bottom_mask has
83 * ones.
84 */
85
86 /** Mask for the lower non-compressible bits of an mfn */
87 unsigned long __ro_after_init pfn_pdx_bottom_mask = ~0UL;
88
89 /** Mask for the lower non-compressible bits of an maddr or vaddr */
90 unsigned long __ro_after_init ma_va_bottom_mask = ~0UL;
91
92 /** Mask for the higher non-compressible bits of an mfn */
93 unsigned long __ro_after_init pfn_top_mask = 0;
94
95 /** Mask for the higher non-compressible bits of an maddr or vaddr */
96 unsigned long __ro_after_init ma_top_mask = 0;
97
98 /**
99 * Mask for a pdx compression bit slice.
100 *
101 * Invariant: valid(mfn) implies (mfn & pfn_hole_mask) == 0
102 */
103 unsigned long __ro_after_init pfn_hole_mask = 0;
104
105 /** Number of bits of the "compressible" bit slice of an mfn */
106 unsigned int __ro_after_init pfn_pdx_hole_shift = 0;
107
108 /* Sets all bits from the most-significant 1-bit down to the LSB */
fill_mask(uint64_t mask)109 static uint64_t fill_mask(uint64_t mask)
110 {
111 while (mask & (mask + 1))
112 mask |= mask + 1;
113
114 return mask;
115 }
116
pdx_is_region_compressible(paddr_t base,unsigned long npages)117 bool pdx_is_region_compressible(paddr_t base, unsigned long npages)
118 {
119 return !(paddr_to_pfn(base) & pfn_hole_mask) &&
120 !(pdx_region_mask(base, npages * PAGE_SIZE) & ~ma_va_bottom_mask);
121 }
122
123 /* We don't want to compress the low MAX_ORDER bits of the addresses. */
pdx_init_mask(uint64_t base_addr)124 uint64_t __init pdx_init_mask(uint64_t base_addr)
125 {
126 return fill_mask(max(base_addr,
127 (uint64_t)1 << (MAX_ORDER + PAGE_SHIFT)) - 1);
128 }
129
pdx_region_mask(uint64_t base,uint64_t len)130 uint64_t pdx_region_mask(uint64_t base, uint64_t len)
131 {
132 /*
133 * We say a bit "moves" in a range if there exist 2 addresses in that
134 * range that have that bit both set and cleared respectively. We want
135 * to create a mask of _all_ moving bits in this range. We do this by
136 * comparing the first and last addresses in the range, discarding the
137 * bits that remain the same (this is logically an XOR operation). The
138 * MSB of the resulting expression is the most significant moving bit
139 * in the range. Then it's a matter of setting every bit in lower
140 * positions in order to get the mask of moving bits.
141 */
142 return fill_mask(base ^ (base + len - 1));
143 }
144
pfn_pdx_hole_setup(unsigned long mask)145 void __init pfn_pdx_hole_setup(unsigned long mask)
146 {
147 unsigned int i, j, bottom_shift = 0, hole_shift = 0;
148
149 /*
150 * We skip the first MAX_ORDER bits, as we never want to compress them.
151 * This guarantees that page-pointer arithmetic remains valid within
152 * contiguous aligned ranges of 2^MAX_ORDER pages. Among others, our
153 * buddy allocator relies on this assumption.
154 *
155 * If the logic changes here, we might have to update the ARM specific
156 * init_pdx too.
157 */
158 for ( j = MAX_ORDER-1; ; )
159 {
160 i = find_next_zero_bit(&mask, BITS_PER_LONG, j + 1);
161 if ( i >= BITS_PER_LONG )
162 break;
163 j = find_next_bit(&mask, BITS_PER_LONG, i + 1);
164 if ( j >= BITS_PER_LONG )
165 break;
166 if ( j - i > hole_shift )
167 {
168 hole_shift = j - i;
169 bottom_shift = i;
170 }
171 }
172 if ( !hole_shift )
173 return;
174
175 printk(KERN_INFO "PFN compression on bits %u...%u\n",
176 bottom_shift, bottom_shift + hole_shift - 1);
177
178 pfn_pdx_hole_shift = hole_shift;
179 pfn_pdx_bottom_mask = (1UL << bottom_shift) - 1;
180 ma_va_bottom_mask = (PAGE_SIZE << bottom_shift) - 1;
181 pfn_hole_mask = ((1UL << hole_shift) - 1) << bottom_shift;
182 pfn_top_mask = ~(pfn_pdx_bottom_mask | pfn_hole_mask);
183 ma_top_mask = pfn_top_mask << PAGE_SHIFT;
184 }
185
186 #endif /* CONFIG_PDX_COMPRESSION */
187
188 /*
189 * Local variables:
190 * mode: C
191 * c-file-style: "BSD"
192 * c-basic-offset: 4
193 * indent-tabs-mode: nil
194 * End:
195 */
196