1 /*
2 * powernow - AMD Architectural P-state Driver ($Revision: 1.4 $)
3 *
4 * Copyright (C) 2008 Mark Langsdorf <mark.langsdorf@amd.com>
5 *
6 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or (at
11 * your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; If not, see <http://www.gnu.org/licenses/>.
20 *
21 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
22 */
23
24 #include <xen/types.h>
25 #include <xen/errno.h>
26 #include <xen/init.h>
27 #include <xen/delay.h>
28 #include <xen/cpumask.h>
29 #include <xen/timer.h>
30 #include <xen/xmalloc.h>
31 #include <asm/bug.h>
32 #include <asm/msr.h>
33 #include <asm/io.h>
34 #include <asm/processor.h>
35 #include <asm/percpu.h>
36 #include <asm/cpufeature.h>
37 #include <acpi/acpi.h>
38 #include <acpi/cpufreq/cpufreq.h>
39
40 #define CPUID_FREQ_VOLT_CAPABILITIES 0x80000007
41 #define CPB_CAPABLE 0x00000200
42 #define USE_HW_PSTATE 0x00000080
43 #define HW_PSTATE_MASK 0x00000007
44 #define HW_PSTATE_VALID_MASK 0x80000000
45 #define HW_PSTATE_MAX_MASK 0x000000f0
46 #define HW_PSTATE_MAX_SHIFT 4
47 #define MSR_PSTATE_DEF_BASE 0xc0010064 /* base of Pstate MSRs */
48 #define MSR_PSTATE_STATUS 0xc0010063 /* Pstate Status MSR */
49 #define MSR_PSTATE_CTRL 0xc0010062 /* Pstate control MSR */
50 #define MSR_PSTATE_CUR_LIMIT 0xc0010061 /* pstate current limit MSR */
51 #define MSR_HWCR_CPBDIS_MASK 0x02000000ULL
52
53 #define ARCH_CPU_FLAG_RESUME 1
54
55 static struct cpufreq_driver powernow_cpufreq_driver;
56
transition_pstate(void * pstate)57 static void transition_pstate(void *pstate)
58 {
59 wrmsrl(MSR_PSTATE_CTRL, *(unsigned int *)pstate);
60 }
61
update_cpb(void * data)62 static void update_cpb(void *data)
63 {
64 struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
65
66 if (policy->turbo != CPUFREQ_TURBO_UNSUPPORTED) {
67 uint64_t msr_content;
68
69 rdmsrl(MSR_K8_HWCR, msr_content);
70
71 if (policy->turbo == CPUFREQ_TURBO_ENABLED)
72 msr_content &= ~MSR_HWCR_CPBDIS_MASK;
73 else
74 msr_content |= MSR_HWCR_CPBDIS_MASK;
75
76 wrmsrl(MSR_K8_HWCR, msr_content);
77 }
78 }
79
powernow_cpufreq_update(int cpuid,struct cpufreq_policy * policy)80 static int powernow_cpufreq_update (int cpuid,
81 struct cpufreq_policy *policy)
82 {
83 if (!cpumask_test_cpu(cpuid, &cpu_online_map))
84 return -EINVAL;
85
86 on_selected_cpus(cpumask_of(cpuid), update_cpb, policy, 1);
87
88 return 0;
89 }
90
powernow_cpufreq_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)91 static int powernow_cpufreq_target(struct cpufreq_policy *policy,
92 unsigned int target_freq, unsigned int relation)
93 {
94 struct acpi_cpufreq_data *data = cpufreq_drv_data[policy->cpu];
95 struct processor_performance *perf;
96 unsigned int next_state; /* Index into freq_table */
97 unsigned int next_perf_state; /* Index into perf table */
98 int result;
99
100 if (unlikely(data == NULL ||
101 data->acpi_data == NULL || data->freq_table == NULL)) {
102 return -ENODEV;
103 }
104
105 perf = data->acpi_data;
106 result = cpufreq_frequency_table_target(policy,
107 data->freq_table,
108 target_freq,
109 relation, &next_state);
110 if (unlikely(result))
111 return result;
112
113 next_perf_state = data->freq_table[next_state].index;
114 if (perf->state == next_perf_state) {
115 if (unlikely(data->arch_cpu_flags & ARCH_CPU_FLAG_RESUME))
116 data->arch_cpu_flags &= ~ARCH_CPU_FLAG_RESUME;
117 else
118 return 0;
119 }
120
121 if (policy->shared_type == CPUFREQ_SHARED_TYPE_HW &&
122 likely(policy->cpu == smp_processor_id())) {
123 transition_pstate(&next_perf_state);
124 cpufreq_statistic_update(policy->cpu, perf->state, next_perf_state);
125 } else {
126 cpumask_t online_policy_cpus;
127 unsigned int cpu;
128
129 cpumask_and(&online_policy_cpus, policy->cpus, &cpu_online_map);
130
131 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
132 unlikely(policy->cpu != smp_processor_id()))
133 on_selected_cpus(&online_policy_cpus, transition_pstate,
134 &next_perf_state, 1);
135 else
136 transition_pstate(&next_perf_state);
137
138 for_each_cpu(cpu, &online_policy_cpus)
139 cpufreq_statistic_update(cpu, perf->state, next_perf_state);
140 }
141
142 perf->state = next_perf_state;
143 policy->cur = data->freq_table[next_state].frequency;
144
145 return 0;
146 }
147
amd_fixup_frequency(struct xen_processor_px * px)148 static void amd_fixup_frequency(struct xen_processor_px *px)
149 {
150 u32 hi, lo, fid, did;
151 int index = px->control & 0x00000007;
152 const struct cpuinfo_x86 *c = ¤t_cpu_data;
153
154 if ((c->x86 != 0x10 || c->x86_model >= 10) && c->x86 != 0x11)
155 return;
156
157 rdmsr(MSR_PSTATE_DEF_BASE + index, lo, hi);
158 /*
159 * MSR C001_0064+:
160 * Bit 63: PstateEn. Read-write. If set, the P-state is valid.
161 */
162 if (!(hi & (1U << 31)))
163 return;
164
165 fid = lo & 0x3f;
166 did = (lo >> 6) & 7;
167 if (c->x86 == 0x10)
168 px->core_frequency = (100 * (fid + 16)) >> did;
169 else
170 px->core_frequency = (100 * (fid + 8)) >> did;
171 }
172
173 struct amd_cpu_data {
174 struct processor_performance *perf;
175 u32 max_hw_pstate;
176 };
177
get_cpu_data(void * arg)178 static void get_cpu_data(void *arg)
179 {
180 struct amd_cpu_data *data = arg;
181 struct processor_performance *perf = data->perf;
182 uint64_t msr_content;
183 unsigned int i;
184
185 rdmsrl(MSR_PSTATE_CUR_LIMIT, msr_content);
186 data->max_hw_pstate = (msr_content & HW_PSTATE_MAX_MASK) >>
187 HW_PSTATE_MAX_SHIFT;
188
189 for (i = 0; i < perf->state_count && i <= data->max_hw_pstate; i++)
190 amd_fixup_frequency(&perf->states[i]);
191 }
192
powernow_cpufreq_verify(struct cpufreq_policy * policy)193 static int powernow_cpufreq_verify(struct cpufreq_policy *policy)
194 {
195 struct acpi_cpufreq_data *data;
196 struct processor_performance *perf;
197
198 if (!policy || !(data = cpufreq_drv_data[policy->cpu]) ||
199 !processor_pminfo[policy->cpu])
200 return -EINVAL;
201
202 perf = &processor_pminfo[policy->cpu]->perf;
203
204 cpufreq_verify_within_limits(policy, 0,
205 perf->states[perf->platform_limit].core_frequency * 1000);
206
207 return cpufreq_frequency_table_verify(policy, data->freq_table);
208 }
209
feature_detect(void * info)210 static void feature_detect(void *info)
211 {
212 struct cpufreq_policy *policy = info;
213 unsigned int edx;
214
215 if ( cpu_has_aperfmperf )
216 {
217 policy->aperf_mperf = 1;
218 powernow_cpufreq_driver.getavg = get_measured_perf;
219 }
220
221 edx = cpuid_edx(CPUID_FREQ_VOLT_CAPABILITIES);
222 if ((edx & CPB_CAPABLE) == CPB_CAPABLE) {
223 policy->turbo = CPUFREQ_TURBO_ENABLED;
224 if (cpufreq_verbose)
225 printk(XENLOG_INFO
226 "CPU%u: Core Boost/Turbo detected and enabled\n",
227 smp_processor_id());
228 }
229 }
230
powernow_cpufreq_cpu_init(struct cpufreq_policy * policy)231 static int powernow_cpufreq_cpu_init(struct cpufreq_policy *policy)
232 {
233 unsigned int i;
234 unsigned int valid_states = 0;
235 unsigned int cpu = policy->cpu;
236 struct acpi_cpufreq_data *data;
237 unsigned int result = 0;
238 struct processor_performance *perf;
239 struct amd_cpu_data info;
240 struct cpuinfo_x86 *c = &cpu_data[policy->cpu];
241
242 data = xzalloc(struct acpi_cpufreq_data);
243 if (!data)
244 return -ENOMEM;
245
246 cpufreq_drv_data[cpu] = data;
247
248 data->acpi_data = &processor_pminfo[cpu]->perf;
249
250 info.perf = perf = data->acpi_data;
251 policy->shared_type = perf->shared_type;
252
253 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
254 policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
255 cpumask_set_cpu(cpu, policy->cpus);
256 if (cpumask_weight(policy->cpus) != 1) {
257 printk(XENLOG_WARNING "Unsupported sharing type %d (%u CPUs)\n",
258 policy->shared_type, cpumask_weight(policy->cpus));
259 result = -ENODEV;
260 goto err_unreg;
261 }
262 } else {
263 cpumask_copy(policy->cpus, cpumask_of(cpu));
264 }
265
266 /* capability check */
267 if (perf->state_count <= 1) {
268 printk("No P-States\n");
269 result = -ENODEV;
270 goto err_unreg;
271 }
272
273 if (perf->control_register.space_id != perf->status_register.space_id) {
274 result = -ENODEV;
275 goto err_unreg;
276 }
277
278 data->freq_table = xmalloc_array(struct cpufreq_frequency_table,
279 (perf->state_count+1));
280 if (!data->freq_table) {
281 result = -ENOMEM;
282 goto err_unreg;
283 }
284
285 /* detect transition latency */
286 policy->cpuinfo.transition_latency = 0;
287 for (i=0; i<perf->state_count; i++) {
288 if ((perf->states[i].transition_latency * 1000) >
289 policy->cpuinfo.transition_latency)
290 policy->cpuinfo.transition_latency =
291 perf->states[i].transition_latency * 1000;
292 }
293
294 policy->governor = cpufreq_opt_governor ? : CPUFREQ_DEFAULT_GOVERNOR;
295
296 on_selected_cpus(cpumask_of(cpu), get_cpu_data, &info, 1);
297
298 /* table init */
299 for (i = 0; i < perf->state_count && i <= info.max_hw_pstate; i++) {
300 if (i > 0 && perf->states[i].core_frequency >=
301 data->freq_table[valid_states-1].frequency / 1000)
302 continue;
303
304 data->freq_table[valid_states].index = perf->states[i].control & HW_PSTATE_MASK;
305 data->freq_table[valid_states].frequency =
306 perf->states[i].core_frequency * 1000;
307 valid_states++;
308 }
309 data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END;
310 perf->state = 0;
311
312 result = cpufreq_frequency_table_cpuinfo(policy, data->freq_table);
313 if (result)
314 goto err_freqfree;
315
316 if (c->cpuid_level >= 6)
317 on_selected_cpus(cpumask_of(cpu), feature_detect, policy, 1);
318
319 /*
320 * the first call to ->target() should result in us actually
321 * writing something to the appropriate registers.
322 */
323 data->arch_cpu_flags |= ARCH_CPU_FLAG_RESUME;
324
325 policy->cur = data->freq_table[i].frequency;
326 return result;
327
328 err_freqfree:
329 xfree(data->freq_table);
330 err_unreg:
331 xfree(data);
332 cpufreq_drv_data[cpu] = NULL;
333
334 return result;
335 }
336
powernow_cpufreq_cpu_exit(struct cpufreq_policy * policy)337 static int powernow_cpufreq_cpu_exit(struct cpufreq_policy *policy)
338 {
339 struct acpi_cpufreq_data *data = cpufreq_drv_data[policy->cpu];
340
341 if (data) {
342 cpufreq_drv_data[policy->cpu] = NULL;
343 xfree(data->freq_table);
344 xfree(data);
345 }
346
347 return 0;
348 }
349
350 static struct cpufreq_driver powernow_cpufreq_driver = {
351 .verify = powernow_cpufreq_verify,
352 .target = powernow_cpufreq_target,
353 .init = powernow_cpufreq_cpu_init,
354 .exit = powernow_cpufreq_cpu_exit,
355 .update = powernow_cpufreq_update
356 };
357
powernow_register_driver()358 unsigned int __init powernow_register_driver()
359 {
360 unsigned int i, ret = 0;
361
362 for_each_online_cpu(i) {
363 struct cpuinfo_x86 *c = &cpu_data[i];
364 if (c->x86_vendor != X86_VENDOR_AMD)
365 ret = -ENODEV;
366 else
367 {
368 u32 eax, ebx, ecx, edx;
369 cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
370 if ((edx & USE_HW_PSTATE) != USE_HW_PSTATE)
371 ret = -ENODEV;
372 }
373 if (ret)
374 return ret;
375 }
376
377 ret = cpufreq_register_driver(&powernow_cpufreq_driver);
378 return ret;
379 }
380