1 /*
2 * x2APIC driver.
3 *
4 * Copyright (c) 2008, Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program; If not, see <http://www.gnu.org/licenses/>.
17 */
18
19 #include <xen/init.h>
20 #include <xen/cpu.h>
21 #include <xen/cpumask.h>
22 #include <asm/apicdef.h>
23 #include <asm/genapic.h>
24 #include <asm/apic.h>
25 #include <asm/io_apic.h>
26 #include <asm/msr.h>
27 #include <asm/processor.h>
28 #include <xen/smp.h>
29 #include <asm/mach-default/mach_mpparse.h>
30
31 static DEFINE_PER_CPU_READ_MOSTLY(u32, cpu_2_logical_apicid);
32 static DEFINE_PER_CPU_READ_MOSTLY(cpumask_t *, cluster_cpus);
33 static cpumask_t *cluster_cpus_spare;
34 static DEFINE_PER_CPU(cpumask_var_t, scratch_mask);
35
x2apic_cluster(unsigned int cpu)36 static inline u32 x2apic_cluster(unsigned int cpu)
37 {
38 return per_cpu(cpu_2_logical_apicid, cpu) >> 16;
39 }
40
init_apic_ldr_x2apic_phys(void)41 static void init_apic_ldr_x2apic_phys(void)
42 {
43 }
44
init_apic_ldr_x2apic_cluster(void)45 static void init_apic_ldr_x2apic_cluster(void)
46 {
47 unsigned int cpu, this_cpu = smp_processor_id();
48
49 per_cpu(cpu_2_logical_apicid, this_cpu) = apic_read(APIC_LDR);
50
51 if ( per_cpu(cluster_cpus, this_cpu) )
52 {
53 ASSERT(cpumask_test_cpu(this_cpu, per_cpu(cluster_cpus, this_cpu)));
54 return;
55 }
56
57 per_cpu(cluster_cpus, this_cpu) = cluster_cpus_spare;
58 for_each_online_cpu ( cpu )
59 {
60 if (this_cpu == cpu || x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
61 continue;
62 per_cpu(cluster_cpus, this_cpu) = per_cpu(cluster_cpus, cpu);
63 break;
64 }
65 if ( per_cpu(cluster_cpus, this_cpu) == cluster_cpus_spare )
66 cluster_cpus_spare = NULL;
67
68 cpumask_set_cpu(this_cpu, per_cpu(cluster_cpus, this_cpu));
69 }
70
clustered_apic_check_x2apic(void)71 static void __init clustered_apic_check_x2apic(void)
72 {
73 }
74
vector_allocation_cpumask_x2apic_cluster(int cpu)75 static const cpumask_t *vector_allocation_cpumask_x2apic_cluster(int cpu)
76 {
77 return per_cpu(cluster_cpus, cpu);
78 }
79
cpu_mask_to_apicid_x2apic_cluster(const cpumask_t * cpumask)80 static unsigned int cpu_mask_to_apicid_x2apic_cluster(const cpumask_t *cpumask)
81 {
82 unsigned int cpu = cpumask_any(cpumask);
83 unsigned int dest = per_cpu(cpu_2_logical_apicid, cpu);
84 const cpumask_t *cluster_cpus = per_cpu(cluster_cpus, cpu);
85
86 for_each_cpu ( cpu, cluster_cpus )
87 if ( cpumask_test_cpu(cpu, cpumask) )
88 dest |= per_cpu(cpu_2_logical_apicid, cpu);
89
90 return dest;
91 }
92
send_IPI_self_x2apic(uint8_t vector)93 static void send_IPI_self_x2apic(uint8_t vector)
94 {
95 apic_wrmsr(APIC_SELF_IPI, vector);
96 }
97
send_IPI_mask_x2apic_phys(const cpumask_t * cpumask,int vector)98 static void send_IPI_mask_x2apic_phys(const cpumask_t *cpumask, int vector)
99 {
100 unsigned int cpu;
101 unsigned long flags;
102 uint64_t msr_content;
103
104 /*
105 * Ensure that any synchronisation data written in program order by this
106 * CPU is seen by notified remote CPUs. The WRMSR contained within
107 * apic_icr_write() can otherwise be executed early.
108 *
109 * The reason mb() is sufficient here is subtle: the register arguments
110 * to WRMSR must depend on a memory read executed after the barrier. This
111 * is guaranteed by cpu_physical_id(), which reads from a global array (and
112 * so cannot be hoisted above the barrier even by a clever compiler).
113 */
114 mb();
115
116 local_irq_save(flags);
117
118 for_each_cpu ( cpu, cpumask )
119 {
120 if ( !cpu_online(cpu) || (cpu == smp_processor_id()) )
121 continue;
122 msr_content = cpu_physical_id(cpu);
123 msr_content = (msr_content << 32) | APIC_DM_FIXED |
124 APIC_DEST_PHYSICAL | vector;
125 apic_wrmsr(APIC_ICR, msr_content);
126 }
127
128 local_irq_restore(flags);
129 }
130
send_IPI_mask_x2apic_cluster(const cpumask_t * cpumask,int vector)131 static void send_IPI_mask_x2apic_cluster(const cpumask_t *cpumask, int vector)
132 {
133 unsigned int cpu = smp_processor_id();
134 cpumask_t *ipimask = per_cpu(scratch_mask, cpu);
135 const cpumask_t *cluster_cpus;
136 unsigned long flags;
137
138 mb(); /* See above for an explanation. */
139
140 local_irq_save(flags);
141
142 cpumask_andnot(ipimask, &cpu_online_map, cpumask_of(cpu));
143
144 for ( cpumask_and(ipimask, cpumask, ipimask); !cpumask_empty(ipimask);
145 cpumask_andnot(ipimask, ipimask, cluster_cpus) )
146 {
147 uint64_t msr_content = 0;
148
149 cluster_cpus = per_cpu(cluster_cpus, cpumask_first(ipimask));
150 for_each_cpu ( cpu, cluster_cpus )
151 {
152 if ( !cpumask_test_cpu(cpu, ipimask) )
153 continue;
154 msr_content |= per_cpu(cpu_2_logical_apicid, cpu);
155 }
156
157 BUG_ON(!msr_content);
158 msr_content = (msr_content << 32) | APIC_DM_FIXED |
159 APIC_DEST_LOGICAL | vector;
160 apic_wrmsr(APIC_ICR, msr_content);
161 }
162
163 local_irq_restore(flags);
164 }
165
166 static const struct genapic apic_x2apic_phys = {
167 APIC_INIT("x2apic_phys", NULL),
168 .int_delivery_mode = dest_Fixed,
169 .int_dest_mode = 0 /* physical delivery */,
170 .init_apic_ldr = init_apic_ldr_x2apic_phys,
171 .clustered_apic_check = clustered_apic_check_x2apic,
172 .target_cpus = target_cpus_all,
173 .vector_allocation_cpumask = vector_allocation_cpumask_phys,
174 .cpu_mask_to_apicid = cpu_mask_to_apicid_phys,
175 .send_IPI_mask = send_IPI_mask_x2apic_phys,
176 .send_IPI_self = send_IPI_self_x2apic
177 };
178
179 static const struct genapic apic_x2apic_cluster = {
180 APIC_INIT("x2apic_cluster", NULL),
181 .int_delivery_mode = dest_LowestPrio,
182 .int_dest_mode = 1 /* logical delivery */,
183 .init_apic_ldr = init_apic_ldr_x2apic_cluster,
184 .clustered_apic_check = clustered_apic_check_x2apic,
185 .target_cpus = target_cpus_all,
186 .vector_allocation_cpumask = vector_allocation_cpumask_x2apic_cluster,
187 .cpu_mask_to_apicid = cpu_mask_to_apicid_x2apic_cluster,
188 .send_IPI_mask = send_IPI_mask_x2apic_cluster,
189 .send_IPI_self = send_IPI_self_x2apic
190 };
191
update_clusterinfo(struct notifier_block * nfb,unsigned long action,void * hcpu)192 static int update_clusterinfo(
193 struct notifier_block *nfb, unsigned long action, void *hcpu)
194 {
195 unsigned int cpu = (unsigned long)hcpu;
196 int err = 0;
197
198 switch (action) {
199 case CPU_UP_PREPARE:
200 per_cpu(cpu_2_logical_apicid, cpu) = BAD_APICID;
201 if ( !cluster_cpus_spare )
202 cluster_cpus_spare = xzalloc(cpumask_t);
203 if ( !cluster_cpus_spare ||
204 !alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) )
205 err = -ENOMEM;
206 break;
207 case CPU_UP_CANCELED:
208 case CPU_DEAD:
209 if ( per_cpu(cluster_cpus, cpu) )
210 {
211 cpumask_clear_cpu(cpu, per_cpu(cluster_cpus, cpu));
212 if ( cpumask_empty(per_cpu(cluster_cpus, cpu)) )
213 xfree(per_cpu(cluster_cpus, cpu));
214 }
215 free_cpumask_var(per_cpu(scratch_mask, cpu));
216 break;
217 }
218
219 return !err ? NOTIFY_DONE : notifier_from_errno(err);
220 }
221
222 static struct notifier_block x2apic_cpu_nfb = {
223 .notifier_call = update_clusterinfo
224 };
225
226 static s8 __initdata x2apic_phys = -1; /* By default we use logical cluster mode. */
227 boolean_param("x2apic_phys", x2apic_phys);
228
apic_x2apic_probe(void)229 const struct genapic *__init apic_x2apic_probe(void)
230 {
231 if ( x2apic_phys < 0 )
232 x2apic_phys = !!(acpi_gbl_FADT.flags & ACPI_FADT_APIC_PHYSICAL);
233
234 if ( x2apic_phys )
235 return &apic_x2apic_phys;
236
237 if ( !this_cpu(cluster_cpus) )
238 {
239 update_clusterinfo(NULL, CPU_UP_PREPARE,
240 (void *)(long)smp_processor_id());
241 init_apic_ldr_x2apic_cluster();
242 register_cpu_notifier(&x2apic_cpu_nfb);
243 }
244
245 return &apic_x2apic_cluster;
246 }
247
check_x2apic_preenabled(void)248 void __init check_x2apic_preenabled(void)
249 {
250 u32 lo, hi;
251
252 if ( !cpu_has_x2apic )
253 return;
254
255 /* Check whether x2apic mode was already enabled by the BIOS. */
256 rdmsr(MSR_IA32_APICBASE, lo, hi);
257 if ( lo & MSR_IA32_APICBASE_EXTD )
258 {
259 printk("x2APIC mode is already enabled by BIOS.\n");
260 x2apic_enabled = 1;
261 genapic = apic_x2apic_probe();
262 }
263 }
264