blob: 77fa155becf654ad90372d6ec9bd3dac04d356dc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Intel IO-APIC support for multi-Pentium hosts.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
5 *
6 * Many thanks to Stig Venaas for trying out countless experimental
7 * patches and reporting/debugging problems patiently!
8 *
9 * (c) 1999, Multiple IO-APIC support, developed by
10 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
11 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
12 * further tested and cleaned up by Zach Brown <zab@redhat.com>
13 * and Ingo Molnar <mingo@redhat.com>
14 *
15 * Fixes
16 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
17 * thanks to Eric Gilmore
18 * and Rolf G. Tews
19 * for testing these extensively
20 * Paul Diefenbaugh : Added full ACPI support
21 */
22
23#include <linux/mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
Yinghai Lud4057bd2008-08-19 20:50:38 -070028#include <linux/pci.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
Alexey Dobriyan129f6942005-06-23 00:08:33 -070032#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <linux/sysdev.h>
Eric W. Biederman3b7d1922006-10-04 02:16:59 -070034#include <linux/msi.h>
Eric W. Biederman95d77882006-10-04 02:17:01 -070035#include <linux/htirq.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080036#include <linux/freezer.h>
Eric W. Biedermanf26d6a22007-05-02 19:27:19 +020037#include <linux/kthread.h>
Ingo Molnar54168ed2008-08-20 09:07:45 +020038#include <linux/jiffies.h> /* time_after() */
Yinghai Lud4057bd2008-08-19 20:50:38 -070039#ifdef CONFIG_ACPI
40#include <acpi/acpi_bus.h>
41#endif
42#include <linux/bootmem.h>
43#include <linux/dmar.h>
venkatesh.pallipadi@intel.com58ac1e72008-09-05 18:02:17 -070044#include <linux/hpet.h>
Ashok Raj54d5d422005-09-06 15:16:15 -070045
Yinghai Lud4057bd2008-08-19 20:50:38 -070046#include <asm/idle.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <asm/io.h>
48#include <asm/smp.h>
49#include <asm/desc.h>
Yinghai Lud4057bd2008-08-19 20:50:38 -070050#include <asm/proto.h>
51#include <asm/acpi.h>
52#include <asm/dma.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <asm/timer.h>
Ingo Molnar306e4402005-06-30 02:58:55 -070054#include <asm/i8259.h>
Don Zickus3e4ff112006-06-26 13:57:01 +020055#include <asm/nmi.h>
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -070056#include <asm/msidef.h>
Eric W. Biederman8b955b02006-10-04 02:16:55 -070057#include <asm/hypertransport.h>
Yinghai Lua4dbc342008-07-25 02:14:28 -070058#include <asm/setup.h>
Yinghai Lud4057bd2008-08-19 20:50:38 -070059#include <asm/irq_remapping.h>
venkatesh.pallipadi@intel.com58ac1e72008-09-05 18:02:17 -070060#include <asm/hpet.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
Yinghai Lu497c9a12008-08-19 20:50:28 -070062#include <mach_ipi.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070063#include <mach_apic.h>
Andi Kleen874c4fe2006-09-26 10:52:26 +020064#include <mach_apicdef.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +010066#define __apicdebuginit(type) static type __init
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068/*
Ingo Molnar54168ed2008-08-20 09:07:45 +020069 * Is the SiS APIC rmw bug present ?
70 * -1 = don't know, 0 = no, 1 = yes
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 */
72int sis_apic_bug = -1;
73
Yinghai Luefa25592008-08-19 20:50:36 -070074static DEFINE_SPINLOCK(ioapic_lock);
75static DEFINE_SPINLOCK(vector_lock);
76
Yinghai Luefa25592008-08-19 20:50:36 -070077/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 * # of IRQ routing registers
79 */
80int nr_ioapic_registers[MAX_IO_APICS];
81
Alexey Starikovskiy9f640cc2008-04-04 23:41:13 +040082/* I/O APIC entries */
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +040083struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
Alexey Starikovskiy9f640cc2008-04-04 23:41:13 +040084int nr_ioapics;
85
Alexey Starikovskiy584f7342008-04-04 23:41:32 +040086/* MP IRQ source entries */
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +040087struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
Alexey Starikovskiy584f7342008-04-04 23:41:32 +040088
89/* # of MP IRQ source entries */
90int mp_irq_entries;
91
Alexey Starikovskiy8732fc42008-05-19 19:47:16 +040092#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
93int mp_bus_id_to_type[MAX_MP_BUSSES];
94#endif
95
96DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
97
Yinghai Luefa25592008-08-19 20:50:36 -070098int skip_ioapic_setup;
99
Ingo Molnar54168ed2008-08-20 09:07:45 +0200100static int __init parse_noapic(char *str)
Yinghai Luefa25592008-08-19 20:50:36 -0700101{
102 /* disable IO-APIC */
103 disable_ioapic_setup();
104 return 0;
105}
106early_param("noapic", parse_noapic);
Chuck Ebbert66759a02005-09-12 18:49:25 +0200107
Yinghai Luda51a822008-08-19 20:50:25 -0700108struct irq_cfg;
Yinghai Lu0f978f42008-08-19 20:50:26 -0700109struct irq_pin_list;
Yinghai Lua1420f32008-08-19 20:50:24 -0700110struct irq_cfg {
Yinghai Luda51a822008-08-19 20:50:25 -0700111 unsigned int irq;
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700112#ifdef CONFIG_HAVE_SPARSE_IRQ
Yinghai Luda51a822008-08-19 20:50:25 -0700113 struct irq_cfg *next;
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700114#endif
Yinghai Lu0f978f42008-08-19 20:50:26 -0700115 struct irq_pin_list *irq_2_pin;
Yinghai Lu497c9a12008-08-19 20:50:28 -0700116 cpumask_t domain;
117 cpumask_t old_domain;
118 unsigned move_cleanup_count;
Yinghai Lua1420f32008-08-19 20:50:24 -0700119 u8 vector;
Yinghai Lu497c9a12008-08-19 20:50:28 -0700120 u8 move_in_progress : 1;
Yinghai Lua1420f32008-08-19 20:50:24 -0700121};
122
Yinghai Lua1420f32008-08-19 20:50:24 -0700123/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
124static struct irq_cfg irq_cfg_legacy[] __initdata = {
Yinghai Lu497c9a12008-08-19 20:50:28 -0700125 [0] = { .irq = 0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, },
126 [1] = { .irq = 1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, },
127 [2] = { .irq = 2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, },
128 [3] = { .irq = 3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, },
129 [4] = { .irq = 4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, },
130 [5] = { .irq = 5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, },
131 [6] = { .irq = 6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, },
132 [7] = { .irq = 7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, },
133 [8] = { .irq = 8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, },
134 [9] = { .irq = 9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, },
135 [10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
136 [11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
137 [12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
138 [13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
139 [14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
140 [15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
Yinghai Lua1420f32008-08-19 20:50:24 -0700141};
142
Yinghai Luda51a822008-08-19 20:50:25 -0700143static struct irq_cfg irq_cfg_init = { .irq = -1U, };
Yinghai Lua1420f32008-08-19 20:50:24 -0700144
Yinghai Luda51a822008-08-19 20:50:25 -0700145static void init_one_irq_cfg(struct irq_cfg *cfg)
146{
147 memcpy(cfg, &irq_cfg_init, sizeof(struct irq_cfg));
Yinghai Lua1420f32008-08-19 20:50:24 -0700148}
149
150static struct irq_cfg *irq_cfgx;
Yinghai Lue89eb432008-08-20 20:46:25 -0700151
Cyrill Gorcunovac54a6c2008-09-04 22:37:50 +0400152#ifdef CONFIG_HAVE_SPARSE_IRQ
Yinghai Lue89eb432008-08-20 20:46:25 -0700153/*
154 * Protect the irq_cfgx_free freelist:
155 */
156static DEFINE_SPINLOCK(irq_cfg_lock);
157
Yinghai Luda51a822008-08-19 20:50:25 -0700158static struct irq_cfg *irq_cfgx_free;
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700159#endif
Cyrill Gorcunovac54a6c2008-09-04 22:37:50 +0400160
Yinghai Luda51a822008-08-19 20:50:25 -0700161static void __init init_work(void *data)
162{
163 struct dyn_array *da = data;
164 struct irq_cfg *cfg;
165 int legacy_count;
166 int i;
167
168 cfg = *da->name;
169
170 memcpy(cfg, irq_cfg_legacy, sizeof(irq_cfg_legacy));
171
Cyrill Gorcunov676f4a92008-09-04 22:37:49 +0400172 legacy_count = ARRAY_SIZE(irq_cfg_legacy);
Yinghai Luda51a822008-08-19 20:50:25 -0700173 for (i = legacy_count; i < *da->nr; i++)
174 init_one_irq_cfg(&cfg[i]);
175
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700176#ifdef CONFIG_HAVE_SPARSE_IRQ
Yinghai Luda51a822008-08-19 20:50:25 -0700177 for (i = 1; i < *da->nr; i++)
178 cfg[i-1].next = &cfg[i];
179
180 irq_cfgx_free = &irq_cfgx[legacy_count];
181 irq_cfgx[legacy_count - 1].next = NULL;
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700182#endif
Yinghai Luda51a822008-08-19 20:50:25 -0700183}
184
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700185#ifdef CONFIG_HAVE_SPARSE_IRQ
186/* need to be biger than size of irq_cfg_legacy */
187static int nr_irq_cfg = 32;
188
189static int __init parse_nr_irq_cfg(char *arg)
190{
191 if (arg) {
192 nr_irq_cfg = simple_strtoul(arg, NULL, 0);
193 if (nr_irq_cfg < 32)
194 nr_irq_cfg = 32;
195 }
196 return 0;
197}
198
199early_param("nr_irq_cfg", parse_nr_irq_cfg);
200
201#define for_each_irq_cfg(irqX, cfg) \
202 for (cfg = irq_cfgx, irqX = cfg->irq; cfg; cfg = cfg->next, irqX = cfg ? cfg->irq : -1U)
203
Yinghai Luda51a822008-08-19 20:50:25 -0700204
205DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irq_cfg, PAGE_SIZE, init_work);
Yinghai Lua1420f32008-08-19 20:50:24 -0700206
207static struct irq_cfg *irq_cfg(unsigned int irq)
208{
Yinghai Luda51a822008-08-19 20:50:25 -0700209 struct irq_cfg *cfg;
Yinghai Lua1420f32008-08-19 20:50:24 -0700210
Yinghai Luda51a822008-08-19 20:50:25 -0700211 cfg = irq_cfgx;
212 while (cfg) {
213 if (cfg->irq == irq)
214 return cfg;
215
216 cfg = cfg->next;
217 }
218
219 return NULL;
220}
221
222static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
223{
224 struct irq_cfg *cfg, *cfg_pri;
Yinghai Lue89eb432008-08-20 20:46:25 -0700225 unsigned long flags;
Yinghai Luda51a822008-08-19 20:50:25 -0700226 int count = 0;
Yinghai Lue89eb432008-08-20 20:46:25 -0700227 int i;
Yinghai Luda51a822008-08-19 20:50:25 -0700228
229 cfg_pri = cfg = irq_cfgx;
230 while (cfg) {
231 if (cfg->irq == irq)
232 return cfg;
233
234 cfg_pri = cfg;
235 cfg = cfg->next;
236 count++;
237 }
238
Yinghai Lue89eb432008-08-20 20:46:25 -0700239 spin_lock_irqsave(&irq_cfg_lock, flags);
Yinghai Luda51a822008-08-19 20:50:25 -0700240 if (!irq_cfgx_free) {
241 unsigned long phys;
242 unsigned long total_bytes;
243 /*
244 * we run out of pre-allocate ones, allocate more
245 */
246 printk(KERN_DEBUG "try to get more irq_cfg %d\n", nr_irq_cfg);
247
248 total_bytes = sizeof(struct irq_cfg) * nr_irq_cfg;
249 if (after_bootmem)
250 cfg = kzalloc(total_bytes, GFP_ATOMIC);
251 else
252 cfg = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0);
253
254 if (!cfg)
255 panic("please boot with nr_irq_cfg= %d\n", count * 2);
256
257 phys = __pa(cfg);
258 printk(KERN_DEBUG "irq_irq ==> [%#lx - %#lx]\n", phys, phys + total_bytes);
259
260 for (i = 0; i < nr_irq_cfg; i++)
261 init_one_irq_cfg(&cfg[i]);
262
263 for (i = 1; i < nr_irq_cfg; i++)
264 cfg[i-1].next = &cfg[i];
265
266 irq_cfgx_free = cfg;
267 }
268
269 cfg = irq_cfgx_free;
270 irq_cfgx_free = irq_cfgx_free->next;
271 cfg->next = NULL;
272 if (cfg_pri)
273 cfg_pri->next = cfg;
274 else
275 irq_cfgx = cfg;
276 cfg->irq = irq;
Yinghai Lue89eb432008-08-20 20:46:25 -0700277
278 spin_unlock_irqrestore(&irq_cfg_lock, flags);
279
Yinghai Luda51a822008-08-19 20:50:25 -0700280 printk(KERN_DEBUG "found new irq_cfg for irq %d\n", cfg->irq);
Yinghai Luda51a822008-08-19 20:50:25 -0700281#ifdef CONFIG_HAVE_SPARSE_IRQ_DEBUG
282 {
283 /* dump the results */
284 struct irq_cfg *cfg;
285 unsigned long phys;
286 unsigned long bytes = sizeof(struct irq_cfg);
287
288 printk(KERN_DEBUG "=========================== %d\n", irq);
289 printk(KERN_DEBUG "irq_cfg dump after get that for %d\n", irq);
290 for_each_irq_cfg(cfg) {
291 phys = __pa(cfg);
292 printk(KERN_DEBUG "irq_cfg %d ==> [%#lx - %#lx]\n", cfg->irq, phys, phys + bytes);
293 }
294 printk(KERN_DEBUG "===========================\n");
295 }
296#endif
297 return cfg;
Yinghai Lua1420f32008-08-19 20:50:24 -0700298}
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700299#else
Yinghai Lua1420f32008-08-19 20:50:24 -0700300
Yinghai Lu8f09cd22008-08-19 20:50:51 -0700301#define for_each_irq_cfg(irq, cfg) \
302 for (irq = 0, cfg = &irq_cfgx[irq]; irq < nr_irqs; irq++, cfg = &irq_cfgx[irq])
303
304DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irqs, PAGE_SIZE, init_work);
305
306struct irq_cfg *irq_cfg(unsigned int irq)
307{
308 if (irq < nr_irqs)
309 return &irq_cfgx[irq];
310
311 return NULL;
312}
313struct irq_cfg *irq_cfg_alloc(unsigned int irq)
314{
315 return irq_cfg(irq);
316}
317
318#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319/*
320 * This is performance-critical, we want to do it O(1)
321 *
322 * the indexing order of this array favors 1:1 mappings
323 * between pins and IRQs.
324 */
325
Yinghai Lu0f978f42008-08-19 20:50:26 -0700326struct irq_pin_list {
327 int apic, pin;
328 struct irq_pin_list *next;
329};
Yinghai Lu301e6192008-08-19 20:50:02 -0700330
Yinghai Lu0f978f42008-08-19 20:50:26 -0700331static struct irq_pin_list *irq_2_pin_head;
332/* fill one page ? */
333static int nr_irq_2_pin = 0x100;
334static struct irq_pin_list *irq_2_pin_ptr;
335static void __init irq_2_pin_init_work(void *data)
336{
337 struct dyn_array *da = data;
338 struct irq_pin_list *pin;
339 int i;
340
341 pin = *da->name;
342
343 for (i = 1; i < *da->nr; i++)
344 pin[i-1].next = &pin[i];
345
346 irq_2_pin_ptr = &pin[0];
347}
348DEFINE_DYN_ARRAY(irq_2_pin_head, sizeof(struct irq_pin_list), nr_irq_2_pin, PAGE_SIZE, irq_2_pin_init_work);
349
350static struct irq_pin_list *get_one_free_irq_2_pin(void)
351{
352 struct irq_pin_list *pin;
353 int i;
354
355 pin = irq_2_pin_ptr;
356
357 if (pin) {
358 irq_2_pin_ptr = pin->next;
359 pin->next = NULL;
360 return pin;
361 }
362
363 /*
364 * we run out of pre-allocate ones, allocate more
365 */
366 printk(KERN_DEBUG "try to get more irq_2_pin %d\n", nr_irq_2_pin);
367
368 if (after_bootmem)
369 pin = kzalloc(sizeof(struct irq_pin_list)*nr_irq_2_pin,
370 GFP_ATOMIC);
371 else
372 pin = __alloc_bootmem_nopanic(sizeof(struct irq_pin_list) *
373 nr_irq_2_pin, PAGE_SIZE, 0);
374
375 if (!pin)
376 panic("can not get more irq_2_pin\n");
377
378 for (i = 1; i < nr_irq_2_pin; i++)
379 pin[i-1].next = &pin[i];
380
381 irq_2_pin_ptr = pin->next;
382 pin->next = NULL;
383
384 return pin;
385}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386
Linus Torvalds130fe052006-11-01 09:11:00 -0800387struct io_apic {
388 unsigned int index;
389 unsigned int unused[3];
390 unsigned int data;
391};
392
393static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
394{
395 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +0400396 + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
Linus Torvalds130fe052006-11-01 09:11:00 -0800397}
398
399static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
400{
401 struct io_apic __iomem *io_apic = io_apic_base(apic);
402 writel(reg, &io_apic->index);
403 return readl(&io_apic->data);
404}
405
406static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
407{
408 struct io_apic __iomem *io_apic = io_apic_base(apic);
409 writel(reg, &io_apic->index);
410 writel(value, &io_apic->data);
411}
412
413/*
414 * Re-write a value: to be used for read-modify-write
415 * cycles where the read already set up the index register.
416 *
417 * Older SiS APIC requires we rewrite the index register
418 */
419static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
420{
Ingo Molnar54168ed2008-08-20 09:07:45 +0200421 struct io_apic __iomem *io_apic = io_apic_base(apic);
422 if (sis_apic_bug)
423 writel(reg, &io_apic->index);
Linus Torvalds130fe052006-11-01 09:11:00 -0800424 writel(value, &io_apic->data);
425}
426
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700427static bool io_apic_level_ack_pending(unsigned int irq)
428{
429 struct irq_pin_list *entry;
430 unsigned long flags;
431 struct irq_cfg *cfg = irq_cfg(irq);
432
433 spin_lock_irqsave(&ioapic_lock, flags);
434 entry = cfg->irq_2_pin;
435 for (;;) {
436 unsigned int reg;
437 int pin;
438
439 if (!entry)
440 break;
441 pin = entry->pin;
442 reg = io_apic_read(entry->apic, 0x10 + pin*2);
443 /* Is the remote IRR bit set? */
444 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
445 spin_unlock_irqrestore(&ioapic_lock, flags);
446 return true;
447 }
448 if (!entry->next)
449 break;
450 entry = entry->next;
451 }
452 spin_unlock_irqrestore(&ioapic_lock, flags);
453
454 return false;
455}
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700456
Andi Kleencf4c6a22006-09-26 10:52:30 +0200457union entry_union {
458 struct { u32 w1, w2; };
459 struct IO_APIC_route_entry entry;
460};
461
462static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
463{
464 union entry_union eu;
465 unsigned long flags;
466 spin_lock_irqsave(&ioapic_lock, flags);
467 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
468 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
469 spin_unlock_irqrestore(&ioapic_lock, flags);
470 return eu.entry;
471}
472
Linus Torvaldsf9dadfa2006-11-01 10:05:35 -0800473/*
474 * When we write a new IO APIC routing entry, we need to write the high
475 * word first! If the mask bit in the low word is clear, we will enable
476 * the interrupt, and we need to make sure the entry is fully populated
477 * before that happens.
478 */
Andi Kleend15512f2006-12-07 02:14:07 +0100479static void
480__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
481{
482 union entry_union eu;
483 eu.entry = e;
484 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
485 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
486}
487
Andi Kleencf4c6a22006-09-26 10:52:30 +0200488static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
489{
490 unsigned long flags;
Andi Kleencf4c6a22006-09-26 10:52:30 +0200491 spin_lock_irqsave(&ioapic_lock, flags);
Andi Kleend15512f2006-12-07 02:14:07 +0100492 __ioapic_write_entry(apic, pin, e);
Linus Torvaldsf9dadfa2006-11-01 10:05:35 -0800493 spin_unlock_irqrestore(&ioapic_lock, flags);
494}
495
496/*
497 * When we mask an IO APIC routing entry, we need to write the low
498 * word first, in order to set the mask bit before we change the
499 * high bits!
500 */
501static void ioapic_mask_entry(int apic, int pin)
502{
503 unsigned long flags;
504 union entry_union eu = { .entry.mask = 1 };
505
506 spin_lock_irqsave(&ioapic_lock, flags);
Andi Kleencf4c6a22006-09-26 10:52:30 +0200507 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
508 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
509 spin_unlock_irqrestore(&ioapic_lock, flags);
510}
511
Yinghai Lu497c9a12008-08-19 20:50:28 -0700512#ifdef CONFIG_SMP
513static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
514{
515 int apic, pin;
516 struct irq_cfg *cfg;
517 struct irq_pin_list *entry;
518
519 cfg = irq_cfg(irq);
520 entry = cfg->irq_2_pin;
521 for (;;) {
522 unsigned int reg;
523
524 if (!entry)
525 break;
526
527 apic = entry->apic;
528 pin = entry->pin;
Ingo Molnar54168ed2008-08-20 09:07:45 +0200529#ifdef CONFIG_INTR_REMAP
530 /*
531 * With interrupt-remapping, destination information comes
532 * from interrupt-remapping table entry.
533 */
534 if (!irq_remapped(irq))
535 io_apic_write(apic, 0x11 + pin*2, dest);
536#else
Yinghai Lu497c9a12008-08-19 20:50:28 -0700537 io_apic_write(apic, 0x11 + pin*2, dest);
Ingo Molnar54168ed2008-08-20 09:07:45 +0200538#endif
Yinghai Lu497c9a12008-08-19 20:50:28 -0700539 reg = io_apic_read(apic, 0x10 + pin*2);
540 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
541 reg |= vector;
Ingo Molnar54168ed2008-08-20 09:07:45 +0200542 io_apic_modify(apic, 0x10 + pin*2, reg);
Yinghai Lu497c9a12008-08-19 20:50:28 -0700543 if (!entry->next)
544 break;
545 entry = entry->next;
546 }
547}
Yinghai Luefa25592008-08-19 20:50:36 -0700548
549static int assign_irq_vector(int irq, cpumask_t mask);
550
Yinghai Lu497c9a12008-08-19 20:50:28 -0700551static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
552{
553 struct irq_cfg *cfg;
554 unsigned long flags;
555 unsigned int dest;
556 cpumask_t tmp;
Ingo Molnar54168ed2008-08-20 09:07:45 +0200557 struct irq_desc *desc;
Yinghai Lu497c9a12008-08-19 20:50:28 -0700558
Yinghai Lu497c9a12008-08-19 20:50:28 -0700559 cpus_and(tmp, mask, cpu_online_map);
560 if (cpus_empty(tmp))
561 return;
562
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700563 cfg = irq_cfg(irq);
Yinghai Lu497c9a12008-08-19 20:50:28 -0700564 if (assign_irq_vector(irq, mask))
565 return;
566
567 cpus_and(tmp, cfg->domain, mask);
Yinghai Lu497c9a12008-08-19 20:50:28 -0700568 dest = cpu_mask_to_apicid(tmp);
569 /*
570 * Only the high 8 bits are valid.
571 */
572 dest = SET_APIC_LOGICAL_ID(dest);
573
Ingo Molnar54168ed2008-08-20 09:07:45 +0200574 desc = irq_to_desc(irq);
Yinghai Lu497c9a12008-08-19 20:50:28 -0700575 spin_lock_irqsave(&ioapic_lock, flags);
576 __target_IO_APIC_irq(irq, dest, cfg->vector);
Ingo Molnar54168ed2008-08-20 09:07:45 +0200577 desc->affinity = mask;
Yinghai Lu497c9a12008-08-19 20:50:28 -0700578 spin_unlock_irqrestore(&ioapic_lock, flags);
579}
Yinghai Lu497c9a12008-08-19 20:50:28 -0700580#endif /* CONFIG_SMP */
581
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582/*
583 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
584 * shared ISA-space IRQs, so we have to support them. We are super
585 * fast in the common case, and fast for shared ISA-space IRQs.
586 */
587static void add_pin_to_irq(unsigned int irq, int apic, int pin)
588{
Yinghai Lu0f978f42008-08-19 20:50:26 -0700589 struct irq_cfg *cfg;
590 struct irq_pin_list *entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591
Yinghai Lu0f978f42008-08-19 20:50:26 -0700592 /* first time to refer irq_cfg, so with new */
593 cfg = irq_cfg_alloc(irq);
594 entry = cfg->irq_2_pin;
595 if (!entry) {
596 entry = get_one_free_irq_2_pin();
597 cfg->irq_2_pin = entry;
598 entry->apic = apic;
599 entry->pin = pin;
600 printk(KERN_DEBUG " 0 add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
601 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 }
Yinghai Lu0f978f42008-08-19 20:50:26 -0700603
604 while (entry->next) {
605 /* not again, please */
606 if (entry->apic == apic && entry->pin == pin)
607 return;
608
609 entry = entry->next;
610 }
611
612 entry->next = get_one_free_irq_2_pin();
613 entry = entry->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 entry->apic = apic;
615 entry->pin = pin;
Yinghai Lu0f978f42008-08-19 20:50:26 -0700616 printk(KERN_DEBUG " x add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617}
618
619/*
620 * Reroute an IRQ to a different pin.
621 */
622static void __init replace_pin_at_irq(unsigned int irq,
623 int oldapic, int oldpin,
624 int newapic, int newpin)
625{
Yinghai Lu0f978f42008-08-19 20:50:26 -0700626 struct irq_cfg *cfg = irq_cfg(irq);
627 struct irq_pin_list *entry = cfg->irq_2_pin;
628 int replaced = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629
Yinghai Lu0f978f42008-08-19 20:50:26 -0700630 while (entry) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 if (entry->apic == oldapic && entry->pin == oldpin) {
632 entry->apic = newapic;
633 entry->pin = newpin;
Yinghai Lu0f978f42008-08-19 20:50:26 -0700634 replaced = 1;
635 /* every one is different, right? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 break;
Yinghai Lu0f978f42008-08-19 20:50:26 -0700637 }
638 entry = entry->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 }
Yinghai Lu0f978f42008-08-19 20:50:26 -0700640
641 /* why? call replace before add? */
642 if (!replaced)
643 add_pin_to_irq(irq, newapic, newpin);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644}
645
Yinghai Lu4e738e22008-08-19 20:50:47 -0700646#define __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL) \
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700647 \
648{ \
649 int pin; \
650 struct irq_cfg *cfg; \
651 struct irq_pin_list *entry; \
652 \
653 cfg = irq_cfg(irq); \
654 entry = cfg->irq_2_pin; \
655 for (;;) { \
656 unsigned int reg; \
657 if (!entry) \
658 break; \
659 pin = entry->pin; \
660 reg = io_apic_read(entry->apic, 0x10 + R + pin*2); \
Yinghai Lu4e738e22008-08-19 20:50:47 -0700661 reg ACTION_DISABLE; \
662 reg ACTION_ENABLE; \
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700663 io_apic_modify(entry->apic, 0x10 + R + pin*2, reg); \
664 FINAL; \
665 if (!entry->next) \
666 break; \
667 entry = entry->next; \
668 } \
669}
670
Yinghai Lu4e738e22008-08-19 20:50:47 -0700671#define DO_ACTION(name,R, ACTION_ENABLE, ACTION_DISABLE, FINAL) \
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700672 \
673 static void name##_IO_APIC_irq (unsigned int irq) \
Yinghai Lu4e738e22008-08-19 20:50:47 -0700674 __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL)
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700675
676/* mask = 0 */
Yinghai Lu4e738e22008-08-19 20:50:47 -0700677DO_ACTION(__unmask, 0, |= 0, &= ~IO_APIC_REDIR_MASKED, )
678
679#ifdef CONFIG_X86_64
680/*
681 * Synchronize the IO-APIC and the CPU by doing
682 * a dummy read from the IO-APIC
683 */
684static inline void io_apic_sync(unsigned int apic)
685{
686 struct io_apic __iomem *io_apic = io_apic_base(apic);
687 readl(&io_apic->data);
688}
689
690/* mask = 1 */
691DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, &= ~0, io_apic_sync(entry->apic))
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700692
693#else
694
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695/* mask = 1 */
Yinghai Lu4e738e22008-08-19 20:50:47 -0700696DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, &= ~0, )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697
698/* mask = 1, trigger = 0 */
Yinghai Lu4e738e22008-08-19 20:50:47 -0700699DO_ACTION(__mask_and_edge, 0, |= IO_APIC_REDIR_MASKED, &= ~IO_APIC_REDIR_LEVEL_TRIGGER, )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700
701/* mask = 0, trigger = 1 */
Yinghai Lu4e738e22008-08-19 20:50:47 -0700702DO_ACTION(__unmask_and_level, 0, |= IO_APIC_REDIR_LEVEL_TRIGGER, &= ~IO_APIC_REDIR_MASKED, )
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703
Yinghai Lu047c8fd2008-08-19 20:50:41 -0700704#endif
705
Ingo Molnar54168ed2008-08-20 09:07:45 +0200706static void mask_IO_APIC_irq (unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707{
708 unsigned long flags;
709
710 spin_lock_irqsave(&ioapic_lock, flags);
711 __mask_IO_APIC_irq(irq);
712 spin_unlock_irqrestore(&ioapic_lock, flags);
713}
714
Ingo Molnar54168ed2008-08-20 09:07:45 +0200715static void unmask_IO_APIC_irq (unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716{
717 unsigned long flags;
718
719 spin_lock_irqsave(&ioapic_lock, flags);
720 __unmask_IO_APIC_irq(irq);
721 spin_unlock_irqrestore(&ioapic_lock, flags);
722}
723
724static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
725{
726 struct IO_APIC_route_entry entry;
Paolo Ciarrocchi36062442008-06-08 13:07:18 +0200727
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 /* Check delivery_mode to be sure we're not clearing an SMI pin */
Andi Kleencf4c6a22006-09-26 10:52:30 +0200729 entry = ioapic_read_entry(apic, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 if (entry.delivery_mode == dest_SMI)
731 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 /*
733 * Disable it in the IO-APIC irq-routing table:
734 */
Linus Torvaldsf9dadfa2006-11-01 10:05:35 -0800735 ioapic_mask_entry(apic, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736}
737
Ingo Molnar54168ed2008-08-20 09:07:45 +0200738static void clear_IO_APIC (void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739{
740 int apic, pin;
741
742 for (apic = 0; apic < nr_ioapics; apic++)
743 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
744 clear_IO_APIC_pin(apic, pin);
745}
746
Ingo Molnar54168ed2008-08-20 09:07:45 +0200747#if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
Harvey Harrison75604d72008-01-30 13:31:17 +0100748void send_IPI_self(int vector)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749{
750 unsigned int cfg;
751
752 /*
753 * Wait for idle.
754 */
755 apic_wait_icr_idle();
756 cfg = APIC_DM_FIXED | APIC_DEST_SELF | vector | APIC_DEST_LOGICAL;
757 /*
758 * Send the IPI. The write to APIC_ICR fires this off.
759 */
Maciej W. Rozycki593f4a72008-07-16 19:15:30 +0100760 apic_write(APIC_ICR, cfg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761}
Ingo Molnar54168ed2008-08-20 09:07:45 +0200762#endif /* !CONFIG_SMP && CONFIG_X86_32*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
Ingo Molnar54168ed2008-08-20 09:07:45 +0200764#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765/*
766 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
767 * specific CPU-side IRQs.
768 */
769
770#define MAX_PIRQS 8
771static int pirq_entries [MAX_PIRQS];
772static int pirqs_enabled;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774static int __init ioapic_pirq_setup(char *str)
775{
776 int i, max;
777 int ints[MAX_PIRQS+1];
778
779 get_options(str, ARRAY_SIZE(ints), ints);
780
781 for (i = 0; i < MAX_PIRQS; i++)
782 pirq_entries[i] = -1;
783
784 pirqs_enabled = 1;
785 apic_printk(APIC_VERBOSE, KERN_INFO
786 "PIRQ redirection, working around broken MP-BIOS.\n");
787 max = MAX_PIRQS;
788 if (ints[0] < MAX_PIRQS)
789 max = ints[0];
790
791 for (i = 0; i < max; i++) {
792 apic_printk(APIC_VERBOSE, KERN_DEBUG
793 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
794 /*
795 * PIRQs are mapped upside down, usually.
796 */
797 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
798 }
799 return 1;
800}
801
802__setup("pirq=", ioapic_pirq_setup);
Ingo Molnar54168ed2008-08-20 09:07:45 +0200803#endif /* CONFIG_X86_32 */
804
805#ifdef CONFIG_INTR_REMAP
806/* I/O APIC RTE contents at the OS boot up */
807static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];
808
809/*
810 * Saves and masks all the unmasked IO-APIC RTE's
811 */
812int save_mask_IO_APIC_setup(void)
813{
814 union IO_APIC_reg_01 reg_01;
815 unsigned long flags;
816 int apic, pin;
817
818 /*
819 * The number of IO-APIC IRQ registers (== #pins):
820 */
821 for (apic = 0; apic < nr_ioapics; apic++) {
822 spin_lock_irqsave(&ioapic_lock, flags);
823 reg_01.raw = io_apic_read(apic, 1);
824 spin_unlock_irqrestore(&ioapic_lock, flags);
825 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
826 }
827
828 for (apic = 0; apic < nr_ioapics; apic++) {
829 early_ioapic_entries[apic] =
830 kzalloc(sizeof(struct IO_APIC_route_entry) *
831 nr_ioapic_registers[apic], GFP_KERNEL);
832 if (!early_ioapic_entries[apic])
833 return -ENOMEM;
834 }
835
836 for (apic = 0; apic < nr_ioapics; apic++)
837 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
838 struct IO_APIC_route_entry entry;
839
840 entry = early_ioapic_entries[apic][pin] =
841 ioapic_read_entry(apic, pin);
842 if (!entry.mask) {
843 entry.mask = 1;
844 ioapic_write_entry(apic, pin, entry);
845 }
846 }
847 return 0;
848}
849
850void restore_IO_APIC_setup(void)
851{
852 int apic, pin;
853
854 for (apic = 0; apic < nr_ioapics; apic++)
855 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
856 ioapic_write_entry(apic, pin,
857 early_ioapic_entries[apic][pin]);
858}
859
860void reinit_intr_remapped_IO_APIC(int intr_remapping)
861{
862 /*
863 * for now plain restore of previous settings.
864 * TBD: In the case of OS enabling interrupt-remapping,
865 * IO-APIC RTE's need to be setup to point to interrupt-remapping
866 * table entries. for now, do a plain restore, and wait for
867 * the setup_IO_APIC_irqs() to do proper initialization.
868 */
869 restore_IO_APIC_setup();
870}
871#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872
873/*
874 * Find the IRQ entry number of a certain pin.
875 */
876static int find_irq_entry(int apic, int pin, int type)
877{
878 int i;
879
880 for (i = 0; i < mp_irq_entries; i++)
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400881 if (mp_irqs[i].mp_irqtype == type &&
882 (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
883 mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
884 mp_irqs[i].mp_dstirq == pin)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 return i;
886
887 return -1;
888}
889
890/*
891 * Find the pin to which IRQ[irq] (ISA) is connected
892 */
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -0800893static int __init find_isa_irq_pin(int irq, int type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894{
895 int i;
896
897 for (i = 0; i < mp_irq_entries; i++) {
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400898 int lbus = mp_irqs[i].mp_srcbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899
Alexey Starikovskiyd27e2b82008-03-20 14:54:18 +0300900 if (test_bit(lbus, mp_bus_not_pci) &&
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400901 (mp_irqs[i].mp_irqtype == type) &&
902 (mp_irqs[i].mp_srcbusirq == irq))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400904 return mp_irqs[i].mp_dstirq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 }
906 return -1;
907}
908
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -0800909static int __init find_isa_irq_apic(int irq, int type)
910{
911 int i;
912
913 for (i = 0; i < mp_irq_entries; i++) {
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400914 int lbus = mp_irqs[i].mp_srcbus;
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -0800915
Alexey Starikovskiy73b29612008-03-20 14:54:24 +0300916 if (test_bit(lbus, mp_bus_not_pci) &&
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400917 (mp_irqs[i].mp_irqtype == type) &&
918 (mp_irqs[i].mp_srcbusirq == irq))
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -0800919 break;
920 }
921 if (i < mp_irq_entries) {
922 int apic;
Ingo Molnar54168ed2008-08-20 09:07:45 +0200923 for(apic = 0; apic < nr_ioapics; apic++) {
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400924 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -0800925 return apic;
926 }
927 }
928
929 return -1;
930}
931
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932/*
933 * Find a specific PCI IRQ entry.
934 * Not an __init, possibly needed by modules
935 */
936static int pin_2_irq(int idx, int apic, int pin);
937
938int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
939{
940 int apic, i, best_guess = -1;
941
Ingo Molnar54168ed2008-08-20 09:07:45 +0200942 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
943 bus, slot, pin);
Alexey Starikovskiyce6444d2008-05-19 19:47:09 +0400944 if (test_bit(bus, mp_bus_not_pci)) {
Ingo Molnar54168ed2008-08-20 09:07:45 +0200945 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 return -1;
947 }
948 for (i = 0; i < mp_irq_entries; i++) {
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400949 int lbus = mp_irqs[i].mp_srcbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950
951 for (apic = 0; apic < nr_ioapics; apic++)
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400952 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
953 mp_irqs[i].mp_dstapic == MP_APIC_ALL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 break;
955
Alexey Starikovskiy47cab822008-03-20 14:54:30 +0300956 if (!test_bit(lbus, mp_bus_not_pci) &&
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400957 !mp_irqs[i].mp_irqtype &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 (bus == lbus) &&
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400959 (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
Ingo Molnar54168ed2008-08-20 09:07:45 +0200960 int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961
962 if (!(apic || IO_APIC_IRQ(irq)))
963 continue;
964
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +0400965 if (pin == (mp_irqs[i].mp_srcbusirq & 3))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966 return irq;
967 /*
968 * Use the first all-but-pin matching entry as a
969 * best-guess fuzzy result for broken mptables.
970 */
971 if (best_guess < 0)
972 best_guess = irq;
973 }
974 }
975 return best_guess;
976}
Ingo Molnar54168ed2008-08-20 09:07:45 +0200977
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700978EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979
Alexey Starikovskiyc0a282c2008-03-20 14:55:02 +0300980#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981/*
982 * EISA Edge/Level control register, ELCR
983 */
984static int EISA_ELCR(unsigned int irq)
985{
986 if (irq < 16) {
987 unsigned int port = 0x4d0 + (irq >> 3);
988 return (inb(port) >> (irq & 7)) & 1;
989 }
990 apic_printk(APIC_VERBOSE, KERN_INFO
991 "Broken MPtable reports ISA irq %d\n", irq);
992 return 0;
993}
Ingo Molnar54168ed2008-08-20 09:07:45 +0200994
Alexey Starikovskiyc0a282c2008-03-20 14:55:02 +0300995#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996
Alexey Starikovskiy67288012008-03-20 14:54:36 +0300997/* ISA interrupts are always polarity zero edge triggered,
998 * when listed as conforming in the MP table. */
999
1000#define default_ISA_trigger(idx) (0)
1001#define default_ISA_polarity(idx) (0)
1002
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003/* EISA interrupts are always polarity zero and can be edge or level
1004 * trigger depending on the ELCR value. If an interrupt is listed as
1005 * EISA conforming in the MP table, that means its trigger type must
1006 * be read in from the ELCR */
1007
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04001008#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
Alexey Starikovskiy67288012008-03-20 14:54:36 +03001009#define default_EISA_polarity(idx) default_ISA_polarity(idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
1011/* PCI interrupts are always polarity one level triggered,
1012 * when listed as conforming in the MP table. */
1013
1014#define default_PCI_trigger(idx) (1)
1015#define default_PCI_polarity(idx) (1)
1016
1017/* MCA interrupts are always polarity zero level triggered,
1018 * when listed as conforming in the MP table. */
1019
1020#define default_MCA_trigger(idx) (1)
Alexey Starikovskiy67288012008-03-20 14:54:36 +03001021#define default_MCA_polarity(idx) default_ISA_polarity(idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022
Shaohua Li61fd47e2007-11-17 01:05:28 -05001023static int MPBIOS_polarity(int idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024{
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04001025 int bus = mp_irqs[idx].mp_srcbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 int polarity;
1027
1028 /*
1029 * Determine IRQ line polarity (high active or low active):
1030 */
Ingo Molnar54168ed2008-08-20 09:07:45 +02001031 switch (mp_irqs[idx].mp_irqflag & 3)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 {
Ingo Molnar54168ed2008-08-20 09:07:45 +02001033 case 0: /* conforms, ie. bus-type dependent polarity */
1034 if (test_bit(bus, mp_bus_not_pci))
1035 polarity = default_ISA_polarity(idx);
1036 else
1037 polarity = default_PCI_polarity(idx);
1038 break;
1039 case 1: /* high active */
1040 {
1041 polarity = 0;
1042 break;
1043 }
1044 case 2: /* reserved */
1045 {
1046 printk(KERN_WARNING "broken BIOS!!\n");
1047 polarity = 1;
1048 break;
1049 }
1050 case 3: /* low active */
1051 {
1052 polarity = 1;
1053 break;
1054 }
1055 default: /* invalid */
1056 {
1057 printk(KERN_WARNING "broken BIOS!!\n");
1058 polarity = 1;
1059 break;
1060 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061 }
1062 return polarity;
1063}
1064
1065static int MPBIOS_trigger(int idx)
1066{
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04001067 int bus = mp_irqs[idx].mp_srcbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068 int trigger;
1069
1070 /*
1071 * Determine IRQ trigger mode (edge or level sensitive):
1072 */
Ingo Molnar54168ed2008-08-20 09:07:45 +02001073 switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 {
Ingo Molnar54168ed2008-08-20 09:07:45 +02001075 case 0: /* conforms, ie. bus-type dependent */
1076 if (test_bit(bus, mp_bus_not_pci))
1077 trigger = default_ISA_trigger(idx);
1078 else
1079 trigger = default_PCI_trigger(idx);
Alexey Starikovskiyc0a282c2008-03-20 14:55:02 +03001080#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
Ingo Molnar54168ed2008-08-20 09:07:45 +02001081 switch (mp_bus_id_to_type[bus]) {
1082 case MP_BUS_ISA: /* ISA pin */
1083 {
1084 /* set before the switch */
1085 break;
1086 }
1087 case MP_BUS_EISA: /* EISA pin */
1088 {
1089 trigger = default_EISA_trigger(idx);
1090 break;
1091 }
1092 case MP_BUS_PCI: /* PCI pin */
1093 {
1094 /* set before the switch */
1095 break;
1096 }
1097 case MP_BUS_MCA: /* MCA pin */
1098 {
1099 trigger = default_MCA_trigger(idx);
1100 break;
1101 }
1102 default:
1103 {
1104 printk(KERN_WARNING "broken BIOS!!\n");
1105 trigger = 1;
1106 break;
1107 }
1108 }
1109#endif
1110 break;
1111 case 1: /* edge */
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02001112 {
Ingo Molnar54168ed2008-08-20 09:07:45 +02001113 trigger = 0;
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02001114 break;
1115 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02001116 case 2: /* reserved */
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02001117 {
1118 printk(KERN_WARNING "broken BIOS!!\n");
1119 trigger = 1;
1120 break;
1121 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02001122 case 3: /* level */
1123 {
1124 trigger = 1;
1125 break;
1126 }
1127 default: /* invalid */
1128 {
1129 printk(KERN_WARNING "broken BIOS!!\n");
1130 trigger = 0;
1131 break;
1132 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133 }
1134 return trigger;
1135}
1136
1137static inline int irq_polarity(int idx)
1138{
1139 return MPBIOS_polarity(idx);
1140}
1141
1142static inline int irq_trigger(int idx)
1143{
1144 return MPBIOS_trigger(idx);
1145}
1146
Yinghai Luefa25592008-08-19 20:50:36 -07001147int (*ioapic_renumber_irq)(int ioapic, int irq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148static int pin_2_irq(int idx, int apic, int pin)
1149{
1150 int irq, i;
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04001151 int bus = mp_irqs[idx].mp_srcbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152
1153 /*
1154 * Debugging check, we are in big trouble if this message pops up!
1155 */
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04001156 if (mp_irqs[idx].mp_dstirq != pin)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
1158
Ingo Molnar54168ed2008-08-20 09:07:45 +02001159 if (test_bit(bus, mp_bus_not_pci)) {
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04001160 irq = mp_irqs[idx].mp_srcbusirq;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001161 } else {
Alexey Starikovskiy643befe2008-03-20 14:54:49 +03001162 /*
1163 * PCI IRQs are mapped in order
1164 */
1165 i = irq = 0;
1166 while (i < apic)
1167 irq += nr_ioapic_registers[i++];
1168 irq += pin;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001169 /*
1170 * For MPS mode, so far only needed by ES7000 platform
1171 */
1172 if (ioapic_renumber_irq)
1173 irq = ioapic_renumber_irq(apic, irq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 }
1175
Ingo Molnar54168ed2008-08-20 09:07:45 +02001176#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 /*
1178 * PCI IRQ command line redirection. Yes, limits are hardcoded.
1179 */
1180 if ((pin >= 16) && (pin <= 23)) {
1181 if (pirq_entries[pin-16] != -1) {
1182 if (!pirq_entries[pin-16]) {
1183 apic_printk(APIC_VERBOSE, KERN_DEBUG
1184 "disabling PIRQ%d\n", pin-16);
1185 } else {
1186 irq = pirq_entries[pin-16];
1187 apic_printk(APIC_VERBOSE, KERN_DEBUG
1188 "using PIRQ%d -> IRQ %d\n",
1189 pin-16, irq);
1190 }
1191 }
1192 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02001193#endif
1194
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 return irq;
1196}
1197
Yinghai Lu497c9a12008-08-19 20:50:28 -07001198void lock_vector_lock(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199{
Yinghai Lu497c9a12008-08-19 20:50:28 -07001200 /* Used to the online set of cpus does not change
1201 * during assign_irq_vector.
1202 */
1203 spin_lock(&vector_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204}
1205
Yinghai Lu497c9a12008-08-19 20:50:28 -07001206void unlock_vector_lock(void)
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07001207{
Yinghai Lu497c9a12008-08-19 20:50:28 -07001208 spin_unlock(&vector_lock);
1209}
1210
1211static int __assign_irq_vector(int irq, cpumask_t mask)
1212{
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001213 /*
1214 * NOTE! The local APIC isn't very good at handling
1215 * multiple interrupts at the same interrupt level.
1216 * As the interrupt level is determined by taking the
1217 * vector number and shifting that right by 4, we
1218 * want to spread these out a bit so that they don't
1219 * all fall in the same interrupt level.
1220 *
1221 * Also, we've got to be careful not to trash gate
1222 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1223 */
Ingo Molnar54168ed2008-08-20 09:07:45 +02001224 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
1225 unsigned int old_vector;
1226 int cpu;
1227 struct irq_cfg *cfg;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001228
Ingo Molnar54168ed2008-08-20 09:07:45 +02001229 cfg = irq_cfg(irq);
Yinghai Lu497c9a12008-08-19 20:50:28 -07001230
Ingo Molnar54168ed2008-08-20 09:07:45 +02001231 /* Only try and allocate irqs on cpus that are present */
1232 cpus_and(mask, mask, cpu_online_map);
Yinghai Lu497c9a12008-08-19 20:50:28 -07001233
Ingo Molnar54168ed2008-08-20 09:07:45 +02001234 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
1235 return -EBUSY;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001236
Ingo Molnar54168ed2008-08-20 09:07:45 +02001237 old_vector = cfg->vector;
1238 if (old_vector) {
1239 cpumask_t tmp;
1240 cpus_and(tmp, cfg->domain, mask);
1241 if (!cpus_empty(tmp))
1242 return 0;
1243 }
Yinghai Lu497c9a12008-08-19 20:50:28 -07001244
Ingo Molnar54168ed2008-08-20 09:07:45 +02001245 for_each_cpu_mask_nr(cpu, mask) {
1246 cpumask_t domain, new_mask;
1247 int new_cpu;
1248 int vector, offset;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001249
Ingo Molnar54168ed2008-08-20 09:07:45 +02001250 domain = vector_allocation_domain(cpu);
1251 cpus_and(new_mask, domain, cpu_online_map);
Yinghai Lu497c9a12008-08-19 20:50:28 -07001252
Ingo Molnar54168ed2008-08-20 09:07:45 +02001253 vector = current_vector;
1254 offset = current_offset;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001255next:
Ingo Molnar54168ed2008-08-20 09:07:45 +02001256 vector += 8;
1257 if (vector >= first_system_vector) {
1258 /* If we run out of vectors on large boxen, must share them. */
1259 offset = (offset + 1) % 8;
1260 vector = FIRST_DEVICE_VECTOR + offset;
Yinghai Lu7a959cf2008-08-19 20:50:32 -07001261 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02001262 if (unlikely(current_vector == vector))
1263 continue;
1264#ifdef CONFIG_X86_64
1265 if (vector == IA32_SYSCALL_VECTOR)
1266 goto next;
1267#else
1268 if (vector == SYSCALL_VECTOR)
1269 goto next;
1270#endif
1271 for_each_cpu_mask_nr(new_cpu, new_mask)
1272 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1273 goto next;
1274 /* Found one! */
1275 current_vector = vector;
1276 current_offset = offset;
1277 if (old_vector) {
1278 cfg->move_in_progress = 1;
1279 cfg->old_domain = cfg->domain;
1280 }
1281 for_each_cpu_mask_nr(new_cpu, new_mask)
1282 per_cpu(vector_irq, new_cpu)[vector] = irq;
1283 cfg->vector = vector;
1284 cfg->domain = domain;
1285 return 0;
1286 }
1287 return -ENOSPC;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001288}
1289
1290static int assign_irq_vector(int irq, cpumask_t mask)
1291{
1292 int err;
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07001293 unsigned long flags;
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07001294
1295 spin_lock_irqsave(&vector_lock, flags);
Yinghai Lu497c9a12008-08-19 20:50:28 -07001296 err = __assign_irq_vector(irq, mask);
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07001297 spin_unlock_irqrestore(&vector_lock, flags);
Yinghai Lu497c9a12008-08-19 20:50:28 -07001298 return err;
1299}
1300
1301static void __clear_irq_vector(int irq)
1302{
1303 struct irq_cfg *cfg;
1304 cpumask_t mask;
1305 int cpu, vector;
1306
1307 cfg = irq_cfg(irq);
1308 BUG_ON(!cfg->vector);
1309
1310 vector = cfg->vector;
1311 cpus_and(mask, cfg->domain, cpu_online_map);
1312 for_each_cpu_mask_nr(cpu, mask)
1313 per_cpu(vector_irq, cpu)[vector] = -1;
1314
1315 cfg->vector = 0;
1316 cpus_clear(cfg->domain);
1317}
1318
1319void __setup_vector_irq(int cpu)
1320{
1321 /* Initialize vector_irq on a new cpu */
1322 /* This function must be called with vector_lock held */
1323 int irq, vector;
1324 struct irq_cfg *cfg;
1325
1326 /* Mark the inuse vectors */
Yinghai Lu8f09cd22008-08-19 20:50:51 -07001327 for_each_irq_cfg(irq, cfg) {
Yinghai Lu497c9a12008-08-19 20:50:28 -07001328 if (!cpu_isset(cpu, cfg->domain))
1329 continue;
1330 vector = cfg->vector;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001331 per_cpu(vector_irq, cpu)[vector] = irq;
1332 }
1333 /* Mark the free vectors */
1334 for (vector = 0; vector < NR_VECTORS; ++vector) {
1335 irq = per_cpu(vector_irq, cpu)[vector];
1336 if (irq < 0)
1337 continue;
1338
1339 cfg = irq_cfg(irq);
1340 if (!cpu_isset(cpu, cfg->domain))
1341 per_cpu(vector_irq, cpu)[vector] = -1;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001342 }
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07001343}
Glauber Costa3fde6902008-05-28 20:34:19 -07001344
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07001345static struct irq_chip ioapic_chip;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001346#ifdef CONFIG_INTR_REMAP
1347static struct irq_chip ir_ioapic_chip;
1348#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349
Ingo Molnar54168ed2008-08-20 09:07:45 +02001350#define IOAPIC_AUTO -1
1351#define IOAPIC_EDGE 0
1352#define IOAPIC_LEVEL 1
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001354#ifdef CONFIG_X86_32
Yinghai Lu1d025192008-08-19 20:50:34 -07001355static inline int IO_APIC_irq_trigger(int irq)
1356{
Ingo Molnar54168ed2008-08-20 09:07:45 +02001357 int apic, idx, pin;
Yinghai Lu1d025192008-08-19 20:50:34 -07001358
Ingo Molnar54168ed2008-08-20 09:07:45 +02001359 for (apic = 0; apic < nr_ioapics; apic++) {
1360 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1361 idx = find_irq_entry(apic, pin, mp_INT);
1362 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1363 return irq_trigger(idx);
1364 }
1365 }
1366 /*
1367 * nonexistent IRQs are edge default
1368 */
1369 return 0;
Yinghai Lu1d025192008-08-19 20:50:34 -07001370}
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001371#else
1372static inline int IO_APIC_irq_trigger(int irq)
1373{
Ingo Molnar54168ed2008-08-20 09:07:45 +02001374 return 1;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001375}
1376#endif
Yinghai Lu1d025192008-08-19 20:50:34 -07001377
Yinghai Lu497c9a12008-08-19 20:50:28 -07001378static void ioapic_register_intr(int irq, unsigned long trigger)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379{
Yinghai Lu08678b02008-08-19 20:50:05 -07001380 struct irq_desc *desc;
1381
Yinghai Lu199751d2008-08-19 20:50:27 -07001382 /* first time to use this irq_desc */
1383 if (irq < 16)
1384 desc = irq_to_desc(irq);
1385 else
1386 desc = irq_to_desc_alloc(irq);
1387
Jan Beulich6ebcc002006-06-26 13:56:46 +02001388 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001389 trigger == IOAPIC_LEVEL)
Yinghai Lu08678b02008-08-19 20:50:05 -07001390 desc->status |= IRQ_LEVEL;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001391 else
1392 desc->status &= ~IRQ_LEVEL;
1393
Ingo Molnar54168ed2008-08-20 09:07:45 +02001394#ifdef CONFIG_INTR_REMAP
1395 if (irq_remapped(irq)) {
1396 desc->status |= IRQ_MOVE_PCNTXT;
1397 if (trigger)
1398 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1399 handle_fasteoi_irq,
1400 "fasteoi");
1401 else
1402 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1403 handle_edge_irq, "edge");
1404 return;
1405 }
1406#endif
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001407 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1408 trigger == IOAPIC_LEVEL)
Ingo Molnara460e742006-10-17 00:10:03 -07001409 set_irq_chip_and_handler_name(irq, &ioapic_chip,
Ingo Molnar54168ed2008-08-20 09:07:45 +02001410 handle_fasteoi_irq,
1411 "fasteoi");
Yinghai Lu047c8fd2008-08-19 20:50:41 -07001412 else
Ingo Molnara460e742006-10-17 00:10:03 -07001413 set_irq_chip_and_handler_name(irq, &ioapic_chip,
Ingo Molnar54168ed2008-08-20 09:07:45 +02001414 handle_edge_irq, "edge");
Yinghai Lu497c9a12008-08-19 20:50:28 -07001415}
1416
1417static int setup_ioapic_entry(int apic, int irq,
1418 struct IO_APIC_route_entry *entry,
1419 unsigned int destination, int trigger,
1420 int polarity, int vector)
1421{
1422 /*
1423 * add it to the IO-APIC irq-routing table:
1424 */
1425 memset(entry,0,sizeof(*entry));
1426
Ingo Molnar54168ed2008-08-20 09:07:45 +02001427#ifdef CONFIG_INTR_REMAP
1428 if (intr_remapping_enabled) {
1429 struct intel_iommu *iommu = map_ioapic_to_ir(apic);
1430 struct irte irte;
1431 struct IR_IO_APIC_route_entry *ir_entry =
1432 (struct IR_IO_APIC_route_entry *) entry;
1433 int index;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001434
Ingo Molnar54168ed2008-08-20 09:07:45 +02001435 if (!iommu)
1436 panic("No mapping iommu for ioapic %d\n", apic);
1437
1438 index = alloc_irte(iommu, irq, 1);
1439 if (index < 0)
1440 panic("Failed to allocate IRTE for ioapic %d\n", apic);
1441
1442 memset(&irte, 0, sizeof(irte));
1443
1444 irte.present = 1;
1445 irte.dst_mode = INT_DEST_MODE;
1446 irte.trigger_mode = trigger;
1447 irte.dlvry_mode = INT_DELIVERY_MODE;
1448 irte.vector = vector;
1449 irte.dest_id = IRTE_DEST(destination);
1450
1451 modify_irte(irq, &irte);
1452
1453 ir_entry->index2 = (index >> 15) & 0x1;
1454 ir_entry->zero = 0;
1455 ir_entry->format = 1;
1456 ir_entry->index = (index & 0x7fff);
1457 } else
1458#endif
1459 {
1460 entry->delivery_mode = INT_DELIVERY_MODE;
1461 entry->dest_mode = INT_DEST_MODE;
1462 entry->dest = destination;
1463 }
1464
1465 entry->mask = 0; /* enable IRQ */
Yinghai Lu497c9a12008-08-19 20:50:28 -07001466 entry->trigger = trigger;
1467 entry->polarity = polarity;
1468 entry->vector = vector;
1469
1470 /* Mask level triggered irqs.
1471 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1472 */
1473 if (trigger)
1474 entry->mask = 1;
Yinghai Lu497c9a12008-08-19 20:50:28 -07001475 return 0;
1476}
1477
1478static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
Ingo Molnar54168ed2008-08-20 09:07:45 +02001479 int trigger, int polarity)
Yinghai Lu497c9a12008-08-19 20:50:28 -07001480{
1481 struct irq_cfg *cfg;
1482 struct IO_APIC_route_entry entry;
1483 cpumask_t mask;
1484
1485 if (!IO_APIC_IRQ(irq))
1486 return;
1487
1488 cfg = irq_cfg(irq);
1489
1490 mask = TARGET_CPUS;
1491 if (assign_irq_vector(irq, mask))
1492 return;
1493
1494 cpus_and(mask, cfg->domain, mask);
1495
1496 apic_printk(APIC_VERBOSE,KERN_DEBUG
1497 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1498 "IRQ %d Mode:%i Active:%i)\n",
1499 apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
1500 irq, trigger, polarity);
1501
1502
1503 if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
1504 cpu_mask_to_apicid(mask), trigger, polarity,
1505 cfg->vector)) {
1506 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1507 mp_ioapics[apic].mp_apicid, pin);
1508 __clear_irq_vector(irq);
1509 return;
1510 }
1511
1512 ioapic_register_intr(irq, trigger);
1513 if (irq < 16)
1514 disable_8259A_irq(irq);
1515
1516 ioapic_write_entry(apic, pin, entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517}
1518
1519static void __init setup_IO_APIC_irqs(void)
1520{
Yinghai Lu497c9a12008-08-19 20:50:28 -07001521 int apic, pin, idx, irq, first_notcon = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
1523 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1524
1525 for (apic = 0; apic < nr_ioapics; apic++) {
1526 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1527
Yinghai Lu497c9a12008-08-19 20:50:28 -07001528 idx = find_irq_entry(apic,pin,mp_INT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 if (idx == -1) {
1530 if (first_notcon) {
Yinghai Lu497c9a12008-08-19 20:50:28 -07001531 apic_printk(APIC_VERBOSE, KERN_DEBUG " IO-APIC (apicid-pin) %d-%d", mp_ioapics[apic].mp_apicid, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532 first_notcon = 0;
1533 } else
Yinghai Lu497c9a12008-08-19 20:50:28 -07001534 apic_printk(APIC_VERBOSE, ", %d-%d", mp_ioapics[apic].mp_apicid, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535 continue;
1536 }
Yinghai Lu20d225b2007-10-17 18:04:41 +02001537 if (!first_notcon) {
1538 apic_printk(APIC_VERBOSE, " not connected.\n");
1539 first_notcon = 1;
1540 }
1541
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542 irq = pin_2_irq(idx, apic, pin);
Ingo Molnar54168ed2008-08-20 09:07:45 +02001543#ifdef CONFIG_X86_32
Yinghai Lu497c9a12008-08-19 20:50:28 -07001544 if (multi_timer_check(apic, irq))
1545 continue;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001546#endif
Yinghai Lu497c9a12008-08-19 20:50:28 -07001547 add_pin_to_irq(irq, apic, pin);
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02001548
Yinghai Lu497c9a12008-08-19 20:50:28 -07001549 setup_IO_APIC_irq(apic, pin, irq,
1550 irq_trigger(idx), irq_polarity(idx));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 }
1552 }
1553
1554 if (!first_notcon)
1555 apic_printk(APIC_VERBOSE, " not connected.\n");
1556}
1557
1558/*
Maciej W. Rozyckif7633ce2008-05-27 21:19:34 +01001559 * Set up the timer pin, possibly with the 8259A-master behind.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560 */
Maciej W. Rozyckif7633ce2008-05-27 21:19:34 +01001561static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
1562 int vector)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563{
1564 struct IO_APIC_route_entry entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565
Ingo Molnar54168ed2008-08-20 09:07:45 +02001566#ifdef CONFIG_INTR_REMAP
1567 if (intr_remapping_enabled)
1568 return;
1569#endif
1570
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02001571 memset(&entry, 0, sizeof(entry));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572
1573 /*
1574 * We use logical delivery to get the timer IRQ
1575 * to the first CPU.
1576 */
1577 entry.dest_mode = INT_DEST_MODE;
Maciej W. Rozycki03be7502008-05-27 21:19:45 +01001578 entry.mask = 1; /* mask IRQ now */
Yinghai Lud83e94a2008-08-19 20:50:33 -07001579 entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580 entry.delivery_mode = INT_DELIVERY_MODE;
1581 entry.polarity = 0;
1582 entry.trigger = 0;
1583 entry.vector = vector;
1584
1585 /*
1586 * The timer IRQ doesn't have to know that behind the
Maciej W. Rozyckif7633ce2008-05-27 21:19:34 +01001587 * scene we may have a 8259A-master in AEOI mode ...
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588 */
Ingo Molnar54168ed2008-08-20 09:07:45 +02001589 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590
1591 /*
1592 * Add it to the IO-APIC irq-routing table:
1593 */
Andi Kleencf4c6a22006-09-26 10:52:30 +02001594 ioapic_write_entry(apic, pin, entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595}
1596
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +01001597
1598__apicdebuginit(void) print_IO_APIC(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599{
1600 int apic, i;
1601 union IO_APIC_reg_00 reg_00;
1602 union IO_APIC_reg_01 reg_01;
1603 union IO_APIC_reg_02 reg_02;
1604 union IO_APIC_reg_03 reg_03;
1605 unsigned long flags;
Yinghai Lu0f978f42008-08-19 20:50:26 -07001606 struct irq_cfg *cfg;
Yinghai Lu8f09cd22008-08-19 20:50:51 -07001607 unsigned int irq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608
1609 if (apic_verbosity == APIC_QUIET)
1610 return;
1611
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02001612 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613 for (i = 0; i < nr_ioapics; i++)
1614 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04001615 mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616
1617 /*
1618 * We are a bit conservative about what we expect. We have to
1619 * know about every hardware change ASAP.
1620 */
1621 printk(KERN_INFO "testing the IO APIC.......................\n");
1622
1623 for (apic = 0; apic < nr_ioapics; apic++) {
1624
1625 spin_lock_irqsave(&ioapic_lock, flags);
1626 reg_00.raw = io_apic_read(apic, 0);
1627 reg_01.raw = io_apic_read(apic, 1);
1628 if (reg_01.bits.version >= 0x10)
1629 reg_02.raw = io_apic_read(apic, 2);
Ingo Molnar54168ed2008-08-20 09:07:45 +02001630 if (reg_01.bits.version >= 0x20)
1631 reg_03.raw = io_apic_read(apic, 3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632 spin_unlock_irqrestore(&ioapic_lock, flags);
1633
Ingo Molnar54168ed2008-08-20 09:07:45 +02001634 printk("\n");
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04001635 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1637 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1638 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1639 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640
Ingo Molnar54168ed2008-08-20 09:07:45 +02001641 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643
1644 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1645 printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646
1647 /*
1648 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1649 * but the value of reg_02 is read as the previous read register
1650 * value, so ignore it if reg_02 == reg_01.
1651 */
1652 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1653 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1654 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655 }
1656
1657 /*
1658 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1659 * or reg_03, but the value of reg_0[23] is read as the previous read
1660 * register value, so ignore it if reg_03 == reg_0[12].
1661 */
1662 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1663 reg_03.raw != reg_01.raw) {
1664 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1665 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666 }
1667
1668 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1669
Yinghai Lud83e94a2008-08-19 20:50:33 -07001670 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1671 " Stat Dmod Deli Vect: \n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672
1673 for (i = 0; i <= reg_01.bits.entries; i++) {
1674 struct IO_APIC_route_entry entry;
1675
Andi Kleencf4c6a22006-09-26 10:52:30 +02001676 entry = ioapic_read_entry(apic, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677
Ingo Molnar54168ed2008-08-20 09:07:45 +02001678 printk(KERN_DEBUG " %02x %03X ",
1679 i,
1680 entry.dest
1681 );
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682
1683 printk("%1d %1d %1d %1d %1d %1d %1d %02X\n",
1684 entry.mask,
1685 entry.trigger,
1686 entry.irr,
1687 entry.polarity,
1688 entry.delivery_status,
1689 entry.dest_mode,
1690 entry.delivery_mode,
1691 entry.vector
1692 );
1693 }
1694 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 printk(KERN_DEBUG "IRQ to pin mappings:\n");
Yinghai Lu8f09cd22008-08-19 20:50:51 -07001696 for_each_irq_cfg(irq, cfg) {
Yinghai Lu0f978f42008-08-19 20:50:26 -07001697 struct irq_pin_list *entry = cfg->irq_2_pin;
1698 if (!entry)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699 continue;
Yinghai Lu8f09cd22008-08-19 20:50:51 -07001700 printk(KERN_DEBUG "IRQ%d ", irq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701 for (;;) {
1702 printk("-> %d:%d", entry->apic, entry->pin);
1703 if (!entry->next)
1704 break;
Yinghai Lu0f978f42008-08-19 20:50:26 -07001705 entry = entry->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001706 }
1707 printk("\n");
1708 }
1709
1710 printk(KERN_INFO ".................................... done.\n");
1711
1712 return;
1713}
1714
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +01001715__apicdebuginit(void) print_APIC_bitfield(int base)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716{
1717 unsigned int v;
1718 int i, j;
1719
1720 if (apic_verbosity == APIC_QUIET)
1721 return;
1722
1723 printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
1724 for (i = 0; i < 8; i++) {
1725 v = apic_read(base + i*0x10);
1726 for (j = 0; j < 32; j++) {
1727 if (v & (1<<j))
1728 printk("1");
1729 else
1730 printk("0");
1731 }
1732 printk("\n");
1733 }
1734}
1735
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +01001736__apicdebuginit(void) print_local_APIC(void *dummy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737{
1738 unsigned int v, ver, maxlvt;
Hiroshi Shimamoto7ab6af72008-07-30 17:36:48 -07001739 u64 icr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740
1741 if (apic_verbosity == APIC_QUIET)
1742 return;
1743
1744 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1745 smp_processor_id(), hard_smp_processor_id());
Andreas Herrmann66823112008-06-05 16:35:10 +02001746 v = apic_read(APIC_ID);
Ingo Molnar54168ed2008-08-20 09:07:45 +02001747 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001748 v = apic_read(APIC_LVR);
1749 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1750 ver = GET_APIC_VERSION(v);
Thomas Gleixnere05d7232007-02-16 01:27:58 -08001751 maxlvt = lapic_get_maxlvt();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752
1753 v = apic_read(APIC_TASKPRI);
1754 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1755
Ingo Molnar54168ed2008-08-20 09:07:45 +02001756 if (APIC_INTEGRATED(ver)) { /* !82489DX */
Yinghai Lua11b5ab2008-09-03 16:58:31 -07001757 if (!APIC_XAPIC(ver)) {
1758 v = apic_read(APIC_ARBPRI);
1759 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1760 v & APIC_ARBPRI_MASK);
1761 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762 v = apic_read(APIC_PROCPRI);
1763 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1764 }
1765
Yinghai Lua11b5ab2008-09-03 16:58:31 -07001766 /*
1767 * Remote read supported only in the 82489DX and local APIC for
1768 * Pentium processors.
1769 */
1770 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1771 v = apic_read(APIC_RRR);
1772 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1773 }
1774
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775 v = apic_read(APIC_LDR);
1776 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
Yinghai Lua11b5ab2008-09-03 16:58:31 -07001777 if (!x2apic_enabled()) {
1778 v = apic_read(APIC_DFR);
1779 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1780 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001781 v = apic_read(APIC_SPIV);
1782 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1783
1784 printk(KERN_DEBUG "... APIC ISR field:\n");
1785 print_APIC_bitfield(APIC_ISR);
1786 printk(KERN_DEBUG "... APIC TMR field:\n");
1787 print_APIC_bitfield(APIC_TMR);
1788 printk(KERN_DEBUG "... APIC IRR field:\n");
1789 print_APIC_bitfield(APIC_IRR);
1790
Ingo Molnar54168ed2008-08-20 09:07:45 +02001791 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1792 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793 apic_write(APIC_ESR, 0);
Ingo Molnar54168ed2008-08-20 09:07:45 +02001794
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795 v = apic_read(APIC_ESR);
1796 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1797 }
1798
Hiroshi Shimamoto7ab6af72008-07-30 17:36:48 -07001799 icr = apic_icr_read();
Ingo Molnar0c425ce2008-08-18 13:04:26 +02001800 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1801 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802
1803 v = apic_read(APIC_LVTT);
1804 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1805
1806 if (maxlvt > 3) { /* PC is LVT#4. */
1807 v = apic_read(APIC_LVTPC);
1808 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1809 }
1810 v = apic_read(APIC_LVT0);
1811 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1812 v = apic_read(APIC_LVT1);
1813 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1814
1815 if (maxlvt > 2) { /* ERR is LVT#3. */
1816 v = apic_read(APIC_LVTERR);
1817 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1818 }
1819
1820 v = apic_read(APIC_TMICT);
1821 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1822 v = apic_read(APIC_TMCCT);
1823 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1824 v = apic_read(APIC_TDCR);
1825 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1826 printk("\n");
1827}
1828
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +01001829__apicdebuginit(void) print_all_local_APICs(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830{
Yinghai Luffd5aae2008-08-19 20:50:50 -07001831 int cpu;
1832
1833 preempt_disable();
1834 for_each_online_cpu(cpu)
1835 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1836 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001837}
1838
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +01001839__apicdebuginit(void) print_PIC(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841 unsigned int v;
1842 unsigned long flags;
1843
1844 if (apic_verbosity == APIC_QUIET)
1845 return;
1846
1847 printk(KERN_DEBUG "\nprinting PIC contents\n");
1848
1849 spin_lock_irqsave(&i8259A_lock, flags);
1850
1851 v = inb(0xa1) << 8 | inb(0x21);
1852 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1853
1854 v = inb(0xa0) << 8 | inb(0x20);
1855 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1856
Ingo Molnar54168ed2008-08-20 09:07:45 +02001857 outb(0x0b,0xa0);
1858 outb(0x0b,0x20);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 v = inb(0xa0) << 8 | inb(0x20);
Ingo Molnar54168ed2008-08-20 09:07:45 +02001860 outb(0x0a,0xa0);
1861 outb(0x0a,0x20);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001862
1863 spin_unlock_irqrestore(&i8259A_lock, flags);
1864
1865 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1866
1867 v = inb(0x4d1) << 8 | inb(0x4d0);
1868 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1869}
1870
Maciej W. Rozycki32f71af2008-07-21 00:52:49 +01001871__apicdebuginit(int) print_all_ICs(void)
1872{
1873 print_PIC();
1874 print_all_local_APICs();
1875 print_IO_APIC();
1876
1877 return 0;
1878}
1879
1880fs_initcall(print_all_ICs);
1881
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882
Yinghai Luefa25592008-08-19 20:50:36 -07001883/* Where if anywhere is the i8259 connect in external int mode */
1884static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1885
Ingo Molnar54168ed2008-08-20 09:07:45 +02001886void __init enable_IO_APIC(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887{
1888 union IO_APIC_reg_01 reg_01;
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001889 int i8259_apic, i8259_pin;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001890 int apic;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891 unsigned long flags;
1892
Ingo Molnar54168ed2008-08-20 09:07:45 +02001893#ifdef CONFIG_X86_32
1894 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895 if (!pirqs_enabled)
1896 for (i = 0; i < MAX_PIRQS; i++)
1897 pirq_entries[i] = -1;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001898#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899
1900 /*
1901 * The number of IO-APIC IRQ registers (== #pins):
1902 */
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001903 for (apic = 0; apic < nr_ioapics; apic++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 spin_lock_irqsave(&ioapic_lock, flags);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001905 reg_01.raw = io_apic_read(apic, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001906 spin_unlock_irqrestore(&ioapic_lock, flags);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001907 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
1908 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02001909 for(apic = 0; apic < nr_ioapics; apic++) {
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001910 int pin;
1911 /* See if any of the pins is in ExtINT mode */
Eric W. Biederman1008fdd2006-01-11 22:46:06 +01001912 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001913 struct IO_APIC_route_entry entry;
Andi Kleencf4c6a22006-09-26 10:52:30 +02001914 entry = ioapic_read_entry(apic, pin);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001915
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001916 /* If the interrupt line is enabled and in ExtInt mode
1917 * I have found the pin where the i8259 is connected.
1918 */
1919 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1920 ioapic_i8259.apic = apic;
1921 ioapic_i8259.pin = pin;
1922 goto found_i8259;
1923 }
1924 }
1925 }
1926 found_i8259:
1927 /* Look to see what if the MP table has reported the ExtINT */
1928 /* If we could not find the appropriate pin by looking at the ioapic
1929 * the i8259 probably is not connected the ioapic but give the
1930 * mptable a chance anyway.
1931 */
1932 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1933 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1934 /* Trust the MP table if nothing is setup in the hardware */
1935 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1936 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1937 ioapic_i8259.pin = i8259_pin;
1938 ioapic_i8259.apic = i8259_apic;
1939 }
1940 /* Complain if the MP table and the hardware disagree */
1941 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1942 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1943 {
1944 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945 }
1946
1947 /*
1948 * Do not trust the IO-APIC being empty at bootup
1949 */
1950 clear_IO_APIC();
1951}
1952
1953/*
1954 * Not an __init, needed by the reboot code
1955 */
1956void disable_IO_APIC(void)
1957{
1958 /*
1959 * Clear the IO-APIC before rebooting:
1960 */
1961 clear_IO_APIC();
1962
Eric W. Biederman650927e2005-06-25 14:57:44 -07001963 /*
Karsten Wiese0b968d22005-09-09 12:59:04 +02001964 * If the i8259 is routed through an IOAPIC
Eric W. Biederman650927e2005-06-25 14:57:44 -07001965 * Put that IOAPIC in virtual wire mode
Karsten Wiese0b968d22005-09-09 12:59:04 +02001966 * so legacy interrupts can be delivered.
Eric W. Biederman650927e2005-06-25 14:57:44 -07001967 */
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001968 if (ioapic_i8259.pin != -1) {
Eric W. Biederman650927e2005-06-25 14:57:44 -07001969 struct IO_APIC_route_entry entry;
Eric W. Biederman650927e2005-06-25 14:57:44 -07001970
1971 memset(&entry, 0, sizeof(entry));
1972 entry.mask = 0; /* Enabled */
1973 entry.trigger = 0; /* Edge */
1974 entry.irr = 0;
1975 entry.polarity = 0; /* High */
1976 entry.delivery_status = 0;
1977 entry.dest_mode = 0; /* Physical */
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001978 entry.delivery_mode = dest_ExtINT; /* ExtInt */
Eric W. Biederman650927e2005-06-25 14:57:44 -07001979 entry.vector = 0;
Ingo Molnar54168ed2008-08-20 09:07:45 +02001980 entry.dest = read_apic_id();
Eric W. Biederman650927e2005-06-25 14:57:44 -07001981
1982 /*
1983 * Add it to the IO-APIC irq-routing table:
1984 */
Andi Kleencf4c6a22006-09-26 10:52:30 +02001985 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
Eric W. Biederman650927e2005-06-25 14:57:44 -07001986 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02001987
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08001988 disconnect_bsp_APIC(ioapic_i8259.pin != -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989}
1990
Ingo Molnar54168ed2008-08-20 09:07:45 +02001991#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -07001992/*
1993 * function to set the IO-APIC physical IDs based on the
1994 * values stored in the MPC table.
1995 *
1996 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
1997 */
1998
Linus Torvalds1da177e2005-04-16 15:20:36 -07001999static void __init setup_ioapic_ids_from_mpc(void)
2000{
2001 union IO_APIC_reg_00 reg_00;
2002 physid_mask_t phys_id_present_map;
2003 int apic;
2004 int i;
2005 unsigned char old_id;
2006 unsigned long flags;
2007
Yinghai Lua4dbc342008-07-25 02:14:28 -07002008 if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
Yinghai Lud49c4282008-06-08 18:31:54 -07002009 return;
Yinghai Lud49c4282008-06-08 18:31:54 -07002010
Linus Torvalds1da177e2005-04-16 15:20:36 -07002011 /*
Natalie Protasevichca05fea2005-06-23 00:08:22 -07002012 * Don't check I/O APIC IDs for xAPIC systems. They have
2013 * no meaning without the serial APIC bus.
2014 */
Shaohua Li7c5c1e42006-03-23 02:59:53 -08002015 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2016 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
Natalie Protasevichca05fea2005-06-23 00:08:22 -07002017 return;
2018 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 * This is broken; anything with a real cpu count has to
2020 * circumvent this idiocy regardless.
2021 */
2022 phys_id_present_map = ioapic_phys_id_map(phys_cpu_present_map);
2023
2024 /*
2025 * Set the IOAPIC ID to the value stored in the MPC table.
2026 */
2027 for (apic = 0; apic < nr_ioapics; apic++) {
2028
2029 /* Read the register 0 value */
2030 spin_lock_irqsave(&ioapic_lock, flags);
2031 reg_00.raw = io_apic_read(apic, 0);
2032 spin_unlock_irqrestore(&ioapic_lock, flags);
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002033
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002034 old_id = mp_ioapics[apic].mp_apicid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002036 if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002038 apic, mp_ioapics[apic].mp_apicid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2040 reg_00.bits.ID);
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002041 mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042 }
2043
Linus Torvalds1da177e2005-04-16 15:20:36 -07002044 /*
2045 * Sanity check, is the ID really free? Every APIC in a
2046 * system must have a unique ID or we get lots of nice
2047 * 'stuck on smp_invalidate_needed IPI wait' messages.
2048 */
2049 if (check_apicid_used(phys_id_present_map,
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002050 mp_ioapics[apic].mp_apicid)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002051 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002052 apic, mp_ioapics[apic].mp_apicid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053 for (i = 0; i < get_physical_broadcast(); i++)
2054 if (!physid_isset(i, phys_id_present_map))
2055 break;
2056 if (i >= get_physical_broadcast())
2057 panic("Max APIC ID exceeded!\n");
2058 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2059 i);
2060 physid_set(i, phys_id_present_map);
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002061 mp_ioapics[apic].mp_apicid = i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002062 } else {
2063 physid_mask_t tmp;
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002064 tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065 apic_printk(APIC_VERBOSE, "Setting %d in the "
2066 "phys_id_present_map\n",
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002067 mp_ioapics[apic].mp_apicid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2069 }
2070
2071
2072 /*
2073 * We need to adjust the IRQ routing table
2074 * if the ID changed.
2075 */
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002076 if (old_id != mp_ioapics[apic].mp_apicid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077 for (i = 0; i < mp_irq_entries; i++)
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04002078 if (mp_irqs[i].mp_dstapic == old_id)
2079 mp_irqs[i].mp_dstapic
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002080 = mp_ioapics[apic].mp_apicid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002081
2082 /*
2083 * Read the right value from the MPC table and
2084 * write it into the ID register.
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002085 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 apic_printk(APIC_VERBOSE, KERN_INFO
2087 "...changing IO-APIC physical APIC ID to %d ...",
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002088 mp_ioapics[apic].mp_apicid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002090 reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 spin_lock_irqsave(&ioapic_lock, flags);
Yinghai Lua2d332f2008-08-21 12:56:32 -07002092 io_apic_write(apic, 0, reg_00.raw);
2093 spin_unlock_irqrestore(&ioapic_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094
2095 /*
2096 * Sanity check
2097 */
2098 spin_lock_irqsave(&ioapic_lock, flags);
2099 reg_00.raw = io_apic_read(apic, 0);
2100 spin_unlock_irqrestore(&ioapic_lock, flags);
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04002101 if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002102 printk("could not set ID!\n");
2103 else
2104 apic_printk(APIC_VERBOSE, " ok.\n");
2105 }
2106}
Ingo Molnar54168ed2008-08-20 09:07:45 +02002107#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108
Zachary Amsden7ce0bcf2007-02-13 13:26:21 +01002109int no_timer_check __initdata;
Zachary Amsden8542b202006-12-07 02:14:09 +01002110
2111static int __init notimercheck(char *s)
2112{
2113 no_timer_check = 1;
2114 return 1;
2115}
2116__setup("no_timer_check", notimercheck);
2117
Linus Torvalds1da177e2005-04-16 15:20:36 -07002118/*
2119 * There is a nasty bug in some older SMP boards, their mptable lies
2120 * about the timer IRQ. We do the following to work around the situation:
2121 *
2122 * - timer IRQ defaults to IO-APIC IRQ
2123 * - if this function detects that timer IRQs are defunct, then we fall
2124 * back to ISA timer IRQs
2125 */
Adrian Bunkf0a7a5c2007-07-21 17:10:29 +02002126static int __init timer_irq_works(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127{
2128 unsigned long t1 = jiffies;
Ingo Molnar4aae0702007-12-18 18:05:58 +01002129 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130
Zachary Amsden8542b202006-12-07 02:14:09 +01002131 if (no_timer_check)
2132 return 1;
2133
Ingo Molnar4aae0702007-12-18 18:05:58 +01002134 local_save_flags(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002135 local_irq_enable();
2136 /* Let ten ticks pass... */
2137 mdelay((10 * 1000) / HZ);
Ingo Molnar4aae0702007-12-18 18:05:58 +01002138 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002139
2140 /*
2141 * Expect a few ticks at least, to be sure some possible
2142 * glue logic does not lock up after one or two first
2143 * ticks in a non-ExtINT mode. Also the local APIC
2144 * might have cached one ExtINT interrupt. Finally, at
2145 * least one tick may be lost due to delays.
2146 */
Ingo Molnar54168ed2008-08-20 09:07:45 +02002147
2148 /* jiffies wrap? */
Julia Lawall1d16b532008-01-30 13:32:19 +01002149 if (time_after(jiffies, t1 + 4))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002150 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002151 return 0;
2152}
2153
2154/*
2155 * In the SMP+IOAPIC case it might happen that there are an unspecified
2156 * number of pending IRQ events unhandled. These cases are very rare,
2157 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
2158 * better to do it this way as thus we do not have to be aware of
2159 * 'pending' interrupts in the IRQ path, except at this point.
2160 */
2161/*
2162 * Edge triggered needs to resend any interrupt
2163 * that was delayed but this is now handled in the device
2164 * independent code.
2165 */
2166
2167/*
2168 * Starting up a edge-triggered IO-APIC interrupt is
2169 * nasty - we need to make sure that we get the edge.
2170 * If it is already asserted for some reason, we need
2171 * return 1 to indicate that is was pending.
2172 *
2173 * This is not complete - we should be able to fake
2174 * an edge even if it isn't on the 8259A...
2175 */
Ingo Molnar54168ed2008-08-20 09:07:45 +02002176
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07002177static unsigned int startup_ioapic_irq(unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002178{
2179 int was_pending = 0;
2180 unsigned long flags;
2181
2182 spin_lock_irqsave(&ioapic_lock, flags);
2183 if (irq < 16) {
2184 disable_8259A_irq(irq);
2185 if (i8259A_irq_pending(irq))
2186 was_pending = 1;
2187 }
2188 __unmask_IO_APIC_irq(irq);
2189 spin_unlock_irqrestore(&ioapic_lock, flags);
2190
2191 return was_pending;
2192}
2193
Ingo Molnar54168ed2008-08-20 09:07:45 +02002194#ifdef CONFIG_X86_64
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07002195static int ioapic_retrigger_irq(unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002196{
Ingo Molnar54168ed2008-08-20 09:07:45 +02002197
2198 struct irq_cfg *cfg = irq_cfg(irq);
2199 unsigned long flags;
2200
2201 spin_lock_irqsave(&vector_lock, flags);
2202 send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
2203 spin_unlock_irqrestore(&vector_lock, flags);
Ingo Molnarc0ad90a2006-06-29 02:24:44 -07002204
2205 return 1;
2206}
Ingo Molnar54168ed2008-08-20 09:07:45 +02002207#else
2208static int ioapic_retrigger_irq(unsigned int irq)
2209{
2210 send_IPI_self(irq_cfg(irq)->vector);
2211
2212 return 1;
2213}
2214#endif
2215
2216/*
2217 * Level and edge triggered IO-APIC interrupts need different handling,
2218 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
2219 * handled with the level-triggered descriptor, but that one has slightly
2220 * more overhead. Level-triggered interrupts cannot be handled with the
2221 * edge-triggered handler, without risking IRQ storms and other ugly
2222 * races.
2223 */
Ingo Molnarc0ad90a2006-06-29 02:24:44 -07002224
Yinghai Lu497c9a12008-08-19 20:50:28 -07002225#ifdef CONFIG_SMP
Ingo Molnar54168ed2008-08-20 09:07:45 +02002226
2227#ifdef CONFIG_INTR_REMAP
2228static void ir_irq_migration(struct work_struct *work);
2229
2230static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
2231
2232/*
2233 * Migrate the IO-APIC irq in the presence of intr-remapping.
2234 *
2235 * For edge triggered, irq migration is a simple atomic update(of vector
2236 * and cpu destination) of IRTE and flush the hardware cache.
2237 *
2238 * For level triggered, we need to modify the io-apic RTE aswell with the update
2239 * vector information, along with modifying IRTE with vector and destination.
2240 * So irq migration for level triggered is little bit more complex compared to
2241 * edge triggered migration. But the good news is, we use the same algorithm
2242 * for level triggered migration as we have today, only difference being,
2243 * we now initiate the irq migration from process context instead of the
2244 * interrupt context.
2245 *
2246 * In future, when we do a directed EOI (combined with cpu EOI broadcast
2247 * suppression) to the IO-APIC, level triggered irq migration will also be
2248 * as simple as edge triggered migration and we can do the irq migration
2249 * with a simple atomic update to IO-APIC RTE.
2250 */
2251static void migrate_ioapic_irq(int irq, cpumask_t mask)
2252{
2253 struct irq_cfg *cfg;
2254 struct irq_desc *desc;
2255 cpumask_t tmp, cleanup_mask;
2256 struct irte irte;
2257 int modify_ioapic_rte;
2258 unsigned int dest;
2259 unsigned long flags;
2260
2261 cpus_and(tmp, mask, cpu_online_map);
2262 if (cpus_empty(tmp))
2263 return;
2264
2265 if (get_irte(irq, &irte))
2266 return;
2267
2268 if (assign_irq_vector(irq, mask))
2269 return;
2270
2271 cfg = irq_cfg(irq);
2272 cpus_and(tmp, cfg->domain, mask);
2273 dest = cpu_mask_to_apicid(tmp);
2274
2275 desc = irq_to_desc(irq);
2276 modify_ioapic_rte = desc->status & IRQ_LEVEL;
2277 if (modify_ioapic_rte) {
2278 spin_lock_irqsave(&ioapic_lock, flags);
2279 __target_IO_APIC_irq(irq, dest, cfg->vector);
2280 spin_unlock_irqrestore(&ioapic_lock, flags);
2281 }
2282
2283 irte.vector = cfg->vector;
2284 irte.dest_id = IRTE_DEST(dest);
2285
2286 /*
2287 * Modified the IRTE and flushes the Interrupt entry cache.
2288 */
2289 modify_irte(irq, &irte);
2290
2291 if (cfg->move_in_progress) {
2292 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2293 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2294 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2295 cfg->move_in_progress = 0;
2296 }
2297
2298 desc->affinity = mask;
2299}
2300
2301static int migrate_irq_remapped_level(int irq)
2302{
2303 int ret = -1;
2304 struct irq_desc *desc = irq_to_desc(irq);
2305
2306 mask_IO_APIC_irq(irq);
2307
2308 if (io_apic_level_ack_pending(irq)) {
2309 /*
2310 * Interrupt in progress. Migrating irq now will change the
2311 * vector information in the IO-APIC RTE and that will confuse
2312 * the EOI broadcast performed by cpu.
2313 * So, delay the irq migration to the next instance.
2314 */
2315 schedule_delayed_work(&ir_migration_work, 1);
2316 goto unmask;
2317 }
2318
2319 /* everthing is clear. we have right of way */
2320 migrate_ioapic_irq(irq, desc->pending_mask);
2321
2322 ret = 0;
2323 desc->status &= ~IRQ_MOVE_PENDING;
2324 cpus_clear(desc->pending_mask);
2325
2326unmask:
2327 unmask_IO_APIC_irq(irq);
2328 return ret;
2329}
2330
2331static void ir_irq_migration(struct work_struct *work)
2332{
2333 unsigned int irq;
2334 struct irq_desc *desc;
2335
2336 for_each_irq_desc(irq, desc) {
2337 if (desc->status & IRQ_MOVE_PENDING) {
2338 unsigned long flags;
2339
2340 spin_lock_irqsave(&desc->lock, flags);
2341 if (!desc->chip->set_affinity ||
2342 !(desc->status & IRQ_MOVE_PENDING)) {
2343 desc->status &= ~IRQ_MOVE_PENDING;
2344 spin_unlock_irqrestore(&desc->lock, flags);
2345 continue;
2346 }
2347
2348 desc->chip->set_affinity(irq, desc->pending_mask);
2349 spin_unlock_irqrestore(&desc->lock, flags);
2350 }
2351 }
2352}
2353
2354/*
2355 * Migrates the IRQ destination in the process context.
2356 */
2357static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
2358{
2359 struct irq_desc *desc = irq_to_desc(irq);
2360
2361 if (desc->status & IRQ_LEVEL) {
2362 desc->status |= IRQ_MOVE_PENDING;
2363 desc->pending_mask = mask;
2364 migrate_irq_remapped_level(irq);
2365 return;
2366 }
2367
2368 migrate_ioapic_irq(irq, mask);
2369}
2370#endif
2371
Yinghai Lu497c9a12008-08-19 20:50:28 -07002372asmlinkage void smp_irq_move_cleanup_interrupt(void)
2373{
2374 unsigned vector, me;
2375 ack_APIC_irq();
Ingo Molnar54168ed2008-08-20 09:07:45 +02002376#ifdef CONFIG_X86_64
2377 exit_idle();
2378#endif
Yinghai Lu497c9a12008-08-19 20:50:28 -07002379 irq_enter();
2380
2381 me = smp_processor_id();
2382 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2383 unsigned int irq;
2384 struct irq_desc *desc;
2385 struct irq_cfg *cfg;
2386 irq = __get_cpu_var(vector_irq)[vector];
2387
2388 desc = irq_to_desc(irq);
2389 if (!desc)
2390 continue;
2391
2392 cfg = irq_cfg(irq);
2393 spin_lock(&desc->lock);
2394 if (!cfg->move_cleanup_count)
2395 goto unlock;
2396
2397 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
2398 goto unlock;
2399
2400 __get_cpu_var(vector_irq)[vector] = -1;
2401 cfg->move_cleanup_count--;
2402unlock:
2403 spin_unlock(&desc->lock);
2404 }
2405
2406 irq_exit();
2407}
2408
2409static void irq_complete_move(unsigned int irq)
2410{
2411 struct irq_cfg *cfg = irq_cfg(irq);
2412 unsigned vector, me;
2413
2414 if (likely(!cfg->move_in_progress))
2415 return;
2416
2417 vector = ~get_irq_regs()->orig_ax;
2418 me = smp_processor_id();
2419 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
2420 cpumask_t cleanup_mask;
2421
2422 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2423 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2424 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2425 cfg->move_in_progress = 0;
2426 }
2427}
2428#else
2429static inline void irq_complete_move(unsigned int irq) {}
2430#endif
Ingo Molnar54168ed2008-08-20 09:07:45 +02002431#ifdef CONFIG_INTR_REMAP
2432static void ack_x2apic_level(unsigned int irq)
2433{
2434 ack_x2APIC_irq();
2435}
2436
2437static void ack_x2apic_edge(unsigned int irq)
2438{
2439 ack_x2APIC_irq();
2440}
2441#endif
Yinghai Lu497c9a12008-08-19 20:50:28 -07002442
Yinghai Lu1d025192008-08-19 20:50:34 -07002443static void ack_apic_edge(unsigned int irq)
2444{
2445 irq_complete_move(irq);
2446 move_native_irq(irq);
2447 ack_APIC_irq();
2448}
2449
Yinghai Lu3eb2cce2008-08-19 20:50:48 -07002450#ifdef CONFIG_X86_32
2451atomic_t irq_mis_count;
2452#endif
2453
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002454static void ack_apic_level(unsigned int irq)
2455{
Yinghai Lu3eb2cce2008-08-19 20:50:48 -07002456#ifdef CONFIG_X86_32
2457 unsigned long v;
2458 int i;
2459#endif
Ingo Molnar54168ed2008-08-20 09:07:45 +02002460 int do_unmask_irq = 0;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002461
Ingo Molnar54168ed2008-08-20 09:07:45 +02002462 irq_complete_move(irq);
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002463#ifdef CONFIG_GENERIC_PENDING_IRQ
Ingo Molnar54168ed2008-08-20 09:07:45 +02002464 /* If we are moving the irq we need to mask it */
2465 if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) {
2466 do_unmask_irq = 1;
2467 mask_IO_APIC_irq(irq);
2468 }
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002469#endif
2470
Yinghai Lu3eb2cce2008-08-19 20:50:48 -07002471#ifdef CONFIG_X86_32
2472 /*
2473 * It appears there is an erratum which affects at least version 0x11
2474 * of I/O APIC (that's the 82093AA and cores integrated into various
2475 * chipsets). Under certain conditions a level-triggered interrupt is
2476 * erroneously delivered as edge-triggered one but the respective IRR
2477 * bit gets set nevertheless. As a result the I/O unit expects an EOI
2478 * message but it will never arrive and further interrupts are blocked
2479 * from the source. The exact reason is so far unknown, but the
2480 * phenomenon was observed when two consecutive interrupt requests
2481 * from a given source get delivered to the same CPU and the source is
2482 * temporarily disabled in between.
2483 *
2484 * A workaround is to simulate an EOI message manually. We achieve it
2485 * by setting the trigger mode to edge and then to level when the edge
2486 * trigger mode gets detected in the TMR of a local APIC for a
2487 * level-triggered interrupt. We mask the source for the time of the
2488 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2489 * The idea is from Manfred Spraul. --macro
2490 */
2491 i = irq_cfg(irq)->vector;
2492
2493 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2494#endif
2495
Ingo Molnar54168ed2008-08-20 09:07:45 +02002496 /*
2497 * We must acknowledge the irq before we move it or the acknowledge will
2498 * not propagate properly.
2499 */
2500 ack_APIC_irq();
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002501
Ingo Molnar54168ed2008-08-20 09:07:45 +02002502 /* Now we can move and renable the irq */
2503 if (unlikely(do_unmask_irq)) {
2504 /* Only migrate the irq if the ack has been received.
2505 *
2506 * On rare occasions the broadcast level triggered ack gets
2507 * delayed going to ioapics, and if we reprogram the
2508 * vector while Remote IRR is still set the irq will never
2509 * fire again.
2510 *
2511 * To prevent this scenario we read the Remote IRR bit
2512 * of the ioapic. This has two effects.
2513 * - On any sane system the read of the ioapic will
2514 * flush writes (and acks) going to the ioapic from
2515 * this cpu.
2516 * - We get to see if the ACK has actually been delivered.
2517 *
2518 * Based on failed experiments of reprogramming the
2519 * ioapic entry from outside of irq context starting
2520 * with masking the ioapic entry and then polling until
2521 * Remote IRR was clear before reprogramming the
2522 * ioapic I don't trust the Remote IRR bit to be
2523 * completey accurate.
2524 *
2525 * However there appears to be no other way to plug
2526 * this race, so if the Remote IRR bit is not
2527 * accurate and is causing problems then it is a hardware bug
2528 * and you can go talk to the chipset vendor about it.
2529 */
2530 if (!io_apic_level_ack_pending(irq))
2531 move_masked_irq(irq);
2532 unmask_IO_APIC_irq(irq);
2533 }
Yinghai Lu1d025192008-08-19 20:50:34 -07002534
Yinghai Lu3eb2cce2008-08-19 20:50:48 -07002535#ifdef CONFIG_X86_32
Yinghai Lu1d025192008-08-19 20:50:34 -07002536 if (!(v & (1 << (i & 0x1f)))) {
2537 atomic_inc(&irq_mis_count);
2538 spin_lock(&ioapic_lock);
2539 __mask_and_edge_IO_APIC_irq(irq);
2540 __unmask_and_level_IO_APIC_irq(irq);
2541 spin_unlock(&ioapic_lock);
2542 }
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002543#endif
Yinghai Lu3eb2cce2008-08-19 20:50:48 -07002544}
Yinghai Lu1d025192008-08-19 20:50:34 -07002545
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07002546static struct irq_chip ioapic_chip __read_mostly = {
2547 .name = "IO-APIC",
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07002548 .startup = startup_ioapic_irq,
2549 .mask = mask_IO_APIC_irq,
2550 .unmask = unmask_IO_APIC_irq,
Yinghai Lu1d025192008-08-19 20:50:34 -07002551 .ack = ack_apic_edge,
2552 .eoi = ack_apic_level,
Ashok Raj54d5d422005-09-06 15:16:15 -07002553#ifdef CONFIG_SMP
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07002554 .set_affinity = set_ioapic_affinity_irq,
Ashok Raj54d5d422005-09-06 15:16:15 -07002555#endif
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07002556 .retrigger = ioapic_retrigger_irq,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557};
2558
Ingo Molnar54168ed2008-08-20 09:07:45 +02002559#ifdef CONFIG_INTR_REMAP
2560static struct irq_chip ir_ioapic_chip __read_mostly = {
2561 .name = "IR-IO-APIC",
2562 .startup = startup_ioapic_irq,
2563 .mask = mask_IO_APIC_irq,
2564 .unmask = unmask_IO_APIC_irq,
2565 .ack = ack_x2apic_edge,
2566 .eoi = ack_x2apic_level,
2567#ifdef CONFIG_SMP
2568 .set_affinity = set_ir_ioapic_affinity_irq,
2569#endif
2570 .retrigger = ioapic_retrigger_irq,
2571};
2572#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002573
2574static inline void init_IO_APIC_traps(void)
2575{
2576 int irq;
Yinghai Lu08678b02008-08-19 20:50:05 -07002577 struct irq_desc *desc;
Yinghai Luda51a822008-08-19 20:50:25 -07002578 struct irq_cfg *cfg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002579
2580 /*
2581 * NOTE! The local APIC isn't very good at handling
2582 * multiple interrupts at the same interrupt level.
2583 * As the interrupt level is determined by taking the
2584 * vector number and shifting that right by 4, we
2585 * want to spread these out a bit so that they don't
2586 * all fall in the same interrupt level.
2587 *
2588 * Also, we've got to be careful not to trash gate
2589 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2590 */
Yinghai Lu8f09cd22008-08-19 20:50:51 -07002591 for_each_irq_cfg(irq, cfg) {
Yinghai Luda51a822008-08-19 20:50:25 -07002592 if (IO_APIC_IRQ(irq) && !cfg->vector) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002593 /*
2594 * Hmm.. We don't have an entry for this,
2595 * so default to an old-fashioned 8259
2596 * interrupt if we can..
2597 */
2598 if (irq < 16)
2599 make_8259A_irq(irq);
Yinghai Lu08678b02008-08-19 20:50:05 -07002600 else {
2601 desc = irq_to_desc(irq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002602 /* Strange. Oh, well.. */
Yinghai Lu08678b02008-08-19 20:50:05 -07002603 desc->chip = &no_irq_chip;
2604 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002605 }
2606 }
2607}
2608
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07002609/*
2610 * The local APIC irq-chip implementation:
2611 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002612
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002613static void mask_lapic_irq(unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002614{
2615 unsigned long v;
2616
2617 v = apic_read(APIC_LVT0);
Maciej W. Rozycki593f4a72008-07-16 19:15:30 +01002618 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002619}
2620
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002621static void unmask_lapic_irq(unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002622{
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07002623 unsigned long v;
2624
2625 v = apic_read(APIC_LVT0);
Maciej W. Rozycki593f4a72008-07-16 19:15:30 +01002626 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002627}
2628
Ingo Molnar54168ed2008-08-20 09:07:45 +02002629static void ack_lapic_irq (unsigned int irq)
Yinghai Lu1d025192008-08-19 20:50:34 -07002630{
2631 ack_APIC_irq();
2632}
2633
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07002634static struct irq_chip lapic_chip __read_mostly = {
Maciej W. Rozycki9a1c6192008-05-27 21:19:09 +01002635 .name = "local-APIC",
Ingo Molnarf5b9ed72006-10-04 02:16:26 -07002636 .mask = mask_lapic_irq,
2637 .unmask = unmask_lapic_irq,
Maciej W. Rozyckic88ac1d2008-07-11 19:35:17 +01002638 .ack = ack_lapic_irq,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002639};
2640
Yinghai Lu497c9a12008-08-19 20:50:28 -07002641static void lapic_register_intr(int irq)
Maciej W. Rozyckic88ac1d2008-07-11 19:35:17 +01002642{
Yinghai Lu08678b02008-08-19 20:50:05 -07002643 struct irq_desc *desc;
2644
2645 desc = irq_to_desc(irq);
2646 desc->status &= ~IRQ_LEVEL;
Maciej W. Rozyckic88ac1d2008-07-11 19:35:17 +01002647 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2648 "edge");
Maciej W. Rozyckic88ac1d2008-07-11 19:35:17 +01002649}
2650
Jan Beuliche9427102008-01-30 13:31:24 +01002651static void __init setup_nmi(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002652{
2653 /*
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002654 * Dirty trick to enable the NMI watchdog ...
Linus Torvalds1da177e2005-04-16 15:20:36 -07002655 * We put the 8259A master into AEOI mode and
2656 * unmask on all local APICs LVT0 as NMI.
2657 *
2658 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
2659 * is from Maciej W. Rozycki - so we do not have to EOI from
2660 * the NMI handler or the timer interrupt.
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002661 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002662 apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");
2663
Jan Beuliche9427102008-01-30 13:31:24 +01002664 enable_NMI_through_LVT0();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002665
2666 apic_printk(APIC_VERBOSE, " done.\n");
2667}
2668
2669/*
2670 * This looks a bit hackish but it's about the only one way of sending
2671 * a few INTA cycles to 8259As and any associated glue logic. ICR does
2672 * not support the ExtINT mode, unfortunately. We need to send these
2673 * cycles as some i82489DX-based boards have glue logic that keeps the
2674 * 8259A interrupt line asserted until INTA. --macro
2675 */
Jacek Luczak28acf282008-04-12 17:41:12 +02002676static inline void __init unlock_ExtINT_logic(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677{
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002678 int apic, pin, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679 struct IO_APIC_route_entry entry0, entry1;
2680 unsigned char save_control, save_freq_select;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002681
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002682 pin = find_isa_irq_pin(8, mp_INT);
Adrian Bunk956fb532006-12-07 02:14:11 +01002683 if (pin == -1) {
2684 WARN_ON_ONCE(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002685 return;
Adrian Bunk956fb532006-12-07 02:14:11 +01002686 }
2687 apic = find_isa_irq_apic(8, mp_INT);
2688 if (apic == -1) {
2689 WARN_ON_ONCE(1);
2690 return;
2691 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002692
Andi Kleencf4c6a22006-09-26 10:52:30 +02002693 entry0 = ioapic_read_entry(apic, pin);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002694 clear_IO_APIC_pin(apic, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002695
2696 memset(&entry1, 0, sizeof(entry1));
2697
2698 entry1.dest_mode = 0; /* physical delivery */
2699 entry1.mask = 0; /* unmask IRQ now */
Yinghai Lud83e94a2008-08-19 20:50:33 -07002700 entry1.dest = hard_smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701 entry1.delivery_mode = dest_ExtINT;
2702 entry1.polarity = entry0.polarity;
2703 entry1.trigger = 0;
2704 entry1.vector = 0;
2705
Andi Kleencf4c6a22006-09-26 10:52:30 +02002706 ioapic_write_entry(apic, pin, entry1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002707
2708 save_control = CMOS_READ(RTC_CONTROL);
2709 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2710 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2711 RTC_FREQ_SELECT);
2712 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2713
2714 i = 100;
2715 while (i-- > 0) {
2716 mdelay(10);
2717 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2718 i -= 10;
2719 }
2720
2721 CMOS_WRITE(save_control, RTC_CONTROL);
2722 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002723 clear_IO_APIC_pin(apic, pin);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002724
Andi Kleencf4c6a22006-09-26 10:52:30 +02002725 ioapic_write_entry(apic, pin, entry0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002726}
2727
Yinghai Luefa25592008-08-19 20:50:36 -07002728static int disable_timer_pin_1 __initdata;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002729/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
Ingo Molnar54168ed2008-08-20 09:07:45 +02002730static int __init disable_timer_pin_setup(char *arg)
Yinghai Luefa25592008-08-19 20:50:36 -07002731{
2732 disable_timer_pin_1 = 1;
2733 return 0;
2734}
Ingo Molnar54168ed2008-08-20 09:07:45 +02002735early_param("disable_timer_pin_1", disable_timer_pin_setup);
Yinghai Luefa25592008-08-19 20:50:36 -07002736
2737int timer_through_8259 __initdata;
2738
Linus Torvalds1da177e2005-04-16 15:20:36 -07002739/*
2740 * This code may look a bit paranoid, but it's supposed to cooperate with
2741 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2742 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2743 * fanatically on his truly buggy board.
Ingo Molnar54168ed2008-08-20 09:07:45 +02002744 *
2745 * FIXME: really need to revamp this for all platforms.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002746 */
Zachary Amsden8542b202006-12-07 02:14:09 +01002747static inline void __init check_timer(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002748{
Yinghai Lu497c9a12008-08-19 20:50:28 -07002749 struct irq_cfg *cfg = irq_cfg(0);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002750 int apic1, pin1, apic2, pin2;
Ingo Molnar4aae0702007-12-18 18:05:58 +01002751 unsigned long flags;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07002752 unsigned int ver;
2753 int no_pin1 = 0;
Ingo Molnar4aae0702007-12-18 18:05:58 +01002754
2755 local_irq_save(flags);
Maciej W. Rozyckid4d25de2007-11-26 20:42:19 +01002756
Ingo Molnar54168ed2008-08-20 09:07:45 +02002757 ver = apic_read(APIC_LVR);
2758 ver = GET_APIC_VERSION(ver);
Ingo Molnar6e908942008-03-21 14:32:36 +01002759
Linus Torvalds1da177e2005-04-16 15:20:36 -07002760 /*
2761 * get/set the timer IRQ vector:
2762 */
2763 disable_8259A_irq(0);
Yinghai Lu497c9a12008-08-19 20:50:28 -07002764 assign_irq_vector(0, TARGET_CPUS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765
2766 /*
Maciej W. Rozyckid11d5792008-05-21 22:09:11 +01002767 * As IRQ0 is to be enabled in the 8259A, the virtual
2768 * wire has to be disabled in the local APIC. Also
2769 * timer interrupts need to be acknowledged manually in
2770 * the 8259A for the i82489DX when using the NMI
2771 * watchdog as that APIC treats NMIs as level-triggered.
2772 * The AEOI mode will finish them in the 8259A
2773 * automatically.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002774 */
Maciej W. Rozycki593f4a72008-07-16 19:15:30 +01002775 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002776 init_8259A(1);
Ingo Molnar54168ed2008-08-20 09:07:45 +02002777#ifdef CONFIG_X86_32
Maciej W. Rozyckid11d5792008-05-21 22:09:11 +01002778 timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
Ingo Molnar54168ed2008-08-20 09:07:45 +02002779#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002780
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002781 pin1 = find_isa_irq_pin(0, mp_INT);
2782 apic1 = find_isa_irq_apic(0, mp_INT);
2783 pin2 = ioapic_i8259.pin;
2784 apic2 = ioapic_i8259.apic;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002786 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2787 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
Yinghai Lu497c9a12008-08-19 20:50:28 -07002788 cfg->vector, apic1, pin1, apic2, pin2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002789
Maciej W. Rozycki691874f2008-05-27 21:19:51 +01002790 /*
2791 * Some BIOS writers are clueless and report the ExtINTA
2792 * I/O APIC input from the cascaded 8259A as the timer
2793 * interrupt input. So just in case, if only one pin
2794 * was found above, try it both directly and through the
2795 * 8259A.
2796 */
2797 if (pin1 == -1) {
Ingo Molnar54168ed2008-08-20 09:07:45 +02002798#ifdef CONFIG_INTR_REMAP
2799 if (intr_remapping_enabled)
2800 panic("BIOS bug: timer not connected to IO-APIC");
2801#endif
Maciej W. Rozycki691874f2008-05-27 21:19:51 +01002802 pin1 = pin2;
2803 apic1 = apic2;
2804 no_pin1 = 1;
2805 } else if (pin2 == -1) {
2806 pin2 = pin1;
2807 apic2 = apic1;
2808 }
2809
Linus Torvalds1da177e2005-04-16 15:20:36 -07002810 if (pin1 != -1) {
2811 /*
2812 * Ok, does IRQ0 through the IOAPIC work?
2813 */
Maciej W. Rozycki691874f2008-05-27 21:19:51 +01002814 if (no_pin1) {
2815 add_pin_to_irq(0, apic1, pin1);
Yinghai Lu497c9a12008-08-19 20:50:28 -07002816 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
Maciej W. Rozycki691874f2008-05-27 21:19:51 +01002817 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002818 unmask_IO_APIC_irq(0);
2819 if (timer_irq_works()) {
2820 if (nmi_watchdog == NMI_IO_APIC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002821 setup_nmi();
2822 enable_8259A_irq(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823 }
Chuck Ebbert66759a02005-09-12 18:49:25 +02002824 if (disable_timer_pin_1 > 0)
2825 clear_IO_APIC_pin(0, pin1);
Ingo Molnar4aae0702007-12-18 18:05:58 +01002826 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002827 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02002828#ifdef CONFIG_INTR_REMAP
2829 if (intr_remapping_enabled)
2830 panic("timer doesn't work through Interrupt-remapped IO-APIC");
2831#endif
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002832 clear_IO_APIC_pin(apic1, pin1);
Maciej W. Rozycki691874f2008-05-27 21:19:51 +01002833 if (!no_pin1)
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002834 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2835 "8254 timer not connected to IO-APIC\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002837 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2838 "(IRQ0) through the 8259A ...\n");
2839 apic_printk(APIC_QUIET, KERN_INFO
2840 "..... (found apic %d pin %d) ...\n", apic2, pin2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002841 /*
2842 * legacy devices should be connected to IO APIC #0
2843 */
Maciej W. Rozycki691874f2008-05-27 21:19:51 +01002844 replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
Yinghai Lu497c9a12008-08-19 20:50:28 -07002845 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
Maciej W. Rozycki24742ec2008-05-27 21:19:40 +01002846 unmask_IO_APIC_irq(0);
Maciej W. Rozyckiecd29472008-05-21 22:09:19 +01002847 enable_8259A_irq(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002848 if (timer_irq_works()) {
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002849 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
Maciej W. Rozycki35542c52008-05-21 22:10:22 +01002850 timer_through_8259 = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851 if (nmi_watchdog == NMI_IO_APIC) {
Maciej W. Rozycki60134eb2008-05-21 22:09:34 +01002852 disable_8259A_irq(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853 setup_nmi();
Maciej W. Rozycki60134eb2008-05-21 22:09:34 +01002854 enable_8259A_irq(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855 }
Ingo Molnar4aae0702007-12-18 18:05:58 +01002856 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002857 }
2858 /*
2859 * Cleanup, just in case ...
2860 */
Maciej W. Rozyckiecd29472008-05-21 22:09:19 +01002861 disable_8259A_irq(0);
Eric W. Biedermanfcfd6362005-10-30 14:59:39 -08002862 clear_IO_APIC_pin(apic2, pin2);
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002863 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002864 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002865
2866 if (nmi_watchdog == NMI_IO_APIC) {
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002867 apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
2868 "through the IO-APIC - disabling NMI Watchdog!\n");
Cyrill Gorcunov067fa0f2008-05-29 22:32:30 +04002869 nmi_watchdog = NMI_NONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002870 }
Ingo Molnar54168ed2008-08-20 09:07:45 +02002871#ifdef CONFIG_X86_32
Maciej W. Rozyckid11d5792008-05-21 22:09:11 +01002872 timer_ack = 0;
Ingo Molnar54168ed2008-08-20 09:07:45 +02002873#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002874
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002875 apic_printk(APIC_QUIET, KERN_INFO
2876 "...trying to set up timer as Virtual Wire IRQ...\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877
Yinghai Lu497c9a12008-08-19 20:50:28 -07002878 lapic_register_intr(0);
2879 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002880 enable_8259A_irq(0);
2881
2882 if (timer_irq_works()) {
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002883 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
Ingo Molnar4aae0702007-12-18 18:05:58 +01002884 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002885 }
Maciej W. Rozyckie67465f2008-05-21 22:09:26 +01002886 disable_8259A_irq(0);
Yinghai Lu497c9a12008-08-19 20:50:28 -07002887 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002888 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002889
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002890 apic_printk(APIC_QUIET, KERN_INFO
2891 "...trying to set up timer as ExtINT IRQ...\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002892
Linus Torvalds1da177e2005-04-16 15:20:36 -07002893 init_8259A(0);
2894 make_8259A_irq(0);
Maciej W. Rozycki593f4a72008-07-16 19:15:30 +01002895 apic_write(APIC_LVT0, APIC_DM_EXTINT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002896
2897 unlock_ExtINT_logic();
2898
2899 if (timer_irq_works()) {
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002900 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
Ingo Molnar4aae0702007-12-18 18:05:58 +01002901 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002902 }
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002903 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002904 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
Maciej W. Rozycki49a66a02008-07-14 19:08:13 +01002905 "report. Then try booting with the 'noapic' option.\n");
Ingo Molnar4aae0702007-12-18 18:05:58 +01002906out:
2907 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908}
2909
2910/*
Maciej W. Rozyckiaf174782008-07-11 19:35:23 +01002911 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2912 * to devices. However there may be an I/O APIC pin available for
2913 * this interrupt regardless. The pin may be left unconnected, but
2914 * typically it will be reused as an ExtINT cascade interrupt for
2915 * the master 8259A. In the MPS case such a pin will normally be
2916 * reported as an ExtINT interrupt in the MP table. With ACPI
2917 * there is no provision for ExtINT interrupts, and in the absence
2918 * of an override it would be treated as an ordinary ISA I/O APIC
2919 * interrupt, that is edge-triggered and unmasked by default. We
2920 * used to do this, but it caused problems on some systems because
2921 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2922 * the same ExtINT cascade interrupt to drive the local APIC of the
2923 * bootstrap processor. Therefore we refrain from routing IRQ2 to
2924 * the I/O APIC in all cases now. No actual device should request
2925 * it anyway. --macro
Linus Torvalds1da177e2005-04-16 15:20:36 -07002926 */
2927#define PIC_IRQS (1 << PIC_CASCADE_IR)
2928
2929void __init setup_IO_APIC(void)
2930{
Ingo Molnar54168ed2008-08-20 09:07:45 +02002931
2932#ifdef CONFIG_X86_32
Linus Torvalds1da177e2005-04-16 15:20:36 -07002933 enable_IO_APIC();
Ingo Molnar54168ed2008-08-20 09:07:45 +02002934#else
2935 /*
2936 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
2937 */
2938#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002939
Maciej W. Rozyckiaf174782008-07-11 19:35:23 +01002940 io_apic_irqs = ~PIC_IRQS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002941
Ingo Molnar54168ed2008-08-20 09:07:45 +02002942 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2943 /*
2944 * Set up IO-APIC IRQ routing.
2945 */
2946#ifdef CONFIG_X86_32
2947 if (!acpi_ioapic)
2948 setup_ioapic_ids_from_mpc();
2949#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002950 sync_Arb_IDs();
2951 setup_IO_APIC_irqs();
2952 init_IO_APIC_traps();
Linus Torvalds1e4c85f2005-10-31 19:16:17 -08002953 check_timer();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002954}
2955
2956/*
Ingo Molnar54168ed2008-08-20 09:07:45 +02002957 * Called after all the initialization is done. If we didnt find any
2958 * APIC bugs then we can allow the modify fast path
Linus Torvalds1da177e2005-04-16 15:20:36 -07002959 */
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002960
Linus Torvalds1da177e2005-04-16 15:20:36 -07002961static int __init io_apic_bug_finalize(void)
2962{
Ingo Molnar54168ed2008-08-20 09:07:45 +02002963 if (sis_apic_bug == -1)
2964 sis_apic_bug = 0;
2965 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002966}
2967
2968late_initcall(io_apic_bug_finalize);
2969
2970struct sysfs_ioapic_data {
2971 struct sys_device dev;
2972 struct IO_APIC_route_entry entry[0];
2973};
Ingo Molnar54168ed2008-08-20 09:07:45 +02002974static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002975
Pavel Machek438510f2005-04-16 15:25:24 -07002976static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002977{
2978 struct IO_APIC_route_entry *entry;
2979 struct sysfs_ioapic_data *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002980 int i;
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002981
Linus Torvalds1da177e2005-04-16 15:20:36 -07002982 data = container_of(dev, struct sysfs_ioapic_data, dev);
2983 entry = data->entry;
Ingo Molnar54168ed2008-08-20 09:07:45 +02002984 for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
2985 *entry = ioapic_read_entry(dev->id, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002986
2987 return 0;
2988}
2989
2990static int ioapic_resume(struct sys_device *dev)
2991{
2992 struct IO_APIC_route_entry *entry;
2993 struct sysfs_ioapic_data *data;
2994 unsigned long flags;
2995 union IO_APIC_reg_00 reg_00;
2996 int i;
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02002997
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 data = container_of(dev, struct sysfs_ioapic_data, dev);
2999 entry = data->entry;
3000
3001 spin_lock_irqsave(&ioapic_lock, flags);
3002 reg_00.raw = io_apic_read(dev->id, 0);
Alexey Starikovskiyec2cd0a2008-05-14 19:03:10 +04003003 if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
3004 reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003005 io_apic_write(dev->id, 0, reg_00.raw);
3006 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003007 spin_unlock_irqrestore(&ioapic_lock, flags);
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003008 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
Andi Kleencf4c6a22006-09-26 10:52:30 +02003009 ioapic_write_entry(dev->id, i, entry[i]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003010
3011 return 0;
3012}
3013
3014static struct sysdev_class ioapic_sysdev_class = {
Kay Sieversaf5ca3f2007-12-20 02:09:39 +01003015 .name = "ioapic",
Linus Torvalds1da177e2005-04-16 15:20:36 -07003016 .suspend = ioapic_suspend,
3017 .resume = ioapic_resume,
3018};
3019
3020static int __init ioapic_init_sysfs(void)
3021{
Ingo Molnar54168ed2008-08-20 09:07:45 +02003022 struct sys_device * dev;
3023 int i, size, error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003024
3025 error = sysdev_class_register(&ioapic_sysdev_class);
3026 if (error)
3027 return error;
3028
Ingo Molnar54168ed2008-08-20 09:07:45 +02003029 for (i = 0; i < nr_ioapics; i++ ) {
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003030 size = sizeof(struct sys_device) + nr_ioapic_registers[i]
Linus Torvalds1da177e2005-04-16 15:20:36 -07003031 * sizeof(struct IO_APIC_route_entry);
Christophe Jaillet25556c12008-06-22 22:13:48 +02003032 mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003033 if (!mp_ioapic_data[i]) {
3034 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3035 continue;
3036 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003037 dev = &mp_ioapic_data[i]->dev;
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003038 dev->id = i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003039 dev->cls = &ioapic_sysdev_class;
3040 error = sysdev_register(dev);
3041 if (error) {
3042 kfree(mp_ioapic_data[i]);
3043 mp_ioapic_data[i] = NULL;
3044 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3045 continue;
3046 }
3047 }
3048
3049 return 0;
3050}
3051
3052device_initcall(ioapic_init_sysfs);
3053
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003054/*
Eric W. Biederman95d77882006-10-04 02:17:01 -07003055 * Dynamic irq allocate and deallocation
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003056 */
Yinghai Lu199751d2008-08-19 20:50:27 -07003057unsigned int create_irq_nr(unsigned int irq_want)
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003058{
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07003059 /* Allocate an unused irq */
Ingo Molnar54168ed2008-08-20 09:07:45 +02003060 unsigned int irq;
3061 unsigned int new;
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003062 unsigned long flags;
Yinghai Luda51a822008-08-19 20:50:25 -07003063 struct irq_cfg *cfg_new;
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003064
Yinghai Lu497c9a12008-08-19 20:50:28 -07003065#ifndef CONFIG_HAVE_SPARSE_IRQ
Yinghai Lu199751d2008-08-19 20:50:27 -07003066 irq_want = nr_irqs - 1;
Yinghai Lu497c9a12008-08-19 20:50:28 -07003067#endif
Yinghai Lu199751d2008-08-19 20:50:27 -07003068
3069 irq = 0;
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07003070 spin_lock_irqsave(&vector_lock, flags);
Ingo Molnar54168ed2008-08-20 09:07:45 +02003071 for (new = irq_want; new > 0; new--) {
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07003072 if (platform_legacy_irq(new))
3073 continue;
Yinghai Luda51a822008-08-19 20:50:25 -07003074 cfg_new = irq_cfg(new);
3075 if (cfg_new && cfg_new->vector != 0)
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07003076 continue;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07003077 /* check if need to create one */
Yinghai Luda51a822008-08-19 20:50:25 -07003078 if (!cfg_new)
3079 cfg_new = irq_cfg_alloc(new);
Yinghai Lu497c9a12008-08-19 20:50:28 -07003080 if (__assign_irq_vector(new, TARGET_CPUS) == 0)
Eric W. Biedermanace80ab2006-10-04 02:16:47 -07003081 irq = new;
3082 break;
3083 }
3084 spin_unlock_irqrestore(&vector_lock, flags);
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003085
Yinghai Lu199751d2008-08-19 20:50:27 -07003086 if (irq > 0) {
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003087 dynamic_irq_init(irq);
3088 }
3089 return irq;
3090}
3091
Yinghai Lu199751d2008-08-19 20:50:27 -07003092int create_irq(void)
3093{
Ingo Molnar54168ed2008-08-20 09:07:45 +02003094 int irq;
3095
3096 irq = create_irq_nr(nr_irqs - 1);
3097
3098 if (irq == 0)
3099 irq = -1;
3100
3101 return irq;
Yinghai Lu199751d2008-08-19 20:50:27 -07003102}
3103
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003104void destroy_irq(unsigned int irq)
3105{
3106 unsigned long flags;
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003107
3108 dynamic_irq_cleanup(irq);
3109
Ingo Molnar54168ed2008-08-20 09:07:45 +02003110#ifdef CONFIG_INTR_REMAP
3111 free_irte(irq);
3112#endif
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003113 spin_lock_irqsave(&vector_lock, flags);
Yinghai Lu497c9a12008-08-19 20:50:28 -07003114 __clear_irq_vector(irq);
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003115 spin_unlock_irqrestore(&vector_lock, flags);
3116}
Eric W. Biederman3fc471e2006-10-04 02:16:39 -07003117
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003118/*
Simon Arlott27b46d72007-10-20 01:13:56 +02003119 * MSI message composition
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003120 */
3121#ifdef CONFIG_PCI_MSI
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003122static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003123{
Yinghai Lu497c9a12008-08-19 20:50:28 -07003124 struct irq_cfg *cfg;
3125 int err;
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003126 unsigned dest;
Yinghai Lu497c9a12008-08-19 20:50:28 -07003127 cpumask_t tmp;
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003128
Yinghai Lu497c9a12008-08-19 20:50:28 -07003129 tmp = TARGET_CPUS;
3130 err = assign_irq_vector(irq, tmp);
3131 if (err)
3132 return err;
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003133
Yinghai Lu497c9a12008-08-19 20:50:28 -07003134 cfg = irq_cfg(irq);
3135 cpus_and(tmp, cfg->domain, tmp);
3136 dest = cpu_mask_to_apicid(tmp);
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003137
Ingo Molnar54168ed2008-08-20 09:07:45 +02003138#ifdef CONFIG_INTR_REMAP
3139 if (irq_remapped(irq)) {
3140 struct irte irte;
3141 int ir_index;
3142 u16 sub_handle;
Yinghai Lu497c9a12008-08-19 20:50:28 -07003143
Ingo Molnar54168ed2008-08-20 09:07:45 +02003144 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
3145 BUG_ON(ir_index == -1);
Yinghai Lu497c9a12008-08-19 20:50:28 -07003146
Ingo Molnar54168ed2008-08-20 09:07:45 +02003147 memset (&irte, 0, sizeof(irte));
3148
3149 irte.present = 1;
3150 irte.dst_mode = INT_DEST_MODE;
3151 irte.trigger_mode = 0; /* edge */
3152 irte.dlvry_mode = INT_DELIVERY_MODE;
3153 irte.vector = cfg->vector;
3154 irte.dest_id = IRTE_DEST(dest);
3155
3156 modify_irte(irq, &irte);
3157
3158 msg->address_hi = MSI_ADDR_BASE_HI;
3159 msg->data = sub_handle;
3160 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
3161 MSI_ADDR_IR_SHV |
3162 MSI_ADDR_IR_INDEX1(ir_index) |
3163 MSI_ADDR_IR_INDEX2(ir_index);
3164 } else
3165#endif
3166 {
3167 msg->address_hi = MSI_ADDR_BASE_HI;
3168 msg->address_lo =
3169 MSI_ADDR_BASE_LO |
3170 ((INT_DEST_MODE == 0) ?
3171 MSI_ADDR_DEST_MODE_PHYSICAL:
3172 MSI_ADDR_DEST_MODE_LOGICAL) |
3173 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3174 MSI_ADDR_REDIRECTION_CPU:
3175 MSI_ADDR_REDIRECTION_LOWPRI) |
3176 MSI_ADDR_DEST_ID(dest);
3177
3178 msg->data =
3179 MSI_DATA_TRIGGER_EDGE |
3180 MSI_DATA_LEVEL_ASSERT |
3181 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3182 MSI_DATA_DELIVERY_FIXED:
3183 MSI_DATA_DELIVERY_LOWPRI) |
3184 MSI_DATA_VECTOR(cfg->vector);
3185 }
Yinghai Lu497c9a12008-08-19 20:50:28 -07003186 return err;
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003187}
3188
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003189#ifdef CONFIG_SMP
3190static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3191{
Yinghai Lu497c9a12008-08-19 20:50:28 -07003192 struct irq_cfg *cfg;
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003193 struct msi_msg msg;
3194 unsigned int dest;
3195 cpumask_t tmp;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003196 struct irq_desc *desc;
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003197
3198 cpus_and(tmp, mask, cpu_online_map);
3199 if (cpus_empty(tmp))
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003200 return;
3201
Yinghai Lu497c9a12008-08-19 20:50:28 -07003202 if (assign_irq_vector(irq, mask))
3203 return;
3204
3205 cfg = irq_cfg(irq);
3206 cpus_and(tmp, cfg->domain, mask);
3207 dest = cpu_mask_to_apicid(tmp);
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003208
3209 read_msi_msg(irq, &msg);
3210
3211 msg.data &= ~MSI_DATA_VECTOR_MASK;
Yinghai Lu497c9a12008-08-19 20:50:28 -07003212 msg.data |= MSI_DATA_VECTOR(cfg->vector);
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003213 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3214 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3215
3216 write_msi_msg(irq, &msg);
Ingo Molnar54168ed2008-08-20 09:07:45 +02003217 desc = irq_to_desc(irq);
3218 desc->affinity = mask;
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003219}
Ingo Molnar54168ed2008-08-20 09:07:45 +02003220
3221#ifdef CONFIG_INTR_REMAP
3222/*
3223 * Migrate the MSI irq to another cpumask. This migration is
3224 * done in the process context using interrupt-remapping hardware.
3225 */
3226static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3227{
3228 struct irq_cfg *cfg;
3229 unsigned int dest;
3230 cpumask_t tmp, cleanup_mask;
3231 struct irte irte;
3232 struct irq_desc *desc;
3233
3234 cpus_and(tmp, mask, cpu_online_map);
3235 if (cpus_empty(tmp))
3236 return;
3237
3238 if (get_irte(irq, &irte))
3239 return;
3240
3241 if (assign_irq_vector(irq, mask))
3242 return;
3243
3244 cfg = irq_cfg(irq);
3245 cpus_and(tmp, cfg->domain, mask);
3246 dest = cpu_mask_to_apicid(tmp);
3247
3248 irte.vector = cfg->vector;
3249 irte.dest_id = IRTE_DEST(dest);
3250
3251 /*
3252 * atomically update the IRTE with the new destination and vector.
3253 */
3254 modify_irte(irq, &irte);
3255
3256 /*
3257 * After this point, all the interrupts will start arriving
3258 * at the new destination. So, time to cleanup the previous
3259 * vector allocation.
3260 */
3261 if (cfg->move_in_progress) {
3262 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
3263 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
3264 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
3265 cfg->move_in_progress = 0;
3266 }
3267
3268 desc = irq_to_desc(irq);
3269 desc->affinity = mask;
3270}
3271#endif
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003272#endif /* CONFIG_SMP */
3273
3274/*
3275 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
3276 * which implement the MSI or MSI-X Capability Structure.
3277 */
3278static struct irq_chip msi_chip = {
3279 .name = "PCI-MSI",
3280 .unmask = unmask_msi_irq,
3281 .mask = mask_msi_irq,
Yinghai Lu1d025192008-08-19 20:50:34 -07003282 .ack = ack_apic_edge,
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003283#ifdef CONFIG_SMP
3284 .set_affinity = set_msi_irq_affinity,
3285#endif
3286 .retrigger = ioapic_retrigger_irq,
3287};
3288
Ingo Molnar54168ed2008-08-20 09:07:45 +02003289#ifdef CONFIG_INTR_REMAP
3290static struct irq_chip msi_ir_chip = {
3291 .name = "IR-PCI-MSI",
3292 .unmask = unmask_msi_irq,
3293 .mask = mask_msi_irq,
3294 .ack = ack_x2apic_edge,
3295#ifdef CONFIG_SMP
3296 .set_affinity = ir_set_msi_irq_affinity,
3297#endif
3298 .retrigger = ioapic_retrigger_irq,
3299};
3300
3301/*
3302 * Map the PCI dev to the corresponding remapping hardware unit
3303 * and allocate 'nvec' consecutive interrupt-remapping table entries
3304 * in it.
3305 */
3306static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
3307{
3308 struct intel_iommu *iommu;
3309 int index;
3310
3311 iommu = map_dev_to_ir(dev);
3312 if (!iommu) {
3313 printk(KERN_ERR
3314 "Unable to map PCI %s to iommu\n", pci_name(dev));
3315 return -ENOENT;
3316 }
3317
3318 index = alloc_irte(iommu, irq, nvec);
3319 if (index < 0) {
3320 printk(KERN_ERR
3321 "Unable to allocate %d IRTE for PCI %s\n", nvec,
3322 pci_name(dev));
3323 return -ENOSPC;
3324 }
3325 return index;
3326}
3327#endif
Yinghai Lu1d025192008-08-19 20:50:34 -07003328
3329static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
3330{
3331 int ret;
3332 struct msi_msg msg;
3333
3334 ret = msi_compose_msg(dev, irq, &msg);
3335 if (ret < 0)
3336 return ret;
3337
3338 set_irq_msi(irq, desc);
3339 write_msi_msg(irq, &msg);
3340
Ingo Molnar54168ed2008-08-20 09:07:45 +02003341#ifdef CONFIG_INTR_REMAP
3342 if (irq_remapped(irq)) {
3343 struct irq_desc *desc = irq_to_desc(irq);
3344 /*
3345 * irq migration in process context
3346 */
3347 desc->status |= IRQ_MOVE_PCNTXT;
3348 set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
3349 } else
3350#endif
3351 set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
Yinghai Lu1d025192008-08-19 20:50:34 -07003352
3353 return 0;
3354}
3355
Yinghai Lu199751d2008-08-19 20:50:27 -07003356static unsigned int build_irq_for_pci_dev(struct pci_dev *dev)
3357{
3358 unsigned int irq;
3359
3360 irq = dev->bus->number;
3361 irq <<= 8;
3362 irq |= dev->devfn;
3363 irq <<= 12;
3364
3365 return irq;
3366}
3367
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -07003368int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003369{
Ingo Molnar54168ed2008-08-20 09:07:45 +02003370 unsigned int irq;
3371 int ret;
Yinghai Lu199751d2008-08-19 20:50:27 -07003372 unsigned int irq_want;
3373
3374 irq_want = build_irq_for_pci_dev(dev) + 0x100;
3375
3376 irq = create_irq_nr(irq_want);
Yinghai Lu199751d2008-08-19 20:50:27 -07003377 if (irq == 0)
3378 return -1;
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -07003379
Ingo Molnar54168ed2008-08-20 09:07:45 +02003380#ifdef CONFIG_INTR_REMAP
3381 if (!intr_remapping_enabled)
3382 goto no_ir;
3383
3384 ret = msi_alloc_irte(dev, irq, 1);
3385 if (ret < 0)
3386 goto error;
3387no_ir:
3388#endif
Yinghai Lu1d025192008-08-19 20:50:34 -07003389 ret = setup_msi_irq(dev, desc, irq);
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -07003390 if (ret < 0) {
3391 destroy_irq(irq);
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003392 return ret;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003393 }
Michael Ellerman7fe37302007-04-18 19:39:21 +10003394 return 0;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003395
3396#ifdef CONFIG_INTR_REMAP
3397error:
3398 destroy_irq(irq);
3399 return ret;
3400#endif
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003401}
3402
Yinghai Lu047c8fd2008-08-19 20:50:41 -07003403int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3404{
Ingo Molnar54168ed2008-08-20 09:07:45 +02003405 unsigned int irq;
3406 int ret, sub_handle;
3407 struct msi_desc *desc;
3408 unsigned int irq_want;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07003409
Ingo Molnar54168ed2008-08-20 09:07:45 +02003410#ifdef CONFIG_INTR_REMAP
3411 struct intel_iommu *iommu = 0;
3412 int index = 0;
3413#endif
3414
3415 irq_want = build_irq_for_pci_dev(dev) + 0x100;
3416 sub_handle = 0;
3417 list_for_each_entry(desc, &dev->msi_list, list) {
3418 irq = create_irq_nr(irq_want--);
3419 if (irq == 0)
3420 return -1;
3421#ifdef CONFIG_INTR_REMAP
3422 if (!intr_remapping_enabled)
3423 goto no_ir;
3424
3425 if (!sub_handle) {
3426 /*
3427 * allocate the consecutive block of IRTE's
3428 * for 'nvec'
3429 */
3430 index = msi_alloc_irte(dev, irq, nvec);
3431 if (index < 0) {
3432 ret = index;
3433 goto error;
3434 }
3435 } else {
3436 iommu = map_dev_to_ir(dev);
3437 if (!iommu) {
3438 ret = -ENOENT;
3439 goto error;
3440 }
3441 /*
3442 * setup the mapping between the irq and the IRTE
3443 * base index, the sub_handle pointing to the
3444 * appropriate interrupt remap table entry.
3445 */
3446 set_irte_irq(irq, iommu, index, sub_handle);
3447 }
3448no_ir:
3449#endif
3450 ret = setup_msi_irq(dev, desc, irq);
3451 if (ret < 0)
3452 goto error;
3453 sub_handle++;
3454 }
3455 return 0;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07003456
3457error:
Ingo Molnar54168ed2008-08-20 09:07:45 +02003458 destroy_irq(irq);
3459 return ret;
Yinghai Lu047c8fd2008-08-19 20:50:41 -07003460}
3461
Eric W. Biederman3b7d1922006-10-04 02:16:59 -07003462void arch_teardown_msi_irq(unsigned int irq)
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003463{
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -07003464 destroy_irq(irq);
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003465}
3466
Ingo Molnar54168ed2008-08-20 09:07:45 +02003467#ifdef CONFIG_DMAR
3468#ifdef CONFIG_SMP
3469static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
3470{
3471 struct irq_cfg *cfg;
3472 struct msi_msg msg;
3473 unsigned int dest;
3474 cpumask_t tmp;
3475 struct irq_desc *desc;
Eric W. Biederman2d3fcc12006-10-04 02:16:43 -07003476
Ingo Molnar54168ed2008-08-20 09:07:45 +02003477 cpus_and(tmp, mask, cpu_online_map);
3478 if (cpus_empty(tmp))
3479 return;
3480
3481 if (assign_irq_vector(irq, mask))
3482 return;
3483
3484 cfg = irq_cfg(irq);
3485 cpus_and(tmp, cfg->domain, mask);
3486 dest = cpu_mask_to_apicid(tmp);
3487
3488 dmar_msi_read(irq, &msg);
3489
3490 msg.data &= ~MSI_DATA_VECTOR_MASK;
3491 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3492 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3493 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3494
3495 dmar_msi_write(irq, &msg);
3496 desc = irq_to_desc(irq);
3497 desc->affinity = mask;
3498}
3499#endif /* CONFIG_SMP */
3500
3501struct irq_chip dmar_msi_type = {
3502 .name = "DMAR_MSI",
3503 .unmask = dmar_msi_unmask,
3504 .mask = dmar_msi_mask,
3505 .ack = ack_apic_edge,
3506#ifdef CONFIG_SMP
3507 .set_affinity = dmar_msi_set_affinity,
3508#endif
3509 .retrigger = ioapic_retrigger_irq,
3510};
3511
3512int arch_setup_dmar_msi(unsigned int irq)
3513{
3514 int ret;
3515 struct msi_msg msg;
3516
3517 ret = msi_compose_msg(NULL, irq, &msg);
3518 if (ret < 0)
3519 return ret;
3520 dmar_msi_write(irq, &msg);
3521 set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3522 "edge");
3523 return 0;
3524}
3525#endif
3526
venkatesh.pallipadi@intel.com58ac1e72008-09-05 18:02:17 -07003527#ifdef CONFIG_HPET_TIMER
3528
3529#ifdef CONFIG_SMP
3530static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
3531{
3532 struct irq_cfg *cfg;
3533 struct irq_desc *desc;
3534 struct msi_msg msg;
3535 unsigned int dest;
3536 cpumask_t tmp;
3537
3538 cpus_and(tmp, mask, cpu_online_map);
3539 if (cpus_empty(tmp))
3540 return;
3541
3542 if (assign_irq_vector(irq, mask))
3543 return;
3544
3545 cfg = irq_cfg(irq);
3546 cpus_and(tmp, cfg->domain, mask);
3547 dest = cpu_mask_to_apicid(tmp);
3548
3549 hpet_msi_read(irq, &msg);
3550
3551 msg.data &= ~MSI_DATA_VECTOR_MASK;
3552 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3553 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3554 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3555
3556 hpet_msi_write(irq, &msg);
3557 desc = irq_to_desc(irq);
3558 desc->affinity = mask;
3559}
3560#endif /* CONFIG_SMP */
3561
3562struct irq_chip hpet_msi_type = {
3563 .name = "HPET_MSI",
3564 .unmask = hpet_msi_unmask,
3565 .mask = hpet_msi_mask,
3566 .ack = ack_apic_edge,
3567#ifdef CONFIG_SMP
3568 .set_affinity = hpet_msi_set_affinity,
3569#endif
3570 .retrigger = ioapic_retrigger_irq,
3571};
3572
3573int arch_setup_hpet_msi(unsigned int irq)
3574{
3575 int ret;
3576 struct msi_msg msg;
3577
3578 ret = msi_compose_msg(NULL, irq, &msg);
3579 if (ret < 0)
3580 return ret;
3581
3582 hpet_msi_write(irq, &msg);
3583 set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq,
3584 "edge");
3585 return 0;
3586}
3587#endif
3588
Ingo Molnar54168ed2008-08-20 09:07:45 +02003589#endif /* CONFIG_PCI_MSI */
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003590/*
3591 * Hypertransport interrupt support
3592 */
3593#ifdef CONFIG_HT_IRQ
3594
3595#ifdef CONFIG_SMP
3596
Yinghai Lu497c9a12008-08-19 20:50:28 -07003597static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003598{
Eric W. Biedermanec683072006-11-08 17:44:57 -08003599 struct ht_irq_msg msg;
3600 fetch_ht_irq_msg(irq, &msg);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003601
Yinghai Lu497c9a12008-08-19 20:50:28 -07003602 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
Eric W. Biedermanec683072006-11-08 17:44:57 -08003603 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003604
Yinghai Lu497c9a12008-08-19 20:50:28 -07003605 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
Eric W. Biedermanec683072006-11-08 17:44:57 -08003606 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003607
Eric W. Biedermanec683072006-11-08 17:44:57 -08003608 write_ht_irq_msg(irq, &msg);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003609}
3610
3611static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
3612{
Yinghai Lu497c9a12008-08-19 20:50:28 -07003613 struct irq_cfg *cfg;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003614 unsigned int dest;
3615 cpumask_t tmp;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003616 struct irq_desc *desc;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003617
3618 cpus_and(tmp, mask, cpu_online_map);
3619 if (cpus_empty(tmp))
Yinghai Lu497c9a12008-08-19 20:50:28 -07003620 return;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003621
Yinghai Lu497c9a12008-08-19 20:50:28 -07003622 if (assign_irq_vector(irq, mask))
3623 return;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003624
Yinghai Lu497c9a12008-08-19 20:50:28 -07003625 cfg = irq_cfg(irq);
3626 cpus_and(tmp, cfg->domain, mask);
3627 dest = cpu_mask_to_apicid(tmp);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003628
Yinghai Lu497c9a12008-08-19 20:50:28 -07003629 target_ht_irq(irq, dest, cfg->vector);
Ingo Molnar54168ed2008-08-20 09:07:45 +02003630 desc = irq_to_desc(irq);
3631 desc->affinity = mask;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003632}
3633#endif
3634
Aneesh Kumar K.Vc37e1082006-10-11 01:20:43 -07003635static struct irq_chip ht_irq_chip = {
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003636 .name = "PCI-HT",
3637 .mask = mask_ht_irq,
3638 .unmask = unmask_ht_irq,
Yinghai Lu1d025192008-08-19 20:50:34 -07003639 .ack = ack_apic_edge,
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003640#ifdef CONFIG_SMP
3641 .set_affinity = set_ht_irq_affinity,
3642#endif
3643 .retrigger = ioapic_retrigger_irq,
3644};
3645
3646int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3647{
Yinghai Lu497c9a12008-08-19 20:50:28 -07003648 struct irq_cfg *cfg;
3649 int err;
3650 cpumask_t tmp;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003651
Yinghai Lu497c9a12008-08-19 20:50:28 -07003652 tmp = TARGET_CPUS;
3653 err = assign_irq_vector(irq, tmp);
Ingo Molnar54168ed2008-08-20 09:07:45 +02003654 if (!err) {
Eric W. Biedermanec683072006-11-08 17:44:57 -08003655 struct ht_irq_msg msg;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003656 unsigned dest;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003657
Yinghai Lu497c9a12008-08-19 20:50:28 -07003658 cfg = irq_cfg(irq);
3659 cpus_and(tmp, cfg->domain, tmp);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003660 dest = cpu_mask_to_apicid(tmp);
3661
Eric W. Biedermanec683072006-11-08 17:44:57 -08003662 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003663
Eric W. Biedermanec683072006-11-08 17:44:57 -08003664 msg.address_lo =
3665 HT_IRQ_LOW_BASE |
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003666 HT_IRQ_LOW_DEST_ID(dest) |
Yinghai Lu497c9a12008-08-19 20:50:28 -07003667 HT_IRQ_LOW_VECTOR(cfg->vector) |
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003668 ((INT_DEST_MODE == 0) ?
3669 HT_IRQ_LOW_DM_PHYSICAL :
3670 HT_IRQ_LOW_DM_LOGICAL) |
3671 HT_IRQ_LOW_RQEOI_EDGE |
3672 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3673 HT_IRQ_LOW_MT_FIXED :
3674 HT_IRQ_LOW_MT_ARBITRATED) |
3675 HT_IRQ_LOW_IRQ_MASKED;
3676
Eric W. Biedermanec683072006-11-08 17:44:57 -08003677 write_ht_irq_msg(irq, &msg);
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003678
Ingo Molnara460e742006-10-17 00:10:03 -07003679 set_irq_chip_and_handler_name(irq, &ht_irq_chip,
3680 handle_edge_irq, "edge");
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003681 }
Yinghai Lu497c9a12008-08-19 20:50:28 -07003682 return err;
Eric W. Biederman8b955b02006-10-04 02:16:55 -07003683}
3684#endif /* CONFIG_HT_IRQ */
3685
Yinghai Lu9d6a4d02008-08-19 20:50:52 -07003686int __init io_apic_get_redir_entries (int ioapic)
3687{
3688 union IO_APIC_reg_01 reg_01;
3689 unsigned long flags;
3690
3691 spin_lock_irqsave(&ioapic_lock, flags);
3692 reg_01.raw = io_apic_read(ioapic, 1);
3693 spin_unlock_irqrestore(&ioapic_lock, flags);
3694
3695 return reg_01.bits.entries;
3696}
3697
3698int __init probe_nr_irqs(void)
3699{
3700 int idx;
3701 int nr = 0;
Yinghai Lu052c0bf2008-08-21 13:10:09 -07003702#ifndef CONFIG_XEN
3703 int nr_min = 32;
3704#else
3705 int nr_min = NR_IRQS;
3706#endif
Yinghai Lu9d6a4d02008-08-19 20:50:52 -07003707
3708 for (idx = 0; idx < nr_ioapics; idx++)
Yinghai Lu052c0bf2008-08-21 13:10:09 -07003709 nr += io_apic_get_redir_entries(idx) + 1;
Yinghai Lu9d6a4d02008-08-19 20:50:52 -07003710
3711 /* double it for hotplug and msi and nmi */
3712 nr <<= 1;
3713
3714 /* something wrong ? */
Yinghai Lu052c0bf2008-08-21 13:10:09 -07003715 if (nr < nr_min)
3716 nr = nr_min;
Yinghai Lu9d6a4d02008-08-19 20:50:52 -07003717
3718 return nr;
3719}
3720
Linus Torvalds1da177e2005-04-16 15:20:36 -07003721/* --------------------------------------------------------------------------
Ingo Molnar54168ed2008-08-20 09:07:45 +02003722 ACPI-based IOAPIC Configuration
Linus Torvalds1da177e2005-04-16 15:20:36 -07003723 -------------------------------------------------------------------------- */
3724
Len Brown888ba6c2005-08-24 12:07:20 -04003725#ifdef CONFIG_ACPI
Linus Torvalds1da177e2005-04-16 15:20:36 -07003726
Ingo Molnar54168ed2008-08-20 09:07:45 +02003727#ifdef CONFIG_X86_32
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003728int __init io_apic_get_unique_id(int ioapic, int apic_id)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003729{
3730 union IO_APIC_reg_00 reg_00;
3731 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3732 physid_mask_t tmp;
3733 unsigned long flags;
3734 int i = 0;
3735
3736 /*
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003737 * The P4 platform supports up to 256 APIC IDs on two separate APIC
3738 * buses (one for LAPICs, one for IOAPICs), where predecessors only
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739 * supports up to 16 on one shared APIC bus.
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003740 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07003741 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
3742 * advantage of new APIC bus architecture.
3743 */
3744
3745 if (physids_empty(apic_id_map))
3746 apic_id_map = ioapic_phys_id_map(phys_cpu_present_map);
3747
3748 spin_lock_irqsave(&ioapic_lock, flags);
3749 reg_00.raw = io_apic_read(ioapic, 0);
3750 spin_unlock_irqrestore(&ioapic_lock, flags);
3751
3752 if (apic_id >= get_physical_broadcast()) {
3753 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3754 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3755 apic_id = reg_00.bits.ID;
3756 }
3757
3758 /*
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003759 * Every APIC in a system must have a unique ID or we get lots of nice
Linus Torvalds1da177e2005-04-16 15:20:36 -07003760 * 'stuck on smp_invalidate_needed IPI wait' messages.
3761 */
3762 if (check_apicid_used(apic_id_map, apic_id)) {
3763
3764 for (i = 0; i < get_physical_broadcast(); i++) {
3765 if (!check_apicid_used(apic_id_map, i))
3766 break;
3767 }
3768
3769 if (i == get_physical_broadcast())
3770 panic("Max apic_id exceeded!\n");
3771
3772 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3773 "trying %d\n", ioapic, apic_id, i);
3774
3775 apic_id = i;
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003776 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003777
3778 tmp = apicid_to_cpu_present(apic_id);
3779 physids_or(apic_id_map, apic_id_map, tmp);
3780
3781 if (reg_00.bits.ID != apic_id) {
3782 reg_00.bits.ID = apic_id;
3783
3784 spin_lock_irqsave(&ioapic_lock, flags);
3785 io_apic_write(ioapic, 0, reg_00.raw);
3786 reg_00.raw = io_apic_read(ioapic, 0);
3787 spin_unlock_irqrestore(&ioapic_lock, flags);
3788
3789 /* Sanity check */
Andreas Deresch6070f9e2006-02-26 04:18:34 +01003790 if (reg_00.bits.ID != apic_id) {
3791 printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
3792 return -1;
3793 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003794 }
3795
3796 apic_printk(APIC_VERBOSE, KERN_INFO
3797 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3798
3799 return apic_id;
3800}
3801
Paolo Ciarrocchi36062442008-06-08 13:07:18 +02003802int __init io_apic_get_version(int ioapic)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003803{
3804 union IO_APIC_reg_01 reg_01;
3805 unsigned long flags;
3806
3807 spin_lock_irqsave(&ioapic_lock, flags);
3808 reg_01.raw = io_apic_read(ioapic, 1);
3809 spin_unlock_irqrestore(&ioapic_lock, flags);
3810
3811 return reg_01.bits.version;
3812}
Ingo Molnar54168ed2008-08-20 09:07:45 +02003813#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003814
Ingo Molnar54168ed2008-08-20 09:07:45 +02003815int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003816{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003817 if (!IO_APIC_IRQ(irq)) {
Ingo Molnar54168ed2008-08-20 09:07:45 +02003818 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07003819 ioapic);
3820 return -EINVAL;
3821 }
3822
3823 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003824 * IRQs < 16 are already in the irq_2_pin[] map
3825 */
3826 if (irq >= 16)
3827 add_pin_to_irq(irq, ioapic, pin);
3828
Yinghai Lu497c9a12008-08-19 20:50:28 -07003829 setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003830
3831 return 0;
3832}
3833
Ingo Molnar54168ed2008-08-20 09:07:45 +02003834
Shaohua Li61fd47e2007-11-17 01:05:28 -05003835int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
3836{
3837 int i;
3838
3839 if (skip_ioapic_setup)
3840 return -1;
3841
3842 for (i = 0; i < mp_irq_entries; i++)
Alexey Starikovskiy2fddb6e282008-05-14 19:03:17 +04003843 if (mp_irqs[i].mp_irqtype == mp_INT &&
3844 mp_irqs[i].mp_srcbusirq == bus_irq)
Shaohua Li61fd47e2007-11-17 01:05:28 -05003845 break;
3846 if (i >= mp_irq_entries)
3847 return -1;
3848
3849 *trigger = irq_trigger(i);
3850 *polarity = irq_polarity(i);
3851 return 0;
3852}
3853
Len Brown888ba6c2005-08-24 12:07:20 -04003854#endif /* CONFIG_ACPI */
Rusty Russell1a3f2392006-09-26 10:52:32 +02003855
Yinghai Lu497c9a12008-08-19 20:50:28 -07003856/*
3857 * This function currently is only a helper for the i386 smp boot process where
3858 * we need to reprogram the ioredtbls to cater for the cpus which have come online
3859 * so mask in all cases should simply be TARGET_CPUS
3860 */
3861#ifdef CONFIG_SMP
3862void __init setup_ioapic_dest(void)
3863{
3864 int pin, ioapic, irq, irq_entry;
3865 struct irq_cfg *cfg;
Yinghai Lu497c9a12008-08-19 20:50:28 -07003866
3867 if (skip_ioapic_setup == 1)
3868 return;
3869
3870 for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
3871 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
3872 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3873 if (irq_entry == -1)
3874 continue;
3875 irq = pin_2_irq(irq_entry, ioapic, pin);
3876
3877 /* setup_IO_APIC_irqs could fail to get vector for some device
3878 * when you have too many devices, because at that time only boot
3879 * cpu is online.
3880 */
3881 cfg = irq_cfg(irq);
3882 if (!cfg->vector)
3883 setup_IO_APIC_irq(ioapic, pin, irq,
3884 irq_trigger(irq_entry),
3885 irq_polarity(irq_entry));
Ingo Molnar54168ed2008-08-20 09:07:45 +02003886#ifdef CONFIG_INTR_REMAP
3887 else if (intr_remapping_enabled)
3888 set_ir_ioapic_affinity_irq(irq, TARGET_CPUS);
3889#endif
3890 else
Yinghai Lu497c9a12008-08-19 20:50:28 -07003891 set_ioapic_affinity_irq(irq, TARGET_CPUS);
Yinghai Lu497c9a12008-08-19 20:50:28 -07003892 }
3893
3894 }
3895}
3896#endif
3897
Ingo Molnar54168ed2008-08-20 09:07:45 +02003898#define IOAPIC_RESOURCE_NAME_SIZE 11
3899
3900static struct resource *ioapic_resources;
3901
3902static struct resource * __init ioapic_setup_resources(void)
3903{
3904 unsigned long n;
3905 struct resource *res;
3906 char *mem;
3907 int i;
3908
3909 if (nr_ioapics <= 0)
3910 return NULL;
3911
3912 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3913 n *= nr_ioapics;
3914
3915 mem = alloc_bootmem(n);
3916 res = (void *)mem;
3917
3918 if (mem != NULL) {
3919 mem += sizeof(struct resource) * nr_ioapics;
3920
3921 for (i = 0; i < nr_ioapics; i++) {
3922 res[i].name = mem;
3923 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3924 sprintf(mem, "IOAPIC %u", i);
3925 mem += IOAPIC_RESOURCE_NAME_SIZE;
3926 }
3927 }
3928
3929 ioapic_resources = res;
3930
3931 return res;
3932}
Ingo Molnar54168ed2008-08-20 09:07:45 +02003933
Yinghai Luf3294a32008-06-27 01:41:56 -07003934void __init ioapic_init_mappings(void)
3935{
3936 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3937 int i;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003938 struct resource *ioapic_res;
Yinghai Luf3294a32008-06-27 01:41:56 -07003939
Ingo Molnar54168ed2008-08-20 09:07:45 +02003940 ioapic_res = ioapic_setup_resources();
Yinghai Luf3294a32008-06-27 01:41:56 -07003941 for (i = 0; i < nr_ioapics; i++) {
3942 if (smp_found_config) {
3943 ioapic_phys = mp_ioapics[i].mp_apicaddr;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003944#ifdef CONFIG_X86_32
3945 if (!ioapic_phys) {
3946 printk(KERN_ERR
3947 "WARNING: bogus zero IO-APIC "
3948 "address found in MPTABLE, "
3949 "disabling IO/APIC support!\n");
3950 smp_found_config = 0;
3951 skip_ioapic_setup = 1;
3952 goto fake_ioapic_page;
3953 }
3954#endif
Yinghai Luf3294a32008-06-27 01:41:56 -07003955 } else {
Ingo Molnar54168ed2008-08-20 09:07:45 +02003956#ifdef CONFIG_X86_32
Yinghai Luf3294a32008-06-27 01:41:56 -07003957fake_ioapic_page:
Ingo Molnar54168ed2008-08-20 09:07:45 +02003958#endif
Yinghai Luf3294a32008-06-27 01:41:56 -07003959 ioapic_phys = (unsigned long)
Ingo Molnar54168ed2008-08-20 09:07:45 +02003960 alloc_bootmem_pages(PAGE_SIZE);
Yinghai Luf3294a32008-06-27 01:41:56 -07003961 ioapic_phys = __pa(ioapic_phys);
3962 }
3963 set_fixmap_nocache(idx, ioapic_phys);
Ingo Molnar54168ed2008-08-20 09:07:45 +02003964 apic_printk(APIC_VERBOSE,
3965 "mapped IOAPIC to %08lx (%08lx)\n",
3966 __fix_to_virt(idx), ioapic_phys);
Yinghai Luf3294a32008-06-27 01:41:56 -07003967 idx++;
Ingo Molnar54168ed2008-08-20 09:07:45 +02003968
Ingo Molnar54168ed2008-08-20 09:07:45 +02003969 if (ioapic_res != NULL) {
3970 ioapic_res->start = ioapic_phys;
3971 ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
3972 ioapic_res++;
3973 }
Yinghai Luf3294a32008-06-27 01:41:56 -07003974 }
3975}
3976
Ingo Molnar54168ed2008-08-20 09:07:45 +02003977static int __init ioapic_insert_resources(void)
3978{
3979 int i;
3980 struct resource *r = ioapic_resources;
3981
3982 if (!r) {
3983 printk(KERN_ERR
3984 "IO APIC resources could be not be allocated.\n");
3985 return -1;
3986 }
3987
3988 for (i = 0; i < nr_ioapics; i++) {
3989 insert_resource(&iomem_resource, r);
3990 r++;
3991 }
3992
3993 return 0;
3994}
3995
3996/* Insert the IO APIC resources after PCI initialization has occured to handle
3997 * IO APICS that are mapped in on a BAR in PCI space. */
3998late_initcall(ioapic_insert_resources);