27 #include <linux/sched.h>
28 #include <linux/pci.h>
30 #include <linux/compiler.h>
32 #include <linux/module.h>
39 #include <linux/slab.h>
45 #include <linux/hpet.h>
55 #include <asm/timer.h>
56 #include <asm/i8259.h>
57 #include <asm/msidef.h>
59 #include <asm/setup.h>
60 #include <asm/irq_remapping.h>
62 #include <asm/hw_irq.h>
66 #define __apicdebuginit(type) static type __init
68 #define for_each_irq_pin(entry, head) \
69 for (entry = head; entry; entry = entry->next)
71 #ifdef CONFIG_IRQ_REMAP
72 static void irq_remap_modify_chip_defaults(
struct irq_chip *
chip);
73 static inline bool irq_remapped(
struct irq_cfg *
cfg)
75 return cfg->irq_2_iommu.iommu !=
NULL;
78 static inline bool irq_remapped(
struct irq_cfg *
cfg)
82 static inline void irq_remap_modify_chip_defaults(
struct irq_chip *
chip)
96 static struct ioapic {
108 struct mp_ioapic_gsi gsi_config;
112 #define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver
116 return ioapics[ioapic_idx].mp_config.apicid;
121 return ioapics[ioapic_idx].mp_config.apicaddr;
126 return &ioapics[ioapic_idx].gsi_config;
163 static int __init parse_noapic(
char *
str)
171 static int io_apic_setup_irq_pin(
unsigned int irq,
int node,
180 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
191 panic(
"Max # of irq sources exceeded!!\n");
217 ioapics[
i].saved_registers =
220 if (!ioapics[i].saved_registers)
221 pr_err(
"IOAPIC %d: suspend/resume impossible!\n", i);
231 for (i = 0; i <
count; i++) {
239 if (i < legacy_pic->nr_legacy_irqs) {
241 cpumask_set_cpu(0, cfg[i].domain);
250 return irq_get_chip_data(irq);
253 static struct irq_cfg *alloc_irq_cfg(
unsigned int irq,
int node)
257 cfg = kzalloc_node(
sizeof(*cfg),
GFP_KERNEL, node);
266 free_cpumask_var(cfg->
domain);
272 static void free_irq_cfg(
unsigned int at,
struct irq_cfg *cfg)
277 free_cpumask_var(cfg->
domain);
282 static struct irq_cfg *alloc_irq_and_cfg_at(
unsigned int at,
int node)
284 int res = irq_alloc_desc_at(at, node);
290 cfg = irq_get_chip_data(at);
295 cfg = alloc_irq_cfg(at, node);
303 static int alloc_irq_from(
unsigned int from,
int node)
305 return irq_alloc_desc_from(from, node);
308 static void free_irq_at(
unsigned int at,
struct irq_cfg *cfg)
310 free_irq_cfg(at, cfg);
329 static inline void io_apic_eoi(
unsigned int apic,
unsigned int vector)
374 eu.
w1 = io_apic_read(
apic, 0x10 + 2 *
pin);
375 eu.w2 = io_apic_read(
apic, 0x11 + 2 *
pin);
386 eu.entry = __ioapic_read_entry(
apic,
pin);
403 io_apic_write(apic, 0x11 + 2*pin, eu.
w2);
404 io_apic_write(apic, 0x10 + 2*pin, eu.
w1);
412 __ioapic_write_entry(apic, pin, e);
421 static void ioapic_mask_entry(
int apic,
int pin)
427 io_apic_write(apic, 0x10 + 2*pin, eu.
w1);
428 io_apic_write(apic, 0x11 + 2*pin, eu.
w2);
437 static int __add_pin_to_irq_node(
struct irq_cfg *cfg,
int node,
int apic,
int pin)
444 if (entry->
apic == apic && entry->
pin == pin)
449 entry = alloc_irq_pin_list(node);
451 pr_err(
"can not alloc irq_pin_list (%d,%d,%d)\n",
462 static void add_pin_to_irq_node(
struct irq_cfg *cfg,
int node,
int apic,
int pin)
464 if (__add_pin_to_irq_node(cfg, node, apic, pin))
465 panic(
"IO-APIC: failed to add irq-pin. Can not proceed\n");
471 static void __init replace_pin_at_irq_node(
struct irq_cfg *cfg,
int node,
472 int oldapic,
int oldpin,
473 int newapic,
int newpin)
478 if (entry->
apic == oldapic && entry->
pin == oldpin) {
479 entry->
apic = newapic;
487 add_pin_to_irq_node(cfg, node, newapic, newpin);
490 static void __io_apic_modify_irq(
struct irq_pin_list *entry,
491 int mask_and,
int mask_or,
497 reg = io_apic_read(entry->
apic, 0x10 + pin * 2);
500 io_apic_modify(entry->
apic, 0x10 + pin * 2, reg);
505 static void io_apic_modify_irq(
struct irq_cfg *cfg,
506 int mask_and,
int mask_or,
512 __io_apic_modify_irq(entry, mask_and, mask_or, final);
523 io_apic = io_apic_base(entry->apic);
527 static void mask_ioapic(
struct irq_cfg *cfg)
541 static void __unmask_ioapic(
struct irq_cfg *cfg)
546 static void unmask_ioapic(
struct irq_cfg *cfg)
551 __unmask_ioapic(cfg);
576 static void __eoi_ioapic_pin(
int apic,
int pin,
int vector,
struct irq_cfg *cfg)
585 if (cfg && irq_remapped(cfg))
586 io_apic_eoi(apic, pin);
588 io_apic_eoi(apic, vector);
592 entry = entry1 = __ioapic_read_entry(apic, pin);
600 __ioapic_write_entry(apic, pin, entry1);
605 __ioapic_write_entry(apic, pin, entry);
609 static void eoi_ioapic_irq(
unsigned int irq,
struct irq_cfg *cfg)
616 __eoi_ioapic_pin(entry->apic, entry->pin, cfg->vector, cfg);
620 static
void clear_IO_APIC_pin(
unsigned int apic,
unsigned int pin)
625 entry = ioapic_read_entry(apic, pin);
626 if (entry.delivery_mode ==
dest_SMI)
635 ioapic_write_entry(apic, pin, entry);
636 entry = ioapic_read_entry(apic, pin);
647 if (!entry.trigger) {
649 ioapic_write_entry(apic, pin, entry);
653 __eoi_ioapic_pin(apic, pin, entry.vector,
NULL);
661 ioapic_mask_entry(apic, pin);
662 entry = ioapic_read_entry(apic, pin);
664 pr_err(
"Unable to reset IRR for apic: %d, pin :%d\n",
668 static void clear_IO_APIC (
void)
673 for (pin = 0; pin < ioapics[
apic].nr_registers; pin++)
674 clear_IO_APIC_pin(apic, pin);
684 static int pirq_entries[MAX_PIRQS] = {
685 [0 ... MAX_PIRQS - 1] = -1
688 static int __init ioapic_pirq_setup(
char *
str)
691 int ints[MAX_PIRQS+1];
696 "PIRQ redirection, working around broken MP-BIOS.\n");
698 if (ints[0] < MAX_PIRQS)
701 for (i = 0; i <
max; i++) {
703 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
707 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
712 __setup(
"pirq=", ioapic_pirq_setup);
724 if (!ioapics[apic].saved_registers) {
729 for (pin = 0; pin < ioapics[
apic].nr_registers; pin++)
730 ioapics[apic].saved_registers[pin] =
731 ioapic_read_entry(apic, pin);
745 if (!ioapics[apic].saved_registers)
748 for (pin = 0; pin < ioapics[
apic].nr_registers; pin++) {
751 entry = ioapics[
apic].saved_registers[
pin];
754 ioapic_write_entry(apic, pin, entry);
768 if (!ioapics[apic].saved_registers)
771 for (pin = 0; pin < ioapics[
apic].nr_registers; pin++)
772 ioapic_write_entry(apic, pin,
773 ioapics[apic].saved_registers[pin]);
781 static int find_irq_entry(
int ioapic_idx,
int pin,
int type)
786 if (
mp_irqs[i].irqtype == type &&
798 static int __init find_isa_irq_pin(
int irq,
int type)
805 if (
test_bit(lbus, mp_bus_not_pci) &&
806 (
mp_irqs[i].irqtype == type) &&
814 static int __init find_isa_irq_apic(
int irq,
int type)
821 if (
test_bit(lbus, mp_bus_not_pci) &&
822 (
mp_irqs[i].irqtype == type) &&
827 if (i < mp_irq_entries) {
830 for (ioapic_idx = 0; ioapic_idx <
nr_ioapics; ioapic_idx++)
842 static int EISA_ELCR(
unsigned int irq)
844 if (irq < legacy_pic->nr_legacy_irqs) {
845 unsigned int port = 0x4d0 + (irq >> 3);
846 return (
inb(port) >> (irq & 7)) & 1;
849 "Broken MPtable reports ISA irq %d\n", irq);
858 #define default_ISA_trigger(idx) (0)
859 #define default_ISA_polarity(idx) (0)
866 #define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq))
867 #define default_EISA_polarity(idx) default_ISA_polarity(idx)
872 #define default_PCI_trigger(idx) (1)
873 #define default_PCI_polarity(idx) (1)
875 static int irq_polarity(
int idx)
883 switch (
mp_irqs[idx].irqflag & 3)
917 static int irq_trigger(
int idx)
925 switch ((
mp_irqs[idx].irqflag>>2) & 3)
933 switch (mp_bus_id_to_type[bus]) {
984 static int pin_2_irq(
int idx,
int apic,
int pin)
993 if (
mp_irqs[idx].dstirq != pin)
994 pr_err(
"broken BIOS or MPTABLE parser, ayiee!!\n");
996 if (
test_bit(bus, mp_bus_not_pci)) {
1007 #ifdef CONFIG_X86_32
1011 if ((pin >= 16) && (pin <= 23)) {
1012 if (pirq_entries[pin-16] != -1) {
1013 if (!pirq_entries[pin-16]) {
1015 "disabling PIRQ%d\n", pin-16);
1017 irq = pirq_entries[pin-16];
1019 "using PIRQ%d -> IRQ %d\n",
1036 int ioapic_idx,
i, best_guess = -1;
1039 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1041 if (
test_bit(bus, mp_bus_not_pci)) {
1043 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1049 for (ioapic_idx = 0; ioapic_idx <
nr_ioapics; ioapic_idx++)
1054 if (!
test_bit(lbus, mp_bus_not_pci) &&
1057 (slot == ((
mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1058 int irq = pin_2_irq(i, ioapic_idx,
mp_irqs[i].dstirq);
1063 if (pin == (
mp_irqs[i].srcbusirq & 3)) {
1064 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1074 if (best_guess < 0) {
1075 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1115 static int current_offset = VECTOR_OFFSET_START % 16;
1122 if (!alloc_cpumask_var(&tmp_mask,
GFP_ATOMIC))
1129 while (cpu < nr_cpu_ids) {
1132 apic->vector_allocation_domain(cpu, tmp_mask, mask);
1134 if (cpumask_subset(tmp_mask, cfg->
domain)) {
1136 if (cpumask_equal(tmp_mask, cfg->
domain))
1149 vector = current_vector;
1150 offset = current_offset;
1154 offset = (offset + 1) % 16;
1158 if (
unlikely(current_vector == vector)) {
1160 cpumask_andnot(tmp_mask, mask, cfg->
old_domain);
1165 if (
test_bit(vector, used_vectors))
1169 if (
per_cpu(vector_irq, new_cpu)[vector] != -1)
1172 current_vector = vector;
1173 current_offset = offset;
1179 per_cpu(vector_irq, new_cpu)[vector] = irq;
1180 cfg->vector = vector;
1181 cpumask_copy(cfg->domain, tmp_mask);
1185 free_cpumask_var(tmp_mask);
1192 unsigned long flags;
1195 err = __assign_irq_vector(irq, cfg, mask);
1200 static void __clear_irq_vector(
int irq,
struct irq_cfg *cfg)
1208 per_cpu(vector_irq, cpu)[vector] = -1;
1211 cpumask_clear(cfg->domain);
1213 if (
likely(!cfg->move_in_progress))
1218 if (
per_cpu(vector_irq, cpu)[vector] != irq)
1240 for_each_active_irq(irq) {
1241 cfg = irq_get_chip_data(irq);
1248 if (irq < legacy_pic->nr_legacy_irqs && !
IO_APIC_IRQ(irq))
1249 cpumask_set_cpu(cpu, cfg->
domain);
1269 static struct irq_chip ioapic_chip;
1271 #ifdef CONFIG_X86_32
1272 static inline int IO_APIC_irq_trigger(
int irq)
1277 for (pin = 0; pin < ioapics[
apic].nr_registers; pin++) {
1278 idx = find_irq_entry(apic, pin,
mp_INT);
1279 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1280 return irq_trigger(idx);
1289 static inline int IO_APIC_irq_trigger(
int irq)
1295 static void ioapic_register_intr(
unsigned int irq,
struct irq_cfg *cfg,
1296 unsigned long trigger)
1302 if ((trigger ==
IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1311 if (irq_remapped(cfg)) {
1313 irq_remap_modify_chip_defaults(chip);
1314 fasteoi = trigger != 0;
1319 fasteoi ?
"fasteoi" :
"edge");
1323 unsigned int destination,
int vector,
1330 memset(entry, 0,
sizeof(*entry));
1334 entry->
dest = destination;
1350 static void setup_ioapic_irq(
unsigned int irq,
struct irq_cfg *cfg,
1364 if (irq < legacy_pic->nr_legacy_irqs &&
1366 apic->vector_allocation_domain(0, cfg->
domain,
1367 apic->target_cpus());
1372 if (apic->cpu_mask_to_apicid_and(cfg->
domain, apic->target_cpus(),
1374 pr_warn(
"Failed to obtain apicid for ioapic %d, pin %d\n",
1376 __clear_irq_vector(irq, cfg);
1382 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1383 "IRQ %d Mode:%i Active:%i Dest:%d)\n",
1387 if (setup_ioapic_entry(irq, &entry, dest, cfg->
vector, attr)) {
1388 pr_warn(
"Failed to setup ioapic entry for ioapic %d, pin %d\n",
1390 __clear_irq_vector(irq, cfg);
1395 ioapic_register_intr(irq, cfg, attr->
trigger);
1396 if (irq < legacy_pic->nr_legacy_irqs)
1402 static bool __init io_apic_pin_not_connected(
int idx,
int ioapic_idx,
int pin)
1412 static void __init __io_apic_setup_irqs(
unsigned int ioapic_idx)
1416 unsigned int pin, irq;
1418 for (pin = 0; pin < ioapics[ioapic_idx].nr_registers; pin++) {
1419 idx = find_irq_entry(ioapic_idx, pin,
mp_INT);
1420 if (io_apic_pin_not_connected(idx, ioapic_idx, pin))
1423 irq = pin_2_irq(idx, ioapic_idx, pin);
1425 if ((ioapic_idx > 0) && (irq > 16))
1432 if (apic->multi_timer_check &&
1433 apic->multi_timer_check(ioapic_idx, irq))
1436 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1439 io_apic_setup_irq_pin(irq, node, &attr);
1443 static void __init setup_IO_APIC_irqs(
void)
1445 unsigned int ioapic_idx;
1449 for (ioapic_idx = 0; ioapic_idx <
nr_ioapics; ioapic_idx++)
1450 __io_apic_setup_irqs(ioapic_idx);
1471 idx = find_irq_entry(ioapic_idx, pin,
mp_INT);
1475 irq = pin_2_irq(idx, ioapic_idx, pin);
1481 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1490 static void __init setup_timer_IRQ0_pin(
unsigned int ioapic_idx,
1491 unsigned int pin,
int vector)
1499 memset(&entry, 0,
sizeof(entry));
1505 if (
unlikely(apic->cpu_mask_to_apicid_and(apic->target_cpus(),
1506 apic->target_cpus(), &
dest)))
1527 ioapic_write_entry(ioapic_idx, pin, entry);
1537 unsigned long flags;
1540 reg_00.
raw = io_apic_read(ioapic_idx, 0);
1541 reg_01.
raw = io_apic_read(ioapic_idx, 1);
1542 if (reg_01.bits.
version >= 0x10)
1543 reg_02.
raw = io_apic_read(ioapic_idx, 2);
1544 if (reg_01.bits.
version >= 0x20)
1545 reg_03.
raw = io_apic_read(ioapic_idx, 3);
1567 if (reg_01.bits.
version >= 0x10 && reg_02.
raw != reg_01.
raw) {
1577 if (reg_01.bits.
version >= 0x20 && reg_03.
raw != reg_02.
raw &&
1578 reg_03.
raw != reg_01.
raw) {
1587 " Pol Stat Indx2 Zero Vect:\n");
1590 " Stat Dmod Deli Vect:\n");
1593 for (i = 0; i <= reg_01.bits.
entries; i++) {
1598 entry = ioapic_read_entry(ioapic_idx, i);
1604 pr_cont(
"%1d %1d %1d %1d %1d "
1605 "%1d %1d %X %02X\n",
1619 entry = ioapic_read_entry(ioapic_idx, i);
1624 pr_cont(
"%1d %1d %1d %1d %1d "
1630 entry.delivery_status,
1632 entry.delivery_mode,
1647 for (ioapic_idx = 0; ioapic_idx <
nr_ioapics; ioapic_idx++)
1650 ioapics[ioapic_idx].nr_registers);
1658 for (ioapic_idx = 0; ioapic_idx <
nr_ioapics; ioapic_idx++)
1659 print_IO_APIC(ioapic_idx);
1662 for_each_active_irq(irq) {
1665 chip = irq_get_chip(irq);
1666 if (chip != &ioapic_chip)
1669 cfg = irq_get_chip_data(irq);
1677 pr_cont("-> %
d:%
d", entry->apic, entry->pin);
1690 for (i = 0; i < 8; i++)
1691 pr_cont(
"%08x", apic_read(base + i*0x10));
1698 unsigned int i,
v,
ver, maxlvt;
1734 if (!x2apic_enabled()) {
1756 icr = apic_icr_read();
1786 maxlvt = (v >> 16) & 0xff;
1790 for (i = 0; i < maxlvt; i++) {
1817 unsigned long flags;
1826 v =
inb(0xa1) << 8 |
inb(0x21);
1829 v =
inb(0xa0) << 8 |
inb(0x20);
1834 v =
inb(0xa0) << 8 |
inb(0x20);
1842 v =
inb(0x4d1) << 8 |
inb(0x4d0);
1847 static __init int setup_show_lapic(
char *
arg)
1851 if (
strcmp(arg,
"all") == 0) {
1861 __setup(
"show_lapic=", setup_show_lapic);
1871 if (!cpu_has_apic && !apic_from_smp_config())
1874 print_local_APICs(show_lapic);
1884 static struct {
int pin,
apic; } ioapic_i8259 = { -1, -1 };
1888 int i8259_apic, i8259_pin;
1897 for (pin = 0; pin < ioapics[
apic].nr_registers; pin++) {
1899 entry = ioapic_read_entry(apic, pin);
1904 if ((entry.mask == 0) && (entry.delivery_mode ==
dest_ExtINT)) {
1905 ioapic_i8259.apic =
apic;
1906 ioapic_i8259.pin =
pin;
1917 i8259_pin = find_isa_irq_pin(0,
mp_ExtINT);
1918 i8259_apic = find_isa_irq_apic(0,
mp_ExtINT);
1920 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1922 ioapic_i8259.pin = i8259_pin;
1923 ioapic_i8259.apic = i8259_apic;
1926 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1927 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1964 memset(&entry, 0,
sizeof(entry));
1973 entry.
dest = read_apic_id();
1978 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1984 if (cpu_has_apic || apic_from_smp_config())
1986 ioapic_i8259.pin != -1);
1989 #ifdef CONFIG_X86_32
1996 void __init setup_ioapic_ids_from_mpc_nocheck(
void)
2002 unsigned char old_id;
2003 unsigned long flags;
2014 for (ioapic_idx = 0; ioapic_idx <
nr_ioapics; ioapic_idx++) {
2017 reg_00.raw = io_apic_read(ioapic_idx, 0);
2022 if (
mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
2023 printk(
KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
2027 ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
2035 if (apic->check_apicid_used(&phys_id_present_map,
2037 printk(
KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
2039 for (i = 0; i < get_physical_broadcast(); i++)
2042 if (i >= get_physical_broadcast())
2043 panic(
"Max APIC ID exceeded!\n");
2047 ioapics[ioapic_idx].mp_config.apicid =
i;
2053 "phys_id_present_map\n",
2055 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2064 if (
mp_irqs[i].dstapic == old_id)
2076 "...changing IO-APIC physical APIC ID to %d ...",
2081 io_apic_write(ioapic_idx, 0, reg_00.raw);
2088 reg_00.raw = io_apic_read(ioapic_idx, 0);
2091 pr_cont(
"could not set ID!\n");
2109 setup_ioapic_ids_from_mpc_nocheck();
2115 static int __init notimercheck(
char *
s)
2120 __setup(
"no_timer_check", notimercheck);
2130 static int __init timer_irq_works(
void)
2133 unsigned long flags;
2181 static unsigned int startup_ioapic_irq(
struct irq_data *
data)
2183 int was_pending = 0, irq = data->
irq;
2184 unsigned long flags;
2187 if (irq < legacy_pic->nr_legacy_irqs) {
2198 static int ioapic_retrigger_irq(
struct irq_data *data)
2201 unsigned long flags;
2229 cpumask_and(cleanup_mask, cfg->
old_domain, cpu_online_mask);
2230 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2231 free_cpumask_var(cleanup_mask);
2236 asmlinkage void smp_irq_move_cleanup_interrupt(
void)
2275 irr = apic_read(
APIC_IRR + (vector / 32 * 0x10));
2283 if (irr & (1 << (vector % 32))) {
2295 static void __irq_complete_move(
struct irq_cfg *cfg,
unsigned vector)
2308 static void irq_complete_move(
struct irq_cfg *cfg)
2313 void irq_force_complete_move(
int irq)
2315 struct irq_cfg *cfg = irq_get_chip_data(irq);
2320 __irq_complete_move(cfg, cfg->
vector);
2323 static inline void irq_complete_move(
struct irq_cfg *cfg) { }
2326 static void __target_IO_APIC_irq(
unsigned int irq,
unsigned int dest,
struct irq_cfg *cfg)
2341 if (!irq_remapped(cfg))
2342 io_apic_write(apic, 0x11 + pin*2, dest);
2343 reg = io_apic_read(apic, 0x10 + pin*2);
2346 io_apic_modify(apic, 0x10 + pin*2, reg);
2359 unsigned int irq = data->
irq;
2365 if (!cpumask_intersects(mask, cpu_online_mask))
2372 err = apic->cpu_mask_to_apicid_and(mask, cfg->
domain, dest_id);
2375 pr_err(
"Failed to recover vector for irq %d\n", irq);
2379 cpumask_copy(data->
affinity, mask);
2385 ioapic_set_affinity(
struct irq_data *data,
const struct cpumask *mask,
2388 unsigned int dest, irq = data->
irq;
2389 unsigned long flags;
2400 __target_IO_APIC_irq(irq, dest, data->
chip_data);
2407 static void ack_apic_edge(
struct irq_data *data)
2416 #ifdef CONFIG_GENERIC_PENDING_IRQ
2417 static bool io_apic_level_ack_pending(
struct irq_cfg *cfg)
2420 unsigned long flags;
2428 reg = io_apic_read(entry->
apic, 0x10 + pin*2);
2440 static inline bool ioapic_irqd_mask(
struct irq_data *data,
struct irq_cfg *cfg)
2443 if (
unlikely(irqd_is_setaffinity_pending(data))) {
2450 static inline void ioapic_irqd_unmask(
struct irq_data *data,
2451 struct irq_cfg *cfg,
bool masked)
2480 if (!io_apic_level_ack_pending(cfg))
2486 static inline bool ioapic_irqd_mask(
struct irq_data *data,
struct irq_cfg *cfg)
2490 static inline void ioapic_irqd_unmask(
struct irq_data *data,
2491 struct irq_cfg *cfg,
bool masked)
2496 static void ack_apic_level(
struct irq_data *data)
2499 int i, irq = data->
irq;
2503 irq_complete_move(cfg);
2504 masked = ioapic_irqd_mask(data, cfg);
2539 v = apic_read(
APIC_TMR + ((i & ~0x1f) >> 1));
2554 if (!(v & (1 << (i & 0x1f)))) {
2557 eoi_ioapic_irq(irq, cfg);
2560 ioapic_irqd_unmask(data, cfg, masked);
2563 #ifdef CONFIG_IRQ_REMAP
2564 static void ir_ack_apic_edge(
struct irq_data *data)
2569 static void ir_ack_apic_level(
struct irq_data *data)
2580 static void irq_remap_modify_chip_defaults(
struct irq_chip *chip)
2583 chip->
irq_ack = ir_ack_apic_edge;
2584 chip->
irq_eoi = ir_ack_apic_level;
2592 .irq_startup = startup_ioapic_irq,
2593 .irq_mask = mask_ioapic_irq,
2594 .irq_unmask = unmask_ioapic_irq,
2595 .irq_ack = ack_apic_edge,
2596 .irq_eoi = ack_apic_level,
2597 .irq_set_affinity = ioapic_set_affinity,
2598 .irq_retrigger = ioapic_retrigger_irq,
2601 static inline void init_IO_APIC_traps(
void)
2617 for_each_active_irq(irq) {
2618 cfg = irq_get_chip_data(irq);
2625 if (irq < legacy_pic->nr_legacy_irqs)
2638 static void mask_lapic_irq(
struct irq_data *data)
2646 static void unmask_lapic_irq(
struct irq_data *data)
2654 static void ack_lapic_irq(
struct irq_data *data)
2659 static struct irq_chip lapic_chip __read_mostly = {
2660 .
name =
"local-APIC",
2661 .irq_mask = mask_lapic_irq,
2662 .irq_unmask = unmask_lapic_irq,
2663 .irq_ack = ack_lapic_irq,
2666 static void lapic_register_intr(
int irq)
2680 static inline void __init unlock_ExtINT_logic(
void)
2684 unsigned char save_control, save_freq_select;
2686 pin = find_isa_irq_pin(8,
mp_INT);
2691 apic = find_isa_irq_apic(8,
mp_INT);
2697 entry0 = ioapic_read_entry(apic, pin);
2698 clear_IO_APIC_pin(apic, pin);
2700 memset(&entry1, 0,
sizeof(entry1));
2702 entry1.dest_mode = 0;
2706 entry1.polarity = entry0.polarity;
2710 ioapic_write_entry(apic, pin, entry1);
2727 clear_IO_APIC_pin(apic, pin);
2729 ioapic_write_entry(apic, pin, entry0);
2734 static int __init disable_timer_pin_setup(
char *arg)
2736 disable_timer_pin_1 = 1;
2739 early_param(
"disable_timer_pin_1", disable_timer_pin_setup);
2751 static inline void __init check_timer(
void)
2753 struct irq_cfg *cfg = irq_get_chip_data(0);
2755 int apic1, pin1, apic2, pin2;
2756 unsigned long flags;
2779 pin1 = find_isa_irq_pin(0,
mp_INT);
2780 apic1 = find_isa_irq_apic(0,
mp_INT);
2781 pin2 = ioapic_i8259.pin;
2782 apic2 = ioapic_i8259.apic;
2785 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2786 cfg->
vector, apic1, pin1, apic2, pin2);
2797 panic(
"BIOS bug: timer not connected to IO-APIC");
2801 }
else if (pin2 == -1) {
2811 add_pin_to_irq_node(cfg, node, apic1, pin1);
2812 setup_timer_IRQ0_pin(apic1, pin1, cfg->
vector);
2820 idx = find_irq_entry(apic1, pin1,
mp_INT);
2821 if (idx != -1 && irq_trigger(idx))
2824 if (timer_irq_works()) {
2825 if (disable_timer_pin_1 > 0)
2826 clear_IO_APIC_pin(0, pin1);
2830 panic(
"timer doesn't work through Interrupt-remapped IO-APIC");
2832 clear_IO_APIC_pin(apic1, pin1);
2835 "8254 timer not connected to IO-APIC\n");
2838 "(IRQ0) through the 8259A ...\n");
2840 "..... (found apic %d pin %d) ...\n", apic2, pin2);
2844 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2845 setup_timer_IRQ0_pin(apic2, pin2, cfg->
vector);
2847 if (timer_irq_works()) {
2849 timer_through_8259 = 1;
2857 clear_IO_APIC_pin(apic2, pin2);
2862 "...trying to set up timer as Virtual Wire IRQ...\n");
2864 lapic_register_intr(0);
2868 if (timer_irq_works()) {
2878 "...trying to set up timer as ExtINT IRQ...\n");
2884 unlock_ExtINT_logic();
2886 if (timer_irq_works()) {
2892 if (x2apic_preenabled)
2894 "Perhaps problem with the pre-enabled x2apic mode\n"
2895 "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2896 panic(
"IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2897 "report. Then try booting with the 'noapic' option.\n");
2919 #define PIC_IRQS (1UL << PIC_CASCADE_IR)
2933 x86_init.mpparse.setup_ioapic_ids();
2936 setup_IO_APIC_irqs();
2937 init_IO_APIC_traps();
2947 static int __init io_apic_bug_finalize(
void)
2956 static void resume_ioapic_id(
int ioapic_idx)
2958 unsigned long flags;
2962 reg_00.raw = io_apic_read(ioapic_idx, 0);
2965 io_apic_write(ioapic_idx, 0, reg_00.raw);
2970 static void ioapic_resume(
void)
2974 for (ioapic_idx = nr_ioapics - 1; ioapic_idx >= 0; ioapic_idx--)
2975 resume_ioapic_id(ioapic_idx);
2982 .resume = ioapic_resume,
2985 static int __init ioapic_init_ops(
void)
3000 unsigned long flags;
3001 unsigned int ret = 0;
3004 if (from < nr_irqs_gsi)
3007 irq = alloc_irq_from(from, node);
3010 cfg = alloc_irq_cfg(irq, node);
3012 free_irq_at(irq,
NULL);
3017 if (!__assign_irq_vector(irq, cfg, apic->target_cpus()))
3025 free_irq_at(irq, cfg);
3033 unsigned int irq_want;
3036 irq_want = nr_irqs_gsi;
3047 struct irq_cfg *cfg = irq_get_chip_data(irq);
3048 unsigned long flags;
3052 if (irq_remapped(cfg))
3055 __clear_irq_vector(irq, cfg);
3057 free_irq_at(irq, cfg);
3063 #ifdef CONFIG_PCI_MSI
3064 static int msi_compose_msg(
struct pci_dev *pdev,
unsigned int irq,
3079 err = apic->cpu_mask_to_apicid_and(cfg->
domain,
3080 apic->target_cpus(), &
dest);
3084 if (irq_remapped(cfg)) {
3089 if (x2apic_enabled())
3097 ((apic->irq_dest_mode == 0) ?
3117 msi_set_affinity(
struct irq_data *data,
const struct cpumask *mask,
bool force)
3142 static struct irq_chip msi_chip = {
3146 .irq_ack = ack_apic_edge,
3147 .irq_set_affinity = msi_set_affinity,
3148 .irq_retrigger = ioapic_retrigger_irq,
3157 ret = msi_compose_msg(dev, irq, &msg, -1);
3164 if (irq_remapped(irq_get_chip_data(irq))) {
3166 irq_remap_modify_chip_defaults(chip);
3171 dev_printk(
KERN_DEBUG, &dev->
dev,
"irq %d for MSI/MSI-X\n", irq);
3176 int native_setup_msi_irqs(
struct pci_dev *dev,
int nvec,
int type)
3179 unsigned int irq, irq_want;
3186 node = dev_to_node(&dev->
dev);
3187 irq_want = nr_irqs_gsi;
3214 ret = setup_msi_irq(dev, msidesc, irq);
3226 void native_teardown_msi_irq(
unsigned int irq)
3231 #ifdef CONFIG_DMAR_TABLE
3233 dmar_msi_set_affinity(
struct irq_data *data,
const struct cpumask *mask,
3237 unsigned int dest, irq = data->
irq;
3256 static struct irq_chip dmar_msi_type = {
3260 .irq_ack = ack_apic_edge,
3261 .irq_set_affinity = dmar_msi_set_affinity,
3262 .irq_retrigger = ioapic_retrigger_irq,
3270 ret = msi_compose_msg(
NULL, irq, &msg, -1);
3280 #ifdef CONFIG_HPET_TIMER
3282 static int hpet_msi_set_affinity(
struct irq_data *data,
3283 const struct cpumask *mask,
bool force)
3304 static struct irq_chip hpet_msi_type = {
3306 .irq_unmask = hpet_msi_unmask,
3307 .irq_mask = hpet_msi_mask,
3308 .irq_ack = ack_apic_edge,
3309 .irq_set_affinity = hpet_msi_set_affinity,
3310 .irq_retrigger = ioapic_retrigger_irq,
3313 int arch_setup_hpet_msi(
unsigned int irq,
unsigned int id)
3315 struct irq_chip *chip = &hpet_msi_type;
3324 ret = msi_compose_msg(
NULL, irq, &msg,
id);
3328 hpet_msi_write(irq_get_handler_data(irq), &msg);
3330 if (irq_remapped(irq_get_chip_data(irq)))
3331 irq_remap_modify_chip_defaults(chip);
3342 #ifdef CONFIG_HT_IRQ
3344 static void target_ht_irq(
unsigned int irq,
unsigned int dest,
u8 vector)
3359 ht_set_affinity(
struct irq_data *data,
const struct cpumask *mask,
bool force)
3367 target_ht_irq(data->
irq, dest, cfg->
vector);
3371 static struct irq_chip ht_irq_chip = {
3375 .irq_ack = ack_apic_edge,
3376 .irq_set_affinity = ht_set_affinity,
3377 .irq_retrigger = ioapic_retrigger_irq,
3395 err = apic->cpu_mask_to_apicid_and(cfg->
domain,
3396 apic->target_cpus(), &
dest);
3406 ((apic->irq_dest_mode == 0) ?
3427 io_apic_setup_irq_pin(
unsigned int irq,
int node,
struct io_apic_irq_attr *attr)
3429 struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
3436 setup_ioapic_irq(irq, cfg, attr);
3447 if (
test_bit(pin, ioapics[ioapic_idx].pin_programmed)) {
3448 pr_debug(
"Pin %d-%d already programmed\n",
3452 ret = io_apic_setup_irq_pin(irq, node, attr);
3454 set_bit(pin, ioapics[ioapic_idx].pin_programmed);
3458 static int __init io_apic_get_redir_entries(
int ioapic)
3461 unsigned long flags;
3464 reg_01.raw = io_apic_read(ioapic, 1);
3471 return reg_01.bits.entries + 1;
3474 static void __init probe_nr_irqs_gsi(
void)
3479 if (nr > nr_irqs_gsi)
3495 nr_irqs = NR_VECTORS * nr_cpu_ids;
3497 nr = nr_irqs_gsi + 8 * nr_cpu_ids;
3498 #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
3502 nr += nr_irqs_gsi * 16;
3526 #ifdef CONFIG_X86_32
3527 static int __init io_apic_get_unique_id(
int ioapic,
int apic_id)
3532 unsigned long flags;
3548 reg_00.raw = io_apic_read(ioapic, 0);
3551 if (apic_id >= get_physical_broadcast()) {
3553 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3554 apic_id = reg_00.bits.ID;
3561 if (apic->check_apicid_used(&apic_id_map, apic_id)) {
3563 for (i = 0; i < get_physical_broadcast(); i++) {
3564 if (!apic->check_apicid_used(&apic_id_map, i))
3568 if (i == get_physical_broadcast())
3569 panic(
"Max apic_id exceeded!\n");
3572 "trying %d\n", ioapic, apic_id, i);
3577 apic->apicid_to_cpu_present(apic_id, &tmp);
3580 if (reg_00.bits.ID != apic_id) {
3581 reg_00.bits.ID = apic_id;
3584 io_apic_write(ioapic, 0, reg_00.raw);
3585 reg_00.raw = io_apic_read(ioapic, 0);
3589 if (reg_00.bits.ID != apic_id) {
3590 pr_err(
"IOAPIC[%d]: Unable to change apic_id!\n",
3597 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3606 return io_apic_get_unique_id(nr_ioapics,
id);
3616 bitmap_zero(
used, 256);
3626 static int __init io_apic_get_version(
int ioapic)
3629 unsigned long flags;
3632 reg_01.raw = io_apic_read(ioapic, 1);
3635 return reg_01.bits.version;
3653 idx = find_irq_entry(ioapic, pin,
mp_INT);
3657 *trigger = irq_trigger(idx);
3658 *polarity = irq_polarity(idx);
3670 int pin, ioapic, irq, irq_entry;
3677 for (ioapic = 0; ioapic <
nr_ioapics; ioapic++)
3678 for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3679 irq_entry = find_irq_entry(ioapic, pin,
mp_INT);
3680 if (irq_entry == -1)
3682 irq = pin_2_irq(irq_entry, ioapic, pin);
3684 if ((ioapic > 0) && (irq > 16))
3692 if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
3695 mask = apic->target_cpus();
3700 ioapic_set_affinity(idata, mask,
false);
3706 #define IOAPIC_RESOURCE_NAME_SIZE 11
3708 static struct resource *ioapic_resources;
3710 static struct resource *
__init ioapic_setup_resources(
int nr_ioapics)
3717 if (nr_ioapics <= 0)
3735 ioapic_resources =
res;
3742 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3746 ioapic_res = ioapic_setup_resources(nr_ioapics);
3750 #ifdef CONFIG_X86_32
3753 "WARNING: bogus zero IO-APIC "
3754 "address found in MPTABLE, "
3755 "disabling IO/APIC support!\n");
3758 goto fake_ioapic_page;
3762 #ifdef CONFIG_X86_32
3766 ioapic_phys =
__pa(ioapic_phys);
3774 ioapic_res->
start = ioapic_phys;
3779 probe_nr_irqs_gsi();
3790 "IO APIC resources couldn't be allocated.\n");
3804 if (nr_ioapics == 0)
3810 if ((gsi >= gsi_cfg->gsi_base)
3811 && (gsi <= gsi_cfg->gsi_end))
3815 printk(
KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
3821 struct mp_ioapic_gsi *gsi_cfg;
3827 if (
WARN_ON(gsi > gsi_cfg->gsi_end))
3830 return gsi - gsi_cfg->gsi_base;
3836 pr_warn(
"WARNING: Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
3841 pr_warn(
"WARNING: Bogus (zero) I/O APIC address found in table, skipping!\n");
3847 static __init int bad_ioapic_register(
int idx)
3853 reg_00.
raw = io_apic_read(idx, 0);
3854 reg_01.raw = io_apic_read(idx, 1);
3855 reg_02.raw = io_apic_read(idx, 2);
3857 if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
3858 pr_warn(
"I/O APIC 0x%x registers return all ones, skipping!\n",
3870 struct mp_ioapic_gsi *gsi_cfg;
3872 if (bad_ioapic(address))
3883 if (bad_ioapic_register(idx)) {
3888 ioapics[
idx].mp_config.apicid = io_apic_unique_id(
id);
3889 ioapics[
idx].mp_config.apicver = io_apic_get_version(idx);
3895 entries = io_apic_get_redir_entries(idx);
3897 gsi_cfg->gsi_base = gsi_base;
3898 gsi_cfg->gsi_end = gsi_base + entries - 1;
3905 if (gsi_cfg->gsi_end >=
gsi_top)
3906 gsi_top = gsi_cfg->gsi_end + 1;
3908 pr_info(
"IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
3911 gsi_cfg->gsi_base, gsi_cfg->gsi_end);
3928 io_apic_setup_irq_pin(0, 0, &attr);