Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
pci-calgary_64.c
Go to the documentation of this file.
1 /*
2  * Derived from arch/powerpc/kernel/iommu.c
3  *
4  * Copyright IBM Corporation, 2006-2007
5  * Copyright (C) 2006 Jon Mason <[email protected]>
6  *
7  * Author: Jon Mason <[email protected]>
8  * Author: Muli Ben-Yehuda <[email protected]>
9 
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23  */
24 
25 #define pr_fmt(fmt) "Calgary: " fmt
26 
27 #include <linux/kernel.h>
28 #include <linux/init.h>
29 #include <linux/types.h>
30 #include <linux/slab.h>
31 #include <linux/mm.h>
32 #include <linux/spinlock.h>
33 #include <linux/string.h>
34 #include <linux/crash_dump.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/bitmap.h>
37 #include <linux/pci_ids.h>
38 #include <linux/pci.h>
39 #include <linux/delay.h>
40 #include <linux/scatterlist.h>
41 #include <linux/iommu-helper.h>
42 
43 #include <asm/iommu.h>
44 #include <asm/calgary.h>
45 #include <asm/tce.h>
46 #include <asm/pci-direct.h>
47 #include <asm/dma.h>
48 #include <asm/rio.h>
49 #include <asm/bios_ebda.h>
50 #include <asm/x86_init.h>
51 #include <asm/iommu_table.h>
52 
53 #ifdef CONFIG_CALGARY_IOMMU_ENABLED_BY_DEFAULT
55 #else
57 #endif /* CONFIG_CALGARY_DEFAULT_ENABLED */
58 
59 #define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
60 #define PCI_DEVICE_ID_IBM_CALIOC2 0x0308
61 
62 /* register offsets inside the host bridge space */
63 #define CALGARY_CONFIG_REG 0x0108
64 #define PHB_CSR_OFFSET 0x0110 /* Channel Status */
65 #define PHB_PLSSR_OFFSET 0x0120
66 #define PHB_CONFIG_RW_OFFSET 0x0160
67 #define PHB_IOBASE_BAR_LOW 0x0170
68 #define PHB_IOBASE_BAR_HIGH 0x0180
69 #define PHB_MEM_1_LOW 0x0190
70 #define PHB_MEM_1_HIGH 0x01A0
71 #define PHB_IO_ADDR_SIZE 0x01B0
72 #define PHB_MEM_1_SIZE 0x01C0
73 #define PHB_MEM_ST_OFFSET 0x01D0
74 #define PHB_AER_OFFSET 0x0200
75 #define PHB_CONFIG_0_HIGH 0x0220
76 #define PHB_CONFIG_0_LOW 0x0230
77 #define PHB_CONFIG_0_END 0x0240
78 #define PHB_MEM_2_LOW 0x02B0
79 #define PHB_MEM_2_HIGH 0x02C0
80 #define PHB_MEM_2_SIZE_HIGH 0x02D0
81 #define PHB_MEM_2_SIZE_LOW 0x02E0
82 #define PHB_DOSHOLE_OFFSET 0x08E0
83 
84 /* CalIOC2 specific */
85 #define PHB_SAVIOR_L2 0x0DB0
86 #define PHB_PAGE_MIG_CTRL 0x0DA8
87 #define PHB_PAGE_MIG_DEBUG 0x0DA0
88 #define PHB_ROOT_COMPLEX_STATUS 0x0CB0
89 
90 /* PHB_CONFIG_RW */
91 #define PHB_TCE_ENABLE 0x20000000
92 #define PHB_SLOT_DISABLE 0x1C000000
93 #define PHB_DAC_DISABLE 0x01000000
94 #define PHB_MEM2_ENABLE 0x00400000
95 #define PHB_MCSR_ENABLE 0x00100000
96 /* TAR (Table Address Register) */
97 #define TAR_SW_BITS 0x0000ffffffff800fUL
98 #define TAR_VALID 0x0000000000000008UL
99 /* CSR (Channel/DMA Status Register) */
100 #define CSR_AGENT_MASK 0xffe0ffff
101 /* CCR (Calgary Configuration Register) */
102 #define CCR_2SEC_TIMEOUT 0x000000000000000EUL
103 /* PMCR/PMDR (Page Migration Control/Debug Registers */
104 #define PMR_SOFTSTOP 0x80000000
105 #define PMR_SOFTSTOPFAULT 0x40000000
106 #define PMR_HARDSTOP 0x20000000
107 
108 /*
109  * The maximum PHB bus number.
110  * x3950M2 (rare): 8 chassis, 48 PHBs per chassis = 384
111  * x3950M2: 4 chassis, 48 PHBs per chassis = 192
112  * x3950 (PCIE): 8 chassis, 32 PHBs per chassis = 256
113  * x3950 (PCIX): 8 chassis, 16 PHBs per chassis = 128
114  */
115 #define MAX_PHB_BUS_NUM 256
116 
117 #define PHBS_PER_CALGARY 4
118 
119 /* register offsets in Calgary's internal register space */
120 static const unsigned long tar_offsets[] = {
121  0x0580 /* TAR0 */,
122  0x0588 /* TAR1 */,
123  0x0590 /* TAR2 */,
124  0x0598 /* TAR3 */
125 };
126 
127 static const unsigned long split_queue_offsets[] = {
128  0x4870 /* SPLIT QUEUE 0 */,
129  0x5870 /* SPLIT QUEUE 1 */,
130  0x6870 /* SPLIT QUEUE 2 */,
131  0x7870 /* SPLIT QUEUE 3 */
132 };
133 
134 static const unsigned long phb_offsets[] = {
135  0x8000 /* PHB0 */,
136  0x9000 /* PHB1 */,
137  0xA000 /* PHB2 */,
138  0xB000 /* PHB3 */
139 };
140 
141 /* PHB debug registers */
142 
143 static const unsigned long phb_debug_offsets[] = {
144  0x4000 /* PHB 0 DEBUG */,
145  0x5000 /* PHB 1 DEBUG */,
146  0x6000 /* PHB 2 DEBUG */,
147  0x7000 /* PHB 3 DEBUG */
148 };
149 
150 /*
151  * STUFF register for each debug PHB,
152  * byte 1 = start bus number, byte 2 = end bus number
153  */
154 
155 #define PHB_DEBUG_STUFF_OFFSET 0x0020
156 
157 #define EMERGENCY_PAGES 32 /* = 128KB */
158 
160 static int translate_empty_slots __read_mostly = 0;
161 static int calgary_detected __read_mostly = 0;
162 
163 static struct rio_table_hdr *rio_table_hdr __initdata;
164 static struct scal_detail *scal_devs[MAX_NUMNODES] __initdata;
165 static struct rio_detail *rio_devs[MAX_NUMNODES * 4] __initdata;
166 
168  void *tce_space;
169  unsigned char translation_disabled;
170  signed char phbid;
171  void __iomem *bbar;
172 };
173 
174 static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
175 static void calgary_tce_cache_blast(struct iommu_table *tbl);
176 static void calgary_dump_error_regs(struct iommu_table *tbl);
177 static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
178 static void calioc2_tce_cache_blast(struct iommu_table *tbl);
179 static void calioc2_dump_error_regs(struct iommu_table *tbl);
180 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl);
181 static void get_tce_space_from_tar(void);
182 
183 static struct cal_chipset_ops calgary_chip_ops = {
184  .handle_quirks = calgary_handle_quirks,
185  .tce_cache_blast = calgary_tce_cache_blast,
186  .dump_error_regs = calgary_dump_error_regs
187 };
188 
189 static struct cal_chipset_ops calioc2_chip_ops = {
190  .handle_quirks = calioc2_handle_quirks,
191  .tce_cache_blast = calioc2_tce_cache_blast,
192  .dump_error_regs = calioc2_dump_error_regs
193 };
194 
195 static struct calgary_bus_info bus_info[MAX_PHB_BUS_NUM] = { { NULL, 0, 0 }, };
196 
197 static inline int translation_enabled(struct iommu_table *tbl)
198 {
199  /* only PHBs with translation enabled have an IOMMU table */
200  return (tbl != NULL);
201 }
202 
203 static void iommu_range_reserve(struct iommu_table *tbl,
204  unsigned long start_addr, unsigned int npages)
205 {
206  unsigned long index;
207  unsigned long end;
208  unsigned long flags;
209 
210  index = start_addr >> PAGE_SHIFT;
211 
212  /* bail out if we're asked to reserve a region we don't cover */
213  if (index >= tbl->it_size)
214  return;
215 
216  end = index + npages;
217  if (end > tbl->it_size) /* don't go off the table */
218  end = tbl->it_size;
219 
220  spin_lock_irqsave(&tbl->it_lock, flags);
221 
222  bitmap_set(tbl->it_map, index, npages);
223 
224  spin_unlock_irqrestore(&tbl->it_lock, flags);
225 }
226 
227 static unsigned long iommu_range_alloc(struct device *dev,
228  struct iommu_table *tbl,
229  unsigned int npages)
230 {
231  unsigned long flags;
232  unsigned long offset;
233  unsigned long boundary_size;
234 
235  boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
236  PAGE_SIZE) >> PAGE_SHIFT;
237 
238  BUG_ON(npages == 0);
239 
240  spin_lock_irqsave(&tbl->it_lock, flags);
241 
242  offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
243  npages, 0, boundary_size, 0);
244  if (offset == ~0UL) {
245  tbl->chip_ops->tce_cache_blast(tbl);
246 
247  offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
248  npages, 0, boundary_size, 0);
249  if (offset == ~0UL) {
250  pr_warn("IOMMU full\n");
251  spin_unlock_irqrestore(&tbl->it_lock, flags);
252  if (panic_on_overflow)
253  panic("Calgary: fix the allocator.\n");
254  else
255  return DMA_ERROR_CODE;
256  }
257  }
258 
259  tbl->it_hint = offset + npages;
260  BUG_ON(tbl->it_hint > tbl->it_size);
261 
262  spin_unlock_irqrestore(&tbl->it_lock, flags);
263 
264  return offset;
265 }
266 
267 static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
268  void *vaddr, unsigned int npages, int direction)
269 {
270  unsigned long entry;
271  dma_addr_t ret;
272 
273  entry = iommu_range_alloc(dev, tbl, npages);
274 
275  if (unlikely(entry == DMA_ERROR_CODE)) {
276  pr_warn("failed to allocate %u pages in iommu %p\n",
277  npages, tbl);
278  return DMA_ERROR_CODE;
279  }
280 
281  /* set the return dma address */
282  ret = (entry << PAGE_SHIFT) | ((unsigned long)vaddr & ~PAGE_MASK);
283 
284  /* put the TCEs in the HW table */
285  tce_build(tbl, entry, npages, (unsigned long)vaddr & PAGE_MASK,
286  direction);
287  return ret;
288 }
289 
290 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
291  unsigned int npages)
292 {
293  unsigned long entry;
294  unsigned long badend;
295  unsigned long flags;
296 
297  /* were we called with bad_dma_address? */
299  if (unlikely((dma_addr >= DMA_ERROR_CODE) && (dma_addr < badend))) {
300  WARN(1, KERN_ERR "Calgary: driver tried unmapping bad DMA "
301  "address 0x%Lx\n", dma_addr);
302  return;
303  }
304 
305  entry = dma_addr >> PAGE_SHIFT;
306 
307  BUG_ON(entry + npages > tbl->it_size);
308 
309  tce_free(tbl, entry, npages);
310 
311  spin_lock_irqsave(&tbl->it_lock, flags);
312 
313  bitmap_clear(tbl->it_map, entry, npages);
314 
315  spin_unlock_irqrestore(&tbl->it_lock, flags);
316 }
317 
318 static inline struct iommu_table *find_iommu_table(struct device *dev)
319 {
320  struct pci_dev *pdev;
321  struct pci_bus *pbus;
322  struct iommu_table *tbl;
323 
324  pdev = to_pci_dev(dev);
325 
326  /* search up the device tree for an iommu */
327  pbus = pdev->bus;
328  do {
329  tbl = pci_iommu(pbus);
330  if (tbl && tbl->it_busno == pbus->number)
331  break;
332  tbl = NULL;
333  pbus = pbus->parent;
334  } while (pbus);
335 
336  BUG_ON(tbl && (tbl->it_busno != pbus->number));
337 
338  return tbl;
339 }
340 
341 static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist,
342  int nelems,enum dma_data_direction dir,
343  struct dma_attrs *attrs)
344 {
345  struct iommu_table *tbl = find_iommu_table(dev);
346  struct scatterlist *s;
347  int i;
348 
349  if (!translation_enabled(tbl))
350  return;
351 
352  for_each_sg(sglist, s, nelems, i) {
353  unsigned int npages;
355  unsigned int dmalen = s->dma_length;
356 
357  if (dmalen == 0)
358  break;
359 
360  npages = iommu_num_pages(dma, dmalen, PAGE_SIZE);
361  iommu_free(tbl, dma, npages);
362  }
363 }
364 
365 static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
366  int nelems, enum dma_data_direction dir,
367  struct dma_attrs *attrs)
368 {
369  struct iommu_table *tbl = find_iommu_table(dev);
370  struct scatterlist *s;
371  unsigned long vaddr;
372  unsigned int npages;
373  unsigned long entry;
374  int i;
375 
376  for_each_sg(sg, s, nelems, i) {
377  BUG_ON(!sg_page(s));
378 
379  vaddr = (unsigned long) sg_virt(s);
380  npages = iommu_num_pages(vaddr, s->length, PAGE_SIZE);
381 
382  entry = iommu_range_alloc(dev, tbl, npages);
383  if (entry == DMA_ERROR_CODE) {
384  /* makes sure unmap knows to stop */
385  s->dma_length = 0;
386  goto error;
387  }
388 
389  s->dma_address = (entry << PAGE_SHIFT) | s->offset;
390 
391  /* insert into HW table */
392  tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);
393 
394  s->dma_length = s->length;
395  }
396 
397  return nelems;
398 error:
399  calgary_unmap_sg(dev, sg, nelems, dir, NULL);
400  for_each_sg(sg, s, nelems, i) {
402  sg->dma_length = 0;
403  }
404  return 0;
405 }
406 
407 static dma_addr_t calgary_map_page(struct device *dev, struct page *page,
408  unsigned long offset, size_t size,
409  enum dma_data_direction dir,
410  struct dma_attrs *attrs)
411 {
412  void *vaddr = page_address(page) + offset;
413  unsigned long uaddr;
414  unsigned int npages;
415  struct iommu_table *tbl = find_iommu_table(dev);
416 
417  uaddr = (unsigned long)vaddr;
418  npages = iommu_num_pages(uaddr, size, PAGE_SIZE);
419 
420  return iommu_alloc(dev, tbl, vaddr, npages, dir);
421 }
422 
423 static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr,
424  size_t size, enum dma_data_direction dir,
425  struct dma_attrs *attrs)
426 {
427  struct iommu_table *tbl = find_iommu_table(dev);
428  unsigned int npages;
429 
430  npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
431  iommu_free(tbl, dma_addr, npages);
432 }
433 
434 static void* calgary_alloc_coherent(struct device *dev, size_t size,
435  dma_addr_t *dma_handle, gfp_t flag, struct dma_attrs *attrs)
436 {
437  void *ret = NULL;
439  unsigned int npages, order;
440  struct iommu_table *tbl = find_iommu_table(dev);
441 
442  size = PAGE_ALIGN(size); /* size rounded up to full pages */
443  npages = size >> PAGE_SHIFT;
444  order = get_order(size);
445 
446  flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
447 
448  /* alloc enough pages (and possibly more) */
449  ret = (void *)__get_free_pages(flag, order);
450  if (!ret)
451  goto error;
452  memset(ret, 0, size);
453 
454  /* set up tces to cover the allocated range */
455  mapping = iommu_alloc(dev, tbl, ret, npages, DMA_BIDIRECTIONAL);
456  if (mapping == DMA_ERROR_CODE)
457  goto free;
458  *dma_handle = mapping;
459  return ret;
460 free:
461  free_pages((unsigned long)ret, get_order(size));
462  ret = NULL;
463 error:
464  return ret;
465 }
466 
467 static void calgary_free_coherent(struct device *dev, size_t size,
468  void *vaddr, dma_addr_t dma_handle,
469  struct dma_attrs *attrs)
470 {
471  unsigned int npages;
472  struct iommu_table *tbl = find_iommu_table(dev);
473 
474  size = PAGE_ALIGN(size);
475  npages = size >> PAGE_SHIFT;
476 
477  iommu_free(tbl, dma_handle, npages);
478  free_pages((unsigned long)vaddr, get_order(size));
479 }
480 
481 static struct dma_map_ops calgary_dma_ops = {
482  .alloc = calgary_alloc_coherent,
483  .free = calgary_free_coherent,
484  .map_sg = calgary_map_sg,
485  .unmap_sg = calgary_unmap_sg,
486  .map_page = calgary_map_page,
487  .unmap_page = calgary_unmap_page,
488 };
489 
490 static inline void __iomem * busno_to_bbar(unsigned char num)
491 {
492  return bus_info[num].bbar;
493 }
494 
495 static inline int busno_to_phbid(unsigned char num)
496 {
497  return bus_info[num].phbid;
498 }
499 
500 static inline unsigned long split_queue_offset(unsigned char num)
501 {
502  size_t idx = busno_to_phbid(num);
503 
504  return split_queue_offsets[idx];
505 }
506 
507 static inline unsigned long tar_offset(unsigned char num)
508 {
509  size_t idx = busno_to_phbid(num);
510 
511  return tar_offsets[idx];
512 }
513 
514 static inline unsigned long phb_offset(unsigned char num)
515 {
516  size_t idx = busno_to_phbid(num);
517 
518  return phb_offsets[idx];
519 }
520 
521 static inline void __iomem* calgary_reg(void __iomem *bar, unsigned long offset)
522 {
523  unsigned long target = ((unsigned long)bar) | offset;
524  return (void __iomem*)target;
525 }
526 
527 static inline int is_calioc2(unsigned short device)
528 {
529  return (device == PCI_DEVICE_ID_IBM_CALIOC2);
530 }
531 
532 static inline int is_calgary(unsigned short device)
533 {
534  return (device == PCI_DEVICE_ID_IBM_CALGARY);
535 }
536 
537 static inline int is_cal_pci_dev(unsigned short device)
538 {
539  return (is_calgary(device) || is_calioc2(device));
540 }
541 
542 static void calgary_tce_cache_blast(struct iommu_table *tbl)
543 {
544  u64 val;
545  u32 aer;
546  int i = 0;
547  void __iomem *bbar = tbl->bbar;
548  void __iomem *target;
549 
550  /* disable arbitration on the bus */
551  target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
552  aer = readl(target);
553  writel(0, target);
554 
555  /* read plssr to ensure it got there */
556  target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
557  val = readl(target);
558 
559  /* poll split queues until all DMA activity is done */
560  target = calgary_reg(bbar, split_queue_offset(tbl->it_busno));
561  do {
562  val = readq(target);
563  i++;
564  } while ((val & 0xff) != 0xff && i < 100);
565  if (i == 100)
566  pr_warn("PCI bus not quiesced, continuing anyway\n");
567 
568  /* invalidate TCE cache */
569  target = calgary_reg(bbar, tar_offset(tbl->it_busno));
570  writeq(tbl->tar_val, target);
571 
572  /* enable arbitration */
573  target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
574  writel(aer, target);
575  (void)readl(target); /* flush */
576 }
577 
578 static void calioc2_tce_cache_blast(struct iommu_table *tbl)
579 {
580  void __iomem *bbar = tbl->bbar;
581  void __iomem *target;
582  u64 val64;
583  u32 val;
584  int i = 0;
585  int count = 1;
586  unsigned char bus = tbl->it_busno;
587 
588 begin:
589  printk(KERN_DEBUG "Calgary: CalIOC2 bus 0x%x entering tce cache blast "
590  "sequence - count %d\n", bus, count);
591 
592  /* 1. using the Page Migration Control reg set SoftStop */
593  target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
594  val = be32_to_cpu(readl(target));
595  printk(KERN_DEBUG "1a. read 0x%x [LE] from %p\n", val, target);
596  val |= PMR_SOFTSTOP;
597  printk(KERN_DEBUG "1b. writing 0x%x [LE] to %p\n", val, target);
598  writel(cpu_to_be32(val), target);
599 
600  /* 2. poll split queues until all DMA activity is done */
601  printk(KERN_DEBUG "2a. starting to poll split queues\n");
602  target = calgary_reg(bbar, split_queue_offset(bus));
603  do {
604  val64 = readq(target);
605  i++;
606  } while ((val64 & 0xff) != 0xff && i < 100);
607  if (i == 100)
608  pr_warn("CalIOC2: PCI bus not quiesced, continuing anyway\n");
609 
610  /* 3. poll Page Migration DEBUG for SoftStopFault */
611  target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
612  val = be32_to_cpu(readl(target));
613  printk(KERN_DEBUG "3. read 0x%x [LE] from %p\n", val, target);
614 
615  /* 4. if SoftStopFault - goto (1) */
616  if (val & PMR_SOFTSTOPFAULT) {
617  if (++count < 100)
618  goto begin;
619  else {
620  pr_warn("CalIOC2: too many SoftStopFaults, aborting TCE cache flush sequence!\n");
621  return; /* pray for the best */
622  }
623  }
624 
625  /* 5. Slam into HardStop by reading PHB_PAGE_MIG_CTRL */
626  target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
627  printk(KERN_DEBUG "5a. slamming into HardStop by reading %p\n", target);
628  val = be32_to_cpu(readl(target));
629  printk(KERN_DEBUG "5b. read 0x%x [LE] from %p\n", val, target);
630  target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
631  val = be32_to_cpu(readl(target));
632  printk(KERN_DEBUG "5c. read 0x%x [LE] from %p (debug)\n", val, target);
633 
634  /* 6. invalidate TCE cache */
635  printk(KERN_DEBUG "6. invalidating TCE cache\n");
636  target = calgary_reg(bbar, tar_offset(bus));
637  writeq(tbl->tar_val, target);
638 
639  /* 7. Re-read PMCR */
640  printk(KERN_DEBUG "7a. Re-reading PMCR\n");
641  target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
642  val = be32_to_cpu(readl(target));
643  printk(KERN_DEBUG "7b. read 0x%x [LE] from %p\n", val, target);
644 
645  /* 8. Remove HardStop */
646  printk(KERN_DEBUG "8a. removing HardStop from PMCR\n");
647  target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
648  val = 0;
649  printk(KERN_DEBUG "8b. writing 0x%x [LE] to %p\n", val, target);
650  writel(cpu_to_be32(val), target);
651  val = be32_to_cpu(readl(target));
652  printk(KERN_DEBUG "8c. read 0x%x [LE] from %p\n", val, target);
653 }
654 
655 static void __init calgary_reserve_mem_region(struct pci_dev *dev, u64 start,
656  u64 limit)
657 {
658  unsigned int numpages;
659 
660  limit = limit | 0xfffff;
661  limit++;
662 
663  numpages = ((limit - start) >> PAGE_SHIFT);
664  iommu_range_reserve(pci_iommu(dev->bus), start, numpages);
665 }
666 
667 static void __init calgary_reserve_peripheral_mem_1(struct pci_dev *dev)
668 {
669  void __iomem *target;
670  u64 low, high, sizelow;
671  u64 start, limit;
672  struct iommu_table *tbl = pci_iommu(dev->bus);
673  unsigned char busnum = dev->bus->number;
674  void __iomem *bbar = tbl->bbar;
675 
676  /* peripheral MEM_1 region */
677  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_LOW);
678  low = be32_to_cpu(readl(target));
679  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_HIGH);
680  high = be32_to_cpu(readl(target));
681  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_SIZE);
682  sizelow = be32_to_cpu(readl(target));
683 
684  start = (high << 32) | low;
685  limit = sizelow;
686 
687  calgary_reserve_mem_region(dev, start, limit);
688 }
689 
690 static void __init calgary_reserve_peripheral_mem_2(struct pci_dev *dev)
691 {
692  void __iomem *target;
693  u32 val32;
694  u64 low, high, sizelow, sizehigh;
695  u64 start, limit;
696  struct iommu_table *tbl = pci_iommu(dev->bus);
697  unsigned char busnum = dev->bus->number;
698  void __iomem *bbar = tbl->bbar;
699 
700  /* is it enabled? */
701  target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
702  val32 = be32_to_cpu(readl(target));
703  if (!(val32 & PHB_MEM2_ENABLE))
704  return;
705 
706  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_LOW);
707  low = be32_to_cpu(readl(target));
708  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_HIGH);
709  high = be32_to_cpu(readl(target));
710  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_LOW);
711  sizelow = be32_to_cpu(readl(target));
712  target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_HIGH);
713  sizehigh = be32_to_cpu(readl(target));
714 
715  start = (high << 32) | low;
716  limit = (sizehigh << 32) | sizelow;
717 
718  calgary_reserve_mem_region(dev, start, limit);
719 }
720 
721 /*
722  * some regions of the IO address space do not get translated, so we
723  * must not give devices IO addresses in those regions. The regions
724  * are the 640KB-1MB region and the two PCI peripheral memory holes.
725  * Reserve all of them in the IOMMU bitmap to avoid giving them out
726  * later.
727  */
728 static void __init calgary_reserve_regions(struct pci_dev *dev)
729 {
730  unsigned int npages;
731  u64 start;
732  struct iommu_table *tbl = pci_iommu(dev->bus);
733 
734  /* reserve EMERGENCY_PAGES from bad_dma_address and up */
735  iommu_range_reserve(tbl, DMA_ERROR_CODE, EMERGENCY_PAGES);
736 
737  /* avoid the BIOS/VGA first 640KB-1MB region */
738  /* for CalIOC2 - avoid the entire first MB */
739  if (is_calgary(dev->device)) {
740  start = (640 * 1024);
741  npages = ((1024 - 640) * 1024) >> PAGE_SHIFT;
742  } else { /* calioc2 */
743  start = 0;
744  npages = (1 * 1024 * 1024) >> PAGE_SHIFT;
745  }
746  iommu_range_reserve(tbl, start, npages);
747 
748  /* reserve the two PCI peripheral memory regions in IO space */
749  calgary_reserve_peripheral_mem_1(dev);
750  calgary_reserve_peripheral_mem_2(dev);
751 }
752 
753 static int __init calgary_setup_tar(struct pci_dev *dev, void __iomem *bbar)
754 {
755  u64 val64;
756  u64 table_phys;
757  void __iomem *target;
758  int ret;
759  struct iommu_table *tbl;
760 
761  /* build TCE tables for each PHB */
762  ret = build_tce_table(dev, bbar);
763  if (ret)
764  return ret;
765 
766  tbl = pci_iommu(dev->bus);
767  tbl->it_base = (unsigned long)bus_info[dev->bus->number].tce_space;
768 
769  if (is_kdump_kernel())
770  calgary_init_bitmap_from_tce_table(tbl);
771  else
772  tce_free(tbl, 0, tbl->it_size);
773 
774  if (is_calgary(dev->device))
775  tbl->chip_ops = &calgary_chip_ops;
776  else if (is_calioc2(dev->device))
777  tbl->chip_ops = &calioc2_chip_ops;
778  else
779  BUG();
780 
781  calgary_reserve_regions(dev);
782 
783  /* set TARs for each PHB */
784  target = calgary_reg(bbar, tar_offset(dev->bus->number));
785  val64 = be64_to_cpu(readq(target));
786 
787  /* zero out all TAR bits under sw control */
788  val64 &= ~TAR_SW_BITS;
789  table_phys = (u64)__pa(tbl->it_base);
790 
791  val64 |= table_phys;
792 
794  val64 |= (u64) specified_table_size;
795 
796  tbl->tar_val = cpu_to_be64(val64);
797 
798  writeq(tbl->tar_val, target);
799  readq(target); /* flush */
800 
801  return 0;
802 }
803 
804 static void __init calgary_free_bus(struct pci_dev *dev)
805 {
806  u64 val64;
807  struct iommu_table *tbl = pci_iommu(dev->bus);
808  void __iomem *target;
809  unsigned int bitmapsz;
810 
811  target = calgary_reg(tbl->bbar, tar_offset(dev->bus->number));
812  val64 = be64_to_cpu(readq(target));
813  val64 &= ~TAR_SW_BITS;
814  writeq(cpu_to_be64(val64), target);
815  readq(target); /* flush */
816 
817  bitmapsz = tbl->it_size / BITS_PER_BYTE;
818  free_pages((unsigned long)tbl->it_map, get_order(bitmapsz));
819  tbl->it_map = NULL;
820 
821  kfree(tbl);
822 
823  set_pci_iommu(dev->bus, NULL);
824 
825  /* Can't free bootmem allocated memory after system is up :-( */
826  bus_info[dev->bus->number].tce_space = NULL;
827 }
828 
829 static void calgary_dump_error_regs(struct iommu_table *tbl)
830 {
831  void __iomem *bbar = tbl->bbar;
832  void __iomem *target;
833  u32 csr, plssr;
834 
835  target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
836  csr = be32_to_cpu(readl(target));
837 
838  target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
839  plssr = be32_to_cpu(readl(target));
840 
841  /* If no error, the agent ID in the CSR is not valid */
842  pr_emerg("DMA error on Calgary PHB 0x%x, 0x%08x@CSR 0x%08x@PLSSR\n",
843  tbl->it_busno, csr, plssr);
844 }
845 
846 static void calioc2_dump_error_regs(struct iommu_table *tbl)
847 {
848  void __iomem *bbar = tbl->bbar;
849  u32 csr, csmr, plssr, mck, rcstat;
850  void __iomem *target;
851  unsigned long phboff = phb_offset(tbl->it_busno);
852  unsigned long erroff;
853  u32 errregs[7];
854  int i;
855 
856  /* dump CSR */
857  target = calgary_reg(bbar, phboff | PHB_CSR_OFFSET);
858  csr = be32_to_cpu(readl(target));
859  /* dump PLSSR */
860  target = calgary_reg(bbar, phboff | PHB_PLSSR_OFFSET);
861  plssr = be32_to_cpu(readl(target));
862  /* dump CSMR */
863  target = calgary_reg(bbar, phboff | 0x290);
864  csmr = be32_to_cpu(readl(target));
865  /* dump mck */
866  target = calgary_reg(bbar, phboff | 0x800);
867  mck = be32_to_cpu(readl(target));
868 
869  pr_emerg("DMA error on CalIOC2 PHB 0x%x\n", tbl->it_busno);
870 
871  pr_emerg("0x%08x@CSR 0x%08x@PLSSR 0x%08x@CSMR 0x%08x@MCK\n",
872  csr, plssr, csmr, mck);
873 
874  /* dump rest of error regs */
875  pr_emerg("");
876  for (i = 0; i < ARRAY_SIZE(errregs); i++) {
877  /* err regs are at 0x810 - 0x870 */
878  erroff = (0x810 + (i * 0x10));
879  target = calgary_reg(bbar, phboff | erroff);
880  errregs[i] = be32_to_cpu(readl(target));
881  pr_cont("0x%08x@0x%lx ", errregs[i], erroff);
882  }
883  pr_cont("\n");
884 
885  /* root complex status */
886  target = calgary_reg(bbar, phboff | PHB_ROOT_COMPLEX_STATUS);
887  rcstat = be32_to_cpu(readl(target));
888  printk(KERN_EMERG "Calgary: 0x%08x@0x%x\n", rcstat,
890 }
891 
892 static void calgary_watchdog(unsigned long data)
893 {
894  struct pci_dev *dev = (struct pci_dev *)data;
895  struct iommu_table *tbl = pci_iommu(dev->bus);
896  void __iomem *bbar = tbl->bbar;
897  u32 val32;
898  void __iomem *target;
899 
900  target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
901  val32 = be32_to_cpu(readl(target));
902 
903  /* If no error, the agent ID in the CSR is not valid */
904  if (val32 & CSR_AGENT_MASK) {
905  tbl->chip_ops->dump_error_regs(tbl);
906 
907  /* reset error */
908  writel(0, target);
909 
910  /* Disable bus that caused the error */
911  target = calgary_reg(bbar, phb_offset(tbl->it_busno) |
913  val32 = be32_to_cpu(readl(target));
914  val32 |= PHB_SLOT_DISABLE;
915  writel(cpu_to_be32(val32), target);
916  readl(target); /* flush */
917  } else {
918  /* Reset the timer */
919  mod_timer(&tbl->watchdog_timer, jiffies + 2 * HZ);
920  }
921 }
922 
923 static void __init calgary_set_split_completion_timeout(void __iomem *bbar,
924  unsigned char busnum, unsigned long timeout)
925 {
926  u64 val64;
927  void __iomem *target;
928  unsigned int phb_shift = ~0; /* silence gcc */
929  u64 mask;
930 
931  switch (busno_to_phbid(busnum)) {
932  case 0: phb_shift = (63 - 19);
933  break;
934  case 1: phb_shift = (63 - 23);
935  break;
936  case 2: phb_shift = (63 - 27);
937  break;
938  case 3: phb_shift = (63 - 35);
939  break;
940  default:
941  BUG_ON(busno_to_phbid(busnum));
942  }
943 
944  target = calgary_reg(bbar, CALGARY_CONFIG_REG);
945  val64 = be64_to_cpu(readq(target));
946 
947  /* zero out this PHB's timer bits */
948  mask = ~(0xFUL << phb_shift);
949  val64 &= mask;
950  val64 |= (timeout << phb_shift);
951  writeq(cpu_to_be64(val64), target);
952  readq(target); /* flush */
953 }
954 
955 static void __init calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
956 {
957  unsigned char busnum = dev->bus->number;
958  void __iomem *bbar = tbl->bbar;
959  void __iomem *target;
960  u32 val;
961 
962  /*
963  * CalIOC2 designers recommend setting bit 8 in 0xnDB0 to 1
964  */
965  target = calgary_reg(bbar, phb_offset(busnum) | PHB_SAVIOR_L2);
966  val = cpu_to_be32(readl(target));
967  val |= 0x00800000;
968  writel(cpu_to_be32(val), target);
969 }
970 
971 static void __init calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
972 {
973  unsigned char busnum = dev->bus->number;
974 
975  /*
976  * Give split completion a longer timeout on bus 1 for aic94xx
977  * http://bugzilla.kernel.org/show_bug.cgi?id=7180
978  */
979  if (is_calgary(dev->device) && (busnum == 1))
980  calgary_set_split_completion_timeout(tbl->bbar, busnum,
982 }
983 
984 static void __init calgary_enable_translation(struct pci_dev *dev)
985 {
986  u32 val32;
987  unsigned char busnum;
988  void __iomem *target;
989  void __iomem *bbar;
990  struct iommu_table *tbl;
991 
992  busnum = dev->bus->number;
993  tbl = pci_iommu(dev->bus);
994  bbar = tbl->bbar;
995 
996  /* enable TCE in PHB Config Register */
997  target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
998  val32 = be32_to_cpu(readl(target));
1000 
1001  printk(KERN_INFO "Calgary: enabling translation on %s PHB %#x\n",
1002  (dev->device == PCI_DEVICE_ID_IBM_CALGARY) ?
1003  "Calgary" : "CalIOC2", busnum);
1004  printk(KERN_INFO "Calgary: errant DMAs will now be prevented on this "
1005  "bus.\n");
1006 
1007  writel(cpu_to_be32(val32), target);
1008  readl(target); /* flush */
1009 
1010  init_timer(&tbl->watchdog_timer);
1011  tbl->watchdog_timer.function = &calgary_watchdog;
1012  tbl->watchdog_timer.data = (unsigned long)dev;
1013  mod_timer(&tbl->watchdog_timer, jiffies);
1014 }
1015 
1016 static void __init calgary_disable_translation(struct pci_dev *dev)
1017 {
1018  u32 val32;
1019  unsigned char busnum;
1020  void __iomem *target;
1021  void __iomem *bbar;
1022  struct iommu_table *tbl;
1023 
1024  busnum = dev->bus->number;
1025  tbl = pci_iommu(dev->bus);
1026  bbar = tbl->bbar;
1027 
1028  /* disable TCE in PHB Config Register */
1029  target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
1030  val32 = be32_to_cpu(readl(target));
1032 
1033  printk(KERN_INFO "Calgary: disabling translation on PHB %#x!\n", busnum);
1034  writel(cpu_to_be32(val32), target);
1035  readl(target); /* flush */
1036 
1038 }
1039 
1040 static void __init calgary_init_one_nontraslated(struct pci_dev *dev)
1041 {
1042  pci_dev_get(dev);
1043  set_pci_iommu(dev->bus, NULL);
1044 
1045  /* is the device behind a bridge? */
1046  if (dev->bus->parent)
1047  dev->bus->parent->self = dev;
1048  else
1049  dev->bus->self = dev;
1050 }
1051 
1052 static int __init calgary_init_one(struct pci_dev *dev)
1053 {
1054  void __iomem *bbar;
1055  struct iommu_table *tbl;
1056  int ret;
1057 
1058  bbar = busno_to_bbar(dev->bus->number);
1059  ret = calgary_setup_tar(dev, bbar);
1060  if (ret)
1061  goto done;
1062 
1063  pci_dev_get(dev);
1064 
1065  if (dev->bus->parent) {
1066  if (dev->bus->parent->self)
1067  printk(KERN_WARNING "Calgary: IEEEE, dev %p has "
1068  "bus->parent->self!\n", dev);
1069  dev->bus->parent->self = dev;
1070  } else
1071  dev->bus->self = dev;
1072 
1073  tbl = pci_iommu(dev->bus);
1074  tbl->chip_ops->handle_quirks(tbl, dev);
1075 
1076  calgary_enable_translation(dev);
1077 
1078  return 0;
1079 
1080 done:
1081  return ret;
1082 }
1083 
1084 static int __init calgary_locate_bbars(void)
1085 {
1086  int ret;
1087  int rioidx, phb, bus;
1088  void __iomem *bbar;
1089  void __iomem *target;
1090  unsigned long offset;
1091  u8 start_bus, end_bus;
1092  u32 val;
1093 
1094  ret = -ENODATA;
1095  for (rioidx = 0; rioidx < rio_table_hdr->num_rio_dev; rioidx++) {
1096  struct rio_detail *rio = rio_devs[rioidx];
1097 
1098  if ((rio->type != COMPAT_CALGARY) && (rio->type != ALT_CALGARY))
1099  continue;
1100 
1101  /* map entire 1MB of Calgary config space */
1102  bbar = ioremap_nocache(rio->BBAR, 1024 * 1024);
1103  if (!bbar)
1104  goto error;
1105 
1106  for (phb = 0; phb < PHBS_PER_CALGARY; phb++) {
1107  offset = phb_debug_offsets[phb] | PHB_DEBUG_STUFF_OFFSET;
1108  target = calgary_reg(bbar, offset);
1109 
1110  val = be32_to_cpu(readl(target));
1111 
1112  start_bus = (u8)((val & 0x00FF0000) >> 16);
1113  end_bus = (u8)((val & 0x0000FF00) >> 8);
1114 
1115  if (end_bus) {
1116  for (bus = start_bus; bus <= end_bus; bus++) {
1117  bus_info[bus].bbar = bbar;
1118  bus_info[bus].phbid = phb;
1119  }
1120  } else {
1121  bus_info[start_bus].bbar = bbar;
1122  bus_info[start_bus].phbid = phb;
1123  }
1124  }
1125  }
1126 
1127  return 0;
1128 
1129 error:
1130  /* scan bus_info and iounmap any bbars we previously ioremap'd */
1131  for (bus = 0; bus < ARRAY_SIZE(bus_info); bus++)
1132  if (bus_info[bus].bbar)
1133  iounmap(bus_info[bus].bbar);
1134 
1135  return ret;
1136 }
1137 
1138 static int __init calgary_init(void)
1139 {
1140  int ret;
1141  struct pci_dev *dev = NULL;
1142  struct calgary_bus_info *info;
1143 
1144  ret = calgary_locate_bbars();
1145  if (ret)
1146  return ret;
1147 
1148  /* Purely for kdump kernel case */
1149  if (is_kdump_kernel())
1150  get_tce_space_from_tar();
1151 
1152  do {
1154  if (!dev)
1155  break;
1156  if (!is_cal_pci_dev(dev->device))
1157  continue;
1158 
1159  info = &bus_info[dev->bus->number];
1160  if (info->translation_disabled) {
1161  calgary_init_one_nontraslated(dev);
1162  continue;
1163  }
1164 
1165  if (!info->tce_space && !translate_empty_slots)
1166  continue;
1167 
1168  ret = calgary_init_one(dev);
1169  if (ret)
1170  goto error;
1171  } while (1);
1172 
1173  dev = NULL;
1174  for_each_pci_dev(dev) {
1175  struct iommu_table *tbl;
1176 
1177  tbl = find_iommu_table(&dev->dev);
1178 
1179  if (translation_enabled(tbl))
1180  dev->dev.archdata.dma_ops = &calgary_dma_ops;
1181  }
1182 
1183  return ret;
1184 
1185 error:
1186  do {
1188  if (!dev)
1189  break;
1190  if (!is_cal_pci_dev(dev->device))
1191  continue;
1192 
1193  info = &bus_info[dev->bus->number];
1194  if (info->translation_disabled) {
1195  pci_dev_put(dev);
1196  continue;
1197  }
1198  if (!info->tce_space && !translate_empty_slots)
1199  continue;
1200 
1201  calgary_disable_translation(dev);
1202  calgary_free_bus(dev);
1203  pci_dev_put(dev); /* Undo calgary_init_one()'s pci_dev_get() */
1204  dev->dev.archdata.dma_ops = NULL;
1205  } while (1);
1206 
1207  return ret;
1208 }
1209 
1210 static inline int __init determine_tce_table_size(u64 ram)
1211 {
1212  int ret;
1213 
1215  return specified_table_size;
1216 
1217  /*
1218  * Table sizes are from 0 to 7 (TCE_TABLE_SIZE_64K to
1219  * TCE_TABLE_SIZE_8M). Table size 0 has 8K entries and each
1220  * larger table size has twice as many entries, so shift the
1221  * max ram address by 13 to divide by 8K and then look at the
1222  * order of the result to choose between 0-7.
1223  */
1224  ret = get_order(ram >> 13);
1225  if (ret > TCE_TABLE_SIZE_8M)
1226  ret = TCE_TABLE_SIZE_8M;
1227 
1228  return ret;
1229 }
1230 
1231 static int __init build_detail_arrays(void)
1232 {
1233  unsigned long ptr;
1234  unsigned numnodes, i;
1235  int scal_detail_size, rio_detail_size;
1236 
1237  numnodes = rio_table_hdr->num_scal_dev;
1238  if (numnodes > MAX_NUMNODES){
1240  "Calgary: MAX_NUMNODES too low! Defined as %d, "
1241  "but system has %d nodes.\n",
1242  MAX_NUMNODES, numnodes);
1243  return -ENODEV;
1244  }
1245 
1246  switch (rio_table_hdr->version){
1247  case 2:
1248  scal_detail_size = 11;
1249  rio_detail_size = 13;
1250  break;
1251  case 3:
1252  scal_detail_size = 12;
1253  rio_detail_size = 15;
1254  break;
1255  default:
1257  "Calgary: Invalid Rio Grande Table Version: %d\n",
1259  return -EPROTO;
1260  }
1261 
1262  ptr = ((unsigned long)rio_table_hdr) + 3;
1263  for (i = 0; i < numnodes; i++, ptr += scal_detail_size)
1264  scal_devs[i] = (struct scal_detail *)ptr;
1265 
1266  for (i = 0; i < rio_table_hdr->num_rio_dev;
1267  i++, ptr += rio_detail_size)
1268  rio_devs[i] = (struct rio_detail *)ptr;
1269 
1270  return 0;
1271 }
1272 
1273 static int __init calgary_bus_has_devices(int bus, unsigned short pci_dev)
1274 {
1275  int dev;
1276  u32 val;
1277 
1278  if (pci_dev == PCI_DEVICE_ID_IBM_CALIOC2) {
1279  /*
1280  * FIXME: properly scan for devices across the
1281  * PCI-to-PCI bridge on every CalIOC2 port.
1282  */
1283  return 1;
1284  }
1285 
1286  for (dev = 1; dev < 8; dev++) {
1287  val = read_pci_config(bus, dev, 0, 0);
1288  if (val != 0xffffffff)
1289  break;
1290  }
1291  return (val != 0xffffffff);
1292 }
1293 
1294 /*
1295  * calgary_init_bitmap_from_tce_table():
1296  * Function for kdump case. In the second/kdump kernel initialize
1297  * the bitmap based on the tce table entries obtained from first kernel
1298  */
1299 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl)
1300 {
1301  u64 *tp;
1302  unsigned int index;
1303  tp = ((u64 *)tbl->it_base);
1304  for (index = 0 ; index < tbl->it_size; index++) {
1305  if (*tp != 0x0)
1306  set_bit(index, tbl->it_map);
1307  tp++;
1308  }
1309 }
1310 
1311 /*
1312  * get_tce_space_from_tar():
1313  * Function for kdump case. Get the tce tables from first kernel
1314  * by reading the contents of the base address register of calgary iommu
1315  */
1316 static void __init get_tce_space_from_tar(void)
1317 {
1318  int bus;
1319  void __iomem *target;
1320  unsigned long tce_space;
1321 
1322  for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1323  struct calgary_bus_info *info = &bus_info[bus];
1324  unsigned short pci_device;
1325  u32 val;
1326 
1327  val = read_pci_config(bus, 0, 0, 0);
1328  pci_device = (val & 0xFFFF0000) >> 16;
1329 
1330  if (!is_cal_pci_dev(pci_device))
1331  continue;
1332  if (info->translation_disabled)
1333  continue;
1334 
1335  if (calgary_bus_has_devices(bus, pci_device) ||
1336  translate_empty_slots) {
1337  target = calgary_reg(bus_info[bus].bbar,
1338  tar_offset(bus));
1339  tce_space = be64_to_cpu(readq(target));
1340  tce_space = tce_space & TAR_SW_BITS;
1341 
1342  tce_space = tce_space & (~specified_table_size);
1343  info->tce_space = (u64 *)__va(tce_space);
1344  }
1345  }
1346  return;
1347 }
1348 
1349 static int __init calgary_iommu_init(void)
1350 {
1351  int ret;
1352 
1353  /* ok, we're trying to use Calgary - let's roll */
1354  printk(KERN_INFO "PCI-DMA: Using Calgary IOMMU\n");
1355 
1356  ret = calgary_init();
1357  if (ret) {
1358  printk(KERN_ERR "PCI-DMA: Calgary init failed %d, "
1359  "falling back to no_iommu\n", ret);
1360  return ret;
1361  }
1362 
1363  return 0;
1364 }
1365 
1367 {
1368  int bus;
1369  void *tbl;
1370  int calgary_found = 0;
1371  unsigned long ptr;
1372  unsigned int offset, prev_offset;
1373  int ret;
1374 
1375  /*
1376  * if the user specified iommu=off or iommu=soft or we found
1377  * another HW IOMMU already, bail out.
1378  */
1379  if (no_iommu || iommu_detected)
1380  return -ENODEV;
1381 
1382  if (!use_calgary)
1383  return -ENODEV;
1384 
1385  if (!early_pci_allowed())
1386  return -ENODEV;
1387 
1388  printk(KERN_DEBUG "Calgary: detecting Calgary via BIOS EBDA area\n");
1389 
1390  ptr = (unsigned long)phys_to_virt(get_bios_ebda());
1391 
1392  rio_table_hdr = NULL;
1393  prev_offset = 0;
1394  offset = 0x180;
1395  /*
1396  * The next offset is stored in the 1st word.
1397  * Only parse up until the offset increases:
1398  */
1399  while (offset > prev_offset) {
1400  /* The block id is stored in the 2nd word */
1401  if (*((unsigned short *)(ptr + offset + 2)) == 0x4752){
1402  /* set the pointer past the offset & block id */
1403  rio_table_hdr = (struct rio_table_hdr *)(ptr + offset + 4);
1404  break;
1405  }
1406  prev_offset = offset;
1407  offset = *((unsigned short *)(ptr + offset));
1408  }
1409  if (!rio_table_hdr) {
1410  printk(KERN_DEBUG "Calgary: Unable to locate Rio Grande table "
1411  "in EBDA - bailing!\n");
1412  return -ENODEV;
1413  }
1414 
1415  ret = build_detail_arrays();
1416  if (ret) {
1417  printk(KERN_DEBUG "Calgary: build_detail_arrays ret %d\n", ret);
1418  return -ENOMEM;
1419  }
1420 
1421  specified_table_size = determine_tce_table_size((is_kdump_kernel() ?
1423 
1424  for (bus = 0; bus < MAX_PHB_BUS_NUM; bus++) {
1425  struct calgary_bus_info *info = &bus_info[bus];
1426  unsigned short pci_device;
1427  u32 val;
1428 
1429  val = read_pci_config(bus, 0, 0, 0);
1430  pci_device = (val & 0xFFFF0000) >> 16;
1431 
1432  if (!is_cal_pci_dev(pci_device))
1433  continue;
1434 
1435  if (info->translation_disabled)
1436  continue;
1437 
1438  if (calgary_bus_has_devices(bus, pci_device) ||
1439  translate_empty_slots) {
1440  /*
1441  * If it is kdump kernel, find and use tce tables
1442  * from first kernel, else allocate tce tables here
1443  */
1444  if (!is_kdump_kernel()) {
1445  tbl = alloc_tce_table();
1446  if (!tbl)
1447  goto cleanup;
1448  info->tce_space = tbl;
1449  }
1450  calgary_found = 1;
1451  }
1452  }
1453 
1454  printk(KERN_DEBUG "Calgary: finished detection, Calgary %s\n",
1455  calgary_found ? "found" : "not found");
1456 
1457  if (calgary_found) {
1458  iommu_detected = 1;
1459  calgary_detected = 1;
1460  printk(KERN_INFO "PCI-DMA: Calgary IOMMU detected.\n");
1461  printk(KERN_INFO "PCI-DMA: Calgary TCE table spec is %d\n",
1463 
1464  x86_init.iommu.iommu_init = calgary_iommu_init;
1465  }
1466  return calgary_found;
1467 
1468 cleanup:
1469  for (--bus; bus >= 0; --bus) {
1470  struct calgary_bus_info *info = &bus_info[bus];
1471 
1472  if (info->tce_space)
1473  free_tce_table(info->tce_space);
1474  }
1475  return -ENOMEM;
1476 }
1477 
1478 static int __init calgary_parse_options(char *p)
1479 {
1480  unsigned int bridge;
1481  unsigned long val;
1482  size_t len;
1483  ssize_t ret;
1484 
1485  while (*p) {
1486  if (!strncmp(p, "64k", 3))
1488  else if (!strncmp(p, "128k", 4))
1490  else if (!strncmp(p, "256k", 4))
1492  else if (!strncmp(p, "512k", 4))
1494  else if (!strncmp(p, "1M", 2))
1496  else if (!strncmp(p, "2M", 2))
1498  else if (!strncmp(p, "4M", 2))
1500  else if (!strncmp(p, "8M", 2))
1502 
1503  len = strlen("translate_empty_slots");
1504  if (!strncmp(p, "translate_empty_slots", len))
1505  translate_empty_slots = 1;
1506 
1507  len = strlen("disable");
1508  if (!strncmp(p, "disable", len)) {
1509  p += len;
1510  if (*p == '=')
1511  ++p;
1512  if (*p == '\0')
1513  break;
1514  ret = kstrtoul(p, 0, &val);
1515  if (ret)
1516  break;
1517 
1518  bridge = val;
1519  if (bridge < MAX_PHB_BUS_NUM) {
1520  printk(KERN_INFO "Calgary: disabling "
1521  "translation for PHB %#x\n", bridge);
1522  bus_info[bridge].translation_disabled = 1;
1523  }
1524  }
1525 
1526  p = strpbrk(p, ",");
1527  if (!p)
1528  break;
1529 
1530  p++; /* skip ',' */
1531  }
1532  return 1;
1533 }
1534 __setup("calgary=", calgary_parse_options);
1535 
1536 static void __init calgary_fixup_one_tce_space(struct pci_dev *dev)
1537 {
1538  struct iommu_table *tbl;
1539  unsigned int npages;
1540  int i;
1541 
1542  tbl = pci_iommu(dev->bus);
1543 
1544  for (i = 0; i < 4; i++) {
1545  struct resource *r = &dev->resource[PCI_BRIDGE_RESOURCES + i];
1546 
1547  /* Don't give out TCEs that map MEM resources */
1548  if (!(r->flags & IORESOURCE_MEM))
1549  continue;
1550 
1551  /* 0-based? we reserve the whole 1st MB anyway */
1552  if (!r->start)
1553  continue;
1554 
1555  /* cover the whole region */
1556  npages = resource_size(r) >> PAGE_SHIFT;
1557  npages++;
1558 
1559  iommu_range_reserve(tbl, r->start, npages);
1560  }
1561 }
1562 
1563 static int __init calgary_fixup_tce_spaces(void)
1564 {
1565  struct pci_dev *dev = NULL;
1566  struct calgary_bus_info *info;
1567 
1568  if (no_iommu || swiotlb || !calgary_detected)
1569  return -ENODEV;
1570 
1571  printk(KERN_DEBUG "Calgary: fixing up tce spaces\n");
1572 
1573  do {
1575  if (!dev)
1576  break;
1577  if (!is_cal_pci_dev(dev->device))
1578  continue;
1579 
1580  info = &bus_info[dev->bus->number];
1581  if (info->translation_disabled)
1582  continue;
1583 
1584  if (!info->tce_space)
1585  continue;
1586 
1587  calgary_fixup_one_tce_space(dev);
1588 
1589  } while (1);
1590 
1591  return 0;
1592 }
1593 
1594 /*
1595  * We need to be call after pcibios_assign_resources (fs_initcall level)
1596  * and before device_initcall.
1597  */
1598 rootfs_initcall(calgary_fixup_tce_spaces);
1599