Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
exynos-iommu.c
Go to the documentation of this file.
1 /* linux/drivers/iommu/exynos_iommu.c
2  *
3  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
4  * http://www.samsung.com
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 
11 #ifdef CONFIG_EXYNOS_IOMMU_DEBUG
12 #define DEBUG
13 #endif
14 
15 #include <linux/io.h>
16 #include <linux/interrupt.h>
17 #include <linux/platform_device.h>
18 #include <linux/slab.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/clk.h>
21 #include <linux/err.h>
22 #include <linux/mm.h>
23 #include <linux/iommu.h>
24 #include <linux/errno.h>
25 #include <linux/list.h>
26 #include <linux/memblock.h>
27 #include <linux/export.h>
28 
29 #include <asm/cacheflush.h>
30 #include <asm/pgtable.h>
31 
32 #include <mach/sysmmu.h>
33 
34 /* We does not consider super section mapping (16MB) */
35 #define SECT_ORDER 20
36 #define LPAGE_ORDER 16
37 #define SPAGE_ORDER 12
38 
39 #define SECT_SIZE (1 << SECT_ORDER)
40 #define LPAGE_SIZE (1 << LPAGE_ORDER)
41 #define SPAGE_SIZE (1 << SPAGE_ORDER)
42 
43 #define SECT_MASK (~(SECT_SIZE - 1))
44 #define LPAGE_MASK (~(LPAGE_SIZE - 1))
45 #define SPAGE_MASK (~(SPAGE_SIZE - 1))
46 
47 #define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
48 #define lv1ent_page(sent) ((*(sent) & 3) == 1)
49 #define lv1ent_section(sent) ((*(sent) & 3) == 2)
50 
51 #define lv2ent_fault(pent) ((*(pent) & 3) == 0)
52 #define lv2ent_small(pent) ((*(pent) & 2) == 2)
53 #define lv2ent_large(pent) ((*(pent) & 3) == 1)
54 
55 #define section_phys(sent) (*(sent) & SECT_MASK)
56 #define section_offs(iova) ((iova) & 0xFFFFF)
57 #define lpage_phys(pent) (*(pent) & LPAGE_MASK)
58 #define lpage_offs(iova) ((iova) & 0xFFFF)
59 #define spage_phys(pent) (*(pent) & SPAGE_MASK)
60 #define spage_offs(iova) ((iova) & 0xFFF)
61 
62 #define lv1ent_offset(iova) ((iova) >> SECT_ORDER)
63 #define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER)
64 
65 #define NUM_LV1ENTRIES 4096
66 #define NUM_LV2ENTRIES 256
67 
68 #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long))
69 
70 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
71 
72 #define lv2table_base(sent) (*(sent) & 0xFFFFFC00)
73 
74 #define mk_lv1ent_sect(pa) ((pa) | 2)
75 #define mk_lv1ent_page(pa) ((pa) | 1)
76 #define mk_lv2ent_lpage(pa) ((pa) | 1)
77 #define mk_lv2ent_spage(pa) ((pa) | 2)
78 
79 #define CTRL_ENABLE 0x5
80 #define CTRL_BLOCK 0x7
81 #define CTRL_DISABLE 0x0
82 
83 #define REG_MMU_CTRL 0x000
84 #define REG_MMU_CFG 0x004
85 #define REG_MMU_STATUS 0x008
86 #define REG_MMU_FLUSH 0x00C
87 #define REG_MMU_FLUSH_ENTRY 0x010
88 #define REG_PT_BASE_ADDR 0x014
89 #define REG_INT_STATUS 0x018
90 #define REG_INT_CLEAR 0x01C
91 
92 #define REG_PAGE_FAULT_ADDR 0x024
93 #define REG_AW_FAULT_ADDR 0x028
94 #define REG_AR_FAULT_ADDR 0x02C
95 #define REG_DEFAULT_SLAVE_ADDR 0x030
96 
97 #define REG_MMU_VERSION 0x034
98 
99 #define REG_PB0_SADDR 0x04C
100 #define REG_PB0_EADDR 0x050
101 #define REG_PB1_SADDR 0x054
102 #define REG_PB1_EADDR 0x058
103 
104 static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova)
105 {
106  return pgtable + lv1ent_offset(iova);
107 }
108 
109 static unsigned long *page_entry(unsigned long *sent, unsigned long iova)
110 {
111  return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova);
112 }
113 
125 };
126 
127 /*
128  * @itype: type of fault.
129  * @pgtable_base: the physical address of page table base. This is 0 if @itype
130  * is SYSMMU_BUSERROR.
131  * @fault_addr: the device (virtual) address that the System MMU tried to
132  * translated. This is 0 if @itype is SYSMMU_BUSERROR.
133  */
135  unsigned long pgtable_base, unsigned long fault_addr);
136 
137 static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
145  REG_AW_FAULT_ADDR
146 };
147 
148 static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
149  "PAGE FAULT",
150  "AR MULTI-HIT FAULT",
151  "AW MULTI-HIT FAULT",
152  "BUS ERROR",
153  "AR SECURITY PROTECTION FAULT",
154  "AR ACCESS PROTECTION FAULT",
155  "AW SECURITY PROTECTION FAULT",
156  "AW ACCESS PROTECTION FAULT",
157  "UNKNOWN FAULT"
158 };
159 
161  struct list_head clients; /* list of sysmmu_drvdata.node */
162  unsigned long *pgtable; /* lv1 page table, 16KB */
163  short *lv2entcnt; /* free lv2 entry counter for each section */
164  spinlock_t lock; /* lock for this structure */
165  spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
166 };
167 
169  struct list_head node; /* entry of exynos_iommu_domain.clients */
170  struct device *sysmmu; /* System MMU's device descriptor */
171  struct device *dev; /* Owner of system MMU */
172  char *dbgname;
173  int nsfrs;
175  struct clk *clk[2];
180  unsigned long pgtable;
181 };
182 
183 static bool set_sysmmu_active(struct sysmmu_drvdata *data)
184 {
185  /* return true if the System MMU was not active previously
186  and it needs to be initialized */
187  return ++data->activations == 1;
188 }
189 
190 static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)
191 {
192  /* return true if the System MMU is needed to be disabled */
193  BUG_ON(data->activations < 1);
194  return --data->activations == 0;
195 }
196 
197 static bool is_sysmmu_active(struct sysmmu_drvdata *data)
198 {
199  return data->activations > 0;
200 }
201 
202 static void sysmmu_unblock(void __iomem *sfrbase)
203 {
205 }
206 
207 static bool sysmmu_block(void __iomem *sfrbase)
208 {
209  int i = 120;
210 
212  while ((i > 0) && !(__raw_readl(sfrbase + REG_MMU_STATUS) & 1))
213  --i;
214 
215  if (!(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) {
216  sysmmu_unblock(sfrbase);
217  return false;
218  }
219 
220  return true;
221 }
222 
223 static void __sysmmu_tlb_invalidate(void __iomem *sfrbase)
224 {
225  __raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
226 }
227 
228 static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
229  unsigned long iova)
230 {
231  __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY);
232 }
233 
234 static void __sysmmu_set_ptbase(void __iomem *sfrbase,
235  unsigned long pgd)
236 {
237  __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */
238  __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);
239 
240  __sysmmu_tlb_invalidate(sfrbase);
241 }
242 
243 static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base,
244  unsigned long size, int idx)
245 {
246  __raw_writel(base, sfrbase + REG_PB0_SADDR + idx * 8);
247  __raw_writel(size - 1 + base, sfrbase + REG_PB0_EADDR + idx * 8);
248 }
249 
251  unsigned long base0, unsigned long size0,
252  unsigned long base1, unsigned long size1)
253 {
254  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
255  unsigned long flags;
256  int i;
257 
258  BUG_ON((base0 + size0) <= base0);
259  BUG_ON((size1 > 0) && ((base1 + size1) <= base1));
260 
261  read_lock_irqsave(&data->lock, flags);
262  if (!is_sysmmu_active(data))
263  goto finish;
264 
265  for (i = 0; i < data->nsfrs; i++) {
266  if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) {
267  if (!sysmmu_block(data->sfrbases[i]))
268  continue;
269 
270  if (size1 == 0) {
271  if (size0 <= SZ_128K) {
272  base1 = base0;
273  size1 = size0;
274  } else {
275  size1 = size0 -
276  ALIGN(size0 / 2, SZ_64K);
277  size0 = size0 - size1;
278  base1 = base0 + size0;
279  }
280  }
281 
282  __sysmmu_set_prefbuf(
283  data->sfrbases[i], base0, size0, 0);
284  __sysmmu_set_prefbuf(
285  data->sfrbases[i], base1, size1, 1);
286 
287  sysmmu_unblock(data->sfrbases[i]);
288  }
289  }
290 finish:
291  read_unlock_irqrestore(&data->lock, flags);
292 }
293 
294 static void __set_fault_handler(struct sysmmu_drvdata *data,
295  sysmmu_fault_handler_t handler)
296 {
297  unsigned long flags;
298 
299  write_lock_irqsave(&data->lock, flags);
300  data->fault_handler = handler;
301  write_unlock_irqrestore(&data->lock, flags);
302 }
303 
305  sysmmu_fault_handler_t handler)
306 {
307  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
308 
309  __set_fault_handler(data, handler);
310 }
311 
312 static int default_fault_handler(enum exynos_sysmmu_inttype itype,
313  unsigned long pgtable_base, unsigned long fault_addr)
314 {
315  unsigned long *ent;
316 
317  if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
318  itype = SYSMMU_FAULT_UNKNOWN;
319 
320  pr_err("%s occurred at 0x%lx(Page table base: 0x%lx)\n",
321  sysmmu_fault_name[itype], fault_addr, pgtable_base);
322 
323  ent = section_entry(__va(pgtable_base), fault_addr);
324  pr_err("\tLv1 entry: 0x%lx\n", *ent);
325 
326  if (lv1ent_page(ent)) {
327  ent = page_entry(ent, fault_addr);
328  pr_err("\t Lv2 entry: 0x%lx\n", *ent);
329  }
330 
331  pr_err("Generating Kernel OOPS... because it is unrecoverable.\n");
332 
333  BUG();
334 
335  return 0;
336 }
337 
338 static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
339 {
340  /* SYSMMU is in blocked when interrupt occurred. */
341  struct sysmmu_drvdata *data = dev_id;
342  struct resource *irqres;
343  struct platform_device *pdev;
344  enum exynos_sysmmu_inttype itype;
345  unsigned long addr = -1;
346 
347  int i, ret = -ENOSYS;
348 
349  read_lock(&data->lock);
350 
351  WARN_ON(!is_sysmmu_active(data));
352 
353  pdev = to_platform_device(data->sysmmu);
354  for (i = 0; i < (pdev->num_resources / 2); i++) {
355  irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
356  if (irqres && ((int)irqres->start == irq))
357  break;
358  }
359 
360  if (i == pdev->num_resources) {
361  itype = SYSMMU_FAULT_UNKNOWN;
362  } else {
363  itype = (enum exynos_sysmmu_inttype)
365  if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
366  itype = SYSMMU_FAULT_UNKNOWN;
367  else
368  addr = __raw_readl(
369  data->sfrbases[i] + fault_reg_offset[itype]);
370  }
371 
372  if (data->domain)
373  ret = report_iommu_fault(data->domain, data->dev,
374  addr, itype);
375 
376  if ((ret == -ENOSYS) && data->fault_handler) {
377  unsigned long base = data->pgtable;
378  if (itype != SYSMMU_FAULT_UNKNOWN)
379  base = __raw_readl(
380  data->sfrbases[i] + REG_PT_BASE_ADDR);
381  ret = data->fault_handler(itype, base, addr);
382  }
383 
384  if (!ret && (itype != SYSMMU_FAULT_UNKNOWN))
385  __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR);
386  else
387  dev_dbg(data->sysmmu, "(%s) %s is not handled.\n",
388  data->dbgname, sysmmu_fault_name[itype]);
389 
390  if (itype != SYSMMU_FAULT_UNKNOWN)
391  sysmmu_unblock(data->sfrbases[i]);
392 
393  read_unlock(&data->lock);
394 
395  return IRQ_HANDLED;
396 }
397 
398 static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data)
399 {
400  unsigned long flags;
401  bool disabled = false;
402  int i;
403 
404  write_lock_irqsave(&data->lock, flags);
405 
406  if (!set_sysmmu_inactive(data))
407  goto finish;
408 
409  for (i = 0; i < data->nsfrs; i++)
411 
412  if (data->clk[1])
413  clk_disable(data->clk[1]);
414  if (data->clk[0])
415  clk_disable(data->clk[0]);
416 
417  disabled = true;
418  data->pgtable = 0;
419  data->domain = NULL;
420 finish:
421  write_unlock_irqrestore(&data->lock, flags);
422 
423  if (disabled)
424  dev_dbg(data->sysmmu, "(%s) Disabled\n", data->dbgname);
425  else
426  dev_dbg(data->sysmmu, "(%s) %d times left to be disabled\n",
427  data->dbgname, data->activations);
428 
429  return disabled;
430 }
431 
432 /* __exynos_sysmmu_enable: Enables System MMU
433  *
434  * returns -error if an error occurred and System MMU is not enabled,
435  * 0 if the System MMU has been just enabled and 1 if System MMU was already
436  * enabled before.
437  */
438 static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data,
439  unsigned long pgtable, struct iommu_domain *domain)
440 {
441  int i, ret = 0;
442  unsigned long flags;
443 
444  write_lock_irqsave(&data->lock, flags);
445 
446  if (!set_sysmmu_active(data)) {
447  if (WARN_ON(pgtable != data->pgtable)) {
448  ret = -EBUSY;
449  set_sysmmu_inactive(data);
450  } else {
451  ret = 1;
452  }
453 
454  dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname);
455  goto finish;
456  }
457 
458  if (data->clk[0])
459  clk_enable(data->clk[0]);
460  if (data->clk[1])
461  clk_enable(data->clk[1]);
462 
463  data->pgtable = pgtable;
464 
465  for (i = 0; i < data->nsfrs; i++) {
466  __sysmmu_set_ptbase(data->sfrbases[i], pgtable);
467 
468  if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) {
469  /* System MMU version is 3.x */
470  __raw_writel((1 << 12) | (2 << 28),
471  data->sfrbases[i] + REG_MMU_CFG);
472  __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0);
473  __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1);
474  }
475 
477  }
478 
479  data->domain = domain;
480 
481  dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname);
482 finish:
483  write_unlock_irqrestore(&data->lock, flags);
484 
485  return ret;
486 }
487 
488 int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable)
489 {
490  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
491  int ret;
492 
493  BUG_ON(!memblock_is_memory(pgtable));
494 
495  ret = pm_runtime_get_sync(data->sysmmu);
496  if (ret < 0) {
497  dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname);
498  return ret;
499  }
500 
501  ret = __exynos_sysmmu_enable(data, pgtable, NULL);
502  if (WARN_ON(ret < 0)) {
503  pm_runtime_put(data->sysmmu);
504  dev_err(data->sysmmu,
505  "(%s) Already enabled with page table %#lx\n",
506  data->dbgname, data->pgtable);
507  } else {
508  data->dev = dev;
509  }
510 
511  return ret;
512 }
513 
515 {
516  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
517  bool disabled;
518 
519  disabled = __exynos_sysmmu_disable(data);
520  pm_runtime_put(data->sysmmu);
521 
522  return disabled;
523 }
524 
525 static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova)
526 {
527  unsigned long flags;
528  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
529 
530  read_lock_irqsave(&data->lock, flags);
531 
532  if (is_sysmmu_active(data)) {
533  int i;
534  for (i = 0; i < data->nsfrs; i++) {
535  if (sysmmu_block(data->sfrbases[i])) {
536  __sysmmu_tlb_invalidate_entry(
537  data->sfrbases[i], iova);
538  sysmmu_unblock(data->sfrbases[i]);
539  }
540  }
541  } else {
542  dev_dbg(data->sysmmu,
543  "(%s) Disabled. Skipping invalidating TLB.\n",
544  data->dbgname);
545  }
546 
547  read_unlock_irqrestore(&data->lock, flags);
548 }
549 
551 {
552  unsigned long flags;
553  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
554 
555  read_lock_irqsave(&data->lock, flags);
556 
557  if (is_sysmmu_active(data)) {
558  int i;
559  for (i = 0; i < data->nsfrs; i++) {
560  if (sysmmu_block(data->sfrbases[i])) {
561  __sysmmu_tlb_invalidate(data->sfrbases[i]);
562  sysmmu_unblock(data->sfrbases[i]);
563  }
564  }
565  } else {
566  dev_dbg(data->sysmmu,
567  "(%s) Disabled. Skipping invalidating TLB.\n",
568  data->dbgname);
569  }
570 
571  read_unlock_irqrestore(&data->lock, flags);
572 }
573 
574 static int exynos_sysmmu_probe(struct platform_device *pdev)
575 {
576  int i, ret;
577  struct device *dev;
578  struct sysmmu_drvdata *data;
579 
580  dev = &pdev->dev;
581 
582  data = kzalloc(sizeof(*data), GFP_KERNEL);
583  if (!data) {
584  dev_dbg(dev, "Not enough memory\n");
585  ret = -ENOMEM;
586  goto err_alloc;
587  }
588 
589  ret = dev_set_drvdata(dev, data);
590  if (ret) {
591  dev_dbg(dev, "Unabled to initialize driver data\n");
592  goto err_init;
593  }
594 
595  data->nsfrs = pdev->num_resources / 2;
596  data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs,
597  GFP_KERNEL);
598  if (data->sfrbases == NULL) {
599  dev_dbg(dev, "Not enough memory\n");
600  ret = -ENOMEM;
601  goto err_init;
602  }
603 
604  for (i = 0; i < data->nsfrs; i++) {
605  struct resource *res;
606  res = platform_get_resource(pdev, IORESOURCE_MEM, i);
607  if (!res) {
608  dev_dbg(dev, "Unable to find IOMEM region\n");
609  ret = -ENOENT;
610  goto err_res;
611  }
612 
613  data->sfrbases[i] = ioremap(res->start, resource_size(res));
614  if (!data->sfrbases[i]) {
615  dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
616  res->start);
617  ret = -ENOENT;
618  goto err_res;
619  }
620  }
621 
622  for (i = 0; i < data->nsfrs; i++) {
623  ret = platform_get_irq(pdev, i);
624  if (ret <= 0) {
625  dev_dbg(dev, "Unable to find IRQ resource\n");
626  goto err_irq;
627  }
628 
629  ret = request_irq(ret, exynos_sysmmu_irq, 0,
630  dev_name(dev), data);
631  if (ret) {
632  dev_dbg(dev, "Unabled to register interrupt handler\n");
633  goto err_irq;
634  }
635  }
636 
637  if (dev_get_platdata(dev)) {
638  char *deli, *beg;
639  struct sysmmu_platform_data *platdata = dev_get_platdata(dev);
640 
641  beg = platdata->clockname;
642 
643  for (deli = beg; (*deli != '\0') && (*deli != ','); deli++)
644  /* NOTHING */;
645 
646  if (*deli == '\0')
647  deli = NULL;
648  else
649  *deli = '\0';
650 
651  data->clk[0] = clk_get(dev, beg);
652  if (IS_ERR(data->clk[0])) {
653  data->clk[0] = NULL;
654  dev_dbg(dev, "No clock descriptor registered\n");
655  }
656 
657  if (data->clk[0] && deli) {
658  *deli = ',';
659  data->clk[1] = clk_get(dev, deli + 1);
660  if (IS_ERR(data->clk[1]))
661  data->clk[1] = NULL;
662  }
663 
664  data->dbgname = platdata->dbgname;
665  }
666 
667  data->sysmmu = dev;
668  rwlock_init(&data->lock);
669  INIT_LIST_HEAD(&data->node);
670 
671  __set_fault_handler(data, &default_fault_handler);
672 
673  if (dev->parent)
674  pm_runtime_enable(dev);
675 
676  dev_dbg(dev, "(%s) Initialized\n", data->dbgname);
677  return 0;
678 err_irq:
679  while (i-- > 0) {
680  int irq;
681 
682  irq = platform_get_irq(pdev, i);
683  free_irq(irq, data);
684  }
685 err_res:
686  while (data->nsfrs-- > 0)
687  iounmap(data->sfrbases[data->nsfrs]);
688  kfree(data->sfrbases);
689 err_init:
690  kfree(data);
691 err_alloc:
692  dev_err(dev, "Failed to initialize\n");
693  return ret;
694 }
695 
696 static struct platform_driver exynos_sysmmu_driver = {
697  .probe = exynos_sysmmu_probe,
698  .driver = {
699  .owner = THIS_MODULE,
700  .name = "exynos-sysmmu",
701  }
702 };
703 
704 static inline void pgtable_flush(void *vastart, void *vaend)
705 {
706  dmac_flush_range(vastart, vaend);
707  outer_flush_range(virt_to_phys(vastart),
708  virt_to_phys(vaend));
709 }
710 
711 static int exynos_iommu_domain_init(struct iommu_domain *domain)
712 {
713  struct exynos_iommu_domain *priv;
714 
715  priv = kzalloc(sizeof(*priv), GFP_KERNEL);
716  if (!priv)
717  return -ENOMEM;
718 
719  priv->pgtable = (unsigned long *)__get_free_pages(
720  GFP_KERNEL | __GFP_ZERO, 2);
721  if (!priv->pgtable)
722  goto err_pgtable;
723 
724  priv->lv2entcnt = (short *)__get_free_pages(
725  GFP_KERNEL | __GFP_ZERO, 1);
726  if (!priv->lv2entcnt)
727  goto err_counter;
728 
729  pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES);
730 
731  spin_lock_init(&priv->lock);
732  spin_lock_init(&priv->pgtablelock);
733  INIT_LIST_HEAD(&priv->clients);
734 
735  domain->geometry.aperture_start = 0;
736  domain->geometry.aperture_end = ~0UL;
737  domain->geometry.force_aperture = true;
738 
739  domain->priv = priv;
740  return 0;
741 
742 err_counter:
743  free_pages((unsigned long)priv->pgtable, 2);
744 err_pgtable:
745  kfree(priv);
746  return -ENOMEM;
747 }
748 
749 static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
750 {
751  struct exynos_iommu_domain *priv = domain->priv;
752  struct sysmmu_drvdata *data;
753  unsigned long flags;
754  int i;
755 
756  WARN_ON(!list_empty(&priv->clients));
757 
758  spin_lock_irqsave(&priv->lock, flags);
759 
760  list_for_each_entry(data, &priv->clients, node) {
761  while (!exynos_sysmmu_disable(data->dev))
762  ; /* until System MMU is actually disabled */
763  }
764 
765  spin_unlock_irqrestore(&priv->lock, flags);
766 
767  for (i = 0; i < NUM_LV1ENTRIES; i++)
768  if (lv1ent_page(priv->pgtable + i))
769  kfree(__va(lv2table_base(priv->pgtable + i)));
770 
771  free_pages((unsigned long)priv->pgtable, 2);
772  free_pages((unsigned long)priv->lv2entcnt, 1);
773  kfree(domain->priv);
774  domain->priv = NULL;
775 }
776 
777 static int exynos_iommu_attach_device(struct iommu_domain *domain,
778  struct device *dev)
779 {
780  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
781  struct exynos_iommu_domain *priv = domain->priv;
782  unsigned long flags;
783  int ret;
784 
785  ret = pm_runtime_get_sync(data->sysmmu);
786  if (ret < 0)
787  return ret;
788 
789  ret = 0;
790 
791  spin_lock_irqsave(&priv->lock, flags);
792 
793  ret = __exynos_sysmmu_enable(data, __pa(priv->pgtable), domain);
794 
795  if (ret == 0) {
796  /* 'data->node' must not be appeared in priv->clients */
797  BUG_ON(!list_empty(&data->node));
798  data->dev = dev;
799  list_add_tail(&data->node, &priv->clients);
800  }
801 
802  spin_unlock_irqrestore(&priv->lock, flags);
803 
804  if (ret < 0) {
805  dev_err(dev, "%s: Failed to attach IOMMU with pgtable %#lx\n",
806  __func__, __pa(priv->pgtable));
807  pm_runtime_put(data->sysmmu);
808  } else if (ret > 0) {
809  dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n",
810  __func__, __pa(priv->pgtable));
811  } else {
812  dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n",
813  __func__, __pa(priv->pgtable));
814  }
815 
816  return ret;
817 }
818 
819 static void exynos_iommu_detach_device(struct iommu_domain *domain,
820  struct device *dev)
821 {
822  struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
823  struct exynos_iommu_domain *priv = domain->priv;
824  struct list_head *pos;
825  unsigned long flags;
826  bool found = false;
827 
828  spin_lock_irqsave(&priv->lock, flags);
829 
830  list_for_each(pos, &priv->clients) {
831  if (list_entry(pos, struct sysmmu_drvdata, node) == data) {
832  found = true;
833  break;
834  }
835  }
836 
837  if (!found)
838  goto finish;
839 
840  if (__exynos_sysmmu_disable(data)) {
841  dev_dbg(dev, "%s: Detached IOMMU with pgtable %#lx\n",
842  __func__, __pa(priv->pgtable));
843  list_del_init(&data->node);
844 
845  } else {
846  dev_dbg(dev, "%s: Detaching IOMMU with pgtable %#lx delayed",
847  __func__, __pa(priv->pgtable));
848  }
849 
850 finish:
851  spin_unlock_irqrestore(&priv->lock, flags);
852 
853  if (found)
854  pm_runtime_put(data->sysmmu);
855 }
856 
857 static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova,
858  short *pgcounter)
859 {
860  if (lv1ent_fault(sent)) {
861  unsigned long *pent;
862 
863  pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC);
864  BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1));
865  if (!pent)
866  return NULL;
867 
868  *sent = mk_lv1ent_page(__pa(pent));
869  *pgcounter = NUM_LV2ENTRIES;
870  pgtable_flush(pent, pent + NUM_LV2ENTRIES);
871  pgtable_flush(sent, sent + 1);
872  }
873 
874  return page_entry(sent, iova);
875 }
876 
877 static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt)
878 {
879  if (lv1ent_section(sent))
880  return -EADDRINUSE;
881 
882  if (lv1ent_page(sent)) {
883  if (*pgcnt != NUM_LV2ENTRIES)
884  return -EADDRINUSE;
885 
886  kfree(page_entry(sent, 0));
887 
888  *pgcnt = 0;
889  }
890 
891  *sent = mk_lv1ent_sect(paddr);
892 
893  pgtable_flush(sent, sent + 1);
894 
895  return 0;
896 }
897 
898 static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
899  short *pgcnt)
900 {
901  if (size == SPAGE_SIZE) {
902  if (!lv2ent_fault(pent))
903  return -EADDRINUSE;
904 
905  *pent = mk_lv2ent_spage(paddr);
906  pgtable_flush(pent, pent + 1);
907  *pgcnt -= 1;
908  } else { /* size == LPAGE_SIZE */
909  int i;
910  for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
911  if (!lv2ent_fault(pent)) {
912  memset(pent, 0, sizeof(*pent) * i);
913  return -EADDRINUSE;
914  }
915 
916  *pent = mk_lv2ent_lpage(paddr);
917  }
918  pgtable_flush(pent - SPAGES_PER_LPAGE, pent);
919  *pgcnt -= SPAGES_PER_LPAGE;
920  }
921 
922  return 0;
923 }
924 
925 static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova,
926  phys_addr_t paddr, size_t size, int prot)
927 {
928  struct exynos_iommu_domain *priv = domain->priv;
929  unsigned long *entry;
930  unsigned long flags;
931  int ret = -ENOMEM;
932 
933  BUG_ON(priv->pgtable == NULL);
934 
935  spin_lock_irqsave(&priv->pgtablelock, flags);
936 
937  entry = section_entry(priv->pgtable, iova);
938 
939  if (size == SECT_SIZE) {
940  ret = lv1set_section(entry, paddr,
941  &priv->lv2entcnt[lv1ent_offset(iova)]);
942  } else {
943  unsigned long *pent;
944 
945  pent = alloc_lv2entry(entry, iova,
946  &priv->lv2entcnt[lv1ent_offset(iova)]);
947 
948  if (!pent)
949  ret = -ENOMEM;
950  else
951  ret = lv2set_page(pent, paddr, size,
952  &priv->lv2entcnt[lv1ent_offset(iova)]);
953  }
954 
955  if (ret) {
956  pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n",
957  __func__, iova, size);
958  }
959 
960  spin_unlock_irqrestore(&priv->pgtablelock, flags);
961 
962  return ret;
963 }
964 
965 static size_t exynos_iommu_unmap(struct iommu_domain *domain,
966  unsigned long iova, size_t size)
967 {
968  struct exynos_iommu_domain *priv = domain->priv;
969  struct sysmmu_drvdata *data;
970  unsigned long flags;
971  unsigned long *ent;
972 
973  BUG_ON(priv->pgtable == NULL);
974 
975  spin_lock_irqsave(&priv->pgtablelock, flags);
976 
977  ent = section_entry(priv->pgtable, iova);
978 
979  if (lv1ent_section(ent)) {
980  BUG_ON(size < SECT_SIZE);
981 
982  *ent = 0;
983  pgtable_flush(ent, ent + 1);
984  size = SECT_SIZE;
985  goto done;
986  }
987 
988  if (unlikely(lv1ent_fault(ent))) {
989  if (size > SECT_SIZE)
990  size = SECT_SIZE;
991  goto done;
992  }
993 
994  /* lv1ent_page(sent) == true here */
995 
996  ent = page_entry(ent, iova);
997 
998  if (unlikely(lv2ent_fault(ent))) {
999  size = SPAGE_SIZE;
1000  goto done;
1001  }
1002 
1003  if (lv2ent_small(ent)) {
1004  *ent = 0;
1005  size = SPAGE_SIZE;
1006  priv->lv2entcnt[lv1ent_offset(iova)] += 1;
1007  goto done;
1008  }
1009 
1010  /* lv1ent_large(ent) == true here */
1011  BUG_ON(size < LPAGE_SIZE);
1012 
1013  memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
1014 
1015  size = LPAGE_SIZE;
1016  priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
1017 done:
1018  spin_unlock_irqrestore(&priv->pgtablelock, flags);
1019 
1020  spin_lock_irqsave(&priv->lock, flags);
1021  list_for_each_entry(data, &priv->clients, node)
1022  sysmmu_tlb_invalidate_entry(data->dev, iova);
1023  spin_unlock_irqrestore(&priv->lock, flags);
1024 
1025 
1026  return size;
1027 }
1028 
1029 static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
1030  unsigned long iova)
1031 {
1032  struct exynos_iommu_domain *priv = domain->priv;
1033  unsigned long *entry;
1034  unsigned long flags;
1035  phys_addr_t phys = 0;
1036 
1037  spin_lock_irqsave(&priv->pgtablelock, flags);
1038 
1039  entry = section_entry(priv->pgtable, iova);
1040 
1041  if (lv1ent_section(entry)) {
1042  phys = section_phys(entry) + section_offs(iova);
1043  } else if (lv1ent_page(entry)) {
1044  entry = page_entry(entry, iova);
1045 
1046  if (lv2ent_large(entry))
1047  phys = lpage_phys(entry) + lpage_offs(iova);
1048  else if (lv2ent_small(entry))
1049  phys = spage_phys(entry) + spage_offs(iova);
1050  }
1051 
1052  spin_unlock_irqrestore(&priv->pgtablelock, flags);
1053 
1054  return phys;
1055 }
1056 
1057 static struct iommu_ops exynos_iommu_ops = {
1058  .domain_init = &exynos_iommu_domain_init,
1059  .domain_destroy = &exynos_iommu_domain_destroy,
1060  .attach_dev = &exynos_iommu_attach_device,
1061  .detach_dev = &exynos_iommu_detach_device,
1062  .map = &exynos_iommu_map,
1063  .unmap = &exynos_iommu_unmap,
1064  .iova_to_phys = &exynos_iommu_iova_to_phys,
1065  .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
1066 };
1067 
1068 static int __init exynos_iommu_init(void)
1069 {
1070  int ret;
1071 
1072  ret = platform_driver_register(&exynos_sysmmu_driver);
1073 
1074  if (ret == 0)
1075  bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
1076 
1077  return ret;
1078 }
1079 subsys_initcall(exynos_iommu_init);