Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
pat.c
Go to the documentation of this file.
1 /*
2  * Handle caching attributes in page tables (PAT)
3  *
4  * Authors: Venkatesh Pallipadi <[email protected]>
5  * Suresh B Siddha <[email protected]>
6  *
7  * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
8  */
9 
10 #include <linux/seq_file.h>
11 #include <linux/bootmem.h>
12 #include <linux/debugfs.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/slab.h>
16 #include <linux/mm.h>
17 #include <linux/fs.h>
18 #include <linux/rbtree.h>
19 
20 #include <asm/cacheflush.h>
21 #include <asm/processor.h>
22 #include <asm/tlbflush.h>
23 #include <asm/x86_init.h>
24 #include <asm/pgtable.h>
25 #include <asm/fcntl.h>
26 #include <asm/e820.h>
27 #include <asm/mtrr.h>
28 #include <asm/page.h>
29 #include <asm/msr.h>
30 #include <asm/pat.h>
31 #include <asm/io.h>
32 
33 #include "pat_internal.h"
34 
35 #ifdef CONFIG_X86_PAT
36 int __read_mostly pat_enabled = 1;
37 
38 static inline void pat_disable(const char *reason)
39 {
40  pat_enabled = 0;
41  printk(KERN_INFO "%s\n", reason);
42 }
43 
44 static int __init nopat(char *str)
45 {
46  pat_disable("PAT support disabled.");
47  return 0;
48 }
49 early_param("nopat", nopat);
50 #else
51 static inline void pat_disable(const char *reason)
52 {
53  (void)reason;
54 }
55 #endif
56 
57 
59 
60 static int __init pat_debug_setup(char *str)
61 {
62  pat_debug_enable = 1;
63  return 0;
64 }
65 __setup("debugpat", pat_debug_setup);
66 
67 static u64 __read_mostly boot_pat_state;
68 
69 enum {
70  PAT_UC = 0, /* uncached */
71  PAT_WC = 1, /* Write combining */
72  PAT_WT = 4, /* Write Through */
73  PAT_WP = 5, /* Write Protected */
74  PAT_WB = 6, /* Write Back (default) */
75  PAT_UC_MINUS = 7, /* UC, but can be overriden by MTRR */
76 };
77 
78 #define PAT(x, y) ((u64)PAT_ ## y << ((x)*8))
79 
80 void pat_init(void)
81 {
82  u64 pat;
83  bool boot_cpu = !boot_pat_state;
84 
85  if (!pat_enabled)
86  return;
87 
88  if (!cpu_has_pat) {
89  if (!boot_pat_state) {
90  pat_disable("PAT not supported by CPU.");
91  return;
92  } else {
93  /*
94  * If this happens we are on a secondary CPU, but
95  * switched to PAT on the boot CPU. We have no way to
96  * undo PAT.
97  */
98  printk(KERN_ERR "PAT enabled, "
99  "but not supported by secondary CPU\n");
100  BUG();
101  }
102  }
103 
104  /* Set PWT to Write-Combining. All other bits stay the same */
105  /*
106  * PTE encoding used in Linux:
107  * PAT
108  * |PCD
109  * ||PWT
110  * |||
111  * 000 WB _PAGE_CACHE_WB
112  * 001 WC _PAGE_CACHE_WC
113  * 010 UC- _PAGE_CACHE_UC_MINUS
114  * 011 UC _PAGE_CACHE_UC
115  * PAT bit unused
116  */
117  pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
118  PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
119 
120  /* Boot CPU check */
121  if (!boot_pat_state)
122  rdmsrl(MSR_IA32_CR_PAT, boot_pat_state);
123 
124  wrmsrl(MSR_IA32_CR_PAT, pat);
125 
126  if (boot_cpu)
127  printk(KERN_INFO "x86 PAT enabled: cpu %d, old 0x%Lx, new 0x%Lx\n",
128  smp_processor_id(), boot_pat_state, pat);
129 }
130 
131 #undef PAT
132 
133 static DEFINE_SPINLOCK(memtype_lock); /* protects memtype accesses */
134 
135 /*
136  * Does intersection of PAT memory type and MTRR memory type and returns
137  * the resulting memory type as PAT understands it.
138  * (Type in pat and mtrr will not have same value)
139  * The intersection is based on "Effective Memory Type" tables in IA-32
140  * SDM vol 3a
141  */
142 static unsigned long pat_x_mtrr_type(u64 start, u64 end, unsigned long req_type)
143 {
144  /*
145  * Look for MTRR hint to get the effective type in case where PAT
146  * request is for WB.
147  */
148  if (req_type == _PAGE_CACHE_WB) {
149  u8 mtrr_type;
150 
151  mtrr_type = mtrr_type_lookup(start, end);
152  if (mtrr_type != MTRR_TYPE_WRBACK)
153  return _PAGE_CACHE_UC_MINUS;
154 
155  return _PAGE_CACHE_WB;
156  }
157 
158  return req_type;
159 }
160 
162  unsigned long cur_pfn;
163  int ram;
164  int not_ram;
165 };
166 
167 static int
168 pagerange_is_ram_callback(unsigned long initial_pfn, unsigned long total_nr_pages, void *arg)
169 {
170  struct pagerange_state *state = arg;
171 
172  state->not_ram |= initial_pfn > state->cur_pfn;
173  state->ram |= total_nr_pages > 0;
174  state->cur_pfn = initial_pfn + total_nr_pages;
175 
176  return state->ram && state->not_ram;
177 }
178 
179 static int pat_pagerange_is_ram(resource_size_t start, resource_size_t end)
180 {
181  int ret = 0;
182  unsigned long start_pfn = start >> PAGE_SHIFT;
183  unsigned long end_pfn = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
184  struct pagerange_state state = {start_pfn, 0, 0};
185 
186  /*
187  * For legacy reasons, physical address range in the legacy ISA
188  * region is tracked as non-RAM. This will allow users of
189  * /dev/mem to map portions of legacy ISA region, even when
190  * some of those portions are listed(or not even listed) with
191  * different e820 types(RAM/reserved/..)
192  */
193  if (start_pfn < ISA_END_ADDRESS >> PAGE_SHIFT)
194  start_pfn = ISA_END_ADDRESS >> PAGE_SHIFT;
195 
196  if (start_pfn < end_pfn) {
197  ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
198  &state, pagerange_is_ram_callback);
199  }
200 
201  return (ret > 0) ? -1 : (state.ram ? 1 : 0);
202 }
203 
204 /*
205  * For RAM pages, we use page flags to mark the pages with appropriate type.
206  * Here we do two pass:
207  * - Find the memtype of all the pages in the range, look for any conflicts
208  * - In case of no conflicts, set the new memtype for pages in the range
209  */
210 static int reserve_ram_pages_type(u64 start, u64 end, unsigned long req_type,
211  unsigned long *new_type)
212 {
213  struct page *page;
214  u64 pfn;
215 
216  if (req_type == _PAGE_CACHE_UC) {
217  /* We do not support strong UC */
218  WARN_ON_ONCE(1);
219  req_type = _PAGE_CACHE_UC_MINUS;
220  }
221 
222  for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
223  unsigned long type;
224 
225  page = pfn_to_page(pfn);
226  type = get_page_memtype(page);
227  if (type != -1) {
228  printk(KERN_INFO "reserve_ram_pages_type failed [mem %#010Lx-%#010Lx], track 0x%lx, req 0x%lx\n",
229  start, end - 1, type, req_type);
230  if (new_type)
231  *new_type = type;
232 
233  return -EBUSY;
234  }
235  }
236 
237  if (new_type)
238  *new_type = req_type;
239 
240  for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
241  page = pfn_to_page(pfn);
242  set_page_memtype(page, req_type);
243  }
244  return 0;
245 }
246 
247 static int free_ram_pages_type(u64 start, u64 end)
248 {
249  struct page *page;
250  u64 pfn;
251 
252  for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
253  page = pfn_to_page(pfn);
254  set_page_memtype(page, -1);
255  }
256  return 0;
257 }
258 
259 /*
260  * req_type typically has one of the:
261  * - _PAGE_CACHE_WB
262  * - _PAGE_CACHE_WC
263  * - _PAGE_CACHE_UC_MINUS
264  * - _PAGE_CACHE_UC
265  *
266  * If new_type is NULL, function will return an error if it cannot reserve the
267  * region with req_type. If new_type is non-NULL, function will return
268  * available type in new_type in case of no error. In case of any error
269  * it will return a negative return value.
270  */
271 int reserve_memtype(u64 start, u64 end, unsigned long req_type,
272  unsigned long *new_type)
273 {
274  struct memtype *new;
275  unsigned long actual_type;
276  int is_range_ram;
277  int err = 0;
278 
279  BUG_ON(start >= end); /* end is exclusive */
280 
281  if (!pat_enabled) {
282  /* This is identical to page table setting without PAT */
283  if (new_type) {
284  if (req_type == _PAGE_CACHE_WC)
285  *new_type = _PAGE_CACHE_UC_MINUS;
286  else
287  *new_type = req_type & _PAGE_CACHE_MASK;
288  }
289  return 0;
290  }
291 
292  /* Low ISA region is always mapped WB in page table. No need to track */
293  if (x86_platform.is_untracked_pat_range(start, end)) {
294  if (new_type)
295  *new_type = _PAGE_CACHE_WB;
296  return 0;
297  }
298 
299  /*
300  * Call mtrr_lookup to get the type hint. This is an
301  * optimization for /dev/mem mmap'ers into WB memory (BIOS
302  * tools and ACPI tools). Use WB request for WB memory and use
303  * UC_MINUS otherwise.
304  */
305  actual_type = pat_x_mtrr_type(start, end, req_type & _PAGE_CACHE_MASK);
306 
307  if (new_type)
308  *new_type = actual_type;
309 
310  is_range_ram = pat_pagerange_is_ram(start, end);
311  if (is_range_ram == 1) {
312 
313  err = reserve_ram_pages_type(start, end, req_type, new_type);
314 
315  return err;
316  } else if (is_range_ram < 0) {
317  return -EINVAL;
318  }
319 
320  new = kzalloc(sizeof(struct memtype), GFP_KERNEL);
321  if (!new)
322  return -ENOMEM;
323 
324  new->start = start;
325  new->end = end;
326  new->type = actual_type;
327 
328  spin_lock(&memtype_lock);
329 
330  err = rbt_memtype_check_insert(new, new_type);
331  if (err) {
332  printk(KERN_INFO "reserve_memtype failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
333  start, end - 1,
334  cattr_name(new->type), cattr_name(req_type));
335  kfree(new);
336  spin_unlock(&memtype_lock);
337 
338  return err;
339  }
340 
341  spin_unlock(&memtype_lock);
342 
343  dprintk("reserve_memtype added [mem %#010Lx-%#010Lx], track %s, req %s, ret %s\n",
344  start, end - 1, cattr_name(new->type), cattr_name(req_type),
345  new_type ? cattr_name(*new_type) : "-");
346 
347  return err;
348 }
349 
350 int free_memtype(u64 start, u64 end)
351 {
352  int err = -EINVAL;
353  int is_range_ram;
354  struct memtype *entry;
355 
356  if (!pat_enabled)
357  return 0;
358 
359  /* Low ISA region is always mapped WB. No need to track */
360  if (x86_platform.is_untracked_pat_range(start, end))
361  return 0;
362 
363  is_range_ram = pat_pagerange_is_ram(start, end);
364  if (is_range_ram == 1) {
365 
366  err = free_ram_pages_type(start, end);
367 
368  return err;
369  } else if (is_range_ram < 0) {
370  return -EINVAL;
371  }
372 
373  spin_lock(&memtype_lock);
374  entry = rbt_memtype_erase(start, end);
375  spin_unlock(&memtype_lock);
376 
377  if (!entry) {
378  printk(KERN_INFO "%s:%d freeing invalid memtype [mem %#010Lx-%#010Lx]\n",
379  current->comm, current->pid, start, end - 1);
380  return -EINVAL;
381  }
382 
383  kfree(entry);
384 
385  dprintk("free_memtype request [mem %#010Lx-%#010Lx]\n", start, end - 1);
386 
387  return 0;
388 }
389 
390 
400 static unsigned long lookup_memtype(u64 paddr)
401 {
402  int rettype = _PAGE_CACHE_WB;
403  struct memtype *entry;
404 
405  if (x86_platform.is_untracked_pat_range(paddr, paddr + PAGE_SIZE))
406  return rettype;
407 
408  if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
409  struct page *page;
410  page = pfn_to_page(paddr >> PAGE_SHIFT);
411  rettype = get_page_memtype(page);
412  /*
413  * -1 from get_page_memtype() implies RAM page is in its
414  * default state and not reserved, and hence of type WB
415  */
416  if (rettype == -1)
417  rettype = _PAGE_CACHE_WB;
418 
419  return rettype;
420  }
421 
422  spin_lock(&memtype_lock);
423 
424  entry = rbt_memtype_lookup(paddr);
425  if (entry != NULL)
426  rettype = entry->type;
427  else
428  rettype = _PAGE_CACHE_UC_MINUS;
429 
430  spin_unlock(&memtype_lock);
431  return rettype;
432 }
433 
445  unsigned long *type)
446 {
447  resource_size_t size = end - start;
448  unsigned long req_type = *type;
449  unsigned long new_type;
450  int ret;
451 
452  WARN_ON_ONCE(iomem_map_sanity_check(start, size));
453 
454  ret = reserve_memtype(start, end, req_type, &new_type);
455  if (ret)
456  goto out_err;
457 
458  if (!is_new_memtype_allowed(start, size, req_type, new_type))
459  goto out_free;
460 
461  if (kernel_map_sync_memtype(start, size, new_type) < 0)
462  goto out_free;
463 
464  *type = new_type;
465  return 0;
466 
467 out_free:
468  free_memtype(start, end);
469  ret = -EBUSY;
470 out_err:
471  return ret;
472 }
473 
480 {
481  free_memtype(start, end);
482 }
483 
484 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
485  unsigned long size, pgprot_t vma_prot)
486 {
487  return vma_prot;
488 }
489 
490 #ifdef CONFIG_STRICT_DEVMEM
491 /* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM*/
492 static inline int range_is_allowed(unsigned long pfn, unsigned long size)
493 {
494  return 1;
495 }
496 #else
497 /* This check is needed to avoid cache aliasing when PAT is enabled */
498 static inline int range_is_allowed(unsigned long pfn, unsigned long size)
499 {
500  u64 from = ((u64)pfn) << PAGE_SHIFT;
501  u64 to = from + size;
502  u64 cursor = from;
503 
504  if (!pat_enabled)
505  return 1;
506 
507  while (cursor < to) {
508  if (!devmem_is_allowed(pfn)) {
509  printk(KERN_INFO "Program %s tried to access /dev/mem between [mem %#010Lx-%#010Lx]\n",
510  current->comm, from, to - 1);
511  return 0;
512  }
513  cursor += PAGE_SIZE;
514  pfn++;
515  }
516  return 1;
517 }
518 #endif /* CONFIG_STRICT_DEVMEM */
519 
520 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
521  unsigned long size, pgprot_t *vma_prot)
522 {
523  unsigned long flags = _PAGE_CACHE_WB;
524 
525  if (!range_is_allowed(pfn, size))
526  return 0;
527 
528  if (file->f_flags & O_DSYNC)
529  flags = _PAGE_CACHE_UC_MINUS;
530 
531 #ifdef CONFIG_X86_32
532  /*
533  * On the PPro and successors, the MTRRs are used to set
534  * memory types for physical addresses outside main memory,
535  * so blindly setting UC or PWT on those pages is wrong.
536  * For Pentiums and earlier, the surround logic should disable
537  * caching for the high addresses through the KEN pin, but
538  * we maintain the tradition of paranoia in this code.
539  */
540  if (!pat_enabled &&
541  !(boot_cpu_has(X86_FEATURE_MTRR) ||
542  boot_cpu_has(X86_FEATURE_K6_MTRR) ||
543  boot_cpu_has(X86_FEATURE_CYRIX_ARR) ||
544  boot_cpu_has(X86_FEATURE_CENTAUR_MCR)) &&
545  (pfn << PAGE_SHIFT) >= __pa(high_memory)) {
546  flags = _PAGE_CACHE_UC;
547  }
548 #endif
549 
550  *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) |
551  flags);
552  return 1;
553 }
554 
555 /*
556  * Change the memory type for the physial address range in kernel identity
557  * mapping space if that range is a part of identity map.
558  */
559 int kernel_map_sync_memtype(u64 base, unsigned long size, unsigned long flags)
560 {
561  unsigned long id_sz;
562 
563  if (base >= __pa(high_memory))
564  return 0;
565 
566  id_sz = (__pa(high_memory) < base + size) ?
567  __pa(high_memory) - base :
568  size;
569 
570  if (ioremap_change_attr((unsigned long)__va(base), id_sz, flags) < 0) {
571  printk(KERN_INFO "%s:%d ioremap_change_attr failed %s "
572  "for [mem %#010Lx-%#010Lx]\n",
573  current->comm, current->pid,
574  cattr_name(flags),
575  base, (unsigned long long)(base + size-1));
576  return -EINVAL;
577  }
578  return 0;
579 }
580 
581 /*
582  * Internal interface to reserve a range of physical memory with prot.
583  * Reserved non RAM regions only and after successful reserve_memtype,
584  * this func also keeps identity mapping (if any) in sync with this new prot.
585  */
586 static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
587  int strict_prot)
588 {
589  int is_ram = 0;
590  int ret;
591  unsigned long want_flags = (pgprot_val(*vma_prot) & _PAGE_CACHE_MASK);
592  unsigned long flags = want_flags;
593 
594  is_ram = pat_pagerange_is_ram(paddr, paddr + size);
595 
596  /*
597  * reserve_pfn_range() for RAM pages. We do not refcount to keep
598  * track of number of mappings of RAM pages. We can assert that
599  * the type requested matches the type of first page in the range.
600  */
601  if (is_ram) {
602  if (!pat_enabled)
603  return 0;
604 
605  flags = lookup_memtype(paddr);
606  if (want_flags != flags) {
607  printk(KERN_WARNING "%s:%d map pfn RAM range req %s for [mem %#010Lx-%#010Lx], got %s\n",
608  current->comm, current->pid,
609  cattr_name(want_flags),
610  (unsigned long long)paddr,
611  (unsigned long long)(paddr + size - 1),
612  cattr_name(flags));
613  *vma_prot = __pgprot((pgprot_val(*vma_prot) &
614  (~_PAGE_CACHE_MASK)) |
615  flags);
616  }
617  return 0;
618  }
619 
620  ret = reserve_memtype(paddr, paddr + size, want_flags, &flags);
621  if (ret)
622  return ret;
623 
624  if (flags != want_flags) {
625  if (strict_prot ||
626  !is_new_memtype_allowed(paddr, size, want_flags, flags)) {
627  free_memtype(paddr, paddr + size);
628  printk(KERN_ERR "%s:%d map pfn expected mapping type %s"
629  " for [mem %#010Lx-%#010Lx], got %s\n",
630  current->comm, current->pid,
631  cattr_name(want_flags),
632  (unsigned long long)paddr,
633  (unsigned long long)(paddr + size - 1),
634  cattr_name(flags));
635  return -EINVAL;
636  }
637  /*
638  * We allow returning different type than the one requested in
639  * non strict case.
640  */
641  *vma_prot = __pgprot((pgprot_val(*vma_prot) &
642  (~_PAGE_CACHE_MASK)) |
643  flags);
644  }
645 
646  if (kernel_map_sync_memtype(paddr, size, flags) < 0) {
647  free_memtype(paddr, paddr + size);
648  return -EINVAL;
649  }
650  return 0;
651 }
652 
653 /*
654  * Internal interface to free a range of physical memory.
655  * Frees non RAM regions only.
656  */
657 static void free_pfn_range(u64 paddr, unsigned long size)
658 {
659  int is_ram;
660 
661  is_ram = pat_pagerange_is_ram(paddr, paddr + size);
662  if (is_ram == 0)
663  free_memtype(paddr, paddr + size);
664 }
665 
666 /*
667  * track_pfn_copy is called when vma that is covering the pfnmap gets
668  * copied through copy_page_range().
669  *
670  * If the vma has a linear pfn mapping for the entire range, we get the prot
671  * from pte and reserve the entire vma range with single reserve_pfn_range call.
672  */
674 {
676  unsigned long prot;
677  unsigned long vma_size = vma->vm_end - vma->vm_start;
679 
680  if (vma->vm_flags & VM_PAT) {
681  /*
682  * reserve the whole chunk covered by vma. We need the
683  * starting address and protection from pte.
684  */
685  if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
686  WARN_ON_ONCE(1);
687  return -EINVAL;
688  }
689  pgprot = __pgprot(prot);
690  return reserve_pfn_range(paddr, vma_size, &pgprot, 1);
691  }
692 
693  return 0;
694 }
695 
696 /*
697  * prot is passed in as a parameter for the new mapping. If the vma has a
698  * linear pfn mapping for the entire range reserve the entire vma range with
699  * single reserve_pfn_range call.
700  */
701 int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
702  unsigned long pfn, unsigned long addr, unsigned long size)
703 {
704  resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
705  unsigned long flags;
706 
707  /* reserve the whole chunk starting from paddr */
708  if (addr == vma->vm_start && size == (vma->vm_end - vma->vm_start)) {
709  int ret;
710 
711  ret = reserve_pfn_range(paddr, size, prot, 0);
712  if (!ret)
713  vma->vm_flags |= VM_PAT;
714  return ret;
715  }
716 
717  if (!pat_enabled)
718  return 0;
719 
720  /*
721  * For anything smaller than the vma size we set prot based on the
722  * lookup.
723  */
724  flags = lookup_memtype(paddr);
725 
726  /* Check memtype for the remaining pages */
727  while (size > PAGE_SIZE) {
728  size -= PAGE_SIZE;
729  paddr += PAGE_SIZE;
730  if (flags != lookup_memtype(paddr))
731  return -EINVAL;
732  }
733 
734  *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
735  flags);
736 
737  return 0;
738 }
739 
740 int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
741  unsigned long pfn)
742 {
743  unsigned long flags;
744 
745  if (!pat_enabled)
746  return 0;
747 
748  /* Set prot based on lookup */
749  flags = lookup_memtype((resource_size_t)pfn << PAGE_SHIFT);
750  *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
751  flags);
752 
753  return 0;
754 }
755 
756 /*
757  * untrack_pfn is called while unmapping a pfnmap for a region.
758  * untrack can be called for a specific region indicated by pfn and size or
759  * can be for the entire vma (in which case pfn, size are zero).
760  */
761 void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
762  unsigned long size)
763 {
765  unsigned long prot;
766 
767  if (!(vma->vm_flags & VM_PAT))
768  return;
769 
770  /* free the chunk starting from pfn or the whole chunk */
771  paddr = (resource_size_t)pfn << PAGE_SHIFT;
772  if (!paddr && !size) {
773  if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
774  WARN_ON_ONCE(1);
775  return;
776  }
777 
778  size = vma->vm_end - vma->vm_start;
779  }
780  free_pfn_range(paddr, size);
781  vma->vm_flags &= ~VM_PAT;
782 }
783 
785 {
786  if (pat_enabled)
787  return __pgprot(pgprot_val(prot) | _PAGE_CACHE_WC);
788  else
789  return pgprot_noncached(prot);
790 }
792 
793 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT)
794 
795 static struct memtype *memtype_get_idx(loff_t pos)
796 {
797  struct memtype *print_entry;
798  int ret;
799 
800  print_entry = kzalloc(sizeof(struct memtype), GFP_KERNEL);
801  if (!print_entry)
802  return NULL;
803 
804  spin_lock(&memtype_lock);
805  ret = rbt_memtype_copy_nth_element(print_entry, pos);
806  spin_unlock(&memtype_lock);
807 
808  if (!ret) {
809  return print_entry;
810  } else {
811  kfree(print_entry);
812  return NULL;
813  }
814 }
815 
816 static void *memtype_seq_start(struct seq_file *seq, loff_t *pos)
817 {
818  if (*pos == 0) {
819  ++*pos;
820  seq_printf(seq, "PAT memtype list:\n");
821  }
822 
823  return memtype_get_idx(*pos);
824 }
825 
826 static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
827 {
828  ++*pos;
829  return memtype_get_idx(*pos);
830 }
831 
832 static void memtype_seq_stop(struct seq_file *seq, void *v)
833 {
834 }
835 
836 static int memtype_seq_show(struct seq_file *seq, void *v)
837 {
838  struct memtype *print_entry = (struct memtype *)v;
839 
840  seq_printf(seq, "%s @ 0x%Lx-0x%Lx\n", cattr_name(print_entry->type),
841  print_entry->start, print_entry->end);
842  kfree(print_entry);
843 
844  return 0;
845 }
846 
847 static const struct seq_operations memtype_seq_ops = {
848  .start = memtype_seq_start,
849  .next = memtype_seq_next,
850  .stop = memtype_seq_stop,
851  .show = memtype_seq_show,
852 };
853 
854 static int memtype_seq_open(struct inode *inode, struct file *file)
855 {
856  return seq_open(file, &memtype_seq_ops);
857 }
858 
859 static const struct file_operations memtype_fops = {
860  .open = memtype_seq_open,
861  .read = seq_read,
862  .llseek = seq_lseek,
863  .release = seq_release,
864 };
865 
866 static int __init pat_memtype_list_init(void)
867 {
868  if (pat_enabled) {
869  debugfs_create_file("pat_memtype_list", S_IRUSR,
870  arch_debugfs_dir, NULL, &memtype_fops);
871  }
872  return 0;
873 }
874 
875 late_initcall(pat_memtype_list_init);
876 
877 #endif /* CONFIG_DEBUG_FS && CONFIG_X86_PAT */