152 #include <linux/module.h>
153 #include <linux/list.h>
154 #include <linux/hash.h>
155 #include <linux/sched.h>
158 #include <asm/cache.h>
159 #include <asm/setup.h>
161 #include <asm/xen/page.h>
162 #include <asm/xen/hypercall.h>
163 #include <asm/xen/hypervisor.h>
169 static void __init m2p_override_init(
void);
173 #define P2M_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
174 #define P2M_MID_PER_PAGE (PAGE_SIZE / sizeof(unsigned long *))
175 #define P2M_TOP_PER_PAGE (PAGE_SIZE / sizeof(unsigned long **))
177 #define MAX_P2M_PFN (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
180 static RESERVE_BRK_ARRAY(
unsigned long, p2m_missing,
P2M_PER_PAGE);
181 static RESERVE_BRK_ARRAY(
unsigned long *, p2m_mid_missing,
P2M_MID_PER_PAGE);
182 static RESERVE_BRK_ARRAY(
unsigned long, p2m_mid_missing_mfn,
P2M_MID_PER_PAGE);
188 static RESERVE_BRK_ARRAY(
unsigned long, p2m_identity,
P2M_PER_PAGE);
204 static inline unsigned p2m_top_index(
unsigned long pfn)
210 static inline unsigned p2m_mid_index(
unsigned long pfn)
215 static inline unsigned p2m_index(
unsigned long pfn)
220 static void p2m_top_init(
unsigned long ***
top)
225 top[i] = p2m_mid_missing;
228 static void p2m_top_mfn_init(
unsigned long *top)
236 static void p2m_top_mfn_p_init(
unsigned long **top)
241 top[i] = p2m_mid_missing_mfn;
244 static void p2m_mid_init(
unsigned long **
mid)
249 mid[i] = p2m_missing;
252 static void p2m_mid_mfn_init(
unsigned long *mid)
260 static void p2m_init(
unsigned long *p2m)
283 if (p2m_top_mfn ==
NULL) {
285 p2m_mid_mfn_init(p2m_mid_missing_mfn);
288 p2m_top_mfn_p_init(p2m_top_mfn_p);
291 p2m_top_mfn_init(p2m_top_mfn);
294 p2m_mid_mfn_init(p2m_mid_missing_mfn);
298 unsigned topidx = p2m_top_index(pfn);
299 unsigned mididx = p2m_mid_index(pfn);
301 unsigned long *mid_mfn_p;
303 mid = p2m_top[topidx];
304 mid_mfn_p = p2m_top_mfn_p[topidx];
310 if (mid == p2m_mid_missing) {
312 BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
313 p2m_top_mfn[topidx] =
virt_to_mfn(p2m_mid_missing_mfn);
318 if (mid_mfn_p == p2m_mid_missing_mfn) {
326 p2m_mid_mfn_init(mid_mfn_p);
328 p2m_top_mfn_p[topidx] = mid_mfn_p;
348 unsigned long *mfn_list = (
unsigned long *)
xen_start_info->mfn_list;
355 p2m_init(p2m_missing);
358 p2m_mid_init(p2m_mid_missing);
361 p2m_top_init(p2m_top);
364 p2m_init(p2m_identity);
372 unsigned topidx = p2m_top_index(pfn);
373 unsigned mididx = p2m_mid_index(pfn);
375 if (p2m_top[topidx] == p2m_mid_missing) {
379 p2m_top[topidx] =
mid;
388 unsigned long p2midx;
394 p2m_top[topidx][mididx] = &mfn_list[pfn];
406 unsigned long pfn_free = 0;
407 unsigned long *mfn_list =
NULL;
414 va_end = va_start +
size;
417 if (va_start <= __START_KERNEL_map && va_start >=
__PAGE_OFFSET)
422 pr_warn(
"Could not allocate space for a new P2M tree!\n");
426 memset(mfn_list, 0xFF, size);
429 unsigned topidx = p2m_top_index(pfn);
431 unsigned long *mid_p;
433 if (!p2m_top[topidx])
436 if (p2m_top[topidx] == p2m_mid_missing)
439 mididx = p2m_mid_index(pfn);
440 mid_p = p2m_top[topidx][mididx];
443 if ((mid_p == p2m_missing) || (mid_p == p2m_identity))
450 if (mid_p >= (
unsigned long *)va_start && mid_p <= (
unsigned long *)va_end) {
453 if (pfn_free > (size /
sizeof(
unsigned long))) {
454 WARN(1,
"Only allocated for %ld pages, but we want %ld!\n",
455 size /
sizeof(
unsigned long), pfn_free);
458 new = &mfn_list[pfn_free];
461 p2m_top[topidx][mididx] = &mfn_list[pfn_free];
462 p2m_top_mfn_p[topidx][mididx] =
virt_to_mfn(&mfn_list[pfn_free]);
469 return (
unsigned long)mfn_list;
480 unsigned topidx, mididx,
idx;
485 topidx = p2m_top_index(pfn);
486 mididx = p2m_mid_index(pfn);
487 idx = p2m_index(pfn);
494 if (p2m_top[topidx][mididx] == p2m_identity)
497 return p2m_top[topidx][mididx][
idx];
501 static void *alloc_p2m_page(
void)
506 static void free_p2m_page(
void *
p)
518 static bool alloc_p2m(
unsigned long pfn)
520 unsigned topidx, mididx;
521 unsigned long ***top_p, **
mid;
522 unsigned long *top_mfn_p, *mid_mfn;
524 topidx = p2m_top_index(pfn);
525 mididx = p2m_mid_index(pfn);
527 top_p = &p2m_top[topidx];
530 if (mid == p2m_mid_missing) {
532 mid = alloc_p2m_page();
538 if (
cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing)
542 top_mfn_p = &p2m_top_mfn[topidx];
543 mid_mfn = p2m_top_mfn_p[topidx];
547 if (mid_mfn == p2m_mid_missing_mfn) {
549 unsigned long missing_mfn;
550 unsigned long mid_mfn_mfn;
552 mid_mfn = alloc_p2m_page();
556 p2m_mid_mfn_init(mid_mfn);
560 if (
cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn) != missing_mfn)
561 free_p2m_page(mid_mfn);
563 p2m_top_mfn_p[topidx] = mid_mfn;
566 if (p2m_top[topidx][mididx] == p2m_identity ||
567 p2m_top[topidx][mididx] == p2m_missing) {
570 unsigned long *p2m_orig = p2m_top[topidx][mididx];
572 p2m = alloc_p2m_page();
578 if (
cmpxchg(&mid[mididx], p2m_orig, p2m) != p2m_orig)
587 static bool __init early_alloc_p2m_middle(
unsigned long pfn,
bool check_boundary)
589 unsigned topidx, mididx,
idx;
591 unsigned long *mid_mfn_p;
593 topidx = p2m_top_index(pfn);
594 mididx = p2m_mid_index(pfn);
595 idx = p2m_index(pfn);
598 if (!idx && check_boundary)
601 WARN(p2m_top[topidx][mididx] == p2m_identity,
602 "P2M[%d][%d] == IDENTITY, should be MISSING (or alloced)!\n",
608 if (p2m_top[topidx][mididx] != p2m_missing)
616 p2m_top[topidx][mididx] = p2m;
620 mid_mfn_p = p2m_top_mfn_p[topidx];
622 "P2M_TOP_P[%d][%d] != MFN of p2m_missing!\n",
629 static bool __init early_alloc_p2m(
unsigned long pfn)
631 unsigned topidx = p2m_top_index(pfn);
632 unsigned long *mid_mfn_p;
635 mid = p2m_top[topidx];
636 mid_mfn_p = p2m_top_mfn_p[topidx];
637 if (mid == p2m_mid_missing) {
642 p2m_top[topidx] =
mid;
644 BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
647 if (mid_mfn_p == p2m_mid_missing_mfn) {
649 p2m_mid_mfn_init(mid_mfn_p);
651 p2m_top_mfn_p[topidx] = mid_mfn_p;
672 unsigned long *mid_mfn_p;
677 if (p2m_index(set_pfn))
681 topidx = p2m_top_index(pfn);
683 if (!p2m_top[topidx])
686 if (p2m_top[topidx] == p2m_mid_missing)
689 mididx = p2m_mid_index(pfn);
690 p2m = p2m_top[topidx][mididx];
694 if ((p2m == p2m_missing) || (p2m == p2m_identity))
711 if ((ident_pfns == P2M_PER_PAGE) || (inv_pfns == P2M_PER_PAGE))
717 p2m_top[topidx][mididx] = (ident_pfns ? p2m_identity : p2m_missing);
719 mid_mfn_p = p2m_top_mfn_p[topidx];
725 topidx = p2m_top_index(set_pfn);
726 mididx = p2m_mid_index(set_pfn);
729 if (
WARN_ON(p2m_top[topidx] == p2m_mid_missing))
730 early_alloc_p2m(set_pfn);
732 if (
WARN_ON(p2m_top[topidx][mididx] != p2m_missing))
736 p2m_top[topidx][mididx] = p2m;
737 mid_mfn_p = p2m_top_mfn_p[topidx];
745 if (!early_alloc_p2m(pfn))
751 if (!early_alloc_p2m_middle(pfn,
false ))
769 return pfn_e - pfn_s;
774 for (pfn = (pfn_s & ~(P2M_MID_PER_PAGE *
P2M_PER_PAGE - 1));
778 WARN_ON(!early_alloc_p2m(pfn));
781 early_alloc_p2m_middle(pfn_s,
true);
782 early_alloc_p2m_middle(pfn_e,
true);
784 for (pfn = pfn_s; pfn < pfn_e; pfn++)
788 if (!
WARN((pfn - pfn_s) != (pfn_e - pfn_s),
789 "Identity mapping failed. We are %ld short of 1-1 mappings!\n",
790 (pfn_e - pfn_s) - (pfn - pfn_s)))
799 unsigned topidx, mididx,
idx;
810 topidx = p2m_top_index(pfn);
811 mididx = p2m_mid_index(pfn);
812 idx = p2m_index(pfn);
818 if (p2m_top[topidx][mididx] == p2m_identity)
822 if (p2m_top[topidx][mididx] == p2m_missing) {
824 p2m_identity) != p2m_missing);
829 if (p2m_top[topidx][mididx] == p2m_missing)
832 p2m_top[topidx][mididx][
idx] = mfn;
850 #define M2P_OVERRIDE_HASH_SHIFT 10
851 #define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT)
856 static void __init m2p_override_init(
void)
861 sizeof(
unsigned long));
864 INIT_LIST_HEAD(&m2p_overrides[i]);
867 static unsigned long mfn_hash(
unsigned long mfn)
884 if (!PageHighMem(page)) {
888 "m2p_add_override: pfn %lx not mapped", pfn))
892 SetPagePrivate(page);
893 set_page_private(page, mfn);
899 if (kmap_op !=
NULL) {
900 if (!PageHighMem(page)) {
902 xen_mc_entry(
sizeof(*kmap_op));
904 MULTI_grant_table_op(mcs.
mc,
911 list_add(&page->
lru, &m2p_overrides[mfn_hash(mfn)]);
912 spin_unlock_irqrestore(&m2p_override_lock, flags);
951 if (!PageHighMem(page)) {
956 "m2p_remove_override: pfn %lx not mapped", pfn))
962 spin_unlock_irqrestore(&m2p_override_lock, flags);
964 ClearPagePrivate(page);
967 if (kmap_op !=
NULL) {
968 if (!PageHighMem(page)) {
979 if (kmap_op->
handle == -1)
987 "pfn %lx mfn %lx, failed to modify kernel mappings",
999 MULTI_grant_table_op(mcs.
mc,
1021 mfn &= ~FOREIGN_FRAME_BIT;
1033 unsigned long flags;
1034 struct list_head *bucket = &m2p_overrides[mfn_hash(mfn)];
1042 if (page_private(p) == mfn) {
1048 spin_unlock_irqrestore(&m2p_override_lock, flags);
1056 unsigned long ret = pfn;
1065 #ifdef CONFIG_XEN_DEBUG_FS
1067 #include "debugfs.h"
1068 static int p2m_dump_show(
struct seq_file *
m,
void *
v)
1070 static const char *
const level_name[] = {
"top",
"middle",
1071 "entry",
"abnormal",
"error"};
1072 #define TYPE_IDENTITY 0
1073 #define TYPE_MISSING 1
1075 #define TYPE_UNKNOWN 3
1076 static const char *
const type_name[] = {
1077 [TYPE_IDENTITY] =
"identity",
1078 [TYPE_MISSING] =
"missing",
1081 unsigned long pfn, prev_pfn_type = 0, prev_pfn_level = 0;
1089 unsigned topidx = p2m_top_index(pfn);
1090 unsigned mididx = p2m_mid_index(pfn);
1091 unsigned idx = p2m_index(pfn);
1096 if (p2m_top[topidx] == p2m_mid_missing) {
1097 lvl = 0; type = TYPE_MISSING;
1098 }
else if (p2m_top[topidx] ==
NULL) {
1100 }
else if (p2m_top[topidx][mididx] ==
NULL) {
1102 }
else if (p2m_top[topidx][mididx] == p2m_identity) {
1103 lvl = 1; type = TYPE_IDENTITY;
1104 }
else if (p2m_top[topidx][mididx] == p2m_missing) {
1105 lvl = 1; type = TYPE_MISSING;
1106 }
else if (p2m_top[topidx][mididx][idx] == 0) {
1108 }
else if (p2m_top[topidx][mididx][idx] ==
IDENTITY_FRAME(pfn)) {
1109 lvl = 2; type = TYPE_IDENTITY;
1111 lvl = 2; type = TYPE_MISSING;
1112 }
else if (p2m_top[topidx][mididx][idx] == pfn) {
1113 lvl = 2; type = TYPE_PFN;
1114 }
else if (p2m_top[topidx][mididx][idx] != pfn) {
1115 lvl = 2; type = TYPE_PFN;
1121 if (pfn == MAX_DOMAIN_PAGES-1) {
1125 if (prev_type != type) {
1127 prev_pfn_type, pfn, type_name[prev_type]);
1128 prev_pfn_type = pfn;
1131 if (prev_level != lvl) {
1133 prev_pfn_level, pfn, level_name[prev_level]);
1134 prev_pfn_level = pfn;
1139 #undef TYPE_IDENTITY
1151 .
open = p2m_dump_open,
1157 static struct dentry *d_mmu_debug;
1159 static int __init xen_p2m_debugfs(
void)