24 #include <linux/slab.h>
26 #include <linux/sysctl.h>
27 #include <linux/mman.h>
29 #include <asm/tlbflush.h>
30 #include <asm/setup.h>
32 #ifdef CONFIG_HUGETLB_SUPER_PAGES
42 #define ADDITIONAL_HUGE_SIZE (1024 * 1024UL)
45 int huge_shift[HUGE_SHIFT_ENTRIES] = {
46 #ifdef ADDITIONAL_HUGE_SIZE
47 #define ADDITIONAL_HUGE_SHIFT __builtin_ctzl(ADDITIONAL_HUGE_SIZE / PAGE_SIZE)
48 [HUGE_SHIFT_PAGE] = ADDITIONAL_HUGE_SHIFT
87 unsigned long addr,
unsigned long sz)
97 #ifdef CONFIG_HUGETLB_SUPER_PAGES
100 sz !=
PGDIR_SIZE << huge_shift[HUGE_SHIFT_PGDIR]);
106 sz != (
PMD_SIZE << huge_shift[HUGE_SHIFT_PMD]));
110 if (sz !=
PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
111 panic(
"Unexpected page size %#lx\n", sz);
112 return pte_alloc_hugetlb(mm, pmd, addr);
124 #ifdef CONFIG_HUGETLB_SUPER_PAGES
125 if (!
pte_present(*ptep) && huge_shift[level] != 0) {
127 pte_t *super_ptep = base + (index &
mask);
141 #ifdef CONFIG_HUGETLB_SUPER_PAGES
152 #ifndef __PAGETABLE_PUD_FOLDED
153 # error support fourth page table level
157 #ifndef __PAGETABLE_PMD_FOLDED
173 #ifdef CONFIG_HUGETLB_SUPER_PAGES
228 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
229 static unsigned long hugetlb_get_unmapped_area_bottomup(
struct file *
file,
230 unsigned long addr,
unsigned long len,
231 unsigned long pgoff,
unsigned long flags)
262 if (!vma || addr + len <= vma->
vm_start) {
272 static unsigned long hugetlb_get_unmapped_area_topdown(
struct file *file,
273 unsigned long addr0,
unsigned long len,
274 unsigned long pgoff,
unsigned long flags)
279 unsigned long base = mm->
mmap_base, addr = addr0;
287 if (len <= largest_hole) {
313 if (addr + len <= vma->vm_start &&
314 (!prev_vma || (addr >= prev_vma->
vm_end))) {
328 if (addr + largest_hole < vma->vm_start)
334 }
while (len <= vma->vm_start);
355 addr = hugetlb_get_unmapped_area_bottomup(file, addr0,
368 unsigned long len,
unsigned long pgoff,
unsigned long flags)
389 (!vma || addr + len <= vma->vm_start))
393 return hugetlb_get_unmapped_area_bottomup(file, addr, len,
396 return hugetlb_get_unmapped_area_topdown(file, addr, len,
401 #ifdef CONFIG_HUGETLB_SUPER_PAGES
402 static __init int __setup_hugepagesz(
unsigned long ps)
404 int log_ps = __builtin_ctzl(ps);
405 int level, base_shift;
407 if ((1
UL << log_ps) != ps || (log_ps & 1) != 0) {
408 pr_warn(
"Not enabling %ld byte huge pages;"
409 " must be a power of four.\n", ps);
413 if (ps > 64*1024*1024*1024
UL) {
414 pr_warn(
"Not enabling %ld MB huge pages;"
415 " largest legal value is 64 GB .\n", ps >> 20);
418 static long hv_jpage_size;
419 if (hv_jpage_size == 0)
422 pr_warn(
"Not enabling >= %ld MB huge pages:"
423 " hypervisor reports size %ld\n",
436 pr_err(
"hugepagesz: huge page size %ld too small\n", ps);
440 if (log_ps != base_shift) {
441 int shift_val = log_ps - base_shift;
442 if (huge_shift[level] != 0) {
443 int old_shift = base_shift + huge_shift[
level];
444 pr_warn(
"Not enabling %ld MB huge pages;"
445 " already have size %ld MB.\n",
446 ps >> 20, (1
UL << old_shift) >> 20);
450 pr_warn(
"Not enabling %ld MB huge pages;"
451 " no hypervisor support.\n", ps >> 20);
455 huge_shift[
level] = shift_val;
463 static bool saw_hugepagesz;
465 static __init int setup_hugepagesz(
char *
opt)
467 if (!saw_hugepagesz) {
468 saw_hugepagesz =
true;
469 memset(huge_shift, 0,
sizeof(huge_shift));
473 __setup(
"hugepagesz=", setup_hugepagesz);
475 #ifdef ADDITIONAL_HUGE_SIZE
482 static __init int add_default_hugepagesz(
void)
484 if (!saw_hugepagesz) {
488 ADDITIONAL_HUGE_SIZE);