Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
init_32.c
Go to the documentation of this file.
1 /*
2  *
3  * Copyright (C) 1995 Linus Torvalds
4  *
5  * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6  */
7 
8 #include <linux/module.h>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
17 #include <linux/mm.h>
18 #include <linux/hugetlb.h>
19 #include <linux/swap.h>
20 #include <linux/smp.h>
21 #include <linux/init.h>
22 #include <linux/highmem.h>
23 #include <linux/pagemap.h>
24 #include <linux/pci.h>
25 #include <linux/pfn.h>
26 #include <linux/poison.h>
27 #include <linux/bootmem.h>
28 #include <linux/memblock.h>
29 #include <linux/proc_fs.h>
30 #include <linux/memory_hotplug.h>
31 #include <linux/initrd.h>
32 #include <linux/cpumask.h>
33 #include <linux/gfp.h>
34 
35 #include <asm/asm.h>
36 #include <asm/bios_ebda.h>
37 #include <asm/processor.h>
38 #include <asm/uaccess.h>
39 #include <asm/pgtable.h>
40 #include <asm/dma.h>
41 #include <asm/fixmap.h>
42 #include <asm/e820.h>
43 #include <asm/apic.h>
44 #include <asm/bugs.h>
45 #include <asm/tlb.h>
46 #include <asm/tlbflush.h>
47 #include <asm/olpc_ofw.h>
48 #include <asm/pgalloc.h>
49 #include <asm/sections.h>
50 #include <asm/paravirt.h>
51 #include <asm/setup.h>
52 #include <asm/cacheflush.h>
53 #include <asm/page_types.h>
54 #include <asm/init.h>
55 
56 unsigned long highstart_pfn, highend_pfn;
57 
58 static noinline int do_test_wp_bit(void);
59 
61 
62 static __init void *alloc_low_page(void)
63 {
64  unsigned long pfn = pgt_buf_end++;
65  void *adr;
66 
67  if (pfn >= pgt_buf_top)
68  panic("alloc_low_page: ran out of memory");
69 
70  adr = __va(pfn * PAGE_SIZE);
71  clear_page(adr);
72  return adr;
73 }
74 
75 /*
76  * Creates a middle page table and puts a pointer to it in the
77  * given global directory entry. This only returns the gd entry
78  * in non-PAE compilation mode, since the middle layer is folded.
79  */
80 static pmd_t * __init one_md_table_init(pgd_t *pgd)
81 {
82  pud_t *pud;
83  pmd_t *pmd_table;
84 
85 #ifdef CONFIG_X86_PAE
86  if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
87  if (after_bootmem)
88  pmd_table = (pmd_t *)alloc_bootmem_pages(PAGE_SIZE);
89  else
90  pmd_table = (pmd_t *)alloc_low_page();
91  paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
92  set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
93  pud = pud_offset(pgd, 0);
94  BUG_ON(pmd_table != pmd_offset(pud, 0));
95 
96  return pmd_table;
97  }
98 #endif
99  pud = pud_offset(pgd, 0);
100  pmd_table = pmd_offset(pud, 0);
101 
102  return pmd_table;
103 }
104 
105 /*
106  * Create a page table and place a pointer to it in a middle page
107  * directory entry:
108  */
109 static pte_t * __init one_page_table_init(pmd_t *pmd)
110 {
111  if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
112  pte_t *page_table = NULL;
113 
114  if (after_bootmem) {
115 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
116  page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
117 #endif
118  if (!page_table)
119  page_table =
121  } else
122  page_table = (pte_t *)alloc_low_page();
123 
124  paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
125  set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
126  BUG_ON(page_table != pte_offset_kernel(pmd, 0));
127  }
128 
129  return pte_offset_kernel(pmd, 0);
130 }
131 
133 {
134  int pgd_idx = pgd_index(vaddr);
135  int pmd_idx = pmd_index(vaddr);
136 
137  return one_md_table_init(swapper_pg_dir + pgd_idx) + pmd_idx;
138 }
139 
141 {
142  int pte_idx = pte_index(vaddr);
143  pmd_t *pmd;
144 
145  pmd = populate_extra_pmd(vaddr);
146  return one_page_table_init(pmd) + pte_idx;
147 }
148 
149 static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
150  unsigned long vaddr, pte_t *lastpte)
151 {
152 #ifdef CONFIG_HIGHMEM
153  /*
154  * Something (early fixmap) may already have put a pte
155  * page here, which causes the page table allocation
156  * to become nonlinear. Attempt to fix it, and if it
157  * is still nonlinear then we have to bug.
158  */
159  int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
160  int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;
161 
162  if (pmd_idx_kmap_begin != pmd_idx_kmap_end
163  && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
164  && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end
165  && ((__pa(pte) >> PAGE_SHIFT) < pgt_buf_start
166  || (__pa(pte) >> PAGE_SHIFT) >= pgt_buf_end)) {
167  pte_t *newpte;
168  int i;
169 
171  newpte = alloc_low_page();
172  for (i = 0; i < PTRS_PER_PTE; i++)
173  set_pte(newpte + i, pte[i]);
174 
175  paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
176  set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
177  BUG_ON(newpte != pte_offset_kernel(pmd, 0));
178  __flush_tlb_all();
179 
180  paravirt_release_pte(__pa(pte) >> PAGE_SHIFT);
181  pte = newpte;
182  }
183  BUG_ON(vaddr < fix_to_virt(FIX_KMAP_BEGIN - 1)
184  && vaddr > fix_to_virt(FIX_KMAP_END)
185  && lastpte && lastpte + PTRS_PER_PTE != pte);
186 #endif
187  return pte;
188 }
189 
190 /*
191  * This function initializes a certain range of kernel virtual memory
192  * with new bootmem page tables, everywhere page tables are missing in
193  * the given range.
194  *
195  * NOTE: The pagetables are allocated contiguous on the physical space
196  * so we can cache the place of the first one and move around without
197  * checking the pgd every time.
198  */
199 static void __init
200 page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
201 {
202  int pgd_idx, pmd_idx;
203  unsigned long vaddr;
204  pgd_t *pgd;
205  pmd_t *pmd;
206  pte_t *pte = NULL;
207 
208  vaddr = start;
209  pgd_idx = pgd_index(vaddr);
210  pmd_idx = pmd_index(vaddr);
211  pgd = pgd_base + pgd_idx;
212 
213  for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
214  pmd = one_md_table_init(pgd);
215  pmd = pmd + pmd_index(vaddr);
216  for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
217  pmd++, pmd_idx++) {
218  pte = page_table_kmap_check(one_page_table_init(pmd),
219  pmd, vaddr, pte);
220 
221  vaddr += PMD_SIZE;
222  }
223  pmd_idx = 0;
224  }
225 }
226 
227 static inline int is_kernel_text(unsigned long addr)
228 {
229  if (addr >= (unsigned long)_text && addr <= (unsigned long)__init_end)
230  return 1;
231  return 0;
232 }
233 
234 /*
235  * This maps the physical memory to kernel virtual address space, a total
236  * of max_low_pfn pages, by creating page tables starting from address
237  * PAGE_OFFSET:
238  */
239 unsigned long __init
240 kernel_physical_mapping_init(unsigned long start,
241  unsigned long end,
242  unsigned long page_size_mask)
243 {
244  int use_pse = page_size_mask == (1<<PG_LEVEL_2M);
245  unsigned long last_map_addr = end;
246  unsigned long start_pfn, end_pfn;
247  pgd_t *pgd_base = swapper_pg_dir;
248  int pgd_idx, pmd_idx, pte_ofs;
249  unsigned long pfn;
250  pgd_t *pgd;
251  pmd_t *pmd;
252  pte_t *pte;
253  unsigned pages_2m, pages_4k;
254  int mapping_iter;
255 
256  start_pfn = start >> PAGE_SHIFT;
257  end_pfn = end >> PAGE_SHIFT;
258 
259  /*
260  * First iteration will setup identity mapping using large/small pages
261  * based on use_pse, with other attributes same as set by
262  * the early code in head_32.S
263  *
264  * Second iteration will setup the appropriate attributes (NX, GLOBAL..)
265  * as desired for the kernel identity mapping.
266  *
267  * This two pass mechanism conforms to the TLB app note which says:
268  *
269  * "Software should not write to a paging-structure entry in a way
270  * that would change, for any linear address, both the page size
271  * and either the page frame or attributes."
272  */
273  mapping_iter = 1;
274 
275  if (!cpu_has_pse)
276  use_pse = 0;
277 
278 repeat:
279  pages_2m = pages_4k = 0;
280  pfn = start_pfn;
281  pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
282  pgd = pgd_base + pgd_idx;
283  for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
284  pmd = one_md_table_init(pgd);
285 
286  if (pfn >= end_pfn)
287  continue;
288 #ifdef CONFIG_X86_PAE
289  pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
290  pmd += pmd_idx;
291 #else
292  pmd_idx = 0;
293 #endif
294  for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn;
295  pmd++, pmd_idx++) {
296  unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
297 
298  /*
299  * Map with big pages if possible, otherwise
300  * create normal page tables:
301  */
302  if (use_pse) {
303  unsigned int addr2;
305  /*
306  * first pass will use the same initial
307  * identity mapping attribute + _PAGE_PSE.
308  */
309  pgprot_t init_prot =
311  _PAGE_PSE);
312 
313  addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
314  PAGE_OFFSET + PAGE_SIZE-1;
315 
316  if (is_kernel_text(addr) ||
317  is_kernel_text(addr2))
318  prot = PAGE_KERNEL_LARGE_EXEC;
319 
320  pages_2m++;
321  if (mapping_iter == 1)
322  set_pmd(pmd, pfn_pmd(pfn, init_prot));
323  else
324  set_pmd(pmd, pfn_pmd(pfn, prot));
325 
326  pfn += PTRS_PER_PTE;
327  continue;
328  }
329  pte = one_page_table_init(pmd);
330 
331  pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET);
332  pte += pte_ofs;
333  for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn;
334  pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
335  pgprot_t prot = PAGE_KERNEL;
336  /*
337  * first pass will use the same initial
338  * identity mapping attribute.
339  */
340  pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR);
341 
342  if (is_kernel_text(addr))
343  prot = PAGE_KERNEL_EXEC;
344 
345  pages_4k++;
346  if (mapping_iter == 1) {
347  set_pte(pte, pfn_pte(pfn, init_prot));
348  last_map_addr = (pfn << PAGE_SHIFT) + PAGE_SIZE;
349  } else
350  set_pte(pte, pfn_pte(pfn, prot));
351  }
352  }
353  }
354  if (mapping_iter == 1) {
355  /*
356  * update direct mapping page count only in the first
357  * iteration.
358  */
359  update_page_count(PG_LEVEL_2M, pages_2m);
360  update_page_count(PG_LEVEL_4K, pages_4k);
361 
362  /*
363  * local global flush tlb, which will flush the previous
364  * mappings present in both small and large page TLB's.
365  */
366  __flush_tlb_all();
367 
368  /*
369  * Second iteration will set the actual desired PTE attributes.
370  */
371  mapping_iter = 2;
372  goto repeat;
373  }
374  return last_map_addr;
375 }
376 
379 
380 static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
381 {
383  vaddr), vaddr), vaddr);
384 }
385 
386 static void __init kmap_init(void)
387 {
388  unsigned long kmap_vstart;
389 
390  /*
391  * Cache the first kmap pte:
392  */
393  kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
394  kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
395 
396  kmap_prot = PAGE_KERNEL;
397 }
398 
399 #ifdef CONFIG_HIGHMEM
400 static void __init permanent_kmaps_init(pgd_t *pgd_base)
401 {
402  unsigned long vaddr;
403  pgd_t *pgd;
404  pud_t *pud;
405  pmd_t *pmd;
406  pte_t *pte;
407 
408  vaddr = PKMAP_BASE;
409  page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
410 
411  pgd = swapper_pg_dir + pgd_index(vaddr);
412  pud = pud_offset(pgd, vaddr);
413  pmd = pmd_offset(pud, vaddr);
414  pte = pte_offset_kernel(pmd, vaddr);
416 }
417 
418 static void __init add_one_highpage_init(struct page *page)
419 {
420  ClearPageReserved(page);
421  init_page_count(page);
422  __free_page(page);
423  totalhigh_pages++;
424 }
425 
426 void __init add_highpages_with_active_regions(int nid,
427  unsigned long start_pfn, unsigned long end_pfn)
428 {
430  u64 i;
431 
432  for_each_free_mem_range(i, nid, &start, &end, NULL) {
433  unsigned long pfn = clamp_t(unsigned long, PFN_UP(start),
434  start_pfn, end_pfn);
435  unsigned long e_pfn = clamp_t(unsigned long, PFN_DOWN(end),
436  start_pfn, end_pfn);
437  for ( ; pfn < e_pfn; pfn++)
438  if (pfn_valid(pfn))
439  add_one_highpage_init(pfn_to_page(pfn));
440  }
441 }
442 #else
443 static inline void permanent_kmaps_init(pgd_t *pgd_base)
444 {
445 }
446 #endif /* CONFIG_HIGHMEM */
447 
449 {
450  unsigned long pfn, va;
451  pgd_t *pgd, *base = swapper_pg_dir;
452  pud_t *pud;
453  pmd_t *pmd;
454  pte_t *pte;
455 
456  /*
457  * Remove any mappings which extend past the end of physical
458  * memory from the boot time page table:
459  */
460  for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
461  va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
462  pgd = base + pgd_index(va);
463  if (!pgd_present(*pgd))
464  break;
465 
466  pud = pud_offset(pgd, va);
467  pmd = pmd_offset(pud, va);
468  if (!pmd_present(*pmd))
469  break;
470 
471  pte = pte_offset_kernel(pmd, va);
472  if (!pte_present(*pte))
473  break;
474 
475  pte_clear(NULL, va, pte);
476  }
477  paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
478  paging_init();
479 }
480 
481 /*
482  * Build a proper pagetable for the kernel mappings. Up until this
483  * point, we've been running on some set of pagetables constructed by
484  * the boot process.
485  *
486  * If we're booting on native hardware, this will be a pagetable
487  * constructed in arch/x86/kernel/head_32.S. The root of the
488  * pagetable will be swapper_pg_dir.
489  *
490  * If we're booting paravirtualized under a hypervisor, then there are
491  * more options: we may already be running PAE, and the pagetable may
492  * or may not be based in swapper_pg_dir. In any case,
493  * paravirt_pagetable_init() will set up swapper_pg_dir
494  * appropriately for the rest of the initialization to work.
495  *
496  * In general, pagetable_init() assumes that the pagetable may already
497  * be partially populated, and so it avoids stomping on any existing
498  * mappings.
499  */
501 {
502  pgd_t *pgd_base = swapper_pg_dir;
503  unsigned long vaddr, end;
504 
505  /*
506  * Fixed mappings, only the page table structure has to be
507  * created - mappings will be set by set_fixmap():
508  */
510  end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
511  page_table_range_init(vaddr, end, pgd_base);
513 }
514 
515 static void __init pagetable_init(void)
516 {
517  pgd_t *pgd_base = swapper_pg_dir;
518 
519  permanent_kmaps_init(pgd_base);
520 }
521 
523 EXPORT_SYMBOL_GPL(__supported_pte_mask);
524 
525 /* user-defined highmem size */
526 static unsigned int highmem_pages = -1;
527 
528 /*
529  * highmem=size forces highmem to be exactly 'size' bytes.
530  * This works even on boxes that have no highmem otherwise.
531  * This also works to reduce highmem size on bigger boxes.
532  */
533 static int __init parse_highmem(char *arg)
534 {
535  if (!arg)
536  return -EINVAL;
537 
538  highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT;
539  return 0;
540 }
541 early_param("highmem", parse_highmem);
542 
543 #define MSG_HIGHMEM_TOO_BIG \
544  "highmem size (%luMB) is bigger than pages available (%luMB)!\n"
545 
546 #define MSG_LOWMEM_TOO_SMALL \
547  "highmem size (%luMB) results in <64MB lowmem, ignoring it!\n"
548 /*
549  * All of RAM fits into lowmem - but if user wants highmem
550  * artificially via the highmem=x boot parameter then create
551  * it:
552  */
554 {
555  /* max_low_pfn is 0, we already have early_res support */
557 
558  if (highmem_pages == -1)
559  highmem_pages = 0;
560 #ifdef CONFIG_HIGHMEM
561  if (highmem_pages >= max_pfn) {
563  pages_to_mb(highmem_pages), pages_to_mb(max_pfn));
564  highmem_pages = 0;
565  }
566  if (highmem_pages) {
567  if (max_low_pfn - highmem_pages < 64*1024*1024/PAGE_SIZE) {
569  pages_to_mb(highmem_pages));
570  highmem_pages = 0;
571  }
572  max_low_pfn -= highmem_pages;
573  }
574 #else
575  if (highmem_pages)
576  printk(KERN_ERR "ignoring highmem size on non-highmem kernel!\n");
577 #endif
578 }
579 
580 #define MSG_HIGHMEM_TOO_SMALL \
581  "only %luMB highmem pages available, ignoring highmem size of %luMB!\n"
582 
583 #define MSG_HIGHMEM_TRIMMED \
584  "Warning: only 4GB will be used. Use a HIGHMEM64G enabled kernel!\n"
585 /*
586  * We have more RAM than fits into lowmem - we try to put it into
587  * highmem, also taking the highmem=x boot parameter into account:
588  */
590 {
592 
593  if (highmem_pages == -1)
594  highmem_pages = max_pfn - MAXMEM_PFN;
595 
596  if (highmem_pages + MAXMEM_PFN < max_pfn)
597  max_pfn = MAXMEM_PFN + highmem_pages;
598 
599  if (highmem_pages + MAXMEM_PFN > max_pfn) {
601  pages_to_mb(max_pfn - MAXMEM_PFN),
602  pages_to_mb(highmem_pages));
603  highmem_pages = 0;
604  }
605 #ifndef CONFIG_HIGHMEM
606  /* Maximum memory usable is what is directly addressable */
607  printk(KERN_WARNING "Warning only %ldMB will be used.\n", MAXMEM>>20);
608  if (max_pfn > MAX_NONPAE_PFN)
609  printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n");
610  else
611  printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n");
613 #else /* !CONFIG_HIGHMEM */
614 #ifndef CONFIG_HIGHMEM64G
615  if (max_pfn > MAX_NONPAE_PFN) {
616  max_pfn = MAX_NONPAE_PFN;
618  }
619 #endif /* !CONFIG_HIGHMEM64G */
620 #endif /* !CONFIG_HIGHMEM */
621 }
622 
623 /*
624  * Determine low and high memory ranges:
625  */
627 {
628  /* it could update max_pfn */
629 
630  if (max_pfn <= MAXMEM_PFN)
631  lowmem_pfn_init();
632  else
634 }
635 
636 #ifndef CONFIG_NEED_MULTIPLE_NODES
638 {
639 #ifdef CONFIG_HIGHMEM
641  if (max_pfn > max_low_pfn)
643  printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
646  high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
647 #else
649  high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
650 #endif
651 
652  memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);
653  sparse_memory_present_with_active_regions(0);
654 
655 #ifdef CONFIG_FLATMEM
657 #endif
658  __vmalloc_start_set = true;
659 
660  printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
662 
664 }
665 #endif /* !CONFIG_NEED_MULTIPLE_NODES */
666 
668 {
669  printk(KERN_INFO " mapped low ram: 0 - %08lx\n",
671  printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
672 
673  after_bootmem = 1;
674 }
675 
676 /*
677  * paging_init() sets up the page tables - note that the first 8MB are
678  * already mapped by head.S.
679  *
680  * This routines also unmaps the page at virtual kernel address 0, so
681  * that we can trap those pesky NULL-reference errors in the kernel.
682  */
683 void __init paging_init(void)
684 {
685  pagetable_init();
686 
687  __flush_tlb_all();
688 
689  kmap_init();
690 
691  /*
692  * NOTE: at this point the bootmem allocator is fully available.
693  */
695  sparse_memory_present_with_active_regions(MAX_NUMNODES);
696  sparse_init();
697  zone_sizes_init();
698 }
699 
700 /*
701  * Test if the WP bit works in supervisor mode. It isn't supported on 386's
702  * and also on some strange 486's. All 586+'s are OK. This used to involve
703  * black magic jumps to work around some nasty CPU bugs, but fortunately the
704  * switch to using exceptions got rid of all that.
705  */
706 static void __init test_wp_bit(void)
707 {
709  "Checking if this processor honours the WP bit even in supervisor mode...");
710 
711  /* Any page-aligned address will do, the test is non-destructive */
713  boot_cpu_data.wp_works_ok = do_test_wp_bit();
714  clear_fixmap(FIX_WP_TEST);
715 
716  if (!boot_cpu_data.wp_works_ok) {
717  printk(KERN_CONT "No.\n");
718 #ifdef CONFIG_X86_WP_WORKS_OK
719  panic(
720  "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
721 #endif
722  } else {
723  printk(KERN_CONT "Ok.\n");
724  }
725 }
726 
727 void __init mem_init(void)
728 {
729  int codesize, reservedpages, datasize, initsize;
730  int tmp;
731 
732  pci_iommu_alloc();
733 
734 #ifdef CONFIG_FLATMEM
735  BUG_ON(!mem_map);
736 #endif
737  /*
738  * With CONFIG_DEBUG_PAGEALLOC initialization of highmem pages has to
739  * be done before free_all_bootmem(). Memblock use free low memory for
740  * temporary data (see find_range_array()) and for this purpose can use
741  * pages that was already passed to the buddy allocator, hence marked as
742  * not accessible in the page tables when compiled with
743  * CONFIG_DEBUG_PAGEALLOC. Otherwise order of initialization is not
744  * important here.
745  */
747 
748  /* this will put all low memory onto the freelists */
749  totalram_pages += free_all_bootmem();
750 
751  reservedpages = 0;
752  for (tmp = 0; tmp < max_low_pfn; tmp++)
753  /*
754  * Only count reserved RAM pages:
755  */
756  if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
757  reservedpages++;
758 
759  codesize = (unsigned long) &_etext - (unsigned long) &_text;
760  datasize = (unsigned long) &_edata - (unsigned long) &_etext;
761  initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
762 
763  printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, "
764  "%dk reserved, %dk data, %dk init, %ldk highmem)\n",
765  nr_free_pages() << (PAGE_SHIFT-10),
766  num_physpages << (PAGE_SHIFT-10),
767  codesize >> 10,
768  reservedpages << (PAGE_SHIFT-10),
769  datasize >> 10,
770  initsize >> 10,
771  totalhigh_pages << (PAGE_SHIFT-10));
772 
773  printk(KERN_INFO "virtual kernel memory layout:\n"
774  " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
775 #ifdef CONFIG_HIGHMEM
776  " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
777 #endif
778  " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
779  " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
780  " .init : 0x%08lx - 0x%08lx (%4ld kB)\n"
781  " .data : 0x%08lx - 0x%08lx (%4ld kB)\n"
782  " .text : 0x%08lx - 0x%08lx (%4ld kB)\n",
784  (FIXADDR_TOP - FIXADDR_START) >> 10,
785 
786 #ifdef CONFIG_HIGHMEM
787  PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
788  (LAST_PKMAP*PAGE_SIZE) >> 10,
789 #endif
790 
792  (VMALLOC_END - VMALLOC_START) >> 20,
793 
794  (unsigned long)__va(0), (unsigned long)high_memory,
795  ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
796 
797  (unsigned long)&__init_begin, (unsigned long)&__init_end,
798  ((unsigned long)&__init_end -
799  (unsigned long)&__init_begin) >> 10,
800 
801  (unsigned long)&_etext, (unsigned long)&_edata,
802  ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
803 
804  (unsigned long)&_text, (unsigned long)&_etext,
805  ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
806 
807  /*
808  * Check boundaries twice: Some fundamental inconsistencies can
809  * be detected at build time already.
810  */
811 #define __FIXADDR_TOP (-PAGE_SIZE)
812 #ifdef CONFIG_HIGHMEM
813  BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START);
814  BUILD_BUG_ON(VMALLOC_END > PKMAP_BASE);
815 #endif
816 #define high_memory (-128UL << 20)
817  BUILD_BUG_ON(VMALLOC_START >= VMALLOC_END);
818 #undef high_memory
819 #undef __FIXADDR_TOP
820 
821 #ifdef CONFIG_HIGHMEM
822  BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START);
823  BUG_ON(VMALLOC_END > PKMAP_BASE);
824 #endif
825  BUG_ON(VMALLOC_START >= VMALLOC_END);
826  BUG_ON((unsigned long)high_memory > VMALLOC_START);
827 
828  if (boot_cpu_data.wp_works_ok < 0)
829  test_wp_bit();
830 }
831 
832 #ifdef CONFIG_MEMORY_HOTPLUG
833 int arch_add_memory(int nid, u64 start, u64 size)
834 {
835  struct pglist_data *pgdata = NODE_DATA(nid);
836  struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM;
837  unsigned long start_pfn = start >> PAGE_SHIFT;
838  unsigned long nr_pages = size >> PAGE_SHIFT;
839 
840  return __add_pages(nid, zone, start_pfn, nr_pages);
841 }
842 #endif
843 
844 /*
845  * This function cannot be __init, since exceptions don't work in that
846  * section. Put this after the callers, so that it cannot be inlined.
847  */
848 static noinline int do_test_wp_bit(void)
849 {
850  char tmp_reg;
851  int flag;
852 
853  __asm__ __volatile__(
854  " movb %0, %1 \n"
855  "1: movb %1, %0 \n"
856  " xorl %2, %2 \n"
857  "2: \n"
858  _ASM_EXTABLE(1b,2b)
859  :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
860  "=q" (tmp_reg),
861  "=r" (flag)
862  :"2" (1)
863  :"memory");
864 
865  return flag;
866 }
867 
868 #ifdef CONFIG_DEBUG_RODATA
869 const int rodata_test_data = 0xC3;
870 EXPORT_SYMBOL_GPL(rodata_test_data);
871 
872 int kernel_set_to_readonly __read_mostly;
873 
874 void set_kernel_text_rw(void)
875 {
876  unsigned long start = PFN_ALIGN(_text);
877  unsigned long size = PFN_ALIGN(_etext) - start;
878 
879  if (!kernel_set_to_readonly)
880  return;
881 
882  pr_debug("Set kernel text: %lx - %lx for read write\n",
883  start, start+size);
884 
885  set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
886 }
887 
888 void set_kernel_text_ro(void)
889 {
890  unsigned long start = PFN_ALIGN(_text);
891  unsigned long size = PFN_ALIGN(_etext) - start;
892 
893  if (!kernel_set_to_readonly)
894  return;
895 
896  pr_debug("Set kernel text: %lx - %lx for read only\n",
897  start, start+size);
898 
899  set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
900 }
901 
902 static void mark_nxdata_nx(void)
903 {
904  /*
905  * When this called, init has already been executed and released,
906  * so everything past _etext should be NX.
907  */
908  unsigned long start = PFN_ALIGN(_etext);
909  /*
910  * This comes from is_kernel_text upper limit. Also HPAGE where used:
911  */
912  unsigned long size = (((unsigned long)__init_end + HPAGE_SIZE) & HPAGE_MASK) - start;
913 
914  if (__supported_pte_mask & _PAGE_NX)
915  printk(KERN_INFO "NX-protecting the kernel data: %luk\n", size >> 10);
916  set_pages_nx(virt_to_page(start), size >> PAGE_SHIFT);
917 }
918 
919 void mark_rodata_ro(void)
920 {
921  unsigned long start = PFN_ALIGN(_text);
922  unsigned long size = PFN_ALIGN(_etext) - start;
923 
924  set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
925  printk(KERN_INFO "Write protecting the kernel text: %luk\n",
926  size >> 10);
927 
928  kernel_set_to_readonly = 1;
929 
930 #ifdef CONFIG_CPA_DEBUG
931  printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n",
932  start, start+size);
933  set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
934 
935  printk(KERN_INFO "Testing CPA: write protecting again\n");
936  set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
937 #endif
938 
939  start += size;
940  size = (unsigned long)__end_rodata - start;
941  set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
942  printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
943  size >> 10);
944  rodata_test();
945 
946 #ifdef CONFIG_CPA_DEBUG
947  printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size);
948  set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
949 
950  printk(KERN_INFO "Testing CPA: write protecting again\n");
951  set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
952 #endif
953  mark_nxdata_nx();
954 }
955 #endif
956