blob: 4ab3d40aac9013f4587c3ac2766162425ca4569a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86_64/mm/init.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
7 */
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/types.h>
15#include <linux/ptrace.h>
16#include <linux/mman.h>
17#include <linux/mm.h>
18#include <linux/swap.h>
19#include <linux/smp.h>
20#include <linux/init.h>
21#include <linux/pagemap.h>
22#include <linux/bootmem.h>
23#include <linux/proc_fs.h>
Andi Kleen59170892005-11-05 17:25:53 +010024#include <linux/pci.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070025#include <linux/poison.h>
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010026#include <linux/dma-mapping.h>
Matt Tolentino44df75e2006-01-17 07:03:41 +010027#include <linux/module.h>
28#include <linux/memory_hotplug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
30#include <asm/processor.h>
31#include <asm/system.h>
32#include <asm/uaccess.h>
33#include <asm/pgtable.h>
34#include <asm/pgalloc.h>
35#include <asm/dma.h>
36#include <asm/fixmap.h>
37#include <asm/e820.h>
38#include <asm/apic.h>
39#include <asm/tlb.h>
40#include <asm/mmu_context.h>
41#include <asm/proto.h>
42#include <asm/smp.h>
Andi Kleen2bc04142005-11-05 17:25:53 +010043#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
45#ifndef Dprintk
46#define Dprintk(x...)
47#endif
48
Stephen Hemmingere6584502007-05-02 19:27:06 +020049const struct dma_mapping_ops* dma_ops;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010050EXPORT_SYMBOL(dma_ops);
51
Andi Kleene18c6872005-11-05 17:25:53 +010052static unsigned long dma_reserve __initdata;
53
Linus Torvalds1da177e2005-04-16 15:20:36 -070054DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
55
56/*
57 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
58 * physical space so we can cache the place of the first one and move
59 * around without checking the pgd every time.
60 */
61
62void show_mem(void)
63{
Andi Kleene92343c2005-09-12 18:49:24 +020064 long i, total = 0, reserved = 0;
65 long shared = 0, cached = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 pg_data_t *pgdat;
67 struct page *page;
68
Andi Kleene92343c2005-09-12 18:49:24 +020069 printk(KERN_INFO "Mem-info:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 show_free_areas();
Andi Kleene92343c2005-09-12 18:49:24 +020071 printk(KERN_INFO "Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -080073 for_each_online_pgdat(pgdat) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070074 for (i = 0; i < pgdat->node_spanned_pages; ++i) {
75 page = pfn_to_page(pgdat->node_start_pfn + i);
76 total++;
Andi Kleene92343c2005-09-12 18:49:24 +020077 if (PageReserved(page))
78 reserved++;
79 else if (PageSwapCache(page))
80 cached++;
81 else if (page_count(page))
82 shared += page_count(page) - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 }
84 }
Andi Kleene92343c2005-09-12 18:49:24 +020085 printk(KERN_INFO "%lu pages of RAM\n", total);
86 printk(KERN_INFO "%lu reserved pages\n",reserved);
87 printk(KERN_INFO "%lu pages shared\n",shared);
88 printk(KERN_INFO "%lu pages swap cached\n",cached);
Linus Torvalds1da177e2005-04-16 15:20:36 -070089}
90
Linus Torvalds1da177e2005-04-16 15:20:36 -070091int after_bootmem;
92
Andi Kleen5f44a662006-03-25 16:30:25 +010093static __init void *spp_getpage(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070094{
95 void *ptr;
96 if (after_bootmem)
97 ptr = (void *) get_zeroed_page(GFP_ATOMIC);
98 else
99 ptr = alloc_bootmem_pages(PAGE_SIZE);
100 if (!ptr || ((unsigned long)ptr & ~PAGE_MASK))
101 panic("set_pte_phys: cannot allocate page data %s\n", after_bootmem?"after bootmem":"");
102
103 Dprintk("spp_getpage %p\n", ptr);
104 return ptr;
105}
106
Andi Kleen5f44a662006-03-25 16:30:25 +0100107static __init void set_pte_phys(unsigned long vaddr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 unsigned long phys, pgprot_t prot)
109{
110 pgd_t *pgd;
111 pud_t *pud;
112 pmd_t *pmd;
113 pte_t *pte, new_pte;
114
115 Dprintk("set_pte_phys %lx to %lx\n", vaddr, phys);
116
117 pgd = pgd_offset_k(vaddr);
118 if (pgd_none(*pgd)) {
119 printk("PGD FIXMAP MISSING, it should be setup in head.S!\n");
120 return;
121 }
122 pud = pud_offset(pgd, vaddr);
123 if (pud_none(*pud)) {
124 pmd = (pmd_t *) spp_getpage();
125 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
126 if (pmd != pmd_offset(pud, 0)) {
127 printk("PAGETABLE BUG #01! %p <-> %p\n", pmd, pmd_offset(pud,0));
128 return;
129 }
130 }
131 pmd = pmd_offset(pud, vaddr);
132 if (pmd_none(*pmd)) {
133 pte = (pte_t *) spp_getpage();
134 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
135 if (pte != pte_offset_kernel(pmd, 0)) {
136 printk("PAGETABLE BUG #02!\n");
137 return;
138 }
139 }
140 new_pte = pfn_pte(phys >> PAGE_SHIFT, prot);
141
142 pte = pte_offset_kernel(pmd, vaddr);
143 if (!pte_none(*pte) &&
144 pte_val(*pte) != (pte_val(new_pte) & __supported_pte_mask))
145 pte_ERROR(*pte);
146 set_pte(pte, new_pte);
147
148 /*
149 * It's enough to flush this one mapping.
150 * (PGE mappings get flushed as well)
151 */
152 __flush_tlb_one(vaddr);
153}
154
155/* NOTE: this is meant to be run only at boot */
Andi Kleen5f44a662006-03-25 16:30:25 +0100156void __init
157__set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158{
159 unsigned long address = __fix_to_virt(idx);
160
161 if (idx >= __end_of_fixed_addresses) {
162 printk("Invalid __set_fixmap\n");
163 return;
164 }
165 set_pte_phys(address, phys, prot);
166}
167
168unsigned long __initdata table_start, table_end;
169
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200170static __meminit void *alloc_low_page(unsigned long *phys)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171{
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200172 unsigned long pfn = table_end++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 void *adr;
174
Matt Tolentino44df75e2006-01-17 07:03:41 +0100175 if (after_bootmem) {
176 adr = (void *)get_zeroed_page(GFP_ATOMIC);
177 *phys = __pa(adr);
178 return adr;
179 }
180
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 if (pfn >= end_pfn)
182 panic("alloc_low_page: ran out of memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200184 adr = early_ioremap(pfn * PAGE_SIZE, PAGE_SIZE);
185 memset(adr, 0, PAGE_SIZE);
186 *phys = pfn * PAGE_SIZE;
187 return adr;
188}
189
190static __meminit void unmap_low_page(void *adr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191{
Matt Tolentino44df75e2006-01-17 07:03:41 +0100192
193 if (after_bootmem)
194 return;
195
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200196 early_iounmap(adr, PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197}
198
Andi Kleenf2d3efe2006-03-25 16:30:22 +0100199/* Must run before zap_low_mappings */
200__init void *early_ioremap(unsigned long addr, unsigned long size)
201{
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200202 unsigned long vaddr;
203 pmd_t *pmd, *last_pmd;
204 int i, pmds;
Andi Kleenf2d3efe2006-03-25 16:30:22 +0100205
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200206 pmds = ((addr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
207 vaddr = __START_KERNEL_map;
208 pmd = level2_kernel_pgt;
209 last_pmd = level2_kernel_pgt + PTRS_PER_PMD - 1;
210 for (; pmd <= last_pmd; pmd++, vaddr += PMD_SIZE) {
211 for (i = 0; i < pmds; i++) {
212 if (pmd_present(pmd[i]))
213 goto next;
214 }
215 vaddr += addr & ~PMD_MASK;
216 addr &= PMD_MASK;
217 for (i = 0; i < pmds; i++, addr += PMD_SIZE)
218 set_pmd(pmd + i,__pmd(addr | _KERNPG_TABLE | _PAGE_PSE));
219 __flush_tlb();
220 return (void *)vaddr;
221 next:
222 ;
Andi Kleenf2d3efe2006-03-25 16:30:22 +0100223 }
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200224 printk("early_ioremap(0x%lx, %lu) failed\n", addr, size);
225 return NULL;
Andi Kleenf2d3efe2006-03-25 16:30:22 +0100226}
227
228/* To avoid virtual aliases later */
229__init void early_iounmap(void *addr, unsigned long size)
230{
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200231 unsigned long vaddr;
232 pmd_t *pmd;
233 int i, pmds;
234
235 vaddr = (unsigned long)addr;
236 pmds = ((vaddr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
237 pmd = level2_kernel_pgt + pmd_index(vaddr);
238 for (i = 0; i < pmds; i++)
239 pmd_clear(pmd + i);
Andi Kleenf2d3efe2006-03-25 16:30:22 +0100240 __flush_tlb();
241}
242
Matt Tolentino44df75e2006-01-17 07:03:41 +0100243static void __meminit
Keith Mannthey6ad91652006-09-26 10:52:36 +0200244phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end)
Matt Tolentino44df75e2006-01-17 07:03:41 +0100245{
Keith Mannthey6ad91652006-09-26 10:52:36 +0200246 int i = pmd_index(address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247
Keith Mannthey6ad91652006-09-26 10:52:36 +0200248 for (; i < PTRS_PER_PMD; i++, address += PMD_SIZE) {
Matt Tolentino44df75e2006-01-17 07:03:41 +0100249 unsigned long entry;
Keith Mannthey6ad91652006-09-26 10:52:36 +0200250 pmd_t *pmd = pmd_page + pmd_index(address);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100251
Jan Beulich5f51e132006-06-26 13:59:02 +0200252 if (address >= end) {
253 if (!after_bootmem)
254 for (; i < PTRS_PER_PMD; i++, pmd++)
255 set_pmd(pmd, __pmd(0));
Matt Tolentino44df75e2006-01-17 07:03:41 +0100256 break;
257 }
Keith Mannthey6ad91652006-09-26 10:52:36 +0200258
259 if (pmd_val(*pmd))
260 continue;
261
Matt Tolentino44df75e2006-01-17 07:03:41 +0100262 entry = _PAGE_NX|_PAGE_PSE|_KERNPG_TABLE|_PAGE_GLOBAL|address;
263 entry &= __supported_pte_mask;
264 set_pmd(pmd, __pmd(entry));
265 }
266}
267
268static void __meminit
269phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end)
270{
Keith Mannthey6ad91652006-09-26 10:52:36 +0200271 pmd_t *pmd = pmd_offset(pud,0);
272 spin_lock(&init_mm.page_table_lock);
273 phys_pmd_init(pmd, address, end);
274 spin_unlock(&init_mm.page_table_lock);
275 __flush_tlb_all();
Matt Tolentino44df75e2006-01-17 07:03:41 +0100276}
277
Keith Mannthey6ad91652006-09-26 10:52:36 +0200278static void __meminit phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end)
Matt Tolentino44df75e2006-01-17 07:03:41 +0100279{
Keith Mannthey6ad91652006-09-26 10:52:36 +0200280 int i = pud_index(addr);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100281
Matt Tolentino44df75e2006-01-17 07:03:41 +0100282
Keith Mannthey6ad91652006-09-26 10:52:36 +0200283 for (; i < PTRS_PER_PUD; i++, addr = (addr & PUD_MASK) + PUD_SIZE ) {
Keith Mannthey6ad91652006-09-26 10:52:36 +0200284 unsigned long pmd_phys;
285 pud_t *pud = pud_page + pud_index(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 pmd_t *pmd;
287
Keith Mannthey6ad91652006-09-26 10:52:36 +0200288 if (addr >= end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290
Keith Mannthey6ad91652006-09-26 10:52:36 +0200291 if (!after_bootmem && !e820_any_mapped(addr,addr+PUD_SIZE,0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 set_pud(pud, __pud(0));
293 continue;
294 }
295
Keith Mannthey6ad91652006-09-26 10:52:36 +0200296 if (pud_val(*pud)) {
297 phys_pmd_update(pud, addr, end);
298 continue;
299 }
300
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200301 pmd = alloc_low_page(&pmd_phys);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100302 spin_lock(&init_mm.page_table_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
Keith Mannthey6ad91652006-09-26 10:52:36 +0200304 phys_pmd_init(pmd, addr, end);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100305 spin_unlock(&init_mm.page_table_lock);
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200306 unmap_low_page(pmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 }
308 __flush_tlb();
309}
310
311static void __init find_early_table_space(unsigned long end)
312{
Andi Kleen6c5acd12006-01-11 22:46:57 +0100313 unsigned long puds, pmds, tables, start;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
315 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
316 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
317 tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
318 round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
319
Andi Kleenee408c72006-01-16 01:56:51 +0100320 /* RED-PEN putting page tables only on node 0 could
321 cause a hotspot and fill up ZONE_DMA. The page tables
322 need roughly 0.5KB per GB. */
323 start = 0x8000;
324 table_start = find_e820_area(start, end, tables);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 if (table_start == -1UL)
326 panic("Cannot find space for the kernel page tables");
327
328 table_start >>= PAGE_SHIFT;
329 table_end = table_start;
Matt Tolentino44df75e2006-01-17 07:03:41 +0100330
331 early_printk("kernel direct mapping tables up to %lx @ %lx-%lx\n",
Jan Beulich5f51e132006-06-26 13:59:02 +0200332 end, table_start << PAGE_SHIFT,
333 (table_start << PAGE_SHIFT) + tables);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
335
336/* Setup the direct mapping of the physical memory at PAGE_OFFSET.
337 This runs before bootmem is initialized and gets pages directly from the
338 physical memory. To access them they are temporarily mapped. */
Matt Tolentino44df75e2006-01-17 07:03:41 +0100339void __meminit init_memory_mapping(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340{
341 unsigned long next;
342
343 Dprintk("init_memory_mapping\n");
344
345 /*
346 * Find space for the kernel direct mapping tables.
347 * Later we should allocate these tables in the local node of the memory
348 * mapped. Unfortunately this is done currently before the nodes are
349 * discovered.
350 */
Matt Tolentino44df75e2006-01-17 07:03:41 +0100351 if (!after_bootmem)
352 find_early_table_space(end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
354 start = (unsigned long)__va(start);
355 end = (unsigned long)__va(end);
356
357 for (; start < end; start = next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 unsigned long pud_phys;
Matt Tolentino44df75e2006-01-17 07:03:41 +0100359 pgd_t *pgd = pgd_offset_k(start);
360 pud_t *pud;
361
362 if (after_bootmem)
Andi Kleend2ae5b52006-06-26 13:57:56 +0200363 pud = pud_offset(pgd, start & PGDIR_MASK);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100364 else
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200365 pud = alloc_low_page(&pud_phys);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100366
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 next = start + PGDIR_SIZE;
368 if (next > end)
369 next = end;
370 phys_pud_init(pud, __pa(start), __pa(next));
Matt Tolentino44df75e2006-01-17 07:03:41 +0100371 if (!after_bootmem)
372 set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
Vivek Goyaldafe41e2007-05-02 19:27:06 +0200373 unmap_low_page(pud);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 }
375
Matt Tolentino44df75e2006-01-17 07:03:41 +0100376 if (!after_bootmem)
377 asm volatile("movq %%cr4,%0" : "=r" (mmu_cr4_features));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 __flush_tlb_all();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379}
380
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +0100381void __cpuinit zap_low_mappings(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382{
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +0100383 if (cpu == 0) {
384 pgd_t *pgd = pgd_offset_k(0UL);
385 pgd_clear(pgd);
386 } else {
387 /*
388 * For AP's, zap the low identity mappings by changing the cr3
389 * to init_level4_pgt and doing local flush tlb all
390 */
391 asm volatile("movq %0,%%cr3" :: "r" (__pa_symbol(&init_level4_pgt)));
392 }
393 __flush_tlb_all();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394}
395
Matt Tolentino2b976902005-06-23 00:08:06 -0700396#ifndef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397void __init paging_init(void)
398{
Mel Gorman6391af12006-10-11 01:20:39 -0700399 unsigned long max_zone_pfns[MAX_NR_ZONES];
400 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
401 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
402 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
403 max_zone_pfns[ZONE_NORMAL] = end_pfn;
404
Matt Tolentino44df75e2006-01-17 07:03:41 +0100405 memory_present(0, 0, end_pfn);
406 sparse_init();
Mel Gorman5cb248a2006-09-27 01:49:52 -0700407 free_area_init_nodes(max_zone_pfns);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408}
409#endif
410
411/* Unmap a kernel mapping if it exists. This is useful to avoid prefetches
412 from the CPU leading to inconsistent cache lines. address and size
413 must be aligned to 2MB boundaries.
414 Does nothing when the mapping doesn't exist. */
415void __init clear_kernel_mapping(unsigned long address, unsigned long size)
416{
417 unsigned long end = address + size;
418
419 BUG_ON(address & ~LARGE_PAGE_MASK);
420 BUG_ON(size & ~LARGE_PAGE_MASK);
421
422 for (; address < end; address += LARGE_PAGE_SIZE) {
423 pgd_t *pgd = pgd_offset_k(address);
424 pud_t *pud;
425 pmd_t *pmd;
426 if (pgd_none(*pgd))
427 continue;
428 pud = pud_offset(pgd, address);
429 if (pud_none(*pud))
430 continue;
431 pmd = pmd_offset(pud, address);
432 if (!pmd || pmd_none(*pmd))
433 continue;
434 if (0 == (pmd_val(*pmd) & _PAGE_PSE)) {
435 /* Could handle this, but it should not happen currently. */
436 printk(KERN_ERR
437 "clear_kernel_mapping: mapping has been split. will leak memory\n");
438 pmd_ERROR(*pmd);
439 }
440 set_pmd(pmd, __pmd(0));
441 }
442 __flush_tlb_all();
443}
444
Matt Tolentino44df75e2006-01-17 07:03:41 +0100445/*
446 * Memory hotplug specific functions
Matt Tolentino44df75e2006-01-17 07:03:41 +0100447 */
Matt Tolentino44df75e2006-01-17 07:03:41 +0100448void online_page(struct page *page)
449{
450 ClearPageReserved(page);
Nick Piggin7835e982006-03-22 00:08:40 -0800451 init_page_count(page);
Matt Tolentino44df75e2006-01-17 07:03:41 +0100452 __free_page(page);
453 totalram_pages++;
454 num_physpages++;
455}
456
Yasunori Gotobc02af92006-06-27 02:53:30 -0700457#ifdef CONFIG_MEMORY_HOTPLUG
458/*
Yasunori Gotobc02af92006-06-27 02:53:30 -0700459 * Memory is added always to NORMAL zone. This means you will never get
460 * additional DMA/DMA32 memory.
461 */
462int arch_add_memory(int nid, u64 start, u64 size)
463{
464 struct pglist_data *pgdat = NODE_DATA(nid);
Christoph Lameter776ed982006-09-25 23:31:09 -0700465 struct zone *zone = pgdat->node_zones + ZONE_NORMAL;
Yasunori Gotobc02af92006-06-27 02:53:30 -0700466 unsigned long start_pfn = start >> PAGE_SHIFT;
467 unsigned long nr_pages = size >> PAGE_SHIFT;
468 int ret;
469
Keith Mannthey45e0b782006-09-30 23:27:09 -0700470 init_memory_mapping(start, (start + size -1));
471
Yasunori Gotobc02af92006-06-27 02:53:30 -0700472 ret = __add_pages(zone, start_pfn, nr_pages);
473 if (ret)
474 goto error;
475
Yasunori Gotobc02af92006-06-27 02:53:30 -0700476 return ret;
477error:
478 printk("%s: Problem encountered in __add_pages!\n", __func__);
479 return ret;
480}
481EXPORT_SYMBOL_GPL(arch_add_memory);
482
483int remove_memory(u64 start, u64 size)
484{
485 return -EINVAL;
486}
487EXPORT_SYMBOL_GPL(remove_memory);
488
Yasunori Goto82432292006-11-18 22:19:40 -0800489#if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
Keith Mannthey4942e992006-09-30 23:27:06 -0700490int memory_add_physaddr_to_nid(u64 start)
491{
492 return 0;
493}
Keith Mannthey8c2676a2006-09-30 23:27:07 -0700494EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
Keith Mannthey4942e992006-09-30 23:27:06 -0700495#endif
496
Keith Mannthey45e0b782006-09-30 23:27:09 -0700497#endif /* CONFIG_MEMORY_HOTPLUG */
498
499#ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
Andi Kleen9d99aaa2006-04-07 19:49:15 +0200500/*
501 * Memory Hotadd without sparsemem. The mem_maps have been allocated in advance,
502 * just online the pages.
503 */
504int __add_pages(struct zone *z, unsigned long start_pfn, unsigned long nr_pages)
505{
506 int err = -EIO;
507 unsigned long pfn;
508 unsigned long total = 0, mem = 0;
509 for (pfn = start_pfn; pfn < start_pfn + nr_pages; pfn++) {
Andi Kleen68a3a7f2006-04-07 19:49:18 +0200510 if (pfn_valid(pfn)) {
Andi Kleen9d99aaa2006-04-07 19:49:15 +0200511 online_page(pfn_to_page(pfn));
512 err = 0;
513 mem++;
514 }
515 total++;
516 }
517 if (!err) {
518 z->spanned_pages += total;
519 z->present_pages += mem;
520 z->zone_pgdat->node_spanned_pages += total;
521 z->zone_pgdat->node_present_pages += mem;
522 }
523 return err;
524}
Keith Mannthey45e0b782006-09-30 23:27:09 -0700525#endif
Matt Tolentino44df75e2006-01-17 07:03:41 +0100526
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
528 kcore_vsyscall;
529
530void __init mem_init(void)
531{
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200532 long codesize, reservedpages, datasize, initsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533
Jon Mason0dc243a2006-06-26 13:58:11 +0200534 pci_iommu_alloc();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 /* clear the zero-page */
537 memset(empty_zero_page, 0, PAGE_SIZE);
538
539 reservedpages = 0;
540
541 /* this will put all low memory onto the freelists */
Matt Tolentino2b976902005-06-23 00:08:06 -0700542#ifdef CONFIG_NUMA
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200543 totalram_pages = numa_free_all_bootmem();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544#else
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200545 totalram_pages = free_all_bootmem();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546#endif
Mel Gorman5cb248a2006-09-27 01:49:52 -0700547 reservedpages = end_pfn - totalram_pages -
548 absent_pages_in_range(0, end_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549
550 after_bootmem = 1;
551
552 codesize = (unsigned long) &_etext - (unsigned long) &_text;
553 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
554 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
555
556 /* Register memory areas for /proc/kcore */
557 kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
558 kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
559 VMALLOC_END-VMALLOC_START);
560 kclist_add(&kcore_kernel, &_stext, _end - _stext);
561 kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
562 kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START,
563 VSYSCALL_END - VSYSCALL_START);
564
Andi Kleen0a43e4b2005-09-12 18:49:24 +0200565 printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, %ldk data, %ldk init)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
567 end_pfn << (PAGE_SHIFT-10),
568 codesize >> 10,
569 reservedpages << (PAGE_SHIFT-10),
570 datasize >> 10,
571 initsize >> 10);
572
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +0100573#ifdef CONFIG_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 /*
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +0100575 * Sync boot_level4_pgt mappings with the init_level4_pgt
576 * except for the low identity mappings which are already zapped
577 * in init_level4_pgt. This sync-up is essential for AP's bringup
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 */
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +0100579 memcpy(boot_level4_pgt+1, init_level4_pgt+1, (PTRS_PER_PGD-1)*sizeof(pgd_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580#endif
581}
582
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200583void free_init_pages(char *what, unsigned long begin, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584{
585 unsigned long addr;
586
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200587 if (begin >= end)
588 return;
589
590 printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
591 for (addr = begin; addr < end; addr += PAGE_SIZE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 ClearPageReserved(virt_to_page(addr));
Nick Piggin7835e982006-03-22 00:08:40 -0800593 init_page_count(virt_to_page(addr));
Randy Dunlapc9cf5522006-06-27 02:53:52 -0700594 memset((void *)(addr & ~(PAGE_SIZE-1)),
595 POISON_FREE_INITMEM, PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 free_page(addr);
597 totalram_pages++;
598 }
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200599}
600
601void free_initmem(void)
602{
Randy Dunlapc9cf5522006-06-27 02:53:52 -0700603 memset(__initdata_begin, POISON_FREE_INITDATA,
604 __initdata_end - __initdata_begin);
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200605 free_init_pages("unused kernel memory",
606 (unsigned long)(&__init_begin),
607 (unsigned long)(&__init_end));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608}
609
Arjan van de Ven67df1972006-01-06 00:12:04 -0800610#ifdef CONFIG_DEBUG_RODATA
611
Arjan van de Ven67df1972006-01-06 00:12:04 -0800612void mark_rodata_ro(void)
613{
Heiko Carstensa581c2a2006-07-01 04:36:30 -0700614 unsigned long addr = (unsigned long)__start_rodata;
Arjan van de Ven67df1972006-01-06 00:12:04 -0800615
Heiko Carstensa581c2a2006-07-01 04:36:30 -0700616 for (; addr < (unsigned long)__end_rodata; addr += PAGE_SIZE)
Arjan van de Ven67df1972006-01-06 00:12:04 -0800617 change_page_attr_addr(addr, 1, PAGE_KERNEL_RO);
618
619 printk ("Write protecting the kernel read-only data: %luk\n",
Heiko Carstensa581c2a2006-07-01 04:36:30 -0700620 (__end_rodata - __start_rodata) >> 10);
Arjan van de Ven67df1972006-01-06 00:12:04 -0800621
622 /*
623 * change_page_attr_addr() requires a global_flush_tlb() call after it.
624 * We do this after the printk so that if something went wrong in the
625 * change, the printk gets out at least to give a better debug hint
626 * of who is the culprit.
627 */
628 global_flush_tlb();
629}
630#endif
631
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632#ifdef CONFIG_BLK_DEV_INITRD
633void free_initrd_mem(unsigned long start, unsigned long end)
634{
Gerd Hoffmannd167a512006-06-26 13:56:16 +0200635 free_init_pages("initrd memory", start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636}
637#endif
638
639void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
640{
Matt Tolentino2b976902005-06-23 00:08:06 -0700641#ifdef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 int nid = phys_to_nid(phys);
Andi Kleen5e58a022006-11-14 16:57:46 +0100643#endif
644 unsigned long pfn = phys >> PAGE_SHIFT;
645 if (pfn >= end_pfn) {
646 /* This can happen with kdump kernels when accessing firmware
647 tables. */
648 if (pfn < end_pfn_map)
649 return;
650 printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
651 phys, len);
652 return;
653 }
654
655 /* Should check here against the e820 map to avoid double free */
656#ifdef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 reserve_bootmem_node(NODE_DATA(nid), phys, len);
658#else
659 reserve_bootmem(phys, len);
660#endif
Mel Gorman0e0b8642006-09-27 01:49:56 -0700661 if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
Andi Kleene18c6872005-11-05 17:25:53 +0100662 dma_reserve += len / PAGE_SIZE;
Mel Gorman0e0b8642006-09-27 01:49:56 -0700663 set_dma_reserve(dma_reserve);
664 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665}
666
667int kern_addr_valid(unsigned long addr)
668{
669 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
670 pgd_t *pgd;
671 pud_t *pud;
672 pmd_t *pmd;
673 pte_t *pte;
674
675 if (above != 0 && above != -1UL)
676 return 0;
677
678 pgd = pgd_offset_k(addr);
679 if (pgd_none(*pgd))
680 return 0;
681
682 pud = pud_offset(pgd, addr);
683 if (pud_none(*pud))
684 return 0;
685
686 pmd = pmd_offset(pud, addr);
687 if (pmd_none(*pmd))
688 return 0;
689 if (pmd_large(*pmd))
690 return pfn_valid(pmd_pfn(*pmd));
691
692 pte = pte_offset_kernel(pmd, addr);
693 if (pte_none(*pte))
694 return 0;
695 return pfn_valid(pte_pfn(*pte));
696}
697
698#ifdef CONFIG_SYSCTL
699#include <linux/sysctl.h>
700
701extern int exception_trace, page_fault_trace;
702
703static ctl_table debug_table2[] = {
Eric W. Biedermanc37ce032007-02-14 00:33:51 -0800704 {
705 .ctl_name = 99,
706 .procname = "exception-trace",
707 .data = &exception_trace,
708 .maxlen = sizeof(int),
709 .mode = 0644,
710 .proc_handler = proc_dointvec
711 },
712 {}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713};
714
715static ctl_table debug_root_table2[] = {
Eric W. Biedermanc37ce032007-02-14 00:33:51 -0800716 {
717 .ctl_name = CTL_DEBUG,
718 .procname = "debug",
719 .mode = 0555,
720 .child = debug_table2
721 },
722 {}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723};
724
725static __init int x8664_sysctl_init(void)
726{
Eric W. Biederman0b4d4142007-02-14 00:34:09 -0800727 register_sysctl_table(debug_root_table2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 return 0;
729}
730__initcall(x8664_sysctl_init);
731#endif
732
Ernie Petrides103efcd2006-12-07 02:14:09 +0100733/* A pseudo VMA to allow ptrace access for the vsyscall page. This only
Andi Kleen1e014412005-04-16 15:24:55 -0700734 covers the 64bit vsyscall page now. 32bit has a real VMA now and does
735 not need special handling anymore. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736
737static struct vm_area_struct gate_vma = {
738 .vm_start = VSYSCALL_START,
Ernie Petrides103efcd2006-12-07 02:14:09 +0100739 .vm_end = VSYSCALL_START + (VSYSCALL_MAPPED_PAGES << PAGE_SHIFT),
740 .vm_page_prot = PAGE_READONLY_EXEC,
741 .vm_flags = VM_READ | VM_EXEC
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742};
743
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
745{
746#ifdef CONFIG_IA32_EMULATION
Andi Kleen1e014412005-04-16 15:24:55 -0700747 if (test_tsk_thread_flag(tsk, TIF_IA32))
748 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749#endif
750 return &gate_vma;
751}
752
753int in_gate_area(struct task_struct *task, unsigned long addr)
754{
755 struct vm_area_struct *vma = get_gate_vma(task);
Andi Kleen1e014412005-04-16 15:24:55 -0700756 if (!vma)
757 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 return (addr >= vma->vm_start) && (addr < vma->vm_end);
759}
760
761/* Use this when you have no reliable task/vma, typically from interrupt
762 * context. It is less reliable than using the task's vma and may give
763 * false positives.
764 */
765int in_gate_area_no_task(unsigned long addr)
766{
Andi Kleen1e014412005-04-16 15:24:55 -0700767 return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768}