| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 1 | /* | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 2 |  *    Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. | 
 | 3 |  *			 <benh@kernel.crashing.org> | 
 | 4 |  * | 
 | 5 |  *  This program is free software; you can redistribute it and/or | 
 | 6 |  *  modify it under the terms of the GNU General Public License | 
 | 7 |  *  as published by the Free Software Foundation; either version | 
 | 8 |  *  2 of the License, or (at your option) any later version. | 
 | 9 |  */ | 
 | 10 |  | 
 | 11 | #include <linux/config.h> | 
 | 12 | #include <linux/module.h> | 
 | 13 | #include <linux/errno.h> | 
 | 14 | #include <linux/sched.h> | 
 | 15 | #include <linux/kernel.h> | 
 | 16 | #include <linux/mm.h> | 
 | 17 | #include <linux/smp.h> | 
 | 18 | #include <linux/smp_lock.h> | 
 | 19 | #include <linux/stddef.h> | 
 | 20 | #include <linux/unistd.h> | 
 | 21 | #include <linux/slab.h> | 
 | 22 | #include <linux/user.h> | 
 | 23 | #include <linux/elf.h> | 
 | 24 | #include <linux/security.h> | 
 | 25 | #include <linux/bootmem.h> | 
 | 26 |  | 
 | 27 | #include <asm/pgtable.h> | 
 | 28 | #include <asm/system.h> | 
 | 29 | #include <asm/processor.h> | 
 | 30 | #include <asm/mmu.h> | 
 | 31 | #include <asm/mmu_context.h> | 
 | 32 | #include <asm/lmb.h> | 
 | 33 | #include <asm/machdep.h> | 
 | 34 | #include <asm/cputable.h> | 
 | 35 | #include <asm/sections.h> | 
| Benjamin Herrenschmidt | e822250 | 2006-03-28 23:15:54 +1100 | [diff] [blame] | 36 | #include <asm/firmware.h> | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 37 | #include <asm/vdso.h> | 
 | 38 | #include <asm/vdso_datapage.h> | 
 | 39 |  | 
 | 40 | #undef DEBUG | 
 | 41 |  | 
 | 42 | #ifdef DEBUG | 
 | 43 | #define DBG(fmt...) printk(fmt) | 
 | 44 | #else | 
 | 45 | #define DBG(fmt...) | 
 | 46 | #endif | 
 | 47 |  | 
 | 48 | /* Max supported size for symbol names */ | 
 | 49 | #define MAX_SYMNAME	64 | 
 | 50 |  | 
 | 51 | extern char vdso32_start, vdso32_end; | 
 | 52 | static void *vdso32_kbase = &vdso32_start; | 
 | 53 | unsigned int vdso32_pages; | 
 | 54 | unsigned long vdso32_sigtramp; | 
 | 55 | unsigned long vdso32_rt_sigtramp; | 
 | 56 |  | 
 | 57 | #ifdef CONFIG_PPC64 | 
 | 58 | extern char vdso64_start, vdso64_end; | 
 | 59 | static void *vdso64_kbase = &vdso64_start; | 
 | 60 | unsigned int vdso64_pages; | 
 | 61 | unsigned long vdso64_rt_sigtramp; | 
 | 62 | #endif /* CONFIG_PPC64 */ | 
 | 63 |  | 
 | 64 | /* | 
 | 65 |  * The vdso data page (aka. systemcfg for old ppc64 fans) is here. | 
 | 66 |  * Once the early boot kernel code no longer needs to muck around | 
 | 67 |  * with it, it will become dynamically allocated | 
 | 68 |  */ | 
 | 69 | static union { | 
 | 70 | 	struct vdso_data	data; | 
 | 71 | 	u8			page[PAGE_SIZE]; | 
 | 72 | } vdso_data_store __attribute__((__section__(".data.page_aligned"))); | 
 | 73 | struct vdso_data *vdso_data = &vdso_data_store.data; | 
 | 74 |  | 
 | 75 | /* Format of the patch table */ | 
 | 76 | struct vdso_patch_def | 
 | 77 | { | 
 | 78 | 	unsigned long	ftr_mask, ftr_value; | 
 | 79 | 	const char	*gen_name; | 
 | 80 | 	const char	*fix_name; | 
 | 81 | }; | 
 | 82 |  | 
 | 83 | /* Table of functions to patch based on the CPU type/revision | 
 | 84 |  * | 
 | 85 |  * Currently, we only change sync_dicache to do nothing on processors | 
 | 86 |  * with a coherent icache | 
 | 87 |  */ | 
 | 88 | static struct vdso_patch_def vdso_patches[] = { | 
 | 89 | 	{ | 
 | 90 | 		CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE, | 
 | 91 | 		"__kernel_sync_dicache", "__kernel_sync_dicache_p5" | 
 | 92 | 	}, | 
 | 93 | 	{ | 
 | 94 | 		CPU_FTR_USE_TB, 0, | 
 | 95 | 		"__kernel_gettimeofday", NULL | 
 | 96 | 	}, | 
 | 97 | }; | 
 | 98 |  | 
 | 99 | /* | 
 | 100 |  * Some infos carried around for each of them during parsing at | 
 | 101 |  * boot time. | 
 | 102 |  */ | 
 | 103 | struct lib32_elfinfo | 
 | 104 | { | 
 | 105 | 	Elf32_Ehdr	*hdr;		/* ptr to ELF */ | 
 | 106 | 	Elf32_Sym	*dynsym;	/* ptr to .dynsym section */ | 
 | 107 | 	unsigned long	dynsymsize;	/* size of .dynsym section */ | 
 | 108 | 	char		*dynstr;	/* ptr to .dynstr section */ | 
 | 109 | 	unsigned long	text;		/* offset of .text section in .so */ | 
 | 110 | }; | 
 | 111 |  | 
 | 112 | struct lib64_elfinfo | 
 | 113 | { | 
 | 114 | 	Elf64_Ehdr	*hdr; | 
 | 115 | 	Elf64_Sym	*dynsym; | 
 | 116 | 	unsigned long	dynsymsize; | 
 | 117 | 	char		*dynstr; | 
 | 118 | 	unsigned long	text; | 
 | 119 | }; | 
 | 120 |  | 
 | 121 |  | 
 | 122 | #ifdef __DEBUG | 
 | 123 | static void dump_one_vdso_page(struct page *pg, struct page *upg) | 
 | 124 | { | 
 | 125 | 	printk("kpg: %p (c:%d,f:%08lx)", __va(page_to_pfn(pg) << PAGE_SHIFT), | 
 | 126 | 	       page_count(pg), | 
 | 127 | 	       pg->flags); | 
 | 128 | 	if (upg/* && pg != upg*/) { | 
 | 129 | 		printk(" upg: %p (c:%d,f:%08lx)", __va(page_to_pfn(upg) | 
 | 130 | 						       << PAGE_SHIFT), | 
 | 131 | 		       page_count(upg), | 
 | 132 | 		       upg->flags); | 
 | 133 | 	} | 
 | 134 | 	printk("\n"); | 
 | 135 | } | 
 | 136 |  | 
 | 137 | static void dump_vdso_pages(struct vm_area_struct * vma) | 
 | 138 | { | 
 | 139 | 	int i; | 
 | 140 |  | 
 | 141 | 	if (!vma || test_thread_flag(TIF_32BIT)) { | 
 | 142 | 		printk("vDSO32 @ %016lx:\n", (unsigned long)vdso32_kbase); | 
 | 143 | 		for (i=0; i<vdso32_pages; i++) { | 
 | 144 | 			struct page *pg = virt_to_page(vdso32_kbase + | 
 | 145 | 						       i*PAGE_SIZE); | 
 | 146 | 			struct page *upg = (vma && vma->vm_mm) ? | 
| Linus Torvalds | 6aab341 | 2005-11-28 14:34:23 -0800 | [diff] [blame] | 147 | 				follow_page(vma, vma->vm_start + i*PAGE_SIZE, 0) | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 148 | 				: NULL; | 
 | 149 | 			dump_one_vdso_page(pg, upg); | 
 | 150 | 		} | 
 | 151 | 	} | 
 | 152 | 	if (!vma || !test_thread_flag(TIF_32BIT)) { | 
 | 153 | 		printk("vDSO64 @ %016lx:\n", (unsigned long)vdso64_kbase); | 
 | 154 | 		for (i=0; i<vdso64_pages; i++) { | 
 | 155 | 			struct page *pg = virt_to_page(vdso64_kbase + | 
 | 156 | 						       i*PAGE_SIZE); | 
 | 157 | 			struct page *upg = (vma && vma->vm_mm) ? | 
| Linus Torvalds | 6aab341 | 2005-11-28 14:34:23 -0800 | [diff] [blame] | 158 | 				follow_page(vma, vma->vm_start + i*PAGE_SIZE, 0) | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 159 | 				: NULL; | 
 | 160 | 			dump_one_vdso_page(pg, upg); | 
 | 161 | 		} | 
 | 162 | 	} | 
 | 163 | } | 
 | 164 | #endif /* DEBUG */ | 
 | 165 |  | 
 | 166 | /* | 
 | 167 |  * Keep a dummy vma_close for now, it will prevent VMA merging. | 
 | 168 |  */ | 
 | 169 | static void vdso_vma_close(struct vm_area_struct * vma) | 
 | 170 | { | 
 | 171 | } | 
 | 172 |  | 
 | 173 | /* | 
 | 174 |  * Our nopage() function, maps in the actual vDSO kernel pages, they will | 
 | 175 |  * be mapped read-only by do_no_page(), and eventually COW'ed, either | 
 | 176 |  * right away for an initial write access, or by do_wp_page(). | 
 | 177 |  */ | 
 | 178 | static struct page * vdso_vma_nopage(struct vm_area_struct * vma, | 
 | 179 | 				     unsigned long address, int *type) | 
 | 180 | { | 
 | 181 | 	unsigned long offset = address - vma->vm_start; | 
 | 182 | 	struct page *pg; | 
 | 183 | #ifdef CONFIG_PPC64 | 
| Benjamin Herrenschmidt | 0551fbd | 2006-02-28 16:59:19 -0800 | [diff] [blame] | 184 | 	void *vbase = (vma->vm_mm->task_size > TASK_SIZE_USER32) ? | 
 | 185 | 		vdso64_kbase : vdso32_kbase; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 186 | #else | 
 | 187 | 	void *vbase = vdso32_kbase; | 
 | 188 | #endif | 
 | 189 |  | 
 | 190 | 	DBG("vdso_vma_nopage(current: %s, address: %016lx, off: %lx)\n", | 
 | 191 | 	    current->comm, address, offset); | 
 | 192 |  | 
 | 193 | 	if (address < vma->vm_start || address > vma->vm_end) | 
 | 194 | 		return NOPAGE_SIGBUS; | 
 | 195 |  | 
 | 196 | 	/* | 
 | 197 | 	 * Last page is systemcfg. | 
 | 198 | 	 */ | 
 | 199 | 	if ((vma->vm_end - address) <= PAGE_SIZE) | 
 | 200 | 		pg = virt_to_page(vdso_data); | 
 | 201 | 	else | 
 | 202 | 		pg = virt_to_page(vbase + offset); | 
 | 203 |  | 
 | 204 | 	get_page(pg); | 
 | 205 | 	DBG(" ->page count: %d\n", page_count(pg)); | 
 | 206 |  | 
 | 207 | 	return pg; | 
 | 208 | } | 
 | 209 |  | 
 | 210 | static struct vm_operations_struct vdso_vmops = { | 
 | 211 | 	.close	= vdso_vma_close, | 
 | 212 | 	.nopage	= vdso_vma_nopage, | 
 | 213 | }; | 
 | 214 |  | 
 | 215 | /* | 
 | 216 |  * This is called from binfmt_elf, we create the special vma for the | 
 | 217 |  * vDSO and insert it into the mm struct tree | 
 | 218 |  */ | 
 | 219 | int arch_setup_additional_pages(struct linux_binprm *bprm, | 
 | 220 | 				int executable_stack) | 
 | 221 | { | 
 | 222 | 	struct mm_struct *mm = current->mm; | 
 | 223 | 	struct vm_area_struct *vma; | 
 | 224 | 	unsigned long vdso_pages; | 
 | 225 | 	unsigned long vdso_base; | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 226 | 	int rc; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 227 |  | 
 | 228 | #ifdef CONFIG_PPC64 | 
 | 229 | 	if (test_thread_flag(TIF_32BIT)) { | 
 | 230 | 		vdso_pages = vdso32_pages; | 
 | 231 | 		vdso_base = VDSO32_MBASE; | 
 | 232 | 	} else { | 
 | 233 | 		vdso_pages = vdso64_pages; | 
 | 234 | 		vdso_base = VDSO64_MBASE; | 
 | 235 | 	} | 
 | 236 | #else | 
 | 237 | 	vdso_pages = vdso32_pages; | 
 | 238 | 	vdso_base = VDSO32_MBASE; | 
 | 239 | #endif | 
 | 240 |  | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 241 | 	current->mm->context.vdso_base = 0; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 242 |  | 
 | 243 | 	/* vDSO has a problem and was disabled, just don't "enable" it for the | 
 | 244 | 	 * process | 
 | 245 | 	 */ | 
 | 246 | 	if (vdso_pages == 0) | 
 | 247 | 		return 0; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 248 | 	/* Add a page to the vdso size for the data page */ | 
 | 249 | 	vdso_pages ++; | 
 | 250 |  | 
 | 251 | 	/* | 
 | 252 | 	 * pick a base address for the vDSO in process space. We try to put it | 
 | 253 | 	 * at vdso_base which is the "natural" base for it, but we might fail | 
 | 254 | 	 * and end up putting it elsewhere. | 
 | 255 | 	 */ | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 256 | 	down_write(&mm->mmap_sem); | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 257 | 	vdso_base = get_unmapped_area(NULL, vdso_base, | 
 | 258 | 				      vdso_pages << PAGE_SHIFT, 0, 0); | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 259 | 	if (IS_ERR_VALUE(vdso_base)) { | 
 | 260 | 		rc = vdso_base; | 
 | 261 | 		goto fail_mmapsem; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 262 | 	} | 
 | 263 |  | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 264 |  | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 265 | 	/* Allocate a VMA structure and fill it up */ | 
 | 266 | 	vma = kmem_cache_zalloc(vm_area_cachep, SLAB_KERNEL); | 
 | 267 | 	if (vma == NULL) { | 
 | 268 | 		rc = -ENOMEM; | 
 | 269 | 		goto fail_mmapsem; | 
 | 270 | 	} | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 271 | 	vma->vm_mm = mm; | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 272 | 	vma->vm_start = vdso_base; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 273 | 	vma->vm_end = vma->vm_start + (vdso_pages << PAGE_SHIFT); | 
 | 274 |  | 
 | 275 | 	/* | 
 | 276 | 	 * our vma flags don't have VM_WRITE so by default, the process isn't | 
 | 277 | 	 * allowed to write those pages. | 
 | 278 | 	 * gdb can break that with ptrace interface, and thus trigger COW on | 
 | 279 | 	 * those pages but it's then your responsibility to never do that on | 
 | 280 | 	 * the "data" page of the vDSO or you'll stop getting kernel updates | 
 | 281 | 	 * and your nice userland gettimeofday will be totally dead. | 
 | 282 | 	 * It's fine to use that for setting breakpoints in the vDSO code | 
 | 283 | 	 * pages though | 
 | 284 | 	 */ | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 285 | 	vma->vm_flags = VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 286 | 	vma->vm_flags |= mm->def_flags; | 
 | 287 | 	vma->vm_page_prot = protection_map[vma->vm_flags & 0x7]; | 
 | 288 | 	vma->vm_ops = &vdso_vmops; | 
 | 289 |  | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 290 | 	/* Insert new VMA */ | 
 | 291 | 	rc = insert_vm_struct(mm, vma); | 
 | 292 | 	if (rc) | 
 | 293 | 		goto fail_vma; | 
 | 294 |  | 
 | 295 | 	/* Put vDSO base into mm struct and account for memory usage */ | 
 | 296 | 	current->mm->context.vdso_base = vdso_base; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 297 | 	mm->total_vm += (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; | 
 | 298 | 	up_write(&mm->mmap_sem); | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 299 | 	return 0; | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 300 |  | 
 | 301 |  fail_vma: | 
 | 302 | 	kmem_cache_free(vm_area_cachep, vma); | 
 | 303 |  fail_mmapsem: | 
 | 304 | 	up_write(&mm->mmap_sem); | 
 | 305 | 	return rc; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 306 | } | 
 | 307 |  | 
| Benjamin Herrenschmidt | a5bba93 | 2006-05-30 13:51:37 +1000 | [diff] [blame] | 308 | const char *arch_vma_name(struct vm_area_struct *vma) | 
 | 309 | { | 
 | 310 | 	if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base) | 
 | 311 | 		return "[vdso]"; | 
 | 312 | 	return NULL; | 
 | 313 | } | 
 | 314 |  | 
 | 315 |  | 
 | 316 |  | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 317 | static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname, | 
 | 318 | 				  unsigned long *size) | 
 | 319 | { | 
 | 320 | 	Elf32_Shdr *sechdrs; | 
 | 321 | 	unsigned int i; | 
 | 322 | 	char *secnames; | 
 | 323 |  | 
 | 324 | 	/* Grab section headers and strings so we can tell who is who */ | 
 | 325 | 	sechdrs = (void *)ehdr + ehdr->e_shoff; | 
 | 326 | 	secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; | 
 | 327 |  | 
 | 328 | 	/* Find the section they want */ | 
 | 329 | 	for (i = 1; i < ehdr->e_shnum; i++) { | 
 | 330 | 		if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) { | 
 | 331 | 			if (size) | 
 | 332 | 				*size = sechdrs[i].sh_size; | 
 | 333 | 			return (void *)ehdr + sechdrs[i].sh_offset; | 
 | 334 | 		} | 
 | 335 | 	} | 
 | 336 | 	*size = 0; | 
 | 337 | 	return NULL; | 
 | 338 | } | 
 | 339 |  | 
 | 340 | static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib, | 
 | 341 | 					const char *symname) | 
 | 342 | { | 
 | 343 | 	unsigned int i; | 
 | 344 | 	char name[MAX_SYMNAME], *c; | 
 | 345 |  | 
 | 346 | 	for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) { | 
 | 347 | 		if (lib->dynsym[i].st_name == 0) | 
 | 348 | 			continue; | 
 | 349 | 		strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, | 
 | 350 | 			MAX_SYMNAME); | 
 | 351 | 		c = strchr(name, '@'); | 
 | 352 | 		if (c) | 
 | 353 | 			*c = 0; | 
 | 354 | 		if (strcmp(symname, name) == 0) | 
 | 355 | 			return &lib->dynsym[i]; | 
 | 356 | 	} | 
 | 357 | 	return NULL; | 
 | 358 | } | 
 | 359 |  | 
 | 360 | /* Note that we assume the section is .text and the symbol is relative to | 
 | 361 |  * the library base | 
 | 362 |  */ | 
 | 363 | static unsigned long __init find_function32(struct lib32_elfinfo *lib, | 
 | 364 | 					    const char *symname) | 
 | 365 | { | 
 | 366 | 	Elf32_Sym *sym = find_symbol32(lib, symname); | 
 | 367 |  | 
 | 368 | 	if (sym == NULL) { | 
 | 369 | 		printk(KERN_WARNING "vDSO32: function %s not found !\n", | 
 | 370 | 		       symname); | 
 | 371 | 		return 0; | 
 | 372 | 	} | 
 | 373 | 	return sym->st_value - VDSO32_LBASE; | 
 | 374 | } | 
 | 375 |  | 
 | 376 | static int vdso_do_func_patch32(struct lib32_elfinfo *v32, | 
 | 377 | 				struct lib64_elfinfo *v64, | 
 | 378 | 				const char *orig, const char *fix) | 
 | 379 | { | 
 | 380 | 	Elf32_Sym *sym32_gen, *sym32_fix; | 
 | 381 |  | 
 | 382 | 	sym32_gen = find_symbol32(v32, orig); | 
 | 383 | 	if (sym32_gen == NULL) { | 
 | 384 | 		printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig); | 
 | 385 | 		return -1; | 
 | 386 | 	} | 
 | 387 | 	if (fix == NULL) { | 
 | 388 | 		sym32_gen->st_name = 0; | 
 | 389 | 		return 0; | 
 | 390 | 	} | 
 | 391 | 	sym32_fix = find_symbol32(v32, fix); | 
 | 392 | 	if (sym32_fix == NULL) { | 
 | 393 | 		printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix); | 
 | 394 | 		return -1; | 
 | 395 | 	} | 
 | 396 | 	sym32_gen->st_value = sym32_fix->st_value; | 
 | 397 | 	sym32_gen->st_size = sym32_fix->st_size; | 
 | 398 | 	sym32_gen->st_info = sym32_fix->st_info; | 
 | 399 | 	sym32_gen->st_other = sym32_fix->st_other; | 
 | 400 | 	sym32_gen->st_shndx = sym32_fix->st_shndx; | 
 | 401 |  | 
 | 402 | 	return 0; | 
 | 403 | } | 
 | 404 |  | 
 | 405 |  | 
 | 406 | #ifdef CONFIG_PPC64 | 
 | 407 |  | 
 | 408 | static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname, | 
 | 409 | 				  unsigned long *size) | 
 | 410 | { | 
 | 411 | 	Elf64_Shdr *sechdrs; | 
 | 412 | 	unsigned int i; | 
 | 413 | 	char *secnames; | 
 | 414 |  | 
 | 415 | 	/* Grab section headers and strings so we can tell who is who */ | 
 | 416 | 	sechdrs = (void *)ehdr + ehdr->e_shoff; | 
 | 417 | 	secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; | 
 | 418 |  | 
 | 419 | 	/* Find the section they want */ | 
 | 420 | 	for (i = 1; i < ehdr->e_shnum; i++) { | 
 | 421 | 		if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) { | 
 | 422 | 			if (size) | 
 | 423 | 				*size = sechdrs[i].sh_size; | 
 | 424 | 			return (void *)ehdr + sechdrs[i].sh_offset; | 
 | 425 | 		} | 
 | 426 | 	} | 
 | 427 | 	if (size) | 
 | 428 | 		*size = 0; | 
 | 429 | 	return NULL; | 
 | 430 | } | 
 | 431 |  | 
 | 432 | static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib, | 
 | 433 | 					const char *symname) | 
 | 434 | { | 
 | 435 | 	unsigned int i; | 
 | 436 | 	char name[MAX_SYMNAME], *c; | 
 | 437 |  | 
 | 438 | 	for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) { | 
 | 439 | 		if (lib->dynsym[i].st_name == 0) | 
 | 440 | 			continue; | 
 | 441 | 		strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, | 
 | 442 | 			MAX_SYMNAME); | 
 | 443 | 		c = strchr(name, '@'); | 
 | 444 | 		if (c) | 
 | 445 | 			*c = 0; | 
 | 446 | 		if (strcmp(symname, name) == 0) | 
 | 447 | 			return &lib->dynsym[i]; | 
 | 448 | 	} | 
 | 449 | 	return NULL; | 
 | 450 | } | 
 | 451 |  | 
 | 452 | /* Note that we assume the section is .text and the symbol is relative to | 
 | 453 |  * the library base | 
 | 454 |  */ | 
 | 455 | static unsigned long __init find_function64(struct lib64_elfinfo *lib, | 
 | 456 | 					    const char *symname) | 
 | 457 | { | 
 | 458 | 	Elf64_Sym *sym = find_symbol64(lib, symname); | 
 | 459 |  | 
 | 460 | 	if (sym == NULL) { | 
 | 461 | 		printk(KERN_WARNING "vDSO64: function %s not found !\n", | 
 | 462 | 		       symname); | 
 | 463 | 		return 0; | 
 | 464 | 	} | 
 | 465 | #ifdef VDS64_HAS_DESCRIPTORS | 
 | 466 | 	return *((u64 *)(vdso64_kbase + sym->st_value - VDSO64_LBASE)) - | 
 | 467 | 		VDSO64_LBASE; | 
 | 468 | #else | 
 | 469 | 	return sym->st_value - VDSO64_LBASE; | 
 | 470 | #endif | 
 | 471 | } | 
 | 472 |  | 
 | 473 | static int vdso_do_func_patch64(struct lib32_elfinfo *v32, | 
 | 474 | 				struct lib64_elfinfo *v64, | 
 | 475 | 				const char *orig, const char *fix) | 
 | 476 | { | 
 | 477 | 	Elf64_Sym *sym64_gen, *sym64_fix; | 
 | 478 |  | 
 | 479 | 	sym64_gen = find_symbol64(v64, orig); | 
 | 480 | 	if (sym64_gen == NULL) { | 
 | 481 | 		printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig); | 
 | 482 | 		return -1; | 
 | 483 | 	} | 
 | 484 | 	if (fix == NULL) { | 
 | 485 | 		sym64_gen->st_name = 0; | 
 | 486 | 		return 0; | 
 | 487 | 	} | 
 | 488 | 	sym64_fix = find_symbol64(v64, fix); | 
 | 489 | 	if (sym64_fix == NULL) { | 
 | 490 | 		printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix); | 
 | 491 | 		return -1; | 
 | 492 | 	} | 
 | 493 | 	sym64_gen->st_value = sym64_fix->st_value; | 
 | 494 | 	sym64_gen->st_size = sym64_fix->st_size; | 
 | 495 | 	sym64_gen->st_info = sym64_fix->st_info; | 
 | 496 | 	sym64_gen->st_other = sym64_fix->st_other; | 
 | 497 | 	sym64_gen->st_shndx = sym64_fix->st_shndx; | 
 | 498 |  | 
 | 499 | 	return 0; | 
 | 500 | } | 
 | 501 |  | 
 | 502 | #endif /* CONFIG_PPC64 */ | 
 | 503 |  | 
 | 504 |  | 
 | 505 | static __init int vdso_do_find_sections(struct lib32_elfinfo *v32, | 
 | 506 | 					struct lib64_elfinfo *v64) | 
 | 507 | { | 
 | 508 | 	void *sect; | 
 | 509 |  | 
 | 510 | 	/* | 
 | 511 | 	 * Locate symbol tables & text section | 
 | 512 | 	 */ | 
 | 513 |  | 
 | 514 | 	v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize); | 
 | 515 | 	v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL); | 
 | 516 | 	if (v32->dynsym == NULL || v32->dynstr == NULL) { | 
 | 517 | 		printk(KERN_ERR "vDSO32: required symbol section not found\n"); | 
 | 518 | 		return -1; | 
 | 519 | 	} | 
 | 520 | 	sect = find_section32(v32->hdr, ".text", NULL); | 
 | 521 | 	if (sect == NULL) { | 
 | 522 | 		printk(KERN_ERR "vDSO32: the .text section was not found\n"); | 
 | 523 | 		return -1; | 
 | 524 | 	} | 
 | 525 | 	v32->text = sect - vdso32_kbase; | 
 | 526 |  | 
 | 527 | #ifdef CONFIG_PPC64 | 
 | 528 | 	v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize); | 
 | 529 | 	v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL); | 
 | 530 | 	if (v64->dynsym == NULL || v64->dynstr == NULL) { | 
 | 531 | 		printk(KERN_ERR "vDSO64: required symbol section not found\n"); | 
 | 532 | 		return -1; | 
 | 533 | 	} | 
 | 534 | 	sect = find_section64(v64->hdr, ".text", NULL); | 
 | 535 | 	if (sect == NULL) { | 
 | 536 | 		printk(KERN_ERR "vDSO64: the .text section was not found\n"); | 
 | 537 | 		return -1; | 
 | 538 | 	} | 
 | 539 | 	v64->text = sect - vdso64_kbase; | 
 | 540 | #endif /* CONFIG_PPC64 */ | 
 | 541 |  | 
 | 542 | 	return 0; | 
 | 543 | } | 
 | 544 |  | 
 | 545 | static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32, | 
 | 546 | 					  struct lib64_elfinfo *v64) | 
 | 547 | { | 
 | 548 | 	/* | 
 | 549 | 	 * Find signal trampolines | 
 | 550 | 	 */ | 
 | 551 |  | 
 | 552 | #ifdef CONFIG_PPC64 | 
 | 553 | 	vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64"); | 
 | 554 | #endif | 
 | 555 | 	vdso32_sigtramp	   = find_function32(v32, "__kernel_sigtramp32"); | 
 | 556 | 	vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32"); | 
 | 557 | } | 
 | 558 |  | 
 | 559 | static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32, | 
 | 560 | 				       struct lib64_elfinfo *v64) | 
 | 561 | { | 
 | 562 | 	Elf32_Sym *sym32; | 
 | 563 | #ifdef CONFIG_PPC64 | 
 | 564 | 	Elf64_Sym *sym64; | 
 | 565 |  | 
 | 566 |        	sym64 = find_symbol64(v64, "__kernel_datapage_offset"); | 
 | 567 | 	if (sym64 == NULL) { | 
 | 568 | 		printk(KERN_ERR "vDSO64: Can't find symbol " | 
 | 569 | 		       "__kernel_datapage_offset !\n"); | 
 | 570 | 		return -1; | 
 | 571 | 	} | 
 | 572 | 	*((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) = | 
 | 573 | 		(vdso64_pages << PAGE_SHIFT) - | 
 | 574 | 		(sym64->st_value - VDSO64_LBASE); | 
 | 575 | #endif /* CONFIG_PPC64 */ | 
 | 576 |  | 
 | 577 | 	sym32 = find_symbol32(v32, "__kernel_datapage_offset"); | 
 | 578 | 	if (sym32 == NULL) { | 
 | 579 | 		printk(KERN_ERR "vDSO32: Can't find symbol " | 
 | 580 | 		       "__kernel_datapage_offset !\n"); | 
 | 581 | 		return -1; | 
 | 582 | 	} | 
 | 583 | 	*((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) = | 
 | 584 | 		(vdso32_pages << PAGE_SHIFT) - | 
 | 585 | 		(sym32->st_value - VDSO32_LBASE); | 
 | 586 |  | 
 | 587 | 	return 0; | 
 | 588 | } | 
 | 589 |  | 
 | 590 | static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32, | 
 | 591 | 				       struct lib64_elfinfo *v64) | 
 | 592 | { | 
 | 593 | 	int i; | 
 | 594 |  | 
 | 595 | 	for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) { | 
 | 596 | 		struct vdso_patch_def *patch = &vdso_patches[i]; | 
 | 597 | 		int match = (cur_cpu_spec->cpu_features & patch->ftr_mask) | 
 | 598 | 			== patch->ftr_value; | 
 | 599 | 		if (!match) | 
 | 600 | 			continue; | 
 | 601 |  | 
 | 602 | 		DBG("replacing %s with %s...\n", patch->gen_name, | 
 | 603 | 		    patch->fix_name ? "NONE" : patch->fix_name); | 
 | 604 |  | 
 | 605 | 		/* | 
 | 606 | 		 * Patch the 32 bits and 64 bits symbols. Note that we do not | 
 | 607 | 		 * patch the "." symbol on 64 bits. | 
 | 608 | 		 * It would be easy to do, but doesn't seem to be necessary, | 
 | 609 | 		 * patching the OPD symbol is enough. | 
 | 610 | 		 */ | 
 | 611 | 		vdso_do_func_patch32(v32, v64, patch->gen_name, | 
 | 612 | 				     patch->fix_name); | 
 | 613 | #ifdef CONFIG_PPC64 | 
 | 614 | 		vdso_do_func_patch64(v32, v64, patch->gen_name, | 
 | 615 | 				     patch->fix_name); | 
 | 616 | #endif /* CONFIG_PPC64 */ | 
 | 617 | 	} | 
 | 618 |  | 
 | 619 | 	return 0; | 
 | 620 | } | 
 | 621 |  | 
 | 622 |  | 
 | 623 | static __init int vdso_setup(void) | 
 | 624 | { | 
 | 625 | 	struct lib32_elfinfo	v32; | 
 | 626 | 	struct lib64_elfinfo	v64; | 
 | 627 |  | 
 | 628 | 	v32.hdr = vdso32_kbase; | 
 | 629 | #ifdef CONFIG_PPC64 | 
 | 630 | 	v64.hdr = vdso64_kbase; | 
 | 631 | #endif | 
 | 632 | 	if (vdso_do_find_sections(&v32, &v64)) | 
 | 633 | 		return -1; | 
 | 634 |  | 
 | 635 | 	if (vdso_fixup_datapage(&v32, &v64)) | 
 | 636 | 		return -1; | 
 | 637 |  | 
 | 638 | 	if (vdso_fixup_alt_funcs(&v32, &v64)) | 
 | 639 | 		return -1; | 
 | 640 |  | 
 | 641 | 	vdso_setup_trampolines(&v32, &v64); | 
 | 642 |  | 
 | 643 | 	return 0; | 
 | 644 | } | 
 | 645 |  | 
 | 646 | /* | 
 | 647 |  * Called from setup_arch to initialize the bitmap of available | 
 | 648 |  * syscalls in the systemcfg page | 
 | 649 |  */ | 
 | 650 | static void __init vdso_setup_syscall_map(void) | 
 | 651 | { | 
 | 652 | 	unsigned int i; | 
 | 653 | 	extern unsigned long *sys_call_table; | 
 | 654 | 	extern unsigned long sys_ni_syscall; | 
 | 655 |  | 
 | 656 |  | 
 | 657 | 	for (i = 0; i < __NR_syscalls; i++) { | 
 | 658 | #ifdef CONFIG_PPC64 | 
 | 659 | 		if (sys_call_table[i*2] != sys_ni_syscall) | 
 | 660 | 			vdso_data->syscall_map_64[i >> 5] |= | 
 | 661 | 				0x80000000UL >> (i & 0x1f); | 
 | 662 | 		if (sys_call_table[i*2+1] != sys_ni_syscall) | 
 | 663 | 			vdso_data->syscall_map_32[i >> 5] |= | 
 | 664 | 				0x80000000UL >> (i & 0x1f); | 
 | 665 | #else /* CONFIG_PPC64 */ | 
 | 666 | 		if (sys_call_table[i] != sys_ni_syscall) | 
 | 667 | 			vdso_data->syscall_map_32[i >> 5] |= | 
 | 668 | 				0x80000000UL >> (i & 0x1f); | 
 | 669 | #endif /* CONFIG_PPC64 */ | 
 | 670 | 	} | 
 | 671 | } | 
 | 672 |  | 
 | 673 |  | 
 | 674 | void __init vdso_init(void) | 
 | 675 | { | 
 | 676 | 	int i; | 
 | 677 |  | 
 | 678 | #ifdef CONFIG_PPC64 | 
 | 679 | 	/* | 
 | 680 | 	 * Fill up the "systemcfg" stuff for backward compatiblity | 
 | 681 | 	 */ | 
 | 682 | 	strcpy(vdso_data->eye_catcher, "SYSTEMCFG:PPC64"); | 
 | 683 | 	vdso_data->version.major = SYSTEMCFG_MAJOR; | 
 | 684 | 	vdso_data->version.minor = SYSTEMCFG_MINOR; | 
 | 685 | 	vdso_data->processor = mfspr(SPRN_PVR); | 
| Benjamin Herrenschmidt | e822250 | 2006-03-28 23:15:54 +1100 | [diff] [blame] | 686 | 	/* | 
 | 687 | 	 * Fake the old platform number for pSeries and iSeries and add | 
 | 688 | 	 * in LPAR bit if necessary | 
 | 689 | 	 */ | 
 | 690 | 	vdso_data->platform = machine_is(iseries) ? 0x200 : 0x100; | 
 | 691 | 	if (firmware_has_feature(FW_FEATURE_LPAR)) | 
 | 692 | 		vdso_data->platform |= 1; | 
| Benjamin Herrenschmidt | a7f290d | 2005-11-11 21:15:21 +1100 | [diff] [blame] | 693 | 	vdso_data->physicalMemorySize = lmb_phys_mem_size(); | 
 | 694 | 	vdso_data->dcache_size = ppc64_caches.dsize; | 
 | 695 | 	vdso_data->dcache_line_size = ppc64_caches.dline_size; | 
 | 696 | 	vdso_data->icache_size = ppc64_caches.isize; | 
 | 697 | 	vdso_data->icache_line_size = ppc64_caches.iline_size; | 
 | 698 |  | 
 | 699 | 	/* | 
 | 700 | 	 * Calculate the size of the 64 bits vDSO | 
 | 701 | 	 */ | 
 | 702 | 	vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT; | 
 | 703 | 	DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages); | 
 | 704 | #endif /* CONFIG_PPC64 */ | 
 | 705 |  | 
 | 706 |  | 
 | 707 | 	/* | 
 | 708 | 	 * Calculate the size of the 32 bits vDSO | 
 | 709 | 	 */ | 
 | 710 | 	vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT; | 
 | 711 | 	DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages); | 
 | 712 |  | 
 | 713 |  | 
 | 714 | 	/* | 
 | 715 | 	 * Setup the syscall map in the vDOS | 
 | 716 | 	 */ | 
 | 717 | 	vdso_setup_syscall_map(); | 
 | 718 | 	/* | 
 | 719 | 	 * Initialize the vDSO images in memory, that is do necessary | 
 | 720 | 	 * fixups of vDSO symbols, locate trampolines, etc... | 
 | 721 | 	 */ | 
 | 722 | 	if (vdso_setup()) { | 
 | 723 | 		printk(KERN_ERR "vDSO setup failure, not enabled !\n"); | 
 | 724 | 		vdso32_pages = 0; | 
 | 725 | #ifdef CONFIG_PPC64 | 
 | 726 | 		vdso64_pages = 0; | 
 | 727 | #endif | 
 | 728 | 		return; | 
 | 729 | 	} | 
 | 730 |  | 
 | 731 | 	/* Make sure pages are in the correct state */ | 
 | 732 | 	for (i = 0; i < vdso32_pages; i++) { | 
 | 733 | 		struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE); | 
 | 734 | 		ClearPageReserved(pg); | 
 | 735 | 		get_page(pg); | 
 | 736 |  | 
 | 737 | 	} | 
 | 738 | #ifdef CONFIG_PPC64 | 
 | 739 | 	for (i = 0; i < vdso64_pages; i++) { | 
 | 740 | 		struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE); | 
 | 741 | 		ClearPageReserved(pg); | 
 | 742 | 		get_page(pg); | 
 | 743 | 	} | 
 | 744 | #endif /* CONFIG_PPC64 */ | 
 | 745 |  | 
 | 746 | 	get_page(virt_to_page(vdso_data)); | 
 | 747 | } | 
 | 748 |  | 
 | 749 | int in_gate_area_no_task(unsigned long addr) | 
 | 750 | { | 
 | 751 | 	return 0; | 
 | 752 | } | 
 | 753 |  | 
 | 754 | int in_gate_area(struct task_struct *task, unsigned long addr) | 
 | 755 | { | 
 | 756 | 	return 0; | 
 | 757 | } | 
 | 758 |  | 
 | 759 | struct vm_area_struct *get_gate_vma(struct task_struct *tsk) | 
 | 760 | { | 
 | 761 | 	return NULL; | 
 | 762 | } | 
 | 763 |  |