Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation, version 2. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, but |
| 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or |
| 11 | * NON INFRINGEMENT. See the GNU General Public License for |
| 12 | * more details. |
| 13 | */ |
| 14 | |
| 15 | #include <linux/sched.h> |
| 16 | #include <linux/preempt.h> |
| 17 | #include <linux/module.h> |
| 18 | #include <linux/fs.h> |
| 19 | #include <linux/kprobes.h> |
| 20 | #include <linux/elfcore.h> |
| 21 | #include <linux/tick.h> |
| 22 | #include <linux/init.h> |
| 23 | #include <linux/mm.h> |
| 24 | #include <linux/compat.h> |
| 25 | #include <linux/hardirq.h> |
| 26 | #include <linux/syscalls.h> |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 27 | #include <linux/kernel.h> |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 28 | #include <linux/tracehook.h> |
| 29 | #include <linux/signal.h> |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 30 | #include <asm/stack.h> |
Paul Gortmaker | 34f2c0a | 2012-04-01 16:38:46 -0400 | [diff] [blame] | 31 | #include <asm/switch_to.h> |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 32 | #include <asm/homecache.h> |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 33 | #include <asm/syscalls.h> |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 34 | #include <asm/traps.h> |
David Howells | bd119c6 | 2012-03-28 18:30:03 +0100 | [diff] [blame] | 35 | #include <asm/setup.h> |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 36 | #ifdef CONFIG_HARDWALL |
| 37 | #include <asm/hardwall.h> |
| 38 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 39 | #include <arch/chip.h> |
| 40 | #include <arch/abi.h> |
David Howells | bd119c6 | 2012-03-28 18:30:03 +0100 | [diff] [blame] | 41 | #include <arch/sim_def.h> |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 42 | |
| 43 | |
| 44 | /* |
| 45 | * Use the (x86) "idle=poll" option to prefer low latency when leaving the |
| 46 | * idle loop over low power while in the idle loop, e.g. if we have |
| 47 | * one thread per core and we want to get threads out of futex waits fast. |
| 48 | */ |
| 49 | static int no_idle_nap; |
| 50 | static int __init idle_setup(char *str) |
| 51 | { |
| 52 | if (!str) |
| 53 | return -EINVAL; |
| 54 | |
| 55 | if (!strcmp(str, "poll")) { |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 56 | pr_info("using polling idle threads.\n"); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 57 | no_idle_nap = 1; |
| 58 | } else if (!strcmp(str, "halt")) |
| 59 | no_idle_nap = 0; |
| 60 | else |
| 61 | return -1; |
| 62 | |
| 63 | return 0; |
| 64 | } |
| 65 | early_param("idle", idle_setup); |
| 66 | |
| 67 | /* |
| 68 | * The idle thread. There's no useful work to be |
| 69 | * done, so just try to conserve power and have a |
| 70 | * low exit latency (ie sit in a loop waiting for |
| 71 | * somebody to say that they'd like to reschedule) |
| 72 | */ |
| 73 | void cpu_idle(void) |
| 74 | { |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 75 | int cpu = smp_processor_id(); |
| 76 | |
| 77 | |
| 78 | current_thread_info()->status |= TS_POLLING; |
| 79 | |
| 80 | if (no_idle_nap) { |
| 81 | while (1) { |
| 82 | while (!need_resched()) |
| 83 | cpu_relax(); |
| 84 | schedule(); |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | /* endless idle loop with no priority at all */ |
| 89 | while (1) { |
Frederic Weisbecker | 1268fbc | 2011-11-17 18:48:14 +0100 | [diff] [blame] | 90 | tick_nohz_idle_enter(); |
| 91 | rcu_idle_enter(); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 92 | while (!need_resched()) { |
| 93 | if (cpu_is_offline(cpu)) |
| 94 | BUG(); /* no HOTPLUG_CPU */ |
| 95 | |
| 96 | local_irq_disable(); |
| 97 | __get_cpu_var(irq_stat).idle_timestamp = jiffies; |
| 98 | current_thread_info()->status &= ~TS_POLLING; |
| 99 | /* |
| 100 | * TS_POLLING-cleared state must be visible before we |
| 101 | * test NEED_RESCHED: |
| 102 | */ |
| 103 | smp_mb(); |
| 104 | |
| 105 | if (!need_resched()) |
| 106 | _cpu_idle(); |
| 107 | else |
| 108 | local_irq_enable(); |
| 109 | current_thread_info()->status |= TS_POLLING; |
| 110 | } |
Frederic Weisbecker | 1268fbc | 2011-11-17 18:48:14 +0100 | [diff] [blame] | 111 | rcu_idle_exit(); |
| 112 | tick_nohz_idle_exit(); |
Thomas Gleixner | bd2f553 | 2011-03-21 12:33:18 +0100 | [diff] [blame] | 113 | schedule_preempt_disabled(); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 114 | } |
| 115 | } |
| 116 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 117 | /* |
Thomas Gleixner | d909a81 | 2012-05-05 15:05:47 +0000 | [diff] [blame] | 118 | * Release a thread_info structure |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 119 | */ |
Thomas Gleixner | d909a81 | 2012-05-05 15:05:47 +0000 | [diff] [blame] | 120 | void arch_release_thread_info(struct thread_info *info) |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 121 | { |
| 122 | struct single_step_state *step_state = info->step_state; |
| 123 | |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 124 | #ifdef CONFIG_HARDWALL |
| 125 | /* |
| 126 | * We free a thread_info from the context of the task that has |
| 127 | * been scheduled next, so the original task is already dead. |
| 128 | * Calling deactivate here just frees up the data structures. |
| 129 | * If the task we're freeing held the last reference to a |
| 130 | * hardwall fd, it would have been released prior to this point |
Chris Metcalf | b8ace08 | 2012-03-30 16:01:48 -0400 | [diff] [blame] | 131 | * anyway via exit_files(), and the hardwall_task.info pointers |
| 132 | * would be NULL by now. |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 133 | */ |
Chris Metcalf | b8ace08 | 2012-03-30 16:01:48 -0400 | [diff] [blame] | 134 | hardwall_deactivate_all(info->task); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 135 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 136 | |
| 137 | if (step_state) { |
| 138 | |
| 139 | /* |
| 140 | * FIXME: we don't munmap step_state->buffer |
| 141 | * because the mm_struct for this process (info->task->mm) |
| 142 | * has already been zeroed in exit_mm(). Keeping a |
| 143 | * reference to it here seems like a bad move, so this |
| 144 | * means we can't munmap() the buffer, and therefore if we |
| 145 | * ptrace multiple threads in a process, we will slowly |
| 146 | * leak user memory. (Note that as soon as the last |
| 147 | * thread in a process dies, we will reclaim all user |
| 148 | * memory including single-step buffers in the usual way.) |
| 149 | * We should either assign a kernel VA to this buffer |
| 150 | * somehow, or we should associate the buffer(s) with the |
| 151 | * mm itself so we can clean them up that way. |
| 152 | */ |
| 153 | kfree(step_state); |
| 154 | } |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 155 | } |
| 156 | |
| 157 | static void save_arch_state(struct thread_struct *t); |
| 158 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 159 | int copy_thread(unsigned long clone_flags, unsigned long sp, |
Al Viro | afa86fc | 2012-10-22 22:51:14 -0400 | [diff] [blame^] | 160 | unsigned long arg, struct task_struct *p) |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 161 | { |
Al Viro | afa86fc | 2012-10-22 22:51:14 -0400 | [diff] [blame^] | 162 | struct pt_regs *childregs = task_pt_regs(p), *regs = current_pt_regs(); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 163 | unsigned long ksp; |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 164 | unsigned long *callee_regs; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 165 | |
| 166 | /* |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 167 | * Set up the stack and stack pointer appropriately for the |
| 168 | * new child to find itself woken up in __switch_to(). |
| 169 | * The callee-saved registers must be on the stack to be read; |
| 170 | * the new task will then jump to assembly support to handle |
| 171 | * calling schedule_tail(), etc., and (for userspace tasks) |
| 172 | * returning to the context set up in the pt_regs. |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 173 | */ |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 174 | ksp = (unsigned long) childregs; |
| 175 | ksp -= C_ABI_SAVE_AREA_SIZE; /* interrupt-entry save area */ |
| 176 | ((long *)ksp)[0] = ((long *)ksp)[1] = 0; |
| 177 | ksp -= CALLEE_SAVED_REGS_COUNT * sizeof(unsigned long); |
| 178 | callee_regs = (unsigned long *)ksp; |
| 179 | ksp -= C_ABI_SAVE_AREA_SIZE; /* __switch_to() save area */ |
| 180 | ((long *)ksp)[0] = ((long *)ksp)[1] = 0; |
| 181 | p->thread.ksp = ksp; |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 182 | |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 183 | /* Record the pid of the task that created this one. */ |
| 184 | p->thread.creator_pid = current->pid; |
| 185 | |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 186 | if (unlikely(p->flags & PF_KTHREAD)) { |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 187 | /* kernel thread */ |
| 188 | memset(childregs, 0, sizeof(struct pt_regs)); |
| 189 | memset(&callee_regs[2], 0, |
| 190 | (CALLEE_SAVED_REGS_COUNT - 2) * sizeof(unsigned long)); |
| 191 | callee_regs[0] = sp; /* r30 = function */ |
| 192 | callee_regs[1] = arg; /* r31 = arg */ |
| 193 | childregs->ex1 = PL_ICS_EX1(KERNEL_PL, 0); |
| 194 | p->thread.pc = (unsigned long) ret_from_kernel_thread; |
| 195 | return 0; |
| 196 | } |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 197 | |
| 198 | /* |
| 199 | * Start new thread in ret_from_fork so it schedules properly |
| 200 | * and then return from interrupt like the parent. |
| 201 | */ |
| 202 | p->thread.pc = (unsigned long) ret_from_fork; |
| 203 | |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 204 | /* |
| 205 | * Do not clone step state from the parent; each thread |
| 206 | * must make its own lazily. |
| 207 | */ |
| 208 | task_thread_info(p)->step_state = NULL; |
| 209 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 210 | /* |
| 211 | * Copy the registers onto the kernel stack so the |
| 212 | * return-from-interrupt code will reload it into registers. |
| 213 | */ |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 214 | *childregs = *current_pt_regs(); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 215 | childregs->regs[0] = 0; /* return value is zero */ |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 216 | if (sp) |
| 217 | childregs->sp = sp; /* override with new user stack pointer */ |
| 218 | memcpy(callee_regs, &childregs->regs[CALLEE_SAVED_FIRST_REG], |
Chris Metcalf | 0f8b983 | 2012-10-19 16:25:12 -0400 | [diff] [blame] | 219 | CALLEE_SAVED_REGS_COUNT * sizeof(unsigned long)); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 220 | |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 221 | /* Save user stack top pointer so we can ID the stack vm area later. */ |
| 222 | p->thread.usp0 = childregs->sp; |
| 223 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 224 | /* |
Chris Metcalf | bc4cf2b | 2010-12-14 15:57:49 -0500 | [diff] [blame] | 225 | * If CLONE_SETTLS is set, set "tp" in the new task to "r4", |
| 226 | * which is passed in as arg #5 to sys_clone(). |
| 227 | */ |
| 228 | if (clone_flags & CLONE_SETTLS) |
Al Viro | 008f179 | 2012-10-23 23:32:21 -0400 | [diff] [blame] | 229 | childregs->tp = childregs->regs[4]; |
Chris Metcalf | bc4cf2b | 2010-12-14 15:57:49 -0500 | [diff] [blame] | 230 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 231 | |
| 232 | #if CHIP_HAS_TILE_DMA() |
| 233 | /* |
| 234 | * No DMA in the new thread. We model this on the fact that |
| 235 | * fork() clears the pending signals, alarms, and aio for the child. |
| 236 | */ |
| 237 | memset(&p->thread.tile_dma_state, 0, sizeof(struct tile_dma_state)); |
| 238 | memset(&p->thread.dma_async_tlb, 0, sizeof(struct async_tlb)); |
| 239 | #endif |
| 240 | |
| 241 | #if CHIP_HAS_SN_PROC() |
| 242 | /* Likewise, the new thread is not running static processor code. */ |
| 243 | p->thread.sn_proc_running = 0; |
| 244 | memset(&p->thread.sn_async_tlb, 0, sizeof(struct async_tlb)); |
| 245 | #endif |
| 246 | |
| 247 | #if CHIP_HAS_PROC_STATUS_SPR() |
| 248 | /* New thread has its miscellaneous processor state bits clear. */ |
| 249 | p->thread.proc_status = 0; |
| 250 | #endif |
| 251 | |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 252 | #ifdef CONFIG_HARDWALL |
| 253 | /* New thread does not own any networks. */ |
Chris Metcalf | b8ace08 | 2012-03-30 16:01:48 -0400 | [diff] [blame] | 254 | memset(&p->thread.hardwall[0], 0, |
| 255 | sizeof(struct hardwall_task) * HARDWALL_TYPES); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 256 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 257 | |
| 258 | |
| 259 | /* |
| 260 | * Start the new thread with the current architecture state |
| 261 | * (user interrupt masks, etc.). |
| 262 | */ |
| 263 | save_arch_state(&p->thread); |
| 264 | |
| 265 | return 0; |
| 266 | } |
| 267 | |
| 268 | /* |
| 269 | * Return "current" if it looks plausible, or else a pointer to a dummy. |
| 270 | * This can be helpful if we are just trying to emit a clean panic. |
| 271 | */ |
| 272 | struct task_struct *validate_current(void) |
| 273 | { |
| 274 | static struct task_struct corrupt = { .comm = "<corrupt>" }; |
| 275 | struct task_struct *tsk = current; |
| 276 | if (unlikely((unsigned long)tsk < PAGE_OFFSET || |
Chris Metcalf | b287f69 | 2012-03-29 14:02:52 -0400 | [diff] [blame] | 277 | (high_memory && (void *)tsk > high_memory) || |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 278 | ((unsigned long)tsk & (__alignof__(*tsk) - 1)) != 0)) { |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 279 | pr_err("Corrupt 'current' %p (sp %#lx)\n", tsk, stack_pointer); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 280 | tsk = &corrupt; |
| 281 | } |
| 282 | return tsk; |
| 283 | } |
| 284 | |
| 285 | /* Take and return the pointer to the previous task, for schedule_tail(). */ |
| 286 | struct task_struct *sim_notify_fork(struct task_struct *prev) |
| 287 | { |
| 288 | struct task_struct *tsk = current; |
| 289 | __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_FORK_PARENT | |
| 290 | (tsk->thread.creator_pid << _SIM_CONTROL_OPERATOR_BITS)); |
| 291 | __insn_mtspr(SPR_SIM_CONTROL, SIM_CONTROL_OS_FORK | |
| 292 | (tsk->pid << _SIM_CONTROL_OPERATOR_BITS)); |
| 293 | return prev; |
| 294 | } |
| 295 | |
| 296 | int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs) |
| 297 | { |
| 298 | struct pt_regs *ptregs = task_pt_regs(tsk); |
| 299 | elf_core_copy_regs(regs, ptregs); |
| 300 | return 1; |
| 301 | } |
| 302 | |
| 303 | #if CHIP_HAS_TILE_DMA() |
| 304 | |
| 305 | /* Allow user processes to access the DMA SPRs */ |
| 306 | void grant_dma_mpls(void) |
| 307 | { |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 308 | #if CONFIG_KERNEL_PL == 2 |
| 309 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_1, 1); |
| 310 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_1, 1); |
| 311 | #else |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 312 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_0, 1); |
| 313 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_0, 1); |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 314 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 315 | } |
| 316 | |
| 317 | /* Forbid user processes from accessing the DMA SPRs */ |
| 318 | void restrict_dma_mpls(void) |
| 319 | { |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 320 | #if CONFIG_KERNEL_PL == 2 |
| 321 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_2, 1); |
| 322 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_2, 1); |
| 323 | #else |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 324 | __insn_mtspr(SPR_MPL_DMA_CPL_SET_1, 1); |
| 325 | __insn_mtspr(SPR_MPL_DMA_NOTIFY_SET_1, 1); |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 326 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 327 | } |
| 328 | |
| 329 | /* Pause the DMA engine, then save off its state registers. */ |
| 330 | static void save_tile_dma_state(struct tile_dma_state *dma) |
| 331 | { |
| 332 | unsigned long state = __insn_mfspr(SPR_DMA_USER_STATUS); |
| 333 | unsigned long post_suspend_state; |
| 334 | |
| 335 | /* If we're running, suspend the engine. */ |
| 336 | if ((state & DMA_STATUS_MASK) == SPR_DMA_STATUS__RUNNING_MASK) |
| 337 | __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK); |
| 338 | |
| 339 | /* |
| 340 | * Wait for the engine to idle, then save regs. Note that we |
| 341 | * want to record the "running" bit from before suspension, |
| 342 | * and the "done" bit from after, so that we can properly |
| 343 | * distinguish a case where the user suspended the engine from |
| 344 | * the case where the kernel suspended as part of the context |
| 345 | * swap. |
| 346 | */ |
| 347 | do { |
| 348 | post_suspend_state = __insn_mfspr(SPR_DMA_USER_STATUS); |
| 349 | } while (post_suspend_state & SPR_DMA_STATUS__BUSY_MASK); |
| 350 | |
| 351 | dma->src = __insn_mfspr(SPR_DMA_SRC_ADDR); |
| 352 | dma->src_chunk = __insn_mfspr(SPR_DMA_SRC_CHUNK_ADDR); |
| 353 | dma->dest = __insn_mfspr(SPR_DMA_DST_ADDR); |
| 354 | dma->dest_chunk = __insn_mfspr(SPR_DMA_DST_CHUNK_ADDR); |
| 355 | dma->strides = __insn_mfspr(SPR_DMA_STRIDE); |
| 356 | dma->chunk_size = __insn_mfspr(SPR_DMA_CHUNK_SIZE); |
| 357 | dma->byte = __insn_mfspr(SPR_DMA_BYTE); |
| 358 | dma->status = (state & SPR_DMA_STATUS__RUNNING_MASK) | |
| 359 | (post_suspend_state & SPR_DMA_STATUS__DONE_MASK); |
| 360 | } |
| 361 | |
| 362 | /* Restart a DMA that was running before we were context-switched out. */ |
| 363 | static void restore_tile_dma_state(struct thread_struct *t) |
| 364 | { |
| 365 | const struct tile_dma_state *dma = &t->tile_dma_state; |
| 366 | |
| 367 | /* |
| 368 | * The only way to restore the done bit is to run a zero |
| 369 | * length transaction. |
| 370 | */ |
| 371 | if ((dma->status & SPR_DMA_STATUS__DONE_MASK) && |
| 372 | !(__insn_mfspr(SPR_DMA_USER_STATUS) & SPR_DMA_STATUS__DONE_MASK)) { |
| 373 | __insn_mtspr(SPR_DMA_BYTE, 0); |
| 374 | __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); |
| 375 | while (__insn_mfspr(SPR_DMA_USER_STATUS) & |
| 376 | SPR_DMA_STATUS__BUSY_MASK) |
| 377 | ; |
| 378 | } |
| 379 | |
| 380 | __insn_mtspr(SPR_DMA_SRC_ADDR, dma->src); |
| 381 | __insn_mtspr(SPR_DMA_SRC_CHUNK_ADDR, dma->src_chunk); |
| 382 | __insn_mtspr(SPR_DMA_DST_ADDR, dma->dest); |
| 383 | __insn_mtspr(SPR_DMA_DST_CHUNK_ADDR, dma->dest_chunk); |
| 384 | __insn_mtspr(SPR_DMA_STRIDE, dma->strides); |
| 385 | __insn_mtspr(SPR_DMA_CHUNK_SIZE, dma->chunk_size); |
| 386 | __insn_mtspr(SPR_DMA_BYTE, dma->byte); |
| 387 | |
| 388 | /* |
| 389 | * Restart the engine if we were running and not done. |
| 390 | * Clear a pending async DMA fault that we were waiting on return |
| 391 | * to user space to execute, since we expect the DMA engine |
| 392 | * to regenerate those faults for us now. Note that we don't |
| 393 | * try to clear the TIF_ASYNC_TLB flag, since it's relatively |
| 394 | * harmless if set, and it covers both DMA and the SN processor. |
| 395 | */ |
| 396 | if ((dma->status & DMA_STATUS_MASK) == SPR_DMA_STATUS__RUNNING_MASK) { |
| 397 | t->dma_async_tlb.fault_num = 0; |
| 398 | __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | #endif |
| 403 | |
| 404 | static void save_arch_state(struct thread_struct *t) |
| 405 | { |
| 406 | #if CHIP_HAS_SPLIT_INTR_MASK() |
| 407 | t->interrupt_mask = __insn_mfspr(SPR_INTERRUPT_MASK_0_0) | |
| 408 | ((u64)__insn_mfspr(SPR_INTERRUPT_MASK_0_1) << 32); |
| 409 | #else |
| 410 | t->interrupt_mask = __insn_mfspr(SPR_INTERRUPT_MASK_0); |
| 411 | #endif |
| 412 | t->ex_context[0] = __insn_mfspr(SPR_EX_CONTEXT_0_0); |
| 413 | t->ex_context[1] = __insn_mfspr(SPR_EX_CONTEXT_0_1); |
| 414 | t->system_save[0] = __insn_mfspr(SPR_SYSTEM_SAVE_0_0); |
| 415 | t->system_save[1] = __insn_mfspr(SPR_SYSTEM_SAVE_0_1); |
| 416 | t->system_save[2] = __insn_mfspr(SPR_SYSTEM_SAVE_0_2); |
| 417 | t->system_save[3] = __insn_mfspr(SPR_SYSTEM_SAVE_0_3); |
| 418 | t->intctrl_0 = __insn_mfspr(SPR_INTCTRL_0_STATUS); |
| 419 | #if CHIP_HAS_PROC_STATUS_SPR() |
| 420 | t->proc_status = __insn_mfspr(SPR_PROC_STATUS); |
| 421 | #endif |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 422 | #if !CHIP_HAS_FIXED_INTVEC_BASE() |
| 423 | t->interrupt_vector_base = __insn_mfspr(SPR_INTERRUPT_VECTOR_BASE_0); |
| 424 | #endif |
| 425 | #if CHIP_HAS_TILE_RTF_HWM() |
| 426 | t->tile_rtf_hwm = __insn_mfspr(SPR_TILE_RTF_HWM); |
| 427 | #endif |
| 428 | #if CHIP_HAS_DSTREAM_PF() |
| 429 | t->dstream_pf = __insn_mfspr(SPR_DSTREAM_PF); |
| 430 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 431 | } |
| 432 | |
| 433 | static void restore_arch_state(const struct thread_struct *t) |
| 434 | { |
| 435 | #if CHIP_HAS_SPLIT_INTR_MASK() |
| 436 | __insn_mtspr(SPR_INTERRUPT_MASK_0_0, (u32) t->interrupt_mask); |
| 437 | __insn_mtspr(SPR_INTERRUPT_MASK_0_1, t->interrupt_mask >> 32); |
| 438 | #else |
| 439 | __insn_mtspr(SPR_INTERRUPT_MASK_0, t->interrupt_mask); |
| 440 | #endif |
| 441 | __insn_mtspr(SPR_EX_CONTEXT_0_0, t->ex_context[0]); |
| 442 | __insn_mtspr(SPR_EX_CONTEXT_0_1, t->ex_context[1]); |
| 443 | __insn_mtspr(SPR_SYSTEM_SAVE_0_0, t->system_save[0]); |
| 444 | __insn_mtspr(SPR_SYSTEM_SAVE_0_1, t->system_save[1]); |
| 445 | __insn_mtspr(SPR_SYSTEM_SAVE_0_2, t->system_save[2]); |
| 446 | __insn_mtspr(SPR_SYSTEM_SAVE_0_3, t->system_save[3]); |
| 447 | __insn_mtspr(SPR_INTCTRL_0_STATUS, t->intctrl_0); |
| 448 | #if CHIP_HAS_PROC_STATUS_SPR() |
| 449 | __insn_mtspr(SPR_PROC_STATUS, t->proc_status); |
| 450 | #endif |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 451 | #if !CHIP_HAS_FIXED_INTVEC_BASE() |
| 452 | __insn_mtspr(SPR_INTERRUPT_VECTOR_BASE_0, t->interrupt_vector_base); |
| 453 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 454 | #if CHIP_HAS_TILE_RTF_HWM() |
Chris Metcalf | a802fc6 | 2010-09-15 11:16:10 -0400 | [diff] [blame] | 455 | __insn_mtspr(SPR_TILE_RTF_HWM, t->tile_rtf_hwm); |
| 456 | #endif |
| 457 | #if CHIP_HAS_DSTREAM_PF() |
| 458 | __insn_mtspr(SPR_DSTREAM_PF, t->dstream_pf); |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 459 | #endif |
| 460 | } |
| 461 | |
| 462 | |
| 463 | void _prepare_arch_switch(struct task_struct *next) |
| 464 | { |
| 465 | #if CHIP_HAS_SN_PROC() |
| 466 | int snctl; |
| 467 | #endif |
| 468 | #if CHIP_HAS_TILE_DMA() |
| 469 | struct tile_dma_state *dma = ¤t->thread.tile_dma_state; |
| 470 | if (dma->enabled) |
| 471 | save_tile_dma_state(dma); |
| 472 | #endif |
| 473 | #if CHIP_HAS_SN_PROC() |
| 474 | /* |
| 475 | * Suspend the static network processor if it was running. |
| 476 | * We do not suspend the fabric itself, just like we don't |
| 477 | * try to suspend the UDN. |
| 478 | */ |
| 479 | snctl = __insn_mfspr(SPR_SNCTL); |
| 480 | current->thread.sn_proc_running = |
| 481 | (snctl & SPR_SNCTL__FRZPROC_MASK) == 0; |
| 482 | if (current->thread.sn_proc_running) |
| 483 | __insn_mtspr(SPR_SNCTL, snctl | SPR_SNCTL__FRZPROC_MASK); |
| 484 | #endif |
| 485 | } |
| 486 | |
| 487 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 488 | struct task_struct *__sched _switch_to(struct task_struct *prev, |
| 489 | struct task_struct *next) |
| 490 | { |
| 491 | /* DMA state is already saved; save off other arch state. */ |
| 492 | save_arch_state(&prev->thread); |
| 493 | |
| 494 | #if CHIP_HAS_TILE_DMA() |
| 495 | /* |
| 496 | * Restore DMA in new task if desired. |
| 497 | * Note that it is only safe to restart here since interrupts |
| 498 | * are disabled, so we can't take any DMATLB miss or access |
| 499 | * interrupts before we have finished switching stacks. |
| 500 | */ |
| 501 | if (next->thread.tile_dma_state.enabled) { |
| 502 | restore_tile_dma_state(&next->thread); |
| 503 | grant_dma_mpls(); |
| 504 | } else { |
| 505 | restrict_dma_mpls(); |
| 506 | } |
| 507 | #endif |
| 508 | |
| 509 | /* Restore other arch state. */ |
| 510 | restore_arch_state(&next->thread); |
| 511 | |
| 512 | #if CHIP_HAS_SN_PROC() |
| 513 | /* |
| 514 | * Restart static network processor in the new process |
| 515 | * if it was running before. |
| 516 | */ |
| 517 | if (next->thread.sn_proc_running) { |
| 518 | int snctl = __insn_mfspr(SPR_SNCTL); |
| 519 | __insn_mtspr(SPR_SNCTL, snctl & ~SPR_SNCTL__FRZPROC_MASK); |
| 520 | } |
| 521 | #endif |
| 522 | |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 523 | #ifdef CONFIG_HARDWALL |
| 524 | /* Enable or disable access to the network registers appropriately. */ |
Chris Metcalf | b8ace08 | 2012-03-30 16:01:48 -0400 | [diff] [blame] | 525 | hardwall_switch_tasks(prev, next); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 526 | #endif |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 527 | |
| 528 | /* |
| 529 | * Switch kernel SP, PC, and callee-saved registers. |
| 530 | * In the context of the new task, return the old task pointer |
| 531 | * (i.e. the task that actually called __switch_to). |
Chris Metcalf | a78c942 | 2010-10-14 16:23:03 -0400 | [diff] [blame] | 532 | * Pass the value to use for SYSTEM_SAVE_K_0 when we reset our sp. |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 533 | */ |
| 534 | return __switch_to(prev, next, next_current_ksp0(next)); |
| 535 | } |
| 536 | |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 537 | /* |
| 538 | * This routine is called on return from interrupt if any of the |
| 539 | * TIF_WORK_MASK flags are set in thread_info->flags. It is |
| 540 | * entered with interrupts disabled so we don't miss an event |
| 541 | * that modified the thread_info flags. If any flag is set, we |
| 542 | * handle it and return, and the calling assembly code will |
| 543 | * re-disable interrupts, reload the thread flags, and call back |
| 544 | * if more flags need to be handled. |
| 545 | * |
| 546 | * We return whether we need to check the thread_info flags again |
| 547 | * or not. Note that we don't clear TIF_SINGLESTEP here, so it's |
| 548 | * important that it be tested last, and then claim that we don't |
| 549 | * need to recheck the flags. |
| 550 | */ |
| 551 | int do_work_pending(struct pt_regs *regs, u32 thread_info_flags) |
| 552 | { |
Chris Metcalf | fc327e2 | 2012-04-28 18:51:43 -0400 | [diff] [blame] | 553 | /* If we enter in kernel mode, do nothing and exit the caller loop. */ |
| 554 | if (!user_mode(regs)) |
| 555 | return 0; |
| 556 | |
Chris Metcalf | c19c6c9 | 2012-10-12 15:38:54 -0400 | [diff] [blame] | 557 | /* Enable interrupts; they are disabled again on return to caller. */ |
| 558 | local_irq_enable(); |
| 559 | |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 560 | if (thread_info_flags & _TIF_NEED_RESCHED) { |
| 561 | schedule(); |
| 562 | return 1; |
| 563 | } |
| 564 | #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC() |
| 565 | if (thread_info_flags & _TIF_ASYNC_TLB) { |
| 566 | do_async_page_fault(regs); |
| 567 | return 1; |
| 568 | } |
| 569 | #endif |
| 570 | if (thread_info_flags & _TIF_SIGPENDING) { |
| 571 | do_signal(regs); |
| 572 | return 1; |
| 573 | } |
| 574 | if (thread_info_flags & _TIF_NOTIFY_RESUME) { |
| 575 | clear_thread_flag(TIF_NOTIFY_RESUME); |
| 576 | tracehook_notify_resume(regs); |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 577 | return 1; |
| 578 | } |
| 579 | if (thread_info_flags & _TIF_SINGLESTEP) { |
Chris Metcalf | fc327e2 | 2012-04-28 18:51:43 -0400 | [diff] [blame] | 580 | single_step_once(regs); |
Chris Metcalf | 313ce67 | 2011-05-02 14:50:06 -0400 | [diff] [blame] | 581 | return 0; |
| 582 | } |
| 583 | panic("work_pending: bad flags %#x\n", thread_info_flags); |
| 584 | } |
| 585 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 586 | unsigned long get_wchan(struct task_struct *p) |
| 587 | { |
| 588 | struct KBacktraceIterator kbt; |
| 589 | |
| 590 | if (!p || p == current || p->state == TASK_RUNNING) |
| 591 | return 0; |
| 592 | |
| 593 | for (KBacktraceIterator_init(&kbt, p, NULL); |
| 594 | !KBacktraceIterator_end(&kbt); |
| 595 | KBacktraceIterator_next(&kbt)) { |
| 596 | if (!in_sched_functions(kbt.it.pc)) |
| 597 | return kbt.it.pc; |
| 598 | } |
| 599 | |
| 600 | return 0; |
| 601 | } |
| 602 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 603 | /* Flush thread state. */ |
| 604 | void flush_thread(void) |
| 605 | { |
| 606 | /* Nothing */ |
| 607 | } |
| 608 | |
| 609 | /* |
| 610 | * Free current thread data structures etc.. |
| 611 | */ |
| 612 | void exit_thread(void) |
| 613 | { |
| 614 | /* Nothing */ |
| 615 | } |
| 616 | |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 617 | void show_regs(struct pt_regs *regs) |
| 618 | { |
| 619 | struct task_struct *tsk = validate_current(); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 620 | int i; |
| 621 | |
| 622 | pr_err("\n"); |
| 623 | pr_err(" Pid: %d, comm: %20s, CPU: %d\n", |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 624 | tsk->pid, tsk->comm, smp_processor_id()); |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 625 | #ifdef __tilegx__ |
| 626 | for (i = 0; i < 51; i += 3) |
| 627 | pr_err(" r%-2d: "REGFMT" r%-2d: "REGFMT" r%-2d: "REGFMT"\n", |
| 628 | i, regs->regs[i], i+1, regs->regs[i+1], |
| 629 | i+2, regs->regs[i+2]); |
| 630 | pr_err(" r51: "REGFMT" r52: "REGFMT" tp : "REGFMT"\n", |
| 631 | regs->regs[51], regs->regs[52], regs->tp); |
| 632 | pr_err(" sp : "REGFMT" lr : "REGFMT"\n", regs->sp, regs->lr); |
| 633 | #else |
Chris Metcalf | 7040dea | 2010-09-15 11:17:05 -0400 | [diff] [blame] | 634 | for (i = 0; i < 52; i += 4) |
Chris Metcalf | 0707ad3 | 2010-06-25 17:04:17 -0400 | [diff] [blame] | 635 | pr_err(" r%-2d: "REGFMT" r%-2d: "REGFMT |
| 636 | " r%-2d: "REGFMT" r%-2d: "REGFMT"\n", |
| 637 | i, regs->regs[i], i+1, regs->regs[i+1], |
| 638 | i+2, regs->regs[i+2], i+3, regs->regs[i+3]); |
| 639 | pr_err(" r52: "REGFMT" tp : "REGFMT" sp : "REGFMT" lr : "REGFMT"\n", |
| 640 | regs->regs[52], regs->tp, regs->sp, regs->lr); |
| 641 | #endif |
| 642 | pr_err(" pc : "REGFMT" ex1: %ld faultnum: %ld\n", |
Chris Metcalf | 867e359 | 2010-05-28 23:09:12 -0400 | [diff] [blame] | 643 | regs->pc, regs->ex1, regs->faultnum); |
| 644 | |
| 645 | dump_stack_regs(regs); |
| 646 | } |