blob: 675a05012449d32ccaaa4abf0a0fb4abd095ed84 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1994 Linus Torvalds
3 *
4 * Pentium III FXSR, SSE support
5 * General FPU state handling cleanups
6 * Gareth Hughes <gareth@valinux.com>, May 2000
7 */
Alexey Dobriyan129f6942005-06-23 00:08:33 -07008#include <linux/module.h>
Roland McGrath44210112008-01-30 13:31:50 +01009#include <linux/regset.h>
Ingo Molnarf6689642008-03-05 15:37:32 +010010#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090011#include <linux/slab.h>
Ingo Molnarf6689642008-03-05 15:37:32 +010012
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <asm/sigcontext.h>
Ingo Molnarf6689642008-03-05 15:37:32 +010014#include <asm/processor.h>
15#include <asm/math_emu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <asm/uaccess.h>
Ingo Molnarf6689642008-03-05 15:37:32 +010017#include <asm/ptrace.h>
18#include <asm/i387.h>
Linus Torvalds1361b832012-02-21 13:19:22 -080019#include <asm/fpu-internal.h>
Ingo Molnarf6689642008-03-05 15:37:32 +010020#include <asm/user.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
Linus Torvalds8546c002012-02-21 10:25:45 -080022/*
23 * Were we in an interrupt that interrupted kernel mode?
24 *
Suresh Siddha5d2bd702012-09-06 14:58:52 -070025 * For now, with eagerfpu we will return interrupted kernel FPU
26 * state as not-idle. TBD: Ideally we can change the return value
Suresh Siddha304bced2012-08-24 14:13:02 -070027 * to something like __thread_has_fpu(current). But we need to
28 * be careful of doing __thread_clear_has_fpu() before saving
29 * the FPU etc for supporting nested uses etc. For now, take
30 * the simple route!
31 *
32 * On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
Linus Torvalds8546c002012-02-21 10:25:45 -080033 * pair does nothing at all: the thread must not have fpu (so
34 * that we don't try to save the FPU state), and TS must
35 * be set (so that the clts/stts pair does nothing that is
36 * visible in the interrupted kernel thread).
37 */
38static inline bool interrupted_kernel_fpu_idle(void)
39{
Suresh Siddha5d2bd702012-09-06 14:58:52 -070040 if (use_eager_fpu())
Suresh Siddha304bced2012-08-24 14:13:02 -070041 return 0;
42
Linus Torvalds8546c002012-02-21 10:25:45 -080043 return !__thread_has_fpu(current) &&
44 (read_cr0() & X86_CR0_TS);
45}
46
47/*
48 * Were we in user mode (or vm86 mode) when we were
49 * interrupted?
50 *
51 * Doing kernel_fpu_begin/end() is ok if we are running
52 * in an interrupt context from user mode - we'll just
53 * save the FPU state as required.
54 */
55static inline bool interrupted_user_mode(void)
56{
57 struct pt_regs *regs = get_irq_regs();
58 return regs && user_mode_vm(regs);
59}
60
61/*
62 * Can we use the FPU in kernel mode with the
63 * whole "kernel_fpu_begin/end()" sequence?
64 *
65 * It's always ok in process context (ie "not interrupt")
66 * but it is sometimes ok even from an irq.
67 */
68bool irq_fpu_usable(void)
69{
70 return !in_interrupt() ||
71 interrupted_user_mode() ||
72 interrupted_kernel_fpu_idle();
73}
74EXPORT_SYMBOL(irq_fpu_usable);
75
Suresh Siddhab1a74bf2012-09-20 11:01:49 -070076void __kernel_fpu_begin(void)
Linus Torvalds8546c002012-02-21 10:25:45 -080077{
78 struct task_struct *me = current;
79
Linus Torvalds8546c002012-02-21 10:25:45 -080080 if (__thread_has_fpu(me)) {
81 __save_init_fpu(me);
82 __thread_clear_has_fpu(me);
Suresh Siddhab1a74bf2012-09-20 11:01:49 -070083 /* We do 'stts()' in __kernel_fpu_end() */
Suresh Siddha5d2bd702012-09-06 14:58:52 -070084 } else if (!use_eager_fpu()) {
Alex Shic6ae41e2012-05-11 15:35:27 +080085 this_cpu_write(fpu_owner_task, NULL);
Linus Torvalds8546c002012-02-21 10:25:45 -080086 clts();
87 }
88}
Suresh Siddhab1a74bf2012-09-20 11:01:49 -070089EXPORT_SYMBOL(__kernel_fpu_begin);
Linus Torvalds8546c002012-02-21 10:25:45 -080090
Suresh Siddhab1a74bf2012-09-20 11:01:49 -070091void __kernel_fpu_end(void)
Linus Torvalds8546c002012-02-21 10:25:45 -080092{
Suresh Siddha5d2bd702012-09-06 14:58:52 -070093 if (use_eager_fpu())
Suresh Siddha304bced2012-08-24 14:13:02 -070094 math_state_restore();
95 else
96 stts();
Linus Torvalds8546c002012-02-21 10:25:45 -080097}
Suresh Siddhab1a74bf2012-09-20 11:01:49 -070098EXPORT_SYMBOL(__kernel_fpu_end);
Linus Torvalds8546c002012-02-21 10:25:45 -080099
100void unlazy_fpu(struct task_struct *tsk)
101{
102 preempt_disable();
103 if (__thread_has_fpu(tsk)) {
104 __save_init_fpu(tsk);
105 __thread_fpu_end(tsk);
106 } else
107 tsk->fpu_counter = 0;
108 preempt_enable();
109}
110EXPORT_SYMBOL(unlazy_fpu);
111
Suresh Siddha72a671c2012-07-24 16:05:29 -0700112unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
Suresh Siddha61c46282008-03-10 15:28:04 -0700113unsigned int xstate_size;
Xiaotian Fengf45755b2010-08-13 15:19:11 +0800114EXPORT_SYMBOL_GPL(xstate_size);
Suresh Siddha61c46282008-03-10 15:28:04 -0700115static struct i387_fxsave_struct fx_scratch __cpuinitdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116
Linus Torvalds1361b832012-02-21 13:19:22 -0800117static void __cpuinit mxcsr_feature_mask_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118{
119 unsigned long mask = 0;
Ingo Molnarf6689642008-03-05 15:37:32 +0100120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 if (cpu_has_fxsr) {
Suresh Siddha61c46282008-03-10 15:28:04 -0700122 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
123 asm volatile("fxsave %0" : : "m" (fx_scratch));
124 mask = fx_scratch.mxcsr_mask;
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100125 if (mask == 0)
126 mask = 0x0000ffbf;
127 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 mxcsr_feature_mask &= mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129}
130
Robert Richter0e49bf62010-07-21 19:03:52 +0200131static void __cpuinit init_thread_xstate(void)
Suresh Siddha61c46282008-03-10 15:28:04 -0700132{
Robert Richter0e49bf62010-07-21 19:03:52 +0200133 /*
134 * Note that xstate_size might be overwriten later during
135 * xsave_init().
136 */
137
Suresh Siddhae8a496a2008-05-23 16:26:37 -0700138 if (!HAVE_HWFP) {
Robert Richter1f999ab2010-07-21 19:03:57 +0200139 /*
140 * Disable xsave as we do not support it if i387
141 * emulation is enabled.
142 */
143 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
144 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
Suresh Siddhae8a496a2008-05-23 16:26:37 -0700145 xstate_size = sizeof(struct i387_soft_struct);
146 return;
147 }
148
Suresh Siddha61c46282008-03-10 15:28:04 -0700149 if (cpu_has_fxsr)
150 xstate_size = sizeof(struct i387_fxsave_struct);
Suresh Siddha61c46282008-03-10 15:28:04 -0700151 else
152 xstate_size = sizeof(struct i387_fsave_struct);
Suresh Siddha61c46282008-03-10 15:28:04 -0700153}
154
Roland McGrath44210112008-01-30 13:31:50 +0100155/*
156 * Called at bootup to set up the initial FPU state that is later cloned
157 * into all processes.
158 */
Robert Richter0e49bf62010-07-21 19:03:52 +0200159
Roland McGrath44210112008-01-30 13:31:50 +0100160void __cpuinit fpu_init(void)
161{
Brian Gerst6ac8bac2010-09-03 21:17:09 -0400162 unsigned long cr0;
163 unsigned long cr4_mask = 0;
Ingo Molnarf6689642008-03-05 15:37:32 +0100164
Brian Gerst6ac8bac2010-09-03 21:17:09 -0400165 if (cpu_has_fxsr)
166 cr4_mask |= X86_CR4_OSFXSR;
167 if (cpu_has_xmm)
168 cr4_mask |= X86_CR4_OSXMMEXCPT;
169 if (cr4_mask)
170 set_in_cr4(cr4_mask);
Roland McGrath44210112008-01-30 13:31:50 +0100171
Brian Gerst6ac8bac2010-09-03 21:17:09 -0400172 cr0 = read_cr0();
173 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
174 if (!HAVE_HWFP)
175 cr0 |= X86_CR0_EM;
176 write_cr0(cr0);
Roland McGrath44210112008-01-30 13:31:50 +0100177
Suresh Siddhadc1e35c2008-07-29 10:29:19 -0700178 if (!smp_processor_id())
179 init_thread_xstate();
Suresh Siddhadc1e35c2008-07-29 10:29:19 -0700180
Roland McGrath44210112008-01-30 13:31:50 +0100181 mxcsr_feature_mask_init();
Suresh Siddha5d2bd702012-09-06 14:58:52 -0700182 xsave_init();
183 eager_fpu_init();
Roland McGrath44210112008-01-30 13:31:50 +0100184}
Robert Richter0e49bf62010-07-21 19:03:52 +0200185
Sheng Yang5ee481d2010-05-17 17:22:23 +0800186void fpu_finit(struct fpu *fpu)
Avi Kivity86603282010-05-06 11:45:46 +0300187{
Avi Kivity86603282010-05-06 11:45:46 +0300188 if (!HAVE_HWFP) {
189 finit_soft_fpu(&fpu->state->soft);
190 return;
191 }
Avi Kivity86603282010-05-06 11:45:46 +0300192
193 if (cpu_has_fxsr) {
Suresh Siddha5d2bd702012-09-06 14:58:52 -0700194 fx_finit(&fpu->state->fxsave);
Avi Kivity86603282010-05-06 11:45:46 +0300195 } else {
196 struct i387_fsave_struct *fp = &fpu->state->fsave;
197 memset(fp, 0, xstate_size);
198 fp->cwd = 0xffff037fu;
199 fp->swd = 0xffff0000u;
200 fp->twd = 0xffffffffu;
201 fp->fos = 0xffff0000u;
202 }
203}
Sheng Yang5ee481d2010-05-17 17:22:23 +0800204EXPORT_SYMBOL_GPL(fpu_finit);
Avi Kivity86603282010-05-06 11:45:46 +0300205
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206/*
207 * The _current_ task is using the FPU for the first time
208 * so initialize it and set the mxcsr to its default
209 * value at reset if we support XMM instructions and then
Lucas De Marchi0d2eb442011-03-17 16:24:16 -0300210 * remember the current task has used the FPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 */
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700212int init_fpu(struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213{
Avi Kivity86603282010-05-06 11:45:46 +0300214 int ret;
215
Roland McGrath44210112008-01-30 13:31:50 +0100216 if (tsk_used_math(tsk)) {
Suresh Siddhae8a496a2008-05-23 16:26:37 -0700217 if (HAVE_HWFP && tsk == current)
Roland McGrath44210112008-01-30 13:31:50 +0100218 unlazy_fpu(tsk);
Oleg Nesterov089f9fb2012-04-16 22:48:15 +0200219 tsk->thread.fpu.last_cpu = ~0;
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700220 return 0;
221 }
222
223 /*
224 * Memory allocation at the first usage of the FPU and other state.
225 */
Avi Kivity86603282010-05-06 11:45:46 +0300226 ret = fpu_alloc(&tsk->thread.fpu);
227 if (ret)
228 return ret;
Roland McGrath44210112008-01-30 13:31:50 +0100229
Avi Kivity86603282010-05-06 11:45:46 +0300230 fpu_finit(&tsk->thread.fpu);
Suresh Siddhae8a496a2008-05-23 16:26:37 -0700231
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 set_stopped_child_used_math(tsk);
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700233 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234}
Avi Kivitye5c30142011-01-11 12:15:54 +0200235EXPORT_SYMBOL_GPL(init_fpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800237/*
238 * The xstateregs_active() routine is the same as the fpregs_active() routine,
239 * as the "regset->n" for the xstate regset will be updated based on the feature
240 * capabilites supported by the xsave.
241 */
Roland McGrath44210112008-01-30 13:31:50 +0100242int fpregs_active(struct task_struct *target, const struct user_regset *regset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243{
Roland McGrath44210112008-01-30 13:31:50 +0100244 return tsk_used_math(target) ? regset->n : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245}
Roland McGrath44210112008-01-30 13:31:50 +0100246
247int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
248{
249 return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
250}
251
252int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
253 unsigned int pos, unsigned int count,
254 void *kbuf, void __user *ubuf)
255{
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700256 int ret;
257
Roland McGrath44210112008-01-30 13:31:50 +0100258 if (!cpu_has_fxsr)
259 return -ENODEV;
260
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700261 ret = init_fpu(target);
262 if (ret)
263 return ret;
Roland McGrath44210112008-01-30 13:31:50 +0100264
Suresh Siddha29104e12010-07-19 16:05:49 -0700265 sanitize_i387_state(target);
266
Roland McGrath44210112008-01-30 13:31:50 +0100267 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
Avi Kivity86603282010-05-06 11:45:46 +0300268 &target->thread.fpu.state->fxsave, 0, -1);
Roland McGrath44210112008-01-30 13:31:50 +0100269}
270
271int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
272 unsigned int pos, unsigned int count,
273 const void *kbuf, const void __user *ubuf)
274{
275 int ret;
276
277 if (!cpu_has_fxsr)
278 return -ENODEV;
279
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700280 ret = init_fpu(target);
281 if (ret)
282 return ret;
283
Suresh Siddha29104e12010-07-19 16:05:49 -0700284 sanitize_i387_state(target);
285
Roland McGrath44210112008-01-30 13:31:50 +0100286 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
Avi Kivity86603282010-05-06 11:45:46 +0300287 &target->thread.fpu.state->fxsave, 0, -1);
Roland McGrath44210112008-01-30 13:31:50 +0100288
289 /*
290 * mxcsr reserved bits must be masked to zero for security reasons.
291 */
Avi Kivity86603282010-05-06 11:45:46 +0300292 target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
Roland McGrath44210112008-01-30 13:31:50 +0100293
Suresh Siddha42deec62008-07-29 10:29:26 -0700294 /*
295 * update the header bits in the xsave header, indicating the
296 * presence of FP and SSE state.
297 */
298 if (cpu_has_xsave)
Avi Kivity86603282010-05-06 11:45:46 +0300299 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
Suresh Siddha42deec62008-07-29 10:29:26 -0700300
Roland McGrath44210112008-01-30 13:31:50 +0100301 return ret;
302}
303
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800304int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
305 unsigned int pos, unsigned int count,
306 void *kbuf, void __user *ubuf)
307{
308 int ret;
309
310 if (!cpu_has_xsave)
311 return -ENODEV;
312
313 ret = init_fpu(target);
314 if (ret)
315 return ret;
316
317 /*
Suresh Siddhaff7fbc72010-02-22 14:51:33 -0800318 * Copy the 48bytes defined by the software first into the xstate
319 * memory layout in the thread struct, so that we can copy the entire
320 * xstateregs to the user using one user_regset_copyout().
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800321 */
Avi Kivity86603282010-05-06 11:45:46 +0300322 memcpy(&target->thread.fpu.state->fxsave.sw_reserved,
Suresh Siddhaff7fbc72010-02-22 14:51:33 -0800323 xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800324
325 /*
Suresh Siddhaff7fbc72010-02-22 14:51:33 -0800326 * Copy the xstate memory layout.
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800327 */
328 ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
Avi Kivity86603282010-05-06 11:45:46 +0300329 &target->thread.fpu.state->xsave, 0, -1);
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800330 return ret;
331}
332
333int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
334 unsigned int pos, unsigned int count,
335 const void *kbuf, const void __user *ubuf)
336{
337 int ret;
338 struct xsave_hdr_struct *xsave_hdr;
339
340 if (!cpu_has_xsave)
341 return -ENODEV;
342
343 ret = init_fpu(target);
344 if (ret)
345 return ret;
346
347 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
Avi Kivity86603282010-05-06 11:45:46 +0300348 &target->thread.fpu.state->xsave, 0, -1);
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800349
350 /*
351 * mxcsr reserved bits must be masked to zero for security reasons.
352 */
Avi Kivity86603282010-05-06 11:45:46 +0300353 target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800354
Avi Kivity86603282010-05-06 11:45:46 +0300355 xsave_hdr = &target->thread.fpu.state->xsave.xsave_hdr;
Suresh Siddha5b3efd52010-02-11 11:50:59 -0800356
357 xsave_hdr->xstate_bv &= pcntxt_mask;
358 /*
359 * These bits must be zero.
360 */
361 xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
362
363 return ret;
364}
365
Roland McGrath44210112008-01-30 13:31:50 +0100366#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368/*
369 * FPU tag word conversions.
370 */
371
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100372static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 unsigned int tmp; /* to avoid 16 bit prefixes in the code */
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100375
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 /* Transform each pair of bits into 01 (valid) or 00 (empty) */
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100377 tmp = ~twd;
Roland McGrath44210112008-01-30 13:31:50 +0100378 tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100379 /* and move the valid bits to the lower byte. */
380 tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
381 tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
382 tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
Ingo Molnarf6689642008-03-05 15:37:32 +0100383
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100384 return tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
Phil Carmody497888c2011-07-14 15:07:13 +0300387#define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16)
Roland McGrath44210112008-01-30 13:31:50 +0100388#define FP_EXP_TAG_VALID 0
389#define FP_EXP_TAG_ZERO 1
390#define FP_EXP_TAG_SPECIAL 2
391#define FP_EXP_TAG_EMPTY 3
392
393static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394{
Roland McGrath44210112008-01-30 13:31:50 +0100395 struct _fpxreg *st;
396 u32 tos = (fxsave->swd >> 11) & 7;
397 u32 twd = (unsigned long) fxsave->twd;
398 u32 tag;
399 u32 ret = 0xffff0000u;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 int i;
401
Roland McGrath44210112008-01-30 13:31:50 +0100402 for (i = 0; i < 8; i++, twd >>= 1) {
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100403 if (twd & 0x1) {
404 st = FPREG_ADDR(fxsave, (i - tos) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100406 switch (st->exponent & 0x7fff) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 case 0x7fff:
Roland McGrath44210112008-01-30 13:31:50 +0100408 tag = FP_EXP_TAG_SPECIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 break;
410 case 0x0000:
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100411 if (!st->significand[0] &&
412 !st->significand[1] &&
413 !st->significand[2] &&
Roland McGrath44210112008-01-30 13:31:50 +0100414 !st->significand[3])
415 tag = FP_EXP_TAG_ZERO;
416 else
417 tag = FP_EXP_TAG_SPECIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 break;
419 default:
Roland McGrath44210112008-01-30 13:31:50 +0100420 if (st->significand[3] & 0x8000)
421 tag = FP_EXP_TAG_VALID;
422 else
423 tag = FP_EXP_TAG_SPECIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 break;
425 }
426 } else {
Roland McGrath44210112008-01-30 13:31:50 +0100427 tag = FP_EXP_TAG_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 }
Roland McGrath44210112008-01-30 13:31:50 +0100429 ret |= tag << (2 * i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 }
431 return ret;
432}
433
434/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 * FXSR floating point environment conversions.
436 */
437
Suresh Siddha72a671c2012-07-24 16:05:29 -0700438void
Ingo Molnarf6689642008-03-05 15:37:32 +0100439convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440{
Avi Kivity86603282010-05-06 11:45:46 +0300441 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
Roland McGrath44210112008-01-30 13:31:50 +0100442 struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
443 struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 int i;
445
Roland McGrath44210112008-01-30 13:31:50 +0100446 env->cwd = fxsave->cwd | 0xffff0000u;
447 env->swd = fxsave->swd | 0xffff0000u;
448 env->twd = twd_fxsr_to_i387(fxsave);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449
Roland McGrath44210112008-01-30 13:31:50 +0100450#ifdef CONFIG_X86_64
451 env->fip = fxsave->rip;
452 env->foo = fxsave->rdp;
Brian Gerst10c11f32010-09-03 21:17:13 -0400453 /*
454 * should be actually ds/cs at fpu exception time, but
455 * that information is not available in 64bit mode.
456 */
457 env->fcs = task_pt_regs(tsk)->cs;
Roland McGrath44210112008-01-30 13:31:50 +0100458 if (tsk == current) {
Brian Gerst10c11f32010-09-03 21:17:13 -0400459 savesegment(ds, env->fos);
Roland McGrath44210112008-01-30 13:31:50 +0100460 } else {
Brian Gerst10c11f32010-09-03 21:17:13 -0400461 env->fos = tsk->thread.ds;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 }
Brian Gerst10c11f32010-09-03 21:17:13 -0400463 env->fos |= 0xffff0000;
Roland McGrath44210112008-01-30 13:31:50 +0100464#else
465 env->fip = fxsave->fip;
Jan Beulich609b5292008-03-05 08:35:14 +0000466 env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
Roland McGrath44210112008-01-30 13:31:50 +0100467 env->foo = fxsave->foo;
468 env->fos = fxsave->fos;
469#endif
470
471 for (i = 0; i < 8; ++i)
472 memcpy(&to[i], &from[i], sizeof(to[0]));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473}
474
Suresh Siddha72a671c2012-07-24 16:05:29 -0700475void convert_to_fxsr(struct task_struct *tsk,
476 const struct user_i387_ia32_struct *env)
Roland McGrath44210112008-01-30 13:31:50 +0100477
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478{
Avi Kivity86603282010-05-06 11:45:46 +0300479 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
Roland McGrath44210112008-01-30 13:31:50 +0100480 struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
481 struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482 int i;
483
Roland McGrath44210112008-01-30 13:31:50 +0100484 fxsave->cwd = env->cwd;
485 fxsave->swd = env->swd;
486 fxsave->twd = twd_i387_to_fxsr(env->twd);
487 fxsave->fop = (u16) ((u32) env->fcs >> 16);
488#ifdef CONFIG_X86_64
489 fxsave->rip = env->fip;
490 fxsave->rdp = env->foo;
491 /* cs and ds ignored */
492#else
493 fxsave->fip = env->fip;
494 fxsave->fcs = (env->fcs & 0xffff);
495 fxsave->foo = env->foo;
496 fxsave->fos = env->fos;
497#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
Roland McGrath44210112008-01-30 13:31:50 +0100499 for (i = 0; i < 8; ++i)
500 memcpy(&to[i], &from[i], sizeof(from[0]));
501}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
Roland McGrath44210112008-01-30 13:31:50 +0100503int fpregs_get(struct task_struct *target, const struct user_regset *regset,
504 unsigned int pos, unsigned int count,
505 void *kbuf, void __user *ubuf)
506{
507 struct user_i387_ia32_struct env;
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700508 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700510 ret = init_fpu(target);
511 if (ret)
512 return ret;
Roland McGrath44210112008-01-30 13:31:50 +0100513
Suresh Siddhae8a496a2008-05-23 16:26:37 -0700514 if (!HAVE_HWFP)
515 return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
516
Ingo Molnarf6689642008-03-05 15:37:32 +0100517 if (!cpu_has_fxsr) {
Roland McGrath44210112008-01-30 13:31:50 +0100518 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
Avi Kivity86603282010-05-06 11:45:46 +0300519 &target->thread.fpu.state->fsave, 0,
Suresh Siddha61c46282008-03-10 15:28:04 -0700520 -1);
Ingo Molnarf6689642008-03-05 15:37:32 +0100521 }
Roland McGrath44210112008-01-30 13:31:50 +0100522
Suresh Siddha29104e12010-07-19 16:05:49 -0700523 sanitize_i387_state(target);
524
Roland McGrath44210112008-01-30 13:31:50 +0100525 if (kbuf && pos == 0 && count == sizeof(env)) {
526 convert_from_fxsr(kbuf, target);
527 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 }
Roland McGrath44210112008-01-30 13:31:50 +0100529
530 convert_from_fxsr(&env, target);
Ingo Molnarf6689642008-03-05 15:37:32 +0100531
Roland McGrath44210112008-01-30 13:31:50 +0100532 return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
533}
534
535int fpregs_set(struct task_struct *target, const struct user_regset *regset,
536 unsigned int pos, unsigned int count,
537 const void *kbuf, const void __user *ubuf)
538{
539 struct user_i387_ia32_struct env;
540 int ret;
541
Suresh Siddhaaa283f42008-03-10 15:28:05 -0700542 ret = init_fpu(target);
543 if (ret)
544 return ret;
545
Suresh Siddha29104e12010-07-19 16:05:49 -0700546 sanitize_i387_state(target);
547
Suresh Siddhae8a496a2008-05-23 16:26:37 -0700548 if (!HAVE_HWFP)
549 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
550
Ingo Molnarf6689642008-03-05 15:37:32 +0100551 if (!cpu_has_fxsr) {
Roland McGrath44210112008-01-30 13:31:50 +0100552 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
Avi Kivity86603282010-05-06 11:45:46 +0300553 &target->thread.fpu.state->fsave, 0, -1);
Ingo Molnarf6689642008-03-05 15:37:32 +0100554 }
Roland McGrath44210112008-01-30 13:31:50 +0100555
556 if (pos > 0 || count < sizeof(env))
557 convert_from_fxsr(&env, target);
558
559 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
560 if (!ret)
561 convert_to_fxsr(target, &env);
562
Suresh Siddha42deec62008-07-29 10:29:26 -0700563 /*
564 * update the header bit in the xsave header, indicating the
565 * presence of FP.
566 */
567 if (cpu_has_xsave)
Avi Kivity86603282010-05-06 11:45:46 +0300568 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
Roland McGrath44210112008-01-30 13:31:50 +0100569 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570}
571
572/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 * FPU state for core dumps.
Roland McGrath60b3b9a2008-01-30 13:31:55 +0100574 * This is only used for a.out dumps now.
575 * It is declared generically using elf_fpregset_t (which is
576 * struct user_i387_struct) but is in fact only used for 32-bit
577 * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 */
Cyrill Gorcunov3b095a02008-01-30 13:31:26 +0100579int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 struct task_struct *tsk = current;
Ingo Molnarf6689642008-03-05 15:37:32 +0100582 int fpvalid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
584 fpvalid = !!used_math();
Roland McGrath60b3b9a2008-01-30 13:31:55 +0100585 if (fpvalid)
586 fpvalid = !fpregs_get(tsk, NULL,
587 0, sizeof(struct user_i387_ia32_struct),
588 fpu, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
590 return fpvalid;
591}
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700592EXPORT_SYMBOL(dump_fpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593
Roland McGrath60b3b9a2008-01-30 13:31:55 +0100594#endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */