[IA64] pvops: introduce pv_cpu_ops to paravirtualize privileged instructions.
introduce pv_cpu_ops to paravirtualize privleged instructions
which are defined by ia64 intrinsics.
make them indirect C function calls by introducing function
tables, pv_cpu_ops.
Signed-off-by: Yaozu (Eddie) Dong <eddie.dong@intel.com>
Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
Signed-off-by: Tony Luck <tony.luck@intel.com>
diff --git a/include/asm-ia64/Kbuild b/include/asm-ia64/Kbuild
index eb24a3f..ccbe8ae 100644
--- a/include/asm-ia64/Kbuild
+++ b/include/asm-ia64/Kbuild
@@ -5,12 +5,12 @@
header-y += fpswa.h
header-y += ia64regs.h
header-y += intel_intrin.h
-header-y += intrinsics.h
header-y += perfmon_default_smpl.h
header-y += ptrace_offsets.h
header-y += rse.h
header-y += ucontext.h
unifdef-y += gcc_intrin.h
+unifdef-y += intrinsics.h
unifdef-y += perfmon.h
unifdef-y += ustack.h
diff --git a/include/asm-ia64/gcc_intrin.h b/include/asm-ia64/gcc_intrin.h
index 2fe292c..0f5b559 100644
--- a/include/asm-ia64/gcc_intrin.h
+++ b/include/asm-ia64/gcc_intrin.h
@@ -32,7 +32,7 @@
register unsigned long ia64_r13 asm ("r13") __used;
#endif
-#define ia64_setreg(regnum, val) \
+#define ia64_native_setreg(regnum, val) \
({ \
switch (regnum) { \
case _IA64_REG_PSR_L: \
@@ -61,7 +61,7 @@
} \
})
-#define ia64_getreg(regnum) \
+#define ia64_native_getreg(regnum) \
({ \
__u64 ia64_intri_res; \
\
@@ -385,7 +385,7 @@
#define ia64_invala() asm volatile ("invala" ::: "memory")
-#define ia64_thash(addr) \
+#define ia64_native_thash(addr) \
({ \
__u64 ia64_intri_res; \
asm volatile ("thash %0=%1" : "=r"(ia64_intri_res) : "r" (addr)); \
@@ -438,10 +438,10 @@
#define ia64_set_pmd(index, val) \
asm volatile ("mov pmd[%0]=%1" :: "r"(index), "r"(val) : "memory")
-#define ia64_set_rr(index, val) \
+#define ia64_native_set_rr(index, val) \
asm volatile ("mov rr[%0]=%1" :: "r"(index), "r"(val) : "memory");
-#define ia64_get_cpuid(index) \
+#define ia64_native_get_cpuid(index) \
({ \
__u64 ia64_intri_res; \
asm volatile ("mov %0=cpuid[%r1]" : "=r"(ia64_intri_res) : "rO"(index)); \
@@ -477,33 +477,33 @@
})
-#define ia64_get_pmd(index) \
+#define ia64_native_get_pmd(index) \
({ \
__u64 ia64_intri_res; \
asm volatile ("mov %0=pmd[%1]" : "=r"(ia64_intri_res) : "r"(index)); \
ia64_intri_res; \
})
-#define ia64_get_rr(index) \
+#define ia64_native_get_rr(index) \
({ \
__u64 ia64_intri_res; \
asm volatile ("mov %0=rr[%1]" : "=r"(ia64_intri_res) : "r" (index)); \
ia64_intri_res; \
})
-#define ia64_fc(addr) asm volatile ("fc %0" :: "r"(addr) : "memory")
+#define ia64_native_fc(addr) asm volatile ("fc %0" :: "r"(addr) : "memory")
#define ia64_sync_i() asm volatile (";; sync.i" ::: "memory")
-#define ia64_ssm(mask) asm volatile ("ssm %0":: "i"((mask)) : "memory")
-#define ia64_rsm(mask) asm volatile ("rsm %0":: "i"((mask)) : "memory")
+#define ia64_native_ssm(mask) asm volatile ("ssm %0":: "i"((mask)) : "memory")
+#define ia64_native_rsm(mask) asm volatile ("rsm %0":: "i"((mask)) : "memory")
#define ia64_sum(mask) asm volatile ("sum %0":: "i"((mask)) : "memory")
#define ia64_rum(mask) asm volatile ("rum %0":: "i"((mask)) : "memory")
#define ia64_ptce(addr) asm volatile ("ptc.e %0" :: "r"(addr))
-#define ia64_ptcga(addr, size) \
+#define ia64_native_ptcga(addr, size) \
do { \
asm volatile ("ptc.ga %0,%1" :: "r"(addr), "r"(size) : "memory"); \
ia64_dv_serialize_data(); \
@@ -608,7 +608,7 @@
} \
})
-#define ia64_intrin_local_irq_restore(x) \
+#define ia64_native_intrin_local_irq_restore(x) \
do { \
asm volatile (";; cmp.ne p6,p7=%0,r0;;" \
"(p6) ssm psr.i;" \
diff --git a/include/asm-ia64/intel_intrin.h b/include/asm-ia64/intel_intrin.h
index a520d10..53cec57 100644
--- a/include/asm-ia64/intel_intrin.h
+++ b/include/asm-ia64/intel_intrin.h
@@ -16,8 +16,8 @@
* intrinsic
*/
-#define ia64_getreg __getReg
-#define ia64_setreg __setReg
+#define ia64_native_getreg __getReg
+#define ia64_native_setreg __setReg
#define ia64_hint __hint
#define ia64_hint_pause __hint_pause
@@ -39,10 +39,10 @@
#define ia64_invala_fr __invala_fr
#define ia64_nop __nop
#define ia64_sum __sum
-#define ia64_ssm __ssm
+#define ia64_native_ssm __ssm
#define ia64_rum __rum
-#define ia64_rsm __rsm
-#define ia64_fc __fc
+#define ia64_native_rsm __rsm
+#define ia64_native_fc __fc
#define ia64_ldfs __ldfs
#define ia64_ldfd __ldfd
@@ -88,16 +88,17 @@
__setIndReg(_IA64_REG_INDR_PMC, index, val)
#define ia64_set_pmd(index, val) \
__setIndReg(_IA64_REG_INDR_PMD, index, val)
-#define ia64_set_rr(index, val) \
+#define ia64_native_set_rr(index, val) \
__setIndReg(_IA64_REG_INDR_RR, index, val)
-#define ia64_get_cpuid(index) __getIndReg(_IA64_REG_INDR_CPUID, index)
-#define __ia64_get_dbr(index) __getIndReg(_IA64_REG_INDR_DBR, index)
-#define ia64_get_ibr(index) __getIndReg(_IA64_REG_INDR_IBR, index)
-#define ia64_get_pkr(index) __getIndReg(_IA64_REG_INDR_PKR, index)
-#define ia64_get_pmc(index) __getIndReg(_IA64_REG_INDR_PMC, index)
-#define ia64_get_pmd(index) __getIndReg(_IA64_REG_INDR_PMD, index)
-#define ia64_get_rr(index) __getIndReg(_IA64_REG_INDR_RR, index)
+#define ia64_native_get_cpuid(index) \
+ __getIndReg(_IA64_REG_INDR_CPUID, index)
+#define __ia64_get_dbr(index) __getIndReg(_IA64_REG_INDR_DBR, index)
+#define ia64_get_ibr(index) __getIndReg(_IA64_REG_INDR_IBR, index)
+#define ia64_get_pkr(index) __getIndReg(_IA64_REG_INDR_PKR, index)
+#define ia64_get_pmc(index) __getIndReg(_IA64_REG_INDR_PMC, index)
+#define ia64_native_get_pmd(index) __getIndReg(_IA64_REG_INDR_PMD, index)
+#define ia64_native_get_rr(index) __getIndReg(_IA64_REG_INDR_RR, index)
#define ia64_srlz_d __dsrlz
#define ia64_srlz_i __isrlz
@@ -119,16 +120,16 @@
#define ia64_ld8_acq __ld8_acq
#define ia64_sync_i __synci
-#define ia64_thash __thash
-#define ia64_ttag __ttag
+#define ia64_native_thash __thash
+#define ia64_native_ttag __ttag
#define ia64_itcd __itcd
#define ia64_itci __itci
#define ia64_itrd __itrd
#define ia64_itri __itri
#define ia64_ptce __ptce
#define ia64_ptcl __ptcl
-#define ia64_ptcg __ptcg
-#define ia64_ptcga __ptcga
+#define ia64_native_ptcg __ptcg
+#define ia64_native_ptcga __ptcga
#define ia64_ptri __ptri
#define ia64_ptrd __ptrd
#define ia64_dep_mi _m64_dep_mi
@@ -145,13 +146,13 @@
#define ia64_lfetch_fault __lfetch_fault
#define ia64_lfetch_fault_excl __lfetch_fault_excl
-#define ia64_intrin_local_irq_restore(x) \
+#define ia64_native_intrin_local_irq_restore(x) \
do { \
if ((x) != 0) { \
- ia64_ssm(IA64_PSR_I); \
+ ia64_native_ssm(IA64_PSR_I); \
ia64_srlz_d(); \
} else { \
- ia64_rsm(IA64_PSR_I); \
+ ia64_native_rsm(IA64_PSR_I); \
} \
} while (0)
diff --git a/include/asm-ia64/intrinsics.h b/include/asm-ia64/intrinsics.h
index a3b9689..47d686d 100644
--- a/include/asm-ia64/intrinsics.h
+++ b/include/asm-ia64/intrinsics.h
@@ -18,15 +18,15 @@
# include <asm/gcc_intrin.h>
#endif
-#define ia64_get_psr_i() (ia64_getreg(_IA64_REG_PSR) & IA64_PSR_I)
+#define ia64_native_get_psr_i() (ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I)
-#define ia64_set_rr0_to_rr4(val0, val1, val2, val3, val4) \
-do { \
- ia64_set_rr(0x0000000000000000UL, (val0)); \
- ia64_set_rr(0x2000000000000000UL, (val1)); \
- ia64_set_rr(0x4000000000000000UL, (val2)); \
- ia64_set_rr(0x6000000000000000UL, (val3)); \
- ia64_set_rr(0x8000000000000000UL, (val4)); \
+#define ia64_native_set_rr0_to_rr4(val0, val1, val2, val3, val4) \
+do { \
+ ia64_native_set_rr(0x0000000000000000UL, (val0)); \
+ ia64_native_set_rr(0x2000000000000000UL, (val1)); \
+ ia64_native_set_rr(0x4000000000000000UL, (val2)); \
+ ia64_native_set_rr(0x6000000000000000UL, (val3)); \
+ ia64_native_set_rr(0x8000000000000000UL, (val4)); \
} while (0)
/*
@@ -194,4 +194,48 @@
#endif /* !CONFIG_IA64_DEBUG_CMPXCHG */
#endif
+
+#ifdef __KERNEL__
+#include <asm/paravirt_privop.h>
+#endif
+
+#ifndef __ASSEMBLY__
+#if defined(CONFIG_PARAVIRT) && defined(__KERNEL__)
+#define IA64_INTRINSIC_API(name) pv_cpu_ops.name
+#define IA64_INTRINSIC_MACRO(name) paravirt_ ## name
+#else
+#define IA64_INTRINSIC_API(name) ia64_native_ ## name
+#define IA64_INTRINSIC_MACRO(name) ia64_native_ ## name
+#endif
+
+/************************************************/
+/* Instructions paravirtualized for correctness */
+/************************************************/
+/* fc, thash, get_cpuid, get_pmd, get_eflags, set_eflags */
+/* Note that "ttag" and "cover" are also privilege-sensitive; "ttag"
+ * is not currently used (though it may be in a long-format VHPT system!)
+ */
+#define ia64_fc IA64_INTRINSIC_API(fc)
+#define ia64_thash IA64_INTRINSIC_API(thash)
+#define ia64_get_cpuid IA64_INTRINSIC_API(get_cpuid)
+#define ia64_get_pmd IA64_INTRINSIC_API(get_pmd)
+
+
+/************************************************/
+/* Instructions paravirtualized for performance */
+/************************************************/
+#define ia64_ssm IA64_INTRINSIC_MACRO(ssm)
+#define ia64_rsm IA64_INTRINSIC_MACRO(rsm)
+#define ia64_getreg IA64_INTRINSIC_API(getreg)
+#define ia64_setreg IA64_INTRINSIC_API(setreg)
+#define ia64_set_rr IA64_INTRINSIC_API(set_rr)
+#define ia64_get_rr IA64_INTRINSIC_API(get_rr)
+#define ia64_ptcga IA64_INTRINSIC_API(ptcga)
+#define ia64_get_psr_i IA64_INTRINSIC_API(get_psr_i)
+#define ia64_intrin_local_irq_restore \
+ IA64_INTRINSIC_API(intrin_local_irq_restore)
+#define ia64_set_rr0_to_rr4 IA64_INTRINSIC_API(set_rr0_to_rr4)
+
+#endif /* !__ASSEMBLY__ */
+
#endif /* _ASM_IA64_INTRINSICS_H */
diff --git a/include/asm-ia64/paravirt_privop.h b/include/asm-ia64/paravirt_privop.h
new file mode 100644
index 0000000..7b133ae
--- /dev/null
+++ b/include/asm-ia64/paravirt_privop.h
@@ -0,0 +1,91 @@
+/******************************************************************************
+ * include/asm-ia64/paravirt_privops.h
+ *
+ * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
+ * VA Linux Systems Japan K.K.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ */
+
+#ifndef _ASM_IA64_PARAVIRT_PRIVOP_H
+#define _ASM_IA64_PARAVIRT_PRIVOP_H
+
+#ifdef CONFIG_PARAVIRT
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+#include <asm/kregs.h> /* for IA64_PSR_I */
+
+/******************************************************************************
+ * replacement of intrinsics operations.
+ */
+
+struct pv_cpu_ops {
+ void (*fc)(unsigned long addr);
+ unsigned long (*thash)(unsigned long addr);
+ unsigned long (*get_cpuid)(int index);
+ unsigned long (*get_pmd)(int index);
+ unsigned long (*getreg)(int reg);
+ void (*setreg)(int reg, unsigned long val);
+ void (*ptcga)(unsigned long addr, unsigned long size);
+ unsigned long (*get_rr)(unsigned long index);
+ void (*set_rr)(unsigned long index, unsigned long val);
+ void (*set_rr0_to_rr4)(unsigned long val0, unsigned long val1,
+ unsigned long val2, unsigned long val3,
+ unsigned long val4);
+ void (*ssm_i)(void);
+ void (*rsm_i)(void);
+ unsigned long (*get_psr_i)(void);
+ void (*intrin_local_irq_restore)(unsigned long flags);
+};
+
+extern struct pv_cpu_ops pv_cpu_ops;
+
+extern void ia64_native_setreg_func(int regnum, unsigned long val);
+extern unsigned long ia64_native_getreg_func(int regnum);
+
+/************************************************/
+/* Instructions paravirtualized for performance */
+/************************************************/
+
+/* mask for ia64_native_ssm/rsm() must be constant.("i" constraing).
+ * static inline function doesn't satisfy it. */
+#define paravirt_ssm(mask) \
+ do { \
+ if ((mask) == IA64_PSR_I) \
+ pv_cpu_ops.ssm_i(); \
+ else \
+ ia64_native_ssm(mask); \
+ } while (0)
+
+#define paravirt_rsm(mask) \
+ do { \
+ if ((mask) == IA64_PSR_I) \
+ pv_cpu_ops.rsm_i(); \
+ else \
+ ia64_native_rsm(mask); \
+ } while (0)
+
+#endif /* __ASSEMBLY__ */
+
+#else
+
+/* fallback for native case */
+
+#endif /* CONFIG_PARAVIRT */
+
+#endif /* _ASM_IA64_PARAVIRT_PRIVOP_H */