blob: e82e1d24877227ba65ab716dc6c87e110617bd79 [file] [log] [blame]
Russell King03b505e2010-12-20 14:44:32 +00001/*
2 * linux/arch/arm/kernel/smp_tlb.c
3 *
4 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10#include <linux/preempt.h>
11#include <linux/smp.h>
12
13#include <asm/smp_plat.h>
14#include <asm/tlbflush.h>
Catalin Marinas93dc6882013-03-26 23:35:04 +010015#include <asm/mmu_context.h>
Russell King03b505e2010-12-20 14:44:32 +000016
Russell King03b505e2010-12-20 14:44:32 +000017/**********************************************************************/
18
19/*
20 * TLB operations
21 */
22struct tlb_args {
23 struct vm_area_struct *ta_vma;
24 unsigned long ta_start;
25 unsigned long ta_end;
26};
27
28static inline void ipi_flush_tlb_all(void *ignored)
29{
30 local_flush_tlb_all();
31}
32
33static inline void ipi_flush_tlb_mm(void *arg)
34{
35 struct mm_struct *mm = (struct mm_struct *)arg;
36
37 local_flush_tlb_mm(mm);
38}
39
40static inline void ipi_flush_tlb_page(void *arg)
41{
42 struct tlb_args *ta = (struct tlb_args *)arg;
43
44 local_flush_tlb_page(ta->ta_vma, ta->ta_start);
45}
46
47static inline void ipi_flush_tlb_kernel_page(void *arg)
48{
49 struct tlb_args *ta = (struct tlb_args *)arg;
50
51 local_flush_tlb_kernel_page(ta->ta_start);
52}
53
54static inline void ipi_flush_tlb_range(void *arg)
55{
56 struct tlb_args *ta = (struct tlb_args *)arg;
57
58 local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
59}
60
61static inline void ipi_flush_tlb_kernel_range(void *arg)
62{
63 struct tlb_args *ta = (struct tlb_args *)arg;
64
65 local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
66}
67
Will Deacon862c5882013-02-28 17:48:11 +010068static inline void ipi_flush_bp_all(void *ignored)
69{
70 local_flush_bp_all();
71}
72
Catalin Marinas93dc6882013-03-26 23:35:04 +010073#ifdef CONFIG_ARM_ERRATA_798181
74static int erratum_a15_798181(void)
75{
76 unsigned int midr = read_cpuid_id();
77
78 /* Cortex-A15 r0p0..r3p2 affected */
79 if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
80 return 0;
81 return 1;
82}
83#else
84static int erratum_a15_798181(void)
85{
86 return 0;
87}
88#endif
89
90static void ipi_flush_tlb_a15_erratum(void *arg)
91{
92 dmb();
93}
94
95static void broadcast_tlb_a15_erratum(void)
96{
97 if (!erratum_a15_798181())
98 return;
99
100 dummy_flush_tlb_a15_erratum();
101 smp_call_function_many(cpu_online_mask, ipi_flush_tlb_a15_erratum,
102 NULL, 1);
103}
104
105static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
106{
107 int cpu;
108 cpumask_t mask = { CPU_BITS_NONE };
109
110 if (!erratum_a15_798181())
111 return;
112
113 dummy_flush_tlb_a15_erratum();
114 for_each_online_cpu(cpu) {
115 if (cpu == smp_processor_id())
116 continue;
117 /*
118 * We only need to send an IPI if the other CPUs are running
119 * the same ASID as the one being invalidated. There is no
120 * need for locking around the active_asids check since the
121 * switch_mm() function has at least one dmb() (as required by
122 * this workaround) in case a context switch happens on
123 * another CPU after the condition below.
124 */
125 if (atomic64_read(&mm->context.id) ==
126 atomic64_read(&per_cpu(active_asids, cpu)))
127 cpumask_set_cpu(cpu, &mask);
128 }
129 smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
130}
131
Russell King03b505e2010-12-20 14:44:32 +0000132void flush_tlb_all(void)
133{
134 if (tlb_ops_need_broadcast())
135 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
136 else
137 local_flush_tlb_all();
Catalin Marinas93dc6882013-03-26 23:35:04 +0100138 broadcast_tlb_a15_erratum();
Russell King03b505e2010-12-20 14:44:32 +0000139}
140
141void flush_tlb_mm(struct mm_struct *mm)
142{
143 if (tlb_ops_need_broadcast())
Gilad Ben-Yossef3fc498f2012-03-28 14:42:43 -0700144 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
Russell King03b505e2010-12-20 14:44:32 +0000145 else
146 local_flush_tlb_mm(mm);
Catalin Marinas93dc6882013-03-26 23:35:04 +0100147 broadcast_tlb_mm_a15_erratum(mm);
Russell King03b505e2010-12-20 14:44:32 +0000148}
149
150void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
151{
152 if (tlb_ops_need_broadcast()) {
153 struct tlb_args ta;
154 ta.ta_vma = vma;
155 ta.ta_start = uaddr;
Gilad Ben-Yossef3fc498f2012-03-28 14:42:43 -0700156 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
157 &ta, 1);
Russell King03b505e2010-12-20 14:44:32 +0000158 } else
159 local_flush_tlb_page(vma, uaddr);
Catalin Marinas93dc6882013-03-26 23:35:04 +0100160 broadcast_tlb_mm_a15_erratum(vma->vm_mm);
Russell King03b505e2010-12-20 14:44:32 +0000161}
162
163void flush_tlb_kernel_page(unsigned long kaddr)
164{
165 if (tlb_ops_need_broadcast()) {
166 struct tlb_args ta;
167 ta.ta_start = kaddr;
168 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
169 } else
170 local_flush_tlb_kernel_page(kaddr);
Catalin Marinas93dc6882013-03-26 23:35:04 +0100171 broadcast_tlb_a15_erratum();
Russell King03b505e2010-12-20 14:44:32 +0000172}
173
174void flush_tlb_range(struct vm_area_struct *vma,
175 unsigned long start, unsigned long end)
176{
177 if (tlb_ops_need_broadcast()) {
178 struct tlb_args ta;
179 ta.ta_vma = vma;
180 ta.ta_start = start;
181 ta.ta_end = end;
Gilad Ben-Yossef3fc498f2012-03-28 14:42:43 -0700182 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
183 &ta, 1);
Russell King03b505e2010-12-20 14:44:32 +0000184 } else
185 local_flush_tlb_range(vma, start, end);
Catalin Marinas93dc6882013-03-26 23:35:04 +0100186 broadcast_tlb_mm_a15_erratum(vma->vm_mm);
Russell King03b505e2010-12-20 14:44:32 +0000187}
188
189void flush_tlb_kernel_range(unsigned long start, unsigned long end)
190{
191 if (tlb_ops_need_broadcast()) {
192 struct tlb_args ta;
193 ta.ta_start = start;
194 ta.ta_end = end;
195 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
196 } else
197 local_flush_tlb_kernel_range(start, end);
Catalin Marinas93dc6882013-03-26 23:35:04 +0100198 broadcast_tlb_a15_erratum();
Russell King03b505e2010-12-20 14:44:32 +0000199}
200
Will Deacon862c5882013-02-28 17:48:11 +0100201void flush_bp_all(void)
202{
203 if (tlb_ops_need_broadcast())
204 on_each_cpu(ipi_flush_bp_all, NULL, 1);
205 else
206 local_flush_bp_all();
207}