blob: 9576a2eb375ed81f5aee52e657676cd74dbad4f9 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Dynamic DMA mapping support.
3 */
4
5#include <linux/types.h>
6#include <linux/mm.h>
7#include <linux/string.h>
8#include <linux/pci.h>
9#include <linux/module.h>
10#include <asm/io.h>
Yinghai Luf2cf8e02007-07-21 17:11:31 +020011#include <asm/iommu.h>
Jon Masone4650582006-06-26 13:58:14 +020012#include <asm/calgary.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010014int iommu_merge __read_mostly = 0;
15EXPORT_SYMBOL(iommu_merge);
16
17dma_addr_t bad_dma_address __read_mostly;
18EXPORT_SYMBOL(bad_dma_address);
19
20/* This tells the BIO block layer to assume merging. Default to off
21 because we cannot guarantee merging later. */
22int iommu_bio_merge __read_mostly = 0;
23EXPORT_SYMBOL(iommu_bio_merge);
24
Jan Beulichcaa51712007-07-09 11:55:51 -070025static int iommu_sac_force __read_mostly = 0;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010026
27int no_iommu __read_mostly;
28#ifdef CONFIG_IOMMU_DEBUG
29int panic_on_overflow __read_mostly = 1;
30int force_iommu __read_mostly = 1;
31#else
32int panic_on_overflow __read_mostly = 0;
33int force_iommu __read_mostly= 0;
34#endif
35
Jon Mason8d4f6b92006-06-26 13:58:05 +020036/* Set this to 1 if there is a HW IOMMU in the system */
37int iommu_detected __read_mostly = 0;
38
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010039/* Dummy device used for NULL arguments (normally ISA). Better would
40 be probably a smaller DMA mask, but this is bug-to-bug compatible
41 to i386. */
42struct device fallback_dev = {
43 .bus_id = "fallback device",
Jon Mason9f2036f2006-06-26 13:56:19 +020044 .coherent_dma_mask = DMA_32BIT_MASK,
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010045 .dma_mask = &fallback_dev.coherent_dma_mask,
46};
47
48/* Allocate DMA memory on node near device */
49noinline static void *
50dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
Linus Torvalds1da177e2005-04-16 15:20:36 -070051{
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010052 struct page *page;
53 int node;
Andi Kleenfa47dd02006-04-07 19:49:33 +020054#ifdef CONFIG_PCI
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010055 if (dev->bus == &pci_bus_type)
56 node = pcibus_to_node(to_pci_dev(dev)->bus);
57 else
Andi Kleenfa47dd02006-04-07 19:49:33 +020058#endif
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010059 node = numa_node_id();
Daniel Yeisley0d015322006-05-30 22:47:57 +020060
61 if (node < first_node(node_online_map))
62 node = first_node(node_online_map);
63
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010064 page = alloc_pages_node(node, gfp, order);
65 return page ? page_address(page) : NULL;
66}
Linus Torvalds1da177e2005-04-16 15:20:36 -070067
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010068/*
69 * Allocate memory for a coherent mapping.
70 */
71void *
72dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
73 gfp_t gfp)
74{
75 void *memory;
76 unsigned long dma_mask = 0;
77 u64 bus;
78
79 if (!dev)
80 dev = &fallback_dev;
81 dma_mask = dev->coherent_dma_mask;
82 if (dma_mask == 0)
Jon Mason9f2036f2006-06-26 13:56:19 +020083 dma_mask = DMA_32BIT_MASK;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010084
Andi Kleen81545492007-08-15 02:40:34 +020085 /* Device not DMA able */
86 if (dev->dma_mask == NULL)
87 return NULL;
88
Andi Kleen3056d6b2006-03-25 16:30:43 +010089 /* Don't invoke OOM killer */
90 gfp |= __GFP_NORETRY;
91
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +010092 /* Kludge to make it bug-to-bug compatible with i386. i386
93 uses the normal dma_mask for alloc_coherent. */
94 dma_mask &= *dev->dma_mask;
95
96 /* Why <=? Even when the mask is smaller than 4GB it is often
97 larger than 16MB and in this case we have a chance of
98 finding fitting memory in the next higher zone first. If
99 not retry with true GFP_DMA. -AK */
Jon Mason9f2036f2006-06-26 13:56:19 +0200100 if (dma_mask <= DMA_32BIT_MASK)
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100101 gfp |= GFP_DMA32;
102
103 again:
104 memory = dma_alloc_pages(dev, gfp, get_order(size));
105 if (memory == NULL)
106 return NULL;
107
108 {
109 int high, mmu;
110 bus = virt_to_bus(memory);
111 high = (bus + size) >= dma_mask;
112 mmu = high;
113 if (force_iommu && !(gfp & GFP_DMA))
114 mmu = 1;
115 else if (high) {
116 free_pages((unsigned long)memory,
117 get_order(size));
118
119 /* Don't use the 16MB ZONE_DMA unless absolutely
120 needed. It's better to use remapping first. */
Jon Mason9f2036f2006-06-26 13:56:19 +0200121 if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100122 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
123 goto again;
124 }
125
Andi Kleen6bca52b2006-02-03 21:50:59 +0100126 /* Let low level make its own zone decisions */
127 gfp &= ~(GFP_DMA32|GFP_DMA);
128
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100129 if (dma_ops->alloc_coherent)
130 return dma_ops->alloc_coherent(dev, size,
131 dma_handle, gfp);
132 return NULL;
133 }
134
135 memset(memory, 0, size);
136 if (!mmu) {
137 *dma_handle = virt_to_bus(memory);
138 return memory;
139 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 }
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100141
142 if (dma_ops->alloc_coherent) {
143 free_pages((unsigned long)memory, get_order(size));
144 gfp &= ~(GFP_DMA|GFP_DMA32);
145 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
146 }
147
148 if (dma_ops->map_simple) {
149 *dma_handle = dma_ops->map_simple(dev, memory,
150 size,
151 PCI_DMA_BIDIRECTIONAL);
152 if (*dma_handle != bad_dma_address)
153 return memory;
154 }
155
156 if (panic_on_overflow)
157 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
158 free_pages((unsigned long)memory, get_order(size));
159 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160}
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100161EXPORT_SYMBOL(dma_alloc_coherent);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100163/*
164 * Unmap coherent memory.
165 * The caller must ensure that the device has finished accessing the mapping.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 */
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100167void dma_free_coherent(struct device *dev, size_t size,
168 void *vaddr, dma_addr_t bus)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169{
David Brownellaa248862007-08-10 13:10:27 -0700170 WARN_ON(irqs_disabled()); /* for portability */
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100171 if (dma_ops->unmap_single)
172 dma_ops->unmap_single(dev, bus, size, 0);
173 free_pages((unsigned long)vaddr, get_order(size));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174}
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100175EXPORT_SYMBOL(dma_free_coherent);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176
Andi Kleenece66842006-09-30 01:47:55 +0200177static int forbid_dac __read_mostly;
178
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100179int dma_supported(struct device *dev, u64 mask)
180{
Andi Kleenece66842006-09-30 01:47:55 +0200181#ifdef CONFIG_PCI
182 if (mask > 0xffffffff && forbid_dac > 0) {
183
184
185
186 printk(KERN_INFO "PCI: Disallowing DAC for device %s\n", dev->bus_id);
187 return 0;
188 }
189#endif
190
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100191 if (dma_ops->dma_supported)
192 return dma_ops->dma_supported(dev, mask);
193
194 /* Copied from i386. Doesn't make much sense, because it will
195 only work for pci_alloc_coherent.
196 The caller just has to use GFP_DMA in this case. */
Jon Mason9f2036f2006-06-26 13:56:19 +0200197 if (mask < DMA_24BIT_MASK)
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100198 return 0;
199
200 /* Tell the device to use SAC when IOMMU force is on. This
201 allows the driver to use cheaper accesses in some cases.
202
203 Problem with this is that if we overflow the IOMMU area and
204 return DAC as fallback address the device may not handle it
205 correctly.
206
207 As a special case some controllers have a 39bit address
208 mode that is as efficient as 32bit (aic79xx). Don't force
209 SAC for these. Assume all masks <= 40 bits are of this
210 type. Normally this doesn't make any difference, but gives
211 more gentle handling of IOMMU overflow. */
Jon Mason9f2036f2006-06-26 13:56:19 +0200212 if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100213 printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
214 return 0;
215 }
216
217 return 1;
218}
219EXPORT_SYMBOL(dma_supported);
220
221int dma_set_mask(struct device *dev, u64 mask)
222{
223 if (!dev->dma_mask || !dma_supported(dev, mask))
224 return -EIO;
225 *dev->dma_mask = mask;
226 return 0;
227}
228EXPORT_SYMBOL(dma_set_mask);
229
Karsten Weiss55588702007-02-13 13:26:21 +0100230/*
231 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
232 * documentation.
233 */
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100234__init int iommu_setup(char *p)
235{
Andi Kleended318e2006-09-30 01:47:55 +0200236 iommu_merge = 1;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100237
Andi Kleen2c8c0e62006-09-26 10:52:32 +0200238 if (!p)
239 return -EINVAL;
240
Andi Kleended318e2006-09-30 01:47:55 +0200241 while (*p) {
242 if (!strncmp(p,"off",3))
243 no_iommu = 1;
244 /* gart_parse_options has more force support */
245 if (!strncmp(p,"force",5))
246 force_iommu = 1;
247 if (!strncmp(p,"noforce",7)) {
248 iommu_merge = 0;
249 force_iommu = 0;
250 }
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100251
Andi Kleended318e2006-09-30 01:47:55 +0200252 if (!strncmp(p, "biomerge",8)) {
253 iommu_bio_merge = 4096;
254 iommu_merge = 1;
255 force_iommu = 1;
256 }
257 if (!strncmp(p, "panic",5))
258 panic_on_overflow = 1;
259 if (!strncmp(p, "nopanic",7))
260 panic_on_overflow = 0;
261 if (!strncmp(p, "merge",5)) {
262 iommu_merge = 1;
263 force_iommu = 1;
264 }
265 if (!strncmp(p, "nomerge",7))
266 iommu_merge = 0;
267 if (!strncmp(p, "forcesac",8))
268 iommu_sac_force = 1;
269 if (!strncmp(p, "allowdac", 8))
270 forbid_dac = 0;
271 if (!strncmp(p, "nodac", 5))
272 forbid_dac = -1;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100273
274#ifdef CONFIG_SWIOTLB
Andi Kleended318e2006-09-30 01:47:55 +0200275 if (!strncmp(p, "soft",4))
276 swiotlb = 1;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100277#endif
278
Andi Kleena813ce42006-06-26 13:57:22 +0200279#ifdef CONFIG_IOMMU
Andi Kleended318e2006-09-30 01:47:55 +0200280 gart_parse_options(p);
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100281#endif
282
Muli Ben-Yehudabff65472006-12-07 02:14:07 +0100283#ifdef CONFIG_CALGARY_IOMMU
284 if (!strncmp(p, "calgary", 7))
285 use_calgary = 1;
286#endif /* CONFIG_CALGARY_IOMMU */
287
Andi Kleended318e2006-09-30 01:47:55 +0200288 p += strcspn(p, ",");
289 if (*p == ',')
290 ++p;
291 }
292 return 0;
Muli Ben-Yehuda17a941d2006-01-11 22:44:42 +0100293}
Andi Kleen2c8c0e62006-09-26 10:52:32 +0200294early_param("iommu", iommu_setup);
Jon Mason0dc243a2006-06-26 13:58:11 +0200295
296void __init pci_iommu_alloc(void)
297{
298 /*
299 * The order of these functions is important for
300 * fall-back/fail-over reasons
301 */
302#ifdef CONFIG_IOMMU
303 iommu_hole_init();
304#endif
305
Jon Masone4650582006-06-26 13:58:14 +0200306#ifdef CONFIG_CALGARY_IOMMU
307 detect_calgary();
308#endif
309
Jon Mason0dc243a2006-06-26 13:58:11 +0200310#ifdef CONFIG_SWIOTLB
311 pci_swiotlb_init();
312#endif
313}
314
315static int __init pci_iommu_init(void)
316{
Jon Masone4650582006-06-26 13:58:14 +0200317#ifdef CONFIG_CALGARY_IOMMU
318 calgary_iommu_init();
319#endif
320
Jon Mason0dc243a2006-06-26 13:58:11 +0200321#ifdef CONFIG_IOMMU
322 gart_iommu_init();
323#endif
324
325 no_iommu_init();
326 return 0;
327}
328
Yinghai Lubc2cea62007-07-21 17:11:28 +0200329void pci_iommu_shutdown(void)
330{
331 gart_iommu_shutdown();
332}
333
Andi Kleen388c19e2007-06-20 12:23:32 +0200334#ifdef CONFIG_PCI
335/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
336
337static __devinit void via_no_dac(struct pci_dev *dev)
338{
339 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
340 printk(KERN_INFO "PCI: VIA PCI bridge detected. Disabling DAC.\n");
341 forbid_dac = 1;
342 }
343}
344DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
345#endif
Jon Mason0dc243a2006-06-26 13:58:11 +0200346/* Must execute after PCI subsystem */
347fs_initcall(pci_iommu_init);