blob: 0b11bf18f540bf1970cbd43031da3229e7228c6b [file] [log] [blame]
Glauber Costa459121c92008-04-08 13:20:43 -03001#include <linux/dma-mapping.h>
Joerg Roedel2118d0c2009-01-09 15:13:15 +01002#include <linux/dma-debug.h>
Glauber Costacb5867a2008-04-08 13:20:51 -03003#include <linux/dmar.h>
Glauber Costa116890d2008-04-08 13:20:54 -03004#include <linux/bootmem.h>
Glauber Costabca5c092008-04-08 13:20:53 -03005#include <linux/pci.h>
Catalin Marinasacde31d2009-08-27 14:29:20 +01006#include <linux/kmemleak.h>
Glauber Costacb5867a2008-04-08 13:20:51 -03007
Glauber Costa116890d2008-04-08 13:20:54 -03008#include <asm/proto.h>
9#include <asm/dma.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090010#include <asm/iommu.h>
Joerg Roedel1d9b16d2008-11-27 18:39:15 +010011#include <asm/gart.h>
Glauber Costacb5867a2008-04-08 13:20:51 -030012#include <asm/calgary.h>
Joerg Roedela69ca342008-06-26 21:28:08 +020013#include <asm/amd_iommu.h>
Ingo Molnarb4941a92009-11-10 14:37:58 +010014#include <asm/x86_init.h>
Glauber Costa459121c92008-04-08 13:20:43 -030015
Fenghua Yu3b15e582008-10-23 16:51:00 -070016static int forbid_dac __read_mostly;
17
FUJITA Tomonori160c1d82009-01-05 23:59:02 +090018struct dma_map_ops *dma_ops;
Glauber Costa85c246e2008-04-08 13:20:50 -030019EXPORT_SYMBOL(dma_ops);
20
Dmitri Vorobievb4cdc432008-04-28 03:15:58 +040021static int iommu_sac_force __read_mostly;
Glauber Costa8e0c3792008-04-08 13:20:55 -030022
Glauber Costaf9c258d2008-04-08 13:20:52 -030023#ifdef CONFIG_IOMMU_DEBUG
24int panic_on_overflow __read_mostly = 1;
25int force_iommu __read_mostly = 1;
26#else
27int panic_on_overflow __read_mostly = 0;
28int force_iommu __read_mostly = 0;
29#endif
30
Glauber Costafae9a0d2008-04-08 13:20:56 -030031int iommu_merge __read_mostly = 0;
32
33int no_iommu __read_mostly;
34/* Set this to 1 if there is a HW IOMMU in the system */
35int iommu_detected __read_mostly = 0;
36
Joerg Roedelac0101d2009-09-01 16:00:35 +020037/*
38 * This variable becomes 1 if iommu=pt is passed on the kernel command line.
Marin Mitove3be7852009-10-03 20:45:02 +030039 * If this variable is 1, IOMMU implementations do no DMA translation for
Joerg Roedelac0101d2009-09-01 16:00:35 +020040 * devices and allow every device to access to whole physical memory. This is
41 * useful if a user want to use an IOMMU only for KVM device assignment to
42 * guests and not for driver dma translation.
43 */
44int iommu_pass_through __read_mostly;
Fenghua Yuaed5d5f2009-04-30 17:57:11 -070045
Glauber Costacac67872008-04-08 13:21:00 -030046dma_addr_t bad_dma_address __read_mostly = 0;
47EXPORT_SYMBOL(bad_dma_address);
Glauber Costafae9a0d2008-04-08 13:20:56 -030048
Glauber Costa098cb7f2008-04-09 13:18:10 -030049/* Dummy device used for NULL arguments (normally ISA). Better would
50 be probably a smaller DMA mask, but this is bug-to-bug compatible
51 to older i386. */
Joerg Roedel6c505ce2008-08-19 16:32:45 +020052struct device x86_dma_fallback_dev = {
Kay Sievers1a927132008-10-30 02:17:49 +010053 .init_name = "fallback device",
Yang Hongyang284901a2009-04-06 19:01:15 -070054 .coherent_dma_mask = DMA_BIT_MASK(32),
Joerg Roedel6c505ce2008-08-19 16:32:45 +020055 .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
Glauber Costa098cb7f2008-04-09 13:18:10 -030056};
Joerg Roedel6c505ce2008-08-19 16:32:45 +020057EXPORT_SYMBOL(x86_dma_fallback_dev);
Glauber Costa098cb7f2008-04-09 13:18:10 -030058
Joerg Roedel2118d0c2009-01-09 15:13:15 +010059/* Number of entries preallocated for DMA-API debugging */
60#define PREALLOC_DMA_DEBUG_ENTRIES 32768
61
Glauber Costa459121c92008-04-08 13:20:43 -030062int dma_set_mask(struct device *dev, u64 mask)
63{
64 if (!dev->dma_mask || !dma_supported(dev, mask))
65 return -EIO;
66
67 *dev->dma_mask = mask;
68
69 return 0;
70}
71EXPORT_SYMBOL(dma_set_mask);
72
Glauber Costa116890d2008-04-08 13:20:54 -030073#ifdef CONFIG_X86_64
74static __initdata void *dma32_bootmem_ptr;
75static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
76
77static int __init parse_dma32_size_opt(char *p)
78{
79 if (!p)
80 return -EINVAL;
81 dma32_bootmem_size = memparse(p, &p);
82 return 0;
83}
84early_param("dma32_size", parse_dma32_size_opt);
85
86void __init dma32_reserve_bootmem(void)
87{
88 unsigned long size, align;
Yinghai Luc987d122008-06-24 22:14:09 -070089 if (max_pfn <= MAX_DMA32_PFN)
Glauber Costa116890d2008-04-08 13:20:54 -030090 return;
91
Yinghai Lu7677b2e2008-04-14 20:40:37 -070092 /*
93 * check aperture_64.c allocate_aperture() for reason about
94 * using 512M as goal
95 */
Glauber Costa116890d2008-04-08 13:20:54 -030096 align = 64ULL<<20;
Joerg Roedel1ddb5512008-07-25 16:48:55 +020097 size = roundup(dma32_bootmem_size, align);
Glauber Costa116890d2008-04-08 13:20:54 -030098 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
Yinghai Lu7677b2e2008-04-14 20:40:37 -070099 512ULL<<20);
Catalin Marinasacde31d2009-08-27 14:29:20 +0100100 /*
101 * Kmemleak should not scan this block as it may not be mapped via the
102 * kernel direct mapping.
103 */
104 kmemleak_ignore(dma32_bootmem_ptr);
Glauber Costa116890d2008-04-08 13:20:54 -0300105 if (dma32_bootmem_ptr)
106 dma32_bootmem_size = size;
107 else
108 dma32_bootmem_size = 0;
109}
110static void __init dma32_free_bootmem(void)
111{
Glauber Costa116890d2008-04-08 13:20:54 -0300112
Yinghai Luc987d122008-06-24 22:14:09 -0700113 if (max_pfn <= MAX_DMA32_PFN)
Glauber Costa116890d2008-04-08 13:20:54 -0300114 return;
115
116 if (!dma32_bootmem_ptr)
117 return;
118
Yinghai Lu330fce22008-04-19 01:31:45 -0700119 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
Glauber Costa116890d2008-04-08 13:20:54 -0300120
121 dma32_bootmem_ptr = NULL;
122 dma32_bootmem_size = 0;
123}
Jeremy Fitzhardingecfb80c92008-12-16 12:17:36 -0800124#endif
Glauber Costa116890d2008-04-08 13:20:54 -0300125
126void __init pci_iommu_alloc(void)
127{
Jeremy Fitzhardingecfb80c92008-12-16 12:17:36 -0800128#ifdef CONFIG_X86_64
Glauber Costa116890d2008-04-08 13:20:54 -0300129 /* free the range so iommu could get some range less than 4G */
130 dma32_free_bootmem();
FUJITA Tomonori72d03802009-11-10 21:35:17 +0900131#else
132 dma_ops = &nommu_dma_ops;
Jeremy Fitzhardingecfb80c92008-12-16 12:17:36 -0800133#endif
FUJITA Tomonorib18485e2009-11-12 00:03:28 +0900134 if (pci_swiotlb_init())
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +0900135 return;
Jeremy Fitzhardingecfb80c92008-12-16 12:17:36 -0800136
Glauber Costa116890d2008-04-08 13:20:54 -0300137 gart_iommu_hole_init();
Glauber Costa116890d2008-04-08 13:20:54 -0300138
Glauber Costa116890d2008-04-08 13:20:54 -0300139 detect_calgary();
Glauber Costa116890d2008-04-08 13:20:54 -0300140
141 detect_intel_iommu();
142
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +0900143 /* needs to be called after gart_iommu_hole_init */
Joerg Roedela69ca342008-06-26 21:28:08 +0200144 amd_iommu_detect();
Glauber Costa116890d2008-04-08 13:20:54 -0300145}
FUJITA Tomonori8978b742008-07-29 13:38:53 +0900146
FUJITA Tomonori9f6ac572008-09-24 20:48:35 +0900147void *dma_generic_alloc_coherent(struct device *dev, size_t size,
148 dma_addr_t *dma_addr, gfp_t flag)
149{
150 unsigned long dma_mask;
151 struct page *page;
152 dma_addr_t addr;
153
154 dma_mask = dma_alloc_coherent_mask(dev, flag);
155
156 flag |= __GFP_ZERO;
157again:
158 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
159 if (!page)
160 return NULL;
161
162 addr = page_to_phys(page);
FUJITA Tomonoria4c2baa2009-07-10 10:04:55 +0900163 if (addr + size > dma_mask) {
FUJITA Tomonori9f6ac572008-09-24 20:48:35 +0900164 __free_pages(page, get_order(size));
165
Yang Hongyang284901a2009-04-06 19:01:15 -0700166 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
FUJITA Tomonori9f6ac572008-09-24 20:48:35 +0900167 flag = (flag & ~GFP_DMA32) | GFP_DMA;
168 goto again;
169 }
170
171 return NULL;
172 }
173
174 *dma_addr = addr;
175 return page_address(page);
176}
177
Glauber Costafae9a0d2008-04-08 13:20:56 -0300178/*
179 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
180 * documentation.
181 */
182static __init int iommu_setup(char *p)
183{
184 iommu_merge = 1;
185
186 if (!p)
187 return -EINVAL;
188
189 while (*p) {
190 if (!strncmp(p, "off", 3))
191 no_iommu = 1;
192 /* gart_parse_options has more force support */
193 if (!strncmp(p, "force", 5))
194 force_iommu = 1;
195 if (!strncmp(p, "noforce", 7)) {
196 iommu_merge = 0;
197 force_iommu = 0;
198 }
199
200 if (!strncmp(p, "biomerge", 8)) {
Glauber Costafae9a0d2008-04-08 13:20:56 -0300201 iommu_merge = 1;
202 force_iommu = 1;
203 }
204 if (!strncmp(p, "panic", 5))
205 panic_on_overflow = 1;
206 if (!strncmp(p, "nopanic", 7))
207 panic_on_overflow = 0;
208 if (!strncmp(p, "merge", 5)) {
209 iommu_merge = 1;
210 force_iommu = 1;
211 }
212 if (!strncmp(p, "nomerge", 7))
213 iommu_merge = 0;
214 if (!strncmp(p, "forcesac", 8))
215 iommu_sac_force = 1;
216 if (!strncmp(p, "allowdac", 8))
217 forbid_dac = 0;
218 if (!strncmp(p, "nodac", 5))
Tejun Heo2ae8bb72009-10-26 15:41:46 +0100219 forbid_dac = 1;
Glauber Costafae9a0d2008-04-08 13:20:56 -0300220 if (!strncmp(p, "usedac", 6)) {
221 forbid_dac = -1;
222 return 1;
223 }
224#ifdef CONFIG_SWIOTLB
225 if (!strncmp(p, "soft", 4))
226 swiotlb = 1;
David Woodhouse3238c0c2009-07-01 18:56:16 +0100227#endif
Alex Williamson80286872009-07-30 16:15:18 -0600228 if (!strncmp(p, "pt", 2))
Fenghua Yu4ed0d3e2009-04-24 17:30:20 -0700229 iommu_pass_through = 1;
Glauber Costafae9a0d2008-04-08 13:20:56 -0300230
Glauber Costafae9a0d2008-04-08 13:20:56 -0300231 gart_parse_options(p);
Glauber Costafae9a0d2008-04-08 13:20:56 -0300232
233#ifdef CONFIG_CALGARY_IOMMU
234 if (!strncmp(p, "calgary", 7))
235 use_calgary = 1;
236#endif /* CONFIG_CALGARY_IOMMU */
237
238 p += strcspn(p, ",");
239 if (*p == ',')
240 ++p;
241 }
242 return 0;
243}
244early_param("iommu", iommu_setup);
245
Glauber Costa8e0c3792008-04-08 13:20:55 -0300246int dma_supported(struct device *dev, u64 mask)
247{
FUJITA Tomonori160c1d82009-01-05 23:59:02 +0900248 struct dma_map_ops *ops = get_dma_ops(dev);
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700249
Glauber Costa8e0c3792008-04-08 13:20:55 -0300250#ifdef CONFIG_PCI
251 if (mask > 0xffffffff && forbid_dac > 0) {
Greg Kroah-Hartmanfc3a8822008-05-02 06:02:41 +0200252 dev_info(dev, "PCI: Disallowing DAC for device\n");
Glauber Costa8e0c3792008-04-08 13:20:55 -0300253 return 0;
254 }
255#endif
256
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700257 if (ops->dma_supported)
258 return ops->dma_supported(dev, mask);
Glauber Costa8e0c3792008-04-08 13:20:55 -0300259
260 /* Copied from i386. Doesn't make much sense, because it will
261 only work for pci_alloc_coherent.
262 The caller just has to use GFP_DMA in this case. */
Yang Hongyang2f4f27d2009-04-06 19:01:18 -0700263 if (mask < DMA_BIT_MASK(24))
Glauber Costa8e0c3792008-04-08 13:20:55 -0300264 return 0;
265
266 /* Tell the device to use SAC when IOMMU force is on. This
267 allows the driver to use cheaper accesses in some cases.
268
269 Problem with this is that if we overflow the IOMMU area and
270 return DAC as fallback address the device may not handle it
271 correctly.
272
273 As a special case some controllers have a 39bit address
274 mode that is as efficient as 32bit (aic79xx). Don't force
275 SAC for these. Assume all masks <= 40 bits are of this
276 type. Normally this doesn't make any difference, but gives
277 more gentle handling of IOMMU overflow. */
Yang Hongyang50cf1562009-04-06 19:01:14 -0700278 if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
Greg Kroah-Hartmanfc3a8822008-05-02 06:02:41 +0200279 dev_info(dev, "Force SAC with mask %Lx\n", mask);
Glauber Costa8e0c3792008-04-08 13:20:55 -0300280 return 0;
281 }
282
283 return 1;
284}
285EXPORT_SYMBOL(dma_supported);
286
Glauber Costacb5867a2008-04-08 13:20:51 -0300287static int __init pci_iommu_init(void)
288{
Joerg Roedel2118d0c2009-01-09 15:13:15 +0100289 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
290
Joerg Roedel86f31952009-03-16 17:50:28 +0100291#ifdef CONFIG_PCI
292 dma_debug_add_bus(&pci_bus_type);
293#endif
FUJITA Tomonorid07c1be2009-11-10 19:46:12 +0900294 x86_init.iommu.iommu_init();
295
FUJITA Tomonori75f1cdf2009-11-10 19:46:20 +0900296 if (swiotlb) {
297 printk(KERN_INFO "PCI-DMA: "
298 "Using software bounce buffering for IO (SWIOTLB)\n");
299 swiotlb_print_info();
300 } else
301 swiotlb_free();
302
Glauber Costacb5867a2008-04-08 13:20:51 -0300303 return 0;
304}
Glauber Costacb5867a2008-04-08 13:20:51 -0300305/* Must execute after PCI subsystem */
David Woodhouse9a821b22009-10-12 12:59:29 +0100306rootfs_initcall(pci_iommu_init);
Fenghua Yu3b15e582008-10-23 16:51:00 -0700307
308#ifdef CONFIG_PCI
309/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
310
311static __devinit void via_no_dac(struct pci_dev *dev)
312{
313 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
Bjorn Helgaas13bf7572009-02-24 10:38:22 -0700314 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
Fenghua Yu3b15e582008-10-23 16:51:00 -0700315 forbid_dac = 1;
316 }
317}
318DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
319#endif