blob: 7d0233c12ee3febac282f37083648ce703cbd349 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +11002 * Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corporation
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +11004 * Provide default implementations of the DMA mapping callbacks for
Becky Bruce8dd0e952008-09-08 09:09:53 +00005 * directly mapped busses.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 */
7
8#include <linux/device.h>
9#include <linux/dma-mapping.h>
FUJITA Tomonori80d3e8a2009-08-04 19:08:28 +000010#include <linux/dma-debug.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090011#include <linux/gfp.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100012#include <linux/memblock.h>
Paul Gortmaker66b15db2011-05-27 10:46:24 -040013#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <asm/bug.h>
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110015#include <asm/abs_addr.h>
Benjamin Herrenschmidt5b6e9ff2010-08-30 19:23:52 +000016#include <asm/machdep.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110018/*
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110019 * Generic direct DMA implementation
Benjamin Herrenschmidt92b20c42006-11-11 17:25:14 +110020 *
Michael Ellerman31d1b492008-01-21 16:42:48 +110021 * This implementation supports a per-device offset that can be applied if
22 * the address at which memory is visible to devices is not 0. Platform code
23 * can set archdata.dma_data to an unsigned long holding the offset. By
Becky Bruce4fc665b2008-09-12 10:34:46 +000024 * default the offset is PCI_DRAM_OFFSET.
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110025 */
26
Michael Ellerman35e4a6e2008-01-21 16:42:43 +110027
Becky Bruce4fc665b2008-09-12 10:34:46 +000028void *dma_direct_alloc_coherent(struct device *dev, size_t size,
29 dma_addr_t *dma_handle, gfp_t flag)
Linus Torvalds1da177e2005-04-16 15:20:36 -070030{
Benjamin Herrenschmidt8aa26592008-10-09 17:06:24 +000031 void *ret;
Becky Bruce4fc665b2008-09-12 10:34:46 +000032#ifdef CONFIG_NOT_COHERENT_CACHE
Benjamin Herrenschmidt8b31e492009-05-27 13:50:33 +100033 ret = __dma_alloc_coherent(dev, size, dma_handle, flag);
Benjamin Herrenschmidt8aa26592008-10-09 17:06:24 +000034 if (ret == NULL)
35 return NULL;
Becky Bruce1cebd7a2009-09-21 08:26:34 +000036 *dma_handle += get_dma_offset(dev);
Benjamin Herrenschmidt8aa26592008-10-09 17:06:24 +000037 return ret;
Becky Bruce4fc665b2008-09-12 10:34:46 +000038#else
Benjamin Herrenschmidtc80d9132006-11-11 17:25:16 +110039 struct page *page;
Becky Bruce8fae0352008-09-08 09:09:54 +000040 int node = dev_to_node(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
Becky Bruce4fc665b2008-09-12 10:34:46 +000042 /* ignore region specifiers */
43 flag &= ~(__GFP_HIGHMEM);
44
Benjamin Herrenschmidtc80d9132006-11-11 17:25:16 +110045 page = alloc_pages_node(node, flag, get_order(size));
46 if (page == NULL)
47 return NULL;
48 ret = page_address(page);
49 memset(ret, 0, size);
Becky Bruce1cebd7a2009-09-21 08:26:34 +000050 *dma_handle = virt_to_abs(ret) + get_dma_offset(dev);
Benjamin Herrenschmidtc80d9132006-11-11 17:25:16 +110051
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110052 return ret;
Becky Bruce4fc665b2008-09-12 10:34:46 +000053#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070054}
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
Becky Bruce4fc665b2008-09-12 10:34:46 +000056void dma_direct_free_coherent(struct device *dev, size_t size,
57 void *vaddr, dma_addr_t dma_handle)
Linus Torvalds1da177e2005-04-16 15:20:36 -070058{
Becky Bruce4fc665b2008-09-12 10:34:46 +000059#ifdef CONFIG_NOT_COHERENT_CACHE
60 __dma_free_coherent(size, vaddr);
61#else
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110062 free_pages((unsigned long)vaddr, get_order(size));
Becky Bruce4fc665b2008-09-12 10:34:46 +000063#endif
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110064}
65
Jens Axboe78bdc312007-10-12 13:44:12 +020066static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
Mark Nelson3affedc2008-07-05 05:05:42 +100067 int nents, enum dma_data_direction direction,
68 struct dma_attrs *attrs)
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110069{
Jens Axboe78bdc312007-10-12 13:44:12 +020070 struct scatterlist *sg;
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110071 int i;
72
Jens Axboe78bdc312007-10-12 13:44:12 +020073 for_each_sg(sgl, sg, nents, i) {
Becky Bruce1cebd7a2009-09-21 08:26:34 +000074 sg->dma_address = sg_phys(sg) + get_dma_offset(dev);
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110075 sg->dma_length = sg->length;
Benjamin Herrenschmidt2434bbb2008-11-30 18:53:40 +000076 __dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110077 }
78
79 return nents;
80}
81
82static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sg,
Mark Nelson3affedc2008-07-05 05:05:42 +100083 int nents, enum dma_data_direction direction,
84 struct dma_attrs *attrs)
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110085{
86}
87
88static int dma_direct_dma_supported(struct device *dev, u64 mask)
89{
Becky Bruce4fc665b2008-09-12 10:34:46 +000090#ifdef CONFIG_PPC64
Benjamin Herrenschmidtb2f2e8f2009-08-10 16:36:38 +100091 /* Could be improved so platforms can set the limit in case
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +110092 * they have limited DMA windows
93 */
Nishanth Aravamudanffa56e552010-09-15 08:05:46 +000094 return mask >= get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);
Becky Bruce4fc665b2008-09-12 10:34:46 +000095#else
96 return 1;
97#endif
98}
99
Milton Millerd24f9c62011-06-24 09:05:24 +0000100static u64 dma_direct_get_required_mask(struct device *dev)
101{
102 u64 end, mask;
103
104 end = memblock_end_of_DRAM() + get_dma_offset(dev);
105
106 mask = 1ULL << (fls64(end) - 1);
107 mask += mask - 1;
108
109 return mask;
110}
111
Becky Bruce4fc665b2008-09-12 10:34:46 +0000112static inline dma_addr_t dma_direct_map_page(struct device *dev,
113 struct page *page,
114 unsigned long offset,
115 size_t size,
116 enum dma_data_direction dir,
117 struct dma_attrs *attrs)
118{
119 BUG_ON(dir == DMA_NONE);
120 __dma_sync_page(page, offset, size, dir);
Becky Bruce1cebd7a2009-09-21 08:26:34 +0000121 return page_to_phys(page) + offset + get_dma_offset(dev);
Becky Bruce4fc665b2008-09-12 10:34:46 +0000122}
123
124static inline void dma_direct_unmap_page(struct device *dev,
125 dma_addr_t dma_address,
126 size_t size,
127 enum dma_data_direction direction,
128 struct dma_attrs *attrs)
129{
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +1100130}
131
Becky Bruce15e09c02008-11-20 06:49:16 +0000132#ifdef CONFIG_NOT_COHERENT_CACHE
133static inline void dma_direct_sync_sg(struct device *dev,
134 struct scatterlist *sgl, int nents,
135 enum dma_data_direction direction)
136{
137 struct scatterlist *sg;
138 int i;
139
140 for_each_sg(sgl, sg, nents, i)
141 __dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
142}
143
FUJITA Tomonori712d3e22010-05-26 14:44:17 -0700144static inline void dma_direct_sync_single(struct device *dev,
145 dma_addr_t dma_handle, size_t size,
146 enum dma_data_direction direction)
Becky Bruce15e09c02008-11-20 06:49:16 +0000147{
FUJITA Tomonori712d3e22010-05-26 14:44:17 -0700148 __dma_sync(bus_to_virt(dma_handle), size, direction);
Becky Bruce15e09c02008-11-20 06:49:16 +0000149}
150#endif
151
FUJITA Tomonori45223c52009-08-04 19:08:25 +0000152struct dma_map_ops dma_direct_ops = {
Milton Miller2eccacd2011-06-24 09:05:25 +0000153 .alloc_coherent = dma_direct_alloc_coherent,
154 .free_coherent = dma_direct_free_coherent,
155 .map_sg = dma_direct_map_sg,
156 .unmap_sg = dma_direct_unmap_sg,
157 .dma_supported = dma_direct_dma_supported,
158 .map_page = dma_direct_map_page,
159 .unmap_page = dma_direct_unmap_page,
160 .get_required_mask = dma_direct_get_required_mask,
Becky Bruce15e09c02008-11-20 06:49:16 +0000161#ifdef CONFIG_NOT_COHERENT_CACHE
FUJITA Tomonori712d3e22010-05-26 14:44:17 -0700162 .sync_single_for_cpu = dma_direct_sync_single,
163 .sync_single_for_device = dma_direct_sync_single,
Becky Bruce15e09c02008-11-20 06:49:16 +0000164 .sync_sg_for_cpu = dma_direct_sync_sg,
165 .sync_sg_for_device = dma_direct_sync_sg,
166#endif
Benjamin Herrenschmidt12d04ee2006-11-11 17:25:02 +1100167};
168EXPORT_SYMBOL(dma_direct_ops);
FUJITA Tomonori80d3e8a2009-08-04 19:08:28 +0000169
170#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
171
Benjamin Herrenschmidt5b6e9ff2010-08-30 19:23:52 +0000172int dma_set_mask(struct device *dev, u64 dma_mask)
173{
174 struct dma_map_ops *dma_ops = get_dma_ops(dev);
175
176 if (ppc_md.dma_set_mask)
177 return ppc_md.dma_set_mask(dev, dma_mask);
Kumar Gala6471fc62011-06-10 02:22:06 -0500178 if ((dma_ops != NULL) && (dma_ops->set_dma_mask != NULL))
Benjamin Herrenschmidt5b6e9ff2010-08-30 19:23:52 +0000179 return dma_ops->set_dma_mask(dev, dma_mask);
180 if (!dev->dma_mask || !dma_supported(dev, dma_mask))
181 return -EIO;
182 *dev->dma_mask = dma_mask;
183 return 0;
184}
185EXPORT_SYMBOL(dma_set_mask);
186
Milton Miller6a5c7be2011-06-24 09:05:22 +0000187u64 dma_get_required_mask(struct device *dev)
188{
189 struct dma_map_ops *dma_ops = get_dma_ops(dev);
Milton Miller6a5c7be2011-06-24 09:05:22 +0000190
191 if (ppc_md.dma_get_required_mask)
192 return ppc_md.dma_get_required_mask(dev);
193
194 if (unlikely(dma_ops == NULL))
195 return 0;
196
Milton Millerd24f9c62011-06-24 09:05:24 +0000197 if (dma_ops->get_required_mask)
198 return dma_ops->get_required_mask(dev);
Milton Miller6a5c7be2011-06-24 09:05:22 +0000199
Milton Millerd24f9c62011-06-24 09:05:24 +0000200 return DMA_BIT_MASK(8 * sizeof(dma_addr_t));
Milton Miller6a5c7be2011-06-24 09:05:22 +0000201}
202EXPORT_SYMBOL_GPL(dma_get_required_mask);
203
FUJITA Tomonori80d3e8a2009-08-04 19:08:28 +0000204static int __init dma_init(void)
205{
206 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
207
208 return 0;
209}
210fs_initcall(dma_init);
Benjamin Herrenschmidt60909122011-03-24 20:50:06 +0000211
212int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
213 void *cpu_addr, dma_addr_t handle, size_t size)
214{
215 unsigned long pfn;
216
217#ifdef CONFIG_NOT_COHERENT_CACHE
218 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
219 pfn = __dma_get_coherent_pfn((unsigned long)cpu_addr);
220#else
221 pfn = page_to_pfn(virt_to_page(cpu_addr));
222#endif
223 return remap_pfn_range(vma, vma->vm_start,
224 pfn + vma->vm_pgoff,
225 vma->vm_end - vma->vm_start,
226 vma->vm_page_prot);
227}
228EXPORT_SYMBOL_GPL(dma_mmap_coherent);