blob: 794163ae97b49e2c93a7e2f785bf2ebf7e8fa306 [file] [log] [blame]
Joerg Roedelb6c02712008-06-26 21:27:53 +02001/*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/pci.h>
21#include <linux/gfp.h>
22#include <linux/bitops.h>
Joerg Roedel7f265082008-12-12 13:50:21 +010023#include <linux/debugfs.h>
Joerg Roedelb6c02712008-06-26 21:27:53 +020024#include <linux/scatterlist.h>
FUJITA Tomonori51491362009-01-05 23:47:25 +090025#include <linux/dma-mapping.h>
Joerg Roedelb6c02712008-06-26 21:27:53 +020026#include <linux/iommu-helper.h>
Joerg Roedelc156e342008-12-02 18:13:27 +010027#include <linux/iommu.h>
Joerg Roedelb6c02712008-06-26 21:27:53 +020028#include <asm/proto.h>
FUJITA Tomonori46a7fa22008-07-11 10:23:42 +090029#include <asm/iommu.h>
Joerg Roedel1d9b16d2008-11-27 18:39:15 +010030#include <asm/gart.h>
Joerg Roedelb6c02712008-06-26 21:27:53 +020031#include <asm/amd_iommu_types.h>
Joerg Roedelc6da9922008-06-26 21:28:06 +020032#include <asm/amd_iommu.h>
Joerg Roedelb6c02712008-06-26 21:27:53 +020033
34#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
35
Joerg Roedel136f78a2008-07-11 17:14:27 +020036#define EXIT_LOOP_COUNT 10000000
37
Joerg Roedelb6c02712008-06-26 21:27:53 +020038static DEFINE_RWLOCK(amd_iommu_devtable_lock);
39
Joerg Roedelbd60b732008-09-11 10:24:48 +020040/* A list of preallocated protection domains */
41static LIST_HEAD(iommu_pd_list);
42static DEFINE_SPINLOCK(iommu_pd_list_lock);
43
Joerg Roedel26961ef2008-12-03 17:00:17 +010044#ifdef CONFIG_IOMMU_API
45static struct iommu_ops amd_iommu_ops;
46#endif
47
Joerg Roedel431b2a22008-07-11 17:14:22 +020048/*
49 * general struct to manage commands send to an IOMMU
50 */
Joerg Roedeld6449532008-07-11 17:14:28 +020051struct iommu_cmd {
Joerg Roedelb6c02712008-06-26 21:27:53 +020052 u32 data[4];
53};
54
Joerg Roedelbd0e5212008-06-26 21:27:56 +020055static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
56 struct unity_map_entry *e);
Joerg Roedele275a2a2008-12-10 18:27:25 +010057static struct dma_ops_domain *find_protection_domain(u16 devid);
Joerg Roedel8bda3092009-05-12 12:02:46 +020058static u64* alloc_pte(struct protection_domain *dom,
59 unsigned long address, u64
60 **pte_page, gfp_t gfp);
Joerg Roedelbd0e5212008-06-26 21:27:56 +020061
Joerg Roedel7f265082008-12-12 13:50:21 +010062#ifdef CONFIG_AMD_IOMMU_STATS
63
64/*
65 * Initialization code for statistics collection
66 */
67
Joerg Roedelda49f6d2008-12-12 14:59:58 +010068DECLARE_STATS_COUNTER(compl_wait);
Joerg Roedel0f2a86f2008-12-12 15:05:16 +010069DECLARE_STATS_COUNTER(cnt_map_single);
Joerg Roedel146a6912008-12-12 15:07:12 +010070DECLARE_STATS_COUNTER(cnt_unmap_single);
Joerg Roedeld03f0672008-12-12 15:09:48 +010071DECLARE_STATS_COUNTER(cnt_map_sg);
Joerg Roedel55877a62008-12-12 15:12:14 +010072DECLARE_STATS_COUNTER(cnt_unmap_sg);
Joerg Roedelc8f0fb32008-12-12 15:14:21 +010073DECLARE_STATS_COUNTER(cnt_alloc_coherent);
Joerg Roedel5d31ee72008-12-12 15:16:38 +010074DECLARE_STATS_COUNTER(cnt_free_coherent);
Joerg Roedelc1858972008-12-12 15:42:39 +010075DECLARE_STATS_COUNTER(cross_page);
Joerg Roedelf57d98a2008-12-12 15:46:29 +010076DECLARE_STATS_COUNTER(domain_flush_single);
Joerg Roedel18811f52008-12-12 15:48:28 +010077DECLARE_STATS_COUNTER(domain_flush_all);
Joerg Roedel5774f7c2008-12-12 15:57:30 +010078DECLARE_STATS_COUNTER(alloced_io_mem);
Joerg Roedel8ecaf8f2008-12-12 16:13:04 +010079DECLARE_STATS_COUNTER(total_map_requests);
Joerg Roedelda49f6d2008-12-12 14:59:58 +010080
Joerg Roedel7f265082008-12-12 13:50:21 +010081static struct dentry *stats_dir;
82static struct dentry *de_isolate;
83static struct dentry *de_fflush;
84
85static void amd_iommu_stats_add(struct __iommu_counter *cnt)
86{
87 if (stats_dir == NULL)
88 return;
89
90 cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
91 &cnt->value);
92}
93
94static void amd_iommu_stats_init(void)
95{
96 stats_dir = debugfs_create_dir("amd-iommu", NULL);
97 if (stats_dir == NULL)
98 return;
99
100 de_isolate = debugfs_create_bool("isolation", 0444, stats_dir,
101 (u32 *)&amd_iommu_isolate);
102
103 de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
104 (u32 *)&amd_iommu_unmap_flush);
Joerg Roedelda49f6d2008-12-12 14:59:58 +0100105
106 amd_iommu_stats_add(&compl_wait);
Joerg Roedel0f2a86f2008-12-12 15:05:16 +0100107 amd_iommu_stats_add(&cnt_map_single);
Joerg Roedel146a6912008-12-12 15:07:12 +0100108 amd_iommu_stats_add(&cnt_unmap_single);
Joerg Roedeld03f0672008-12-12 15:09:48 +0100109 amd_iommu_stats_add(&cnt_map_sg);
Joerg Roedel55877a62008-12-12 15:12:14 +0100110 amd_iommu_stats_add(&cnt_unmap_sg);
Joerg Roedelc8f0fb32008-12-12 15:14:21 +0100111 amd_iommu_stats_add(&cnt_alloc_coherent);
Joerg Roedel5d31ee72008-12-12 15:16:38 +0100112 amd_iommu_stats_add(&cnt_free_coherent);
Joerg Roedelc1858972008-12-12 15:42:39 +0100113 amd_iommu_stats_add(&cross_page);
Joerg Roedelf57d98a2008-12-12 15:46:29 +0100114 amd_iommu_stats_add(&domain_flush_single);
Joerg Roedel18811f52008-12-12 15:48:28 +0100115 amd_iommu_stats_add(&domain_flush_all);
Joerg Roedel5774f7c2008-12-12 15:57:30 +0100116 amd_iommu_stats_add(&alloced_io_mem);
Joerg Roedel8ecaf8f2008-12-12 16:13:04 +0100117 amd_iommu_stats_add(&total_map_requests);
Joerg Roedel7f265082008-12-12 13:50:21 +0100118}
119
120#endif
121
Joerg Roedel431b2a22008-07-11 17:14:22 +0200122/* returns !0 if the IOMMU is caching non-present entries in its TLB */
Joerg Roedel4da70b92008-06-26 21:28:01 +0200123static int iommu_has_npcache(struct amd_iommu *iommu)
124{
Joerg Roedelae9b9402008-10-30 17:43:57 +0100125 return iommu->cap & (1UL << IOMMU_CAP_NPCACHE);
Joerg Roedel4da70b92008-06-26 21:28:01 +0200126}
127
Joerg Roedel431b2a22008-07-11 17:14:22 +0200128/****************************************************************************
129 *
Joerg Roedela80dc3e2008-09-11 16:51:41 +0200130 * Interrupt handling functions
131 *
132 ****************************************************************************/
133
Joerg Roedel90008ee2008-09-09 16:41:05 +0200134static void iommu_print_event(void *__evt)
135{
136 u32 *event = __evt;
137 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
138 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
139 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
140 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
141 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
142
143 printk(KERN_ERR "AMD IOMMU: Event logged [");
144
145 switch (type) {
146 case EVENT_TYPE_ILL_DEV:
147 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
148 "address=0x%016llx flags=0x%04x]\n",
149 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
150 address, flags);
151 break;
152 case EVENT_TYPE_IO_FAULT:
153 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
154 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
155 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
156 domid, address, flags);
157 break;
158 case EVENT_TYPE_DEV_TAB_ERR:
159 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
160 "address=0x%016llx flags=0x%04x]\n",
161 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
162 address, flags);
163 break;
164 case EVENT_TYPE_PAGE_TAB_ERR:
165 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
166 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
167 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
168 domid, address, flags);
169 break;
170 case EVENT_TYPE_ILL_CMD:
171 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
172 break;
173 case EVENT_TYPE_CMD_HARD_ERR:
174 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
175 "flags=0x%04x]\n", address, flags);
176 break;
177 case EVENT_TYPE_IOTLB_INV_TO:
178 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
179 "address=0x%016llx]\n",
180 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
181 address);
182 break;
183 case EVENT_TYPE_INV_DEV_REQ:
184 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
185 "address=0x%016llx flags=0x%04x]\n",
186 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
187 address, flags);
188 break;
189 default:
190 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
191 }
192}
193
194static void iommu_poll_events(struct amd_iommu *iommu)
195{
196 u32 head, tail;
197 unsigned long flags;
198
199 spin_lock_irqsave(&iommu->lock, flags);
200
201 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
202 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
203
204 while (head != tail) {
205 iommu_print_event(iommu->evt_buf + head);
206 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
207 }
208
209 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
210
211 spin_unlock_irqrestore(&iommu->lock, flags);
212}
213
Joerg Roedela80dc3e2008-09-11 16:51:41 +0200214irqreturn_t amd_iommu_int_handler(int irq, void *data)
215{
Joerg Roedel90008ee2008-09-09 16:41:05 +0200216 struct amd_iommu *iommu;
217
218 list_for_each_entry(iommu, &amd_iommu_list, list)
219 iommu_poll_events(iommu);
220
221 return IRQ_HANDLED;
Joerg Roedela80dc3e2008-09-11 16:51:41 +0200222}
223
224/****************************************************************************
225 *
Joerg Roedel431b2a22008-07-11 17:14:22 +0200226 * IOMMU command queuing functions
227 *
228 ****************************************************************************/
229
230/*
231 * Writes the command to the IOMMUs command buffer and informs the
232 * hardware about the new command. Must be called with iommu->lock held.
233 */
Joerg Roedeld6449532008-07-11 17:14:28 +0200234static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200235{
236 u32 tail, head;
237 u8 *target;
238
239 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
Jiri Kosina8a7c5ef2008-08-19 02:13:55 +0200240 target = iommu->cmd_buf + tail;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200241 memcpy_toio(target, cmd, sizeof(*cmd));
242 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
243 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
244 if (tail == head)
245 return -ENOMEM;
246 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
247
248 return 0;
249}
250
Joerg Roedel431b2a22008-07-11 17:14:22 +0200251/*
252 * General queuing function for commands. Takes iommu->lock and calls
253 * __iommu_queue_command().
254 */
Joerg Roedeld6449532008-07-11 17:14:28 +0200255static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200256{
257 unsigned long flags;
258 int ret;
259
260 spin_lock_irqsave(&iommu->lock, flags);
261 ret = __iommu_queue_command(iommu, cmd);
Joerg Roedel09ee17e2008-12-03 12:19:27 +0100262 if (!ret)
Joerg Roedel0cfd7aa2008-12-10 19:58:00 +0100263 iommu->need_sync = true;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200264 spin_unlock_irqrestore(&iommu->lock, flags);
265
266 return ret;
267}
268
Joerg Roedel431b2a22008-07-11 17:14:22 +0200269/*
Joerg Roedel8d201962008-12-02 20:34:41 +0100270 * This function waits until an IOMMU has completed a completion
271 * wait command
Joerg Roedel431b2a22008-07-11 17:14:22 +0200272 */
Joerg Roedel8d201962008-12-02 20:34:41 +0100273static void __iommu_wait_for_completion(struct amd_iommu *iommu)
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200274{
Joerg Roedel8d201962008-12-02 20:34:41 +0100275 int ready = 0;
Joerg Roedel519c31b2008-08-14 19:55:15 +0200276 unsigned status = 0;
Joerg Roedel8d201962008-12-02 20:34:41 +0100277 unsigned long i = 0;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200278
Joerg Roedelda49f6d2008-12-12 14:59:58 +0100279 INC_STATS_COUNTER(compl_wait);
280
Joerg Roedel136f78a2008-07-11 17:14:27 +0200281 while (!ready && (i < EXIT_LOOP_COUNT)) {
282 ++i;
Joerg Roedel519c31b2008-08-14 19:55:15 +0200283 /* wait for the bit to become one */
284 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
285 ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
Joerg Roedel136f78a2008-07-11 17:14:27 +0200286 }
287
Joerg Roedel519c31b2008-08-14 19:55:15 +0200288 /* set bit back to zero */
289 status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
290 writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
291
Joerg Roedel84df8172008-12-17 16:36:44 +0100292 if (unlikely(i == EXIT_LOOP_COUNT))
293 panic("AMD IOMMU: Completion wait loop failed\n");
Joerg Roedel8d201962008-12-02 20:34:41 +0100294}
295
296/*
297 * This function queues a completion wait command into the command
298 * buffer of an IOMMU
299 */
300static int __iommu_completion_wait(struct amd_iommu *iommu)
301{
302 struct iommu_cmd cmd;
303
304 memset(&cmd, 0, sizeof(cmd));
305 cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
306 CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
307
308 return __iommu_queue_command(iommu, &cmd);
309}
310
311/*
312 * This function is called whenever we need to ensure that the IOMMU has
313 * completed execution of all commands we sent. It sends a
314 * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
315 * us about that by writing a value to a physical address we pass with
316 * the command.
317 */
318static int iommu_completion_wait(struct amd_iommu *iommu)
319{
320 int ret = 0;
321 unsigned long flags;
322
323 spin_lock_irqsave(&iommu->lock, flags);
324
325 if (!iommu->need_sync)
326 goto out;
327
328 ret = __iommu_completion_wait(iommu);
329
Joerg Roedel0cfd7aa2008-12-10 19:58:00 +0100330 iommu->need_sync = false;
Joerg Roedel8d201962008-12-02 20:34:41 +0100331
332 if (ret)
333 goto out;
334
335 __iommu_wait_for_completion(iommu);
Joerg Roedel84df8172008-12-17 16:36:44 +0100336
Joerg Roedel7e4f88d2008-09-17 14:19:15 +0200337out:
338 spin_unlock_irqrestore(&iommu->lock, flags);
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200339
340 return 0;
341}
342
Joerg Roedel431b2a22008-07-11 17:14:22 +0200343/*
344 * Command send function for invalidating a device table entry
345 */
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200346static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
347{
Joerg Roedeld6449532008-07-11 17:14:28 +0200348 struct iommu_cmd cmd;
Joerg Roedelee2fa742008-09-17 13:47:25 +0200349 int ret;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200350
351 BUG_ON(iommu == NULL);
352
353 memset(&cmd, 0, sizeof(cmd));
354 CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
355 cmd.data[0] = devid;
356
Joerg Roedelee2fa742008-09-17 13:47:25 +0200357 ret = iommu_queue_command(iommu, &cmd);
358
Joerg Roedelee2fa742008-09-17 13:47:25 +0200359 return ret;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200360}
361
Joerg Roedel237b6f32008-12-02 20:54:37 +0100362static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
363 u16 domid, int pde, int s)
364{
365 memset(cmd, 0, sizeof(*cmd));
366 address &= PAGE_MASK;
367 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
368 cmd->data[1] |= domid;
369 cmd->data[2] = lower_32_bits(address);
370 cmd->data[3] = upper_32_bits(address);
371 if (s) /* size bit - we flush more than one 4kb page */
372 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
373 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
374 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
375}
376
Joerg Roedel431b2a22008-07-11 17:14:22 +0200377/*
378 * Generic command send function for invalidaing TLB entries
379 */
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200380static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
381 u64 address, u16 domid, int pde, int s)
382{
Joerg Roedeld6449532008-07-11 17:14:28 +0200383 struct iommu_cmd cmd;
Joerg Roedelee2fa742008-09-17 13:47:25 +0200384 int ret;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200385
Joerg Roedel237b6f32008-12-02 20:54:37 +0100386 __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200387
Joerg Roedelee2fa742008-09-17 13:47:25 +0200388 ret = iommu_queue_command(iommu, &cmd);
389
Joerg Roedelee2fa742008-09-17 13:47:25 +0200390 return ret;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200391}
392
Joerg Roedel431b2a22008-07-11 17:14:22 +0200393/*
394 * TLB invalidation function which is called from the mapping functions.
395 * It invalidates a single PTE if the range to flush is within a single
396 * page. Otherwise it flushes the whole TLB of the IOMMU.
397 */
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200398static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
399 u64 address, size_t size)
400{
Joerg Roedel999ba412008-07-03 19:35:08 +0200401 int s = 0;
Joerg Roedele3c449f2008-10-15 22:02:11 -0700402 unsigned pages = iommu_num_pages(address, size, PAGE_SIZE);
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200403
404 address &= PAGE_MASK;
405
Joerg Roedel999ba412008-07-03 19:35:08 +0200406 if (pages > 1) {
407 /*
408 * If we have to flush more than one page, flush all
409 * TLB entries for this domain
410 */
411 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
412 s = 1;
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200413 }
414
Joerg Roedel999ba412008-07-03 19:35:08 +0200415 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, s);
416
Joerg Roedela19ae1e2008-06-26 21:27:55 +0200417 return 0;
418}
Joerg Roedelb6c02712008-06-26 21:27:53 +0200419
Joerg Roedel1c655772008-09-04 18:40:05 +0200420/* Flush the whole IO/TLB for a given protection domain */
421static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
422{
423 u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
424
Joerg Roedelf57d98a2008-12-12 15:46:29 +0100425 INC_STATS_COUNTER(domain_flush_single);
426
Joerg Roedel1c655772008-09-04 18:40:05 +0200427 iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
428}
429
Joerg Roedel43f49602008-12-02 21:01:12 +0100430/*
431 * This function is used to flush the IO/TLB for a given protection domain
432 * on every IOMMU in the system
433 */
434static void iommu_flush_domain(u16 domid)
435{
436 unsigned long flags;
437 struct amd_iommu *iommu;
438 struct iommu_cmd cmd;
439
Joerg Roedel18811f52008-12-12 15:48:28 +0100440 INC_STATS_COUNTER(domain_flush_all);
441
Joerg Roedel43f49602008-12-02 21:01:12 +0100442 __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
443 domid, 1, 1);
444
445 list_for_each_entry(iommu, &amd_iommu_list, list) {
446 spin_lock_irqsave(&iommu->lock, flags);
447 __iommu_queue_command(iommu, &cmd);
448 __iommu_completion_wait(iommu);
449 __iommu_wait_for_completion(iommu);
450 spin_unlock_irqrestore(&iommu->lock, flags);
451 }
452}
Joerg Roedel43f49602008-12-02 21:01:12 +0100453
Joerg Roedel431b2a22008-07-11 17:14:22 +0200454/****************************************************************************
455 *
456 * The functions below are used the create the page table mappings for
457 * unity mapped regions.
458 *
459 ****************************************************************************/
460
461/*
462 * Generic mapping functions. It maps a physical address into a DMA
463 * address space. It allocates the page table pages if necessary.
464 * In the future it can be extended to a generic mapping function
465 * supporting all features of AMD IOMMU page tables like level skipping
466 * and full 64 bit address spaces.
467 */
Joerg Roedel38e817f2008-12-02 17:27:52 +0100468static int iommu_map_page(struct protection_domain *dom,
469 unsigned long bus_addr,
470 unsigned long phys_addr,
471 int prot)
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200472{
Joerg Roedel8bda3092009-05-12 12:02:46 +0200473 u64 __pte, *pte;
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200474
475 bus_addr = PAGE_ALIGN(bus_addr);
Joerg Roedelbb9d4ff2008-12-04 15:59:48 +0100476 phys_addr = PAGE_ALIGN(phys_addr);
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200477
478 /* only support 512GB address spaces for now */
479 if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))
480 return -EINVAL;
481
Joerg Roedel8bda3092009-05-12 12:02:46 +0200482 pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL);
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200483
484 if (IOMMU_PTE_PRESENT(*pte))
485 return -EBUSY;
486
487 __pte = phys_addr | IOMMU_PTE_P;
488 if (prot & IOMMU_PROT_IR)
489 __pte |= IOMMU_PTE_IR;
490 if (prot & IOMMU_PROT_IW)
491 __pte |= IOMMU_PTE_IW;
492
493 *pte = __pte;
494
495 return 0;
496}
497
Joerg Roedeleb74ff62008-12-02 19:59:10 +0100498static void iommu_unmap_page(struct protection_domain *dom,
499 unsigned long bus_addr)
500{
501 u64 *pte;
502
503 pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];
504
505 if (!IOMMU_PTE_PRESENT(*pte))
506 return;
507
508 pte = IOMMU_PTE_PAGE(*pte);
509 pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
510
511 if (!IOMMU_PTE_PRESENT(*pte))
512 return;
513
514 pte = IOMMU_PTE_PAGE(*pte);
515 pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];
516
517 *pte = 0;
518}
Joerg Roedeleb74ff62008-12-02 19:59:10 +0100519
Joerg Roedel431b2a22008-07-11 17:14:22 +0200520/*
521 * This function checks if a specific unity mapping entry is needed for
522 * this specific IOMMU.
523 */
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200524static int iommu_for_unity_map(struct amd_iommu *iommu,
525 struct unity_map_entry *entry)
526{
527 u16 bdf, i;
528
529 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
530 bdf = amd_iommu_alias_table[i];
531 if (amd_iommu_rlookup_table[bdf] == iommu)
532 return 1;
533 }
534
535 return 0;
536}
537
Joerg Roedel431b2a22008-07-11 17:14:22 +0200538/*
539 * Init the unity mappings for a specific IOMMU in the system
540 *
541 * Basically iterates over all unity mapping entries and applies them to
542 * the default domain DMA of that IOMMU if necessary.
543 */
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200544static int iommu_init_unity_mappings(struct amd_iommu *iommu)
545{
546 struct unity_map_entry *entry;
547 int ret;
548
549 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
550 if (!iommu_for_unity_map(iommu, entry))
551 continue;
552 ret = dma_ops_unity_map(iommu->default_dom, entry);
553 if (ret)
554 return ret;
555 }
556
557 return 0;
558}
559
Joerg Roedel431b2a22008-07-11 17:14:22 +0200560/*
561 * This function actually applies the mapping to the page table of the
562 * dma_ops domain.
563 */
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200564static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
565 struct unity_map_entry *e)
566{
567 u64 addr;
568 int ret;
569
570 for (addr = e->address_start; addr < e->address_end;
571 addr += PAGE_SIZE) {
Joerg Roedel38e817f2008-12-02 17:27:52 +0100572 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot);
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200573 if (ret)
574 return ret;
575 /*
576 * if unity mapping is in aperture range mark the page
577 * as allocated in the aperture
578 */
579 if (addr < dma_dom->aperture_size)
Joerg Roedelc3239562009-05-12 10:56:44 +0200580 __set_bit(addr >> PAGE_SHIFT,
Joerg Roedel384de722009-05-15 12:30:05 +0200581 dma_dom->aperture[0]->bitmap);
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200582 }
583
584 return 0;
585}
586
Joerg Roedel431b2a22008-07-11 17:14:22 +0200587/*
588 * Inits the unity mappings required for a specific device
589 */
Joerg Roedelbd0e5212008-06-26 21:27:56 +0200590static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
591 u16 devid)
592{
593 struct unity_map_entry *e;
594 int ret;
595
596 list_for_each_entry(e, &amd_iommu_unity_map, list) {
597 if (!(devid >= e->devid_start && devid <= e->devid_end))
598 continue;
599 ret = dma_ops_unity_map(dma_dom, e);
600 if (ret)
601 return ret;
602 }
603
604 return 0;
605}
606
Joerg Roedel431b2a22008-07-11 17:14:22 +0200607/****************************************************************************
608 *
609 * The next functions belong to the address allocator for the dma_ops
610 * interface functions. They work like the allocators in the other IOMMU
611 * drivers. Its basically a bitmap which marks the allocated pages in
612 * the aperture. Maybe it could be enhanced in the future to a more
613 * efficient allocator.
614 *
615 ****************************************************************************/
Joerg Roedeld3086442008-06-26 21:27:57 +0200616
Joerg Roedel431b2a22008-07-11 17:14:22 +0200617/*
Joerg Roedel384de722009-05-15 12:30:05 +0200618 * The address allocator core functions.
Joerg Roedel431b2a22008-07-11 17:14:22 +0200619 *
620 * called with domain->lock held
621 */
Joerg Roedel384de722009-05-15 12:30:05 +0200622
623static unsigned long dma_ops_area_alloc(struct device *dev,
624 struct dma_ops_domain *dom,
625 unsigned int pages,
626 unsigned long align_mask,
627 u64 dma_mask,
628 unsigned long start)
629{
630 unsigned long next_bit = dom->next_bit % APERTURE_RANGE_PAGES;
631 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
632 int i = start >> APERTURE_RANGE_SHIFT;
633 unsigned long boundary_size;
634 unsigned long address = -1;
635 unsigned long limit;
636
637 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
638 PAGE_SIZE) >> PAGE_SHIFT;
639
640 for (;i < max_index; ++i) {
641 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
642
643 if (dom->aperture[i]->offset >= dma_mask)
644 break;
645
646 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
647 dma_mask >> PAGE_SHIFT);
648
649 address = iommu_area_alloc(dom->aperture[i]->bitmap,
650 limit, next_bit, pages, 0,
651 boundary_size, align_mask);
652 if (address != -1) {
653 address = dom->aperture[i]->offset +
654 (address << PAGE_SHIFT);
655 dom->next_bit = (address >> PAGE_SHIFT) + pages;
656 break;
657 }
658
659 next_bit = 0;
660 }
661
662 return address;
663}
664
Joerg Roedeld3086442008-06-26 21:27:57 +0200665static unsigned long dma_ops_alloc_addresses(struct device *dev,
666 struct dma_ops_domain *dom,
Joerg Roedel6d4f3432008-09-04 19:18:02 +0200667 unsigned int pages,
Joerg Roedel832a90c2008-09-18 15:54:23 +0200668 unsigned long align_mask,
669 u64 dma_mask)
Joerg Roedeld3086442008-06-26 21:27:57 +0200670{
Joerg Roedeld3086442008-06-26 21:27:57 +0200671 unsigned long address;
Joerg Roedel384de722009-05-15 12:30:05 +0200672 unsigned long start = dom->next_bit << PAGE_SHIFT;
Joerg Roedeld3086442008-06-26 21:27:57 +0200673
Joerg Roedeld3086442008-06-26 21:27:57 +0200674
Joerg Roedel384de722009-05-15 12:30:05 +0200675 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
676 dma_mask, start);
Joerg Roedeld3086442008-06-26 21:27:57 +0200677
Joerg Roedel1c655772008-09-04 18:40:05 +0200678 if (address == -1) {
Joerg Roedel384de722009-05-15 12:30:05 +0200679 dom->next_bit = 0;
680 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
681 dma_mask, 0);
Joerg Roedel1c655772008-09-04 18:40:05 +0200682 dom->need_flush = true;
683 }
Joerg Roedeld3086442008-06-26 21:27:57 +0200684
Joerg Roedel384de722009-05-15 12:30:05 +0200685 if (unlikely(address == -1))
Joerg Roedeld3086442008-06-26 21:27:57 +0200686 address = bad_dma_address;
687
688 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
689
690 return address;
691}
692
Joerg Roedel431b2a22008-07-11 17:14:22 +0200693/*
694 * The address free function.
695 *
696 * called with domain->lock held
697 */
Joerg Roedeld3086442008-06-26 21:27:57 +0200698static void dma_ops_free_addresses(struct dma_ops_domain *dom,
699 unsigned long address,
700 unsigned int pages)
701{
Joerg Roedel384de722009-05-15 12:30:05 +0200702 unsigned i = address >> APERTURE_RANGE_SHIFT;
703 struct aperture_range *range = dom->aperture[i];
Joerg Roedel80be3082008-11-06 14:59:05 +0100704
Joerg Roedel384de722009-05-15 12:30:05 +0200705 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
706
707 if ((address >> PAGE_SHIFT) >= dom->next_bit)
Joerg Roedel80be3082008-11-06 14:59:05 +0100708 dom->need_flush = true;
Joerg Roedel384de722009-05-15 12:30:05 +0200709
710 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
711 iommu_area_free(range->bitmap, address, pages);
712
Joerg Roedeld3086442008-06-26 21:27:57 +0200713}
714
Joerg Roedel431b2a22008-07-11 17:14:22 +0200715/****************************************************************************
716 *
717 * The next functions belong to the domain allocation. A domain is
718 * allocated for every IOMMU as the default domain. If device isolation
719 * is enabled, every device get its own domain. The most important thing
720 * about domains is the page table mapping the DMA address space they
721 * contain.
722 *
723 ****************************************************************************/
724
Joerg Roedelec487d12008-06-26 21:27:58 +0200725static u16 domain_id_alloc(void)
726{
727 unsigned long flags;
728 int id;
729
730 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
731 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
732 BUG_ON(id == 0);
733 if (id > 0 && id < MAX_DOMAIN_ID)
734 __set_bit(id, amd_iommu_pd_alloc_bitmap);
735 else
736 id = 0;
737 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
738
739 return id;
740}
741
Joerg Roedela2acfb72008-12-02 18:28:53 +0100742static void domain_id_free(int id)
743{
744 unsigned long flags;
745
746 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
747 if (id > 0 && id < MAX_DOMAIN_ID)
748 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
749 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
750}
Joerg Roedela2acfb72008-12-02 18:28:53 +0100751
Joerg Roedel431b2a22008-07-11 17:14:22 +0200752/*
753 * Used to reserve address ranges in the aperture (e.g. for exclusion
754 * ranges.
755 */
Joerg Roedelec487d12008-06-26 21:27:58 +0200756static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
757 unsigned long start_page,
758 unsigned int pages)
759{
Joerg Roedel384de722009-05-15 12:30:05 +0200760 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
Joerg Roedelec487d12008-06-26 21:27:58 +0200761
762 if (start_page + pages > last_page)
763 pages = last_page - start_page;
764
Joerg Roedel384de722009-05-15 12:30:05 +0200765 for (i = start_page; i < start_page + pages; ++i) {
766 int index = i / APERTURE_RANGE_PAGES;
767 int page = i % APERTURE_RANGE_PAGES;
768 __set_bit(page, dom->aperture[index]->bitmap);
769 }
Joerg Roedelec487d12008-06-26 21:27:58 +0200770}
771
Joerg Roedel86db2e52008-12-02 18:20:21 +0100772static void free_pagetable(struct protection_domain *domain)
Joerg Roedelec487d12008-06-26 21:27:58 +0200773{
774 int i, j;
775 u64 *p1, *p2, *p3;
776
Joerg Roedel86db2e52008-12-02 18:20:21 +0100777 p1 = domain->pt_root;
Joerg Roedelec487d12008-06-26 21:27:58 +0200778
779 if (!p1)
780 return;
781
782 for (i = 0; i < 512; ++i) {
783 if (!IOMMU_PTE_PRESENT(p1[i]))
784 continue;
785
786 p2 = IOMMU_PTE_PAGE(p1[i]);
Joerg Roedel3cc3d842008-12-04 16:44:31 +0100787 for (j = 0; j < 512; ++j) {
Joerg Roedelec487d12008-06-26 21:27:58 +0200788 if (!IOMMU_PTE_PRESENT(p2[j]))
789 continue;
790 p3 = IOMMU_PTE_PAGE(p2[j]);
791 free_page((unsigned long)p3);
792 }
793
794 free_page((unsigned long)p2);
795 }
796
797 free_page((unsigned long)p1);
Joerg Roedel86db2e52008-12-02 18:20:21 +0100798
799 domain->pt_root = NULL;
Joerg Roedelec487d12008-06-26 21:27:58 +0200800}
801
Joerg Roedel431b2a22008-07-11 17:14:22 +0200802/*
803 * Free a domain, only used if something went wrong in the
804 * allocation path and we need to free an already allocated page table
805 */
Joerg Roedelec487d12008-06-26 21:27:58 +0200806static void dma_ops_domain_free(struct dma_ops_domain *dom)
807{
Joerg Roedel384de722009-05-15 12:30:05 +0200808 int i;
809
Joerg Roedelec487d12008-06-26 21:27:58 +0200810 if (!dom)
811 return;
812
Joerg Roedel86db2e52008-12-02 18:20:21 +0100813 free_pagetable(&dom->domain);
Joerg Roedelec487d12008-06-26 21:27:58 +0200814
Joerg Roedel384de722009-05-15 12:30:05 +0200815 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
816 if (!dom->aperture[i])
817 continue;
818 free_page((unsigned long)dom->aperture[i]->bitmap);
819 kfree(dom->aperture[i]);
820 }
Joerg Roedelec487d12008-06-26 21:27:58 +0200821
822 kfree(dom);
823}
824
Joerg Roedel431b2a22008-07-11 17:14:22 +0200825/*
826 * Allocates a new protection domain usable for the dma_ops functions.
827 * It also intializes the page table and the address allocator data
828 * structures required for the dma_ops interface
829 */
Joerg Roedelec487d12008-06-26 21:27:58 +0200830static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
831 unsigned order)
832{
833 struct dma_ops_domain *dma_dom;
834 unsigned i, num_pte_pages;
835 u64 *l2_pde;
836 u64 address;
837
838 /*
839 * Currently the DMA aperture must be between 32 MB and 1GB in size
840 */
841 if ((order < 25) || (order > 30))
842 return NULL;
843
844 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
845 if (!dma_dom)
846 return NULL;
847
Joerg Roedel384de722009-05-15 12:30:05 +0200848 dma_dom->aperture[0] = kzalloc(sizeof(struct aperture_range),
849 GFP_KERNEL);
850 if (!dma_dom->aperture[0])
851 goto free_dma_dom;
852
Joerg Roedelec487d12008-06-26 21:27:58 +0200853 spin_lock_init(&dma_dom->domain.lock);
854
855 dma_dom->domain.id = domain_id_alloc();
856 if (dma_dom->domain.id == 0)
857 goto free_dma_dom;
858 dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
859 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
Joerg Roedel9fdb19d2008-12-02 17:46:25 +0100860 dma_dom->domain.flags = PD_DMA_OPS_MASK;
Joerg Roedelec487d12008-06-26 21:27:58 +0200861 dma_dom->domain.priv = dma_dom;
862 if (!dma_dom->domain.pt_root)
863 goto free_dma_dom;
Joerg Roedelc3239562009-05-12 10:56:44 +0200864 dma_dom->aperture_size = APERTURE_RANGE_SIZE;
Joerg Roedel384de722009-05-15 12:30:05 +0200865 dma_dom->aperture[0]->bitmap = (void *)get_zeroed_page(GFP_KERNEL);
866 if (!dma_dom->aperture[0]->bitmap)
Joerg Roedelec487d12008-06-26 21:27:58 +0200867 goto free_dma_dom;
868 /*
869 * mark the first page as allocated so we never return 0 as
870 * a valid dma-address. So we can use 0 as error value
871 */
Joerg Roedel384de722009-05-15 12:30:05 +0200872 dma_dom->aperture[0]->bitmap[0] = 1;
Joerg Roedelec487d12008-06-26 21:27:58 +0200873 dma_dom->next_bit = 0;
874
Joerg Roedel1c655772008-09-04 18:40:05 +0200875 dma_dom->need_flush = false;
Joerg Roedelbd60b732008-09-11 10:24:48 +0200876 dma_dom->target_dev = 0xffff;
Joerg Roedel1c655772008-09-04 18:40:05 +0200877
Joerg Roedel431b2a22008-07-11 17:14:22 +0200878 /* Intialize the exclusion range if necessary */
Joerg Roedelec487d12008-06-26 21:27:58 +0200879 if (iommu->exclusion_start &&
880 iommu->exclusion_start < dma_dom->aperture_size) {
881 unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
Joerg Roedele3c449f2008-10-15 22:02:11 -0700882 int pages = iommu_num_pages(iommu->exclusion_start,
883 iommu->exclusion_length,
884 PAGE_SIZE);
Joerg Roedelec487d12008-06-26 21:27:58 +0200885 dma_ops_reserve_addresses(dma_dom, startpage, pages);
886 }
887
Joerg Roedel431b2a22008-07-11 17:14:22 +0200888 /*
889 * At the last step, build the page tables so we don't need to
890 * allocate page table pages in the dma_ops mapping/unmapping
Joerg Roedelc3239562009-05-12 10:56:44 +0200891 * path for the first 128MB of dma address space.
Joerg Roedel431b2a22008-07-11 17:14:22 +0200892 */
Joerg Roedelec487d12008-06-26 21:27:58 +0200893 num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);
Joerg Roedelec487d12008-06-26 21:27:58 +0200894
895 l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);
896 if (l2_pde == NULL)
897 goto free_dma_dom;
898
899 dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
900
901 for (i = 0; i < num_pte_pages; ++i) {
Joerg Roedel384de722009-05-15 12:30:05 +0200902 u64 **pte_page = &dma_dom->aperture[0]->pte_pages[i];
Joerg Roedelc3239562009-05-12 10:56:44 +0200903 *pte_page = (u64 *)get_zeroed_page(GFP_KERNEL);
904 if (!*pte_page)
Joerg Roedelec487d12008-06-26 21:27:58 +0200905 goto free_dma_dom;
Joerg Roedelc3239562009-05-12 10:56:44 +0200906 address = virt_to_phys(*pte_page);
Joerg Roedelec487d12008-06-26 21:27:58 +0200907 l2_pde[i] = IOMMU_L1_PDE(address);
908 }
909
910 return dma_dom;
911
912free_dma_dom:
913 dma_ops_domain_free(dma_dom);
914
915 return NULL;
916}
917
Joerg Roedel431b2a22008-07-11 17:14:22 +0200918/*
Joerg Roedel5b28df62008-12-02 17:49:42 +0100919 * little helper function to check whether a given protection domain is a
920 * dma_ops domain
921 */
922static bool dma_ops_domain(struct protection_domain *domain)
923{
924 return domain->flags & PD_DMA_OPS_MASK;
925}
926
927/*
Joerg Roedel431b2a22008-07-11 17:14:22 +0200928 * Find out the protection domain structure for a given PCI device. This
929 * will give us the pointer to the page table root for example.
930 */
Joerg Roedelb20ac0d2008-06-26 21:27:59 +0200931static struct protection_domain *domain_for_device(u16 devid)
932{
933 struct protection_domain *dom;
934 unsigned long flags;
935
936 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
937 dom = amd_iommu_pd_table[devid];
938 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
939
940 return dom;
941}
942
Joerg Roedel431b2a22008-07-11 17:14:22 +0200943/*
944 * If a device is not yet associated with a domain, this function does
945 * assigns it visible for the hardware
946 */
Joerg Roedelf1179dc2008-12-10 14:39:51 +0100947static void attach_device(struct amd_iommu *iommu,
948 struct protection_domain *domain,
949 u16 devid)
Joerg Roedelb20ac0d2008-06-26 21:27:59 +0200950{
951 unsigned long flags;
Joerg Roedelb20ac0d2008-06-26 21:27:59 +0200952 u64 pte_root = virt_to_phys(domain->pt_root);
953
Joerg Roedel863c74e2008-12-02 17:56:36 +0100954 domain->dev_cnt += 1;
955
Joerg Roedel38ddf412008-09-11 10:38:32 +0200956 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
957 << DEV_ENTRY_MODE_SHIFT;
958 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
Joerg Roedelb20ac0d2008-06-26 21:27:59 +0200959
960 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
Joerg Roedel38ddf412008-09-11 10:38:32 +0200961 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
962 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
Joerg Roedelb20ac0d2008-06-26 21:27:59 +0200963 amd_iommu_dev_table[devid].data[2] = domain->id;
964
965 amd_iommu_pd_table[devid] = domain;
966 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
967
968 iommu_queue_inv_dev_entry(iommu, devid);
Joerg Roedelb20ac0d2008-06-26 21:27:59 +0200969}
970
Joerg Roedel355bf552008-12-08 12:02:41 +0100971/*
972 * Removes a device from a protection domain (unlocked)
973 */
974static void __detach_device(struct protection_domain *domain, u16 devid)
975{
976
977 /* lock domain */
978 spin_lock(&domain->lock);
979
980 /* remove domain from the lookup table */
981 amd_iommu_pd_table[devid] = NULL;
982
983 /* remove entry from the device table seen by the hardware */
984 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
985 amd_iommu_dev_table[devid].data[1] = 0;
986 amd_iommu_dev_table[devid].data[2] = 0;
987
988 /* decrease reference counter */
989 domain->dev_cnt -= 1;
990
991 /* ready */
992 spin_unlock(&domain->lock);
993}
994
995/*
996 * Removes a device from a protection domain (with devtable_lock held)
997 */
998static void detach_device(struct protection_domain *domain, u16 devid)
999{
1000 unsigned long flags;
1001
1002 /* lock device table */
1003 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1004 __detach_device(domain, devid);
1005 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1006}
Joerg Roedele275a2a2008-12-10 18:27:25 +01001007
1008static int device_change_notifier(struct notifier_block *nb,
1009 unsigned long action, void *data)
1010{
1011 struct device *dev = data;
1012 struct pci_dev *pdev = to_pci_dev(dev);
1013 u16 devid = calc_devid(pdev->bus->number, pdev->devfn);
1014 struct protection_domain *domain;
1015 struct dma_ops_domain *dma_domain;
1016 struct amd_iommu *iommu;
Joerg Roedel1ac4cbb2008-12-10 19:33:26 +01001017 int order = amd_iommu_aperture_order;
1018 unsigned long flags;
Joerg Roedele275a2a2008-12-10 18:27:25 +01001019
1020 if (devid > amd_iommu_last_bdf)
1021 goto out;
1022
1023 devid = amd_iommu_alias_table[devid];
1024
1025 iommu = amd_iommu_rlookup_table[devid];
1026 if (iommu == NULL)
1027 goto out;
1028
1029 domain = domain_for_device(devid);
1030
1031 if (domain && !dma_ops_domain(domain))
1032 WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
1033 "to a non-dma-ops domain\n", dev_name(dev));
1034
1035 switch (action) {
1036 case BUS_NOTIFY_BOUND_DRIVER:
1037 if (domain)
1038 goto out;
1039 dma_domain = find_protection_domain(devid);
1040 if (!dma_domain)
1041 dma_domain = iommu->default_dom;
1042 attach_device(iommu, &dma_domain->domain, devid);
1043 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
1044 "device %s\n", dma_domain->domain.id, dev_name(dev));
1045 break;
1046 case BUS_NOTIFY_UNBIND_DRIVER:
1047 if (!domain)
1048 goto out;
1049 detach_device(domain, devid);
1050 break;
Joerg Roedel1ac4cbb2008-12-10 19:33:26 +01001051 case BUS_NOTIFY_ADD_DEVICE:
1052 /* allocate a protection domain if a device is added */
1053 dma_domain = find_protection_domain(devid);
1054 if (dma_domain)
1055 goto out;
1056 dma_domain = dma_ops_domain_alloc(iommu, order);
1057 if (!dma_domain)
1058 goto out;
1059 dma_domain->target_dev = devid;
1060
1061 spin_lock_irqsave(&iommu_pd_list_lock, flags);
1062 list_add_tail(&dma_domain->list, &iommu_pd_list);
1063 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
1064
1065 break;
Joerg Roedele275a2a2008-12-10 18:27:25 +01001066 default:
1067 goto out;
1068 }
1069
1070 iommu_queue_inv_dev_entry(iommu, devid);
1071 iommu_completion_wait(iommu);
1072
1073out:
1074 return 0;
1075}
1076
1077struct notifier_block device_nb = {
1078 .notifier_call = device_change_notifier,
1079};
Joerg Roedel355bf552008-12-08 12:02:41 +01001080
Joerg Roedel431b2a22008-07-11 17:14:22 +02001081/*****************************************************************************
1082 *
1083 * The next functions belong to the dma_ops mapping/unmapping code.
1084 *
1085 *****************************************************************************/
1086
1087/*
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001088 * This function checks if the driver got a valid device from the caller to
1089 * avoid dereferencing invalid pointers.
1090 */
1091static bool check_device(struct device *dev)
1092{
1093 if (!dev || !dev->dma_mask)
1094 return false;
1095
1096 return true;
1097}
1098
1099/*
Joerg Roedelbd60b732008-09-11 10:24:48 +02001100 * In this function the list of preallocated protection domains is traversed to
1101 * find the domain for a specific device
1102 */
1103static struct dma_ops_domain *find_protection_domain(u16 devid)
1104{
1105 struct dma_ops_domain *entry, *ret = NULL;
1106 unsigned long flags;
1107
1108 if (list_empty(&iommu_pd_list))
1109 return NULL;
1110
1111 spin_lock_irqsave(&iommu_pd_list_lock, flags);
1112
1113 list_for_each_entry(entry, &iommu_pd_list, list) {
1114 if (entry->target_dev == devid) {
1115 ret = entry;
Joerg Roedelbd60b732008-09-11 10:24:48 +02001116 break;
1117 }
1118 }
1119
1120 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
1121
1122 return ret;
1123}
1124
1125/*
Joerg Roedel431b2a22008-07-11 17:14:22 +02001126 * In the dma_ops path we only have the struct device. This function
1127 * finds the corresponding IOMMU, the protection domain and the
1128 * requestor id for a given device.
1129 * If the device is not yet associated with a domain this is also done
1130 * in this function.
1131 */
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001132static int get_device_resources(struct device *dev,
1133 struct amd_iommu **iommu,
1134 struct protection_domain **domain,
1135 u16 *bdf)
1136{
1137 struct dma_ops_domain *dma_dom;
1138 struct pci_dev *pcidev;
1139 u16 _bdf;
1140
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001141 *iommu = NULL;
1142 *domain = NULL;
1143 *bdf = 0xffff;
1144
1145 if (dev->bus != &pci_bus_type)
1146 return 0;
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001147
1148 pcidev = to_pci_dev(dev);
Joerg Roedeld591b0a2008-07-11 17:14:35 +02001149 _bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001150
Joerg Roedel431b2a22008-07-11 17:14:22 +02001151 /* device not translated by any IOMMU in the system? */
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001152 if (_bdf > amd_iommu_last_bdf)
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001153 return 0;
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001154
1155 *bdf = amd_iommu_alias_table[_bdf];
1156
1157 *iommu = amd_iommu_rlookup_table[*bdf];
1158 if (*iommu == NULL)
1159 return 0;
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001160 *domain = domain_for_device(*bdf);
1161 if (*domain == NULL) {
Joerg Roedelbd60b732008-09-11 10:24:48 +02001162 dma_dom = find_protection_domain(*bdf);
1163 if (!dma_dom)
1164 dma_dom = (*iommu)->default_dom;
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001165 *domain = &dma_dom->domain;
Joerg Roedelf1179dc2008-12-10 14:39:51 +01001166 attach_device(*iommu, *domain, *bdf);
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001167 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
Joerg Roedelab896722008-12-10 19:43:07 +01001168 "device %s\n", (*domain)->id, dev_name(dev));
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001169 }
1170
Joerg Roedelf91ba192008-11-25 12:56:12 +01001171 if (domain_for_device(_bdf) == NULL)
Joerg Roedelf1179dc2008-12-10 14:39:51 +01001172 attach_device(*iommu, *domain, _bdf);
Joerg Roedelf91ba192008-11-25 12:56:12 +01001173
Joerg Roedelb20ac0d2008-06-26 21:27:59 +02001174 return 1;
1175}
1176
Joerg Roedel431b2a22008-07-11 17:14:22 +02001177/*
Joerg Roedel8bda3092009-05-12 12:02:46 +02001178 * If the pte_page is not yet allocated this function is called
1179 */
1180static u64* alloc_pte(struct protection_domain *dom,
1181 unsigned long address, u64 **pte_page, gfp_t gfp)
1182{
1183 u64 *pte, *page;
1184
1185 pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(address)];
1186
1187 if (!IOMMU_PTE_PRESENT(*pte)) {
1188 page = (u64 *)get_zeroed_page(gfp);
1189 if (!page)
1190 return NULL;
1191 *pte = IOMMU_L2_PDE(virt_to_phys(page));
1192 }
1193
1194 pte = IOMMU_PTE_PAGE(*pte);
1195 pte = &pte[IOMMU_PTE_L1_INDEX(address)];
1196
1197 if (!IOMMU_PTE_PRESENT(*pte)) {
1198 page = (u64 *)get_zeroed_page(gfp);
1199 if (!page)
1200 return NULL;
1201 *pte = IOMMU_L1_PDE(virt_to_phys(page));
1202 }
1203
1204 pte = IOMMU_PTE_PAGE(*pte);
1205
1206 if (pte_page)
1207 *pte_page = pte;
1208
1209 pte = &pte[IOMMU_PTE_L0_INDEX(address)];
1210
1211 return pte;
1212}
1213
1214/*
1215 * This function fetches the PTE for a given address in the aperture
1216 */
1217static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
1218 unsigned long address)
1219{
Joerg Roedel384de722009-05-15 12:30:05 +02001220 struct aperture_range *aperture;
Joerg Roedel8bda3092009-05-12 12:02:46 +02001221 u64 *pte, *pte_page;
1222
Joerg Roedel384de722009-05-15 12:30:05 +02001223 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1224 if (!aperture)
1225 return NULL;
1226
1227 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
Joerg Roedel8bda3092009-05-12 12:02:46 +02001228 if (!pte) {
1229 pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC);
Joerg Roedel384de722009-05-15 12:30:05 +02001230 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
1231 } else
1232 pte += IOMMU_PTE_L0_INDEX(address);
Joerg Roedel8bda3092009-05-12 12:02:46 +02001233
1234 return pte;
1235}
1236
1237/*
Joerg Roedel431b2a22008-07-11 17:14:22 +02001238 * This is the generic map function. It maps one 4kb page at paddr to
1239 * the given address in the DMA address space for the domain.
1240 */
Joerg Roedelcb76c322008-06-26 21:28:00 +02001241static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu,
1242 struct dma_ops_domain *dom,
1243 unsigned long address,
1244 phys_addr_t paddr,
1245 int direction)
1246{
1247 u64 *pte, __pte;
1248
1249 WARN_ON(address > dom->aperture_size);
1250
1251 paddr &= PAGE_MASK;
1252
Joerg Roedel8bda3092009-05-12 12:02:46 +02001253 pte = dma_ops_get_pte(dom, address);
Joerg Roedel53812c12009-05-12 12:17:38 +02001254 if (!pte)
1255 return bad_dma_address;
Joerg Roedelcb76c322008-06-26 21:28:00 +02001256
1257 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
1258
1259 if (direction == DMA_TO_DEVICE)
1260 __pte |= IOMMU_PTE_IR;
1261 else if (direction == DMA_FROM_DEVICE)
1262 __pte |= IOMMU_PTE_IW;
1263 else if (direction == DMA_BIDIRECTIONAL)
1264 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
1265
1266 WARN_ON(*pte);
1267
1268 *pte = __pte;
1269
1270 return (dma_addr_t)address;
1271}
1272
Joerg Roedel431b2a22008-07-11 17:14:22 +02001273/*
1274 * The generic unmapping function for on page in the DMA address space.
1275 */
Joerg Roedelcb76c322008-06-26 21:28:00 +02001276static void dma_ops_domain_unmap(struct amd_iommu *iommu,
1277 struct dma_ops_domain *dom,
1278 unsigned long address)
1279{
Joerg Roedel384de722009-05-15 12:30:05 +02001280 struct aperture_range *aperture;
Joerg Roedelcb76c322008-06-26 21:28:00 +02001281 u64 *pte;
1282
1283 if (address >= dom->aperture_size)
1284 return;
1285
Joerg Roedel384de722009-05-15 12:30:05 +02001286 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
1287 if (!aperture)
1288 return;
Joerg Roedelcb76c322008-06-26 21:28:00 +02001289
Joerg Roedel384de722009-05-15 12:30:05 +02001290 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
1291 if (!pte)
1292 return;
1293
Joerg Roedelcb76c322008-06-26 21:28:00 +02001294 pte += IOMMU_PTE_L0_INDEX(address);
1295
1296 WARN_ON(!*pte);
1297
1298 *pte = 0ULL;
1299}
1300
Joerg Roedel431b2a22008-07-11 17:14:22 +02001301/*
1302 * This function contains common code for mapping of a physically
Joerg Roedel24f81162008-12-08 14:25:39 +01001303 * contiguous memory region into DMA address space. It is used by all
1304 * mapping functions provided with this IOMMU driver.
Joerg Roedel431b2a22008-07-11 17:14:22 +02001305 * Must be called with the domain lock held.
1306 */
Joerg Roedelcb76c322008-06-26 21:28:00 +02001307static dma_addr_t __map_single(struct device *dev,
1308 struct amd_iommu *iommu,
1309 struct dma_ops_domain *dma_dom,
1310 phys_addr_t paddr,
1311 size_t size,
Joerg Roedel6d4f3432008-09-04 19:18:02 +02001312 int dir,
Joerg Roedel832a90c2008-09-18 15:54:23 +02001313 bool align,
1314 u64 dma_mask)
Joerg Roedelcb76c322008-06-26 21:28:00 +02001315{
1316 dma_addr_t offset = paddr & ~PAGE_MASK;
Joerg Roedel53812c12009-05-12 12:17:38 +02001317 dma_addr_t address, start, ret;
Joerg Roedelcb76c322008-06-26 21:28:00 +02001318 unsigned int pages;
Joerg Roedel6d4f3432008-09-04 19:18:02 +02001319 unsigned long align_mask = 0;
Joerg Roedelcb76c322008-06-26 21:28:00 +02001320 int i;
1321
Joerg Roedele3c449f2008-10-15 22:02:11 -07001322 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
Joerg Roedelcb76c322008-06-26 21:28:00 +02001323 paddr &= PAGE_MASK;
1324
Joerg Roedel8ecaf8f2008-12-12 16:13:04 +01001325 INC_STATS_COUNTER(total_map_requests);
1326
Joerg Roedelc1858972008-12-12 15:42:39 +01001327 if (pages > 1)
1328 INC_STATS_COUNTER(cross_page);
1329
Joerg Roedel6d4f3432008-09-04 19:18:02 +02001330 if (align)
1331 align_mask = (1UL << get_order(size)) - 1;
1332
Joerg Roedel832a90c2008-09-18 15:54:23 +02001333 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
1334 dma_mask);
Joerg Roedelcb76c322008-06-26 21:28:00 +02001335 if (unlikely(address == bad_dma_address))
1336 goto out;
1337
1338 start = address;
1339 for (i = 0; i < pages; ++i) {
Joerg Roedel53812c12009-05-12 12:17:38 +02001340 ret = dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);
1341 if (ret == bad_dma_address)
1342 goto out_unmap;
1343
Joerg Roedelcb76c322008-06-26 21:28:00 +02001344 paddr += PAGE_SIZE;
1345 start += PAGE_SIZE;
1346 }
1347 address += offset;
1348
Joerg Roedel5774f7c2008-12-12 15:57:30 +01001349 ADD_STATS_COUNTER(alloced_io_mem, size);
1350
FUJITA Tomonoriafa9fdc2008-09-20 01:23:30 +09001351 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
Joerg Roedel1c655772008-09-04 18:40:05 +02001352 iommu_flush_tlb(iommu, dma_dom->domain.id);
1353 dma_dom->need_flush = false;
1354 } else if (unlikely(iommu_has_npcache(iommu)))
Joerg Roedel270cab242008-09-04 15:49:46 +02001355 iommu_flush_pages(iommu, dma_dom->domain.id, address, size);
1356
Joerg Roedelcb76c322008-06-26 21:28:00 +02001357out:
1358 return address;
Joerg Roedel53812c12009-05-12 12:17:38 +02001359
1360out_unmap:
1361
1362 for (--i; i >= 0; --i) {
1363 start -= PAGE_SIZE;
1364 dma_ops_domain_unmap(iommu, dma_dom, start);
1365 }
1366
1367 dma_ops_free_addresses(dma_dom, address, pages);
1368
1369 return bad_dma_address;
Joerg Roedelcb76c322008-06-26 21:28:00 +02001370}
1371
Joerg Roedel431b2a22008-07-11 17:14:22 +02001372/*
1373 * Does the reverse of the __map_single function. Must be called with
1374 * the domain lock held too
1375 */
Joerg Roedelcb76c322008-06-26 21:28:00 +02001376static void __unmap_single(struct amd_iommu *iommu,
1377 struct dma_ops_domain *dma_dom,
1378 dma_addr_t dma_addr,
1379 size_t size,
1380 int dir)
1381{
1382 dma_addr_t i, start;
1383 unsigned int pages;
1384
Joerg Roedelb8d99052008-12-08 14:40:26 +01001385 if ((dma_addr == bad_dma_address) ||
1386 (dma_addr + size > dma_dom->aperture_size))
Joerg Roedelcb76c322008-06-26 21:28:00 +02001387 return;
1388
Joerg Roedele3c449f2008-10-15 22:02:11 -07001389 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
Joerg Roedelcb76c322008-06-26 21:28:00 +02001390 dma_addr &= PAGE_MASK;
1391 start = dma_addr;
1392
1393 for (i = 0; i < pages; ++i) {
1394 dma_ops_domain_unmap(iommu, dma_dom, start);
1395 start += PAGE_SIZE;
1396 }
1397
Joerg Roedel5774f7c2008-12-12 15:57:30 +01001398 SUB_STATS_COUNTER(alloced_io_mem, size);
1399
Joerg Roedelcb76c322008-06-26 21:28:00 +02001400 dma_ops_free_addresses(dma_dom, dma_addr, pages);
Joerg Roedel270cab242008-09-04 15:49:46 +02001401
Joerg Roedel80be3082008-11-06 14:59:05 +01001402 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
Joerg Roedel1c655772008-09-04 18:40:05 +02001403 iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
Joerg Roedel80be3082008-11-06 14:59:05 +01001404 dma_dom->need_flush = false;
1405 }
Joerg Roedelcb76c322008-06-26 21:28:00 +02001406}
1407
Joerg Roedel431b2a22008-07-11 17:14:22 +02001408/*
1409 * The exported map_single function for dma_ops.
1410 */
FUJITA Tomonori51491362009-01-05 23:47:25 +09001411static dma_addr_t map_page(struct device *dev, struct page *page,
1412 unsigned long offset, size_t size,
1413 enum dma_data_direction dir,
1414 struct dma_attrs *attrs)
Joerg Roedel4da70b92008-06-26 21:28:01 +02001415{
1416 unsigned long flags;
1417 struct amd_iommu *iommu;
1418 struct protection_domain *domain;
1419 u16 devid;
1420 dma_addr_t addr;
Joerg Roedel832a90c2008-09-18 15:54:23 +02001421 u64 dma_mask;
FUJITA Tomonori51491362009-01-05 23:47:25 +09001422 phys_addr_t paddr = page_to_phys(page) + offset;
Joerg Roedel4da70b92008-06-26 21:28:01 +02001423
Joerg Roedel0f2a86f2008-12-12 15:05:16 +01001424 INC_STATS_COUNTER(cnt_map_single);
1425
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001426 if (!check_device(dev))
1427 return bad_dma_address;
1428
Joerg Roedel832a90c2008-09-18 15:54:23 +02001429 dma_mask = *dev->dma_mask;
Joerg Roedel4da70b92008-06-26 21:28:01 +02001430
1431 get_device_resources(dev, &iommu, &domain, &devid);
1432
1433 if (iommu == NULL || domain == NULL)
Joerg Roedel431b2a22008-07-11 17:14:22 +02001434 /* device not handled by any AMD IOMMU */
Joerg Roedel4da70b92008-06-26 21:28:01 +02001435 return (dma_addr_t)paddr;
1436
Joerg Roedel5b28df62008-12-02 17:49:42 +01001437 if (!dma_ops_domain(domain))
1438 return bad_dma_address;
1439
Joerg Roedel4da70b92008-06-26 21:28:01 +02001440 spin_lock_irqsave(&domain->lock, flags);
Joerg Roedel832a90c2008-09-18 15:54:23 +02001441 addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
1442 dma_mask);
Joerg Roedel4da70b92008-06-26 21:28:01 +02001443 if (addr == bad_dma_address)
1444 goto out;
1445
Joerg Roedel09ee17e2008-12-03 12:19:27 +01001446 iommu_completion_wait(iommu);
Joerg Roedel4da70b92008-06-26 21:28:01 +02001447
1448out:
1449 spin_unlock_irqrestore(&domain->lock, flags);
1450
1451 return addr;
1452}
1453
Joerg Roedel431b2a22008-07-11 17:14:22 +02001454/*
1455 * The exported unmap_single function for dma_ops.
1456 */
FUJITA Tomonori51491362009-01-05 23:47:25 +09001457static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
1458 enum dma_data_direction dir, struct dma_attrs *attrs)
Joerg Roedel4da70b92008-06-26 21:28:01 +02001459{
1460 unsigned long flags;
1461 struct amd_iommu *iommu;
1462 struct protection_domain *domain;
1463 u16 devid;
1464
Joerg Roedel146a6912008-12-12 15:07:12 +01001465 INC_STATS_COUNTER(cnt_unmap_single);
1466
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001467 if (!check_device(dev) ||
1468 !get_device_resources(dev, &iommu, &domain, &devid))
Joerg Roedel431b2a22008-07-11 17:14:22 +02001469 /* device not handled by any AMD IOMMU */
Joerg Roedel4da70b92008-06-26 21:28:01 +02001470 return;
1471
Joerg Roedel5b28df62008-12-02 17:49:42 +01001472 if (!dma_ops_domain(domain))
1473 return;
1474
Joerg Roedel4da70b92008-06-26 21:28:01 +02001475 spin_lock_irqsave(&domain->lock, flags);
1476
1477 __unmap_single(iommu, domain->priv, dma_addr, size, dir);
1478
Joerg Roedel09ee17e2008-12-03 12:19:27 +01001479 iommu_completion_wait(iommu);
Joerg Roedel4da70b92008-06-26 21:28:01 +02001480
1481 spin_unlock_irqrestore(&domain->lock, flags);
1482}
1483
Joerg Roedel431b2a22008-07-11 17:14:22 +02001484/*
1485 * This is a special map_sg function which is used if we should map a
1486 * device which is not handled by an AMD IOMMU in the system.
1487 */
Joerg Roedel65b050a2008-06-26 21:28:02 +02001488static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
1489 int nelems, int dir)
1490{
1491 struct scatterlist *s;
1492 int i;
1493
1494 for_each_sg(sglist, s, nelems, i) {
1495 s->dma_address = (dma_addr_t)sg_phys(s);
1496 s->dma_length = s->length;
1497 }
1498
1499 return nelems;
1500}
1501
Joerg Roedel431b2a22008-07-11 17:14:22 +02001502/*
1503 * The exported map_sg function for dma_ops (handles scatter-gather
1504 * lists).
1505 */
Joerg Roedel65b050a2008-06-26 21:28:02 +02001506static int map_sg(struct device *dev, struct scatterlist *sglist,
FUJITA Tomonori160c1d82009-01-05 23:59:02 +09001507 int nelems, enum dma_data_direction dir,
1508 struct dma_attrs *attrs)
Joerg Roedel65b050a2008-06-26 21:28:02 +02001509{
1510 unsigned long flags;
1511 struct amd_iommu *iommu;
1512 struct protection_domain *domain;
1513 u16 devid;
1514 int i;
1515 struct scatterlist *s;
1516 phys_addr_t paddr;
1517 int mapped_elems = 0;
Joerg Roedel832a90c2008-09-18 15:54:23 +02001518 u64 dma_mask;
Joerg Roedel65b050a2008-06-26 21:28:02 +02001519
Joerg Roedeld03f0672008-12-12 15:09:48 +01001520 INC_STATS_COUNTER(cnt_map_sg);
1521
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001522 if (!check_device(dev))
1523 return 0;
1524
Joerg Roedel832a90c2008-09-18 15:54:23 +02001525 dma_mask = *dev->dma_mask;
Joerg Roedel65b050a2008-06-26 21:28:02 +02001526
1527 get_device_resources(dev, &iommu, &domain, &devid);
1528
1529 if (!iommu || !domain)
1530 return map_sg_no_iommu(dev, sglist, nelems, dir);
1531
Joerg Roedel5b28df62008-12-02 17:49:42 +01001532 if (!dma_ops_domain(domain))
1533 return 0;
1534
Joerg Roedel65b050a2008-06-26 21:28:02 +02001535 spin_lock_irqsave(&domain->lock, flags);
1536
1537 for_each_sg(sglist, s, nelems, i) {
1538 paddr = sg_phys(s);
1539
1540 s->dma_address = __map_single(dev, iommu, domain->priv,
Joerg Roedel832a90c2008-09-18 15:54:23 +02001541 paddr, s->length, dir, false,
1542 dma_mask);
Joerg Roedel65b050a2008-06-26 21:28:02 +02001543
1544 if (s->dma_address) {
1545 s->dma_length = s->length;
1546 mapped_elems++;
1547 } else
1548 goto unmap;
Joerg Roedel65b050a2008-06-26 21:28:02 +02001549 }
1550
Joerg Roedel09ee17e2008-12-03 12:19:27 +01001551 iommu_completion_wait(iommu);
Joerg Roedel65b050a2008-06-26 21:28:02 +02001552
1553out:
1554 spin_unlock_irqrestore(&domain->lock, flags);
1555
1556 return mapped_elems;
1557unmap:
1558 for_each_sg(sglist, s, mapped_elems, i) {
1559 if (s->dma_address)
1560 __unmap_single(iommu, domain->priv, s->dma_address,
1561 s->dma_length, dir);
1562 s->dma_address = s->dma_length = 0;
1563 }
1564
1565 mapped_elems = 0;
1566
1567 goto out;
1568}
1569
Joerg Roedel431b2a22008-07-11 17:14:22 +02001570/*
1571 * The exported map_sg function for dma_ops (handles scatter-gather
1572 * lists).
1573 */
Joerg Roedel65b050a2008-06-26 21:28:02 +02001574static void unmap_sg(struct device *dev, struct scatterlist *sglist,
FUJITA Tomonori160c1d82009-01-05 23:59:02 +09001575 int nelems, enum dma_data_direction dir,
1576 struct dma_attrs *attrs)
Joerg Roedel65b050a2008-06-26 21:28:02 +02001577{
1578 unsigned long flags;
1579 struct amd_iommu *iommu;
1580 struct protection_domain *domain;
1581 struct scatterlist *s;
1582 u16 devid;
1583 int i;
1584
Joerg Roedel55877a62008-12-12 15:12:14 +01001585 INC_STATS_COUNTER(cnt_unmap_sg);
1586
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001587 if (!check_device(dev) ||
1588 !get_device_resources(dev, &iommu, &domain, &devid))
Joerg Roedel65b050a2008-06-26 21:28:02 +02001589 return;
1590
Joerg Roedel5b28df62008-12-02 17:49:42 +01001591 if (!dma_ops_domain(domain))
1592 return;
1593
Joerg Roedel65b050a2008-06-26 21:28:02 +02001594 spin_lock_irqsave(&domain->lock, flags);
1595
1596 for_each_sg(sglist, s, nelems, i) {
1597 __unmap_single(iommu, domain->priv, s->dma_address,
1598 s->dma_length, dir);
Joerg Roedel65b050a2008-06-26 21:28:02 +02001599 s->dma_address = s->dma_length = 0;
1600 }
1601
Joerg Roedel09ee17e2008-12-03 12:19:27 +01001602 iommu_completion_wait(iommu);
Joerg Roedel65b050a2008-06-26 21:28:02 +02001603
1604 spin_unlock_irqrestore(&domain->lock, flags);
1605}
1606
Joerg Roedel431b2a22008-07-11 17:14:22 +02001607/*
1608 * The exported alloc_coherent function for dma_ops.
1609 */
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001610static void *alloc_coherent(struct device *dev, size_t size,
1611 dma_addr_t *dma_addr, gfp_t flag)
1612{
1613 unsigned long flags;
1614 void *virt_addr;
1615 struct amd_iommu *iommu;
1616 struct protection_domain *domain;
1617 u16 devid;
1618 phys_addr_t paddr;
Joerg Roedel832a90c2008-09-18 15:54:23 +02001619 u64 dma_mask = dev->coherent_dma_mask;
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001620
Joerg Roedelc8f0fb32008-12-12 15:14:21 +01001621 INC_STATS_COUNTER(cnt_alloc_coherent);
1622
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001623 if (!check_device(dev))
1624 return NULL;
1625
FUJITA Tomonori13d9fea2008-09-10 20:19:40 +09001626 if (!get_device_resources(dev, &iommu, &domain, &devid))
1627 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
1628
Joerg Roedelc97ac532008-09-11 10:59:15 +02001629 flag |= __GFP_ZERO;
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001630 virt_addr = (void *)__get_free_pages(flag, get_order(size));
1631 if (!virt_addr)
1632 return 0;
1633
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001634 paddr = virt_to_phys(virt_addr);
1635
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001636 if (!iommu || !domain) {
1637 *dma_addr = (dma_addr_t)paddr;
1638 return virt_addr;
1639 }
1640
Joerg Roedel5b28df62008-12-02 17:49:42 +01001641 if (!dma_ops_domain(domain))
1642 goto out_free;
1643
Joerg Roedel832a90c2008-09-18 15:54:23 +02001644 if (!dma_mask)
1645 dma_mask = *dev->dma_mask;
1646
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001647 spin_lock_irqsave(&domain->lock, flags);
1648
1649 *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
Joerg Roedel832a90c2008-09-18 15:54:23 +02001650 size, DMA_BIDIRECTIONAL, true, dma_mask);
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001651
Joerg Roedel5b28df62008-12-02 17:49:42 +01001652 if (*dma_addr == bad_dma_address)
1653 goto out_free;
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001654
Joerg Roedel09ee17e2008-12-03 12:19:27 +01001655 iommu_completion_wait(iommu);
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001656
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001657 spin_unlock_irqrestore(&domain->lock, flags);
1658
1659 return virt_addr;
Joerg Roedel5b28df62008-12-02 17:49:42 +01001660
1661out_free:
1662
1663 free_pages((unsigned long)virt_addr, get_order(size));
1664
1665 return NULL;
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001666}
1667
Joerg Roedel431b2a22008-07-11 17:14:22 +02001668/*
1669 * The exported free_coherent function for dma_ops.
Joerg Roedel431b2a22008-07-11 17:14:22 +02001670 */
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001671static void free_coherent(struct device *dev, size_t size,
1672 void *virt_addr, dma_addr_t dma_addr)
1673{
1674 unsigned long flags;
1675 struct amd_iommu *iommu;
1676 struct protection_domain *domain;
1677 u16 devid;
1678
Joerg Roedel5d31ee72008-12-12 15:16:38 +01001679 INC_STATS_COUNTER(cnt_free_coherent);
1680
Joerg Roedeldbcc1122008-09-04 15:04:26 +02001681 if (!check_device(dev))
1682 return;
1683
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001684 get_device_resources(dev, &iommu, &domain, &devid);
1685
1686 if (!iommu || !domain)
1687 goto free_mem;
1688
Joerg Roedel5b28df62008-12-02 17:49:42 +01001689 if (!dma_ops_domain(domain))
1690 goto free_mem;
1691
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001692 spin_lock_irqsave(&domain->lock, flags);
1693
1694 __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001695
Joerg Roedel09ee17e2008-12-03 12:19:27 +01001696 iommu_completion_wait(iommu);
Joerg Roedel5d8b53cf32008-06-26 21:28:03 +02001697
1698 spin_unlock_irqrestore(&domain->lock, flags);
1699
1700free_mem:
1701 free_pages((unsigned long)virt_addr, get_order(size));
1702}
1703
Joerg Roedelc432f3d2008-06-26 21:28:04 +02001704/*
Joerg Roedelb39ba6a2008-09-09 18:40:46 +02001705 * This function is called by the DMA layer to find out if we can handle a
1706 * particular device. It is part of the dma_ops.
1707 */
1708static int amd_iommu_dma_supported(struct device *dev, u64 mask)
1709{
1710 u16 bdf;
1711 struct pci_dev *pcidev;
1712
1713 /* No device or no PCI device */
1714 if (!dev || dev->bus != &pci_bus_type)
1715 return 0;
1716
1717 pcidev = to_pci_dev(dev);
1718
1719 bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
1720
1721 /* Out of our scope? */
1722 if (bdf > amd_iommu_last_bdf)
1723 return 0;
1724
1725 return 1;
1726}
1727
1728/*
Joerg Roedel431b2a22008-07-11 17:14:22 +02001729 * The function for pre-allocating protection domains.
1730 *
Joerg Roedelc432f3d2008-06-26 21:28:04 +02001731 * If the driver core informs the DMA layer if a driver grabs a device
1732 * we don't need to preallocate the protection domains anymore.
1733 * For now we have to.
1734 */
Jaswinder Singh Rajput0e93dd82008-12-29 21:45:22 +05301735static void prealloc_protection_domains(void)
Joerg Roedelc432f3d2008-06-26 21:28:04 +02001736{
1737 struct pci_dev *dev = NULL;
1738 struct dma_ops_domain *dma_dom;
1739 struct amd_iommu *iommu;
1740 int order = amd_iommu_aperture_order;
1741 u16 devid;
1742
1743 while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
Joerg Roedeledcb34d2008-12-10 20:01:45 +01001744 devid = calc_devid(dev->bus->number, dev->devfn);
Joerg Roedel3a61ec32008-07-25 13:07:50 +02001745 if (devid > amd_iommu_last_bdf)
Joerg Roedelc432f3d2008-06-26 21:28:04 +02001746 continue;
1747 devid = amd_iommu_alias_table[devid];
1748 if (domain_for_device(devid))
1749 continue;
1750 iommu = amd_iommu_rlookup_table[devid];
1751 if (!iommu)
1752 continue;
1753 dma_dom = dma_ops_domain_alloc(iommu, order);
1754 if (!dma_dom)
1755 continue;
1756 init_unity_mappings_for_device(dma_dom, devid);
Joerg Roedelbd60b732008-09-11 10:24:48 +02001757 dma_dom->target_dev = devid;
1758
1759 list_add_tail(&dma_dom->list, &iommu_pd_list);
Joerg Roedelc432f3d2008-06-26 21:28:04 +02001760 }
1761}
1762
FUJITA Tomonori160c1d82009-01-05 23:59:02 +09001763static struct dma_map_ops amd_iommu_dma_ops = {
Joerg Roedel6631ee92008-06-26 21:28:05 +02001764 .alloc_coherent = alloc_coherent,
1765 .free_coherent = free_coherent,
FUJITA Tomonori51491362009-01-05 23:47:25 +09001766 .map_page = map_page,
1767 .unmap_page = unmap_page,
Joerg Roedel6631ee92008-06-26 21:28:05 +02001768 .map_sg = map_sg,
1769 .unmap_sg = unmap_sg,
Joerg Roedelb39ba6a2008-09-09 18:40:46 +02001770 .dma_supported = amd_iommu_dma_supported,
Joerg Roedel6631ee92008-06-26 21:28:05 +02001771};
1772
Joerg Roedel431b2a22008-07-11 17:14:22 +02001773/*
1774 * The function which clues the AMD IOMMU driver into dma_ops.
1775 */
Joerg Roedel6631ee92008-06-26 21:28:05 +02001776int __init amd_iommu_init_dma_ops(void)
1777{
1778 struct amd_iommu *iommu;
1779 int order = amd_iommu_aperture_order;
1780 int ret;
1781
Joerg Roedel431b2a22008-07-11 17:14:22 +02001782 /*
1783 * first allocate a default protection domain for every IOMMU we
1784 * found in the system. Devices not assigned to any other
1785 * protection domain will be assigned to the default one.
1786 */
Joerg Roedel6631ee92008-06-26 21:28:05 +02001787 list_for_each_entry(iommu, &amd_iommu_list, list) {
1788 iommu->default_dom = dma_ops_domain_alloc(iommu, order);
1789 if (iommu->default_dom == NULL)
1790 return -ENOMEM;
Joerg Roedele2dc14a2008-12-10 18:48:59 +01001791 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
Joerg Roedel6631ee92008-06-26 21:28:05 +02001792 ret = iommu_init_unity_mappings(iommu);
1793 if (ret)
1794 goto free_domains;
1795 }
1796
Joerg Roedel431b2a22008-07-11 17:14:22 +02001797 /*
1798 * If device isolation is enabled, pre-allocate the protection
1799 * domains for each device.
1800 */
Joerg Roedel6631ee92008-06-26 21:28:05 +02001801 if (amd_iommu_isolate)
1802 prealloc_protection_domains();
1803
1804 iommu_detected = 1;
1805 force_iommu = 1;
1806 bad_dma_address = 0;
Ingo Molnar92af4e22008-06-27 10:48:16 +02001807#ifdef CONFIG_GART_IOMMU
Joerg Roedel6631ee92008-06-26 21:28:05 +02001808 gart_iommu_aperture_disabled = 1;
1809 gart_iommu_aperture = 0;
Ingo Molnar92af4e22008-06-27 10:48:16 +02001810#endif
Joerg Roedel6631ee92008-06-26 21:28:05 +02001811
Joerg Roedel431b2a22008-07-11 17:14:22 +02001812 /* Make the driver finally visible to the drivers */
Joerg Roedel6631ee92008-06-26 21:28:05 +02001813 dma_ops = &amd_iommu_dma_ops;
1814
Joerg Roedel26961ef2008-12-03 17:00:17 +01001815 register_iommu(&amd_iommu_ops);
Joerg Roedel26961ef2008-12-03 17:00:17 +01001816
Joerg Roedele275a2a2008-12-10 18:27:25 +01001817 bus_register_notifier(&pci_bus_type, &device_nb);
1818
Joerg Roedel7f265082008-12-12 13:50:21 +01001819 amd_iommu_stats_init();
1820
Joerg Roedel6631ee92008-06-26 21:28:05 +02001821 return 0;
1822
1823free_domains:
1824
1825 list_for_each_entry(iommu, &amd_iommu_list, list) {
1826 if (iommu->default_dom)
1827 dma_ops_domain_free(iommu->default_dom);
1828 }
1829
1830 return ret;
1831}
Joerg Roedel6d98cd82008-12-08 12:05:55 +01001832
1833/*****************************************************************************
1834 *
1835 * The following functions belong to the exported interface of AMD IOMMU
1836 *
1837 * This interface allows access to lower level functions of the IOMMU
1838 * like protection domain handling and assignement of devices to domains
1839 * which is not possible with the dma_ops interface.
1840 *
1841 *****************************************************************************/
1842
Joerg Roedel6d98cd82008-12-08 12:05:55 +01001843static void cleanup_domain(struct protection_domain *domain)
1844{
1845 unsigned long flags;
1846 u16 devid;
1847
1848 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1849
1850 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
1851 if (amd_iommu_pd_table[devid] == domain)
1852 __detach_device(domain, devid);
1853
1854 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1855}
1856
Joerg Roedelc156e342008-12-02 18:13:27 +01001857static int amd_iommu_domain_init(struct iommu_domain *dom)
1858{
1859 struct protection_domain *domain;
1860
1861 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1862 if (!domain)
1863 return -ENOMEM;
1864
1865 spin_lock_init(&domain->lock);
1866 domain->mode = PAGE_MODE_3_LEVEL;
1867 domain->id = domain_id_alloc();
1868 if (!domain->id)
1869 goto out_free;
1870 domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1871 if (!domain->pt_root)
1872 goto out_free;
1873
1874 dom->priv = domain;
1875
1876 return 0;
1877
1878out_free:
1879 kfree(domain);
1880
1881 return -ENOMEM;
1882}
1883
Joerg Roedel98383fc2008-12-02 18:34:12 +01001884static void amd_iommu_domain_destroy(struct iommu_domain *dom)
1885{
1886 struct protection_domain *domain = dom->priv;
1887
1888 if (!domain)
1889 return;
1890
1891 if (domain->dev_cnt > 0)
1892 cleanup_domain(domain);
1893
1894 BUG_ON(domain->dev_cnt != 0);
1895
1896 free_pagetable(domain);
1897
1898 domain_id_free(domain->id);
1899
1900 kfree(domain);
1901
1902 dom->priv = NULL;
1903}
1904
Joerg Roedel684f2882008-12-08 12:07:44 +01001905static void amd_iommu_detach_device(struct iommu_domain *dom,
1906 struct device *dev)
1907{
1908 struct protection_domain *domain = dom->priv;
1909 struct amd_iommu *iommu;
1910 struct pci_dev *pdev;
1911 u16 devid;
1912
1913 if (dev->bus != &pci_bus_type)
1914 return;
1915
1916 pdev = to_pci_dev(dev);
1917
1918 devid = calc_devid(pdev->bus->number, pdev->devfn);
1919
1920 if (devid > 0)
1921 detach_device(domain, devid);
1922
1923 iommu = amd_iommu_rlookup_table[devid];
1924 if (!iommu)
1925 return;
1926
1927 iommu_queue_inv_dev_entry(iommu, devid);
1928 iommu_completion_wait(iommu);
1929}
1930
Joerg Roedel01106062008-12-02 19:34:11 +01001931static int amd_iommu_attach_device(struct iommu_domain *dom,
1932 struct device *dev)
1933{
1934 struct protection_domain *domain = dom->priv;
1935 struct protection_domain *old_domain;
1936 struct amd_iommu *iommu;
1937 struct pci_dev *pdev;
1938 u16 devid;
1939
1940 if (dev->bus != &pci_bus_type)
1941 return -EINVAL;
1942
1943 pdev = to_pci_dev(dev);
1944
1945 devid = calc_devid(pdev->bus->number, pdev->devfn);
1946
1947 if (devid >= amd_iommu_last_bdf ||
1948 devid != amd_iommu_alias_table[devid])
1949 return -EINVAL;
1950
1951 iommu = amd_iommu_rlookup_table[devid];
1952 if (!iommu)
1953 return -EINVAL;
1954
1955 old_domain = domain_for_device(devid);
1956 if (old_domain)
1957 return -EBUSY;
1958
1959 attach_device(iommu, domain, devid);
1960
1961 iommu_completion_wait(iommu);
1962
1963 return 0;
1964}
1965
Joerg Roedelc6229ca2008-12-02 19:48:43 +01001966static int amd_iommu_map_range(struct iommu_domain *dom,
1967 unsigned long iova, phys_addr_t paddr,
1968 size_t size, int iommu_prot)
1969{
1970 struct protection_domain *domain = dom->priv;
1971 unsigned long i, npages = iommu_num_pages(paddr, size, PAGE_SIZE);
1972 int prot = 0;
1973 int ret;
1974
1975 if (iommu_prot & IOMMU_READ)
1976 prot |= IOMMU_PROT_IR;
1977 if (iommu_prot & IOMMU_WRITE)
1978 prot |= IOMMU_PROT_IW;
1979
1980 iova &= PAGE_MASK;
1981 paddr &= PAGE_MASK;
1982
1983 for (i = 0; i < npages; ++i) {
1984 ret = iommu_map_page(domain, iova, paddr, prot);
1985 if (ret)
1986 return ret;
1987
1988 iova += PAGE_SIZE;
1989 paddr += PAGE_SIZE;
1990 }
1991
1992 return 0;
1993}
1994
Joerg Roedeleb74ff62008-12-02 19:59:10 +01001995static void amd_iommu_unmap_range(struct iommu_domain *dom,
1996 unsigned long iova, size_t size)
1997{
1998
1999 struct protection_domain *domain = dom->priv;
2000 unsigned long i, npages = iommu_num_pages(iova, size, PAGE_SIZE);
2001
2002 iova &= PAGE_MASK;
2003
2004 for (i = 0; i < npages; ++i) {
2005 iommu_unmap_page(domain, iova);
2006 iova += PAGE_SIZE;
2007 }
2008
2009 iommu_flush_domain(domain->id);
2010}
2011
Joerg Roedel645c4c82008-12-02 20:05:50 +01002012static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2013 unsigned long iova)
2014{
2015 struct protection_domain *domain = dom->priv;
2016 unsigned long offset = iova & ~PAGE_MASK;
2017 phys_addr_t paddr;
2018 u64 *pte;
2019
2020 pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(iova)];
2021
2022 if (!IOMMU_PTE_PRESENT(*pte))
2023 return 0;
2024
2025 pte = IOMMU_PTE_PAGE(*pte);
2026 pte = &pte[IOMMU_PTE_L1_INDEX(iova)];
2027
2028 if (!IOMMU_PTE_PRESENT(*pte))
2029 return 0;
2030
2031 pte = IOMMU_PTE_PAGE(*pte);
2032 pte = &pte[IOMMU_PTE_L0_INDEX(iova)];
2033
2034 if (!IOMMU_PTE_PRESENT(*pte))
2035 return 0;
2036
2037 paddr = *pte & IOMMU_PAGE_MASK;
2038 paddr |= offset;
2039
2040 return paddr;
2041}
2042
Sheng Yangdbb9fd82009-03-18 15:33:06 +08002043static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
2044 unsigned long cap)
2045{
2046 return 0;
2047}
2048
Joerg Roedel26961ef2008-12-03 17:00:17 +01002049static struct iommu_ops amd_iommu_ops = {
2050 .domain_init = amd_iommu_domain_init,
2051 .domain_destroy = amd_iommu_domain_destroy,
2052 .attach_dev = amd_iommu_attach_device,
2053 .detach_dev = amd_iommu_detach_device,
2054 .map = amd_iommu_map_range,
2055 .unmap = amd_iommu_unmap_range,
2056 .iova_to_phys = amd_iommu_iova_to_phys,
Sheng Yangdbb9fd82009-03-18 15:33:06 +08002057 .domain_has_cap = amd_iommu_domain_has_cap,
Joerg Roedel26961ef2008-12-03 17:00:17 +01002058};
2059