Updated iommu/msm_bus
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 57ed244..a5bee8e 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -83,8 +83,6 @@
static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
int amd_iommu_max_glx_val = -1;
-static struct dma_map_ops amd_iommu_dma_ops;
-
/*
* general struct to manage commands send to an IOMMU
*/
@@ -452,27 +450,12 @@
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
{
- int type, devid, domid, flags;
- volatile u32 *event = __evt;
- int count = 0;
- u64 address;
-
-retry:
- type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
- devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
- domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
- flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
- address = (u64)(((u64)event[3]) << 32) | event[2];
-
- if (type == 0) {
- /* Did we hit the erratum? */
- if (++count == LOOP_TIMEOUT) {
- pr_err("AMD-Vi: No event written to event log\n");
- return;
- }
- udelay(1);
- goto retry;
- }
+ u32 *event = __evt;
+ int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
+ int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
+ int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
+ int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
+ u64 address = (u64)(((u64)event[3]) << 32) | event[2];
printk(KERN_ERR "AMD-Vi: Event logged [");
@@ -525,8 +508,6 @@
default:
printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
}
-
- memset(__evt, 0, 4 * sizeof(u32));
}
static void iommu_poll_events(struct amd_iommu *iommu)
@@ -549,12 +530,26 @@
spin_unlock_irqrestore(&iommu->lock, flags);
}
-static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
+static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u32 head)
{
struct amd_iommu_fault fault;
+ volatile u64 *raw;
+ int i;
INC_STATS_COUNTER(pri_requests);
+ raw = (u64 *)(iommu->ppr_log + head);
+
+ /*
+ * Hardware bug: Interrupt may arrive before the entry is written to
+ * memory. If this happens we need to wait for the entry to arrive.
+ */
+ for (i = 0; i < LOOP_TIMEOUT; ++i) {
+ if (PPR_REQ_TYPE(raw[0]) != 0)
+ break;
+ udelay(1);
+ }
+
if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
return;
@@ -566,6 +561,12 @@
fault.tag = PPR_TAG(raw[0]);
fault.flags = PPR_FLAGS(raw[0]);
+ /*
+ * To detect the hardware bug we need to clear the entry
+ * to back to zero.
+ */
+ raw[0] = raw[1] = 0;
+
atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
}
@@ -577,62 +578,25 @@
if (iommu->ppr_log == NULL)
return;
- /* enable ppr interrupts again */
- writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
-
spin_lock_irqsave(&iommu->lock, flags);
head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
while (head != tail) {
- volatile u64 *raw;
- u64 entry[2];
- int i;
-
- raw = (u64 *)(iommu->ppr_log + head);
-
- /*
- * Hardware bug: Interrupt may arrive before the entry is
- * written to memory. If this happens we need to wait for the
- * entry to arrive.
- */
- for (i = 0; i < LOOP_TIMEOUT; ++i) {
- if (PPR_REQ_TYPE(raw[0]) != 0)
- break;
- udelay(1);
- }
-
- /* Avoid memcpy function-call overhead */
- entry[0] = raw[0];
- entry[1] = raw[1];
-
- /*
- * To detect the hardware bug we need to clear the entry
- * back to zero.
- */
- raw[0] = raw[1] = 0UL;
-
- /* Update head pointer of hardware ring-buffer */
- head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
- writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
-
- /*
- * Release iommu->lock because ppr-handling might need to
- * re-aquire it
- */
- spin_unlock_irqrestore(&iommu->lock, flags);
/* Handle PPR entry */
- iommu_handle_ppr_entry(iommu, entry);
+ iommu_handle_ppr_entry(iommu, head);
- spin_lock_irqsave(&iommu->lock, flags);
-
- /* Refresh ring-buffer information */
- head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
+ /* Update and refresh ring-buffer state*/
+ head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
+ writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
}
+ /* enable ppr interrupts again */
+ writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
+
spin_unlock_irqrestore(&iommu->lock, flags);
}
@@ -2071,20 +2035,20 @@
}
/* FIXME: Move this to PCI code */
-#define PCI_PRI_TLP_OFF (1 << 15)
+#define PCI_PRI_TLP_OFF (1 << 2)
bool pci_pri_tlp_required(struct pci_dev *pdev)
{
- u16 status;
+ u16 control;
int pos;
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
if (!pos)
return false;
- pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
+ pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
- return (status & PCI_PRI_TLP_OFF) ? true : false;
+ return (control & PCI_PRI_TLP_OFF) ? true : false;
}
/*
@@ -2254,18 +2218,6 @@
iommu_init_device(dev);
- /*
- * dev_data is still NULL and
- * got initialized in iommu_init_device
- */
- dev_data = get_dev_data(dev);
-
- if (iommu_pass_through || dev_data->iommu_v2) {
- dev_data->passthrough = true;
- attach_device(dev, pt_domain);
- break;
- }
-
domain = domain_for_device(dev);
/* allocate a protection domain if a device is added */
@@ -2281,10 +2233,6 @@
list_add_tail(&dma_domain->list, &iommu_pd_list);
spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
- dev_data = get_dev_data(dev);
-
- dev->archdata.dma_ops = &amd_iommu_dma_ops;
-
break;
case BUS_NOTIFY_DEL_DEVICE:
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index c04ddca..c567903 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -1029,9 +1029,6 @@
if (!iommu->dev)
return 1;
- iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
- PCI_DEVFN(0, 0));
-
iommu->cap_ptr = h->cap_ptr;
iommu->pci_seg = h->pci_seg;
iommu->mmio_phys = h->mmio_phys;
@@ -1326,16 +1323,20 @@
{
int i, j;
u32 ioc_feature_control;
- struct pci_dev *pdev = iommu->root_pdev;
+ struct pci_dev *pdev = NULL;
/* RD890 BIOSes may not have completely reconfigured the iommu */
- if (!is_rd890_iommu(iommu->dev) || !pdev)
+ if (!is_rd890_iommu(iommu->dev))
return;
/*
* First, we need to ensure that the iommu is enabled. This is
* controlled by a register in the northbridge
*/
+ pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0));
+
+ if (!pdev)
+ return;
/* Select Northbridge indirect register 0x75 and enable writing */
pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
@@ -1345,6 +1346,8 @@
if (!(ioc_feature_control & 0x1))
pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
+ pci_dev_put(pdev);
+
/* Restore the iommu BAR */
pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
iommu->stored_addr_lo);
@@ -1641,8 +1644,6 @@
amd_iommu_init_api();
- x86_platform.iommu_shutdown = disable_iommus;
-
if (iommu_pass_through)
goto out;
@@ -1651,6 +1652,8 @@
else
printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n");
+ x86_platform.iommu_shutdown = disable_iommus;
+
out:
return ret;
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 2435555..2452f3b 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -481,9 +481,6 @@
/* Pointer to PCI device of this IOMMU */
struct pci_dev *dev;
- /* Cache pdev to root device for resume quirks */
- struct pci_dev *root_pdev;
-
/* physical address of MMIO space */
u64 mmio_phys;
/* virtual address of MMIO space */
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index a1f1bc8..036fe9b 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -681,8 +681,6 @@
atomic_set(&pasid_state->count, 1);
init_waitqueue_head(&pasid_state->wq);
- spin_lock_init(&pasid_state->lock);
-
pasid_state->task = task;
pasid_state->mm = get_task_mm(task);
pasid_state->device_state = dev_state;
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 97b2e21..35c1e17 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1056,8 +1056,8 @@
const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
{
- if (fault_reason >= 0x20 && (fault_reason - 0x20 <
- ARRAY_SIZE(intr_remap_fault_reasons))) {
+ if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
+ ARRAY_SIZE(intr_remap_fault_reasons))) {
*fault_type = INTR_REMAP;
return intr_remap_fault_reasons[fault_reason - 0x20];
} else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 5fda348..f93d5ac 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -2286,6 +2286,12 @@
if (!info)
return -ENOMEM;
+ ret = domain_context_mapping(domain, pdev, translation);
+ if (ret) {
+ free_devinfo_mem(info);
+ return ret;
+ }
+
info->segment = pci_domain_nr(pdev->bus);
info->bus = pdev->bus->number;
info->devfn = pdev->devfn;
@@ -2298,17 +2304,6 @@
pdev->dev.archdata.iommu = info;
spin_unlock_irqrestore(&device_domain_lock, flags);
- ret = domain_context_mapping(domain, pdev, translation);
- if (ret) {
- spin_lock_irqsave(&device_domain_lock, flags);
- list_del(&info->link);
- list_del(&info->global);
- pdev->dev.archdata.iommu = NULL;
- spin_unlock_irqrestore(&device_domain_lock, flags);
- free_devinfo_mem(info);
- return ret;
- }
-
return 0;
}
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 2f34262..ef69d91 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -83,6 +83,19 @@
bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
}
+/**
+ * bus_set_iommu - set iommu-callbacks for the bus
+ * @bus: bus.
+ * @ops: the callbacks provided by the iommu-driver
+ *
+ * This function is called by an iommu driver to set the iommu methods
+ * used for a particular bus. Drivers for devices on that bus can use
+ * the iommu-api after these ops are registered.
+ * This special function is needed because IOMMUs are usually devices on
+ * the bus itself, so the iommu drivers are not initialized when the bus
+ * is set up. With this function the iommu-driver can set the iommu-ops
+ * afterwards.
+ */
int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops)
{
if (bus->iommu_ops != NULL)
@@ -90,7 +103,7 @@
bus->iommu_ops = ops;
-
+ /* Do IOMMU specific setup for this bus-type */
iommu_bus_init(bus, ops);
return 0;
@@ -103,6 +116,17 @@
}
EXPORT_SYMBOL_GPL(iommu_present);
+/**
+ * iommu_set_fault_handler() - set a fault handler for an iommu domain
+ * @domain: iommu domain
+ * @handler: fault handler
+ *
+ * This function should be used by IOMMU users which want to be notified
+ * whenever an IOMMU fault happens.
+ *
+ * The fault handler itself should return 0 on success, and an appropriate
+ * error code otherwise.
+ */
void iommu_set_fault_handler(struct iommu_domain *domain,
iommu_fault_handler_t handler)
{
@@ -197,9 +221,14 @@
if (unlikely(domain->ops->map == NULL))
return -ENODEV;
-
+ /* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
+ /*
+ * both the virtual address and the physical one, as well as
+ * the size of the mapping, must be aligned (at least) to the
+ * size of the smallest page supported by the hardware
+ */
if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
"0x%x\n", iova, (unsigned long)paddr,
@@ -214,27 +243,27 @@
unsigned long pgsize, addr_merge = iova | paddr;
unsigned int pgsize_idx;
-
+ /* Max page size that still fits into 'size' */
pgsize_idx = __fls(size);
-
+ /* need to consider alignment requirements ? */
if (likely(addr_merge)) {
-
+ /* Max page size allowed by both iova and paddr */
unsigned int align_pgsize_idx = __ffs(addr_merge);
pgsize_idx = min(pgsize_idx, align_pgsize_idx);
}
-
+ /* build a mask of acceptable page sizes */
pgsize = (1UL << (pgsize_idx + 1)) - 1;
-
+ /* throw away page sizes not supported by the hardware */
pgsize &= domain->ops->pgsize_bitmap;
-
+ /* make sure we're still sane */
BUG_ON(!pgsize);
-
+ /* pick the biggest page */
pgsize_idx = __fls(pgsize);
pgsize = 1UL << pgsize_idx;
@@ -250,7 +279,7 @@
size -= pgsize;
}
-
+ /* unroll mapping in case something went wrong */
if (ret)
iommu_unmap(domain, orig_iova, orig_size - size);
@@ -266,9 +295,14 @@
if (unlikely(domain->ops->unmap == NULL))
return -ENODEV;
-
+ /* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
+ /*
+ * The virtual address, as well as the size of the mapping, must be
+ * aligned (at least) to the size of the smallest page supported
+ * by the hardware
+ */
if (!IS_ALIGNED(iova | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
iova, (unsigned long)size, min_pagesz);
@@ -278,6 +312,10 @@
pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
(unsigned long)size);
+ /*
+ * Keep iterating until we either unmap 'size' bytes (or more)
+ * or we hit an area that isn't mapped.
+ */
while (unmapped < size) {
size_t left = size - unmapped;
diff --git a/drivers/iommu/msm_iommu-v2.c b/drivers/iommu/msm_iommu-v2.c
new file mode 100644
index 0000000..15de300
--- /dev/null
+++ b/drivers/iommu/msm_iommu-v2.c
@@ -0,0 +1,720 @@
+/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/iommu.h>
+#include <linux/clk.h>
+#include <linux/scatterlist.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regulator/consumer.h>
+#include <asm/sizes.h>
+
+#include <mach/iommu_hw-v2.h>
+#include <mach/iommu.h>
+
+#include "msm_iommu_pagetable.h"
+
+/* bitmap of the page sizes currently supported */
+#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
+
+static DEFINE_MUTEX(msm_iommu_lock);
+
+struct msm_priv {
+ struct iommu_pt pt;
+ struct list_head list_attached;
+};
+
+static int __enable_clocks(struct msm_iommu_drvdata *drvdata)
+{
+ int ret;
+
+ ret = clk_prepare_enable(drvdata->pclk);
+ if (ret)
+ goto fail;
+
+ ret = clk_prepare_enable(drvdata->clk);
+ if (ret)
+ clk_disable_unprepare(drvdata->pclk);
+
+ if (drvdata->aclk) {
+ ret = clk_prepare_enable(drvdata->aclk);
+ if (ret) {
+ clk_disable_unprepare(drvdata->clk);
+ clk_disable_unprepare(drvdata->pclk);
+ }
+ }
+fail:
+ return ret;
+}
+
+static void __disable_clocks(struct msm_iommu_drvdata *drvdata)
+{
+ if (drvdata->aclk)
+ clk_disable_unprepare(drvdata->aclk);
+ clk_disable_unprepare(drvdata->clk);
+ clk_disable_unprepare(drvdata->pclk);
+}
+
+static void __sync_tlb(void __iomem *base, int ctx)
+{
+ SET_TLBSYNC(base, ctx, 0);
+
+ /* No barrier needed due to register proximity */
+ while (GET_CB_TLBSTATUS_SACTIVE(base, ctx))
+ cpu_relax();
+
+ /* No barrier needed due to read dependency */
+}
+
+static int __flush_iotlb_va(struct iommu_domain *domain, unsigned int va)
+{
+ struct msm_priv *priv = domain->priv;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ int ret = 0;
+ int asid;
+
+ list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
+ BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);
+
+ iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
+ BUG_ON(!iommu_drvdata);
+
+
+ ret = __enable_clocks(iommu_drvdata);
+ if (ret)
+ goto fail;
+
+ asid = GET_CB_CONTEXTIDR_ASID(iommu_drvdata->base,
+ ctx_drvdata->num);
+
+ SET_TLBIVA(iommu_drvdata->base, ctx_drvdata->num,
+ asid | (va & CB_TLBIVA_VA));
+ mb();
+ __sync_tlb(iommu_drvdata->base, ctx_drvdata->num);
+ __disable_clocks(iommu_drvdata);
+ }
+fail:
+ return ret;
+}
+
+static int __flush_iotlb(struct iommu_domain *domain)
+{
+ struct msm_priv *priv = domain->priv;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ int ret = 0;
+ int asid;
+
+ list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
+ BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);
+
+ iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
+ BUG_ON(!iommu_drvdata);
+
+ ret = __enable_clocks(iommu_drvdata);
+ if (ret)
+ goto fail;
+
+ asid = GET_CB_CONTEXTIDR_ASID(iommu_drvdata->base,
+ ctx_drvdata->num);
+
+ SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num, asid);
+ mb();
+ __sync_tlb(iommu_drvdata->base, ctx_drvdata->num);
+ __disable_clocks(iommu_drvdata);
+ }
+
+fail:
+ return ret;
+}
+
+static void __reset_iommu(void __iomem *base, int smt_size)
+{
+ int i;
+
+ SET_ACR(base, 0);
+ SET_NSACR(base, 0);
+ SET_CR2(base, 0);
+ SET_NSCR2(base, 0);
+ SET_GFAR(base, 0);
+ SET_GFSRRESTORE(base, 0);
+ SET_TLBIALLNSNH(base, 0);
+ SET_PMCR(base, 0);
+ SET_SCR1(base, 0);
+ SET_SSDR_N(base, 0, 0);
+
+ for (i = 0; i < smt_size; i++)
+ SET_SMR_VALID(base, i, 0);
+
+ mb();
+}
+
+static void __program_iommu(void __iomem *base, int smt_size)
+{
+ __reset_iommu(base, smt_size);
+
+ SET_CR0_SMCFCFG(base, 1);
+ SET_CR0_USFCFG(base, 1);
+ SET_CR0_STALLD(base, 1);
+ SET_CR0_GCFGFIE(base, 1);
+ SET_CR0_GCFGFRE(base, 1);
+ SET_CR0_GFIE(base, 1);
+ SET_CR0_GFRE(base, 1);
+ SET_CR0_CLIENTPD(base, 0);
+ mb(); /* Make sure writes complete before returning */
+}
+
+static void __reset_context(void __iomem *base, int ctx)
+{
+ SET_ACTLR(base, ctx, 0);
+ SET_FAR(base, ctx, 0);
+ SET_FSRRESTORE(base, ctx, 0);
+ SET_NMRR(base, ctx, 0);
+ SET_PAR(base, ctx, 0);
+ SET_PRRR(base, ctx, 0);
+ SET_SCTLR(base, ctx, 0);
+ SET_TLBIALL(base, ctx, 0);
+ SET_TTBCR(base, ctx, 0);
+ SET_TTBR0(base, ctx, 0);
+ SET_TTBR1(base, ctx, 0);
+ mb();
+}
+
+static void __program_context(void __iomem *base, int ctx, int ncb,
+ phys_addr_t pgtable, int redirect,
+ u32 *sids, int len, int smt_size)
+{
+ unsigned int prrr, nmrr;
+ unsigned int pn;
+ int i, j, found, num = 0;
+
+ __reset_context(base, ctx);
+
+ pn = pgtable >> CB_TTBR0_ADDR_SHIFT;
+ SET_TTBCR(base, ctx, 0);
+ SET_CB_TTBR0_ADDR(base, ctx, pn);
+
+ /* Enable context fault interrupt */
+ SET_CB_SCTLR_CFIE(base, ctx, 1);
+
+ /* Redirect all cacheable requests to L2 slave port. */
+ SET_CB_ACTLR_BPRCISH(base, ctx, 1);
+ SET_CB_ACTLR_BPRCOSH(base, ctx, 1);
+ SET_CB_ACTLR_BPRCNSH(base, ctx, 1);
+
+ /* Turn on TEX Remap */
+ SET_CB_SCTLR_TRE(base, ctx, 1);
+
+ /* Enable private ASID namespace */
+ SET_CB_SCTLR_ASIDPNE(base, ctx, 1);
+
+ /* Set TEX remap attributes */
+ RCP15_PRRR(prrr);
+ RCP15_NMRR(nmrr);
+ SET_PRRR(base, ctx, prrr);
+ SET_NMRR(base, ctx, nmrr);
+
+ /* Configure page tables as inner-cacheable and shareable to reduce
+ * the TLB miss penalty.
+ */
+ if (redirect) {
+ SET_CB_TTBR0_S(base, ctx, 1);
+ SET_CB_TTBR0_NOS(base, ctx, 1);
+ SET_CB_TTBR0_IRGN1(base, ctx, 0); /* WB, WA */
+ SET_CB_TTBR0_IRGN0(base, ctx, 1);
+ SET_CB_TTBR0_RGN(base, ctx, 1); /* WB, WA */
+ }
+
+ /* Program the M2V tables for this context */
+ for (i = 0; i < len / sizeof(*sids); i++) {
+ for (; num < smt_size; num++)
+ if (GET_SMR_VALID(base, num) == 0)
+ break;
+ BUG_ON(num >= smt_size);
+
+ SET_SMR_VALID(base, num, 1);
+ SET_SMR_MASK(base, num, 0);
+ SET_SMR_ID(base, num, sids[i]);
+
+ /* Set VMID = 0 */
+ SET_S2CR_N(base, num, 0);
+ SET_S2CR_CBNDX(base, num, ctx);
+ /* Set security bit override to be Non-secure */
+ SET_S2CR_NSCFG(base, num, 3);
+ }
+
+ SET_CBAR_N(base, ctx, 0);
+ /* Stage 1 Context with Stage 2 bypass */
+ SET_CBAR_TYPE(base, ctx, 1);
+ /* Route page faults to the non-secure interrupt */
+ SET_CBAR_IRPTNDX(base, ctx, 1);
+
+ /* Find if this page table is used elsewhere, and re-use ASID */
+ found = 0;
+ for (i = 0; i < ncb; i++)
+ if ((GET_CB_TTBR0_ADDR(base, i) == pn) && (i != ctx)) {
+ SET_CB_CONTEXTIDR_ASID(base, ctx, \
+ GET_CB_CONTEXTIDR_ASID(base, i));
+ found = 1;
+ break;
+ }
+
+ /* If page table is new, find an unused ASID */
+ if (!found) {
+ for (i = 0; i < ncb; i++) {
+ found = 0;
+ for (j = 0; j < ncb; j++) {
+ if (GET_CB_CONTEXTIDR_ASID(base, j) == i &&
+ j != ctx)
+ found = 1;
+ }
+
+ if (!found) {
+ SET_CB_CONTEXTIDR_ASID(base, ctx, i);
+ break;
+ }
+ }
+ BUG_ON(found);
+ }
+
+ /* Enable the MMU */
+ SET_CB_SCTLR_M(base, ctx, 1);
+ mb();
+}
+
+static int msm_iommu_domain_init(struct iommu_domain *domain, int flags)
+{
+ struct msm_priv *priv;
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ goto fail_nomem;
+
+#ifdef CONFIG_IOMMU_PGTABLES_L2
+ priv->pt.redirect = flags & MSM_IOMMU_DOMAIN_PT_CACHEABLE;
+#endif
+
+ INIT_LIST_HEAD(&priv->list_attached);
+ if (msm_iommu_pagetable_alloc(&priv->pt))
+ goto fail_nomem;
+
+ domain->priv = priv;
+ return 0;
+
+fail_nomem:
+ kfree(priv);
+ return -ENOMEM;
+}
+
+static void msm_iommu_domain_destroy(struct iommu_domain *domain)
+{
+ struct msm_priv *priv;
+
+ mutex_lock(&msm_iommu_lock);
+ priv = domain->priv;
+ domain->priv = NULL;
+
+ if (priv)
+ msm_iommu_pagetable_free(&priv->pt);
+
+ kfree(priv);
+ mutex_unlock(&msm_iommu_lock);
+}
+
+static int msm_iommu_ctx_attached(struct device *dev)
+{
+ struct platform_device *pdev;
+ struct device_node *child;
+ struct msm_iommu_ctx_drvdata *ctx;
+
+ for_each_child_of_node(dev->of_node, child) {
+ pdev = of_find_device_by_node(child);
+
+ ctx = dev_get_drvdata(&pdev->dev);
+ if (ctx->attached_domain) {
+ of_node_put(child);
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+ struct msm_priv *priv;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ struct msm_iommu_ctx_drvdata *tmp_drvdata;
+ int ret;
+
+ mutex_lock(&msm_iommu_lock);
+
+ priv = domain->priv;
+ if (!priv || !dev) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ iommu_drvdata = dev_get_drvdata(dev->parent);
+ ctx_drvdata = dev_get_drvdata(dev);
+ if (!iommu_drvdata || !ctx_drvdata) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ if (!list_empty(&ctx_drvdata->attached_elm)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm)
+ if (tmp_drvdata == ctx_drvdata) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ ret = regulator_enable(iommu_drvdata->gdsc);
+ if (ret)
+ goto fail;
+
+ ret = __enable_clocks(iommu_drvdata);
+ if (ret) {
+ regulator_disable(iommu_drvdata->gdsc);
+ goto fail;
+ }
+
+ if (!msm_iommu_ctx_attached(dev->parent))
+ __program_iommu(iommu_drvdata->base, iommu_drvdata->nsmr);
+
+ __program_context(iommu_drvdata->base, ctx_drvdata->num,
+ iommu_drvdata->ncb, __pa(priv->pt.fl_table),
+ priv->pt.redirect, ctx_drvdata->sids, ctx_drvdata->nsid,
+ iommu_drvdata->nsmr);
+ __disable_clocks(iommu_drvdata);
+
+ list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
+ ctx_drvdata->attached_domain = domain;
+
+fail:
+ mutex_unlock(&msm_iommu_lock);
+ return ret;
+}
+
+static void msm_iommu_detach_dev(struct iommu_domain *domain,
+ struct device *dev)
+{
+ struct msm_priv *priv;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ int ret;
+
+ mutex_lock(&msm_iommu_lock);
+ priv = domain->priv;
+ if (!priv || !dev)
+ goto fail;
+
+ iommu_drvdata = dev_get_drvdata(dev->parent);
+ ctx_drvdata = dev_get_drvdata(dev);
+ if (!iommu_drvdata || !ctx_drvdata || !ctx_drvdata->attached_domain)
+ goto fail;
+
+ ret = __enable_clocks(iommu_drvdata);
+ if (ret)
+ goto fail;
+
+ SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num,
+ GET_CB_CONTEXTIDR_ASID(iommu_drvdata->base, ctx_drvdata->num));
+
+ __reset_context(iommu_drvdata->base, ctx_drvdata->num);
+ __disable_clocks(iommu_drvdata);
+
+ regulator_disable(iommu_drvdata->gdsc);
+
+ list_del_init(&ctx_drvdata->attached_elm);
+ ctx_drvdata->attached_domain = NULL;
+
+fail:
+ mutex_unlock(&msm_iommu_lock);
+}
+
+static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
+ phys_addr_t pa, size_t len, int prot)
+{
+ struct msm_priv *priv;
+ int ret = 0;
+
+ mutex_lock(&msm_iommu_lock);
+
+ priv = domain->priv;
+ if (!priv) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ ret = msm_iommu_pagetable_map(&priv->pt, va, pa, len, prot);
+ if (ret)
+ goto fail;
+
+ ret = __flush_iotlb_va(domain, va);
+fail:
+ mutex_unlock(&msm_iommu_lock);
+ return ret;
+}
+
+static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
+ size_t len)
+{
+ struct msm_priv *priv;
+ int ret = -ENODEV;
+
+ mutex_lock(&msm_iommu_lock);
+
+ priv = domain->priv;
+ if (!priv)
+ goto fail;
+
+ ret = msm_iommu_pagetable_unmap(&priv->pt, va, len);
+ if (ret < 0)
+ goto fail;
+
+ ret = __flush_iotlb_va(domain, va);
+fail:
+ mutex_unlock(&msm_iommu_lock);
+
+ /* the IOMMU API requires us to return how many bytes were unmapped */
+ len = ret ? 0 : len;
+ return len;
+}
+
+static int msm_iommu_map_range(struct iommu_domain *domain, unsigned int va,
+ struct scatterlist *sg, unsigned int len,
+ int prot)
+{
+ int ret;
+ struct msm_priv *priv;
+
+ mutex_lock(&msm_iommu_lock);
+
+ priv = domain->priv;
+ if (!priv) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ ret = msm_iommu_pagetable_map_range(&priv->pt, va, sg, len, prot);
+ if (ret)
+ goto fail;
+
+ __flush_iotlb(domain);
+fail:
+ mutex_unlock(&msm_iommu_lock);
+ return ret;
+}
+
+
+static int msm_iommu_unmap_range(struct iommu_domain *domain, unsigned int va,
+ unsigned int len)
+{
+ struct msm_priv *priv;
+
+ mutex_lock(&msm_iommu_lock);
+
+ priv = domain->priv;
+ msm_iommu_pagetable_unmap_range(&priv->pt, va, len);
+
+ __flush_iotlb(domain);
+ mutex_unlock(&msm_iommu_lock);
+ return 0;
+}
+
+static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
+ unsigned long va)
+{
+ struct msm_priv *priv;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ unsigned int par;
+ void __iomem *base;
+ phys_addr_t ret = 0;
+ int ctx;
+
+ mutex_lock(&msm_iommu_lock);
+
+ priv = domain->priv;
+ if (list_empty(&priv->list_attached))
+ goto fail;
+
+ ctx_drvdata = list_entry(priv->list_attached.next,
+ struct msm_iommu_ctx_drvdata, attached_elm);
+ iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
+
+ base = iommu_drvdata->base;
+ ctx = ctx_drvdata->num;
+
+ ret = __enable_clocks(iommu_drvdata);
+ if (ret) {
+ ret = 0; /* 0 indicates translation failed */
+ goto fail;
+ }
+
+ SET_ATS1PR(base, ctx, va & CB_ATS1PR_ADDR);
+ mb();
+ while (GET_CB_ATSR_ACTIVE(base, ctx))
+ cpu_relax();
+
+ par = GET_PAR(base, ctx);
+ __disable_clocks(iommu_drvdata);
+
+ if (par & CB_PAR_F) {
+ ret = 0;
+ } else {
+ /* We are dealing with a supersection */
+ if (ret & CB_PAR_SS)
+ ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
+ else /* Upper 20 bits from PAR, lower 12 from VA */
+ ret = (par & 0xFFFFF000) | (va & 0x00000FFF);
+ }
+
+fail:
+ mutex_unlock(&msm_iommu_lock);
+ return ret;
+}
+
+static int msm_iommu_domain_has_cap(struct iommu_domain *domain,
+ unsigned long cap)
+{
+ return 0;
+}
+
+static void print_ctx_regs(void __iomem *base, int ctx, unsigned int fsr)
+{
+ pr_err("FAR = %08x PAR = %08x\n",
+ GET_FAR(base, ctx), GET_PAR(base, ctx));
+ pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s]\n", fsr,
+ (fsr & 0x02) ? "TF " : "",
+ (fsr & 0x04) ? "AFF " : "",
+ (fsr & 0x08) ? "PF " : "",
+ (fsr & 0x10) ? "EF " : "",
+ (fsr & 0x20) ? "TLBMCF " : "",
+ (fsr & 0x40) ? "TLBLKF " : "",
+ (fsr & 0x80) ? "MHF " : "",
+ (fsr & 0x40000000) ? "SS " : "",
+ (fsr & 0x80000000) ? "MULTI " : "");
+
+ pr_err("FSYNR0 = %08x FSYNR1 = %08x\n",
+ GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx));
+ pr_err("TTBR0 = %08x TTBR1 = %08x\n",
+ GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));
+ pr_err("SCTLR = %08x ACTLR = %08x\n",
+ GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));
+ pr_err("PRRR = %08x NMRR = %08x\n",
+ GET_PRRR(base, ctx), GET_NMRR(base, ctx));
+}
+
+irqreturn_t msm_iommu_fault_handler_v2(int irq, void *dev_id)
+{
+ struct platform_device *pdev = dev_id;
+ struct msm_iommu_drvdata *drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ unsigned int fsr;
+ int ret;
+
+ mutex_lock(&msm_iommu_lock);
+
+ BUG_ON(!pdev);
+
+ drvdata = dev_get_drvdata(pdev->dev.parent);
+ BUG_ON(!drvdata);
+
+ ctx_drvdata = dev_get_drvdata(&pdev->dev);
+ BUG_ON(!ctx_drvdata);
+
+ ret = __enable_clocks(drvdata);
+ if (ret) {
+ ret = IRQ_NONE;
+ goto fail;
+ }
+
+ fsr = GET_FSR(drvdata->base, ctx_drvdata->num);
+ if (fsr) {
+ if (!ctx_drvdata->attached_domain) {
+ pr_err("Bad domain in interrupt handler\n");
+ ret = -ENOSYS;
+ } else
+ ret = report_iommu_fault(ctx_drvdata->attached_domain,
+ &ctx_drvdata->pdev->dev,
+ GET_FAR(drvdata->base, ctx_drvdata->num), 0);
+
+ if (ret == -ENOSYS) {
+ pr_err("Unexpected IOMMU page fault!\n");
+ pr_err("name = %s\n", drvdata->name);
+ pr_err("context = %s (%d)\n", ctx_drvdata->name,
+ ctx_drvdata->num);
+ pr_err("Interesting registers:\n");
+ print_ctx_regs(drvdata->base, ctx_drvdata->num, fsr);
+ }
+
+ SET_FSR(drvdata->base, ctx_drvdata->num, fsr);
+ ret = IRQ_HANDLED;
+ } else
+ ret = IRQ_NONE;
+
+ __disable_clocks(drvdata);
+fail:
+ mutex_unlock(&msm_iommu_lock);
+ return ret;
+}
+
+static phys_addr_t msm_iommu_get_pt_base_addr(struct iommu_domain *domain)
+{
+ struct msm_priv *priv = domain->priv;
+ return __pa(priv->pt.fl_table);
+}
+
+static struct iommu_ops msm_iommu_ops = {
+ .domain_init = msm_iommu_domain_init,
+ .domain_destroy = msm_iommu_domain_destroy,
+ .attach_dev = msm_iommu_attach_dev,
+ .detach_dev = msm_iommu_detach_dev,
+ .map = msm_iommu_map,
+ .unmap = msm_iommu_unmap,
+ .map_range = msm_iommu_map_range,
+ .unmap_range = msm_iommu_unmap_range,
+ .iova_to_phys = msm_iommu_iova_to_phys,
+ .domain_has_cap = msm_iommu_domain_has_cap,
+ .get_pt_base_addr = msm_iommu_get_pt_base_addr,
+ .pgsize_bitmap = MSM_IOMMU_PGSIZES,
+};
+
+static int __init msm_iommu_init(void)
+{
+ msm_iommu_pagetable_init();
+ bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
+ return 0;
+}
+
+subsys_initcall(msm_iommu_init);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MSM SMMU v2 Driver");
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index ac26522..05bb4a9 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2012, Code Aurora Forum. All rights reserved.
+/* Copyright (c) 2010-2012, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -31,8 +31,6 @@
#include <mach/iommu.h>
#include <mach/msm_smsm.h>
-#include <linux/delay.h>
-
#define MRC(reg, processor, op1, crn, crm, op2) \
__asm__ __volatile__ ( \
" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \
@@ -41,9 +39,11 @@
#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
+/* Sharability attributes of MSM IOMMU mappings */
#define MSM_IOMMU_ATTR_NON_SH 0x0
#define MSM_IOMMU_ATTR_SH 0x4
+/* Cacheability attributes of MSM IOMMU mappings */
#define MSM_IOMMU_ATTR_NONCACHED 0x0
#define MSM_IOMMU_ATTR_CACHED_WB_WA 0x1
#define MSM_IOMMU_ATTR_CACHED_WB_NWA 0x2
@@ -57,19 +57,20 @@
dmac_flush_range(start, end);
}
+/* bitmap of the page sizes currently supported */
#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
static int msm_iommu_tex_class[4];
-static volatile struct iommu_domain *flush_iotlb_domain = NULL;
-static volatile int flush_iotlb_footprint = 0;
-static volatile struct msm_iommu_ctx_drvdata *flush_iotlb_ctx_drvdata = NULL;
-static volatile struct iommu_domain *flush_iotlb_va_domain = NULL;
-static volatile int flush_iotlb_va_footprint = 0;
-static volatile struct msm_iommu_ctx_drvdata *flush_iotlb_va_ctx_drvdata = NULL;
-
DEFINE_MUTEX(msm_iommu_lock);
+/**
+ * Remote spinlock implementation based on Peterson's algorithm to be used
+ * to synchronize IOMMU config port access between CPU and GPU.
+ * This implements Process 0 of the spin lock algorithm. GPU implements
+ * Process 1. Flag and turn is stored in shared memory to allow GPU to
+ * access these.
+ */
struct msm_iommu_remote_lock {
int initialized;
struct remote_iommu_petersons_spinlock *lock;
@@ -164,25 +165,15 @@
int ret = 0;
int asid;
- flush_iotlb_va_domain = domain;
- flush_iotlb_va_footprint = 0;
- dsb();
-
list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
- flush_iotlb_va_ctx_drvdata = ctx_drvdata;
if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent)
BUG();
iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
- flush_iotlb_va_footprint = 1;
- dsb();
-
if (!iommu_drvdata)
BUG();
ret = __enable_clocks(iommu_drvdata);
- flush_iotlb_va_footprint = 2;
- dsb();
if (ret)
goto fail;
@@ -190,26 +181,16 @@
asid = GET_CONTEXTIDR_ASID(iommu_drvdata->base,
ctx_drvdata->num);
- flush_iotlb_va_footprint = 3;
- dsb();
SET_TLBIVA(iommu_drvdata->base, ctx_drvdata->num,
asid | (va & TLBIVA_VA));
- flush_iotlb_va_footprint = 4;
-
mb();
msm_iommu_remote_spin_unlock();
__disable_clocks(iommu_drvdata);
- flush_iotlb_va_footprint = 5;
- dsb();
}
fail:
- flush_iotlb_va_footprint = 6;
- flush_iotlb_va_domain = NULL;
- flush_iotlb_va_ctx_drvdata = NULL;
- dsb();
return ret;
}
@@ -221,24 +202,15 @@
int ret = 0;
int asid;
- flush_iotlb_footprint = 0;
- flush_iotlb_domain = domain;
- dsb();
-
list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
- flush_iotlb_ctx_drvdata = ctx_drvdata;
if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent)
BUG();
iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
if (!iommu_drvdata)
BUG();
- flush_iotlb_footprint = 1;
- dsb();
ret = __enable_clocks(iommu_drvdata);
- flush_iotlb_footprint = 2;
- dsb();
if (ret)
goto fail;
@@ -246,25 +218,15 @@
asid = GET_CONTEXTIDR_ASID(iommu_drvdata->base,
ctx_drvdata->num);
- flush_iotlb_footprint = 3;
- dsb();
SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num, asid);
- flush_iotlb_footprint = 4;
mb();
msm_iommu_remote_spin_unlock();
__disable_clocks(iommu_drvdata);
- flush_iotlb_footprint = 5;
- dsb();
}
fail:
- WARN_ON(ret);
- flush_iotlb_footprint = 6;
- flush_iotlb_domain = NULL;
- flush_iotlb_ctx_drvdata = NULL;
- dsb();
return ret;
}
@@ -303,11 +265,11 @@
__reset_context(base, ctx);
-
-
+ /* Set up HTW mode */
+ /* TLB miss configuration: perform HTW on miss */
SET_TLBMCFG(base, ctx, 0x3);
-
+ /* V2P configuration: HTW for access */
SET_V2PCFG(base, ctx, 0x3);
SET_TTBCR(base, ctx, ttbr_split);
@@ -315,29 +277,32 @@
if (ttbr_split)
SET_TTBR1_PA(base, ctx, (pgtable >> TTBR1_PA_SHIFT));
-
+ /* Enable context fault interrupt */
SET_CFEIE(base, ctx, 1);
-
+ /* Stall access on a context fault and let the handler deal with it */
SET_CFCFG(base, ctx, 1);
-
+ /* Redirect all cacheable requests to L2 slave port. */
SET_RCISH(base, ctx, 1);
SET_RCOSH(base, ctx, 1);
SET_RCNSH(base, ctx, 1);
-
+ /* Turn on TEX Remap */
SET_TRE(base, ctx, 1);
-
+ /* Set TEX remap attributes */
RCP15_PRRR(prrr);
RCP15_NMRR(nmrr);
SET_PRRR(base, ctx, prrr);
SET_NMRR(base, ctx, nmrr);
-
+ /* Turn on BFB prefetch */
SET_BFBDFE(base, ctx, 1);
+ /* Configure page tables as inner-cacheable and shareable to reduce
+ * the TLB miss penalty.
+ */
if (redirect) {
SET_TTBR0_SH(base, ctx, 1);
SET_TTBR1_SH(base, ctx, 1);
@@ -345,17 +310,17 @@
SET_TTBR0_NOS(base, ctx, 1);
SET_TTBR1_NOS(base, ctx, 1);
- SET_TTBR0_IRGNH(base, ctx, 0);
+ SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */
SET_TTBR0_IRGNL(base, ctx, 1);
- SET_TTBR1_IRGNH(base, ctx, 0);
+ SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */
SET_TTBR1_IRGNL(base, ctx, 1);
- SET_TTBR0_ORGN(base, ctx, 1);
- SET_TTBR1_ORGN(base, ctx, 1);
+ SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */
+ SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */
}
-
+ /* Find if this page table is used elsewhere, and re-use ASID */
found = 0;
for (i = 0; i < ncb; i++)
if (GET_TTBR0_PA(base, i) == (pgtable >> TTBR0_PA_SHIFT) &&
@@ -366,7 +331,7 @@
break;
}
-
+ /* If page table is new, find an unused ASID */
if (!found) {
for (i = 0; i < ncb; i++) {
found = 0;
@@ -384,7 +349,7 @@
BUG_ON(found);
}
-
+ /* Enable the MMU */
SET_M(base, ctx, 1);
mb();
@@ -586,6 +551,92 @@
return pgprot;
}
+static unsigned long *make_second_level(struct msm_priv *priv,
+ unsigned long *fl_pte)
+{
+ unsigned long *sl;
+ sl = (unsigned long *) __get_free_pages(GFP_KERNEL,
+ get_order(SZ_4K));
+
+ if (!sl) {
+ pr_debug("Could not allocate second level table\n");
+ goto fail;
+ }
+ memset(sl, 0, SZ_4K);
+ clean_pte(sl, sl + NUM_SL_PTE, priv->redirect);
+
+ *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | \
+ FL_TYPE_TABLE);
+
+ clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+fail:
+ return sl;
+}
+
+static int sl_4k(unsigned long *sl_pte, phys_addr_t pa, unsigned int pgprot)
+{
+ int ret = 0;
+
+ if (*sl_pte) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_NG | SL_SHARED
+ | SL_TYPE_SMALL | pgprot;
+fail:
+ return ret;
+}
+
+static int sl_64k(unsigned long *sl_pte, phys_addr_t pa, unsigned int pgprot)
+{
+ int ret = 0;
+
+ int i;
+
+ for (i = 0; i < 16; i++)
+ if (*(sl_pte+i)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ for (i = 0; i < 16; i++)
+ *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_NG
+ | SL_SHARED | SL_TYPE_LARGE | pgprot;
+
+fail:
+ return ret;
+}
+
+
+static inline int fl_1m(unsigned long *fl_pte, phys_addr_t pa, int pgprot)
+{
+ if (*fl_pte)
+ return -EBUSY;
+
+ *fl_pte = (pa & 0xFFF00000) | FL_NG | FL_TYPE_SECT | FL_SHARED
+ | pgprot;
+
+ return 0;
+}
+
+
+static inline int fl_16m(unsigned long *fl_pte, phys_addr_t pa, int pgprot)
+{
+ int i;
+ int ret = 0;
+ for (i = 0; i < 16; i++)
+ if (*(fl_pte+i)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+ for (i = 0; i < 16; i++)
+ *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION
+ | FL_TYPE_SECT | FL_SHARED | FL_NG | pgprot;
+fail:
+ return ret;
+}
+
static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
phys_addr_t pa, size_t len, int prot)
{
@@ -629,55 +680,31 @@
goto fail;
}
- fl_offset = FL_OFFSET(va);
- fl_pte = fl_table + fl_offset;
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
if (len == SZ_16M) {
- int i = 0;
-
- for (i = 0; i < 16; i++)
- if (*(fl_pte+i)) {
- ret = -EBUSY;
- goto fail;
- }
-
- for (i = 0; i < 16; i++)
- *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION
- | FL_TYPE_SECT | FL_SHARED | FL_NG | pgprot;
+ ret = fl_16m(fl_pte, pa, pgprot);
+ if (ret)
+ goto fail;
clean_pte(fl_pte, fl_pte + 16, priv->redirect);
}
if (len == SZ_1M) {
- if (*fl_pte) {
- ret = -EBUSY;
+ ret = fl_1m(fl_pte, pa, pgprot);
+ if (ret)
goto fail;
- }
-
- *fl_pte = (pa & 0xFFF00000) | FL_NG | FL_TYPE_SECT | FL_SHARED
- | pgprot;
clean_pte(fl_pte, fl_pte + 1, priv->redirect);
}
-
+ /* Need a 2nd level table */
if (len == SZ_4K || len == SZ_64K) {
if (*fl_pte == 0) {
- unsigned long *sl;
- sl = (unsigned long *) __get_free_pages(GFP_KERNEL,
- get_order(SZ_4K));
-
- if (!sl) {
- pr_debug("Could not allocate second level table\n");
+ if (make_second_level(priv, fl_pte) == NULL) {
ret = -ENOMEM;
goto fail;
}
- memset(sl, 0, SZ_4K);
- clean_pte(sl, sl + NUM_SL_PTE, priv->redirect);
-
- *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | \
- FL_TYPE_TABLE);
-
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
}
if (!(*fl_pte & FL_TYPE_TABLE)) {
@@ -691,29 +718,17 @@
sl_pte = sl_table + sl_offset;
if (len == SZ_4K) {
- if (*sl_pte) {
- ret = -EBUSY;
+ ret = sl_4k(sl_pte, pa, pgprot);
+ if (ret)
goto fail;
- }
- *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_NG | SL_SHARED
- | SL_TYPE_SMALL | pgprot;
clean_pte(sl_pte, sl_pte + 1, priv->redirect);
}
if (len == SZ_64K) {
- int i;
-
- for (i = 0; i < 16; i++)
- if (*(sl_pte+i)) {
- ret = -EBUSY;
- goto fail;
- }
-
- for (i = 0; i < 16; i++)
- *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_NG
- | SL_SHARED | SL_TYPE_LARGE | pgprot;
-
+ ret = sl_64k(sl_pte, pa, pgprot);
+ if (ret)
+ goto fail;
clean_pte(sl_pte, sl_pte + 16, priv->redirect);
}
@@ -755,15 +770,15 @@
goto fail;
}
- fl_offset = FL_OFFSET(va);
- fl_pte = fl_table + fl_offset;
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
if (*fl_pte == 0) {
pr_debug("First level PTE is 0\n");
goto fail;
}
-
+ /* Unmap supersection */
if (len == SZ_16M) {
for (i = 0; i < 16; i++)
*(fl_pte+i) = 0;
@@ -813,35 +828,46 @@
fail:
mutex_unlock(&msm_iommu_lock);
-
+ /* the IOMMU API requires us to return how many bytes were unmapped */
len = ret ? 0 : len;
return len;
}
static unsigned int get_phys_addr(struct scatterlist *sg)
{
+ /*
+ * Try sg_dma_address first so that we can
+ * map carveout regions that do not have a
+ * struct page associated with them.
+ */
unsigned int pa = sg_dma_address(sg);
if (pa == 0)
pa = sg_phys(sg);
return pa;
}
+static inline int is_fully_aligned(unsigned int va, phys_addr_t pa, size_t len,
+ int align)
+{
+ return IS_ALIGNED(va, align) && IS_ALIGNED(pa, align)
+ && (len >= align);
+}
+
static int msm_iommu_map_range(struct iommu_domain *domain, unsigned int va,
struct scatterlist *sg, unsigned int len,
int prot)
{
unsigned int pa;
unsigned int offset = 0;
- unsigned int pgprot;
unsigned long *fl_table;
unsigned long *fl_pte;
unsigned long fl_offset;
- unsigned long *sl_table;
+ unsigned long *sl_table = NULL;
unsigned long sl_offset, sl_start;
- unsigned int chunk_offset = 0;
- unsigned int chunk_pa;
+ unsigned int chunk_size, chunk_offset = 0;
int ret = 0;
struct msm_priv *priv;
+ unsigned int pgprot4k, pgprot64k, pgprot1m, pgprot16m;
mutex_lock(&msm_iommu_lock);
@@ -850,68 +876,118 @@
priv = domain->priv;
fl_table = priv->pgtable;
- pgprot = __get_pgprot(prot, SZ_4K);
+ pgprot4k = __get_pgprot(prot, SZ_4K);
+ pgprot64k = __get_pgprot(prot, SZ_64K);
+ pgprot1m = __get_pgprot(prot, SZ_1M);
+ pgprot16m = __get_pgprot(prot, SZ_16M);
- if (!pgprot) {
+ if (!pgprot4k || !pgprot64k || !pgprot1m || !pgprot16m) {
ret = -EINVAL;
goto fail;
}
- fl_offset = FL_OFFSET(va);
- fl_pte = fl_table + fl_offset;
-
- sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
- sl_offset = SL_OFFSET(va);
-
- chunk_pa = get_phys_addr(sg);
- if (chunk_pa == 0) {
- pr_debug("No dma address for sg %p\n", sg);
- ret = -EINVAL;
- goto fail;
- }
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
+ pa = get_phys_addr(sg);
while (offset < len) {
-
- if (*fl_pte == 0) {
- sl_table = (unsigned long *)
- __get_free_pages(GFP_KERNEL, get_order(SZ_4K));
+ chunk_size = SZ_4K;
- if (!sl_table) {
- pr_debug("Could not allocate second level table\n");
- ret = -ENOMEM;
- goto fail;
+ if (is_fully_aligned(va, pa, sg->length - chunk_offset,
+ SZ_16M))
+ chunk_size = SZ_16M;
+ else if (is_fully_aligned(va, pa, sg->length - chunk_offset,
+ SZ_1M))
+ chunk_size = SZ_1M;
+ /* 64k or 4k determined later */
+
+ /* for 1M and 16M, only first level entries are required */
+ if (chunk_size >= SZ_1M) {
+ if (chunk_size == SZ_16M) {
+ ret = fl_16m(fl_pte, pa, pgprot16m);
+ if (ret)
+ goto fail;
+ clean_pte(fl_pte, fl_pte + 16, priv->redirect);
+ fl_pte += 16;
+ } else if (chunk_size == SZ_1M) {
+ ret = fl_1m(fl_pte, pa, pgprot1m);
+ if (ret)
+ goto fail;
+ clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ fl_pte++;
}
- memset(sl_table, 0, SZ_4K);
- clean_pte(sl_table, sl_table + NUM_SL_PTE,
- priv->redirect);
-
- *fl_pte = ((((int)__pa(sl_table)) & FL_BASE_MASK) |
- FL_TYPE_TABLE);
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
- } else
- sl_table = (unsigned long *)
- __va(((*fl_pte) & FL_BASE_MASK));
-
- sl_start = sl_offset;
-
-
- while (offset < len && sl_offset < NUM_SL_PTE) {
- pa = chunk_pa + chunk_offset;
- sl_table[sl_offset] = (pa & SL_BASE_MASK_SMALL) |
- pgprot | SL_NG | SL_SHARED | SL_TYPE_SMALL;
- sl_offset++;
- offset += SZ_4K;
-
- chunk_offset += SZ_4K;
+ offset += chunk_size;
+ chunk_offset += chunk_size;
+ va += chunk_size;
+ pa += chunk_size;
if (chunk_offset >= sg->length && offset < len) {
chunk_offset = 0;
sg = sg_next(sg);
- chunk_pa = get_phys_addr(sg);
- if (chunk_pa == 0) {
+ pa = get_phys_addr(sg);
+ if (pa == 0) {
pr_debug("No dma address for sg %p\n",
- sg);
+ sg);
+ ret = -EINVAL;
+ goto fail;
+ }
+ }
+ continue;
+ }
+ /* for 4K or 64K, make sure there is a second level table */
+ if (*fl_pte == 0) {
+ if (!make_second_level(priv, fl_pte)) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+ }
+ if (!(*fl_pte & FL_TYPE_TABLE)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+ sl_table = __va(((*fl_pte) & FL_BASE_MASK));
+ sl_offset = SL_OFFSET(va);
+ /* Keep track of initial position so we
+ * don't clean more than we have to
+ */
+ sl_start = sl_offset;
+
+ /* Build the 2nd level page table */
+ while (offset < len && sl_offset < NUM_SL_PTE) {
+
+ /* Map a large 64K page if the chunk is large enough and
+ * the pa and va are aligned
+ */
+
+ if (is_fully_aligned(va, pa, sg->length - chunk_offset,
+ SZ_64K))
+ chunk_size = SZ_64K;
+ else
+ chunk_size = SZ_4K;
+
+ if (chunk_size == SZ_4K) {
+ sl_4k(&sl_table[sl_offset], pa, pgprot4k);
+ sl_offset++;
+ } else {
+ BUG_ON(sl_offset + 16 > NUM_SL_PTE);
+ sl_64k(&sl_table[sl_offset], pa, pgprot64k);
+ sl_offset += 16;
+ }
+
+
+ offset += chunk_size;
+ chunk_offset += chunk_size;
+ va += chunk_size;
+ pa += chunk_size;
+
+ if (chunk_offset >= sg->length && offset < len) {
+ chunk_offset = 0;
+ sg = sg_next(sg);
+ pa = get_phys_addr(sg);
+ if (pa == 0) {
+ pr_debug("No dma address for sg %p\n",
+ sg);
ret = -EINVAL;
goto fail;
}
@@ -919,7 +995,7 @@
}
clean_pte(sl_table + sl_start, sl_table + sl_offset,
- priv->redirect);
+ priv->redirect);
fl_pte++;
sl_offset = 0;
@@ -950,40 +1026,56 @@
priv = domain->priv;
fl_table = priv->pgtable;
- fl_offset = FL_OFFSET(va);
- fl_pte = fl_table + fl_offset;
-
- sl_start = SL_OFFSET(va);
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
while (offset < len) {
- sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
- sl_end = ((len - offset) / SZ_4K) + sl_start;
+ if (*fl_pte & FL_TYPE_TABLE) {
+ sl_start = SL_OFFSET(va);
+ sl_table = __va(((*fl_pte) & FL_BASE_MASK));
+ sl_end = ((len - offset) / SZ_4K) + sl_start;
- if (sl_end > NUM_SL_PTE)
- sl_end = NUM_SL_PTE;
+ if (sl_end > NUM_SL_PTE)
+ sl_end = NUM_SL_PTE;
- memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
- clean_pte(sl_table + sl_start, sl_table + sl_end,
- priv->redirect);
+ memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
+ clean_pte(sl_table + sl_start, sl_table + sl_end,
+ priv->redirect);
- offset += (sl_end - sl_start) * SZ_4K;
+ offset += (sl_end - sl_start) * SZ_4K;
+ va += (sl_end - sl_start) * SZ_4K;
- used = 0;
+ /* Unmap and free the 2nd level table if all mappings
+ * in it were removed. This saves memory, but the table
+ * will need to be re-allocated the next time someone
+ * tries to map these VAs.
+ */
+ used = 0;
- if (sl_end - sl_start != NUM_SL_PTE)
- for (i = 0; i < NUM_SL_PTE; i++)
- if (sl_table[i]) {
- used = 1;
- break;
- }
- if (!used) {
- free_page((unsigned long)sl_table);
+ /* If we just unmapped the whole table, don't bother
+ * seeing if there are still used entries left.
+ */
+ if (sl_end - sl_start != NUM_SL_PTE)
+ for (i = 0; i < NUM_SL_PTE; i++)
+ if (sl_table[i]) {
+ used = 1;
+ break;
+ }
+ if (!used) {
+ free_page((unsigned long)sl_table);
+ *fl_pte = 0;
+
+ clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ }
+
+ sl_start = 0;
+ } else {
*fl_pte = 0;
-
clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ va += SZ_1M;
+ offset += SZ_1M;
+ sl_start = 0;
}
-
- sl_start = 0;
fl_pte++;
}
@@ -1027,10 +1119,10 @@
mb();
par = GET_PAR(base, ctx);
-
+ /* We are dealing with a supersection */
if (GET_NOFAULT_SS(base, ctx))
ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
- else
+ else /* Upper 20 bits from PAR, lower 12 from VA */
ret = (par & 0xFFFFF000) | (va & 0x00000FFF);
if (GET_FAULT(base, ctx))
diff --git a/drivers/iommu/msm_iommu_dev-v2.c b/drivers/iommu/msm_iommu_dev-v2.c
new file mode 100644
index 0000000..4c69c8c
--- /dev/null
+++ b/drivers/iommu/msm_iommu_dev-v2.c
@@ -0,0 +1,308 @@
+/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/iommu.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+
+#include <mach/iommu_hw-v2.h>
+#include <mach/iommu.h>
+
+static int msm_iommu_parse_dt(struct platform_device *pdev,
+ struct msm_iommu_drvdata *drvdata)
+{
+ struct device_node *child;
+ int ret = 0;
+ u32 nsmr;
+
+ ret = device_move(&pdev->dev, &msm_iommu_root_dev->dev, DPM_ORDER_NONE);
+ if (ret)
+ goto fail;
+
+ ret = of_property_read_u32(pdev->dev.of_node, "qcom,iommu-smt-size",
+ &nsmr);
+ if (ret)
+ goto fail;
+
+ if (nsmr > MAX_NUM_SMR) {
+ pr_err("Invalid SMT size: %d\n", nsmr);
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ drvdata->nsmr = nsmr;
+ for_each_child_of_node(pdev->dev.of_node, child) {
+ drvdata->ncb++;
+ if (!of_platform_device_create(child, NULL, &pdev->dev))
+ pr_err("Failed to create %s device\n", child->name);
+ }
+
+ drvdata->name = dev_name(&pdev->dev);
+fail:
+ return ret;
+}
+
+static atomic_t msm_iommu_next_id = ATOMIC_INIT(-1);
+
+static int __devinit msm_iommu_probe(struct platform_device *pdev)
+{
+ struct msm_iommu_drvdata *drvdata;
+ struct resource *r;
+ int ret, needs_alt_core_clk;
+
+ if (msm_iommu_root_dev == pdev)
+ return 0;
+
+ if (pdev->id == -1)
+ pdev->id = atomic_inc_return(&msm_iommu_next_id) - 1;
+
+ drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);
+ if (!drvdata)
+ return -ENOMEM;
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!r)
+ return -EINVAL;
+
+ drvdata->base = devm_ioremap(&pdev->dev, r->start, resource_size(r));
+ if (!drvdata->base)
+ return -ENOMEM;
+
+ drvdata->gdsc = devm_regulator_get(&pdev->dev, "vdd");
+ if (IS_ERR(drvdata->gdsc))
+ return -EINVAL;
+
+ drvdata->pclk = devm_clk_get(&pdev->dev, "iface_clk");
+ if (IS_ERR(drvdata->pclk))
+ return PTR_ERR(drvdata->pclk);
+
+ drvdata->clk = devm_clk_get(&pdev->dev, "core_clk");
+ if (IS_ERR(drvdata->clk))
+ return PTR_ERR(drvdata->clk);
+
+ needs_alt_core_clk = of_property_read_bool(pdev->dev.of_node,
+ "qcom,needs-alt-core-clk");
+ if (needs_alt_core_clk) {
+ drvdata->aclk = devm_clk_get(&pdev->dev, "alt_core_clk");
+ if (IS_ERR(drvdata->aclk))
+ return PTR_ERR(drvdata->aclk);
+ }
+
+ if (clk_get_rate(drvdata->clk) == 0) {
+ ret = clk_round_rate(drvdata->clk, 1);
+ clk_set_rate(drvdata->clk, ret);
+ }
+
+ if (drvdata->aclk && clk_get_rate(drvdata->aclk) == 0) {
+ ret = clk_round_rate(drvdata->aclk, 1);
+ clk_set_rate(drvdata->aclk, ret);
+ }
+
+ ret = msm_iommu_parse_dt(pdev, drvdata);
+ if (ret)
+ return ret;
+
+ pr_info("device %s mapped at %p, with %d ctx banks\n",
+ drvdata->name, drvdata->base, drvdata->ncb);
+
+ platform_set_drvdata(pdev, drvdata);
+
+ return 0;
+}
+
+static int __devexit msm_iommu_remove(struct platform_device *pdev)
+{
+ struct msm_iommu_drvdata *drv = NULL;
+
+ drv = platform_get_drvdata(pdev);
+ if (drv) {
+ if (drv->clk)
+ clk_put(drv->clk);
+ clk_put(drv->pclk);
+ platform_set_drvdata(pdev, NULL);
+ }
+ return 0;
+}
+
+static int msm_iommu_ctx_parse_dt(struct platform_device *pdev,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata)
+{
+ struct resource *r, rp;
+ int irq, ret;
+ u32 nsid;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq > 0) {
+ ret = request_threaded_irq(irq, NULL,
+ msm_iommu_fault_handler_v2,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "msm_iommu_nonsecure_irq", pdev);
+ if (ret) {
+ pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
+ return ret;
+ }
+ }
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!r)
+ return -EINVAL;
+
+ ret = of_address_to_resource(pdev->dev.parent->of_node, 0, &rp);
+ if (ret)
+ return -EINVAL;
+
+ /* Calculate the context bank number using the base addresses. The
+ * first 8 pages belong to the global address space which is followed
+ * by the context banks, hence subtract by 8 to get the context bank
+ * number.
+ */
+ ctx_drvdata->num = ((r->start - rp.start) >> CTX_SHIFT) - 8;
+
+ if (of_property_read_string(pdev->dev.of_node, "label",
+ &ctx_drvdata->name))
+ ctx_drvdata->name = dev_name(&pdev->dev);
+
+ if (!of_get_property(pdev->dev.of_node, "qcom,iommu-ctx-sids", &nsid))
+ return -EINVAL;
+
+ if (nsid >= sizeof(ctx_drvdata->sids))
+ return -EINVAL;
+
+ if (of_property_read_u32_array(pdev->dev.of_node, "qcom,iommu-ctx-sids",
+ ctx_drvdata->sids,
+ nsid / sizeof(*ctx_drvdata->sids))) {
+ return -EINVAL;
+ }
+ ctx_drvdata->nsid = nsid;
+
+ return 0;
+}
+
+static int __devinit msm_iommu_ctx_probe(struct platform_device *pdev)
+{
+ struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;
+ int ret;
+
+ if (!pdev->dev.parent)
+ return -EINVAL;
+
+ ctx_drvdata = devm_kzalloc(&pdev->dev, sizeof(*ctx_drvdata),
+ GFP_KERNEL);
+ if (!ctx_drvdata)
+ return -ENOMEM;
+
+ ctx_drvdata->pdev = pdev;
+ INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
+ platform_set_drvdata(pdev, ctx_drvdata);
+
+ ret = msm_iommu_ctx_parse_dt(pdev, ctx_drvdata);
+ if (!ret)
+ dev_info(&pdev->dev, "context %s using bank %d\n",
+ ctx_drvdata->name, ctx_drvdata->num);
+
+ return ret;
+}
+
+static int __devexit msm_iommu_ctx_remove(struct platform_device *pdev)
+{
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+}
+
+static struct of_device_id msm_iommu_match_table[] = {
+ { .compatible = "qcom,msm-smmu-v2", },
+ {}
+};
+
+static struct platform_driver msm_iommu_driver = {
+ .driver = {
+ .name = "msm_iommu_v2",
+ .of_match_table = msm_iommu_match_table,
+ },
+ .probe = msm_iommu_probe,
+ .remove = __devexit_p(msm_iommu_remove),
+};
+
+static struct of_device_id msm_iommu_ctx_match_table[] = {
+ { .name = "qcom,iommu-ctx", },
+ {}
+};
+
+static struct platform_driver msm_iommu_ctx_driver = {
+ .driver = {
+ .name = "msm_iommu_ctx_v2",
+ .of_match_table = msm_iommu_ctx_match_table,
+ },
+ .probe = msm_iommu_ctx_probe,
+ .remove = __devexit_p(msm_iommu_ctx_remove),
+};
+
+static int __init msm_iommu_driver_init(void)
+{
+ struct device_node *node;
+ int ret;
+
+ node = of_find_compatible_node(NULL, NULL, "qcom,msm-smmu-v2");
+ if (!node)
+ return -ENODEV;
+
+ of_node_put(node);
+
+ msm_iommu_root_dev = platform_device_register_simple(
+ "msm_iommu", -1, 0, 0);
+ if (!msm_iommu_root_dev) {
+ pr_err("Failed to create root IOMMU device\n");
+ ret = -ENODEV;
+ goto error;
+ }
+
+ atomic_inc(&msm_iommu_next_id);
+
+ ret = platform_driver_register(&msm_iommu_driver);
+ if (ret != 0) {
+ pr_err("Failed to register IOMMU driver\n");
+ goto error;
+ }
+
+ ret = platform_driver_register(&msm_iommu_ctx_driver);
+ if (ret != 0) {
+ pr_err("Failed to register IOMMU context driver\n");
+ goto error;
+ }
+
+error:
+ return ret;
+}
+
+static void __exit msm_iommu_driver_exit(void)
+{
+ platform_driver_unregister(&msm_iommu_ctx_driver);
+ platform_driver_unregister(&msm_iommu_driver);
+ platform_device_unregister(msm_iommu_root_dev);
+}
+
+subsys_initcall(msm_iommu_driver_init);
+module_exit(msm_iommu_driver_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
index b4e4c1d..fca102a 100644
--- a/drivers/iommu/msm_iommu_dev.c
+++ b/drivers/iommu/msm_iommu_dev.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2012, Code Aurora Forum. All rights reserved.
+/* Copyright (c) 2010-2012, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -26,10 +26,10 @@
#include <mach/iommu.h>
struct iommu_ctx_iter_data {
-
+ /* input */
const char *name;
-
+ /* output */
struct device *dev;
};
@@ -331,7 +331,7 @@
}
}
-
+ /* Program the M2V tables for this context */
for (i = 0; i < MAX_NUM_MIDS; i++) {
int mid = c->mids[i];
if (mid == -1)
@@ -340,22 +340,22 @@
SET_M2VCBR_N(drvdata->base, mid, 0);
SET_CBACR_N(drvdata->base, c->num, 0);
-
+ /* Route page faults to the non-secure interrupt */
SET_IRPTNDX(drvdata->base, c->num, 1);
-
+ /* Set VMID = 0 */
SET_VMID(drvdata->base, mid, 0);
-
+ /* Set the context number for that MID to this context */
SET_CBNDX(drvdata->base, mid, c->num);
-
+ /* Set MID associated with this context bank to 0 */
SET_CBVMID(drvdata->base, c->num, 0);
-
+ /* Set the ASID for TLB tagging for this context to 0 */
SET_CONTEXTIDR_ASID(drvdata->base, c->num, 0);
-
+ /* Set security bit override to be Non-secure */
SET_NSCFG(drvdata->base, mid, 3);
}
mb();
diff --git a/drivers/iommu/msm_iommu_pagetable.c b/drivers/iommu/msm_iommu_pagetable.c
new file mode 100644
index 0000000..2ee9ba6
--- /dev/null
+++ b/drivers/iommu/msm_iommu_pagetable.c
@@ -0,0 +1,519 @@
+/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/iommu.h>
+#include <linux/scatterlist.h>
+
+#include <asm/cacheflush.h>
+
+#include <mach/iommu.h>
+#include "msm_iommu_pagetable.h"
+
+/* Sharability attributes of MSM IOMMU mappings */
+#define MSM_IOMMU_ATTR_NON_SH 0x0
+#define MSM_IOMMU_ATTR_SH 0x4
+
+/* Cacheability attributes of MSM IOMMU mappings */
+#define MSM_IOMMU_ATTR_NONCACHED 0x0
+#define MSM_IOMMU_ATTR_CACHED_WB_WA 0x1
+#define MSM_IOMMU_ATTR_CACHED_WB_NWA 0x2
+#define MSM_IOMMU_ATTR_CACHED_WT 0x3
+
+static int msm_iommu_tex_class[4];
+
+static inline void clean_pte(unsigned long *start, unsigned long *end,
+ int redirect)
+{
+ if (!redirect)
+ dmac_flush_range(start, end);
+}
+
+int msm_iommu_pagetable_alloc(struct iommu_pt *pt)
+{
+ pt->fl_table = (unsigned long *)__get_free_pages(GFP_KERNEL,
+ get_order(SZ_16K));
+ if (!pt->fl_table)
+ return -ENOMEM;
+
+ memset(pt->fl_table, 0, SZ_16K);
+ clean_pte(pt->fl_table, pt->fl_table + NUM_FL_PTE, pt->redirect);
+
+ return 0;
+}
+
+void msm_iommu_pagetable_free(struct iommu_pt *pt)
+{
+ unsigned long *fl_table;
+ int i;
+
+ fl_table = pt->fl_table;
+ for (i = 0; i < NUM_FL_PTE; i++)
+ if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
+ free_page((unsigned long) __va(((fl_table[i]) &
+ FL_BASE_MASK)));
+ free_pages((unsigned long)fl_table, get_order(SZ_16K));
+ pt->fl_table = 0;
+}
+
+static int __get_pgprot(int prot, int len)
+{
+ unsigned int pgprot;
+ int tex;
+
+ if (!(prot & (IOMMU_READ | IOMMU_WRITE))) {
+ prot |= IOMMU_READ | IOMMU_WRITE;
+ WARN_ONCE(1, "No attributes in iommu mapping; assuming RW\n");
+ }
+
+ if ((prot & IOMMU_WRITE) && !(prot & IOMMU_READ)) {
+ prot |= IOMMU_READ;
+ WARN_ONCE(1, "Write-only unsupported; falling back to RW\n");
+ }
+
+ if (prot & IOMMU_CACHE)
+ tex = (pgprot_kernel >> 2) & 0x07;
+ else
+ tex = msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED];
+
+ if (tex < 0 || tex > NUM_TEX_CLASS - 1)
+ return 0;
+
+ if (len == SZ_16M || len == SZ_1M) {
+ pgprot = FL_SHARED;
+ pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0;
+ pgprot |= tex & 0x02 ? FL_CACHEABLE : 0;
+ pgprot |= tex & 0x04 ? FL_TEX0 : 0;
+ pgprot |= FL_AP0 | FL_AP1;
+ pgprot |= prot & IOMMU_WRITE ? 0 : FL_AP2;
+ } else {
+ pgprot = SL_SHARED;
+ pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0;
+ pgprot |= tex & 0x02 ? SL_CACHEABLE : 0;
+ pgprot |= tex & 0x04 ? SL_TEX0 : 0;
+ pgprot |= SL_AP0 | SL_AP1;
+ pgprot |= prot & IOMMU_WRITE ? 0 : SL_AP2;
+ }
+
+ return pgprot;
+}
+
+int msm_iommu_pagetable_map(struct iommu_pt *pt, unsigned long va,
+ phys_addr_t pa, size_t len, int prot)
+{
+ unsigned long *fl_pte;
+ unsigned long fl_offset;
+ unsigned long *sl_table;
+ unsigned long *sl_pte;
+ unsigned long sl_offset;
+ unsigned int pgprot;
+ int ret = 0;
+
+ if (len != SZ_16M && len != SZ_1M &&
+ len != SZ_64K && len != SZ_4K) {
+ pr_debug("Bad size: %d\n", len);
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ if (!pt->fl_table) {
+ pr_debug("Null page table\n");
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ pgprot = __get_pgprot(prot, len);
+ if (!pgprot) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
+
+ if (len == SZ_16M) {
+ int i = 0;
+
+ for (i = 0; i < 16; i++)
+ if (*(fl_pte+i)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ for (i = 0; i < 16; i++)
+ *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION |
+ FL_TYPE_SECT | FL_SHARED | FL_NG | pgprot;
+ clean_pte(fl_pte, fl_pte + 16, pt->redirect);
+ }
+
+ if (len == SZ_1M) {
+ if (*fl_pte) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ *fl_pte = (pa & 0xFFF00000) | FL_NG | FL_TYPE_SECT
+ | FL_SHARED | pgprot;
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ }
+
+ /* Need a 2nd level table */
+ if (len == SZ_4K || len == SZ_64K) {
+
+ if (*fl_pte == 0) {
+ unsigned long *sl;
+ sl = (unsigned long *) __get_free_pages(GFP_KERNEL,
+ get_order(SZ_4K));
+
+ if (!sl) {
+ pr_debug("Could not allocate second level table\n");
+ ret = -ENOMEM;
+ goto fail;
+ }
+ memset(sl, 0, SZ_4K);
+ clean_pte(sl, sl + NUM_SL_PTE, pt->redirect);
+
+ *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | \
+ FL_TYPE_TABLE);
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ }
+
+ if (!(*fl_pte & FL_TYPE_TABLE)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+ }
+
+ sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
+ sl_offset = SL_OFFSET(va);
+ sl_pte = sl_table + sl_offset;
+
+ if (len == SZ_4K) {
+ if (*sl_pte) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_NG | SL_SHARED
+ | SL_TYPE_SMALL | pgprot;
+ clean_pte(sl_pte, sl_pte + 1, pt->redirect);
+ }
+
+ if (len == SZ_64K) {
+ int i;
+
+ for (i = 0; i < 16; i++)
+ if (*(sl_pte+i)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ for (i = 0; i < 16; i++)
+ *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_NG
+ | SL_SHARED | SL_TYPE_LARGE | pgprot;
+
+ clean_pte(sl_pte, sl_pte + 16, pt->redirect);
+ }
+
+fail:
+ return ret;
+}
+
+size_t msm_iommu_pagetable_unmap(struct iommu_pt *pt, unsigned long va,
+ size_t len)
+{
+ unsigned long *fl_pte;
+ unsigned long fl_offset;
+ unsigned long *sl_table;
+ unsigned long *sl_pte;
+ unsigned long sl_offset;
+ int i, ret = 0;
+
+ if (len != SZ_16M && len != SZ_1M &&
+ len != SZ_64K && len != SZ_4K) {
+ pr_debug("Bad length: %d\n", len);
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ if (!pt->fl_table) {
+ pr_debug("Null page table\n");
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
+
+ if (*fl_pte == 0) {
+ pr_debug("First level PTE is 0\n");
+ ret = -ENODEV;
+ goto fail;
+ }
+
+ /* Unmap supersection */
+ if (len == SZ_16M) {
+ for (i = 0; i < 16; i++)
+ *(fl_pte+i) = 0;
+
+ clean_pte(fl_pte, fl_pte + 16, pt->redirect);
+ }
+
+ if (len == SZ_1M) {
+ *fl_pte = 0;
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ }
+
+ sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
+ sl_offset = SL_OFFSET(va);
+ sl_pte = sl_table + sl_offset;
+
+ if (len == SZ_64K) {
+ for (i = 0; i < 16; i++)
+ *(sl_pte+i) = 0;
+
+ clean_pte(sl_pte, sl_pte + 16, pt->redirect);
+ }
+
+ if (len == SZ_4K) {
+ *sl_pte = 0;
+ clean_pte(sl_pte, sl_pte + 1, pt->redirect);
+ }
+
+ if (len == SZ_4K || len == SZ_64K) {
+ int used = 0;
+
+ for (i = 0; i < NUM_SL_PTE; i++)
+ if (sl_table[i])
+ used = 1;
+ if (!used) {
+ free_page((unsigned long)sl_table);
+ *fl_pte = 0;
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ }
+ }
+
+fail:
+ return ret;
+}
+
+static unsigned int get_phys_addr(struct scatterlist *sg)
+{
+ /*
+ * Try sg_dma_address first so that we can
+ * map carveout regions that do not have a
+ * struct page associated with them.
+ */
+ unsigned int pa = sg_dma_address(sg);
+ if (pa == 0)
+ pa = sg_phys(sg);
+ return pa;
+}
+
+int msm_iommu_pagetable_map_range(struct iommu_pt *pt, unsigned int va,
+ struct scatterlist *sg, unsigned int len, int prot)
+{
+ unsigned int pa;
+ unsigned int offset = 0;
+ unsigned int pgprot;
+ unsigned long *fl_pte;
+ unsigned long fl_offset;
+ unsigned long *sl_table;
+ unsigned long sl_offset, sl_start;
+ unsigned int chunk_offset = 0;
+ unsigned int chunk_pa;
+ int ret = 0;
+
+ BUG_ON(len & (SZ_4K - 1));
+
+ pgprot = __get_pgprot(prot, SZ_4K);
+ if (!pgprot) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
+
+ sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
+ sl_offset = SL_OFFSET(va);
+
+ chunk_pa = get_phys_addr(sg);
+ if (chunk_pa == 0) {
+ pr_debug("No dma address for sg %p\n", sg);
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ while (offset < len) {
+ /* Set up a 2nd level page table if one doesn't exist */
+ if (*fl_pte == 0) {
+ sl_table = (unsigned long *)
+ __get_free_pages(GFP_KERNEL, get_order(SZ_4K));
+
+ if (!sl_table) {
+ pr_debug("Could not allocate second level table\n");
+ ret = -ENOMEM;
+ goto fail;
+ }
+
+ memset(sl_table, 0, SZ_4K);
+ clean_pte(sl_table, sl_table + NUM_SL_PTE,
+ pt->redirect);
+
+ *fl_pte = ((((int)__pa(sl_table)) & FL_BASE_MASK) |
+ FL_TYPE_TABLE);
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ } else
+ sl_table = (unsigned long *)
+ __va(((*fl_pte) & FL_BASE_MASK));
+
+ /* Keep track of initial position so we
+ * don't clean more than we have to
+ */
+ sl_start = sl_offset;
+
+ /* Build the 2nd level page table */
+ while (offset < len && sl_offset < NUM_SL_PTE) {
+ pa = chunk_pa + chunk_offset;
+ sl_table[sl_offset] = (pa & SL_BASE_MASK_SMALL) |
+ pgprot | SL_NG | SL_SHARED | SL_TYPE_SMALL;
+ sl_offset++;
+ offset += SZ_4K;
+
+ chunk_offset += SZ_4K;
+
+ if (chunk_offset >= sg->length && offset < len) {
+ chunk_offset = 0;
+ sg = sg_next(sg);
+ chunk_pa = get_phys_addr(sg);
+ if (chunk_pa == 0) {
+ pr_debug("No dma address for sg %p\n",
+ sg);
+ ret = -EINVAL;
+ goto fail;
+ }
+ }
+ }
+
+ clean_pte(sl_table + sl_start, sl_table + sl_offset,
+ pt->redirect);
+ fl_pte++;
+ sl_offset = 0;
+ }
+
+fail:
+ return ret;
+}
+
+void msm_iommu_pagetable_unmap_range(struct iommu_pt *pt, unsigned int va,
+ unsigned int len)
+{
+ unsigned int offset = 0;
+ unsigned long *fl_pte;
+ unsigned long fl_offset;
+ unsigned long *sl_table;
+ unsigned long sl_start, sl_end;
+ int used, i;
+
+ BUG_ON(len & (SZ_4K - 1));
+
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
+
+ sl_start = SL_OFFSET(va);
+
+ while (offset < len) {
+ sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
+ sl_end = ((len - offset) / SZ_4K) + sl_start;
+
+ if (sl_end > NUM_SL_PTE)
+ sl_end = NUM_SL_PTE;
+
+ memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
+ clean_pte(sl_table + sl_start, sl_table + sl_end,
+ pt->redirect);
+
+ offset += (sl_end - sl_start) * SZ_4K;
+
+ /* Unmap and free the 2nd level table if all mappings in it
+ * were removed. This saves memory, but the table will need
+ * to be re-allocated the next time someone tries to map these
+ * VAs.
+ */
+ used = 0;
+
+ /* If we just unmapped the whole table, don't bother
+ * seeing if there are still used entries left.
+ */
+ if (sl_end - sl_start != NUM_SL_PTE)
+ for (i = 0; i < NUM_SL_PTE; i++)
+ if (sl_table[i]) {
+ used = 1;
+ break;
+ }
+ if (!used) {
+ free_page((unsigned long)sl_table);
+ *fl_pte = 0;
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ }
+
+ sl_start = 0;
+ fl_pte++;
+ }
+}
+
+static int __init get_tex_class(int icp, int ocp, int mt, int nos)
+{
+ int i = 0;
+ unsigned int prrr = 0;
+ unsigned int nmrr = 0;
+ int c_icp, c_ocp, c_mt, c_nos;
+
+ RCP15_PRRR(prrr);
+ RCP15_NMRR(nmrr);
+
+ for (i = 0; i < NUM_TEX_CLASS; i++) {
+ c_nos = PRRR_NOS(prrr, i);
+ c_mt = PRRR_MT(prrr, i);
+ c_icp = NMRR_ICP(nmrr, i);
+ c_ocp = NMRR_OCP(nmrr, i);
+
+ if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos)
+ return i;
+ }
+
+ return -ENODEV;
+}
+
+static void __init setup_iommu_tex_classes(void)
+{
+ msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] =
+ get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1);
+
+ msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] =
+ get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1);
+
+ msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] =
+ get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1);
+
+ msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] =
+ get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1);
+}
+
+void __init msm_iommu_pagetable_init(void)
+{
+ setup_iommu_tex_classes();
+}
diff --git a/drivers/iommu/msm_iommu_pagetable.h b/drivers/iommu/msm_iommu_pagetable.h
new file mode 100644
index 0000000..3266681
--- /dev/null
+++ b/drivers/iommu/msm_iommu_pagetable.h
@@ -0,0 +1,90 @@
+/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __ARCH_ARM_MACH_MSM_IOMMU_PAGETABLE_H
+#define __ARCH_ARM_MACH_MSM_IOMMU_PAGETABLE_H
+
+#define NUM_FL_PTE 4096
+#define NUM_SL_PTE 256
+#define NUM_TEX_CLASS 8
+
+/* First-level page table bits */
+#define FL_BASE_MASK 0xFFFFFC00
+#define FL_TYPE_TABLE (1 << 0)
+#define FL_TYPE_SECT (2 << 0)
+#define FL_SUPERSECTION (1 << 18)
+#define FL_AP0 (1 << 10)
+#define FL_AP1 (1 << 11)
+#define FL_AP2 (1 << 15)
+#define FL_SHARED (1 << 16)
+#define FL_BUFFERABLE (1 << 2)
+#define FL_CACHEABLE (1 << 3)
+#define FL_TEX0 (1 << 12)
+#define FL_OFFSET(va) (((va) & 0xFFF00000) >> 20)
+#define FL_NG (1 << 17)
+
+/* Second-level page table bits */
+#define SL_BASE_MASK_LARGE 0xFFFF0000
+#define SL_BASE_MASK_SMALL 0xFFFFF000
+#define SL_TYPE_LARGE (1 << 0)
+#define SL_TYPE_SMALL (2 << 0)
+#define SL_AP0 (1 << 4)
+#define SL_AP1 (2 << 4)
+#define SL_AP2 (1 << 9)
+#define SL_SHARED (1 << 10)
+#define SL_BUFFERABLE (1 << 2)
+#define SL_CACHEABLE (1 << 3)
+#define SL_TEX0 (1 << 6)
+#define SL_OFFSET(va) (((va) & 0xFF000) >> 12)
+#define SL_NG (1 << 11)
+
+/* Memory type and cache policy attributes */
+#define MT_SO 0
+#define MT_DEV 1
+#define MT_NORMAL 2
+#define CP_NONCACHED 0
+#define CP_WB_WA 1
+#define CP_WT 2
+#define CP_WB_NWA 3
+
+/* TEX Remap Registers */
+#define NMRR_ICP(nmrr, n) (((nmrr) & (3 << ((n) * 2))) >> ((n) * 2))
+#define NMRR_OCP(nmrr, n) (((nmrr) & (3 << ((n) * 2 + 16))) >> ((n) * 2 + 16))
+
+#define PRRR_NOS(prrr, n) ((prrr) & (1 << ((n) + 24)) ? 1 : 0)
+#define PRRR_MT(prrr, n) ((((prrr) & (3 << ((n) * 2))) >> ((n) * 2)))
+
+#define MRC(reg, processor, op1, crn, crm, op2) \
+__asm__ __volatile__ ( \
+" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \
+: "=r" (reg))
+
+#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
+#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
+
+struct iommu_pt {
+ unsigned long *fl_table;
+ int redirect;
+};
+
+void msm_iommu_pagetable_init(void);
+int msm_iommu_pagetable_alloc(struct iommu_pt *pt);
+void msm_iommu_pagetable_free(struct iommu_pt *pt);
+int msm_iommu_pagetable_map(struct iommu_pt *pt, unsigned long va,
+ phys_addr_t pa, size_t len, int prot);
+size_t msm_iommu_pagetable_unmap(struct iommu_pt *pt, unsigned long va,
+ size_t len);
+int msm_iommu_pagetable_map_range(struct iommu_pt *pt, unsigned int va,
+ struct scatterlist *sg, unsigned int len, int prot);
+void msm_iommu_pagetable_unmap_range(struct iommu_pt *pt, unsigned int va,
+ unsigned int len);
+#endif
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 17ef6c4..eb93c82 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -550,13 +550,13 @@
return 0;
as->pte_count = devm_kzalloc(smmu->dev,
- sizeof(as->pte_count[0]) * SMMU_PDIR_COUNT, GFP_ATOMIC);
+ sizeof(as->pte_count[0]) * SMMU_PDIR_COUNT, GFP_KERNEL);
if (!as->pte_count) {
dev_err(smmu->dev,
"failed to allocate smmu_device PTE cunters\n");
return -ENOMEM;
}
- as->pdir_page = alloc_page(GFP_ATOMIC | __GFP_DMA);
+ as->pdir_page = alloc_page(GFP_KERNEL | __GFP_DMA);
if (!as->pdir_page) {
dev_err(smmu->dev,
"failed to allocate smmu_device page directory\n");