msm-3.4 (commit 35cca8ba3ee0e6a2085dbcac48fb2ccbaa72ba98) video/gpu/iommu .. and all the hacks that goes with that
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index b1befa0..330c850 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -16,7 +16,7 @@
# MSM IOMMU support
config MSM_IOMMU
bool "MSM IOMMU Support"
- depends on ARCH_MSM8X60 || ARCH_MSM8960 || ARCH_APQ8064 || ARCH_MSM8974
+ depends on ARCH_MSM8X60 || ARCH_MSM8960 || ARCH_APQ8064 || ARCH_MSM8974 || ARCH_MPQ8092 || ARCH_MSM8610 || ARCH_MSM8226 || ARCH_MSMZINC
select IOMMU_API
help
Support for the IOMMUs found on certain Qualcomm SOCs.
@@ -36,6 +36,16 @@
If unsure, say N here.
+config MSM_IOMMU_PMON
+ bool "MSM IOMMU Perfomance Monitoring Support"
+ depends on (ARCH_MSM8974 || ARCH_MSM8610 || ARCH_MSM8226) && MSM_IOMMU
+ help
+ Support for monitoring IOMMUs performance on certain Qualcomm SOCs.
+ It captures TLB statistics per context bank of the IOMMU as an
+ indication of its performance metric.
+
+ If unsure, say N here.
+
config IOMMU_PGTABLES_L2
bool "Allow SMMU page tables in the L2 cache (Experimental)"
depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 066bc3e..096b53e 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -1,8 +1,9 @@
obj-$(CONFIG_IOMMU_API) += iommu.o
-obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
+obj-$(CONFIG_MSM_IOMMU) += msm_iommu-v0.o msm_iommu_dev-v0.o
ifdef CONFIG_OF
-obj-$(CONFIG_MSM_IOMMU) += msm_iommu-v2.o msm_iommu_dev-v2.o msm_iommu_pagetable.o
+obj-$(CONFIG_MSM_IOMMU) += msm_iommu-v1.o msm_iommu_dev-v1.o msm_iommu_pagetable.o msm_iommu_sec.o
endif
+obj-$(CONFIG_MSM_IOMMU_PMON) += msm_iommu_perfmon.o msm_iommu_perfmon-v0.o msm_iommu_perfmon-v1.o
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
obj-$(CONFIG_DMAR_TABLE) += dmar.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index a5bee8e..32c00cd 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3193,26 +3193,6 @@
return 0;
}
-static int amd_iommu_device_group(struct device *dev, unsigned int *groupid)
-{
- struct iommu_dev_data *dev_data = dev->archdata.iommu;
- struct pci_dev *pdev = to_pci_dev(dev);
- u16 devid;
-
- if (!dev_data)
- return -ENODEV;
-
- if (pdev->is_virtfn || !iommu_group_mf)
- devid = dev_data->devid;
- else
- devid = calc_devid(pdev->bus->number,
- PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
-
- *groupid = amd_iommu_alias_table[devid];
-
- return 0;
-}
-
static struct iommu_ops amd_iommu_ops = {
.domain_init = amd_iommu_domain_init,
.domain_destroy = amd_iommu_domain_destroy,
@@ -3222,7 +3202,6 @@
.unmap = amd_iommu_unmap,
.iova_to_phys = amd_iommu_iova_to_phys,
.domain_has_cap = amd_iommu_domain_has_cap,
- .device_group = amd_iommu_device_group,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
};
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index f93d5ac..d4a0ff7 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -4087,54 +4087,6 @@
return 0;
}
-/*
- * Group numbers are arbitrary. Device with the same group number
- * indicate the iommu cannot differentiate between them. To avoid
- * tracking used groups we just use the seg|bus|devfn of the lowest
- * level we're able to differentiate devices
- */
-static int intel_iommu_device_group(struct device *dev, unsigned int *groupid)
-{
- struct pci_dev *pdev = to_pci_dev(dev);
- struct pci_dev *bridge;
- union {
- struct {
- u8 devfn;
- u8 bus;
- u16 segment;
- } pci;
- u32 group;
- } id;
-
- if (iommu_no_mapping(dev))
- return -ENODEV;
-
- id.pci.segment = pci_domain_nr(pdev->bus);
- id.pci.bus = pdev->bus->number;
- id.pci.devfn = pdev->devfn;
-
- if (!device_to_iommu(id.pci.segment, id.pci.bus, id.pci.devfn))
- return -ENODEV;
-
- bridge = pci_find_upstream_pcie_bridge(pdev);
- if (bridge) {
- if (pci_is_pcie(bridge)) {
- id.pci.bus = bridge->subordinate->number;
- id.pci.devfn = 0;
- } else {
- id.pci.bus = bridge->bus->number;
- id.pci.devfn = bridge->devfn;
- }
- }
-
- if (!pdev->is_virtfn && iommu_group_mf)
- id.pci.devfn = PCI_DEVFN(PCI_SLOT(id.pci.devfn), 0);
-
- *groupid = id.group;
-
- return 0;
-}
-
static struct iommu_ops intel_iommu_ops = {
.domain_init = intel_iommu_domain_init,
.domain_destroy = intel_iommu_domain_destroy,
@@ -4144,7 +4096,6 @@
.unmap = intel_iommu_unmap,
.iova_to_phys = intel_iommu_iova_to_phys,
.domain_has_cap = intel_iommu_domain_has_cap,
- .device_group = intel_iommu_device_group,
.pgsize_bitmap = INTEL_IOMMU_PGSIZES,
};
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index ef69d91..a6eae7e 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -27,60 +27,571 @@
#include <linux/errno.h>
#include <linux/iommu.h>
#include <linux/scatterlist.h>
+#include <linux/idr.h>
+#include <linux/notifier.h>
+#include <linux/err.h>
-static ssize_t show_iommu_group(struct device *dev,
- struct device_attribute *attr, char *buf)
+static struct kset *iommu_group_kset;
+static struct idr iommu_group_idr;
+static struct mutex iommu_group_mutex;
+
+struct iommu_group {
+ struct kobject kobj;
+ struct kobject *devices_kobj;
+ struct list_head devices;
+ struct mutex mutex;
+ struct blocking_notifier_head notifier;
+ void *iommu_data;
+ void (*iommu_data_release)(void *iommu_data);
+ char *name;
+ int id;
+};
+
+struct iommu_device {
+ struct list_head list;
+ struct device *dev;
+ char *name;
+};
+
+struct iommu_group_attribute {
+ struct attribute attr;
+ ssize_t (*show)(struct iommu_group *group, char *buf);
+ ssize_t (*store)(struct iommu_group *group,
+ const char *buf, size_t count);
+};
+
+#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
+struct iommu_group_attribute iommu_group_attr_##_name = \
+ __ATTR(_name, _mode, _show, _store)
+
+#define to_iommu_group_attr(_attr) \
+ container_of(_attr, struct iommu_group_attribute, attr)
+#define to_iommu_group(_kobj) \
+ container_of(_kobj, struct iommu_group, kobj)
+
+static ssize_t iommu_group_attr_show(struct kobject *kobj,
+ struct attribute *__attr, char *buf)
{
- unsigned int groupid;
+ struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
+ struct iommu_group *group = to_iommu_group(kobj);
+ ssize_t ret = -EIO;
- if (iommu_device_group(dev, &groupid))
- return 0;
-
- return sprintf(buf, "%u", groupid);
+ if (attr->show)
+ ret = attr->show(group, buf);
+ return ret;
}
-static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL);
+
+static ssize_t iommu_group_attr_store(struct kobject *kobj,
+ struct attribute *__attr,
+ const char *buf, size_t count)
+{
+ struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
+ struct iommu_group *group = to_iommu_group(kobj);
+ ssize_t ret = -EIO;
+
+ if (attr->store)
+ ret = attr->store(group, buf, count);
+ return ret;
+}
+
+static const struct sysfs_ops iommu_group_sysfs_ops = {
+ .show = iommu_group_attr_show,
+ .store = iommu_group_attr_store,
+};
+
+static int iommu_group_create_file(struct iommu_group *group,
+ struct iommu_group_attribute *attr)
+{
+ return sysfs_create_file(&group->kobj, &attr->attr);
+}
+
+static void iommu_group_remove_file(struct iommu_group *group,
+ struct iommu_group_attribute *attr)
+{
+ sysfs_remove_file(&group->kobj, &attr->attr);
+}
+
+static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
+{
+ return sprintf(buf, "%s\n", group->name);
+}
+
+static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
+
+static void iommu_group_release(struct kobject *kobj)
+{
+ struct iommu_group *group = to_iommu_group(kobj);
+
+ if (group->iommu_data_release)
+ group->iommu_data_release(group->iommu_data);
+
+ mutex_lock(&iommu_group_mutex);
+ idr_remove(&iommu_group_idr, group->id);
+ mutex_unlock(&iommu_group_mutex);
+
+ kfree(group->name);
+ kfree(group);
+}
+
+static struct kobj_type iommu_group_ktype = {
+ .sysfs_ops = &iommu_group_sysfs_ops,
+ .release = iommu_group_release,
+};
+
+/**
+ * iommu_group_alloc - Allocate a new group
+ * @name: Optional name to associate with group, visible in sysfs
+ *
+ * This function is called by an iommu driver to allocate a new iommu
+ * group. The iommu group represents the minimum granularity of the iommu.
+ * Upon successful return, the caller holds a reference to the supplied
+ * group in order to hold the group until devices are added. Use
+ * iommu_group_put() to release this extra reference count, allowing the
+ * group to be automatically reclaimed once it has no devices or external
+ * references.
+ */
+struct iommu_group *iommu_group_alloc(void)
+{
+ struct iommu_group *group;
+ int ret;
+
+ group = kzalloc(sizeof(*group), GFP_KERNEL);
+ if (!group)
+ return ERR_PTR(-ENOMEM);
+
+ group->kobj.kset = iommu_group_kset;
+ mutex_init(&group->mutex);
+ INIT_LIST_HEAD(&group->devices);
+ BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
+
+ mutex_lock(&iommu_group_mutex);
+
+again:
+ if (unlikely(0 == idr_pre_get(&iommu_group_idr, GFP_KERNEL))) {
+ kfree(group);
+ mutex_unlock(&iommu_group_mutex);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ret = idr_get_new_above(&iommu_group_idr, group, 1, &group->id);
+ if (ret == -EAGAIN)
+ goto again;
+ mutex_unlock(&iommu_group_mutex);
+
+ if (ret == -ENOSPC) {
+ kfree(group);
+ return ERR_PTR(ret);
+ }
+
+ ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
+ NULL, "%d", group->id);
+ if (ret) {
+ mutex_lock(&iommu_group_mutex);
+ idr_remove(&iommu_group_idr, group->id);
+ mutex_unlock(&iommu_group_mutex);
+ kfree(group);
+ return ERR_PTR(ret);
+ }
+
+ group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
+ if (!group->devices_kobj) {
+ kobject_put(&group->kobj); /* triggers .release & free */
+ return ERR_PTR(-ENOMEM);
+ }
+
+ /*
+ * The devices_kobj holds a reference on the group kobject, so
+ * as long as that exists so will the group. We can therefore
+ * use the devices_kobj for reference counting.
+ */
+ kobject_put(&group->kobj);
+
+ return group;
+}
+EXPORT_SYMBOL_GPL(iommu_group_alloc);
+
+/**
+ * iommu_group_get_iommudata - retrieve iommu_data registered for a group
+ * @group: the group
+ *
+ * iommu drivers can store data in the group for use when doing iommu
+ * operations. This function provides a way to retrieve it. Caller
+ * should hold a group reference.
+ */
+void *iommu_group_get_iommudata(struct iommu_group *group)
+{
+ return group->iommu_data;
+}
+EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
+
+/**
+ * iommu_group_set_iommudata - set iommu_data for a group
+ * @group: the group
+ * @iommu_data: new data
+ * @release: release function for iommu_data
+ *
+ * iommu drivers can store data in the group for use when doing iommu
+ * operations. This function provides a way to set the data after
+ * the group has been allocated. Caller should hold a group reference.
+ */
+void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
+ void (*release)(void *iommu_data))
+{
+ group->iommu_data = iommu_data;
+ group->iommu_data_release = release;
+}
+EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
+
+/**
+ * iommu_group_set_name - set name for a group
+ * @group: the group
+ * @name: name
+ *
+ * Allow iommu driver to set a name for a group. When set it will
+ * appear in a name attribute file under the group in sysfs.
+ */
+int iommu_group_set_name(struct iommu_group *group, const char *name)
+{
+ int ret;
+
+ if (group->name) {
+ iommu_group_remove_file(group, &iommu_group_attr_name);
+ kfree(group->name);
+ group->name = NULL;
+ if (!name)
+ return 0;
+ }
+
+ group->name = kstrdup(name, GFP_KERNEL);
+ if (!group->name)
+ return -ENOMEM;
+
+ ret = iommu_group_create_file(group, &iommu_group_attr_name);
+ if (ret) {
+ kfree(group->name);
+ group->name = NULL;
+ return ret;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(iommu_group_set_name);
+
+/**
+ * iommu_group_add_device - add a device to an iommu group
+ * @group: the group into which to add the device (reference should be held)
+ * @dev: the device
+ *
+ * This function is called by an iommu driver to add a device into a
+ * group. Adding a device increments the group reference count.
+ */
+int iommu_group_add_device(struct iommu_group *group, struct device *dev)
+{
+ int ret, i = 0;
+ struct iommu_device *device;
+
+ device = kzalloc(sizeof(*device), GFP_KERNEL);
+ if (!device)
+ return -ENOMEM;
+
+ device->dev = dev;
+
+ ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
+ if (ret) {
+ kfree(device);
+ return ret;
+ }
+
+ device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
+rename:
+ if (!device->name) {
+ sysfs_remove_link(&dev->kobj, "iommu_group");
+ kfree(device);
+ return -ENOMEM;
+ }
+
+ ret = sysfs_create_link_nowarn(group->devices_kobj,
+ &dev->kobj, device->name);
+ if (ret) {
+ kfree(device->name);
+ if (ret == -EEXIST && i >= 0) {
+ /*
+ * Account for the slim chance of collision
+ * and append an instance to the name.
+ */
+ device->name = kasprintf(GFP_KERNEL, "%s.%d",
+ kobject_name(&dev->kobj), i++);
+ goto rename;
+ }
+
+ sysfs_remove_link(&dev->kobj, "iommu_group");
+ kfree(device);
+ return ret;
+ }
+
+ kobject_get(group->devices_kobj);
+
+ dev->iommu_group = group;
+
+ mutex_lock(&group->mutex);
+ list_add_tail(&device->list, &group->devices);
+ mutex_unlock(&group->mutex);
+
+ /* Notify any listeners about change to group. */
+ blocking_notifier_call_chain(&group->notifier,
+ IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(iommu_group_add_device);
+
+/**
+ * iommu_group_remove_device - remove a device from it's current group
+ * @dev: device to be removed
+ *
+ * This function is called by an iommu driver to remove the device from
+ * it's current group. This decrements the iommu group reference count.
+ */
+void iommu_group_remove_device(struct device *dev)
+{
+ struct iommu_group *group = dev->iommu_group;
+ struct iommu_device *tmp_device, *device = NULL;
+
+ /* Pre-notify listeners that a device is being removed. */
+ blocking_notifier_call_chain(&group->notifier,
+ IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
+
+ mutex_lock(&group->mutex);
+ list_for_each_entry(tmp_device, &group->devices, list) {
+ if (tmp_device->dev == dev) {
+ device = tmp_device;
+ list_del(&device->list);
+ break;
+ }
+ }
+ mutex_unlock(&group->mutex);
+
+ if (!device)
+ return;
+
+ sysfs_remove_link(group->devices_kobj, device->name);
+ sysfs_remove_link(&dev->kobj, "iommu_group");
+
+ kfree(device->name);
+ kfree(device);
+ dev->iommu_group = NULL;
+ kobject_put(group->devices_kobj);
+}
+EXPORT_SYMBOL_GPL(iommu_group_remove_device);
+
+/**
+ * iommu_group_for_each_dev - iterate over each device in the group
+ * @group: the group
+ * @data: caller opaque data to be passed to callback function
+ * @fn: caller supplied callback function
+ *
+ * This function is called by group users to iterate over group devices.
+ * Callers should hold a reference count to the group during callback.
+ * The group->mutex is held across callbacks, which will block calls to
+ * iommu_group_add/remove_device.
+ */
+int iommu_group_for_each_dev(struct iommu_group *group, void *data,
+ int (*fn)(struct device *, void *))
+{
+ struct iommu_device *device;
+ int ret = 0;
+
+ mutex_lock(&group->mutex);
+ list_for_each_entry(device, &group->devices, list) {
+ ret = fn(device->dev, data);
+ if (ret)
+ break;
+ }
+ mutex_unlock(&group->mutex);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
+
+/**
+ * iommu_group_get - Return the group for a device and increment reference
+ * @dev: get the group that this device belongs to
+ *
+ * This function is called by iommu drivers and users to get the group
+ * for the specified device. If found, the group is returned and the group
+ * reference in incremented, else NULL.
+ */
+struct iommu_group *iommu_group_get(struct device *dev)
+{
+ struct iommu_group *group = dev->iommu_group;
+
+ if (group)
+ kobject_get(group->devices_kobj);
+
+ return group;
+}
+EXPORT_SYMBOL_GPL(iommu_group_get);
+
+/**
+ * iommu_group_find - Find and return the group based on the group name.
+ * Also increment the reference count.
+ * @name: the name of the group
+ *
+ * This function is called by iommu drivers and clients to get the group
+ * by the specified name. If found, the group is returned and the group
+ * reference is incremented, else NULL.
+ */
+struct iommu_group *iommu_group_find(const char *name)
+{
+ struct iommu_group *group;
+ int next = 0;
+
+ mutex_lock(&iommu_group_mutex);
+ while ((group = idr_get_next(&iommu_group_idr, &next))) {
+ if (group->name) {
+ if (strcmp(group->name, name) == 0)
+ break;
+ }
+ ++next;
+ }
+ mutex_unlock(&iommu_group_mutex);
+
+ if (group)
+ kobject_get(group->devices_kobj);
+
+ return group;
+}
+EXPORT_SYMBOL_GPL(iommu_group_find);
+
+/**
+ * iommu_group_put - Decrement group reference
+ * @group: the group to use
+ *
+ * This function is called by iommu drivers and users to release the
+ * iommu group. Once the reference count is zero, the group is released.
+ */
+void iommu_group_put(struct iommu_group *group)
+{
+ if (group)
+ kobject_put(group->devices_kobj);
+}
+EXPORT_SYMBOL_GPL(iommu_group_put);
+
+/**
+ * iommu_group_register_notifier - Register a notifier for group changes
+ * @group: the group to watch
+ * @nb: notifier block to signal
+ *
+ * This function allows iommu group users to track changes in a group.
+ * See include/linux/iommu.h for actions sent via this notifier. Caller
+ * should hold a reference to the group throughout notifier registration.
+ */
+int iommu_group_register_notifier(struct iommu_group *group,
+ struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&group->notifier, nb);
+}
+EXPORT_SYMBOL_GPL(iommu_group_register_notifier);
+
+/**
+ * iommu_group_unregister_notifier - Unregister a notifier
+ * @group: the group to watch
+ * @nb: notifier block to signal
+ *
+ * Unregister a previously registered group notifier block.
+ */
+int iommu_group_unregister_notifier(struct iommu_group *group,
+ struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&group->notifier, nb);
+}
+EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier);
+
+/**
+ * iommu_group_id - Return ID for a group
+ * @group: the group to ID
+ *
+ * Return the unique ID for the group matching the sysfs group number.
+ */
+int iommu_group_id(struct iommu_group *group)
+{
+ return group->id;
+}
+EXPORT_SYMBOL_GPL(iommu_group_id);
static int add_iommu_group(struct device *dev, void *data)
{
- unsigned int groupid;
+ struct iommu_ops *ops = data;
- if (iommu_device_group(dev, &groupid) == 0)
- return device_create_file(dev, &dev_attr_iommu_group);
+ if (!ops->add_device)
+ return -ENODEV;
+
+ WARN_ON(dev->iommu_group);
+
+ ops->add_device(dev);
return 0;
}
-static int remove_iommu_group(struct device *dev)
-{
- unsigned int groupid;
-
- if (iommu_device_group(dev, &groupid) == 0)
- device_remove_file(dev, &dev_attr_iommu_group);
-
- return 0;
-}
-
-static int iommu_device_notifier(struct notifier_block *nb,
- unsigned long action, void *data)
+static int iommu_bus_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
{
struct device *dev = data;
+ struct iommu_ops *ops = dev->bus->iommu_ops;
+ struct iommu_group *group;
+ unsigned long group_action = 0;
- if (action == BUS_NOTIFY_ADD_DEVICE)
- return add_iommu_group(dev, NULL);
- else if (action == BUS_NOTIFY_DEL_DEVICE)
- return remove_iommu_group(dev);
+ /*
+ * ADD/DEL call into iommu driver ops if provided, which may
+ * result in ADD/DEL notifiers to group->notifier
+ */
+ if (action == BUS_NOTIFY_ADD_DEVICE) {
+ if (ops->add_device)
+ return ops->add_device(dev);
+ } else if (action == BUS_NOTIFY_DEL_DEVICE) {
+ if (ops->remove_device && dev->iommu_group) {
+ ops->remove_device(dev);
+ return 0;
+ }
+ }
+ /*
+ * Remaining BUS_NOTIFYs get filtered and republished to the
+ * group, if anyone is listening
+ */
+ group = iommu_group_get(dev);
+ if (!group)
+ return 0;
+
+ switch (action) {
+ case BUS_NOTIFY_BIND_DRIVER:
+ group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER;
+ break;
+ case BUS_NOTIFY_BOUND_DRIVER:
+ group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER;
+ break;
+ case BUS_NOTIFY_UNBIND_DRIVER:
+ group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER;
+ break;
+ case BUS_NOTIFY_UNBOUND_DRIVER:
+ group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER;
+ break;
+ }
+
+ if (group_action)
+ blocking_notifier_call_chain(&group->notifier,
+ group_action, dev);
+
+ iommu_group_put(group);
return 0;
}
-static struct notifier_block iommu_device_nb = {
- .notifier_call = iommu_device_notifier,
+static struct notifier_block iommu_bus_nb = {
+ .notifier_call = iommu_bus_notifier,
};
static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops)
{
- bus_register_notifier(bus, &iommu_device_nb);
- bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
+ bus_register_notifier(bus, &iommu_bus_nb);
+ bus_for_each_dev(bus, NULL, ops, add_iommu_group);
}
/**
@@ -120,6 +631,7 @@
* iommu_set_fault_handler() - set a fault handler for an iommu domain
* @domain: iommu domain
* @handler: fault handler
+ * @token: user data, will be passed back to the fault handler
*
* This function should be used by IOMMU users which want to be notified
* whenever an IOMMU fault happens.
@@ -128,11 +640,13 @@
* error code otherwise.
*/
void iommu_set_fault_handler(struct iommu_domain *domain,
- iommu_fault_handler_t handler)
+ iommu_fault_handler_t handler,
+ void *token)
{
BUG_ON(!domain);
domain->handler = handler;
+ domain->handler_token = token;
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
@@ -190,6 +704,45 @@
}
EXPORT_SYMBOL_GPL(iommu_detach_device);
+/*
+ * IOMMU groups are really the natrual working unit of the IOMMU, but
+ * the IOMMU API works on domains and devices. Bridge that gap by
+ * iterating over the devices in a group. Ideally we'd have a single
+ * device which represents the requestor ID of the group, but we also
+ * allow IOMMU drivers to create policy defined minimum sets, where
+ * the physical hardware may be able to distiguish members, but we
+ * wish to group them at a higher level (ex. untrusted multi-function
+ * PCI devices). Thus we attach each device.
+ */
+static int iommu_group_do_attach_device(struct device *dev, void *data)
+{
+ struct iommu_domain *domain = data;
+
+ return iommu_attach_device(domain, dev);
+}
+
+int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
+{
+ return iommu_group_for_each_dev(group, domain,
+ iommu_group_do_attach_device);
+}
+EXPORT_SYMBOL_GPL(iommu_attach_group);
+
+static int iommu_group_do_detach_device(struct device *dev, void *data)
+{
+ struct iommu_domain *domain = data;
+
+ iommu_detach_device(domain, dev);
+
+ return 0;
+}
+
+void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
+{
+ iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device);
+}
+EXPORT_SYMBOL_GPL(iommu_detach_group);
+
phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain,
unsigned long iova)
{
@@ -367,11 +920,15 @@
}
EXPORT_SYMBOL_GPL(iommu_get_pt_base_addr);
-int iommu_device_group(struct device *dev, unsigned int *groupid)
+static int __init iommu_init(void)
{
- if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group)
- return dev->bus->iommu_ops->device_group(dev, groupid);
+ iommu_group_kset = kset_create_and_add("iommu_groups",
+ NULL, kernel_kobj);
+ idr_init(&iommu_group_idr);
+ mutex_init(&iommu_group_mutex);
- return -ENODEV;
+ BUG_ON(!iommu_group_kset);
+
+ return 0;
}
-EXPORT_SYMBOL_GPL(iommu_device_group);
+subsys_initcall(iommu_init);
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu-v0.c
similarity index 82%
rename from drivers/iommu/msm_iommu.c
rename to drivers/iommu/msm_iommu-v0.c
index 05bb4a9..c0a4720 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu-v0.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2012, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2010-2013, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -27,7 +27,9 @@
#include <asm/cacheflush.h>
#include <asm/sizes.h>
-#include <mach/iommu_hw-8xxx.h>
+#include <mach/iommu_perfmon.h>
+#include <mach/iommu_hw-v0.h>
+#include <mach/msm_iommu_priv.h>
#include <mach/iommu.h>
#include <mach/msm_smsm.h>
@@ -49,6 +51,9 @@
#define MSM_IOMMU_ATTR_CACHED_WB_NWA 0x2
#define MSM_IOMMU_ATTR_CACHED_WT 0x3
+struct bus_type msm_iommu_sec_bus_type = {
+ .name = "msm_iommu_sec_bus",
+};
static inline void clean_pte(unsigned long *start, unsigned long *end,
int redirect)
@@ -127,12 +132,6 @@
return msm_iommu_remote_lock.lock;
}
-struct msm_priv {
- unsigned long *pgtable;
- int redirect;
- struct list_head list_attached;
-};
-
static int __enable_clocks(struct msm_iommu_drvdata *drvdata)
{
int ret;
@@ -157,9 +156,40 @@
clk_disable_unprepare(drvdata->pclk);
}
+static int __enable_regulators(struct msm_iommu_drvdata *drvdata)
+{
+ /* No need to do anything. IOMMUv0 is always on. */
+ return 0;
+}
+
+static void __disable_regulators(struct msm_iommu_drvdata *drvdata)
+{
+ /* No need to do anything. IOMMUv0 is always on. */
+}
+
+static void _iommu_lock_acquire(void)
+{
+ msm_iommu_lock();
+}
+
+static void _iommu_lock_release(void)
+{
+ msm_iommu_unlock();
+}
+
+struct iommu_access_ops iommu_access_ops_v0 = {
+ .iommu_power_on = __enable_regulators,
+ .iommu_power_off = __disable_regulators,
+ .iommu_clk_on = __enable_clocks,
+ .iommu_clk_off = __disable_clocks,
+ .iommu_lock_acquire = _iommu_lock_acquire,
+ .iommu_lock_release = _iommu_lock_release,
+};
+EXPORT_SYMBOL(iommu_access_ops_v0);
+
static int __flush_iotlb_va(struct iommu_domain *domain, unsigned int va)
{
- struct msm_priv *priv = domain->priv;
+ struct msm_iommu_priv *priv = domain->priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
int ret = 0;
@@ -196,7 +226,7 @@
static int __flush_iotlb(struct iommu_domain *domain)
{
- struct msm_priv *priv = domain->priv;
+ struct msm_iommu_priv *priv = domain->priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
int ret = 0;
@@ -230,13 +260,13 @@
return ret;
}
-static void __reset_context(void __iomem *base, int ctx)
+static void __reset_context(void __iomem *base, void __iomem *glb_base, int ctx)
{
- SET_BPRCOSH(base, ctx, 0);
- SET_BPRCISH(base, ctx, 0);
- SET_BPRCNSH(base, ctx, 0);
- SET_BPSHCFG(base, ctx, 0);
- SET_BPMTCFG(base, ctx, 0);
+ SET_BPRCOSH(glb_base, ctx, 0);
+ SET_BPRCISH(glb_base, ctx, 0);
+ SET_BPRCNSH(glb_base, ctx, 0);
+ SET_BPSHCFG(glb_base, ctx, 0);
+ SET_BPMTCFG(glb_base, ctx, 0);
SET_ACTLR(base, ctx, 0);
SET_SCTLR(base, ctx, 0);
SET_FSRRESTORE(base, ctx, 0);
@@ -254,16 +284,15 @@
mb();
}
-static void __program_context(void __iomem *base, int ctx, int ncb,
- phys_addr_t pgtable, int redirect,
- int ttbr_split)
+static void __program_context(void __iomem *base, void __iomem *glb_base,
+ int ctx, int ncb, phys_addr_t pgtable,
+ int redirect, int ttbr_split)
{
unsigned int prrr, nmrr;
int i, j, found;
-
msm_iommu_remote_spin_lock();
- __reset_context(base, ctx);
+ __reset_context(base, glb_base, ctx);
/* Set up HTW mode */
/* TLB miss configuration: perform HTW on miss */
@@ -358,26 +387,27 @@
static int msm_iommu_domain_init(struct iommu_domain *domain, int flags)
{
- struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ struct msm_iommu_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
goto fail_nomem;
INIT_LIST_HEAD(&priv->list_attached);
- priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL,
+ priv->pt.fl_table = (unsigned long *)__get_free_pages(GFP_KERNEL,
get_order(SZ_16K));
- if (!priv->pgtable)
+ if (!priv->pt.fl_table)
goto fail_nomem;
#ifdef CONFIG_IOMMU_PGTABLES_L2
- priv->redirect = flags & MSM_IOMMU_DOMAIN_PT_CACHEABLE;
+ priv->pt.redirect = flags & MSM_IOMMU_DOMAIN_PT_CACHEABLE;
#endif
- memset(priv->pgtable, 0, SZ_16K);
+ memset(priv->pt.fl_table, 0, SZ_16K);
domain->priv = priv;
- clean_pte(priv->pgtable, priv->pgtable + NUM_FL_PTE, priv->redirect);
+ clean_pte(priv->pt.fl_table, priv->pt.fl_table + NUM_FL_PTE,
+ priv->pt.redirect);
return 0;
@@ -388,7 +418,7 @@
static void msm_iommu_domain_destroy(struct iommu_domain *domain)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
unsigned long *fl_table;
int i;
@@ -397,15 +427,15 @@
domain->priv = NULL;
if (priv) {
- fl_table = priv->pgtable;
+ fl_table = priv->pt.fl_table;
for (i = 0; i < NUM_FL_PTE; i++)
if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
free_page((unsigned long) __va(((fl_table[i]) &
FL_BASE_MASK)));
- free_pages((unsigned long)priv->pgtable, get_order(SZ_16K));
- priv->pgtable = NULL;
+ free_pages((unsigned long)priv->pt.fl_table, get_order(SZ_16K));
+ priv->pt.fl_table = NULL;
}
kfree(priv);
@@ -414,8 +444,7 @@
static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
- struct msm_priv *priv;
- struct msm_iommu_ctx_dev *ctx_dev;
+ struct msm_iommu_priv *priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
struct msm_iommu_ctx_drvdata *tmp_drvdata;
@@ -427,42 +456,52 @@
if (!priv || !dev) {
ret = -EINVAL;
- goto fail;
+ goto unlock;
}
iommu_drvdata = dev_get_drvdata(dev->parent);
ctx_drvdata = dev_get_drvdata(dev);
- ctx_dev = dev->platform_data;
- if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) {
+ if (!iommu_drvdata || !ctx_drvdata) {
ret = -EINVAL;
- goto fail;
+ goto unlock;
}
+ ++ctx_drvdata->attach_count;
+
+ if (ctx_drvdata->attach_count > 1)
+ goto unlock;
+
if (!list_empty(&ctx_drvdata->attached_elm)) {
ret = -EBUSY;
- goto fail;
+ goto unlock;
}
list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm)
if (tmp_drvdata == ctx_drvdata) {
ret = -EBUSY;
- goto fail;
+ goto unlock;
}
ret = __enable_clocks(iommu_drvdata);
if (ret)
- goto fail;
+ goto unlock;
- __program_context(iommu_drvdata->base, ctx_dev->num, iommu_drvdata->ncb,
- __pa(priv->pgtable), priv->redirect,
+ __program_context(iommu_drvdata->base, iommu_drvdata->glb_base,
+ ctx_drvdata->num, iommu_drvdata->ncb,
+ __pa(priv->pt.fl_table), priv->pt.redirect,
iommu_drvdata->ttbr_split);
__disable_clocks(iommu_drvdata);
list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
ctx_drvdata->attached_domain = domain;
-fail:
+
+ mutex_unlock(&msm_iommu_lock);
+
+ msm_iommu_attached(dev->parent);
+ return ret;
+unlock:
mutex_unlock(&msm_iommu_lock);
return ret;
}
@@ -470,42 +509,49 @@
static void msm_iommu_detach_dev(struct iommu_domain *domain,
struct device *dev)
{
- struct msm_priv *priv;
- struct msm_iommu_ctx_dev *ctx_dev;
+ struct msm_iommu_priv *priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
int ret;
+ msm_iommu_detached(dev->parent);
+
mutex_lock(&msm_iommu_lock);
priv = domain->priv;
if (!priv || !dev)
- goto fail;
+ goto unlock;
iommu_drvdata = dev_get_drvdata(dev->parent);
ctx_drvdata = dev_get_drvdata(dev);
- ctx_dev = dev->platform_data;
- if (!iommu_drvdata || !ctx_drvdata || !ctx_dev)
- goto fail;
+ if (!iommu_drvdata || !ctx_drvdata)
+ goto unlock;
+
+ --ctx_drvdata->attach_count;
+ BUG_ON(ctx_drvdata->attach_count < 0);
+
+ if (ctx_drvdata->attach_count > 0)
+ goto unlock;
ret = __enable_clocks(iommu_drvdata);
if (ret)
- goto fail;
+ goto unlock;
msm_iommu_remote_spin_lock();
- SET_TLBIASID(iommu_drvdata->base, ctx_dev->num,
- GET_CONTEXTIDR_ASID(iommu_drvdata->base, ctx_dev->num));
+ SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num,
+ GET_CONTEXTIDR_ASID(iommu_drvdata->base, ctx_drvdata->num));
- __reset_context(iommu_drvdata->base, ctx_dev->num);
+ __reset_context(iommu_drvdata->base, iommu_drvdata->glb_base,
+ ctx_drvdata->num);
msm_iommu_remote_spin_unlock();
__disable_clocks(iommu_drvdata);
list_del_init(&ctx_drvdata->attached_elm);
ctx_drvdata->attached_domain = NULL;
-fail:
+unlock:
mutex_unlock(&msm_iommu_lock);
}
@@ -551,7 +597,7 @@
return pgprot;
}
-static unsigned long *make_second_level(struct msm_priv *priv,
+static unsigned long *make_second_level(struct msm_iommu_priv *priv,
unsigned long *fl_pte)
{
unsigned long *sl;
@@ -563,12 +609,12 @@
goto fail;
}
memset(sl, 0, SZ_4K);
- clean_pte(sl, sl + NUM_SL_PTE, priv->redirect);
+ clean_pte(sl, sl + NUM_SL_PTE, priv->pt.redirect);
*fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | \
FL_TYPE_TABLE);
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1, priv->pt.redirect);
fail:
return sl;
}
@@ -640,7 +686,7 @@
static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
phys_addr_t pa, size_t len, int prot)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
unsigned long *fl_table;
unsigned long *fl_pte;
unsigned long fl_offset;
@@ -658,7 +704,7 @@
goto fail;
}
- fl_table = priv->pgtable;
+ fl_table = priv->pt.fl_table;
if (len != SZ_16M && len != SZ_1M &&
len != SZ_64K && len != SZ_4K) {
@@ -687,14 +733,14 @@
ret = fl_16m(fl_pte, pa, pgprot);
if (ret)
goto fail;
- clean_pte(fl_pte, fl_pte + 16, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 16, priv->pt.redirect);
}
if (len == SZ_1M) {
ret = fl_1m(fl_pte, pa, pgprot);
if (ret)
goto fail;
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1, priv->pt.redirect);
}
/* Need a 2nd level table */
@@ -722,14 +768,14 @@
if (ret)
goto fail;
- clean_pte(sl_pte, sl_pte + 1, priv->redirect);
+ clean_pte(sl_pte, sl_pte + 1, priv->pt.redirect);
}
if (len == SZ_64K) {
ret = sl_64k(sl_pte, pa, pgprot);
if (ret)
goto fail;
- clean_pte(sl_pte, sl_pte + 16, priv->redirect);
+ clean_pte(sl_pte, sl_pte + 16, priv->pt.redirect);
}
ret = __flush_iotlb_va(domain, va);
@@ -741,7 +787,7 @@
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
size_t len)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
unsigned long *fl_table;
unsigned long *fl_pte;
unsigned long fl_offset;
@@ -757,7 +803,7 @@
if (!priv)
goto fail;
- fl_table = priv->pgtable;
+ fl_table = priv->pt.fl_table;
if (len != SZ_16M && len != SZ_1M &&
len != SZ_64K && len != SZ_4K) {
@@ -783,13 +829,13 @@
for (i = 0; i < 16; i++)
*(fl_pte+i) = 0;
- clean_pte(fl_pte, fl_pte + 16, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 16, priv->pt.redirect);
}
if (len == SZ_1M) {
*fl_pte = 0;
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1, priv->pt.redirect);
}
sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
@@ -800,13 +846,13 @@
for (i = 0; i < 16; i++)
*(sl_pte+i) = 0;
- clean_pte(sl_pte, sl_pte + 16, priv->redirect);
+ clean_pte(sl_pte, sl_pte + 16, priv->pt.redirect);
}
if (len == SZ_4K) {
*sl_pte = 0;
- clean_pte(sl_pte, sl_pte + 1, priv->redirect);
+ clean_pte(sl_pte, sl_pte + 1, priv->pt.redirect);
}
if (len == SZ_4K || len == SZ_64K) {
@@ -819,7 +865,7 @@
free_page((unsigned long)sl_table);
*fl_pte = 0;
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1, priv->pt.redirect);
}
}
@@ -853,6 +899,55 @@
&& (len >= align);
}
+static int check_range(unsigned long *fl_table, unsigned int va,
+ unsigned int len)
+{
+ unsigned int offset = 0;
+ unsigned long *fl_pte;
+ unsigned long fl_offset;
+ unsigned long *sl_table;
+ unsigned long sl_start, sl_end;
+ int i;
+
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
+
+ while (offset < len) {
+ if (*fl_pte & FL_TYPE_TABLE) {
+ sl_start = SL_OFFSET(va);
+ sl_table = __va(((*fl_pte) & FL_BASE_MASK));
+ sl_end = ((len - offset) / SZ_4K) + sl_start;
+
+ if (sl_end > NUM_SL_PTE)
+ sl_end = NUM_SL_PTE;
+
+ for (i = sl_start; i < sl_end; i++) {
+ if (sl_table[i] != 0) {
+ pr_err("%08x - %08x already mapped\n",
+ va, va + SZ_4K);
+ return -EBUSY;
+ }
+ offset += SZ_4K;
+ va += SZ_4K;
+ }
+
+
+ sl_start = 0;
+ } else {
+ if (*fl_pte != 0) {
+ pr_err("%08x - %08x already mapped\n",
+ va, va + SZ_1M);
+ return -EBUSY;
+ }
+ va += SZ_1M;
+ offset += SZ_1M;
+ sl_start = 0;
+ }
+ fl_pte++;
+ }
+ return 0;
+}
+
static int msm_iommu_map_range(struct iommu_domain *domain, unsigned int va,
struct scatterlist *sg, unsigned int len,
int prot)
@@ -866,7 +961,7 @@
unsigned long sl_offset, sl_start;
unsigned int chunk_size, chunk_offset = 0;
int ret = 0;
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
unsigned int pgprot4k, pgprot64k, pgprot1m, pgprot16m;
mutex_lock(&msm_iommu_lock);
@@ -874,7 +969,7 @@
BUG_ON(len & (SZ_4K - 1));
priv = domain->priv;
- fl_table = priv->pgtable;
+ fl_table = priv->pt.fl_table;
pgprot4k = __get_pgprot(prot, SZ_4K);
pgprot64k = __get_pgprot(prot, SZ_64K);
@@ -885,6 +980,9 @@
ret = -EINVAL;
goto fail;
}
+ ret = check_range(fl_table, va, len);
+ if (ret)
+ goto fail;
fl_offset = FL_OFFSET(va); /* Upper 12 bits */
fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
@@ -907,13 +1005,15 @@
ret = fl_16m(fl_pte, pa, pgprot16m);
if (ret)
goto fail;
- clean_pte(fl_pte, fl_pte + 16, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 16,
+ priv->pt.redirect);
fl_pte += 16;
} else if (chunk_size == SZ_1M) {
ret = fl_1m(fl_pte, pa, pgprot1m);
if (ret)
goto fail;
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1,
+ priv->pt.redirect);
fl_pte++;
}
@@ -995,7 +1095,7 @@
}
clean_pte(sl_table + sl_start, sl_table + sl_offset,
- priv->redirect);
+ priv->pt.redirect);
fl_pte++;
sl_offset = 0;
@@ -1017,14 +1117,14 @@
unsigned long *sl_table;
unsigned long sl_start, sl_end;
int used, i;
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
mutex_lock(&msm_iommu_lock);
BUG_ON(len & (SZ_4K - 1));
priv = domain->priv;
- fl_table = priv->pgtable;
+ fl_table = priv->pt.fl_table;
fl_offset = FL_OFFSET(va); /* Upper 12 bits */
fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
@@ -1040,7 +1140,7 @@
memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
clean_pte(sl_table + sl_start, sl_table + sl_end,
- priv->redirect);
+ priv->pt.redirect);
offset += (sl_end - sl_start) * SZ_4K;
va += (sl_end - sl_start) * SZ_4K;
@@ -1065,13 +1165,14 @@
free_page((unsigned long)sl_table);
*fl_pte = 0;
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1,
+ priv->pt.redirect);
}
sl_start = 0;
} else {
*fl_pte = 0;
- clean_pte(fl_pte, fl_pte + 1, priv->redirect);
+ clean_pte(fl_pte, fl_pte + 1, priv->pt.redirect);
va += SZ_1M;
offset += SZ_1M;
sl_start = 0;
@@ -1087,7 +1188,7 @@
static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
unsigned long va)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
unsigned int par;
@@ -1212,7 +1313,12 @@
}
SET_FSR(base, num, fsr);
- SET_RESUME(base, num, 1);
+ /*
+ * Only resume fetches if the registered fault handler
+ * allows it
+ */
+ if (ret != -EBUSY)
+ SET_RESUME(base, num, 1);
ret = IRQ_HANDLED;
} else
@@ -1228,8 +1334,8 @@
static phys_addr_t msm_iommu_get_pt_base_addr(struct iommu_domain *domain)
{
- struct msm_priv *priv = domain->priv;
- return __pa(priv->pgtable);
+ struct msm_iommu_priv *priv = domain->priv;
+ return __pa(priv->pt.fl_table);
}
static struct iommu_ops msm_iommu_ops = {
@@ -1287,7 +1393,7 @@
static int __init msm_iommu_init(void)
{
- if (!msm_soc_version_supports_iommu_v1())
+ if (!msm_soc_version_supports_iommu_v0())
return -ENODEV;
msm_iommu_lock_initialize();
diff --git a/drivers/iommu/msm_iommu-v2.c b/drivers/iommu/msm_iommu-v1.c
similarity index 64%
rename from drivers/iommu/msm_iommu-v2.c
rename to drivers/iommu/msm_iommu-v1.c
index 15de300..8a26003 100644
--- a/drivers/iommu/msm_iommu-v2.c
+++ b/drivers/iommu/msm_iommu-v1.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -28,9 +28,10 @@
#include <linux/regulator/consumer.h>
#include <asm/sizes.h>
-#include <mach/iommu_hw-v2.h>
+#include <mach/iommu_hw-v1.h>
#include <mach/iommu.h>
-
+#include <mach/msm_iommu_priv.h>
+#include <mach/iommu_perfmon.h>
#include "msm_iommu_pagetable.h"
/* bitmap of the page sizes currently supported */
@@ -38,10 +39,28 @@
static DEFINE_MUTEX(msm_iommu_lock);
-struct msm_priv {
- struct iommu_pt pt;
- struct list_head list_attached;
-};
+static int __enable_regulators(struct msm_iommu_drvdata *drvdata)
+{
+ int ret = regulator_enable(drvdata->gdsc);
+ if (ret)
+ goto fail;
+
+ if (drvdata->alt_gdsc)
+ ret = regulator_enable(drvdata->alt_gdsc);
+
+ if (ret)
+ regulator_disable(drvdata->gdsc);
+fail:
+ return ret;
+}
+
+static void __disable_regulators(struct msm_iommu_drvdata *drvdata)
+{
+ if (drvdata->alt_gdsc)
+ regulator_disable(drvdata->alt_gdsc);
+
+ regulator_disable(drvdata->gdsc);
+}
static int __enable_clocks(struct msm_iommu_drvdata *drvdata)
{
@@ -62,6 +81,16 @@
clk_disable_unprepare(drvdata->pclk);
}
}
+
+ if (drvdata->clk_reg_virt) {
+ unsigned int value;
+
+ value = readl_relaxed(drvdata->clk_reg_virt);
+ value &= ~0x1;
+ writel_relaxed(value, drvdata->clk_reg_virt);
+ /* Ensure clock is on before continuing */
+ mb();
+ }
fail:
return ret;
}
@@ -74,6 +103,56 @@
clk_disable_unprepare(drvdata->pclk);
}
+static void _iommu_lock_acquire(void)
+{
+ mutex_lock(&msm_iommu_lock);
+}
+
+static void _iommu_lock_release(void)
+{
+ mutex_unlock(&msm_iommu_lock);
+}
+
+struct iommu_access_ops iommu_access_ops_v1 = {
+ .iommu_power_on = __enable_regulators,
+ .iommu_power_off = __disable_regulators,
+ .iommu_clk_on = __enable_clocks,
+ .iommu_clk_off = __disable_clocks,
+ .iommu_lock_acquire = _iommu_lock_acquire,
+ .iommu_lock_release = _iommu_lock_release,
+};
+EXPORT_SYMBOL(iommu_access_ops_v1);
+
+void iommu_halt(const struct msm_iommu_drvdata *iommu_drvdata)
+{
+ if (iommu_drvdata->halt_enabled) {
+ SET_MICRO_MMU_CTRL_HALT_REQ(iommu_drvdata->base, 1);
+
+ while (GET_MICRO_MMU_CTRL_IDLE(iommu_drvdata->base) == 0)
+ cpu_relax();
+ /* Ensure device is idle before continuing */
+ mb();
+ }
+}
+
+void iommu_resume(const struct msm_iommu_drvdata *iommu_drvdata)
+{
+ if (iommu_drvdata->halt_enabled) {
+ /*
+ * Ensure transactions have completed before releasing
+ * the halt
+ */
+ mb();
+ SET_MICRO_MMU_CTRL_HALT_REQ(iommu_drvdata->base, 0);
+ /*
+ * Ensure write is complete before continuing to ensure
+ * we don't turn off clocks while transaction is still
+ * pending.
+ */
+ mb();
+ }
+}
+
static void __sync_tlb(void __iomem *base, int ctx)
{
SET_TLBSYNC(base, ctx, 0);
@@ -87,11 +166,10 @@
static int __flush_iotlb_va(struct iommu_domain *domain, unsigned int va)
{
- struct msm_priv *priv = domain->priv;
+ struct msm_iommu_priv *priv = domain->priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
int ret = 0;
- int asid;
list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);
@@ -104,11 +182,8 @@
if (ret)
goto fail;
- asid = GET_CB_CONTEXTIDR_ASID(iommu_drvdata->base,
- ctx_drvdata->num);
-
SET_TLBIVA(iommu_drvdata->base, ctx_drvdata->num,
- asid | (va & CB_TLBIVA_VA));
+ ctx_drvdata->asid | (va & CB_TLBIVA_VA));
mb();
__sync_tlb(iommu_drvdata->base, ctx_drvdata->num);
__disable_clocks(iommu_drvdata);
@@ -119,11 +194,10 @@
static int __flush_iotlb(struct iommu_domain *domain)
{
- struct msm_priv *priv = domain->priv;
+ struct msm_iommu_priv *priv = domain->priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
int ret = 0;
- int asid;
list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {
BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);
@@ -135,10 +209,8 @@
if (ret)
goto fail;
- asid = GET_CB_CONTEXTIDR_ASID(iommu_drvdata->base,
- ctx_drvdata->num);
-
- SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num, asid);
+ SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num,
+ ctx_drvdata->asid);
mb();
__sync_tlb(iommu_drvdata->base, ctx_drvdata->num);
__disable_clocks(iommu_drvdata);
@@ -148,20 +220,19 @@
return ret;
}
-static void __reset_iommu(void __iomem *base, int smt_size)
+/*
+ * May only be called for non-secure iommus
+ */
+static void __reset_iommu(void __iomem *base)
{
- int i;
+ int i, smt_size;
SET_ACR(base, 0);
- SET_NSACR(base, 0);
SET_CR2(base, 0);
- SET_NSCR2(base, 0);
SET_GFAR(base, 0);
SET_GFSRRESTORE(base, 0);
SET_TLBIALLNSNH(base, 0);
- SET_PMCR(base, 0);
- SET_SCR1(base, 0);
- SET_SSDR_N(base, 0, 0);
+ smt_size = GET_IDR0_NUMSMRG(base);
for (i = 0; i < smt_size; i++)
SET_SMR_VALID(base, i, 0);
@@ -169,9 +240,12 @@
mb();
}
-static void __program_iommu(void __iomem *base, int smt_size)
+/*
+ * May only be called for non-secure iommus
+ */
+static void __program_iommu(void __iomem *base)
{
- __reset_iommu(base, smt_size);
+ __reset_iommu(base);
SET_CR0_SMCFCFG(base, 1);
SET_CR0_USFCFG(base, 1);
@@ -181,7 +255,20 @@
SET_CR0_GFIE(base, 1);
SET_CR0_GFRE(base, 1);
SET_CR0_CLIENTPD(base, 0);
- mb(); /* Make sure writes complete before returning */
+
+ mb(); /* Make sure writes complete before returning */
+}
+
+void program_iommu_bfb_settings(void __iomem *base,
+ const struct msm_iommu_bfb_settings *bfb_settings)
+{
+ unsigned int i;
+ if (bfb_settings)
+ for (i = 0; i < bfb_settings->length; i++)
+ SET_GLOBAL_REG(base, bfb_settings->regs[i],
+ bfb_settings->data[i]);
+
+ mb(); /* Make sure writes complete before returning */
}
static void __reset_context(void __iomem *base, int ctx)
@@ -200,13 +287,69 @@
mb();
}
-static void __program_context(void __iomem *base, int ctx, int ncb,
- phys_addr_t pgtable, int redirect,
- u32 *sids, int len, int smt_size)
+static void __release_smg(void __iomem *base, int ctx)
+{
+ int i, smt_size;
+ smt_size = GET_IDR0_NUMSMRG(base);
+
+ /* Invalidate any SMGs associated with this context */
+ for (i = 0; i < smt_size; i++)
+ if (GET_SMR_VALID(base, i) &&
+ GET_S2CR_CBNDX(base, i) == ctx)
+ SET_SMR_VALID(base, i, 0);
+}
+
+static void msm_iommu_assign_ASID(const struct msm_iommu_drvdata *iommu_drvdata,
+ struct msm_iommu_ctx_drvdata *curr_ctx,
+ struct msm_iommu_priv *priv)
+{
+ unsigned int found = 0;
+ void __iomem *base = iommu_drvdata->base;
+ unsigned int i;
+ unsigned int ncb = iommu_drvdata->ncb;
+ struct msm_iommu_ctx_drvdata *tmp_drvdata;
+
+ /* Find if this page table is used elsewhere, and re-use ASID */
+ if (!list_empty(&priv->list_attached)) {
+ tmp_drvdata = list_first_entry(&priv->list_attached,
+ struct msm_iommu_ctx_drvdata, attached_elm);
+
+ ++iommu_drvdata->asid[tmp_drvdata->asid - 1];
+ curr_ctx->asid = tmp_drvdata->asid;
+
+ SET_CB_CONTEXTIDR_ASID(base, curr_ctx->num, curr_ctx->asid);
+ found = 1;
+ }
+
+ /* If page table is new, find an unused ASID */
+ if (!found) {
+ for (i = 0; i < ncb; ++i) {
+ if (iommu_drvdata->asid[i] == 0) {
+ ++iommu_drvdata->asid[i];
+ curr_ctx->asid = i + 1;
+
+ SET_CB_CONTEXTIDR_ASID(base, curr_ctx->num,
+ curr_ctx->asid);
+ found = 1;
+ break;
+ }
+ }
+ BUG_ON(!found);
+ }
+}
+
+static void __program_context(struct msm_iommu_drvdata *iommu_drvdata,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata,
+ struct msm_iommu_priv *priv, bool is_secure)
{
unsigned int prrr, nmrr;
unsigned int pn;
- int i, j, found, num = 0;
+ int num = 0, i, smt_size;
+ void __iomem *base = iommu_drvdata->base;
+ unsigned int ctx = ctx_drvdata->num;
+ u32 *sids = ctx_drvdata->sids;
+ int len = ctx_drvdata->nsid;
+ phys_addr_t pgtable = __pa(priv->pt.fl_table);
__reset_context(base, ctx);
@@ -237,7 +380,7 @@
/* Configure page tables as inner-cacheable and shareable to reduce
* the TLB miss penalty.
*/
- if (redirect) {
+ if (priv->pt.redirect) {
SET_CB_TTBR0_S(base, ctx, 1);
SET_CB_TTBR0_NOS(base, ctx, 1);
SET_CB_TTBR0_IRGN1(base, ctx, 0); /* WB, WA */
@@ -245,57 +388,45 @@
SET_CB_TTBR0_RGN(base, ctx, 1); /* WB, WA */
}
- /* Program the M2V tables for this context */
- for (i = 0; i < len / sizeof(*sids); i++) {
- for (; num < smt_size; num++)
- if (GET_SMR_VALID(base, num) == 0)
- break;
- BUG_ON(num >= smt_size);
+ if (!is_secure) {
+ smt_size = GET_IDR0_NUMSMRG(base);
+ /* Program the M2V tables for this context */
+ for (i = 0; i < len / sizeof(*sids); i++) {
+ for (; num < smt_size; num++)
+ if (GET_SMR_VALID(base, num) == 0)
+ break;
+ BUG_ON(num >= smt_size);
- SET_SMR_VALID(base, num, 1);
- SET_SMR_MASK(base, num, 0);
- SET_SMR_ID(base, num, sids[i]);
+ SET_SMR_VALID(base, num, 1);
+ SET_SMR_MASK(base, num, 0);
+ SET_SMR_ID(base, num, sids[i]);
- /* Set VMID = 0 */
- SET_S2CR_N(base, num, 0);
- SET_S2CR_CBNDX(base, num, ctx);
- /* Set security bit override to be Non-secure */
- SET_S2CR_NSCFG(base, num, 3);
+ SET_S2CR_N(base, num, 0);
+ SET_S2CR_CBNDX(base, num, ctx);
+ SET_S2CR_MEMATTR(base, num, 0x0A);
+ /* Set security bit override to be Non-secure */
+ SET_S2CR_NSCFG(base, num, 3);
+ }
+ SET_CBAR_N(base, ctx, 0);
+
+ /* Stage 1 Context with Stage 2 bypass */
+ SET_CBAR_TYPE(base, ctx, 1);
+
+ /* Route page faults to the non-secure interrupt */
+ SET_CBAR_IRPTNDX(base, ctx, 1);
+
+ /* Set VMID to non-secure HLOS */
+ SET_CBAR_VMID(base, ctx, 3);
+
+ /* Bypass is treated as inner-shareable */
+ SET_CBAR_BPSHCFG(base, ctx, 2);
+
+ /* Do not downgrade memory attributes */
+ SET_CBAR_MEMATTR(base, ctx, 0x0A);
+
}
- SET_CBAR_N(base, ctx, 0);
- /* Stage 1 Context with Stage 2 bypass */
- SET_CBAR_TYPE(base, ctx, 1);
- /* Route page faults to the non-secure interrupt */
- SET_CBAR_IRPTNDX(base, ctx, 1);
-
- /* Find if this page table is used elsewhere, and re-use ASID */
- found = 0;
- for (i = 0; i < ncb; i++)
- if ((GET_CB_TTBR0_ADDR(base, i) == pn) && (i != ctx)) {
- SET_CB_CONTEXTIDR_ASID(base, ctx, \
- GET_CB_CONTEXTIDR_ASID(base, i));
- found = 1;
- break;
- }
-
- /* If page table is new, find an unused ASID */
- if (!found) {
- for (i = 0; i < ncb; i++) {
- found = 0;
- for (j = 0; j < ncb; j++) {
- if (GET_CB_CONTEXTIDR_ASID(base, j) == i &&
- j != ctx)
- found = 1;
- }
-
- if (!found) {
- SET_CB_CONTEXTIDR_ASID(base, ctx, i);
- break;
- }
- }
- BUG_ON(found);
- }
+ msm_iommu_assign_ASID(iommu_drvdata, ctx_drvdata, priv);
/* Enable the MMU */
SET_CB_SCTLR_M(base, ctx, 1);
@@ -304,7 +435,7 @@
static int msm_iommu_domain_init(struct iommu_domain *domain, int flags)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
@@ -328,7 +459,7 @@
static void msm_iommu_domain_destroy(struct iommu_domain *domain)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
mutex_lock(&msm_iommu_lock);
priv = domain->priv;
@@ -341,32 +472,14 @@
mutex_unlock(&msm_iommu_lock);
}
-static int msm_iommu_ctx_attached(struct device *dev)
-{
- struct platform_device *pdev;
- struct device_node *child;
- struct msm_iommu_ctx_drvdata *ctx;
-
- for_each_child_of_node(dev->of_node, child) {
- pdev = of_find_device_by_node(child);
-
- ctx = dev_get_drvdata(&pdev->dev);
- if (ctx->attached_domain) {
- of_node_put(child);
- return 1;
- }
- }
-
- return 0;
-}
-
static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
struct msm_iommu_ctx_drvdata *tmp_drvdata;
int ret;
+ int is_secure;
mutex_lock(&msm_iommu_lock);
@@ -394,28 +507,53 @@
goto fail;
}
- ret = regulator_enable(iommu_drvdata->gdsc);
+ is_secure = iommu_drvdata->sec_id != -1;
+
+ ret = __enable_regulators(iommu_drvdata);
if (ret)
goto fail;
ret = __enable_clocks(iommu_drvdata);
if (ret) {
- regulator_disable(iommu_drvdata->gdsc);
+ __disable_regulators(iommu_drvdata);
goto fail;
}
- if (!msm_iommu_ctx_attached(dev->parent))
- __program_iommu(iommu_drvdata->base, iommu_drvdata->nsmr);
+ /* We can only do this once */
+ if (!iommu_drvdata->ctx_attach_count) {
+ if (!is_secure) {
+ iommu_halt(iommu_drvdata);
+ __program_iommu(iommu_drvdata->base);
+ iommu_resume(iommu_drvdata);
+ } else {
+ ret = msm_iommu_sec_program_iommu(
+ iommu_drvdata->sec_id);
+ if (ret) {
+ __disable_regulators(iommu_drvdata);
+ __disable_clocks(iommu_drvdata);
+ goto fail;
+ }
+ }
+ program_iommu_bfb_settings(iommu_drvdata->base,
+ iommu_drvdata->bfb_settings);
+ }
- __program_context(iommu_drvdata->base, ctx_drvdata->num,
- iommu_drvdata->ncb, __pa(priv->pt.fl_table),
- priv->pt.redirect, ctx_drvdata->sids, ctx_drvdata->nsid,
- iommu_drvdata->nsmr);
+ iommu_halt(iommu_drvdata);
+
+ __program_context(iommu_drvdata, ctx_drvdata, priv, is_secure);
+
+ iommu_resume(iommu_drvdata);
+
__disable_clocks(iommu_drvdata);
list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
ctx_drvdata->attached_domain = domain;
+ ++iommu_drvdata->ctx_attach_count;
+ mutex_unlock(&msm_iommu_lock);
+
+ msm_iommu_attached(dev->parent);
+ return ret;
fail:
mutex_unlock(&msm_iommu_lock);
return ret;
@@ -424,10 +562,13 @@
static void msm_iommu_detach_dev(struct iommu_domain *domain,
struct device *dev)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
int ret;
+ int is_secure;
+
+ msm_iommu_detached(dev->parent);
mutex_lock(&msm_iommu_lock);
priv = domain->priv;
@@ -443,17 +584,30 @@
if (ret)
goto fail;
- SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num,
- GET_CB_CONTEXTIDR_ASID(iommu_drvdata->base, ctx_drvdata->num));
+ is_secure = iommu_drvdata->sec_id != -1;
+
+ SET_TLBIASID(iommu_drvdata->base, ctx_drvdata->num, ctx_drvdata->asid);
+
+ BUG_ON(iommu_drvdata->asid[ctx_drvdata->asid - 1] == 0);
+ iommu_drvdata->asid[ctx_drvdata->asid - 1]--;
+ ctx_drvdata->asid = -1;
+
+ iommu_halt(iommu_drvdata);
__reset_context(iommu_drvdata->base, ctx_drvdata->num);
+ if (!is_secure)
+ __release_smg(iommu_drvdata->base, ctx_drvdata->num);
+
+ iommu_resume(iommu_drvdata);
+
__disable_clocks(iommu_drvdata);
- regulator_disable(iommu_drvdata->gdsc);
+ __disable_regulators(iommu_drvdata);
list_del_init(&ctx_drvdata->attached_elm);
ctx_drvdata->attached_domain = NULL;
-
+ BUG_ON(iommu_drvdata->ctx_attach_count == 0);
+ --iommu_drvdata->ctx_attach_count;
fail:
mutex_unlock(&msm_iommu_lock);
}
@@ -461,7 +615,7 @@
static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
phys_addr_t pa, size_t len, int prot)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
int ret = 0;
mutex_lock(&msm_iommu_lock);
@@ -485,7 +639,7 @@
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
size_t len)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
int ret = -ENODEV;
mutex_lock(&msm_iommu_lock);
@@ -512,7 +666,7 @@
int prot)
{
int ret;
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
mutex_lock(&msm_iommu_lock);
@@ -536,7 +690,7 @@
static int msm_iommu_unmap_range(struct iommu_domain *domain, unsigned int va,
unsigned int len)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
mutex_lock(&msm_iommu_lock);
@@ -551,7 +705,7 @@
static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
unsigned long va)
{
- struct msm_priv *priv;
+ struct msm_iommu_priv *priv;
struct msm_iommu_drvdata *iommu_drvdata;
struct msm_iommu_ctx_drvdata *ctx_drvdata;
unsigned int par;
@@ -688,7 +842,7 @@
static phys_addr_t msm_iommu_get_pt_base_addr(struct iommu_domain *domain)
{
- struct msm_priv *priv = domain->priv;
+ struct msm_iommu_priv *priv = domain->priv;
return __pa(priv->pt.fl_table);
}
diff --git a/drivers/iommu/msm_iommu_dev-v0.c b/drivers/iommu/msm_iommu_dev-v0.c
new file mode 100644
index 0000000..549800f
--- /dev/null
+++ b/drivers/iommu/msm_iommu_dev-v0.c
@@ -0,0 +1,695 @@
+/* Copyright (c) 2010-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/iommu.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+
+#include <mach/iommu_perfmon.h>
+#include <mach/iommu_hw-v0.h>
+#include <mach/iommu.h>
+
+static DEFINE_MUTEX(iommu_list_lock);
+static LIST_HEAD(iommu_list);
+
+void msm_iommu_add_drv(struct msm_iommu_drvdata *drv)
+{
+ mutex_lock(&iommu_list_lock);
+ list_add(&drv->list, &iommu_list);
+ mutex_unlock(&iommu_list_lock);
+}
+
+void msm_iommu_remove_drv(struct msm_iommu_drvdata *drv)
+{
+ mutex_lock(&iommu_list_lock);
+ list_del(&drv->list);
+ mutex_unlock(&iommu_list_lock);
+}
+
+static int find_iommu_ctx(struct device *dev, void *data)
+{
+ struct msm_iommu_ctx_drvdata *c;
+
+ c = dev_get_drvdata(dev);
+ if (!c || !c->name)
+ return 0;
+
+ return !strcmp(data, c->name);
+}
+
+static struct device *find_context(struct device *dev, const char *name)
+{
+ return device_find_child(dev, (void *)name, find_iommu_ctx);
+}
+
+struct device *msm_iommu_get_ctx(const char *ctx_name)
+{
+ struct msm_iommu_drvdata *drv;
+ struct device *dev = NULL;
+
+ mutex_lock(&iommu_list_lock);
+ list_for_each_entry(drv, &iommu_list, list) {
+ dev = find_context(drv->dev, ctx_name);
+ if (dev)
+ break;
+ }
+ mutex_unlock(&iommu_list_lock);
+
+ if (!dev || !dev_get_drvdata(dev))
+ pr_err("Could not find context <%s>\n", ctx_name);
+ put_device(dev);
+
+ return dev;
+}
+EXPORT_SYMBOL(msm_iommu_get_ctx);
+
+static void msm_iommu_reset(void __iomem *base, void __iomem *glb_base, int ncb)
+{
+ int ctx;
+
+ SET_RPUE(glb_base, 0);
+ SET_RPUEIE(glb_base, 0);
+ SET_ESRRESTORE(glb_base, 0);
+ SET_TBE(glb_base, 0);
+ SET_CR(glb_base, 0);
+ SET_SPDMBE(glb_base, 0);
+ SET_TESTBUSCR(glb_base, 0);
+ SET_TLBRSW(glb_base, 0);
+ SET_GLOBAL_TLBIALL(glb_base, 0);
+ SET_RPU_ACR(glb_base, 0);
+ SET_TLBLKCRWE(glb_base, 1);
+
+ for (ctx = 0; ctx < ncb; ctx++) {
+ SET_BPRCOSH(glb_base, ctx, 0);
+ SET_BPRCISH(glb_base, ctx, 0);
+ SET_BPRCNSH(glb_base, ctx, 0);
+ SET_BPSHCFG(glb_base, ctx, 0);
+ SET_BPMTCFG(glb_base, ctx, 0);
+ SET_ACTLR(base, ctx, 0);
+ SET_SCTLR(base, ctx, 0);
+ SET_FSRRESTORE(base, ctx, 0);
+ SET_TTBR0(base, ctx, 0);
+ SET_TTBR1(base, ctx, 0);
+ SET_TTBCR(base, ctx, 0);
+ SET_BFBCR(base, ctx, 0);
+ SET_PAR(base, ctx, 0);
+ SET_FAR(base, ctx, 0);
+ SET_TLBFLPTER(base, ctx, 0);
+ SET_TLBSLPTER(base, ctx, 0);
+ SET_TLBLKCR(base, ctx, 0);
+ SET_CTX_TLBIALL(base, ctx, 0);
+ SET_TLBIVA(base, ctx, 0);
+ SET_PRRR(base, ctx, 0);
+ SET_NMRR(base, ctx, 0);
+ SET_CONTEXTIDR(base, ctx, 0);
+ }
+ mb();
+}
+
+static int msm_iommu_parse_dt(struct platform_device *pdev,
+ struct msm_iommu_drvdata *drvdata)
+{
+#ifdef CONFIG_OF_DEVICE
+ struct device_node *child;
+ struct resource *r;
+ u32 glb_offset = 0;
+ int ret;
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!r) {
+ pr_err("%s: Missing property reg\n", __func__);
+ return -EINVAL;
+ }
+ drvdata->base = devm_ioremap(&pdev->dev, r->start, resource_size(r));
+ if (!drvdata->base) {
+ pr_err("%s: Unable to ioremap %pr\n", __func__, r);
+ return -ENOMEM;
+ }
+ drvdata->glb_base = drvdata->base;
+
+ if (!of_property_read_u32(pdev->dev.of_node, "qcom,glb-offset",
+ &glb_offset)) {
+ drvdata->glb_base += glb_offset;
+ } else {
+ pr_err("%s: Missing property qcom,glb-offset\n", __func__);
+ return -EINVAL;
+ }
+
+ for_each_child_of_node(pdev->dev.of_node, child) {
+ drvdata->ncb++;
+ if (!of_platform_device_create(child, NULL, &pdev->dev))
+ pr_err("Failed to create %s device\n", child->name);
+ }
+
+ ret = of_property_read_string(pdev->dev.of_node, "label",
+ &drvdata->name);
+ if (ret) {
+ pr_err("%s: Missing property label\n", __func__);
+ return -EINVAL;
+ }
+ drvdata->sec_id = -1;
+ drvdata->ttbr_split = 0;
+#endif
+ return 0;
+}
+
+static int __get_clocks(struct platform_device *pdev,
+ struct msm_iommu_drvdata *drvdata)
+{
+ int ret = 0;
+
+ drvdata->pclk = clk_get(&pdev->dev, "iface_clk");
+ if (IS_ERR(drvdata->pclk)) {
+ ret = PTR_ERR(drvdata->pclk);
+ drvdata->pclk = NULL;
+ pr_err("Unable to get %s clock for %s IOMMU device\n",
+ dev_name(&pdev->dev), drvdata->name);
+ goto fail;
+ }
+
+ drvdata->clk = clk_get(&pdev->dev, "core_clk");
+
+ if (!IS_ERR(drvdata->clk)) {
+ if (clk_get_rate(drvdata->clk) == 0) {
+ ret = clk_round_rate(drvdata->clk, 1000);
+ clk_set_rate(drvdata->clk, ret);
+ }
+ } else {
+ drvdata->clk = NULL;
+ }
+ return 0;
+fail:
+ return ret;
+}
+
+static void __put_clocks(struct msm_iommu_drvdata *drvdata)
+{
+ if (drvdata->clk)
+ clk_put(drvdata->clk);
+ clk_put(drvdata->pclk);
+}
+
+static int __enable_clocks(struct msm_iommu_drvdata *drvdata)
+{
+ int ret;
+
+ ret = clk_prepare_enable(drvdata->pclk);
+ if (ret)
+ goto fail;
+
+ if (drvdata->clk) {
+ ret = clk_prepare_enable(drvdata->clk);
+ if (ret)
+ clk_disable_unprepare(drvdata->pclk);
+ }
+fail:
+ return ret;
+}
+
+static void __disable_clocks(struct msm_iommu_drvdata *drvdata)
+{
+ if (drvdata->clk)
+ clk_disable_unprepare(drvdata->clk);
+ clk_disable_unprepare(drvdata->pclk);
+}
+
+/*
+ * Do a basic check of the IOMMU by performing an ATS operation
+ * on context bank 0.
+ */
+static int iommu_sanity_check(struct msm_iommu_drvdata *drvdata)
+{
+ int par;
+ int ret = 0;
+
+ SET_M(drvdata->base, 0, 1);
+ SET_PAR(drvdata->base, 0, 0);
+ SET_V2PCFG(drvdata->base, 0, 1);
+ SET_V2PPR(drvdata->base, 0, 0);
+ mb();
+ par = GET_PAR(drvdata->base, 0);
+ SET_V2PCFG(drvdata->base, 0, 0);
+ SET_M(drvdata->base, 0, 0);
+ mb();
+
+ if (!par) {
+ pr_err("%s: Invalid PAR value detected\n", drvdata->name);
+ ret = -ENODEV;
+ }
+ return ret;
+}
+
+static int msm_iommu_pmon_parse_dt(struct platform_device *pdev,
+ struct iommu_pmon *pmon_info)
+{
+ int ret = 0;
+ int irq = platform_get_irq(pdev, 0);
+ unsigned int cls_prop_size;
+
+ if (irq > 0) {
+ pmon_info->iommu.evt_irq = platform_get_irq(pdev, 0);
+
+ ret = of_property_read_u32(pdev->dev.of_node,
+ "qcom,iommu-pmu-ngroups",
+ &pmon_info->num_groups);
+ if (ret) {
+ pr_err("Error reading qcom,iommu-pmu-ngroups\n");
+ goto fail;
+ }
+ ret = of_property_read_u32(pdev->dev.of_node,
+ "qcom,iommu-pmu-ncounters",
+ &pmon_info->num_counters);
+ if (ret) {
+ pr_err("Error reading qcom,iommu-pmu-ncounters\n");
+ goto fail;
+ }
+
+ if (!of_get_property(pdev->dev.of_node,
+ "qcom,iommu-pmu-event-classes",
+ &cls_prop_size)) {
+ pr_err("Error reading qcom,iommu-pmu-event-classes\n");
+ return -EINVAL;
+ }
+
+ pmon_info->event_cls_supported =
+ devm_kzalloc(&pdev->dev, cls_prop_size, GFP_KERNEL);
+
+ if (!pmon_info->event_cls_supported) {
+ pr_err("Unable to get memory for event class array\n");
+ return -ENOMEM;
+ }
+
+ pmon_info->nevent_cls_supported = cls_prop_size / sizeof(u32);
+
+ ret = of_property_read_u32_array(pdev->dev.of_node,
+ "qcom,iommu-pmu-event-classes",
+ pmon_info->event_cls_supported,
+ pmon_info->nevent_cls_supported);
+ if (ret) {
+ pr_err("Error reading qcom,iommu-pmu-event-classes\n");
+ return ret;
+ }
+ } else {
+ pmon_info->iommu.evt_irq = -1;
+ ret = irq;
+ }
+
+fail:
+ return ret;
+}
+
+static int msm_iommu_probe(struct platform_device *pdev)
+{
+ struct iommu_pmon *pmon_info;
+ struct msm_iommu_drvdata *drvdata;
+ struct msm_iommu_dev *iommu_dev = pdev->dev.platform_data;
+ int ret;
+
+ drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);
+
+ if (!drvdata) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+
+ if (pdev->dev.of_node) {
+ ret = msm_iommu_parse_dt(pdev, drvdata);
+ if (ret)
+ goto fail;
+ } else if (pdev->dev.platform_data) {
+ struct resource *r, *r2;
+ resource_size_t len;
+
+ r = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+ "physbase");
+
+ if (!r) {
+ ret = -ENODEV;
+ goto fail;
+ }
+
+ len = resource_size(r);
+
+ r2 = request_mem_region(r->start, len, r->name);
+ if (!r2) {
+ pr_err("Could not request memory region: %pr\n", r);
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ drvdata->base = devm_ioremap(&pdev->dev, r2->start, len);
+
+ if (!drvdata->base) {
+ pr_err("Could not ioremap: %pr\n", r);
+ ret = -EBUSY;
+ goto fail;
+ }
+ /*
+ * Global register space offset for legacy IOMMUv1 hardware
+ * is always 0xFF000
+ */
+ drvdata->glb_base = drvdata->base + 0xFF000;
+ drvdata->name = iommu_dev->name;
+ drvdata->dev = &pdev->dev;
+ drvdata->ncb = iommu_dev->ncb;
+ drvdata->ttbr_split = iommu_dev->ttbr_split;
+ } else {
+ ret = -ENODEV;
+ goto fail;
+ }
+
+ drvdata->dev = &pdev->dev;
+
+ ret = __get_clocks(pdev, drvdata);
+
+ if (ret)
+ goto fail;
+
+ __enable_clocks(drvdata);
+
+ msm_iommu_reset(drvdata->base, drvdata->glb_base, drvdata->ncb);
+
+ ret = iommu_sanity_check(drvdata);
+ if (ret)
+ goto fail_clk;
+
+ pr_info("device %s mapped at %p, with %d ctx banks\n",
+ drvdata->name, drvdata->base, drvdata->ncb);
+
+ msm_iommu_add_drv(drvdata);
+ platform_set_drvdata(pdev, drvdata);
+
+ __disable_clocks(drvdata);
+
+ pmon_info = msm_iommu_pm_alloc(&pdev->dev);
+ if (pmon_info != NULL) {
+ ret = msm_iommu_pmon_parse_dt(pdev, pmon_info);
+ if (ret) {
+ msm_iommu_pm_free(&pdev->dev);
+ pr_info("%s: pmon not available.\n", drvdata->name);
+ } else {
+ pmon_info->iommu.base = drvdata->base;
+ pmon_info->iommu.ops = &iommu_access_ops_v0;
+ pmon_info->iommu.hw_ops = iommu_pm_get_hw_ops_v0();
+ pmon_info->iommu.iommu_name = drvdata->name;
+ ret = msm_iommu_pm_iommu_register(pmon_info);
+ if (ret) {
+ pr_err("%s iommu register fail\n",
+ drvdata->name);
+ msm_iommu_pm_free(&pdev->dev);
+ } else {
+ pr_debug("%s iommu registered for pmon\n",
+ pmon_info->iommu.iommu_name);
+ }
+ }
+ }
+
+ return 0;
+
+fail_clk:
+ __disable_clocks(drvdata);
+ __put_clocks(drvdata);
+fail:
+ return ret;
+}
+
+static int msm_iommu_remove(struct platform_device *pdev)
+{
+ struct msm_iommu_drvdata *drv = NULL;
+
+ drv = platform_get_drvdata(pdev);
+ if (drv) {
+ msm_iommu_remove_drv(drv);
+ if (drv->clk)
+ clk_put(drv->clk);
+ clk_put(drv->pclk);
+ platform_set_drvdata(pdev, NULL);
+ }
+ return 0;
+}
+
+static int msm_iommu_ctx_parse_dt(struct platform_device *pdev,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata)
+{
+ struct resource *r, rp;
+ int irq, ret;
+ u32 nmid_array_size;
+ u32 nmid;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq > 0) {
+ ret = request_threaded_irq(irq, NULL,
+ msm_iommu_fault_handler,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "msm_iommu_nonsecure_irq", ctx_drvdata);
+ if (ret) {
+ pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
+ return ret;
+ }
+ }
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!r) {
+ pr_err("Could not find reg property for context bank\n");
+ return -EINVAL;
+ }
+
+ ret = of_address_to_resource(pdev->dev.parent->of_node, 0, &rp);
+ if (ret) {
+ pr_err("of_address_to_resource failed\n");
+ return -EINVAL;
+ }
+
+ /* Calculate the context bank number using the base addresses. CB0
+ * starts at the base address.
+ */
+ ctx_drvdata->num = ((r->start - rp.start) >> CTX_SHIFT);
+
+ if (of_property_read_string(pdev->dev.of_node, "label",
+ &ctx_drvdata->name)) {
+ pr_err("Could not find label property\n");
+ return -EINVAL;
+ }
+
+ if (!of_get_property(pdev->dev.of_node, "qcom,iommu-ctx-mids",
+ &nmid_array_size)) {
+ pr_err("Could not find iommu-ctx-mids property\n");
+ return -EINVAL;
+ }
+ if (nmid_array_size >= sizeof(ctx_drvdata->sids)) {
+ pr_err("Too many mids defined - array size: %u, mids size: %u\n",
+ nmid_array_size, sizeof(ctx_drvdata->sids));
+ return -EINVAL;
+ }
+ nmid = nmid_array_size / sizeof(*ctx_drvdata->sids);
+
+ if (of_property_read_u32_array(pdev->dev.of_node, "qcom,iommu-ctx-mids",
+ ctx_drvdata->sids, nmid)) {
+ pr_err("Could not find iommu-ctx-mids property\n");
+ return -EINVAL;
+ }
+ ctx_drvdata->nsid = nmid;
+
+ return 0;
+}
+
+static void __program_m2v_tables(struct msm_iommu_drvdata *drvdata,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata)
+{
+ int i;
+
+ /* Program the M2V tables for this context */
+ for (i = 0; i < ctx_drvdata->nsid; i++) {
+ int sid = ctx_drvdata->sids[i];
+ int num = ctx_drvdata->num;
+
+ SET_M2VCBR_N(drvdata->glb_base, sid, 0);
+ SET_CBACR_N(drvdata->glb_base, num, 0);
+
+ /* Route page faults to the non-secure interrupt */
+ SET_IRPTNDX(drvdata->glb_base, num, 1);
+
+ /* Set VMID = 0 */
+ SET_VMID(drvdata->glb_base, sid, 0);
+
+ /* Set the context number for that SID to this context */
+ SET_CBNDX(drvdata->glb_base, sid, num);
+
+ /* Set SID associated with this context bank to 0 */
+ SET_CBVMID(drvdata->glb_base, num, 0);
+
+ /* Set the ASID for TLB tagging for this context to 0 */
+ SET_CONTEXTIDR_ASID(drvdata->base, num, 0);
+
+ /* Set security bit override to be Non-secure */
+ SET_NSCFG(drvdata->glb_base, sid, 3);
+ }
+ mb();
+}
+
+static int msm_iommu_ctx_probe(struct platform_device *pdev)
+{
+ struct msm_iommu_drvdata *drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;
+ int i, ret, irq;
+ if (!pdev->dev.parent) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ drvdata = dev_get_drvdata(pdev->dev.parent);
+
+ if (!drvdata) {
+ ret = -ENODEV;
+ goto fail;
+ }
+
+ ctx_drvdata = devm_kzalloc(&pdev->dev, sizeof(*ctx_drvdata),
+ GFP_KERNEL);
+ if (!ctx_drvdata) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+
+ ctx_drvdata->pdev = pdev;
+ INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
+ platform_set_drvdata(pdev, ctx_drvdata);
+ ctx_drvdata->attach_count = 0;
+
+ if (pdev->dev.of_node) {
+ ret = msm_iommu_ctx_parse_dt(pdev, ctx_drvdata);
+ if (ret)
+ goto fail;
+ } else if (pdev->dev.platform_data) {
+ struct msm_iommu_ctx_dev *c = pdev->dev.platform_data;
+
+ ctx_drvdata->num = c->num;
+ ctx_drvdata->name = c->name;
+
+ for (i = 0; i < MAX_NUM_MIDS; ++i) {
+ if (c->mids[i] == -1) {
+ ctx_drvdata->nsid = i;
+ break;
+ }
+ ctx_drvdata->sids[i] = c->mids[i];
+ }
+ irq = platform_get_irq_byname(
+ to_platform_device(pdev->dev.parent),
+ "nonsecure_irq");
+ if (irq < 0) {
+ ret = -ENODEV;
+ goto fail;
+ }
+
+ ret = request_threaded_irq(irq, NULL, msm_iommu_fault_handler,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "msm_iommu_nonsecure_irq", ctx_drvdata);
+
+ if (ret) {
+ pr_err("request_threaded_irq %d failed: %d\n", irq,
+ ret);
+ goto fail;
+ }
+ } else {
+ ret = -ENODEV;
+ goto fail;
+ }
+
+ __enable_clocks(drvdata);
+ __program_m2v_tables(drvdata, ctx_drvdata);
+ __disable_clocks(drvdata);
+
+ dev_info(&pdev->dev, "context %s using bank %d\n", ctx_drvdata->name,
+ ctx_drvdata->num);
+ return 0;
+fail:
+ return ret;
+}
+
+static int __devexit msm_iommu_ctx_remove(struct platform_device *pdev)
+{
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+}
+
+
+static struct of_device_id msm_iommu_match_table[] = {
+ { .compatible = "qcom,msm-smmu-v0", },
+ {}
+};
+
+static struct platform_driver msm_iommu_driver = {
+ .driver = {
+ .name = "msm_iommu-v0",
+ .of_match_table = msm_iommu_match_table,
+ },
+ .probe = msm_iommu_probe,
+ .remove = __devexit_p(msm_iommu_remove),
+};
+
+static struct of_device_id msm_iommu_ctx_match_table[] = {
+ { .name = "qcom,iommu-ctx", },
+ {}
+};
+
+static struct platform_driver msm_iommu_ctx_driver = {
+ .driver = {
+ .name = "msm_iommu_ctx",
+ .of_match_table = msm_iommu_ctx_match_table,
+ },
+ .probe = msm_iommu_ctx_probe,
+ .remove = __devexit_p(msm_iommu_ctx_remove),
+};
+
+static int __init msm_iommu_driver_init(void)
+{
+ int ret;
+ ret = platform_driver_register(&msm_iommu_driver);
+ if (ret != 0) {
+ pr_err("Failed to register IOMMU driver\n");
+ goto error;
+ }
+
+ ret = platform_driver_register(&msm_iommu_ctx_driver);
+ if (ret != 0) {
+ pr_err("Failed to register IOMMU context driver\n");
+ goto error;
+ }
+
+error:
+ return ret;
+}
+
+static void __exit msm_iommu_driver_exit(void)
+{
+ platform_driver_unregister(&msm_iommu_ctx_driver);
+ platform_driver_unregister(&msm_iommu_driver);
+}
+
+subsys_initcall(msm_iommu_driver_init);
+module_exit(msm_iommu_driver_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/msm_iommu_dev-v1.c b/drivers/iommu/msm_iommu_dev-v1.c
new file mode 100644
index 0000000..418a086
--- /dev/null
+++ b/drivers/iommu/msm_iommu_dev-v1.c
@@ -0,0 +1,462 @@
+/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/iommu.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+
+#include <mach/iommu_hw-v1.h>
+#include <mach/iommu.h>
+#include <mach/iommu_perfmon.h>
+
+static int msm_iommu_parse_bfb_settings(struct platform_device *pdev,
+ struct msm_iommu_drvdata *drvdata)
+{
+ struct msm_iommu_bfb_settings *bfb_settings;
+ u32 nreg, nval;
+ int ret;
+
+ /*
+ * It is not valid for a device to have the qcom,iommu-bfb-regs
+ * property but not the qcom,iommu-bfb-data property, and vice versa.
+ */
+ if (!of_get_property(pdev->dev.of_node, "qcom,iommu-bfb-regs", &nreg)) {
+ if (of_get_property(pdev->dev.of_node, "qcom,iommu-bfb-data",
+ &nval))
+ return -EINVAL;
+ return 0;
+ }
+
+ if (!of_get_property(pdev->dev.of_node, "qcom,iommu-bfb-data", &nval))
+ return -EINVAL;
+
+ if (nreg >= sizeof(bfb_settings->regs))
+ return -EINVAL;
+
+ if (nval >= sizeof(bfb_settings->data))
+ return -EINVAL;
+
+ if (nval != nreg)
+ return -EINVAL;
+
+ bfb_settings = devm_kzalloc(&pdev->dev, sizeof(*bfb_settings),
+ GFP_KERNEL);
+ if (!bfb_settings)
+ return -ENOMEM;
+
+ ret = of_property_read_u32_array(pdev->dev.of_node,
+ "qcom,iommu-bfb-regs",
+ bfb_settings->regs,
+ nreg / sizeof(*bfb_settings->regs));
+ if (ret)
+ return ret;
+
+ ret = of_property_read_u32_array(pdev->dev.of_node,
+ "qcom,iommu-bfb-data",
+ bfb_settings->data,
+ nval / sizeof(*bfb_settings->data));
+ if (ret)
+ return ret;
+
+ bfb_settings->length = nreg / sizeof(*bfb_settings->regs);
+
+ drvdata->bfb_settings = bfb_settings;
+ return 0;
+}
+
+static int msm_iommu_parse_dt(struct platform_device *pdev,
+ struct msm_iommu_drvdata *drvdata)
+{
+ struct device_node *child;
+ int ret = 0;
+ struct resource *r;
+
+ drvdata->dev = &pdev->dev;
+ msm_iommu_add_drv(drvdata);
+
+ ret = msm_iommu_parse_bfb_settings(pdev, drvdata);
+ if (ret)
+ goto fail;
+
+ for_each_child_of_node(pdev->dev.of_node, child) {
+ drvdata->ncb++;
+ if (!of_platform_device_create(child, NULL, &pdev->dev))
+ pr_err("Failed to create %s device\n", child->name);
+ }
+
+ drvdata->asid = devm_kzalloc(&pdev->dev, drvdata->ncb * sizeof(int),
+ GFP_KERNEL);
+
+ if (!drvdata->asid) {
+ pr_err("Unable to get memory for asid array\n");
+ ret = -ENOMEM;
+ goto fail;
+ }
+
+ ret = of_property_read_string(pdev->dev.of_node, "label",
+ &drvdata->name);
+ if (ret)
+ goto fail;
+
+ drvdata->sec_id = -1;
+ of_property_read_u32(pdev->dev.of_node, "qcom,iommu-secure-id",
+ &drvdata->sec_id);
+
+ r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "clk_base");
+ if (r) {
+ drvdata->clk_reg_virt = devm_ioremap(&pdev->dev, r->start,
+ resource_size(r));
+ if (!drvdata->clk_reg_virt) {
+ pr_err("Failed to map resource for iommu clk: %pr\n",
+ r);
+ ret = -ENOMEM;
+ goto fail;
+ }
+ }
+
+ drvdata->halt_enabled = of_property_read_bool(pdev->dev.of_node,
+ "qcom,iommu-enable-halt");
+
+ return 0;
+fail:
+ return ret;
+}
+
+static int msm_iommu_pmon_parse_dt(struct platform_device *pdev,
+ struct iommu_pmon *pmon_info)
+{
+ int ret = 0;
+ int irq = platform_get_irq(pdev, 0);
+ unsigned int cls_prop_size;
+
+ if (irq > 0) {
+ pmon_info->iommu.evt_irq = platform_get_irq(pdev, 0);
+
+ ret = of_property_read_u32(pdev->dev.of_node,
+ "qcom,iommu-pmu-ngroups",
+ &pmon_info->num_groups);
+ if (ret) {
+ pr_err("Error reading qcom,iommu-pmu-ngroups\n");
+ goto fail;
+ }
+ ret = of_property_read_u32(pdev->dev.of_node,
+ "qcom,iommu-pmu-ncounters",
+ &pmon_info->num_counters);
+ if (ret) {
+ pr_err("Error reading qcom,iommu-pmu-ncounters\n");
+ goto fail;
+ }
+
+ if (!of_get_property(pdev->dev.of_node,
+ "qcom,iommu-pmu-event-classes",
+ &cls_prop_size)) {
+ pr_err("Error reading qcom,iommu-pmu-event-classes\n");
+ return -EINVAL;
+ }
+
+ pmon_info->event_cls_supported =
+ devm_kzalloc(&pdev->dev, cls_prop_size, GFP_KERNEL);
+
+ if (!pmon_info->event_cls_supported) {
+ pr_err("Unable to get memory for event class array\n");
+ return -ENOMEM;
+ }
+
+ pmon_info->nevent_cls_supported = cls_prop_size / sizeof(u32);
+
+ ret = of_property_read_u32_array(pdev->dev.of_node,
+ "qcom,iommu-pmu-event-classes",
+ pmon_info->event_cls_supported,
+ pmon_info->nevent_cls_supported);
+ if (ret) {
+ pr_err("Error reading qcom,iommu-pmu-event-classes\n");
+ return ret;
+ }
+ } else {
+ pmon_info->iommu.evt_irq = -1;
+ ret = irq;
+ }
+
+fail:
+ return ret;
+}
+
+static int __devinit msm_iommu_probe(struct platform_device *pdev)
+{
+ struct iommu_pmon *pmon_info;
+ struct msm_iommu_drvdata *drvdata;
+ struct resource *r;
+ int ret, needs_alt_core_clk;
+
+ drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);
+ if (!drvdata)
+ return -ENOMEM;
+
+ r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "iommu_base");
+ if (!r)
+ return -EINVAL;
+
+ drvdata->base = devm_ioremap(&pdev->dev, r->start, resource_size(r));
+ if (!drvdata->base)
+ return -ENOMEM;
+
+ drvdata->glb_base = drvdata->base;
+
+ drvdata->gdsc = devm_regulator_get(&pdev->dev, "vdd");
+ if (IS_ERR(drvdata->gdsc))
+ return -EINVAL;
+
+ drvdata->alt_gdsc = devm_regulator_get(&pdev->dev, "qcom,alt-vdd");
+ if (IS_ERR(drvdata->alt_gdsc))
+ drvdata->alt_gdsc = NULL;
+
+ drvdata->pclk = devm_clk_get(&pdev->dev, "iface_clk");
+ if (IS_ERR(drvdata->pclk))
+ return PTR_ERR(drvdata->pclk);
+
+ drvdata->clk = devm_clk_get(&pdev->dev, "core_clk");
+ if (IS_ERR(drvdata->clk))
+ return PTR_ERR(drvdata->clk);
+
+ needs_alt_core_clk = of_property_read_bool(pdev->dev.of_node,
+ "qcom,needs-alt-core-clk");
+ if (needs_alt_core_clk) {
+ drvdata->aclk = devm_clk_get(&pdev->dev, "alt_core_clk");
+ if (IS_ERR(drvdata->aclk))
+ return PTR_ERR(drvdata->aclk);
+ }
+
+ if (clk_get_rate(drvdata->clk) == 0) {
+ ret = clk_round_rate(drvdata->clk, 1000);
+ clk_set_rate(drvdata->clk, ret);
+ }
+
+ if (drvdata->aclk && clk_get_rate(drvdata->aclk) == 0) {
+ ret = clk_round_rate(drvdata->aclk, 1000);
+ clk_set_rate(drvdata->aclk, ret);
+ }
+
+ ret = msm_iommu_parse_dt(pdev, drvdata);
+ if (ret)
+ return ret;
+
+ dev_info(&pdev->dev, "device %s mapped at %p, with %d ctx banks\n",
+ drvdata->name, drvdata->base, drvdata->ncb);
+
+ platform_set_drvdata(pdev, drvdata);
+
+ msm_iommu_sec_set_access_ops(&iommu_access_ops_v1);
+
+ pmon_info = msm_iommu_pm_alloc(&pdev->dev);
+ if (pmon_info != NULL) {
+ ret = msm_iommu_pmon_parse_dt(pdev, pmon_info);
+ if (ret) {
+ msm_iommu_pm_free(&pdev->dev);
+ pr_info("%s: pmon not available.\n", drvdata->name);
+ } else {
+ pmon_info->iommu.base = drvdata->base;
+ pmon_info->iommu.ops = &iommu_access_ops_v1;
+ pmon_info->iommu.hw_ops = iommu_pm_get_hw_ops_v1();
+ pmon_info->iommu.iommu_name = drvdata->name;
+ ret = msm_iommu_pm_iommu_register(pmon_info);
+ if (ret) {
+ pr_err("%s iommu register fail\n",
+ drvdata->name);
+ msm_iommu_pm_free(&pdev->dev);
+ } else {
+ pr_debug("%s iommu registered for pmon\n",
+ pmon_info->iommu.iommu_name);
+ }
+ }
+ }
+ return 0;
+}
+
+static int __devexit msm_iommu_remove(struct platform_device *pdev)
+{
+ struct msm_iommu_drvdata *drv = NULL;
+
+ msm_iommu_pm_iommu_unregister(&pdev->dev);
+ msm_iommu_pm_free(&pdev->dev);
+
+ drv = platform_get_drvdata(pdev);
+ if (drv) {
+ msm_iommu_remove_drv(drv);
+ if (drv->clk)
+ clk_put(drv->clk);
+ clk_put(drv->pclk);
+ platform_set_drvdata(pdev, NULL);
+ }
+ return 0;
+}
+
+static int msm_iommu_ctx_parse_dt(struct platform_device *pdev,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata)
+{
+ struct resource *r, rp;
+ int irq, ret;
+ u32 nsid;
+
+ ctx_drvdata->secure_context = of_property_read_bool(pdev->dev.of_node,
+ "qcom,secure-context");
+
+ if (!ctx_drvdata->secure_context) {
+ irq = platform_get_irq(pdev, 0);
+ if (irq > 0) {
+ ret = request_threaded_irq(irq, NULL,
+ msm_iommu_fault_handler_v2,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "msm_iommu_nonsecure_irq", pdev);
+ if (ret) {
+ pr_err("Request IRQ %d failed with ret=%d\n",
+ irq, ret);
+ return ret;
+ }
+ }
+ }
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!r)
+ return -EINVAL;
+
+ ret = of_address_to_resource(pdev->dev.parent->of_node, 0, &rp);
+ if (ret)
+ return -EINVAL;
+
+ /* Calculate the context bank number using the base addresses. The
+ * first 8 pages belong to the global address space which is followed
+ * by the context banks, hence subtract by 8 to get the context bank
+ * number.
+ */
+ ctx_drvdata->num = ((r->start - rp.start) >> CTX_SHIFT) - 8;
+
+ if (of_property_read_string(pdev->dev.of_node, "label",
+ &ctx_drvdata->name))
+ ctx_drvdata->name = dev_name(&pdev->dev);
+
+ if (!of_get_property(pdev->dev.of_node, "qcom,iommu-ctx-sids", &nsid))
+ return -EINVAL;
+
+ if (nsid >= sizeof(ctx_drvdata->sids))
+ return -EINVAL;
+
+ if (of_property_read_u32_array(pdev->dev.of_node, "qcom,iommu-ctx-sids",
+ ctx_drvdata->sids,
+ nsid / sizeof(*ctx_drvdata->sids))) {
+ return -EINVAL;
+ }
+ ctx_drvdata->nsid = nsid;
+
+ ctx_drvdata->asid = -1;
+ return 0;
+}
+
+static int __devinit msm_iommu_ctx_probe(struct platform_device *pdev)
+{
+ struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;
+ int ret;
+
+ if (!pdev->dev.parent)
+ return -EINVAL;
+
+ ctx_drvdata = devm_kzalloc(&pdev->dev, sizeof(*ctx_drvdata),
+ GFP_KERNEL);
+ if (!ctx_drvdata)
+ return -ENOMEM;
+
+ ctx_drvdata->pdev = pdev;
+ INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
+ platform_set_drvdata(pdev, ctx_drvdata);
+
+ ret = msm_iommu_ctx_parse_dt(pdev, ctx_drvdata);
+ if (!ret)
+ dev_info(&pdev->dev, "context %s using bank %d\n",
+ ctx_drvdata->name, ctx_drvdata->num);
+
+ return ret;
+}
+
+static int __devexit msm_iommu_ctx_remove(struct platform_device *pdev)
+{
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+}
+
+static struct of_device_id msm_iommu_match_table[] = {
+ { .compatible = "qcom,msm-smmu-v1", },
+ {}
+};
+
+static struct platform_driver msm_iommu_driver = {
+ .driver = {
+ .name = "msm_iommu_v1",
+ .of_match_table = msm_iommu_match_table,
+ },
+ .probe = msm_iommu_probe,
+ .remove = __devexit_p(msm_iommu_remove),
+};
+
+static struct of_device_id msm_iommu_ctx_match_table[] = {
+ { .name = "qcom,iommu-ctx", },
+ {}
+};
+
+static struct platform_driver msm_iommu_ctx_driver = {
+ .driver = {
+ .name = "msm_iommu_ctx_v1",
+ .of_match_table = msm_iommu_ctx_match_table,
+ },
+ .probe = msm_iommu_ctx_probe,
+ .remove = __devexit_p(msm_iommu_ctx_remove),
+};
+
+static int __init msm_iommu_driver_init(void)
+{
+ int ret;
+
+ ret = platform_driver_register(&msm_iommu_driver);
+ if (ret != 0) {
+ pr_err("Failed to register IOMMU driver\n");
+ goto error;
+ }
+
+ ret = platform_driver_register(&msm_iommu_ctx_driver);
+ if (ret != 0) {
+ pr_err("Failed to register IOMMU context driver\n");
+ goto error;
+ }
+
+error:
+ return ret;
+}
+
+static void __exit msm_iommu_driver_exit(void)
+{
+ platform_driver_unregister(&msm_iommu_ctx_driver);
+ platform_driver_unregister(&msm_iommu_driver);
+}
+
+subsys_initcall(msm_iommu_driver_init);
+module_exit(msm_iommu_driver_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/msm_iommu_dev-v2.c b/drivers/iommu/msm_iommu_dev-v2.c
deleted file mode 100644
index 4c69c8c..0000000
--- a/drivers/iommu/msm_iommu_dev-v2.c
+++ /dev/null
@@ -1,308 +0,0 @@
-/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/iommu.h>
-#include <linux/interrupt.h>
-#include <linux/err.h>
-#include <linux/slab.h>
-#include <linux/atomic.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/of_device.h>
-
-#include <mach/iommu_hw-v2.h>
-#include <mach/iommu.h>
-
-static int msm_iommu_parse_dt(struct platform_device *pdev,
- struct msm_iommu_drvdata *drvdata)
-{
- struct device_node *child;
- int ret = 0;
- u32 nsmr;
-
- ret = device_move(&pdev->dev, &msm_iommu_root_dev->dev, DPM_ORDER_NONE);
- if (ret)
- goto fail;
-
- ret = of_property_read_u32(pdev->dev.of_node, "qcom,iommu-smt-size",
- &nsmr);
- if (ret)
- goto fail;
-
- if (nsmr > MAX_NUM_SMR) {
- pr_err("Invalid SMT size: %d\n", nsmr);
- ret = -EINVAL;
- goto fail;
- }
-
- drvdata->nsmr = nsmr;
- for_each_child_of_node(pdev->dev.of_node, child) {
- drvdata->ncb++;
- if (!of_platform_device_create(child, NULL, &pdev->dev))
- pr_err("Failed to create %s device\n", child->name);
- }
-
- drvdata->name = dev_name(&pdev->dev);
-fail:
- return ret;
-}
-
-static atomic_t msm_iommu_next_id = ATOMIC_INIT(-1);
-
-static int __devinit msm_iommu_probe(struct platform_device *pdev)
-{
- struct msm_iommu_drvdata *drvdata;
- struct resource *r;
- int ret, needs_alt_core_clk;
-
- if (msm_iommu_root_dev == pdev)
- return 0;
-
- if (pdev->id == -1)
- pdev->id = atomic_inc_return(&msm_iommu_next_id) - 1;
-
- drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);
- if (!drvdata)
- return -ENOMEM;
-
- r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!r)
- return -EINVAL;
-
- drvdata->base = devm_ioremap(&pdev->dev, r->start, resource_size(r));
- if (!drvdata->base)
- return -ENOMEM;
-
- drvdata->gdsc = devm_regulator_get(&pdev->dev, "vdd");
- if (IS_ERR(drvdata->gdsc))
- return -EINVAL;
-
- drvdata->pclk = devm_clk_get(&pdev->dev, "iface_clk");
- if (IS_ERR(drvdata->pclk))
- return PTR_ERR(drvdata->pclk);
-
- drvdata->clk = devm_clk_get(&pdev->dev, "core_clk");
- if (IS_ERR(drvdata->clk))
- return PTR_ERR(drvdata->clk);
-
- needs_alt_core_clk = of_property_read_bool(pdev->dev.of_node,
- "qcom,needs-alt-core-clk");
- if (needs_alt_core_clk) {
- drvdata->aclk = devm_clk_get(&pdev->dev, "alt_core_clk");
- if (IS_ERR(drvdata->aclk))
- return PTR_ERR(drvdata->aclk);
- }
-
- if (clk_get_rate(drvdata->clk) == 0) {
- ret = clk_round_rate(drvdata->clk, 1);
- clk_set_rate(drvdata->clk, ret);
- }
-
- if (drvdata->aclk && clk_get_rate(drvdata->aclk) == 0) {
- ret = clk_round_rate(drvdata->aclk, 1);
- clk_set_rate(drvdata->aclk, ret);
- }
-
- ret = msm_iommu_parse_dt(pdev, drvdata);
- if (ret)
- return ret;
-
- pr_info("device %s mapped at %p, with %d ctx banks\n",
- drvdata->name, drvdata->base, drvdata->ncb);
-
- platform_set_drvdata(pdev, drvdata);
-
- return 0;
-}
-
-static int __devexit msm_iommu_remove(struct platform_device *pdev)
-{
- struct msm_iommu_drvdata *drv = NULL;
-
- drv = platform_get_drvdata(pdev);
- if (drv) {
- if (drv->clk)
- clk_put(drv->clk);
- clk_put(drv->pclk);
- platform_set_drvdata(pdev, NULL);
- }
- return 0;
-}
-
-static int msm_iommu_ctx_parse_dt(struct platform_device *pdev,
- struct msm_iommu_ctx_drvdata *ctx_drvdata)
-{
- struct resource *r, rp;
- int irq, ret;
- u32 nsid;
-
- irq = platform_get_irq(pdev, 0);
- if (irq > 0) {
- ret = request_threaded_irq(irq, NULL,
- msm_iommu_fault_handler_v2,
- IRQF_ONESHOT | IRQF_SHARED,
- "msm_iommu_nonsecure_irq", pdev);
- if (ret) {
- pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
- return ret;
- }
- }
-
- r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!r)
- return -EINVAL;
-
- ret = of_address_to_resource(pdev->dev.parent->of_node, 0, &rp);
- if (ret)
- return -EINVAL;
-
- /* Calculate the context bank number using the base addresses. The
- * first 8 pages belong to the global address space which is followed
- * by the context banks, hence subtract by 8 to get the context bank
- * number.
- */
- ctx_drvdata->num = ((r->start - rp.start) >> CTX_SHIFT) - 8;
-
- if (of_property_read_string(pdev->dev.of_node, "label",
- &ctx_drvdata->name))
- ctx_drvdata->name = dev_name(&pdev->dev);
-
- if (!of_get_property(pdev->dev.of_node, "qcom,iommu-ctx-sids", &nsid))
- return -EINVAL;
-
- if (nsid >= sizeof(ctx_drvdata->sids))
- return -EINVAL;
-
- if (of_property_read_u32_array(pdev->dev.of_node, "qcom,iommu-ctx-sids",
- ctx_drvdata->sids,
- nsid / sizeof(*ctx_drvdata->sids))) {
- return -EINVAL;
- }
- ctx_drvdata->nsid = nsid;
-
- return 0;
-}
-
-static int __devinit msm_iommu_ctx_probe(struct platform_device *pdev)
-{
- struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;
- int ret;
-
- if (!pdev->dev.parent)
- return -EINVAL;
-
- ctx_drvdata = devm_kzalloc(&pdev->dev, sizeof(*ctx_drvdata),
- GFP_KERNEL);
- if (!ctx_drvdata)
- return -ENOMEM;
-
- ctx_drvdata->pdev = pdev;
- INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
- platform_set_drvdata(pdev, ctx_drvdata);
-
- ret = msm_iommu_ctx_parse_dt(pdev, ctx_drvdata);
- if (!ret)
- dev_info(&pdev->dev, "context %s using bank %d\n",
- ctx_drvdata->name, ctx_drvdata->num);
-
- return ret;
-}
-
-static int __devexit msm_iommu_ctx_remove(struct platform_device *pdev)
-{
- platform_set_drvdata(pdev, NULL);
- return 0;
-}
-
-static struct of_device_id msm_iommu_match_table[] = {
- { .compatible = "qcom,msm-smmu-v2", },
- {}
-};
-
-static struct platform_driver msm_iommu_driver = {
- .driver = {
- .name = "msm_iommu_v2",
- .of_match_table = msm_iommu_match_table,
- },
- .probe = msm_iommu_probe,
- .remove = __devexit_p(msm_iommu_remove),
-};
-
-static struct of_device_id msm_iommu_ctx_match_table[] = {
- { .name = "qcom,iommu-ctx", },
- {}
-};
-
-static struct platform_driver msm_iommu_ctx_driver = {
- .driver = {
- .name = "msm_iommu_ctx_v2",
- .of_match_table = msm_iommu_ctx_match_table,
- },
- .probe = msm_iommu_ctx_probe,
- .remove = __devexit_p(msm_iommu_ctx_remove),
-};
-
-static int __init msm_iommu_driver_init(void)
-{
- struct device_node *node;
- int ret;
-
- node = of_find_compatible_node(NULL, NULL, "qcom,msm-smmu-v2");
- if (!node)
- return -ENODEV;
-
- of_node_put(node);
-
- msm_iommu_root_dev = platform_device_register_simple(
- "msm_iommu", -1, 0, 0);
- if (!msm_iommu_root_dev) {
- pr_err("Failed to create root IOMMU device\n");
- ret = -ENODEV;
- goto error;
- }
-
- atomic_inc(&msm_iommu_next_id);
-
- ret = platform_driver_register(&msm_iommu_driver);
- if (ret != 0) {
- pr_err("Failed to register IOMMU driver\n");
- goto error;
- }
-
- ret = platform_driver_register(&msm_iommu_ctx_driver);
- if (ret != 0) {
- pr_err("Failed to register IOMMU context driver\n");
- goto error;
- }
-
-error:
- return ret;
-}
-
-static void __exit msm_iommu_driver_exit(void)
-{
- platform_driver_unregister(&msm_iommu_ctx_driver);
- platform_driver_unregister(&msm_iommu_driver);
- platform_device_unregister(msm_iommu_root_dev);
-}
-
-subsys_initcall(msm_iommu_driver_init);
-module_exit(msm_iommu_driver_exit);
-
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
deleted file mode 100644
index fca102a..0000000
--- a/drivers/iommu/msm_iommu_dev.c
+++ /dev/null
@@ -1,431 +0,0 @@
-/* Copyright (c) 2010-2012, The Linux Foundation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/iommu.h>
-#include <linux/interrupt.h>
-#include <linux/err.h>
-#include <linux/slab.h>
-
-#include <mach/iommu_hw-8xxx.h>
-#include <mach/iommu.h>
-
-struct iommu_ctx_iter_data {
- /* input */
- const char *name;
-
- /* output */
- struct device *dev;
-};
-
-struct platform_device *msm_iommu_root_dev;
-
-static int each_iommu_ctx(struct device *dev, void *data)
-{
- struct iommu_ctx_iter_data *res = data;
- struct msm_iommu_ctx_drvdata *c;
-
- c = dev_get_drvdata(dev);
- if (!res || !c || !c->name || !res->name)
- return -EINVAL;
-
- if (!strcmp(res->name, c->name)) {
- res->dev = dev;
- return 1;
- }
- return 0;
-}
-
-static int each_iommu(struct device *dev, void *data)
-{
- return device_for_each_child(dev, data, each_iommu_ctx);
-}
-
-struct device *msm_iommu_get_ctx(const char *ctx_name)
-{
- struct iommu_ctx_iter_data r;
- int found;
-
- if (!msm_iommu_root_dev) {
- pr_err("No root IOMMU device.\n");
- goto fail;
- }
-
- r.name = ctx_name;
- found = device_for_each_child(&msm_iommu_root_dev->dev, &r, each_iommu);
-
- if (found <= 0 || !dev_get_drvdata(r.dev)) {
- pr_err("Could not find context <%s>\n", ctx_name);
- goto fail;
- }
-
- return r.dev;
-fail:
- return NULL;
-}
-EXPORT_SYMBOL(msm_iommu_get_ctx);
-
-static void msm_iommu_reset(void __iomem *base, int ncb)
-{
- int ctx;
-
- SET_RPUE(base, 0);
- SET_RPUEIE(base, 0);
- SET_ESRRESTORE(base, 0);
- SET_TBE(base, 0);
- SET_CR(base, 0);
- SET_SPDMBE(base, 0);
- SET_TESTBUSCR(base, 0);
- SET_TLBRSW(base, 0);
- SET_GLOBAL_TLBIALL(base, 0);
- SET_RPU_ACR(base, 0);
- SET_TLBLKCRWE(base, 1);
-
- for (ctx = 0; ctx < ncb; ctx++) {
- SET_BPRCOSH(base, ctx, 0);
- SET_BPRCISH(base, ctx, 0);
- SET_BPRCNSH(base, ctx, 0);
- SET_BPSHCFG(base, ctx, 0);
- SET_BPMTCFG(base, ctx, 0);
- SET_ACTLR(base, ctx, 0);
- SET_SCTLR(base, ctx, 0);
- SET_FSRRESTORE(base, ctx, 0);
- SET_TTBR0(base, ctx, 0);
- SET_TTBR1(base, ctx, 0);
- SET_TTBCR(base, ctx, 0);
- SET_BFBCR(base, ctx, 0);
- SET_PAR(base, ctx, 0);
- SET_FAR(base, ctx, 0);
- SET_TLBFLPTER(base, ctx, 0);
- SET_TLBSLPTER(base, ctx, 0);
- SET_TLBLKCR(base, ctx, 0);
- SET_CTX_TLBIALL(base, ctx, 0);
- SET_TLBIVA(base, ctx, 0);
- SET_PRRR(base, ctx, 0);
- SET_NMRR(base, ctx, 0);
- SET_CONTEXTIDR(base, ctx, 0);
- }
- mb();
-}
-
-static int msm_iommu_probe(struct platform_device *pdev)
-{
- struct resource *r, *r2;
- struct clk *iommu_clk = NULL;
- struct clk *iommu_pclk = NULL;
- struct msm_iommu_drvdata *drvdata;
- struct msm_iommu_dev *iommu_dev = pdev->dev.platform_data;
- void __iomem *regs_base;
- resource_size_t len;
- int ret, par;
-
- if (pdev->id == -1) {
- msm_iommu_root_dev = pdev;
- return 0;
- }
-
- drvdata = kzalloc(sizeof(*drvdata), GFP_KERNEL);
-
- if (!drvdata) {
- ret = -ENOMEM;
- goto fail;
- }
-
- if (!iommu_dev) {
- ret = -ENODEV;
- goto fail;
- }
-
- iommu_pclk = clk_get_sys("msm_iommu", "iface_clk");
- if (IS_ERR(iommu_pclk)) {
- ret = -ENODEV;
- goto fail;
- }
-
- ret = clk_prepare_enable(iommu_pclk);
- if (ret)
- goto fail_enable;
-
- iommu_clk = clk_get(&pdev->dev, "core_clk");
-
- if (!IS_ERR(iommu_clk)) {
- if (clk_get_rate(iommu_clk) == 0) {
- ret = clk_round_rate(iommu_clk, 1);
- clk_set_rate(iommu_clk, ret);
- }
-
- ret = clk_prepare_enable(iommu_clk);
- if (ret) {
- clk_put(iommu_clk);
- goto fail_pclk;
- }
- } else
- iommu_clk = NULL;
-
- r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase");
-
- if (!r) {
- ret = -ENODEV;
- goto fail_clk;
- }
-
- len = resource_size(r);
-
- r2 = request_mem_region(r->start, len, r->name);
- if (!r2) {
- pr_err("Could not request memory region: start=%p, len=%d\n",
- (void *) r->start, len);
- ret = -EBUSY;
- goto fail_clk;
- }
-
- regs_base = ioremap(r2->start, len);
-
- if (!regs_base) {
- pr_err("Could not ioremap: start=%p, len=%d\n",
- (void *) r2->start, len);
- ret = -EBUSY;
- goto fail_mem;
- }
-
- msm_iommu_reset(regs_base, iommu_dev->ncb);
-
- SET_M(regs_base, 0, 1);
- SET_PAR(regs_base, 0, 0);
- SET_V2PCFG(regs_base, 0, 1);
- SET_V2PPR(regs_base, 0, 0);
- mb();
- par = GET_PAR(regs_base, 0);
- SET_V2PCFG(regs_base, 0, 0);
- SET_M(regs_base, 0, 0);
- mb();
-
- if (!par) {
- pr_err("%s: Invalid PAR value detected\n", iommu_dev->name);
- ret = -ENODEV;
- goto fail_io;
- }
-
- drvdata->pclk = iommu_pclk;
- drvdata->clk = iommu_clk;
- drvdata->base = regs_base;
- drvdata->ncb = iommu_dev->ncb;
- drvdata->ttbr_split = iommu_dev->ttbr_split;
- drvdata->name = iommu_dev->name;
-
- pr_info("device %s mapped at %p, with %d ctx banks\n",
- iommu_dev->name, regs_base, iommu_dev->ncb);
-
- platform_set_drvdata(pdev, drvdata);
-
- if (iommu_clk)
- clk_disable_unprepare(iommu_clk);
-
- clk_disable_unprepare(iommu_pclk);
-
- return 0;
-fail_io:
- iounmap(regs_base);
-fail_mem:
- release_mem_region(r->start, len);
-fail_clk:
- if (iommu_clk) {
- clk_disable_unprepare(iommu_clk);
- clk_put(iommu_clk);
- }
-fail_pclk:
- clk_disable_unprepare(iommu_pclk);
-fail_enable:
- clk_put(iommu_pclk);
-fail:
- kfree(drvdata);
- return ret;
-}
-
-static int msm_iommu_remove(struct platform_device *pdev)
-{
- struct msm_iommu_drvdata *drv = NULL;
-
- drv = platform_get_drvdata(pdev);
- if (drv) {
- if (drv->clk)
- clk_put(drv->clk);
- clk_put(drv->pclk);
- memset(drv, 0, sizeof(*drv));
- kfree(drv);
- platform_set_drvdata(pdev, NULL);
- }
- return 0;
-}
-
-static int msm_iommu_ctx_probe(struct platform_device *pdev)
-{
- struct msm_iommu_ctx_dev *c = pdev->dev.platform_data;
- struct msm_iommu_drvdata *drvdata;
- struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;
- int i, ret, irq;
- if (!c || !pdev->dev.parent) {
- ret = -EINVAL;
- goto fail;
- }
-
- drvdata = dev_get_drvdata(pdev->dev.parent);
-
- if (!drvdata) {
- ret = -ENODEV;
- goto fail;
- }
-
- ctx_drvdata = kzalloc(sizeof(*ctx_drvdata), GFP_KERNEL);
- if (!ctx_drvdata) {
- ret = -ENOMEM;
- goto fail;
- }
- ctx_drvdata->num = c->num;
- ctx_drvdata->pdev = pdev;
- ctx_drvdata->name = c->name;
-
- irq = platform_get_irq_byname(to_platform_device(pdev->dev.parent),
- "nonsecure_irq");
- if (irq < 0) {
- ret = -ENODEV;
- goto fail;
- }
-
- ret = request_threaded_irq(irq, NULL, msm_iommu_fault_handler,
- IRQF_ONESHOT | IRQF_SHARED,
- "msm_iommu_nonsecure_irq", ctx_drvdata);
-
- if (ret) {
- pr_err("request_threaded_irq %d failed: %d\n", irq, ret);
- goto fail;
- }
-
- INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
- platform_set_drvdata(pdev, ctx_drvdata);
-
- ret = clk_prepare_enable(drvdata->pclk);
- if (ret)
- goto fail;
-
- if (drvdata->clk) {
- ret = clk_prepare_enable(drvdata->clk);
- if (ret) {
- clk_disable_unprepare(drvdata->pclk);
- goto fail;
- }
- }
-
- /* Program the M2V tables for this context */
- for (i = 0; i < MAX_NUM_MIDS; i++) {
- int mid = c->mids[i];
- if (mid == -1)
- break;
-
- SET_M2VCBR_N(drvdata->base, mid, 0);
- SET_CBACR_N(drvdata->base, c->num, 0);
-
- /* Route page faults to the non-secure interrupt */
- SET_IRPTNDX(drvdata->base, c->num, 1);
-
- /* Set VMID = 0 */
- SET_VMID(drvdata->base, mid, 0);
-
- /* Set the context number for that MID to this context */
- SET_CBNDX(drvdata->base, mid, c->num);
-
- /* Set MID associated with this context bank to 0 */
- SET_CBVMID(drvdata->base, c->num, 0);
-
- /* Set the ASID for TLB tagging for this context to 0 */
- SET_CONTEXTIDR_ASID(drvdata->base, c->num, 0);
-
- /* Set security bit override to be Non-secure */
- SET_NSCFG(drvdata->base, mid, 3);
- }
- mb();
-
- if (drvdata->clk)
- clk_disable_unprepare(drvdata->clk);
- clk_disable_unprepare(drvdata->pclk);
-
- dev_info(&pdev->dev, "context %s using bank %d\n", c->name, c->num);
- return 0;
-fail:
- kfree(ctx_drvdata);
- return ret;
-}
-
-static int msm_iommu_ctx_remove(struct platform_device *pdev)
-{
- struct msm_iommu_ctx_drvdata *drv = NULL;
- drv = platform_get_drvdata(pdev);
- if (drv) {
- memset(drv, 0, sizeof(struct msm_iommu_ctx_drvdata));
- kfree(drv);
- platform_set_drvdata(pdev, NULL);
- }
- return 0;
-}
-
-static struct platform_driver msm_iommu_driver = {
- .driver = {
- .name = "msm_iommu",
- },
- .probe = msm_iommu_probe,
- .remove = msm_iommu_remove,
-};
-
-static struct platform_driver msm_iommu_ctx_driver = {
- .driver = {
- .name = "msm_iommu_ctx",
- },
- .probe = msm_iommu_ctx_probe,
- .remove = msm_iommu_ctx_remove,
-};
-
-static int __init msm_iommu_driver_init(void)
-{
- int ret;
- ret = platform_driver_register(&msm_iommu_driver);
- if (ret != 0) {
- pr_err("Failed to register IOMMU driver\n");
- goto error;
- }
-
- ret = platform_driver_register(&msm_iommu_ctx_driver);
- if (ret != 0) {
- pr_err("Failed to register IOMMU context driver\n");
- goto error;
- }
-
-error:
- return ret;
-}
-
-static void __exit msm_iommu_driver_exit(void)
-{
- platform_driver_unregister(&msm_iommu_ctx_driver);
- platform_driver_unregister(&msm_iommu_driver);
-}
-
-subsys_initcall(msm_iommu_driver_init);
-module_exit(msm_iommu_driver_exit);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/msm_iommu_pagetable.c b/drivers/iommu/msm_iommu_pagetable.c
index 2ee9ba6..b62bb76 100644
--- a/drivers/iommu/msm_iommu_pagetable.c
+++ b/drivers/iommu/msm_iommu_pagetable.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -20,6 +20,7 @@
#include <asm/cacheflush.h>
#include <mach/iommu.h>
+#include <mach/msm_iommu_priv.h>
#include "msm_iommu_pagetable.h"
/* Sharability attributes of MSM IOMMU mappings */
@@ -41,7 +42,7 @@
dmac_flush_range(start, end);
}
-int msm_iommu_pagetable_alloc(struct iommu_pt *pt)
+int msm_iommu_pagetable_alloc(struct msm_iommu_pt *pt)
{
pt->fl_table = (unsigned long *)__get_free_pages(GFP_KERNEL,
get_order(SZ_16K));
@@ -54,7 +55,7 @@
return 0;
}
-void msm_iommu_pagetable_free(struct iommu_pt *pt)
+void msm_iommu_pagetable_free(struct msm_iommu_pt *pt)
{
unsigned long *fl_table;
int i;
@@ -110,7 +111,91 @@
return pgprot;
}
-int msm_iommu_pagetable_map(struct iommu_pt *pt, unsigned long va,
+static unsigned long *make_second_level(struct msm_iommu_pt *pt,
+ unsigned long *fl_pte)
+{
+ unsigned long *sl;
+ sl = (unsigned long *) __get_free_pages(GFP_KERNEL,
+ get_order(SZ_4K));
+
+ if (!sl) {
+ pr_debug("Could not allocate second level table\n");
+ goto fail;
+ }
+ memset(sl, 0, SZ_4K);
+ clean_pte(sl, sl + NUM_SL_PTE, pt->redirect);
+
+ *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | \
+ FL_TYPE_TABLE);
+
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+fail:
+ return sl;
+}
+
+static int sl_4k(unsigned long *sl_pte, phys_addr_t pa, unsigned int pgprot)
+{
+ int ret = 0;
+
+ if (*sl_pte) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_NG | SL_SHARED
+ | SL_TYPE_SMALL | pgprot;
+fail:
+ return ret;
+}
+
+static int sl_64k(unsigned long *sl_pte, phys_addr_t pa, unsigned int pgprot)
+{
+ int ret = 0;
+
+ int i;
+
+ for (i = 0; i < 16; i++)
+ if (*(sl_pte+i)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ for (i = 0; i < 16; i++)
+ *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_NG
+ | SL_SHARED | SL_TYPE_LARGE | pgprot;
+
+fail:
+ return ret;
+}
+
+static inline int fl_1m(unsigned long *fl_pte, phys_addr_t pa, int pgprot)
+{
+ if (*fl_pte)
+ return -EBUSY;
+
+ *fl_pte = (pa & 0xFFF00000) | FL_NG | FL_TYPE_SECT | FL_SHARED
+ | pgprot;
+
+ return 0;
+}
+
+static inline int fl_16m(unsigned long *fl_pte, phys_addr_t pa, int pgprot)
+{
+ int i;
+ int ret = 0;
+ for (i = 0; i < 16; i++)
+ if (*(fl_pte+i)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+ for (i = 0; i < 16; i++)
+ *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION
+ | FL_TYPE_SECT | FL_SHARED | FL_NG | pgprot;
+fail:
+ return ret;
+}
+
+int msm_iommu_pagetable_map(struct msm_iommu_pt *pt, unsigned long va,
phys_addr_t pa, size_t len, int prot)
{
unsigned long *fl_pte;
@@ -144,28 +229,16 @@
fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
if (len == SZ_16M) {
- int i = 0;
-
- for (i = 0; i < 16; i++)
- if (*(fl_pte+i)) {
- ret = -EBUSY;
- goto fail;
- }
-
- for (i = 0; i < 16; i++)
- *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION |
- FL_TYPE_SECT | FL_SHARED | FL_NG | pgprot;
+ ret = fl_16m(fl_pte, pa, pgprot);
+ if (ret)
+ goto fail;
clean_pte(fl_pte, fl_pte + 16, pt->redirect);
}
if (len == SZ_1M) {
- if (*fl_pte) {
- ret = -EBUSY;
+ ret = fl_1m(fl_pte, pa, pgprot);
+ if (ret)
goto fail;
- }
-
- *fl_pte = (pa & 0xFFF00000) | FL_NG | FL_TYPE_SECT
- | FL_SHARED | pgprot;
clean_pte(fl_pte, fl_pte + 1, pt->redirect);
}
@@ -173,21 +246,10 @@
if (len == SZ_4K || len == SZ_64K) {
if (*fl_pte == 0) {
- unsigned long *sl;
- sl = (unsigned long *) __get_free_pages(GFP_KERNEL,
- get_order(SZ_4K));
-
- if (!sl) {
- pr_debug("Could not allocate second level table\n");
+ if (make_second_level(pt, fl_pte) == NULL) {
ret = -ENOMEM;
goto fail;
}
- memset(sl, 0, SZ_4K);
- clean_pte(sl, sl + NUM_SL_PTE, pt->redirect);
-
- *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | \
- FL_TYPE_TABLE);
- clean_pte(fl_pte, fl_pte + 1, pt->redirect);
}
if (!(*fl_pte & FL_TYPE_TABLE)) {
@@ -201,29 +263,16 @@
sl_pte = sl_table + sl_offset;
if (len == SZ_4K) {
- if (*sl_pte) {
- ret = -EBUSY;
+ ret = sl_4k(sl_pte, pa, pgprot);
+ if (ret)
goto fail;
- }
-
- *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_NG | SL_SHARED
- | SL_TYPE_SMALL | pgprot;
clean_pte(sl_pte, sl_pte + 1, pt->redirect);
}
if (len == SZ_64K) {
- int i;
-
- for (i = 0; i < 16; i++)
- if (*(sl_pte+i)) {
- ret = -EBUSY;
- goto fail;
- }
-
- for (i = 0; i < 16; i++)
- *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_NG
- | SL_SHARED | SL_TYPE_LARGE | pgprot;
-
+ ret = sl_64k(sl_pte, pa, pgprot);
+ if (ret)
+ goto fail;
clean_pte(sl_pte, sl_pte + 16, pt->redirect);
}
@@ -231,7 +280,7 @@
return ret;
}
-size_t msm_iommu_pagetable_unmap(struct iommu_pt *pt, unsigned long va,
+size_t msm_iommu_pagetable_unmap(struct msm_iommu_pt *pt, unsigned long va,
size_t len)
{
unsigned long *fl_pte;
@@ -309,77 +358,165 @@
return ret;
}
-static unsigned int get_phys_addr(struct scatterlist *sg)
+static phys_addr_t get_phys_addr(struct scatterlist *sg)
{
/*
* Try sg_dma_address first so that we can
* map carveout regions that do not have a
* struct page associated with them.
*/
- unsigned int pa = sg_dma_address(sg);
+ phys_addr_t pa = sg_dma_address(sg);
if (pa == 0)
pa = sg_phys(sg);
return pa;
}
-int msm_iommu_pagetable_map_range(struct iommu_pt *pt, unsigned int va,
- struct scatterlist *sg, unsigned int len, int prot)
+static int check_range(unsigned long *fl_table, unsigned int va,
+ unsigned int len)
{
- unsigned int pa;
unsigned int offset = 0;
- unsigned int pgprot;
unsigned long *fl_pte;
unsigned long fl_offset;
unsigned long *sl_table;
+ unsigned long sl_start, sl_end;
+ int i;
+
+ fl_offset = FL_OFFSET(va); /* Upper 12 bits */
+ fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
+
+ while (offset < len) {
+ if (*fl_pte & FL_TYPE_TABLE) {
+ sl_start = SL_OFFSET(va);
+ sl_table = __va(((*fl_pte) & FL_BASE_MASK));
+ sl_end = ((len - offset) / SZ_4K) + sl_start;
+
+ if (sl_end > NUM_SL_PTE)
+ sl_end = NUM_SL_PTE;
+
+ for (i = sl_start; i < sl_end; i++) {
+ if (sl_table[i] != 0) {
+ pr_err("%08x - %08x already mapped\n",
+ va, va + SZ_4K);
+ return -EBUSY;
+ }
+ offset += SZ_4K;
+ va += SZ_4K;
+ }
+
+
+ sl_start = 0;
+ } else {
+ if (*fl_pte != 0) {
+ pr_err("%08x - %08x already mapped\n",
+ va, va + SZ_1M);
+ return -EBUSY;
+ }
+ va += SZ_1M;
+ offset += SZ_1M;
+ sl_start = 0;
+ }
+ fl_pte++;
+ }
+ return 0;
+}
+
+static inline int is_fully_aligned(unsigned int va, phys_addr_t pa, size_t len,
+ int align)
+{
+ return IS_ALIGNED(va, align) && IS_ALIGNED(pa, align)
+ && (len >= align);
+}
+
+int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
+ struct scatterlist *sg, unsigned int len, int prot)
+{
+ phys_addr_t pa;
+ unsigned int offset = 0;
+ unsigned long *fl_pte;
+ unsigned long fl_offset;
+ unsigned long *sl_table = NULL;
unsigned long sl_offset, sl_start;
- unsigned int chunk_offset = 0;
- unsigned int chunk_pa;
+ unsigned int chunk_size, chunk_offset = 0;
int ret = 0;
+ unsigned int pgprot4k, pgprot64k, pgprot1m, pgprot16m;
BUG_ON(len & (SZ_4K - 1));
- pgprot = __get_pgprot(prot, SZ_4K);
- if (!pgprot) {
+ pgprot4k = __get_pgprot(prot, SZ_4K);
+ pgprot64k = __get_pgprot(prot, SZ_64K);
+ pgprot1m = __get_pgprot(prot, SZ_1M);
+ pgprot16m = __get_pgprot(prot, SZ_16M);
+ if (!pgprot4k || !pgprot64k || !pgprot1m || !pgprot16m) {
ret = -EINVAL;
goto fail;
}
fl_offset = FL_OFFSET(va); /* Upper 12 bits */
fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
+ pa = get_phys_addr(sg);
- sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
- sl_offset = SL_OFFSET(va);
-
- chunk_pa = get_phys_addr(sg);
- if (chunk_pa == 0) {
- pr_debug("No dma address for sg %p\n", sg);
- ret = -EINVAL;
+ ret = check_range(pt->fl_table, va, len);
+ if (ret)
goto fail;
- }
while (offset < len) {
- /* Set up a 2nd level page table if one doesn't exist */
- if (*fl_pte == 0) {
- sl_table = (unsigned long *)
- __get_free_pages(GFP_KERNEL, get_order(SZ_4K));
+ chunk_size = SZ_4K;
- if (!sl_table) {
- pr_debug("Could not allocate second level table\n");
+ if (is_fully_aligned(va, pa, sg->length - chunk_offset,
+ SZ_16M))
+ chunk_size = SZ_16M;
+ else if (is_fully_aligned(va, pa, sg->length - chunk_offset,
+ SZ_1M))
+ chunk_size = SZ_1M;
+ /* 64k or 4k determined later */
+
+ /* for 1M and 16M, only first level entries are required */
+ if (chunk_size >= SZ_1M) {
+ if (chunk_size == SZ_16M) {
+ ret = fl_16m(fl_pte, pa, pgprot16m);
+ if (ret)
+ goto fail;
+ clean_pte(fl_pte, fl_pte + 16, pt->redirect);
+ fl_pte += 16;
+ } else if (chunk_size == SZ_1M) {
+ ret = fl_1m(fl_pte, pa, pgprot1m);
+ if (ret)
+ goto fail;
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ fl_pte++;
+ }
+
+ offset += chunk_size;
+ chunk_offset += chunk_size;
+ va += chunk_size;
+ pa += chunk_size;
+
+ if (chunk_offset >= sg->length && offset < len) {
+ chunk_offset = 0;
+ sg = sg_next(sg);
+ pa = get_phys_addr(sg);
+ if (pa == 0) {
+ pr_debug("No dma address for sg %p\n",
+ sg);
+ ret = -EINVAL;
+ goto fail;
+ }
+ }
+ continue;
+ }
+ /* for 4K or 64K, make sure there is a second level table */
+ if (*fl_pte == 0) {
+ if (!make_second_level(pt, fl_pte)) {
ret = -ENOMEM;
goto fail;
}
-
- memset(sl_table, 0, SZ_4K);
- clean_pte(sl_table, sl_table + NUM_SL_PTE,
- pt->redirect);
-
- *fl_pte = ((((int)__pa(sl_table)) & FL_BASE_MASK) |
- FL_TYPE_TABLE);
- clean_pte(fl_pte, fl_pte + 1, pt->redirect);
- } else
- sl_table = (unsigned long *)
- __va(((*fl_pte) & FL_BASE_MASK));
-
+ }
+ if (!(*fl_pte & FL_TYPE_TABLE)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+ sl_table = __va(((*fl_pte) & FL_BASE_MASK));
+ sl_offset = SL_OFFSET(va);
/* Keep track of initial position so we
* don't clean more than we have to
*/
@@ -387,21 +524,38 @@
/* Build the 2nd level page table */
while (offset < len && sl_offset < NUM_SL_PTE) {
- pa = chunk_pa + chunk_offset;
- sl_table[sl_offset] = (pa & SL_BASE_MASK_SMALL) |
- pgprot | SL_NG | SL_SHARED | SL_TYPE_SMALL;
- sl_offset++;
- offset += SZ_4K;
+ /* Map a large 64K page if the chunk is large enough and
+ * the pa and va are aligned
+ */
- chunk_offset += SZ_4K;
+ if (is_fully_aligned(va, pa, sg->length - chunk_offset,
+ SZ_64K))
+ chunk_size = SZ_64K;
+ else
+ chunk_size = SZ_4K;
+
+ if (chunk_size == SZ_4K) {
+ sl_4k(&sl_table[sl_offset], pa, pgprot4k);
+ sl_offset++;
+ } else {
+ BUG_ON(sl_offset + 16 > NUM_SL_PTE);
+ sl_64k(&sl_table[sl_offset], pa, pgprot64k);
+ sl_offset += 16;
+ }
+
+
+ offset += chunk_size;
+ chunk_offset += chunk_size;
+ va += chunk_size;
+ pa += chunk_size;
if (chunk_offset >= sg->length && offset < len) {
chunk_offset = 0;
sg = sg_next(sg);
- chunk_pa = get_phys_addr(sg);
- if (chunk_pa == 0) {
+ pa = get_phys_addr(sg);
+ if (pa == 0) {
pr_debug("No dma address for sg %p\n",
- sg);
+ sg);
ret = -EINVAL;
goto fail;
}
@@ -418,7 +572,7 @@
return ret;
}
-void msm_iommu_pagetable_unmap_range(struct iommu_pt *pt, unsigned int va,
+void msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, unsigned int va,
unsigned int len)
{
unsigned int offset = 0;
@@ -433,44 +587,53 @@
fl_offset = FL_OFFSET(va); /* Upper 12 bits */
fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
- sl_start = SL_OFFSET(va);
-
while (offset < len) {
- sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
- sl_end = ((len - offset) / SZ_4K) + sl_start;
+ if (*fl_pte & FL_TYPE_TABLE) {
+ sl_start = SL_OFFSET(va);
+ sl_table = __va(((*fl_pte) & FL_BASE_MASK));
+ sl_end = ((len - offset) / SZ_4K) + sl_start;
- if (sl_end > NUM_SL_PTE)
- sl_end = NUM_SL_PTE;
+ if (sl_end > NUM_SL_PTE)
+ sl_end = NUM_SL_PTE;
- memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
- clean_pte(sl_table + sl_start, sl_table + sl_end,
- pt->redirect);
+ memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
+ clean_pte(sl_table + sl_start, sl_table + sl_end,
+ pt->redirect);
- offset += (sl_end - sl_start) * SZ_4K;
+ offset += (sl_end - sl_start) * SZ_4K;
+ va += (sl_end - sl_start) * SZ_4K;
- /* Unmap and free the 2nd level table if all mappings in it
- * were removed. This saves memory, but the table will need
- * to be re-allocated the next time someone tries to map these
- * VAs.
- */
- used = 0;
+ /* Unmap and free the 2nd level table if all mappings
+ * in it were removed. This saves memory, but the table
+ * will need to be re-allocated the next time someone
+ * tries to map these VAs.
+ */
+ used = 0;
- /* If we just unmapped the whole table, don't bother
- * seeing if there are still used entries left.
- */
- if (sl_end - sl_start != NUM_SL_PTE)
- for (i = 0; i < NUM_SL_PTE; i++)
- if (sl_table[i]) {
- used = 1;
- break;
- }
- if (!used) {
- free_page((unsigned long)sl_table);
+ /* If we just unmapped the whole table, don't bother
+ * seeing if there are still used entries left.
+ */
+ if (sl_end - sl_start != NUM_SL_PTE)
+ for (i = 0; i < NUM_SL_PTE; i++)
+ if (sl_table[i]) {
+ used = 1;
+ break;
+ }
+ if (!used) {
+ free_page((unsigned long)sl_table);
+ *fl_pte = 0;
+
+ clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ }
+
+ sl_start = 0;
+ } else {
*fl_pte = 0;
clean_pte(fl_pte, fl_pte + 1, pt->redirect);
+ va += SZ_1M;
+ offset += SZ_1M;
+ sl_start = 0;
}
-
- sl_start = 0;
fl_pte++;
}
}
diff --git a/drivers/iommu/msm_iommu_pagetable.h b/drivers/iommu/msm_iommu_pagetable.h
index 3266681..7513aa5 100644
--- a/drivers/iommu/msm_iommu_pagetable.h
+++ b/drivers/iommu/msm_iommu_pagetable.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012 The Linux Foundation. All rights reserved.
+/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -71,20 +71,17 @@
#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
-struct iommu_pt {
- unsigned long *fl_table;
- int redirect;
-};
+struct iommu_pt;
void msm_iommu_pagetable_init(void);
-int msm_iommu_pagetable_alloc(struct iommu_pt *pt);
-void msm_iommu_pagetable_free(struct iommu_pt *pt);
-int msm_iommu_pagetable_map(struct iommu_pt *pt, unsigned long va,
+int msm_iommu_pagetable_alloc(struct msm_iommu_pt *pt);
+void msm_iommu_pagetable_free(struct msm_iommu_pt *pt);
+int msm_iommu_pagetable_map(struct msm_iommu_pt *pt, unsigned long va,
phys_addr_t pa, size_t len, int prot);
-size_t msm_iommu_pagetable_unmap(struct iommu_pt *pt, unsigned long va,
+size_t msm_iommu_pagetable_unmap(struct msm_iommu_pt *pt, unsigned long va,
size_t len);
-int msm_iommu_pagetable_map_range(struct iommu_pt *pt, unsigned int va,
+int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
struct scatterlist *sg, unsigned int len, int prot);
-void msm_iommu_pagetable_unmap_range(struct iommu_pt *pt, unsigned int va,
+void msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, unsigned int va,
unsigned int len);
#endif
diff --git a/drivers/iommu/msm_iommu_perfmon-v0.c b/drivers/iommu/msm_iommu_perfmon-v0.c
new file mode 100644
index 0000000..1073623
--- /dev/null
+++ b/drivers/iommu/msm_iommu_perfmon-v0.c
@@ -0,0 +1,313 @@
+/* Copyright (c) 2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+/**
+ * This file contains the part of the IOMMUv0 PMU driver that actually touches
+ * IOMMU PMU registers.
+ */
+
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <mach/iommu_hw-v0.h>
+#include <mach/iommu_perfmon.h>
+#include <mach/iommu.h>
+
+#define PM_RESET_MASK (0xF)
+#define PM_RESET_SHIFT (0x8)
+#define PM_RESET (PM_RESET_MASK << PM_RESET_SHIFT)
+
+#define PM_ENABLE_MASK (0x1)
+#define PM_ENABLE_SHIFT (0x0)
+#define PM_ENABLE (PM_ENABLE_MASK << PM_ENABLE_SHIFT)
+
+#define PM_OVFL_FLAG_MASK (0xF)
+#define PM_OVFL_FLAG_SHIFT (0x0)
+#define PM_OVFL_FLAG (PM_OVFL_FLAG_MASK << PM_OVFL_FLAG_SHIFT)
+
+#define PM_EVENT_TYPE_MASK (0x1F)
+#define PM_EVENT_TYPE_SHIFT (0x2)
+#define PM_EVENT_TYPE (PM_EVENT_TYPE_MASK << PM_EVENT_TYPE_SHIFT)
+
+#define PM_INT_EN_MASK (0x1)
+#define PM_INT_EN_SHIFT (0x0)
+#define PM_INT_EN (PM_INT_EN_MASK << PM_INT_EN_SHIFT)
+
+#define PM_INT_POL_MASK (0x1)
+#define PM_INT_POL_SHIFT (0x2)
+#define PM_INT_ACTIVE_HIGH (0x1)
+
+#define PMEVCNTR_(n) (EMC_N + n*4)
+#define PMEVTYPER_(n) (EMCC_N + n*4)
+
+/**
+ * Translate between SMMUv0 event classes and standard ARM SMMU event classes
+ */
+static int iommu_pm_event_class_translation_table[] = {
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ 0x8,
+ 0x9,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ 0x80,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ 0x12,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ MSM_IOMMU_PMU_NO_EVENT_CLASS,
+ 0x10,
+};
+
+static int iommu_pm_translate_event_class(int event_class)
+{
+ const unsigned int TBL_LEN =
+ ARRAY_SIZE(iommu_pm_event_class_translation_table);
+ unsigned int i;
+
+ if (event_class < 0)
+ return event_class;
+
+ for (i = 0; i < TBL_LEN; ++i) {
+ if (iommu_pm_event_class_translation_table[i] == event_class)
+ return i;
+ }
+ return MSM_IOMMU_PMU_NO_EVENT_CLASS;
+}
+
+static unsigned int iommu_pm_is_hw_access_OK(const struct iommu_pmon *pmon)
+{
+ /*
+ * IOMMUv0 is in always ON domain so we don't care whether we are
+ * attached or not. We only care whether the PMU is enabled or
+ * not meaning clocks are turned on.
+ */
+ return pmon->enabled;
+}
+
+static void iommu_pm_grp_enable(struct iommu_info *iommu, unsigned int grp_no)
+{
+ /* No group concept in v0. */
+}
+
+static void iommu_pm_grp_disable(struct iommu_info *iommu, unsigned int grp_no)
+{
+ /* No group concept in v0. */
+}
+
+static void iommu_pm_set_int_active_high(const struct iommu_info *iommu)
+{
+ unsigned int emmc;
+ emmc = readl_relaxed(iommu->base + EMMC);
+ emmc |= (PM_INT_ACTIVE_HIGH & PM_INT_POL_MASK) << PM_INT_POL_SHIFT;
+ writel_relaxed(emmc, iommu->base + EMMC);
+}
+
+static void iommu_pm_enable(struct iommu_info *iommu)
+{
+ unsigned int emmc;
+ emmc = readl_relaxed(iommu->base + EMMC);
+ emmc |= PM_ENABLE;
+ writel_relaxed(emmc, iommu->base + EMMC);
+}
+
+static void iommu_pm_disable(struct iommu_info *iommu)
+{
+ unsigned int emmc;
+ emmc = readl_relaxed(iommu->base + EMMC);
+ emmc &= ~PM_ENABLE;
+ writel_relaxed(emmc, iommu->base + EMMC);
+}
+
+static void iommu_pm_reset_counters(const struct iommu_info *iommu)
+{
+ unsigned int emmc;
+ emmc = readl_relaxed(iommu->base + EMMC);
+ emmc |= PM_RESET;
+ writel_relaxed(emmc, iommu->base + EMMC);
+}
+
+static void iommu_pm_check_for_overflow(struct iommu_pmon *pmon)
+{
+ struct iommu_pmon_counter *counter;
+ struct iommu_info *iommu = &pmon->iommu;
+ unsigned int reg_value;
+ unsigned int j;
+ struct iommu_pmon_cnt_group *cnt_grp = &pmon->cnt_grp[0];
+
+ reg_value = readl_relaxed(iommu->base + EMCS);
+ reg_value &= PM_OVFL_FLAG;
+
+ for (j = 0; j < cnt_grp->num_counters; ++j) {
+ counter = &cnt_grp->counters[j];
+
+ if (counter->enabled) {
+ if (reg_value & (1 << counter->absolute_counter_no))
+ counter->overflow_count++;
+ }
+ }
+
+ /* Clear overflow */
+ writel_relaxed(reg_value, iommu->base + EMCS);
+}
+
+static irqreturn_t iommu_pm_evt_ovfl_int_handler(int irq, void *dev_id)
+{
+ struct iommu_pmon *pmon = dev_id;
+ struct iommu_info *iommu = &pmon->iommu;
+
+ mutex_lock(&pmon->lock);
+
+ if (!iommu_pm_is_hw_access_OK(pmon)) {
+ mutex_unlock(&pmon->lock);
+ goto out;
+ }
+
+ iommu->ops->iommu_lock_acquire();
+ iommu_pm_check_for_overflow(pmon);
+ iommu->ops->iommu_lock_release();
+
+ mutex_unlock(&pmon->lock);
+
+out:
+ return IRQ_HANDLED;
+}
+
+static void iommu_pm_counter_enable(struct iommu_info *iommu,
+ struct iommu_pmon_counter *counter)
+{
+ unsigned int bit_no = counter->absolute_counter_no;
+ unsigned int reg_value;
+
+ /* Clear overflow of counter */
+ reg_value = readl_relaxed(iommu->base + EMCS);
+ reg_value &= (1 << bit_no);
+ writel_relaxed(reg_value, iommu->base + EMCS);
+
+ /* Enable counter */
+ counter->enabled = 1;
+}
+
+static void iommu_pm_counter_disable(struct iommu_info *iommu,
+ struct iommu_pmon_counter *counter)
+{
+ unsigned int bit_no = counter->absolute_counter_no;
+ unsigned int reg_value;
+
+ /* Disable counter */
+ counter->enabled = 0;
+
+ /* Clear overflow of counter */
+ reg_value = readl_relaxed(iommu->base + EMCS);
+ reg_value &= (1 << bit_no);
+ writel_relaxed(reg_value, iommu->base + EMCS);
+}
+
+/*
+ * Must be called after iommu_start_access() is called
+ */
+static void iommu_pm_ovfl_int_enable(struct iommu_info *iommu,
+ const struct iommu_pmon_counter *counter)
+{
+ unsigned int reg_no = counter->absolute_counter_no;
+ unsigned int reg_value;
+
+ /* Enable overflow interrupt for counter */
+ reg_value = readl_relaxed(iommu->base + PMEVTYPER_(reg_no));
+ reg_value |= PM_INT_EN;
+ writel_relaxed(reg_value, iommu->base + PMEVTYPER_(reg_no));
+}
+
+/*
+ * Must be called after iommu_start_access() is called
+ */
+static void iommu_pm_ovfl_int_disable(struct iommu_info *iommu,
+ const struct iommu_pmon_counter *counter)
+{
+ unsigned int reg_no = counter->absolute_counter_no;
+ unsigned int reg_value;
+
+ /* Disable overflow interrupt for counter */
+ reg_value = readl_relaxed(iommu->base + PMEVTYPER_(reg_no));
+ reg_value &= ~PM_INT_EN;
+ writel_relaxed(reg_value, iommu->base + PMEVTYPER_(reg_no));
+}
+
+static void iommu_pm_set_event_class(struct iommu_pmon *pmon,
+ unsigned int count_no,
+ unsigned int event_class)
+{
+ unsigned int reg_no = count_no;
+ unsigned int reg_value;
+ int event = iommu_pm_translate_event_class(event_class);
+
+ if (event == MSM_IOMMU_PMU_NO_EVENT_CLASS)
+ event = 0;
+
+ reg_value = readl_relaxed(pmon->iommu.base + PMEVTYPER_(reg_no));
+ reg_value &= ~(PM_EVENT_TYPE_MASK << PM_EVENT_TYPE_SHIFT);
+ reg_value |= (event & PM_EVENT_TYPE_MASK) << PM_EVENT_TYPE_SHIFT;
+ writel_relaxed(reg_value, pmon->iommu.base + PMEVTYPER_(reg_no));
+}
+
+static unsigned int iommu_pm_read_counter(struct iommu_pmon_counter *counter)
+{
+ struct iommu_pmon *pmon = counter->cnt_group->pmon;
+ struct iommu_info *info = &pmon->iommu;
+ unsigned int cnt_no = counter->absolute_counter_no;
+ return readl_relaxed(info->base + PMEVCNTR_(cnt_no));
+}
+
+static void iommu_pm_initialize_hw(const struct iommu_pmon *pmon)
+{
+ const struct iommu_info *iommu = &pmon->iommu;
+ struct msm_iommu_drvdata *iommu_drvdata =
+ dev_get_drvdata(iommu->iommu_dev);
+
+ /* This is called during bootup device initialization so no need
+ * for locking here.
+ */
+ iommu->ops->iommu_power_on(iommu_drvdata);
+ iommu->ops->iommu_clk_on(iommu_drvdata);
+ iommu_pm_set_int_active_high(iommu);
+ iommu->ops->iommu_clk_off(iommu_drvdata);
+ iommu->ops->iommu_power_off(iommu_drvdata);
+}
+
+static struct iommu_pm_hw_ops iommu_pm_hw_ops = {
+ .initialize_hw = iommu_pm_initialize_hw,
+ .is_hw_access_OK = iommu_pm_is_hw_access_OK,
+ .grp_enable = iommu_pm_grp_enable,
+ .grp_disable = iommu_pm_grp_disable,
+ .enable_pm = iommu_pm_enable,
+ .disable_pm = iommu_pm_disable,
+ .reset_counters = iommu_pm_reset_counters,
+ .check_for_overflow = iommu_pm_check_for_overflow,
+ .evt_ovfl_int_handler = iommu_pm_evt_ovfl_int_handler,
+ .counter_enable = iommu_pm_counter_enable,
+ .counter_disable = iommu_pm_counter_disable,
+ .ovfl_int_enable = iommu_pm_ovfl_int_enable,
+ .ovfl_int_disable = iommu_pm_ovfl_int_disable,
+ .set_event_class = iommu_pm_set_event_class,
+ .read_counter = iommu_pm_read_counter,
+};
+
+struct iommu_pm_hw_ops *iommu_pm_get_hw_ops_v0(void)
+{
+ return &iommu_pm_hw_ops;
+}
+EXPORT_SYMBOL(iommu_pm_get_hw_ops_v0);
+
diff --git a/drivers/iommu/msm_iommu_perfmon-v1.c b/drivers/iommu/msm_iommu_perfmon-v1.c
new file mode 100644
index 0000000..7d6dd34
--- /dev/null
+++ b/drivers/iommu/msm_iommu_perfmon-v1.c
@@ -0,0 +1,270 @@
+/* Copyright (c) 2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+/**
+ * This file contains the part of the IOMMUv1 PMU driver that actually touches
+ * IOMMU PMU registers.
+ */
+
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <mach/iommu_hw-v1.h>
+#include <mach/iommu_perfmon.h>
+#include <mach/iommu.h>
+
+#define PMCR_P_MASK (0x1)
+#define PMCR_P_SHIFT (1)
+#define PMCR_P (PMCR_P_MASK << PMCR_P_SHIFT)
+#define PMCFGR_NCG_MASK (0xFF)
+#define PMCFGR_NCG_SHIFT (24)
+#define PMCFGR_NCG (PMCFGR_NCG_MASK << PMCFGR_NCG_SHIFT)
+#define PMCFGR_N_MASK (0xFF)
+#define PMCFGR_N_SHIFT (0)
+#define PMCFGR_N (PMCFGR_N_MASK << PMCFGR_N_SHIFT)
+#define CR_E 0x1
+#define CGCR_CEN 0x800
+#define CGCR_CEN_SHFT (1 << 11)
+#define PMCGCR_CGNC_MASK (0x0F)
+#define PMCGCR_CGNC_SHIFT (24)
+#define PMCGCR_CGNC (PMCGCR_CGNC_MASK << PMCGCR_CGNC_SHIFT)
+#define PMCGCR_(group) (PMCGCR_N + group*4)
+
+#define PMOVSCLR_(n) (PMOVSCLR_N + n*4)
+#define PMCNTENSET_(n) (PMCNTENSET_N + n*4)
+#define PMCNTENCLR_(n) (PMCNTENCLR_N + n*4)
+#define PMINTENSET_(n) (PMINTENSET_N + n*4)
+#define PMINTENCLR_(n) (PMINTENCLR_N + n*4)
+
+#define PMEVCNTR_(n) (PMEVCNTR_N + n*4)
+#define PMEVTYPER_(n) (PMEVTYPER_N + n*4)
+
+
+static unsigned int iommu_pm_is_hw_access_OK(const struct iommu_pmon *pmon)
+{
+ /*
+ * IOMMUv1 is not in the always on domain so we need to make sure
+ * the regulators are turned on in addition to clocks before we allow
+ * access to the hardware thus we check if we have attached to the
+ * IOMMU in addition to checking if we have enabled PMU.
+ */
+ return pmon->enabled && (pmon->iommu_attach_count > 0);
+}
+
+static void iommu_pm_grp_enable(struct iommu_info *iommu, unsigned int grp_no)
+{
+ unsigned int pmcgcr;
+ pmcgcr = readl_relaxed(iommu->base + PMCGCR_(grp_no));
+ pmcgcr |= CGCR_CEN;
+ writel_relaxed(pmcgcr, iommu->base + PMCGCR_(grp_no));
+}
+
+static void iommu_pm_grp_disable(struct iommu_info *iommu, unsigned int grp_no)
+{
+ unsigned int pmcgcr;
+ pmcgcr = readl_relaxed(iommu->base + PMCGCR_(grp_no));
+ pmcgcr &= ~CGCR_CEN;
+ writel_relaxed(pmcgcr, iommu->base + PMCGCR_(grp_no));
+}
+
+static void iommu_pm_enable(struct iommu_info *iommu)
+{
+ unsigned int pmcr;
+ pmcr = readl_relaxed(iommu->base + PMCR);
+ pmcr |= CR_E;
+ writel_relaxed(pmcr, iommu->base + PMCR);
+}
+
+static void iommu_pm_disable(struct iommu_info *iommu)
+{
+ unsigned int pmcr;
+ pmcr = readl_relaxed(iommu->base + PMCR);
+ pmcr &= ~CR_E;
+ writel_relaxed(pmcr, iommu->base + PMCR);
+}
+
+static void iommu_pm_reset_counters(const struct iommu_info *iommu)
+{
+ unsigned int pmcr;
+ pmcr = readl_relaxed(iommu->base + PMCR);
+ pmcr |= PMCR_P;
+ writel_relaxed(pmcr, iommu->base + PMCR);
+}
+
+static void iommu_pm_check_for_overflow(struct iommu_pmon *pmon)
+{
+ struct iommu_pmon_counter *counter;
+ struct iommu_info *iommu = &pmon->iommu;
+ unsigned int reg_no = 0;
+ unsigned int bit_no;
+ unsigned int reg_value;
+ unsigned int i;
+ unsigned int j;
+ unsigned int curr_reg = 0;
+
+ reg_value = readl_relaxed(iommu->base + PMOVSCLR_(curr_reg));
+
+ for (i = 0; i < pmon->num_groups; ++i) {
+ struct iommu_pmon_cnt_group *cnt_grp = &pmon->cnt_grp[i];
+ for (j = 0; j < cnt_grp->num_counters; ++j) {
+ counter = &cnt_grp->counters[j];
+ reg_no = counter->absolute_counter_no / 32;
+ bit_no = counter->absolute_counter_no % 32;
+ if (reg_no != curr_reg) {
+ /* Clear overflow bits */
+ writel_relaxed(reg_value, iommu->base +
+ PMOVSCLR_(reg_no));
+ curr_reg = reg_no;
+ reg_value = readl_relaxed(iommu->base +
+ PMOVSCLR_(curr_reg));
+ }
+
+ if (counter->enabled) {
+ if (reg_value & (1 << bit_no))
+ counter->overflow_count++;
+ }
+ }
+ }
+
+ /* Clear overflow */
+ writel_relaxed(reg_value, iommu->base + PMOVSCLR_(reg_no));
+}
+
+static irqreturn_t iommu_pm_evt_ovfl_int_handler(int irq, void *dev_id)
+{
+ struct iommu_pmon *pmon = dev_id;
+ struct iommu_info *iommu = &pmon->iommu;
+
+ mutex_lock(&pmon->lock);
+
+ if (!iommu_pm_is_hw_access_OK(pmon)) {
+ mutex_unlock(&pmon->lock);
+ goto out;
+ }
+
+ iommu->ops->iommu_lock_acquire();
+ iommu_pm_check_for_overflow(pmon);
+ iommu->ops->iommu_lock_release();
+
+ mutex_unlock(&pmon->lock);
+
+out:
+ return IRQ_HANDLED;
+}
+
+static void iommu_pm_counter_enable(struct iommu_info *iommu,
+ struct iommu_pmon_counter *counter)
+{
+ unsigned int reg_no = counter->absolute_counter_no / 32;
+ unsigned int bit_no = counter->absolute_counter_no % 32;
+ unsigned int reg_value;
+
+ /* Clear overflow of counter */
+ reg_value = 1 << bit_no;
+ writel_relaxed(reg_value, iommu->base + PMOVSCLR_(reg_no));
+
+ /* Enable counter */
+ writel_relaxed(reg_value, iommu->base + PMCNTENSET_(reg_no));
+ counter->enabled = 1;
+}
+
+static void iommu_pm_counter_disable(struct iommu_info *iommu,
+ struct iommu_pmon_counter *counter)
+{
+ unsigned int reg_no = counter->absolute_counter_no / 32;
+ unsigned int bit_no = counter->absolute_counter_no % 32;
+ unsigned int reg_value;
+
+ counter->enabled = 0;
+
+ /* Disable counter */
+ reg_value = 1 << bit_no;
+ writel_relaxed(reg_value, iommu->base + PMCNTENCLR_(reg_no));
+
+ /* Clear overflow of counter */
+ writel_relaxed(reg_value, iommu->base + PMOVSCLR_(reg_no));
+}
+
+/*
+ * Must be called after iommu_start_access() is called
+ */
+static void iommu_pm_ovfl_int_enable(struct iommu_info *iommu,
+ const struct iommu_pmon_counter *counter)
+{
+ unsigned int reg_no = counter->absolute_counter_no / 32;
+ unsigned int bit_no = counter->absolute_counter_no % 32;
+ unsigned int reg_value;
+
+ /* Enable overflow interrupt for counter */
+ reg_value = (1 << bit_no);
+ writel_relaxed(reg_value, iommu->base + PMINTENSET_(reg_no));
+}
+
+/*
+ * Must be called after iommu_start_access() is called
+ */
+static void iommu_pm_ovfl_int_disable(struct iommu_info *iommu,
+ const struct iommu_pmon_counter *counter)
+{
+ unsigned int reg_no = counter->absolute_counter_no / 32;
+ unsigned int bit_no = counter->absolute_counter_no % 32;
+ unsigned int reg_value;
+
+ /* Disable overflow interrupt for counter */
+ reg_value = 1 << bit_no;
+ writel_relaxed(reg_value, iommu->base + PMINTENCLR_(reg_no));
+}
+
+static void iommu_pm_set_event_class(struct iommu_pmon *pmon,
+ unsigned int count_no,
+ unsigned int event_class)
+{
+ writel_relaxed(event_class, pmon->iommu.base + PMEVTYPER_(count_no));
+}
+
+static unsigned int iommu_pm_read_counter(struct iommu_pmon_counter *counter)
+{
+ struct iommu_pmon *pmon = counter->cnt_group->pmon;
+ struct iommu_info *info = &pmon->iommu;
+ unsigned int cnt_no = counter->absolute_counter_no;
+ return readl_relaxed(info->base + PMEVCNTR_(cnt_no));
+}
+
+static void iommu_pm_initialize_hw(const struct iommu_pmon *pmon)
+{
+ /* No initialization needed */
+}
+
+static struct iommu_pm_hw_ops iommu_pm_hw_ops = {
+ .initialize_hw = iommu_pm_initialize_hw,
+ .is_hw_access_OK = iommu_pm_is_hw_access_OK,
+ .grp_enable = iommu_pm_grp_enable,
+ .grp_disable = iommu_pm_grp_disable,
+ .enable_pm = iommu_pm_enable,
+ .disable_pm = iommu_pm_disable,
+ .reset_counters = iommu_pm_reset_counters,
+ .check_for_overflow = iommu_pm_check_for_overflow,
+ .evt_ovfl_int_handler = iommu_pm_evt_ovfl_int_handler,
+ .counter_enable = iommu_pm_counter_enable,
+ .counter_disable = iommu_pm_counter_disable,
+ .ovfl_int_enable = iommu_pm_ovfl_int_enable,
+ .ovfl_int_disable = iommu_pm_ovfl_int_disable,
+ .set_event_class = iommu_pm_set_event_class,
+ .read_counter = iommu_pm_read_counter,
+};
+
+struct iommu_pm_hw_ops *iommu_pm_get_hw_ops_v1(void)
+{
+ return &iommu_pm_hw_ops;
+}
+EXPORT_SYMBOL(iommu_pm_get_hw_ops_v1);
+
diff --git a/drivers/iommu/msm_iommu_perfmon.c b/drivers/iommu/msm_iommu_perfmon.c
new file mode 100644
index 0000000..fee8a4a
--- /dev/null
+++ b/drivers/iommu/msm_iommu_perfmon.c
@@ -0,0 +1,819 @@
+/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/iommu.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <mach/iommu.h>
+#include <mach/iommu_perfmon.h>
+
+static LIST_HEAD(iommu_list);
+static struct dentry *msm_iommu_root_debugfs_dir;
+static const char *NO_EVENT_CLASS_NAME = "none";
+static const unsigned int MAX_EVEN_CLASS_NAME_LEN = 36;
+
+struct event_class {
+ unsigned int event_number;
+ const char *desc;
+};
+
+static struct event_class pmu_event_classes[] = {
+ { 0x00, "cycle_count" },
+ { 0x01, "cycle_count64" },
+ { 0x08, "tlb_refill" },
+ { 0x09, "tlb_refill_read" },
+ { 0x0A, "tlb_refill_write" },
+ { 0x10, "access" },
+ { 0x11, "access_read" },
+ { 0x12, "access_write" },
+ { 0x80, "full_misses" },
+ { 0x81, "partial_miss_1lbfb_hit" },
+ { 0x82, "partial_miss_2lbfb_hit" },
+ { 0x83, "full_hit" },
+ { 0x90, "pred_req_full_miss" },
+ { 0x91, "pred_req_partial_miss_1lbfb_hit" },
+ { 0x92, "pred_req_partial_miss_2lbfb_hit" },
+ { 0xb0, "tot_num_miss_axi_htw_read_req" },
+ { 0xb1, "tot_num_pred_axi_htw_read_req" },
+};
+
+static unsigned int iommu_pm_create_sup_cls_str(char **buf,
+ struct iommu_pmon *pmon)
+{
+ unsigned long buf_size = ARRAY_SIZE(pmu_event_classes) *
+ MAX_EVEN_CLASS_NAME_LEN;
+ unsigned int pos = 0;
+ unsigned int nevent_cls = pmon->nevent_cls_supported;
+
+ *buf = kzalloc(buf_size, GFP_KERNEL);
+ if (*buf) {
+ unsigned int j;
+ int i;
+ struct event_class *ptr;
+ size_t array_len = ARRAY_SIZE(pmu_event_classes);
+ ptr = pmu_event_classes;
+
+ for (j = 0; j < nevent_cls; ++j) {
+ for (i = 0; i < array_len; ++i) {
+
+ if (ptr[i].event_number !=
+ pmon->event_cls_supported[j])
+ continue;
+
+ if (pos < buf_size) {
+ pos += snprintf(&(*buf)[pos],
+ buf_size-pos,
+ "[%u] %s\n",
+ ptr[i].event_number,
+ ptr[i].desc);
+ }
+ break;
+ }
+ }
+ }
+ return pos;
+}
+
+static const char *iommu_pm_find_event_class_name(int event_class)
+{
+ size_t array_len;
+ struct event_class *ptr;
+ int i;
+ const char *event_class_name = NO_EVENT_CLASS_NAME;
+ if (event_class < 0)
+ goto out;
+
+ array_len = ARRAY_SIZE(pmu_event_classes);
+ ptr = pmu_event_classes;
+
+ for (i = 0; i < array_len; ++i) {
+ if (ptr[i].event_number == event_class) {
+ event_class_name = ptr[i].desc;
+ break;
+ }
+ }
+
+out:
+ return event_class_name;
+}
+
+static int iommu_pm_find_event_class(const char *event_class_name)
+{
+ size_t array_len;
+ struct event_class *ptr;
+ int i;
+ int event_class = MSM_IOMMU_PMU_NO_EVENT_CLASS;
+
+ if (strcmp(event_class_name, NO_EVENT_CLASS_NAME) == 0)
+ goto out;
+
+ array_len = ARRAY_SIZE(pmu_event_classes);
+ ptr = pmu_event_classes;
+
+ for (i = 0; i < array_len; ++i) {
+ if (strcmp(ptr[i].desc, event_class_name) == 0) {
+ event_class = ptr[i].event_number;
+ goto out;
+ }
+ }
+
+out:
+ return event_class;
+}
+
+static inline void iommu_pm_add_to_iommu_list(struct iommu_pmon *iommu_pmon)
+{
+ list_add(&iommu_pmon->iommu_list, &iommu_list);
+}
+
+static inline void iommu_pm_del_from_iommu_list(struct iommu_pmon *iommu_pmon)
+{
+ list_del(&iommu_pmon->iommu_list);
+}
+
+static struct iommu_pmon *iommu_pm_get_pm_by_dev(struct device *dev)
+{
+ struct iommu_pmon *pmon;
+ struct iommu_info *info;
+ struct list_head *ent;
+ list_for_each(ent, &iommu_list) {
+ pmon = list_entry(ent, struct iommu_pmon, iommu_list);
+ info = &pmon->iommu;
+ if (dev == info->iommu_dev)
+ return pmon;
+ }
+ return NULL;
+}
+
+static void iommu_pm_set_event_type(struct iommu_pmon *pmon,
+ struct iommu_pmon_counter *counter)
+{
+ int event_class;
+ unsigned int count_no;
+ struct iommu_info *iommu = &pmon->iommu;
+
+ event_class = counter->current_event_class;
+ count_no = counter->absolute_counter_no;
+
+ if (event_class == MSM_IOMMU_PMU_NO_EVENT_CLASS) {
+ if (iommu->hw_ops->is_hw_access_OK(pmon)) {
+ iommu->ops->iommu_lock_acquire();
+ iommu->hw_ops->counter_disable(iommu, counter);
+ iommu->hw_ops->ovfl_int_disable(iommu, counter);
+ iommu->hw_ops->set_event_class(pmon, count_no, 0);
+ iommu->ops->iommu_lock_release();
+ }
+ counter->overflow_count = 0;
+ counter->value = 0;
+ } else {
+ counter->overflow_count = 0;
+ counter->value = 0;
+ if (iommu->hw_ops->is_hw_access_OK(pmon)) {
+ iommu->ops->iommu_lock_acquire();
+ iommu->hw_ops->set_event_class(pmon, count_no,
+ event_class);
+ iommu->hw_ops->ovfl_int_enable(iommu, counter);
+ iommu->hw_ops->counter_enable(iommu, counter);
+ iommu->ops->iommu_lock_release();
+ }
+ }
+}
+
+static void iommu_pm_reset_counts(struct iommu_pmon *pmon)
+{
+ unsigned int i;
+ unsigned int j;
+ for (i = 0; i < pmon->num_groups; ++i) {
+ struct iommu_pmon_cnt_group *cnt_grp = &pmon->cnt_grp[i];
+ for (j = 0; j < cnt_grp->num_counters; ++j) {
+ cnt_grp->counters[j].value = 0;
+ cnt_grp->counters[j].overflow_count = 0;
+ }
+ }
+}
+
+static void iommu_pm_set_all_counters(struct iommu_pmon *pmon)
+{
+ unsigned int i;
+ unsigned int j;
+ for (i = 0; i < pmon->num_groups; ++i) {
+ struct iommu_pmon_cnt_group *cnt_grp = &pmon->cnt_grp[i];
+ for (j = 0; j < cnt_grp->num_counters; ++j)
+ iommu_pm_set_event_type(pmon, &cnt_grp->counters[j]);
+ }
+}
+
+static void iommu_pm_read_all_counters(struct iommu_pmon *pmon)
+{
+ unsigned int i;
+ unsigned int j;
+ struct iommu_info *iommu = &pmon->iommu;
+ for (i = 0; i < pmon->num_groups; ++i) {
+ struct iommu_pmon_cnt_group *cnt_grp = &pmon->cnt_grp[i];
+ for (j = 0; j < cnt_grp->num_counters; ++j) {
+ struct iommu_pmon_counter *counter;
+ counter = &cnt_grp->counters[j];
+ counter->value = iommu->hw_ops->read_counter(counter);
+ }
+ }
+}
+
+static void iommu_pm_on(struct iommu_pmon *pmon)
+{
+ unsigned int i;
+ struct iommu_info *iommu = &pmon->iommu;
+ struct msm_iommu_drvdata *iommu_drvdata =
+ dev_get_drvdata(iommu->iommu_dev);
+
+ iommu->ops->iommu_power_on(iommu_drvdata);
+ iommu->ops->iommu_clk_on(iommu_drvdata);
+
+ /* Reset counters in HW */
+ iommu->ops->iommu_lock_acquire();
+ iommu->hw_ops->reset_counters(&pmon->iommu);
+ iommu->ops->iommu_lock_release();
+
+ /* Reset SW counters */
+ iommu_pm_reset_counts(pmon);
+
+ pmon->enabled = 1;
+
+ iommu_pm_set_all_counters(pmon);
+
+ iommu->ops->iommu_lock_acquire();
+
+ /* enable all counter group */
+ for (i = 0; i < pmon->num_groups; ++i)
+ iommu->hw_ops->grp_enable(iommu, i);
+
+ /* enable global counters */
+ iommu->hw_ops->enable_pm(iommu);
+ iommu->ops->iommu_lock_release();
+
+ pr_info("%s: TLB performance monitoring turned ON\n",
+ pmon->iommu.iommu_name);
+}
+
+static void iommu_pm_off(struct iommu_pmon *pmon)
+{
+ unsigned int i;
+ struct iommu_info *iommu = &pmon->iommu;
+ struct msm_iommu_drvdata *iommu_drvdata =
+ dev_get_drvdata(iommu->iommu_dev);
+
+ pmon->enabled = 0;
+
+ iommu->ops->iommu_lock_acquire();
+
+ /* disable global counters */
+ iommu->hw_ops->disable_pm(iommu);
+
+ /* Check if we overflowed just before turning off pmon */
+ iommu->hw_ops->check_for_overflow(pmon);
+
+ /* disable all counter group */
+ for (i = 0; i < pmon->num_groups; ++i)
+ iommu->hw_ops->grp_disable(iommu, i);
+
+ /* Update cached copy of counters before turning off power */
+ iommu_pm_read_all_counters(pmon);
+
+ iommu->ops->iommu_lock_release();
+ iommu->ops->iommu_clk_off(iommu_drvdata);
+ iommu->ops->iommu_power_off(iommu_drvdata);
+
+ pr_info("%s: TLB performance monitoring turned OFF\n",
+ pmon->iommu.iommu_name);
+}
+
+static int iommu_pm_debug_open(struct inode *inode, struct file *file)
+{
+ file->private_data = inode->i_private;
+ return 0;
+}
+
+static ssize_t iommu_pm_count_value_read(struct file *fp,
+ char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t rd_cnt;
+ unsigned long long full_count;
+
+ struct iommu_pmon_counter *counter = fp->private_data;
+ struct iommu_pmon *pmon = counter->cnt_group->pmon;
+ struct iommu_info *iommu = &pmon->iommu;
+ char buf[50];
+ size_t len;
+
+ mutex_lock(&pmon->lock);
+
+ if (iommu->hw_ops->is_hw_access_OK(pmon)) {
+ iommu->ops->iommu_lock_acquire();
+ counter->value = iommu->hw_ops->read_counter(counter);
+ iommu->ops->iommu_lock_release();
+ }
+ full_count = (unsigned long long) counter->value +
+ ((unsigned long long)counter->overflow_count *
+ 0x100000000ULL);
+
+ len = snprintf(buf, 50, "%llu\n", full_count);
+ rd_cnt = simple_read_from_buffer(user_buff, count, pos, buf, len);
+ mutex_unlock(&pmon->lock);
+
+ return rd_cnt;
+}
+
+static const struct file_operations cnt_value_file_ops = {
+ .open = iommu_pm_debug_open,
+ .read = iommu_pm_count_value_read,
+};
+
+static ssize_t iommu_pm_event_class_read(struct file *fp,
+ char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t rd_cnt;
+ struct iommu_pmon_counter *counter = fp->private_data;
+ struct iommu_pmon *pmon = counter->cnt_group->pmon;
+ char buf[50];
+ const char *event_class_name;
+ size_t len;
+
+ mutex_lock(&pmon->lock);
+ event_class_name = iommu_pm_find_event_class_name(
+ counter->current_event_class);
+ len = snprintf(buf, 50, "%s\n", event_class_name);
+
+ rd_cnt = simple_read_from_buffer(user_buff, count, pos, buf, len);
+ mutex_unlock(&pmon->lock);
+ return rd_cnt;
+}
+
+static ssize_t iommu_pm_event_class_write(struct file *fp,
+ const char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t wr_cnt;
+ char buf[50];
+ size_t buf_size = sizeof(buf);
+ struct iommu_pmon_counter *counter = fp->private_data;
+ struct iommu_pmon *pmon = counter->cnt_group->pmon;
+ int current_event_class;
+
+ if ((count + *pos) >= buf_size)
+ return -EINVAL;
+
+ mutex_lock(&pmon->lock);
+ current_event_class = counter->current_event_class;
+ wr_cnt = simple_write_to_buffer(buf, buf_size, pos, user_buff, count);
+ if (wr_cnt >= 1) {
+ int rv;
+ long value;
+ buf[wr_cnt-1] = '\0';
+ rv = kstrtol(buf, 10, &value);
+ if (!rv) {
+ counter->current_event_class =
+ iommu_pm_find_event_class(
+ iommu_pm_find_event_class_name(value));
+ } else {
+ counter->current_event_class =
+ iommu_pm_find_event_class(buf);
+ } }
+
+ if (current_event_class != counter->current_event_class)
+ iommu_pm_set_event_type(pmon, counter);
+
+ mutex_unlock(&pmon->lock);
+ return wr_cnt;
+}
+
+static const struct file_operations event_class_file_ops = {
+ .open = iommu_pm_debug_open,
+ .read = iommu_pm_event_class_read,
+ .write = iommu_pm_event_class_write,
+};
+
+static ssize_t iommu_reset_counters_write(struct file *fp,
+ const char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t wr_cnt;
+ char buf[10];
+ size_t buf_size = sizeof(buf);
+ struct iommu_pmon *pmon = fp->private_data;
+ struct iommu_info *iommu = &pmon->iommu;
+
+ if ((count + *pos) >= buf_size)
+ return -EINVAL;
+
+ mutex_lock(&pmon->lock);
+ wr_cnt = simple_write_to_buffer(buf, buf_size, pos, user_buff, count);
+ if (wr_cnt >= 1) {
+ unsigned long cmd = 0;
+ int rv;
+ buf[wr_cnt-1] = '\0';
+ rv = kstrtoul(buf, 10, &cmd);
+ if (!rv && (cmd == 1)) {
+ if (iommu->hw_ops->is_hw_access_OK(pmon)) {
+ iommu->ops->iommu_lock_acquire();
+ iommu->hw_ops->reset_counters(&pmon->iommu);
+ iommu->ops->iommu_lock_release();
+ }
+ iommu_pm_reset_counts(pmon);
+ pr_info("TLB performance counters reset\n");
+ } else {
+ pr_err("Unknown performance monitor command: %lu\n",
+ cmd);
+ }
+ }
+ mutex_unlock(&pmon->lock);
+ return wr_cnt;
+}
+
+static const struct file_operations reset_file_ops = {
+ .open = iommu_pm_debug_open,
+ .write = iommu_reset_counters_write,
+};
+
+static ssize_t iommu_pm_enable_counters_read(struct file *fp,
+ char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t rd_cnt;
+ char buf[5];
+ size_t len;
+ struct iommu_pmon *pmon = fp->private_data;
+
+ mutex_lock(&pmon->lock);
+ len = snprintf(buf, 5, "%u\n", pmon->enabled);
+ rd_cnt = simple_read_from_buffer(user_buff, count, pos, buf, len);
+ mutex_unlock(&pmon->lock);
+ return rd_cnt;
+}
+
+static ssize_t iommu_pm_enable_counters_write(struct file *fp,
+ const char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t wr_cnt;
+ char buf[10];
+ size_t buf_size = sizeof(buf);
+ struct iommu_pmon *pmon = fp->private_data;
+
+ if ((count + *pos) >= buf_size)
+ return -EINVAL;
+
+ mutex_lock(&pmon->lock);
+ wr_cnt = simple_write_to_buffer(buf, buf_size, pos, user_buff, count);
+ if (wr_cnt >= 1) {
+ unsigned long cmd;
+ int rv;
+ buf[wr_cnt-1] = '\0';
+ rv = kstrtoul(buf, 10, &cmd);
+ if (!rv && (cmd < 2)) {
+ if (pmon->enabled == 1 && cmd == 0) {
+ if (pmon->iommu_attach_count > 0)
+ iommu_pm_off(pmon);
+ } else if (pmon->enabled == 0 && cmd == 1) {
+ /* We can only turn on perf. monitoring if
+ * iommu is attached. Delay turning on perf.
+ * monitoring until we are attached.
+ */
+ if (pmon->iommu_attach_count > 0)
+ iommu_pm_on(pmon);
+ else
+ pmon->enabled = 1;
+ }
+ } else {
+ pr_err("Unknown performance monitor command: %lu\n",
+ cmd);
+ }
+ }
+ mutex_unlock(&pmon->lock);
+ return wr_cnt;
+}
+
+static const struct file_operations event_enable_file_ops = {
+ .open = iommu_pm_debug_open,
+ .read = iommu_pm_enable_counters_read,
+ .write = iommu_pm_enable_counters_write,
+};
+
+static ssize_t iommu_pm_avail_event_cls_read(struct file *fp,
+ char __user *user_buff,
+ size_t count, loff_t *pos)
+{
+ size_t rd_cnt = 0;
+ struct iommu_pmon *pmon = fp->private_data;
+ char *buf;
+ size_t len;
+
+ mutex_lock(&pmon->lock);
+
+ len = iommu_pm_create_sup_cls_str(&buf, pmon);
+ if (buf) {
+ rd_cnt = simple_read_from_buffer(user_buff, count, pos,
+ buf, len);
+ kfree(buf);
+ }
+ mutex_unlock(&pmon->lock);
+ return rd_cnt;
+}
+
+static const struct file_operations available_event_cls_file_ops = {
+ .open = iommu_pm_debug_open,
+ .read = iommu_pm_avail_event_cls_read,
+};
+
+
+
+static int iommu_pm_create_grp_debugfs_counters_hierarchy(
+ struct iommu_pmon_cnt_group *cnt_grp,
+ unsigned int *abs_counter_no)
+{
+ int ret = 0;
+ int j;
+ char name[20];
+
+ for (j = 0; j < cnt_grp->num_counters; ++j) {
+ struct dentry *grp_dir = cnt_grp->group_dir;
+ struct dentry *counter_dir;
+ cnt_grp->counters[j].cnt_group = cnt_grp;
+ cnt_grp->counters[j].counter_no = j;
+ cnt_grp->counters[j].absolute_counter_no = *abs_counter_no;
+ (*abs_counter_no)++;
+ cnt_grp->counters[j].value = 0;
+ cnt_grp->counters[j].overflow_count = 0;
+ cnt_grp->counters[j].current_event_class =
+ MSM_IOMMU_PMU_NO_EVENT_CLASS;
+
+ snprintf(name, 20, "counter%u", j);
+
+ counter_dir = debugfs_create_dir(name, grp_dir);
+
+ if (IS_ERR_OR_NULL(counter_dir)) {
+ pr_err("unable to create counter debugfs dir %s\n",
+ name);
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ cnt_grp->counters[j].counter_dir = counter_dir;
+
+ if (!debugfs_create_file("value", 0644, counter_dir,
+ &cnt_grp->counters[j],
+ &cnt_value_file_ops)) {
+ ret = -EIO;
+ goto out;
+ }
+
+ if (!debugfs_create_file("current_event_class", 0644,
+ counter_dir, &cnt_grp->counters[j],
+ &event_class_file_ops)) {
+ ret = -EIO;
+ goto out;
+ }
+ }
+out:
+ return ret;
+}
+
+static int iommu_pm_create_group_debugfs_hierarchy(struct iommu_info *iommu,
+ struct iommu_pmon *pmon_entry)
+{
+ int i;
+ int ret = 0;
+ char name[20];
+ unsigned int abs_counter_no = 0;
+
+ for (i = 0; i < pmon_entry->num_groups; ++i) {
+ pmon_entry->cnt_grp[i].pmon = pmon_entry;
+ pmon_entry->cnt_grp[i].grp_no = i;
+ pmon_entry->cnt_grp[i].num_counters = pmon_entry->num_counters;
+ pmon_entry->cnt_grp[i].counters =
+ kzalloc(sizeof(*pmon_entry->cnt_grp[i].counters)
+ * pmon_entry->cnt_grp[i].num_counters, GFP_KERNEL);
+
+ if (!pmon_entry->cnt_grp[i].counters) {
+ pr_err("Unable to allocate memory for counters\n");
+ ret = -ENOMEM;
+ goto out;
+ }
+ snprintf(name, 20, "group%u", i);
+ pmon_entry->cnt_grp[i].group_dir = debugfs_create_dir(name,
+ pmon_entry->iommu_dir);
+ if (IS_ERR_OR_NULL(pmon_entry->cnt_grp[i].group_dir)) {
+ pr_err("unable to create group debugfs dir %s\n", name);
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ ret = iommu_pm_create_grp_debugfs_counters_hierarchy(
+ &pmon_entry->cnt_grp[i],
+ &abs_counter_no);
+ if (ret)
+ goto out;
+ }
+out:
+ return ret;
+}
+
+int msm_iommu_pm_iommu_register(struct iommu_pmon *pmon_entry)
+{
+ int ret = 0;
+ struct iommu_info *iommu = &pmon_entry->iommu;
+ int i;
+
+ if (!iommu->ops || !iommu->iommu_name || !iommu->base
+ || !iommu->iommu_dev) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (!msm_iommu_root_debugfs_dir) {
+ msm_iommu_root_debugfs_dir = debugfs_create_dir("iommu", NULL);
+ if (IS_ERR_OR_NULL(msm_iommu_root_debugfs_dir)) {
+ pr_err("Failed creating iommu debugfs dir \"iommu\"\n");
+ ret = -EIO;
+ goto out;
+ }
+ }
+
+ pmon_entry->cnt_grp = kzalloc(sizeof(*pmon_entry->cnt_grp)
+ * pmon_entry->num_groups, GFP_KERNEL);
+ if (!pmon_entry->cnt_grp) {
+ pr_err("Unable to allocate memory for counter groups\n");
+ ret = -ENOMEM;
+ goto file_err;
+ }
+ pmon_entry->iommu_dir = debugfs_create_dir(iommu->iommu_name,
+ msm_iommu_root_debugfs_dir);
+ if (IS_ERR_OR_NULL(pmon_entry->iommu_dir)) {
+ pr_err("unable to create iommu debugfs dir %s\n",
+ iommu->iommu_name);
+ ret = -ENOMEM;
+ goto free_mem;
+ }
+
+ if (!debugfs_create_file("reset_counters", 0644,
+ pmon_entry->iommu_dir, pmon_entry, &reset_file_ops)) {
+ ret = -EIO;
+ goto free_mem;
+ }
+
+ if (!debugfs_create_file("enable_counters", 0644,
+ pmon_entry->iommu_dir, pmon_entry, &event_enable_file_ops)) {
+ ret = -EIO;
+ goto free_mem;
+ }
+
+ if (!debugfs_create_file("available_event_classes", 0644,
+ pmon_entry->iommu_dir, pmon_entry,
+ &available_event_cls_file_ops)) {
+ ret = -EIO;
+ goto free_mem;
+ }
+
+ ret = iommu_pm_create_group_debugfs_hierarchy(iommu, pmon_entry);
+ if (ret)
+ goto free_mem;
+
+ iommu->hw_ops->initialize_hw(pmon_entry);
+
+ if (iommu->evt_irq > 0) {
+ ret = request_threaded_irq(iommu->evt_irq, NULL,
+ iommu->hw_ops->evt_ovfl_int_handler,
+ IRQF_ONESHOT | IRQF_SHARED,
+ "msm_iommu_pmon_nonsecure_irq", pmon_entry);
+ if (ret) {
+ pr_err("Request IRQ %d failed with ret=%d\n",
+ iommu->evt_irq,
+ ret);
+ goto free_mem;
+ }
+ } else {
+ pr_info("%s: Overflow interrupt not available\n", __func__);
+ }
+
+ dev_dbg(iommu->iommu_dev, "%s iommu registered\n", iommu->iommu_name);
+
+ goto out;
+free_mem:
+ if (pmon_entry->cnt_grp) {
+ for (i = 0; i < pmon_entry->num_groups; ++i) {
+ kfree(pmon_entry->cnt_grp[i].counters);
+ pmon_entry->cnt_grp[i].counters = 0;
+ }
+ }
+ kfree(pmon_entry->cnt_grp);
+ pmon_entry->cnt_grp = 0;
+file_err:
+ debugfs_remove_recursive(msm_iommu_root_debugfs_dir);
+out:
+ return ret;
+}
+EXPORT_SYMBOL(msm_iommu_pm_iommu_register);
+
+void msm_iommu_pm_iommu_unregister(struct device *dev)
+{
+ int i;
+ struct iommu_pmon *pmon_entry = iommu_pm_get_pm_by_dev(dev);
+
+ if (!pmon_entry)
+ return;
+
+ free_irq(pmon_entry->iommu.evt_irq, pmon_entry->iommu.iommu_dev);
+
+ if (!pmon_entry)
+ goto remove_debugfs;
+
+ if (pmon_entry->cnt_grp) {
+ for (i = 0; i < pmon_entry->num_groups; ++i)
+ kfree(pmon_entry->cnt_grp[i].counters);
+ }
+
+ kfree(pmon_entry->cnt_grp);
+
+remove_debugfs:
+ debugfs_remove_recursive(msm_iommu_root_debugfs_dir);
+
+ return;
+}
+EXPORT_SYMBOL(msm_iommu_pm_iommu_unregister);
+
+struct iommu_pmon *msm_iommu_pm_alloc(struct device *dev)
+{
+ struct iommu_pmon *pmon_entry;
+ struct iommu_info *info;
+ pmon_entry = devm_kzalloc(dev, sizeof(*pmon_entry), GFP_KERNEL);
+ if (!pmon_entry)
+ return NULL;
+ info = &pmon_entry->iommu;
+ info->iommu_dev = dev;
+ mutex_init(&pmon_entry->lock);
+ iommu_pm_add_to_iommu_list(pmon_entry);
+ return pmon_entry;
+}
+EXPORT_SYMBOL(msm_iommu_pm_alloc);
+
+void msm_iommu_pm_free(struct device *dev)
+{
+ struct iommu_pmon *pmon = iommu_pm_get_pm_by_dev(dev);
+ if (pmon)
+ iommu_pm_del_from_iommu_list(pmon);
+}
+EXPORT_SYMBOL(msm_iommu_pm_free);
+
+void msm_iommu_attached(struct device *dev)
+{
+ struct iommu_pmon *pmon = iommu_pm_get_pm_by_dev(dev);
+ if (pmon) {
+ mutex_lock(&pmon->lock);
+ ++pmon->iommu_attach_count;
+ if (pmon->iommu_attach_count == 1) {
+ /* If perf. mon was enabled before we attached we do
+ * the actual after we attach.
+ */
+ if (pmon->enabled)
+ iommu_pm_on(pmon);
+ }
+ mutex_unlock(&pmon->lock);
+ }
+}
+EXPORT_SYMBOL(msm_iommu_attached);
+
+void msm_iommu_detached(struct device *dev)
+{
+ struct iommu_pmon *pmon = iommu_pm_get_pm_by_dev(dev);
+ if (pmon) {
+ mutex_lock(&pmon->lock);
+ if (pmon->iommu_attach_count == 1) {
+ /* If perf. mon is still enabled we have to disable
+ * before we do the detach.
+ */
+ if (pmon->enabled)
+ iommu_pm_off(pmon);
+ }
+ BUG_ON(pmon->iommu_attach_count == 0);
+ --pmon->iommu_attach_count;
+ mutex_unlock(&pmon->lock);
+ }
+}
+EXPORT_SYMBOL(msm_iommu_detached);
+
diff --git a/drivers/iommu/msm_iommu_sec.c b/drivers/iommu/msm_iommu_sec.c
new file mode 100644
index 0000000..74d8b48
--- /dev/null
+++ b/drivers/iommu/msm_iommu_sec.c
@@ -0,0 +1,567 @@
+/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/iommu.h>
+#include <linux/clk.h>
+#include <linux/scatterlist.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/kmemleak.h>
+
+#include <asm/sizes.h>
+
+#include <mach/iommu_perfmon.h>
+#include <mach/iommu_hw-v1.h>
+#include <mach/msm_iommu_priv.h>
+#include <mach/iommu.h>
+#include <mach/scm.h>
+
+/* bitmap of the page sizes currently supported */
+#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
+
+#define IOMMU_SECURE_CFG 2
+#define IOMMU_SECURE_PTBL_SIZE 3
+#define IOMMU_SECURE_PTBL_INIT 4
+#define IOMMU_SECURE_MAP 6
+#define IOMMU_SECURE_UNMAP 7
+#define IOMMU_SECURE_MAP2 0x0B
+#define IOMMU_SECURE_UNMAP2 0x0C
+#define IOMMU_TLBINVAL_FLAG 0x00000001
+
+static struct iommu_access_ops *iommu_access_ops;
+
+struct msm_scm_paddr_list {
+ unsigned int list;
+ unsigned int list_size;
+ unsigned int size;
+};
+
+struct msm_scm_mapping_info {
+ unsigned int id;
+ unsigned int ctx_id;
+ unsigned int va;
+ unsigned int size;
+};
+
+struct msm_scm_map2_req {
+ struct msm_scm_paddr_list plist;
+ struct msm_scm_mapping_info info;
+ unsigned int flags;
+};
+
+struct msm_scm_unmap2_req {
+ struct msm_scm_mapping_info info;
+ unsigned int flags;
+};
+
+void msm_iommu_sec_set_access_ops(struct iommu_access_ops *access_ops)
+{
+ iommu_access_ops = access_ops;
+}
+
+static int msm_iommu_sec_ptbl_init(void)
+{
+ struct device_node *np;
+ struct msm_scm_ptbl_init {
+ unsigned int paddr;
+ unsigned int size;
+ unsigned int spare;
+ } pinit;
+ unsigned int *buf;
+ int psize[2] = {0, 0};
+ unsigned int spare;
+ int ret, ptbl_ret = 0;
+
+ for_each_compatible_node(np, NULL, "qcom,msm-smmu-v1")
+ if (of_find_property(np, "qcom,iommu-secure-id", NULL))
+ break;
+
+ if (!np)
+ return 0;
+
+ of_node_put(np);
+ ret = scm_call(SCM_SVC_MP, IOMMU_SECURE_PTBL_SIZE, &spare,
+ sizeof(spare), psize, sizeof(psize));
+ if (ret) {
+ pr_err("scm call IOMMU_SECURE_PTBL_SIZE failed\n");
+ goto fail;
+ }
+
+ if (psize[1]) {
+ pr_err("scm call IOMMU_SECURE_PTBL_SIZE failed\n");
+ goto fail;
+ }
+
+ buf = kmalloc(psize[0], GFP_KERNEL);
+ if (!buf) {
+ pr_err("%s: Failed to allocate %d bytes for PTBL\n",
+ __func__, psize[0]);
+ ret = -ENOMEM;
+ goto fail;
+ }
+
+ pinit.paddr = virt_to_phys(buf);
+ pinit.size = psize[0];
+
+ ret = scm_call(SCM_SVC_MP, IOMMU_SECURE_PTBL_INIT, &pinit,
+ sizeof(pinit), &ptbl_ret, sizeof(ptbl_ret));
+ if (ret) {
+ pr_err("scm call IOMMU_SECURE_PTBL_INIT failed\n");
+ goto fail_mem;
+ }
+ if (ptbl_ret) {
+ pr_err("scm call IOMMU_SECURE_PTBL_INIT extended ret fail\n");
+ goto fail_mem;
+ }
+
+ kmemleak_not_leak(buf);
+
+ return 0;
+
+fail_mem:
+ kfree(buf);
+fail:
+ return ret;
+}
+
+int msm_iommu_sec_program_iommu(int sec_id)
+{
+ struct msm_scm_sec_cfg {
+ unsigned int id;
+ unsigned int spare;
+ } cfg;
+ int ret, scm_ret = 0;
+
+ cfg.id = sec_id;
+
+ ret = scm_call(SCM_SVC_MP, IOMMU_SECURE_CFG, &cfg, sizeof(cfg),
+ &scm_ret, sizeof(scm_ret));
+ if (ret || scm_ret) {
+ pr_err("scm call IOMMU_SECURE_CFG failed\n");
+ return ret ? ret : -EINVAL;
+ }
+
+ return ret;
+}
+
+static int msm_iommu_sec_ptbl_map(struct msm_iommu_drvdata *iommu_drvdata,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata,
+ unsigned long va, phys_addr_t pa, size_t len)
+{
+ struct msm_scm_map2_req map;
+ int ret = 0;
+
+ map.plist.list = virt_to_phys(&pa);
+ map.plist.list_size = 1;
+ map.plist.size = len;
+ map.info.id = iommu_drvdata->sec_id;
+ map.info.ctx_id = ctx_drvdata->num;
+ map.info.va = va;
+ map.info.size = len;
+ map.flags = IOMMU_TLBINVAL_FLAG;
+
+ if (scm_call(SCM_SVC_MP, IOMMU_SECURE_MAP2, &map, sizeof(map), &ret,
+ sizeof(ret)))
+ return -EINVAL;
+ if (ret)
+ return -EINVAL;
+
+ return 0;
+}
+
+static unsigned int get_phys_addr(struct scatterlist *sg)
+{
+ /*
+ * Try sg_dma_address first so that we can
+ * map carveout regions that do not have a
+ * struct page associated with them.
+ */
+ unsigned int pa = sg_dma_address(sg);
+ if (pa == 0)
+ pa = sg_phys(sg);
+ return pa;
+}
+
+static int msm_iommu_sec_ptbl_map_range(struct msm_iommu_drvdata *iommu_drvdata,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata,
+ unsigned long va, struct scatterlist *sg, size_t len)
+{
+ struct scatterlist *sgiter;
+ struct msm_scm_map2_req map;
+ unsigned int *pa_list = 0;
+ unsigned int pa, cnt;
+ unsigned int offset = 0, chunk_offset = 0;
+ int ret, scm_ret;
+
+ map.info.id = iommu_drvdata->sec_id;
+ map.info.ctx_id = ctx_drvdata->num;
+ map.info.va = va;
+ map.info.size = len;
+ map.flags = IOMMU_TLBINVAL_FLAG;
+
+ if (sg->length == len) {
+ pa = get_phys_addr(sg);
+ map.plist.list = virt_to_phys(&pa);
+ map.plist.list_size = 1;
+ map.plist.size = len;
+ } else {
+ sgiter = sg;
+ cnt = sg->length / SZ_1M;
+ while ((sgiter = sg_next(sgiter)))
+ cnt += sgiter->length / SZ_1M;
+
+ pa_list = kmalloc(cnt * sizeof(*pa_list), GFP_KERNEL);
+ if (!pa_list)
+ return -ENOMEM;
+
+ sgiter = sg;
+ cnt = 0;
+ pa = get_phys_addr(sgiter);
+ while (offset < len) {
+ pa += chunk_offset;
+ pa_list[cnt] = pa;
+ chunk_offset += SZ_1M;
+ offset += SZ_1M;
+ cnt++;
+
+ if (chunk_offset >= sgiter->length && offset < len) {
+ chunk_offset = 0;
+ sgiter = sg_next(sgiter);
+ pa = get_phys_addr(sgiter);
+ }
+ }
+
+ map.plist.list = virt_to_phys(pa_list);
+ map.plist.list_size = cnt;
+ map.plist.size = SZ_1M;
+ }
+
+ ret = scm_call(SCM_SVC_MP, IOMMU_SECURE_MAP2, &map, sizeof(map),
+ &scm_ret, sizeof(scm_ret));
+ kfree(pa_list);
+ return ret;
+}
+
+static int msm_iommu_sec_ptbl_unmap(struct msm_iommu_drvdata *iommu_drvdata,
+ struct msm_iommu_ctx_drvdata *ctx_drvdata,
+ unsigned long va, size_t len)
+{
+ struct msm_scm_unmap2_req unmap;
+ int ret, scm_ret;
+
+ unmap.info.id = iommu_drvdata->sec_id;
+ unmap.info.ctx_id = ctx_drvdata->num;
+ unmap.info.va = va;
+ unmap.info.size = len;
+ unmap.flags = IOMMU_TLBINVAL_FLAG;
+
+ ret = scm_call(SCM_SVC_MP, IOMMU_SECURE_UNMAP2, &unmap, sizeof(unmap),
+ &scm_ret, sizeof(scm_ret));
+ return ret;
+}
+
+static int msm_iommu_domain_init(struct iommu_domain *domain, int flags)
+{
+ struct msm_iommu_priv *priv;
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&priv->list_attached);
+ domain->priv = priv;
+ return 0;
+}
+
+static void msm_iommu_domain_destroy(struct iommu_domain *domain)
+{
+ struct msm_iommu_priv *priv;
+
+ iommu_access_ops->iommu_lock_acquire();
+ priv = domain->priv;
+ domain->priv = NULL;
+
+ kfree(priv);
+ iommu_access_ops->iommu_lock_release();
+}
+
+static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+ struct msm_iommu_priv *priv;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ struct msm_iommu_ctx_drvdata *tmp_drvdata;
+ int ret = 0;
+
+ iommu_access_ops->iommu_lock_acquire();
+
+ priv = domain->priv;
+ if (!priv || !dev) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ iommu_drvdata = dev_get_drvdata(dev->parent);
+ ctx_drvdata = dev_get_drvdata(dev);
+ if (!iommu_drvdata || !ctx_drvdata) {
+ ret = -EINVAL;
+ goto fail;
+ }
+
+ if (!list_empty(&ctx_drvdata->attached_elm)) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm)
+ if (tmp_drvdata == ctx_drvdata) {
+ ret = -EBUSY;
+ goto fail;
+ }
+
+ ret = iommu_access_ops->iommu_power_on(iommu_drvdata);
+ if (ret)
+ goto fail;
+
+ /* We can only do this once */
+ if (!iommu_drvdata->ctx_attach_count) {
+ ret = iommu_access_ops->iommu_clk_on(iommu_drvdata);
+ if (ret) {
+ iommu_access_ops->iommu_power_off(iommu_drvdata);
+ goto fail;
+ }
+
+ ret = msm_iommu_sec_program_iommu(iommu_drvdata->sec_id);
+
+ /* bfb settings are always programmed by HLOS */
+ program_iommu_bfb_settings(iommu_drvdata->base,
+ iommu_drvdata->bfb_settings);
+
+ iommu_access_ops->iommu_clk_off(iommu_drvdata);
+ if (ret) {
+ iommu_access_ops->iommu_power_off(iommu_drvdata);
+ goto fail;
+ }
+ }
+
+ list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
+ ctx_drvdata->attached_domain = domain;
+ ++iommu_drvdata->ctx_attach_count;
+
+ iommu_access_ops->iommu_lock_release();
+
+ msm_iommu_attached(dev->parent);
+ return ret;
+fail:
+ iommu_access_ops->iommu_lock_release();
+ return ret;
+}
+
+static void msm_iommu_detach_dev(struct iommu_domain *domain,
+ struct device *dev)
+{
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+
+ msm_iommu_detached(dev->parent);
+
+ iommu_access_ops->iommu_lock_acquire();
+ if (!dev)
+ goto fail;
+
+ iommu_drvdata = dev_get_drvdata(dev->parent);
+ ctx_drvdata = dev_get_drvdata(dev);
+ if (!iommu_drvdata || !ctx_drvdata || !ctx_drvdata->attached_domain)
+ goto fail;
+
+ list_del_init(&ctx_drvdata->attached_elm);
+ ctx_drvdata->attached_domain = NULL;
+
+ iommu_access_ops->iommu_power_off(iommu_drvdata);
+ BUG_ON(iommu_drvdata->ctx_attach_count == 0);
+ --iommu_drvdata->ctx_attach_count;
+fail:
+ iommu_access_ops->iommu_lock_release();
+}
+
+static int get_drvdata(struct iommu_domain *domain,
+ struct msm_iommu_drvdata **iommu_drvdata,
+ struct msm_iommu_ctx_drvdata **ctx_drvdata)
+{
+ struct msm_iommu_priv *priv = domain->priv;
+ struct msm_iommu_ctx_drvdata *ctx;
+
+ list_for_each_entry(ctx, &priv->list_attached, attached_elm) {
+ if (ctx->attached_domain == domain)
+ break;
+ }
+
+ if (ctx->attached_domain != domain)
+ return -EINVAL;
+
+ *ctx_drvdata = ctx;
+ *iommu_drvdata = dev_get_drvdata(ctx->pdev->dev.parent);
+ return 0;
+}
+
+static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
+ phys_addr_t pa, size_t len, int prot)
+{
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ int ret = 0;
+
+ iommu_access_ops->iommu_lock_acquire();
+
+ ret = get_drvdata(domain, &iommu_drvdata, &ctx_drvdata);
+ if (ret)
+ goto fail;
+
+ iommu_access_ops->iommu_clk_on(iommu_drvdata);
+ ret = msm_iommu_sec_ptbl_map(iommu_drvdata, ctx_drvdata,
+ va, pa, len);
+ iommu_access_ops->iommu_clk_off(iommu_drvdata);
+fail:
+ iommu_access_ops->iommu_lock_release();
+ return ret;
+}
+
+static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
+ size_t len)
+{
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ int ret = -ENODEV;
+
+ iommu_access_ops->iommu_lock_acquire();
+
+ ret = get_drvdata(domain, &iommu_drvdata, &ctx_drvdata);
+ if (ret)
+ goto fail;
+
+ iommu_access_ops->iommu_clk_on(iommu_drvdata);
+ ret = msm_iommu_sec_ptbl_unmap(iommu_drvdata, ctx_drvdata,
+ va, len);
+ iommu_access_ops->iommu_clk_off(iommu_drvdata);
+fail:
+ iommu_access_ops->iommu_lock_release();
+
+ /* the IOMMU API requires us to return how many bytes were unmapped */
+ len = ret ? 0 : len;
+ return len;
+}
+
+static int msm_iommu_map_range(struct iommu_domain *domain, unsigned int va,
+ struct scatterlist *sg, unsigned int len,
+ int prot)
+{
+ int ret;
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+
+ iommu_access_ops->iommu_lock_acquire();
+
+ ret = get_drvdata(domain, &iommu_drvdata, &ctx_drvdata);
+ if (ret)
+ goto fail;
+ iommu_access_ops->iommu_clk_on(iommu_drvdata);
+ ret = msm_iommu_sec_ptbl_map_range(iommu_drvdata, ctx_drvdata,
+ va, sg, len);
+ iommu_access_ops->iommu_clk_off(iommu_drvdata);
+fail:
+ iommu_access_ops->iommu_lock_release();
+ return ret;
+}
+
+
+static int msm_iommu_unmap_range(struct iommu_domain *domain, unsigned int va,
+ unsigned int len)
+{
+ struct msm_iommu_drvdata *iommu_drvdata;
+ struct msm_iommu_ctx_drvdata *ctx_drvdata;
+ int ret;
+
+ iommu_access_ops->iommu_lock_acquire();
+
+ ret = get_drvdata(domain, &iommu_drvdata, &ctx_drvdata);
+ if (ret)
+ goto fail;
+
+ iommu_access_ops->iommu_clk_on(iommu_drvdata);
+ ret = msm_iommu_sec_ptbl_unmap(iommu_drvdata, ctx_drvdata, va, len);
+ iommu_access_ops->iommu_clk_off(iommu_drvdata);
+
+fail:
+ iommu_access_ops->iommu_lock_release();
+ return 0;
+}
+
+static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
+ unsigned long va)
+{
+ return 0;
+}
+
+static int msm_iommu_domain_has_cap(struct iommu_domain *domain,
+ unsigned long cap)
+{
+ return 0;
+}
+
+static phys_addr_t msm_iommu_get_pt_base_addr(struct iommu_domain *domain)
+{
+ return 0;
+}
+
+static struct iommu_ops msm_iommu_ops = {
+ .domain_init = msm_iommu_domain_init,
+ .domain_destroy = msm_iommu_domain_destroy,
+ .attach_dev = msm_iommu_attach_dev,
+ .detach_dev = msm_iommu_detach_dev,
+ .map = msm_iommu_map,
+ .unmap = msm_iommu_unmap,
+ .map_range = msm_iommu_map_range,
+ .unmap_range = msm_iommu_unmap_range,
+ .iova_to_phys = msm_iommu_iova_to_phys,
+ .domain_has_cap = msm_iommu_domain_has_cap,
+ .get_pt_base_addr = msm_iommu_get_pt_base_addr,
+ .pgsize_bitmap = MSM_IOMMU_PGSIZES,
+};
+
+static int __init msm_iommu_sec_init(void)
+{
+ int ret;
+
+ ret = bus_register(&msm_iommu_sec_bus_type);
+ if (ret)
+ goto fail;
+
+ bus_set_iommu(&msm_iommu_sec_bus_type, &msm_iommu_ops);
+ ret = msm_iommu_sec_ptbl_init();
+fail:
+ return ret;
+}
+
+subsys_initcall(msm_iommu_sec_init);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MSM SMMU Secure Driver");