gpu: ion: Delay unmapping from IOMMU.
Due to limitations in the multimedia architecure
clients might not know when a buffer can be unmapped
from the IOMMU. In addition, the multimedia architecture
causes unnecessary mappings/unmappings for the same buffers
which reduces framerates.
Add logic to delay unmapping from the IOMMU
until buffer memory is freed and unmap any outstanding
mappings to avoid virtual memory address space leak.
Change-Id: Idaeae269d9ba623e25a0cb087a89b4cbb63915af
Signed-off-by: Olav Haugan <ohaugan@codeaurora.org>
diff --git a/drivers/gpu/ion/ion.c b/drivers/gpu/ion/ion.c
index 9b1e548..0cec423 100644
--- a/drivers/gpu/ion/ion.c
+++ b/drivers/gpu/ion/ion.c
@@ -107,6 +107,8 @@
unsigned int iommu_map_cnt;
};
+static void ion_iommu_release(struct kref *kref);
+
static int ion_validate_buffer_flags(struct ion_buffer *buffer,
unsigned long flags)
{
@@ -235,11 +237,44 @@
return buffer;
}
+/**
+ * Check for delayed IOMMU unmapping. Also unmap any outstanding
+ * mappings which would otherwise have been leaked.
+ */
+static void ion_iommu_delayed_unmap(struct ion_buffer *buffer)
+{
+ struct ion_iommu_map *iommu_map;
+ struct rb_node *node;
+ const struct rb_root *rb = &(buffer->iommu_maps);
+ unsigned long ref_count;
+ unsigned int delayed_unmap;
+
+ mutex_lock(&buffer->lock);
+
+ while ((node = rb_first(rb)) != 0) {
+ iommu_map = rb_entry(node, struct ion_iommu_map, node);
+ ref_count = atomic_read(&iommu_map->ref.refcount);
+ delayed_unmap = iommu_map->flags & ION_IOMMU_UNMAP_DELAYED;
+
+ if ((delayed_unmap && ref_count > 1) || !delayed_unmap) {
+ pr_err("%s: Virtual memory address leak in domain %u, partition %u\n",
+ __func__, iommu_map->domain_info[DI_DOMAIN_NUM],
+ iommu_map->domain_info[DI_PARTITION_NUM]);
+ }
+ /* set ref count to 1 to force release */
+ kref_init(&iommu_map->ref);
+ kref_put(&iommu_map->ref, ion_iommu_release);
+ }
+
+ mutex_unlock(&buffer->lock);
+}
+
static void ion_buffer_destroy(struct kref *kref)
{
struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);
struct ion_device *dev = buffer->dev;
+ ion_iommu_delayed_unmap(buffer);
buffer->heap->ops->free(buffer);
mutex_lock(&dev->lock);
rb_erase(&buffer->node, &dev->buffers);
@@ -562,7 +597,7 @@
}
EXPORT_SYMBOL(ion_map_kernel);
-static int __ion_iommu_map(struct ion_buffer *buffer,
+static struct ion_iommu_map *__ion_iommu_map(struct ion_buffer *buffer,
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, unsigned long flags,
unsigned long *iova)
@@ -573,7 +608,7 @@
data = kmalloc(sizeof(*data), GFP_ATOMIC);
if (!data)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
data->buffer = buffer;
iommu_map_domain(data) = domain_num;
@@ -594,20 +629,20 @@
ion_iommu_add(buffer, data);
- return 0;
+ return data;
out:
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
buffer->size);
kfree(data);
- return ret;
+ return ERR_PTR(ret);
}
int ion_map_iommu(struct ion_client *client, struct ion_handle *handle,
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, unsigned long *iova,
unsigned long *buffer_size,
- unsigned long flags)
+ unsigned long flags, unsigned long iommu_flags)
{
struct ion_buffer *buffer;
struct ion_iommu_map *iommu_map;
@@ -665,17 +700,30 @@
}
iommu_map = ion_iommu_lookup(buffer, domain_num, partition_num);
- if (_ion_map(&buffer->iommu_map_cnt, &handle->iommu_map_cnt) ||
- !iommu_map) {
- ret = __ion_iommu_map(buffer, domain_num, partition_num, align,
- iova_length, flags, iova);
- if (ret < 0)
+ _ion_map(&buffer->iommu_map_cnt, &handle->iommu_map_cnt);
+ if (!iommu_map) {
+ iommu_map = __ion_iommu_map(buffer, domain_num, partition_num,
+ align, iova_length, flags, iova);
+ if (IS_ERR_OR_NULL(iommu_map)) {
_ion_unmap(&buffer->iommu_map_cnt,
&handle->iommu_map_cnt);
+ } else {
+ iommu_map->flags = iommu_flags;
+
+ if (iommu_map->flags & ION_IOMMU_UNMAP_DELAYED)
+ kref_get(&iommu_map->ref);
+ }
} else {
- if (iommu_map->mapped_size != iova_length) {
+ if (iommu_map->flags != iommu_flags) {
+ pr_err("%s: handle %p is already mapped with iommu flags %lx, trying to map with flags %lx\n",
+ __func__, handle,
+ iommu_map->flags, iommu_flags);
+ _ion_unmap(&buffer->iommu_map_cnt,
+ &handle->iommu_map_cnt);
+ ret = -EINVAL;
+ } else if (iommu_map->mapped_size != iova_length) {
pr_err("%s: handle %p is already mapped with length"
- " %x, trying to map with length %lx\n",
+ " %x, trying to map with length %lx\n",
__func__, handle, iommu_map->mapped_size,
iova_length);
_ion_unmap(&buffer->iommu_map_cnt,
diff --git a/drivers/gpu/ion/ion_priv.h b/drivers/gpu/ion/ion_priv.h
index 36758fa..98e11cf 100644
--- a/drivers/gpu/ion/ion_priv.h
+++ b/drivers/gpu/ion/ion_priv.h
@@ -37,6 +37,12 @@
void *vaddr;
};
+enum {
+ DI_PARTITION_NUM = 0,
+ DI_DOMAIN_NUM = 1,
+ DI_MAX,
+};
+
/**
* struct ion_iommu_map - represents a mapping of an ion buffer to an iommu
* @iova_addr - iommu virtual address
@@ -47,6 +53,7 @@
* @ref - for reference counting this mapping
* @mapped_size - size of the iova space mapped
* (may not be the same as the buffer size)
+ * @flags - iommu domain/partition specific flags.
*
* Represents a mapping of one ion buffer to a particular iommu domain
* and address range. There may exist other mappings of this buffer in
@@ -57,12 +64,13 @@
unsigned long iova_addr;
struct rb_node node;
union {
- int domain_info[2];
+ int domain_info[DI_MAX];
uint64_t key;
};
struct ion_buffer *buffer;
struct kref ref;
int mapped_size;
+ unsigned long flags;
};
struct ion_buffer *ion_handle_buffer(struct ion_handle *handle);
diff --git a/include/linux/ion.h b/include/linux/ion.h
index ca1f198..d761e1e 100644
--- a/include/linux/ion.h
+++ b/include/linux/ion.h
@@ -110,6 +110,12 @@
#define ION_IS_CACHED(__flags) ((__flags) & (1 << ION_CACHE_SHIFT))
+/*
+ * This flag allows clients when mapping into the IOMMU to specify to
+ * defer un-mapping from the IOMMU until the buffer memory is freed.
+ */
+#define ION_IOMMU_UNMAP_DELAYED 1
+
#ifdef __KERNEL__
#include <linux/err.h>
#include <mach/ion.h>
@@ -407,6 +413,7 @@
* @iova - pointer to store the iova address
* @buffer_size - pointer to store the size of the buffer
* @flags - flags for options to map
+ * @iommu_flags - flags specific to the iommu.
*
* Maps the handle into the iova space specified via domain number. Iova
* will be allocated from the partition specified via partition_num.
@@ -416,7 +423,7 @@
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, unsigned long *iova,
unsigned long *buffer_size,
- unsigned long flags);
+ unsigned long flags, unsigned long iommu_flags);
/**
@@ -588,7 +595,8 @@
int partition_num, unsigned long align,
unsigned long iova_length, unsigned long *iova,
unsigned long *buffer_size,
- unsigned long flags)
+ unsigned long flags,
+ unsigned long iommu_flags)
{
return -ENODEV;
}