msm: kgsl: Introduce Iommu Code
kgsl now supports the use of 2 types of MMU. One is
the GPU's internal MMU and the other is IOMMU. Both
MMU cannot be active at the same time. The MMU type
can be selected at compile time via config option.
A boot command line parameter can be used to override
the type of MMU selected at compile time.
Signed-off-by: Shubhraprakash Das <sadas@codeaurora.org>
diff --git a/drivers/gpu/msm/Kconfig b/drivers/gpu/msm/Kconfig
index 64cbc30..5852e26 100644
--- a/drivers/gpu/msm/Kconfig
+++ b/drivers/gpu/msm/Kconfig
@@ -64,22 +64,30 @@
bool "Build a DRM interface for the MSM_KGSL driver"
depends on MSM_KGSL && DRM
-config MSM_KGSL_MMU
+config MSM_KGSL_GPUMMU
bool "Enable the GPU MMU in the MSM_KGSL driver"
- depends on MSM_KGSL && MMU && !MSM_KGSL_CFF_DUMP
+ depends on MSM_KGSL && !MSM_KGSL_CFF_DUMP
+ default y
+
+config MSM_KGSL_IOMMU
+ bool "Enable the use of IOMMU in the MSM_KGSL driver"
+ depends on MSM_KGSL && MSM_IOMMU && !MSM_KGSL_GPUMMU && !MSM_KGSL_CFF_DUMP
+
+config MSM_KGSL_MMU
+ bool
+ depends on MSM_KGSL_GPUMMU || MSM_KGSL_IOMMU
default y
config KGSL_PER_PROCESS_PAGE_TABLE
bool "Enable Per Process page tables for the KGSL driver"
default n
- depends on MSM_KGSL_MMU && !MSM_KGSL_DRM
+ depends on MSM_KGSL_GPUMMU && !MSM_KGSL_DRM
---help---
The MMU will use per process pagetables when enabled.
config MSM_KGSL_PAGE_TABLE_SIZE
hex "Size of pagetables"
default 0xFFF0000
- depends on MSM_KGSL_MMU
---help---
Sets the pagetable size used by the MMU. The max value
is 0xFFF0000 or (256M - 64K).
@@ -97,7 +105,7 @@
config MSM_KGSL_MMU_PAGE_FAULT
bool "Force the GPU MMU to page fault for unmapped regions"
default y
- depends on MSM_KGSL_MMU
+ depends on MSM_KGSL_GPUMMU
config MSM_KGSL_DISABLE_SHADOW_WRITES
bool "Disable register shadow writes for context switches"
diff --git a/drivers/gpu/msm/Makefile b/drivers/gpu/msm/Makefile
index 5b36c62..b4cd286 100644
--- a/drivers/gpu/msm/Makefile
+++ b/drivers/gpu/msm/Makefile
@@ -4,10 +4,12 @@
kgsl.o \
kgsl_sharedmem.o \
kgsl_pwrctrl.o \
- kgsl_pwrscale.o
+ kgsl_pwrscale.o \
+ kgsl_mmu.o \
+ kgsl_gpummu.o \
+ kgsl_iommu.o
msm_kgsl_core-$(CONFIG_DEBUG_FS) += kgsl_debugfs.o
-msm_kgsl_core-$(CONFIG_MSM_KGSL_MMU) += kgsl_mmu.o
msm_kgsl_core-$(CONFIG_MSM_KGSL_CFF_DUMP) += kgsl_cffdump.o
msm_kgsl_core-$(CONFIG_MSM_KGSL_DRM) += kgsl_drm.o
msm_kgsl_core-$(CONFIG_MSM_SCM) += kgsl_pwrscale_trustzone.o
diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c
index cd9a4b3..247abf5 100644
--- a/drivers/gpu/msm/adreno.c
+++ b/drivers/gpu/msm/adreno.c
@@ -29,6 +29,7 @@
#include "adreno_postmortem.h"
#include "a2xx_reg.h"
+#include "kgsl_mmu.h"
#define DRIVER_VERSION_MAJOR 3
#define DRIVER_VERSION_MINOR 1
@@ -204,7 +205,7 @@
kgsl_mmu_unmap(pagetable, &device->memstore);
- kgsl_mmu_unmap(pagetable, &device->mmu.dummyspace);
+ kgsl_mmu_unmap(pagetable, &device->mmu.setstate_memory);
}
static int adreno_setup_pt(struct kgsl_device *device,
@@ -229,7 +230,7 @@
if (result)
goto unmap_memptrs_desc;
- result = kgsl_mmu_map_global(pagetable, &device->mmu.dummyspace,
+ result = kgsl_mmu_map_global(pagetable, &device->mmu.setstate_memory,
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (result)
goto unmap_memstore_desc;
@@ -249,7 +250,8 @@
return result;
}
-static void adreno_setstate(struct kgsl_device *device, uint32_t flags)
+static void adreno_setstate(struct kgsl_device *device,
+ uint32_t flags)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int link[32];
@@ -257,13 +259,9 @@
int sizedwords = 0;
unsigned int mh_mmu_invalidate = 0x00000003; /*invalidate all and tc */
- if (!kgsl_mmu_enabled() || !flags)
- return;
-
/* If possible, then set the state via the command stream to avoid
a CPU idle. Otherwise, use the default setstate which uses register
writes */
-
if (adreno_dev->drawctxt_active) {
if (flags & KGSL_MMUFLAGS_PTUPDATE) {
/* wait for graphics pipe to be idle */
@@ -272,7 +270,8 @@
/* set page table base */
*cmds++ = cp_type0_packet(MH_MMU_PT_BASE, 1);
- *cmds++ = device->mmu.hwpagetable->base.gpuaddr;
+ *cmds++ = kgsl_pt_get_base_addr(
+ device->mmu.hwpagetable);
sizedwords += 4;
}
@@ -307,13 +306,14 @@
(REG_PA_SU_SC_MODE_CNTL - 0x2000);
*cmds++ = 0; /* disable faceness generation */
*cmds++ = cp_type3_packet(CP_SET_BIN_BASE_OFFSET, 1);
- *cmds++ = device->mmu.dummyspace.gpuaddr;
+ *cmds++ = device->mmu.setstate_memory.gpuaddr;
*cmds++ = cp_type3_packet(CP_DRAW_INDX_BIN, 6);
*cmds++ = 0; /* viz query info */
*cmds++ = 0x0003C004; /* draw indicator */
*cmds++ = 0; /* bin base */
*cmds++ = 3; /* bin size */
- *cmds++ = device->mmu.dummyspace.gpuaddr; /* dma base */
+ *cmds++ =
+ device->mmu.setstate_memory.gpuaddr; /* dma base */
*cmds++ = 6; /* dma size */
*cmds++ = cp_type3_packet(CP_DRAW_INDX_BIN, 6);
*cmds++ = 0; /* viz query info */
@@ -321,13 +321,14 @@
*cmds++ = 0; /* bin base */
*cmds++ = 3; /* bin size */
/* dma base */
- *cmds++ = device->mmu.dummyspace.gpuaddr;
+ *cmds++ = device->mmu.setstate_memory.gpuaddr;
*cmds++ = 6; /* dma size */
*cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1);
*cmds++ = 0x00000000;
sizedwords += 21;
}
+
if (flags & (KGSL_MMUFLAGS_PTUPDATE | KGSL_MMUFLAGS_TLBFLUSH)) {
*cmds++ = cp_type3_packet(CP_INVALIDATE_STATE, 1);
*cmds++ = 0x7fff; /* invalidate all base pointers */
@@ -336,8 +337,9 @@
adreno_ringbuffer_issuecmds(device, KGSL_CMD_FLAGS_PMODE,
&link[0], sizedwords);
- } else
- kgsl_default_setstate(device, flags);
+ } else {
+ kgsl_mmu_device_setstate(device, flags);
+ }
}
static unsigned int
@@ -794,16 +796,13 @@
break;
case KGSL_PROP_MMU_ENABLE:
{
-#ifdef CONFIG_MSM_KGSL_MMU
- int mmuProp = 1;
-#else
- int mmuProp = 0;
-#endif
+ int mmu_prop = kgsl_mmu_enabled();
+
if (sizebytes != sizeof(int)) {
status = -EINVAL;
break;
}
- if (copy_to_user(value, &mmuProp, sizeof(mmuProp))) {
+ if (copy_to_user(value, &mmu_prop, sizeof(mmu_prop))) {
status = -EFAULT;
break;
}
@@ -938,12 +937,8 @@
mutex_lock(&kgsl_driver.process_mutex);
list_for_each_entry(priv, &kgsl_driver.process_list, list) {
- if (pt_base != 0
- && priv->pagetable
- && priv->pagetable->base.gpuaddr != pt_base) {
+ if (!kgsl_mmu_pt_equal(priv->pagetable, pt_base))
continue;
- }
-
spin_lock(&priv->mem_lock);
entry = kgsl_sharedmem_find_region(priv, gpuaddr,
sizeof(unsigned int));
diff --git a/drivers/gpu/msm/adreno_postmortem.c b/drivers/gpu/msm/adreno_postmortem.c
index 9fe0bb6..7c7f0dc 100644
--- a/drivers/gpu/msm/adreno_postmortem.c
+++ b/drivers/gpu/msm/adreno_postmortem.c
@@ -640,7 +640,7 @@
kgsl_regread(device, MH_MMU_MPU_END, &r1);
kgsl_regread(device, MH_MMU_VA_RANGE, &r2);
- kgsl_regread(device, MH_MMU_PT_BASE, &pt_base);
+ pt_base = kgsl_mmu_get_current_ptbase(device);
KGSL_LOG_DUMP(device,
" MPU_END = %08X | VA_RANGE = %08X | PT_BASE ="
" %08X\n", r1, r2, pt_base);
@@ -719,7 +719,7 @@
KGSL_LOG_DUMP(device, "Current pagetable: %x\t"
"pagetable base: %x\n",
- kgsl_get_ptname_from_ptbase(cur_pt_base),
+ kgsl_mmu_get_ptname_from_ptbase(cur_pt_base),
cur_pt_base);
/* Set cur_pt_base to the new pagetable base */
@@ -727,7 +727,7 @@
KGSL_LOG_DUMP(device, "New pagetable: %x\t"
"pagetable base: %x\n",
- kgsl_get_ptname_from_ptbase(cur_pt_base),
+ kgsl_mmu_get_ptname_from_ptbase(cur_pt_base),
cur_pt_base);
}
}
diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c
index 4eb3bdd0..639f219 100644
--- a/drivers/gpu/msm/adreno_ringbuffer.c
+++ b/drivers/gpu/msm/adreno_ringbuffer.c
@@ -601,7 +601,7 @@
}
kgsl_setstate(device,
- kgsl_pt_get_flags(device->mmu.hwpagetable,
+ kgsl_mmu_pt_get_flags(device->mmu.hwpagetable,
device->id));
adreno_drawctxt_switch(adreno_dev, drawctxt, flags);
diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c
index ad9d432..f4cff77 100644
--- a/drivers/gpu/msm/kgsl.c
+++ b/drivers/gpu/msm/kgsl.c
@@ -35,9 +35,13 @@
#define MODULE_PARAM_PREFIX "kgsl."
static int kgsl_pagetable_count = KGSL_PAGETABLE_COUNT;
+static char *ksgl_mmu_type;
module_param_named(ptcount, kgsl_pagetable_count, int, 0);
MODULE_PARM_DESC(kgsl_pagetable_count,
"Minimum number of pagetables for KGSL to allocate at initialization time");
+module_param_named(mmutype, ksgl_mmu_type, charp, 0);
+MODULE_PARM_DESC(ksgl_mmu_type,
+"Type of MMU to be used for graphics. Valid values are 'iommu' or 'gpummu' or 'nommu'");
static inline struct kgsl_mem_entry *
kgsl_mem_entry_create(void)
@@ -479,15 +483,11 @@
INIT_LIST_HEAD(&private->mem_list);
-#ifdef CONFIG_MSM_KGSL_MMU
+ if (kgsl_mmu_enabled())
{
unsigned long pt_name;
-#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE
pt_name = task_tgid_nr(current);
-#else
- pt_name = KGSL_MMU_GLOBAL_PT;
-#endif
private->pagetable = kgsl_mmu_getpagetable(pt_name);
if (private->pagetable == NULL) {
kfree(private);
@@ -495,7 +495,6 @@
goto out;
}
}
-#endif
list_add(&private->list, &kgsl_driver.process_list);
@@ -2065,17 +2064,19 @@
static int __devinit
kgsl_ptdata_init(void)
{
- INIT_LIST_HEAD(&kgsl_driver.pagetable_list);
-
- return kgsl_ptpool_init(&kgsl_driver.ptpool, KGSL_PAGETABLE_SIZE,
- kgsl_pagetable_count);
+ kgsl_driver.ptpool = kgsl_mmu_ptpool_init(KGSL_PAGETABLE_SIZE,
+ kgsl_pagetable_count);
+ if (!kgsl_driver.ptpool)
+ return -ENOMEM;
+ return 0;
}
static void kgsl_core_exit(void)
{
unregister_chrdev_region(kgsl_driver.major, KGSL_DEVICE_MAX);
- kgsl_ptpool_destroy(&kgsl_driver.ptpool);
+ kgsl_mmu_ptpool_destroy(&kgsl_driver.ptpool);
+ kgsl_driver.ptpool = NULL;
device_unregister(&kgsl_driver.virtdev);
@@ -2091,7 +2092,6 @@
static int __init kgsl_core_init(void)
{
int result = 0;
-
/* alloc major and minor device numbers */
result = alloc_chrdev_region(&kgsl_driver.major, 0, KGSL_DEVICE_MAX,
KGSL_NAME);
@@ -2147,15 +2147,23 @@
INIT_LIST_HEAD(&kgsl_driver.process_list);
- result = kgsl_ptdata_init();
- if (result)
- goto err;
+ INIT_LIST_HEAD(&kgsl_driver.pagetable_list);
+
+ kgsl_mmu_set_mmutype(ksgl_mmu_type);
+
+ if (KGSL_MMU_TYPE_GPU == kgsl_mmu_get_mmutype()) {
+ result = kgsl_ptdata_init();
+ if (result)
+ goto err;
+ }
result = kgsl_drm_init(NULL);
if (result)
goto err;
+ kgsl_mmu_set_mmutype(ksgl_mmu_type);
+
return 0;
err:
diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h
index ad016e0..8db2cb4 100644
--- a/drivers/gpu/msm/kgsl.h
+++ b/drivers/gpu/msm/kgsl.h
@@ -40,13 +40,9 @@
#define KGSL_PAGETABLE_ENTRIES(_sz) (((_sz) >> PAGE_SHIFT) + \
KGSL_PT_EXTRA_ENTRIES)
-#ifdef CONFIG_MSM_KGSL_MMU
#define KGSL_PAGETABLE_SIZE \
ALIGN(KGSL_PAGETABLE_ENTRIES(CONFIG_MSM_KGSL_PAGE_TABLE_SIZE) * \
KGSL_PAGETABLE_ENTRY_SIZE, PAGE_SIZE)
-#else
-#define KGSL_PAGETABLE_SIZE 0
-#endif
#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE
#define KGSL_PAGETABLE_COUNT (CONFIG_MSM_KGSL_PAGE_TABLE_COUNT)
@@ -67,15 +63,6 @@
struct kgsl_device;
-struct kgsl_ptpool {
- size_t ptsize;
- struct mutex lock;
- struct list_head list;
- int entries;
- int static_entries;
- int chunks;
-};
-
struct kgsl_driver {
struct cdev cdev;
dev_t major;
@@ -99,7 +86,7 @@
/* Mutex for protecting the device list */
struct mutex devlock;
- struct kgsl_ptpool ptpool;
+ void *ptpool;
struct {
unsigned int vmalloc;
diff --git a/drivers/gpu/msm/kgsl_gpummu.c b/drivers/gpu/msm/kgsl_gpummu.c
new file mode 100644
index 0000000..383b910
--- /dev/null
+++ b/drivers/gpu/msm/kgsl_gpummu.c
@@ -0,0 +1,787 @@
+/* Copyright (c) 2011, Code Aurora Forum. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+#include <linux/genalloc.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+
+#include "kgsl.h"
+#include "kgsl_mmu.h"
+#include "kgsl_device.h"
+#include "kgsl_sharedmem.h"
+
+static ssize_t
+sysfs_show_ptpool_entries(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct kgsl_ptpool *pool = (struct kgsl_ptpool *)
+ kgsl_driver.ptpool;
+ return snprintf(buf, PAGE_SIZE, "%d\n", pool->entries);
+}
+
+static ssize_t
+sysfs_show_ptpool_min(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct kgsl_ptpool *pool = (struct kgsl_ptpool *)
+ kgsl_driver.ptpool;
+ return snprintf(buf, PAGE_SIZE, "%d\n",
+ pool->static_entries);
+}
+
+static ssize_t
+sysfs_show_ptpool_chunks(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct kgsl_ptpool *pool = (struct kgsl_ptpool *)
+ kgsl_driver.ptpool;
+ return snprintf(buf, PAGE_SIZE, "%d\n", pool->chunks);
+}
+
+static ssize_t
+sysfs_show_ptpool_ptsize(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ struct kgsl_ptpool *pool = (struct kgsl_ptpool *)
+ kgsl_driver.ptpool;
+ return snprintf(buf, PAGE_SIZE, "%d\n", pool->ptsize);
+}
+
+static struct kobj_attribute attr_ptpool_entries = {
+ .attr = { .name = "ptpool_entries", .mode = 0444 },
+ .show = sysfs_show_ptpool_entries,
+ .store = NULL,
+};
+
+static struct kobj_attribute attr_ptpool_min = {
+ .attr = { .name = "ptpool_min", .mode = 0444 },
+ .show = sysfs_show_ptpool_min,
+ .store = NULL,
+};
+
+static struct kobj_attribute attr_ptpool_chunks = {
+ .attr = { .name = "ptpool_chunks", .mode = 0444 },
+ .show = sysfs_show_ptpool_chunks,
+ .store = NULL,
+};
+
+static struct kobj_attribute attr_ptpool_ptsize = {
+ .attr = { .name = "ptpool_ptsize", .mode = 0444 },
+ .show = sysfs_show_ptpool_ptsize,
+ .store = NULL,
+};
+
+static struct attribute *ptpool_attrs[] = {
+ &attr_ptpool_entries.attr,
+ &attr_ptpool_min.attr,
+ &attr_ptpool_chunks.attr,
+ &attr_ptpool_ptsize.attr,
+ NULL,
+};
+
+static struct attribute_group ptpool_attr_group = {
+ .attrs = ptpool_attrs,
+};
+
+static int
+_kgsl_ptpool_add_entries(struct kgsl_ptpool *pool, int count, int dynamic)
+{
+ struct kgsl_ptpool_chunk *chunk;
+ size_t size = ALIGN(count * pool->ptsize, PAGE_SIZE);
+
+ BUG_ON(count == 0);
+
+ if (get_order(size) >= MAX_ORDER) {
+ KGSL_CORE_ERR("ptpool allocation is too big: %d\n", size);
+ return -EINVAL;
+ }
+
+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
+ if (chunk == NULL) {
+ KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(*chunk));
+ return -ENOMEM;
+ }
+
+ chunk->size = size;
+ chunk->count = count;
+ chunk->dynamic = dynamic;
+
+ chunk->data = dma_alloc_coherent(NULL, size,
+ &chunk->phys, GFP_KERNEL);
+
+ if (chunk->data == NULL) {
+ KGSL_CORE_ERR("dma_alloc_coherent(%d) failed\n", size);
+ goto err;
+ }
+
+ chunk->bitmap = kzalloc(BITS_TO_LONGS(count) * 4, GFP_KERNEL);
+
+ if (chunk->bitmap == NULL) {
+ KGSL_CORE_ERR("kzalloc(%d) failed\n",
+ BITS_TO_LONGS(count) * 4);
+ goto err_dma;
+ }
+
+ list_add_tail(&chunk->list, &pool->list);
+
+ pool->chunks++;
+ pool->entries += count;
+
+ if (!dynamic)
+ pool->static_entries += count;
+
+ return 0;
+
+err_dma:
+ dma_free_coherent(NULL, chunk->size, chunk->data, chunk->phys);
+err:
+ kfree(chunk);
+ return -ENOMEM;
+}
+
+static void *
+_kgsl_ptpool_get_entry(struct kgsl_ptpool *pool, unsigned int *physaddr)
+{
+ struct kgsl_ptpool_chunk *chunk;
+
+ list_for_each_entry(chunk, &pool->list, list) {
+ int bit = find_first_zero_bit(chunk->bitmap, chunk->count);
+
+ if (bit >= chunk->count)
+ continue;
+
+ set_bit(bit, chunk->bitmap);
+ *physaddr = chunk->phys + (bit * pool->ptsize);
+
+ return chunk->data + (bit * pool->ptsize);
+ }
+
+ return NULL;
+}
+
+/**
+ * kgsl_ptpool_add
+ * @pool: A pointer to a ptpool structure
+ * @entries: Number of entries to add
+ *
+ * Add static entries to the pagetable pool.
+ */
+
+static int
+kgsl_ptpool_add(struct kgsl_ptpool *pool, int count)
+{
+ int ret = 0;
+ BUG_ON(count == 0);
+
+ mutex_lock(&pool->lock);
+
+ /* Only 4MB can be allocated in one chunk, so larger allocations
+ need to be split into multiple sections */
+
+ while (count) {
+ int entries = ((count * pool->ptsize) > SZ_4M) ?
+ SZ_4M / pool->ptsize : count;
+
+ /* Add the entries as static, i.e. they don't ever stand
+ a chance of being removed */
+
+ ret = _kgsl_ptpool_add_entries(pool, entries, 0);
+ if (ret)
+ break;
+
+ count -= entries;
+ }
+
+ mutex_unlock(&pool->lock);
+ return ret;
+}
+
+/**
+ * kgsl_ptpool_alloc
+ * @pool: A pointer to a ptpool structure
+ * @addr: A pointer to store the physical address of the chunk
+ *
+ * Allocate a pagetable from the pool. Returns the virtual address
+ * of the pagetable, the physical address is returned in physaddr
+ */
+
+static void *kgsl_ptpool_alloc(struct kgsl_ptpool *pool,
+ unsigned int *physaddr)
+{
+ void *addr = NULL;
+ int ret;
+
+ mutex_lock(&pool->lock);
+ addr = _kgsl_ptpool_get_entry(pool, physaddr);
+ if (addr)
+ goto done;
+
+ /* Add a chunk for 1 more pagetable and mark it as dynamic */
+ ret = _kgsl_ptpool_add_entries(pool, 1, 1);
+
+ if (ret)
+ goto done;
+
+ addr = _kgsl_ptpool_get_entry(pool, physaddr);
+done:
+ mutex_unlock(&pool->lock);
+ return addr;
+}
+
+static inline void _kgsl_ptpool_rm_chunk(struct kgsl_ptpool_chunk *chunk)
+{
+ list_del(&chunk->list);
+
+ if (chunk->data)
+ dma_free_coherent(NULL, chunk->size, chunk->data,
+ chunk->phys);
+ kfree(chunk->bitmap);
+ kfree(chunk);
+}
+
+/**
+ * kgsl_ptpool_free
+ * @pool: A pointer to a ptpool structure
+ * @addr: A pointer to the virtual address to free
+ *
+ * Free a pagetable allocated from the pool
+ */
+
+static void kgsl_ptpool_free(struct kgsl_ptpool *pool, void *addr)
+{
+ struct kgsl_ptpool_chunk *chunk, *tmp;
+
+ if (pool == NULL || addr == NULL)
+ return;
+
+ mutex_lock(&pool->lock);
+ list_for_each_entry_safe(chunk, tmp, &pool->list, list) {
+ if (addr >= chunk->data &&
+ addr < chunk->data + chunk->size) {
+ int bit = ((unsigned long) (addr - chunk->data)) /
+ pool->ptsize;
+
+ clear_bit(bit, chunk->bitmap);
+ memset(addr, 0, pool->ptsize);
+
+ if (chunk->dynamic &&
+ bitmap_empty(chunk->bitmap, chunk->count))
+ _kgsl_ptpool_rm_chunk(chunk);
+
+ break;
+ }
+ }
+
+ mutex_unlock(&pool->lock);
+}
+
+void kgsl_gpummu_ptpool_destroy(void *ptpool)
+{
+ struct kgsl_ptpool *pool = (struct kgsl_ptpool *)ptpool;
+ struct kgsl_ptpool_chunk *chunk, *tmp;
+
+ if (pool == NULL)
+ return;
+
+ mutex_lock(&pool->lock);
+ list_for_each_entry_safe(chunk, tmp, &pool->list, list)
+ _kgsl_ptpool_rm_chunk(chunk);
+ mutex_unlock(&pool->lock);
+
+ kfree(pool);
+}
+
+/**
+ * kgsl_ptpool_init
+ * @pool: A pointer to a ptpool structure to initialize
+ * @ptsize: The size of each pagetable entry
+ * @entries: The number of inital entries to add to the pool
+ *
+ * Initalize a pool and allocate an initial chunk of entries.
+ */
+void *kgsl_gpummu_ptpool_init(int ptsize, int entries)
+{
+ struct kgsl_ptpool *pool;
+ int ret = 0;
+ BUG_ON(ptsize == 0);
+
+ pool = kzalloc(sizeof(struct kgsl_ptpool), GFP_KERNEL);
+ if (!pool) {
+ KGSL_CORE_ERR("Failed to allocate memory "
+ "for ptpool\n");
+ return NULL;
+ }
+
+ pool->ptsize = ptsize;
+ mutex_init(&pool->lock);
+ INIT_LIST_HEAD(&pool->list);
+
+ if (entries) {
+ ret = kgsl_ptpool_add(pool, entries);
+ if (ret)
+ goto err_ptpool_remove;
+ }
+
+ ret = sysfs_create_group(kgsl_driver.ptkobj, &ptpool_attr_group);
+ if (ret) {
+ KGSL_CORE_ERR("sysfs_create_group failed for ptpool "
+ "statistics: %d\n", ret);
+ goto err_ptpool_remove;
+ }
+ return (void *)pool;
+
+err_ptpool_remove:
+ kgsl_gpummu_ptpool_destroy(pool);
+ return NULL;
+}
+
+int kgsl_gpummu_pt_equal(struct kgsl_pagetable *pt,
+ unsigned int pt_base)
+{
+ struct kgsl_gpummu_pt *gpummu_pt = pt->priv;
+ return pt && pt_base && (gpummu_pt->base.gpuaddr == pt_base);
+}
+
+void kgsl_gpummu_destroy_pagetable(void *mmu_specific_pt)
+{
+ struct kgsl_gpummu_pt *gpummu_pt = (struct kgsl_gpummu_pt *)
+ mmu_specific_pt;
+ kgsl_ptpool_free((struct kgsl_ptpool *)kgsl_driver.ptpool,
+ gpummu_pt->base.hostptr);
+
+ kgsl_driver.stats.coherent -= KGSL_PAGETABLE_SIZE;
+
+ kfree(gpummu_pt->tlbflushfilter.base);
+
+ kfree(gpummu_pt);
+}
+
+static inline uint32_t
+kgsl_pt_entry_get(unsigned int va_base, uint32_t va)
+{
+ return (va - va_base) >> PAGE_SHIFT;
+}
+
+static inline void
+kgsl_pt_map_set(struct kgsl_gpummu_pt *pt, uint32_t pte, uint32_t val)
+{
+ uint32_t *baseptr = (uint32_t *)pt->base.hostptr;
+
+ writel_relaxed(val, &baseptr[pte]);
+}
+
+static inline uint32_t
+kgsl_pt_map_get(struct kgsl_gpummu_pt *pt, uint32_t pte)
+{
+ uint32_t *baseptr = (uint32_t *)pt->base.hostptr;
+ return readl_relaxed(&baseptr[pte]) & GSL_PT_PAGE_ADDR_MASK;
+}
+
+static unsigned int kgsl_gpummu_pt_get_flags(struct kgsl_pagetable *pt,
+ enum kgsl_deviceid id)
+{
+ unsigned int result = 0;
+ struct kgsl_gpummu_pt *gpummu_pt = (struct kgsl_gpummu_pt *)
+ pt->priv;
+
+ if (pt == NULL)
+ return 0;
+
+ spin_lock(&pt->lock);
+ if (gpummu_pt->tlb_flags && (1<<id)) {
+ result = KGSL_MMUFLAGS_TLBFLUSH;
+ gpummu_pt->tlb_flags &= ~(1<<id);
+ }
+ spin_unlock(&pt->lock);
+ return result;
+}
+
+static void kgsl_gpummu_pagefault(struct kgsl_device *device)
+{
+ unsigned int reg;
+ unsigned int ptbase;
+
+ kgsl_regread(device, MH_MMU_PAGE_FAULT, ®);
+ kgsl_regread(device, MH_MMU_PT_BASE, &ptbase);
+
+ KGSL_MEM_CRIT(device,
+ "mmu page fault: page=0x%lx pt=%d op=%s axi=%d\n",
+ reg & ~(PAGE_SIZE - 1),
+ kgsl_mmu_get_ptname_from_ptbase(ptbase),
+ reg & 0x02 ? "WRITE" : "READ", (reg >> 4) & 0xF);
+}
+
+static void *kgsl_gpummu_create_pagetable(void)
+{
+ struct kgsl_gpummu_pt *gpummu_pt;
+
+ gpummu_pt = kzalloc(sizeof(struct kgsl_gpummu_pt),
+ GFP_KERNEL);
+ if (!gpummu_pt)
+ return NULL;
+
+ gpummu_pt->tlb_flags = 0;
+ gpummu_pt->last_superpte = 0;
+
+ gpummu_pt->tlbflushfilter.size = (CONFIG_MSM_KGSL_PAGE_TABLE_SIZE /
+ (PAGE_SIZE * GSL_PT_SUPER_PTE * 8)) + 1;
+ gpummu_pt->tlbflushfilter.base = (unsigned int *)
+ kzalloc(gpummu_pt->tlbflushfilter.size, GFP_KERNEL);
+ if (!gpummu_pt->tlbflushfilter.base) {
+ KGSL_CORE_ERR("kzalloc(%d) failed\n",
+ gpummu_pt->tlbflushfilter.size);
+ goto err_free_gpummu;
+ }
+ GSL_TLBFLUSH_FILTER_RESET();
+
+ gpummu_pt->base.hostptr = kgsl_ptpool_alloc((struct kgsl_ptpool *)
+ kgsl_driver.ptpool,
+ &gpummu_pt->base.physaddr);
+
+ if (gpummu_pt->base.hostptr == NULL)
+ goto err_flushfilter;
+
+ /* ptpool allocations are from coherent memory, so update the
+ device statistics acordingly */
+
+ KGSL_STATS_ADD(KGSL_PAGETABLE_SIZE, kgsl_driver.stats.coherent,
+ kgsl_driver.stats.coherent_max);
+
+ gpummu_pt->base.gpuaddr = gpummu_pt->base.physaddr;
+ gpummu_pt->base.size = KGSL_PAGETABLE_SIZE;
+
+ return (void *)gpummu_pt;
+
+err_flushfilter:
+ kfree(gpummu_pt->tlbflushfilter.base);
+err_free_gpummu:
+ kfree(gpummu_pt);
+
+ return NULL;
+}
+
+static void kgsl_gpummu_default_setstate(struct kgsl_device *device,
+ uint32_t flags)
+{
+ struct kgsl_gpummu_pt *gpummu_pt;
+ if (!kgsl_mmu_enabled())
+ return;
+
+ if (flags & KGSL_MMUFLAGS_PTUPDATE) {
+ kgsl_idle(device, KGSL_TIMEOUT_DEFAULT);
+ gpummu_pt = device->mmu.hwpagetable->priv;
+ kgsl_regwrite(device, MH_MMU_PT_BASE,
+ gpummu_pt->base.gpuaddr);
+ }
+
+ if (flags & KGSL_MMUFLAGS_TLBFLUSH) {
+ /* Invalidate all and tc */
+ kgsl_regwrite(device, MH_MMU_INVALIDATE, 0x00000003);
+ }
+}
+
+static void kgsl_gpummu_setstate(struct kgsl_device *device,
+ struct kgsl_pagetable *pagetable)
+{
+ struct kgsl_mmu *mmu = &device->mmu;
+ struct kgsl_gpummu_pt *gpummu_pt;
+
+ if (mmu->flags & KGSL_FLAGS_STARTED) {
+ /* page table not current, then setup mmu to use new
+ * specified page table
+ */
+ if (mmu->hwpagetable != pagetable) {
+ mmu->hwpagetable = pagetable;
+ spin_lock(&mmu->hwpagetable->lock);
+ gpummu_pt = mmu->hwpagetable->priv;
+ gpummu_pt->tlb_flags &= ~(1<<device->id);
+ spin_unlock(&mmu->hwpagetable->lock);
+
+ /* call device specific set page table */
+ kgsl_setstate(mmu->device, KGSL_MMUFLAGS_TLBFLUSH |
+ KGSL_MMUFLAGS_PTUPDATE);
+ }
+ }
+}
+
+static int kgsl_gpummu_init(struct kgsl_device *device)
+{
+ /*
+ * intialize device mmu
+ *
+ * call this with the global lock held
+ */
+ int status = 0;
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ mmu->device = device;
+
+ /* sub-client MMU lookups require address translation */
+ if ((mmu->config & ~0x1) > 0) {
+ /*make sure virtual address range is a multiple of 64Kb */
+ if (CONFIG_MSM_KGSL_PAGE_TABLE_SIZE & ((1 << 16) - 1)) {
+ KGSL_CORE_ERR("Invalid pagetable size requested "
+ "for GPUMMU: %x\n", CONFIG_MSM_KGSL_PAGE_TABLE_SIZE);
+ return -EINVAL;
+ }
+
+ /* allocate memory used for completing r/w operations that
+ * cannot be mapped by the MMU
+ */
+ status = kgsl_allocate_contiguous(&mmu->setstate_memory, 64);
+ if (!status)
+ kgsl_sharedmem_set(&mmu->setstate_memory, 0, 0,
+ mmu->setstate_memory.size);
+ }
+
+ dev_info(device->dev, "|%s| MMU type set for device is GPUMMU\n",
+ __func__);
+ return status;
+}
+
+static int kgsl_gpummu_start(struct kgsl_device *device)
+{
+ /*
+ * intialize device mmu
+ *
+ * call this with the global lock held
+ */
+
+ struct kgsl_mmu *mmu = &device->mmu;
+ struct kgsl_gpummu_pt *gpummu_pt;
+
+ if (mmu->flags & KGSL_FLAGS_STARTED)
+ return 0;
+
+ /* MMU not enabled */
+ if ((mmu->config & 0x1) == 0)
+ return 0;
+
+ /* setup MMU and sub-client behavior */
+ kgsl_regwrite(device, MH_MMU_CONFIG, mmu->config);
+
+ /* idle device */
+ kgsl_idle(device, KGSL_TIMEOUT_DEFAULT);
+
+ /* enable axi interrupts */
+ kgsl_regwrite(device, MH_INTERRUPT_MASK,
+ GSL_MMU_INT_MASK | MH_INTERRUPT_MASK__MMU_PAGE_FAULT);
+
+ kgsl_sharedmem_set(&mmu->setstate_memory, 0, 0,
+ mmu->setstate_memory.size);
+
+ /* TRAN_ERROR needs a 32 byte (32 byte aligned) chunk of memory
+ * to complete transactions in case of an MMU fault. Note that
+ * we'll leave the bottom 32 bytes of the setstate_memory for other
+ * purposes (e.g. use it when dummy read cycles are needed
+ * for other blocks) */
+ kgsl_regwrite(device, MH_MMU_TRAN_ERROR,
+ mmu->setstate_memory.physaddr + 32);
+
+ if (mmu->defaultpagetable == NULL)
+ mmu->defaultpagetable =
+ kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
+
+ /* Return error if the default pagetable doesn't exist */
+ if (mmu->defaultpagetable == NULL)
+ return -ENOMEM;
+
+ mmu->hwpagetable = mmu->defaultpagetable;
+ gpummu_pt = mmu->hwpagetable->priv;
+ kgsl_regwrite(device, MH_MMU_PT_BASE,
+ gpummu_pt->base.gpuaddr);
+ kgsl_regwrite(device, MH_MMU_VA_RANGE,
+ (KGSL_PAGETABLE_BASE |
+ (CONFIG_MSM_KGSL_PAGE_TABLE_SIZE >> 16)));
+ kgsl_setstate(device, KGSL_MMUFLAGS_TLBFLUSH);
+ mmu->flags |= KGSL_FLAGS_STARTED;
+
+ return 0;
+}
+
+static int
+kgsl_gpummu_unmap(void *mmu_specific_pt,
+ struct kgsl_memdesc *memdesc)
+{
+ unsigned int numpages;
+ unsigned int pte, ptefirst, ptelast, superpte;
+ unsigned int range = memdesc->size;
+ struct kgsl_gpummu_pt *gpummu_pt = mmu_specific_pt;
+
+ /* All GPU addresses as assigned are page aligned, but some
+ functions purturb the gpuaddr with an offset, so apply the
+ mask here to make sure we have the right address */
+
+ unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK;
+
+ numpages = (range >> PAGE_SHIFT);
+ if (range & (PAGE_SIZE - 1))
+ numpages++;
+
+ ptefirst = kgsl_pt_entry_get(KGSL_PAGETABLE_BASE, gpuaddr);
+ ptelast = ptefirst + numpages;
+
+ superpte = ptefirst - (ptefirst & (GSL_PT_SUPER_PTE-1));
+ GSL_TLBFLUSH_FILTER_SETDIRTY(superpte / GSL_PT_SUPER_PTE);
+ for (pte = ptefirst; pte < ptelast; pte++) {
+#ifdef VERBOSE_DEBUG
+ /* check if PTE exists */
+ if (!kgsl_pt_map_get(gpummu_pt, pte))
+ KGSL_CORE_ERR("pt entry %x is already "
+ "unmapped for pagetable %p\n", pte, gpummu_pt);
+#endif
+ kgsl_pt_map_set(gpummu_pt, pte, GSL_PT_PAGE_DIRTY);
+ superpte = pte - (pte & (GSL_PT_SUPER_PTE - 1));
+ if (pte == superpte)
+ GSL_TLBFLUSH_FILTER_SETDIRTY(superpte /
+ GSL_PT_SUPER_PTE);
+ }
+
+ /* Post all writes to the pagetable */
+ wmb();
+
+ return 0;
+}
+
+static int
+kgsl_gpummu_map(void *mmu_specific_pt,
+ struct kgsl_memdesc *memdesc,
+ unsigned int protflags)
+{
+ int numpages;
+ unsigned int pte, ptefirst, ptelast, physaddr;
+ int flushtlb;
+ unsigned int offset = 0;
+ struct kgsl_gpummu_pt *gpummu_pt = mmu_specific_pt;
+
+ if (!protflags ||
+ protflags & ~(GSL_PT_PAGE_RV | GSL_PT_PAGE_WV)) {
+ KGSL_CORE_ERR("Invalid protflags for "
+ "kgsl_mmu_specific_map: %x", protflags);
+ return -EINVAL;
+ }
+
+ numpages = (memdesc->size >> PAGE_SHIFT);
+
+ ptefirst = kgsl_pt_entry_get(KGSL_PAGETABLE_BASE, memdesc->gpuaddr);
+ ptelast = ptefirst + numpages;
+
+ pte = ptefirst;
+ flushtlb = 0;
+
+ /* tlb needs to be flushed when the first and last pte are not at
+ * superpte boundaries */
+ if ((ptefirst & (GSL_PT_SUPER_PTE - 1)) != 0 ||
+ ((ptelast + 1) & (GSL_PT_SUPER_PTE-1)) != 0)
+ flushtlb = 1;
+
+ for (pte = ptefirst; pte < ptelast; pte++, offset += PAGE_SIZE) {
+#ifdef VERBOSE_DEBUG
+ /* check if PTE exists */
+ uint32_t val = kgsl_pt_map_get(gpummu_pt, pte);
+ if (val != 0 && val != GSL_PT_PAGE_DIRTY) {
+ KGSL_CORE_ERR("pt entry %x is already set with "
+ "value %x for pagetable %p\n", pte, val, gpummu_pt);
+ return -EINVAL;
+ }
+#endif
+ if ((pte & (GSL_PT_SUPER_PTE-1)) == 0)
+ if (GSL_TLBFLUSH_FILTER_ISDIRTY(pte / GSL_PT_SUPER_PTE))
+ flushtlb = 1;
+ /* mark pte as in use */
+
+ physaddr = memdesc->ops->physaddr(memdesc, offset);
+ if (!physaddr) {
+ KGSL_CORE_ERR("Failed to convert %x address to "
+ "physical", (unsigned int)memdesc->hostptr + offset);
+ kgsl_gpummu_unmap(mmu_specific_pt, memdesc);
+ return -EFAULT;
+ }
+ kgsl_pt_map_set(gpummu_pt, pte, physaddr | protflags);
+ }
+
+ /* Post all writes to the pagetable */
+ wmb();
+
+ /* Invalidate tlb only if current page table used by GPU is the
+ * pagetable that we used to allocate */
+ if (flushtlb) {
+ /*set all devices as needing flushing*/
+ gpummu_pt->tlb_flags = UINT_MAX;
+ GSL_TLBFLUSH_FILTER_RESET();
+ }
+
+ return 0;
+}
+
+static int kgsl_gpummu_stop(struct kgsl_device *device)
+{
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ kgsl_regwrite(device, MH_MMU_CONFIG, 0x00000000);
+ mmu->flags &= ~KGSL_FLAGS_STARTED;
+
+ return 0;
+}
+
+static int kgsl_gpummu_close(struct kgsl_device *device)
+{
+ /*
+ * close device mmu
+ *
+ * call this with the global lock held
+ */
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ if (mmu->setstate_memory.gpuaddr)
+ kgsl_sharedmem_free(&mmu->setstate_memory);
+
+ if (mmu->defaultpagetable)
+ kgsl_mmu_putpagetable(mmu->defaultpagetable);
+
+ return 0;
+}
+
+static unsigned int
+kgsl_gpummu_get_current_ptbase(struct kgsl_device *device)
+{
+ unsigned int ptbase;
+ kgsl_regread(device, MH_MMU_PT_BASE, &ptbase);
+ return ptbase;
+}
+
+struct kgsl_mmu_ops gpummu_ops = {
+ .mmu_init = kgsl_gpummu_init,
+ .mmu_close = kgsl_gpummu_close,
+ .mmu_start = kgsl_gpummu_start,
+ .mmu_stop = kgsl_gpummu_stop,
+ .mmu_setstate = kgsl_gpummu_setstate,
+ .mmu_device_setstate = kgsl_gpummu_default_setstate,
+ .mmu_pagefault = kgsl_gpummu_pagefault,
+ .mmu_get_current_ptbase = kgsl_gpummu_get_current_ptbase,
+};
+
+struct kgsl_mmu_pt_ops gpummu_pt_ops = {
+ .mmu_map = kgsl_gpummu_map,
+ .mmu_unmap = kgsl_gpummu_unmap,
+ .mmu_create_pagetable = kgsl_gpummu_create_pagetable,
+ .mmu_destroy_pagetable = kgsl_gpummu_destroy_pagetable,
+ .mmu_pt_equal = kgsl_gpummu_pt_equal,
+ .mmu_pt_get_flags = kgsl_gpummu_pt_get_flags,
+};
diff --git a/drivers/gpu/msm/kgsl_gpummu.h b/drivers/gpu/msm/kgsl_gpummu.h
new file mode 100644
index 0000000..46466a8
--- /dev/null
+++ b/drivers/gpu/msm/kgsl_gpummu.h
@@ -0,0 +1,85 @@
+/* Copyright (c) 2011, Code Aurora Forum. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __KGSL_GPUMMU_H
+#define __KGSL_GPUMMU_H
+
+#define GSL_PT_PAGE_BITS_MASK 0x00000007
+#define GSL_PT_PAGE_ADDR_MASK PAGE_MASK
+
+#define GSL_MMU_INT_MASK \
+ (MH_INTERRUPT_MASK__AXI_READ_ERROR | \
+ MH_INTERRUPT_MASK__AXI_WRITE_ERROR)
+
+/* Macros to manage TLB flushing */
+#define GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS (sizeof(unsigned char) * 8)
+#define GSL_TLBFLUSH_FILTER_GET(superpte) \
+ (*((unsigned char *) \
+ (((unsigned int)gpummu_pt->tlbflushfilter.base) \
+ + (superpte / GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS))))
+#define GSL_TLBFLUSH_FILTER_SETDIRTY(superpte) \
+ (GSL_TLBFLUSH_FILTER_GET((superpte)) |= 1 << \
+ (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS))
+#define GSL_TLBFLUSH_FILTER_ISDIRTY(superpte) \
+ (GSL_TLBFLUSH_FILTER_GET((superpte)) & \
+ (1 << (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS)))
+#define GSL_TLBFLUSH_FILTER_RESET() memset(gpummu_pt->tlbflushfilter.base,\
+ 0, gpummu_pt->tlbflushfilter.size)
+
+extern struct kgsl_mmu_ops gpummu_ops;
+extern struct kgsl_mmu_pt_ops gpummu_pt_ops;
+
+struct kgsl_tlbflushfilter {
+ unsigned int *base;
+ unsigned int size;
+};
+
+struct kgsl_gpummu_pt {
+ struct kgsl_memdesc base;
+ unsigned int last_superpte;
+ unsigned int tlb_flags;
+ /* Maintain filter to manage tlb flushing */
+ struct kgsl_tlbflushfilter tlbflushfilter;
+};
+
+struct kgsl_ptpool_chunk {
+ size_t size;
+ unsigned int count;
+ int dynamic;
+
+ void *data;
+ unsigned int phys;
+
+ unsigned long *bitmap;
+ struct list_head list;
+};
+
+struct kgsl_ptpool {
+ size_t ptsize;
+ struct mutex lock;
+ struct list_head list;
+ int entries;
+ int static_entries;
+ int chunks;
+};
+
+void *kgsl_gpummu_ptpool_init(int ptsize,
+ int entries);
+void kgsl_gpummu_ptpool_destroy(void *ptpool);
+
+static inline unsigned int kgsl_pt_get_base_addr(struct kgsl_pagetable *pt)
+{
+ struct kgsl_gpummu_pt *gpummu_pt = pt->priv;
+ return gpummu_pt->base.gpuaddr;
+}
+#endif /* __KGSL_GPUMMU_H */
diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c
new file mode 100644
index 0000000..f9b9b4a
--- /dev/null
+++ b/drivers/gpu/msm/kgsl_iommu.c
@@ -0,0 +1,353 @@
+/* Copyright (c) 2011, Code Aurora Forum. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+#include <linux/genalloc.h>
+#include <linux/slab.h>
+#include <linux/iommu.h>
+#include <mach/iommu.h>
+#include <linux/msm_kgsl.h>
+
+#include "kgsl.h"
+#include "kgsl_device.h"
+#include "kgsl_mmu.h"
+#include "kgsl_sharedmem.h"
+
+struct kgsl_iommu {
+ struct device *iommu_user_dev;
+ int iommu_user_dev_attached;
+ struct device *iommu_priv_dev;
+ int iommu_priv_dev_attached;
+};
+
+static int kgsl_iommu_pt_equal(struct kgsl_pagetable *pt,
+ unsigned int pt_base)
+{
+ struct iommu_domain *domain = pt->priv;
+ return pt && pt_base && ((unsigned int)domain == pt_base);
+}
+
+static void kgsl_iommu_destroy_pagetable(void *mmu_specific_pt)
+{
+ struct iommu_domain *domain = mmu_specific_pt;
+ if (domain)
+ iommu_domain_free(domain);
+}
+
+void *kgsl_iommu_create_pagetable(void)
+{
+ struct iommu_domain *domain = iommu_domain_alloc(0);
+ if (!domain)
+ KGSL_CORE_ERR("Failed to create iommu domain\n");
+
+ return domain;
+}
+
+static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
+{
+ struct iommu_domain *domain;
+ struct kgsl_iommu *iommu = mmu->priv;
+
+ BUG_ON(mmu->hwpagetable == NULL);
+ BUG_ON(mmu->hwpagetable->priv == NULL);
+
+ domain = mmu->hwpagetable->priv;
+
+ if (iommu->iommu_user_dev_attached) {
+ iommu_detach_device(domain, iommu->iommu_user_dev);
+ iommu->iommu_user_dev_attached = 0;
+ KGSL_MEM_INFO(mmu->device,
+ "iommu %p detached from user dev of MMU: %p\n",
+ domain, mmu);
+ }
+ if (iommu->iommu_priv_dev_attached) {
+ iommu_detach_device(domain, iommu->iommu_priv_dev);
+ iommu->iommu_priv_dev_attached = 0;
+ KGSL_MEM_INFO(mmu->device,
+ "iommu %p detached from priv dev of MMU: %p\n",
+ domain, mmu);
+ }
+}
+
+static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu)
+{
+ struct iommu_domain *domain;
+ int ret = 0;
+ struct kgsl_iommu *iommu = mmu->priv;
+
+ BUG_ON(mmu->hwpagetable == NULL);
+ BUG_ON(mmu->hwpagetable->priv == NULL);
+
+ domain = mmu->hwpagetable->priv;
+
+ if (iommu->iommu_user_dev && !iommu->iommu_user_dev_attached) {
+ ret = iommu_attach_device(domain, iommu->iommu_user_dev);
+ if (ret) {
+ KGSL_MEM_ERR(mmu->device,
+ "Failed to attach device, err %d\n", ret);
+ goto done;
+ }
+ iommu->iommu_user_dev_attached = 1;
+ KGSL_MEM_INFO(mmu->device,
+ "iommu %p attached to user dev of MMU: %p\n",
+ domain, mmu);
+ }
+ if (iommu->iommu_priv_dev && !iommu->iommu_priv_dev_attached) {
+ ret = iommu_attach_device(domain, iommu->iommu_priv_dev);
+ if (ret) {
+ KGSL_MEM_ERR(mmu->device,
+ "Failed to attach device, err %d\n", ret);
+ iommu_detach_device(domain, iommu->iommu_user_dev);
+ iommu->iommu_user_dev_attached = 0;
+ goto done;
+ }
+ iommu->iommu_priv_dev_attached = 1;
+ KGSL_MEM_INFO(mmu->device,
+ "iommu %p attached to priv dev of MMU: %p\n",
+ domain, mmu);
+ }
+done:
+ return ret;
+}
+
+static int kgsl_get_iommu_ctxt(struct kgsl_iommu *iommu,
+ struct kgsl_device *device)
+{
+ int status = 0;
+ struct platform_device *pdev =
+ container_of(device->parentdev, struct platform_device, dev);
+ struct kgsl_device_platform_data *pdata_dev = pdev->dev.platform_data;
+ if (pdata_dev->iommu_user_ctx_name)
+ iommu->iommu_user_dev = msm_iommu_get_ctx(
+ pdata_dev->iommu_user_ctx_name);
+ if (pdata_dev->iommu_priv_ctx_name)
+ iommu->iommu_priv_dev = msm_iommu_get_ctx(
+ pdata_dev->iommu_priv_ctx_name);
+ if (!iommu->iommu_user_dev) {
+ KGSL_CORE_ERR("Failed to get user iommu dev handle for "
+ "device %s\n",
+ pdata_dev->iommu_user_ctx_name);
+ status = -EINVAL;
+ }
+ return status;
+}
+
+static void kgsl_iommu_setstate(struct kgsl_device *device,
+ struct kgsl_pagetable *pagetable)
+{
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ if (mmu->flags & KGSL_FLAGS_STARTED) {
+ /* page table not current, then setup mmu to use new
+ * specified page table
+ */
+ if (mmu->hwpagetable != pagetable) {
+ kgsl_idle(device, KGSL_TIMEOUT_DEFAULT);
+ kgsl_detach_pagetable_iommu_domain(mmu);
+ mmu->hwpagetable = pagetable;
+ if (mmu->hwpagetable)
+ kgsl_attach_pagetable_iommu_domain(mmu);
+ }
+ }
+}
+
+static int kgsl_iommu_init(struct kgsl_device *device)
+{
+ /*
+ * intialize device mmu
+ *
+ * call this with the global lock held
+ */
+ int status = 0;
+ struct kgsl_mmu *mmu = &device->mmu;
+ struct kgsl_iommu *iommu;
+
+ mmu->device = device;
+
+ iommu = kzalloc(sizeof(struct kgsl_iommu), GFP_KERNEL);
+ if (!iommu) {
+ KGSL_CORE_ERR("kzalloc(%d) failed\n",
+ sizeof(struct kgsl_iommu));
+ return -ENOMEM;
+ }
+
+ iommu->iommu_priv_dev_attached = 0;
+ iommu->iommu_user_dev_attached = 0;
+ status = kgsl_get_iommu_ctxt(iommu, device);
+ if (status) {
+ kfree(iommu);
+ iommu = NULL;
+ }
+ mmu->priv = iommu;
+
+ dev_info(device->dev, "|%s| MMU type set for device is IOMMU\n",
+ __func__);
+ return status;
+}
+
+static int kgsl_iommu_start(struct kgsl_device *device)
+{
+ int status;
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ if (mmu->flags & KGSL_FLAGS_STARTED)
+ return 0;
+
+ kgsl_regwrite(device, MH_MMU_CONFIG, 0x00000000);
+ if (mmu->defaultpagetable == NULL)
+ mmu->defaultpagetable =
+ kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
+ /* Return error if the default pagetable doesn't exist */
+ if (mmu->defaultpagetable == NULL)
+ return -ENOMEM;
+ mmu->hwpagetable = mmu->defaultpagetable;
+
+ status = kgsl_attach_pagetable_iommu_domain(mmu);
+ if (!status)
+ mmu->flags |= KGSL_FLAGS_STARTED;
+
+ return status;
+}
+
+static int
+kgsl_iommu_unmap(void *mmu_specific_pt,
+ struct kgsl_memdesc *memdesc)
+{
+ int ret;
+ unsigned int range = memdesc->size;
+ unsigned int iommu_map_addr;
+ int map_order = get_order(SZ_4K);
+ struct iommu_domain *domain = (struct iommu_domain *)
+ mmu_specific_pt;
+
+ /* All GPU addresses as assigned are page aligned, but some
+ functions purturb the gpuaddr with an offset, so apply the
+ mask here to make sure we have the right address */
+
+ unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK;
+
+ if (range == 0 || gpuaddr == 0)
+ return 0;
+
+ for (iommu_map_addr = gpuaddr; iommu_map_addr < (gpuaddr + range);
+ iommu_map_addr += SZ_4K) {
+ ret = iommu_unmap(domain, iommu_map_addr, map_order);
+ if (ret)
+ KGSL_CORE_ERR("iommu_unmap(%p, %x, %d) failed "
+ "with err: %d\n", domain, iommu_map_addr,
+ map_order, ret);
+ }
+
+ return 0;
+}
+
+static int
+kgsl_iommu_map(void *mmu_specific_pt,
+ struct kgsl_memdesc *memdesc,
+ unsigned int protflags)
+{
+ int ret = 0;
+ unsigned int physaddr;
+ unsigned int iommu_virt_addr;
+ unsigned int offset = 0;
+ int map_order;
+ struct iommu_domain *domain = (struct iommu_domain *)
+ mmu_specific_pt;
+
+ BUG_ON(NULL == domain);
+
+ map_order = get_order(SZ_4K);
+
+ for (iommu_virt_addr = memdesc->gpuaddr;
+ iommu_virt_addr < (memdesc->gpuaddr + memdesc->size);
+ iommu_virt_addr += SZ_4K, offset += PAGE_SIZE) {
+ physaddr = memdesc->ops->physaddr(memdesc, offset);
+ if (!physaddr) {
+ KGSL_CORE_ERR("Failed to convert %x address to "
+ "physical\n", (unsigned int)memdesc->hostptr + offset);
+ kgsl_iommu_unmap(mmu_specific_pt, memdesc);
+ return -EFAULT;
+ }
+ ret = iommu_map(domain, iommu_virt_addr, physaddr,
+ map_order, MSM_IOMMU_ATTR_NONCACHED);
+ if (ret) {
+ KGSL_CORE_ERR("iommu_map(%p, %x, %x, %d, %d) "
+ "failed with err: %d\n", domain,
+ iommu_virt_addr, physaddr, map_order,
+ MSM_IOMMU_ATTR_NONCACHED, ret);
+ kgsl_iommu_unmap(mmu_specific_pt, memdesc);
+ return ret;
+ }
+ }
+
+ return ret;
+}
+
+static int kgsl_iommu_stop(struct kgsl_device *device)
+{
+ /*
+ * stop device mmu
+ *
+ * call this with the global lock held
+ */
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ if (mmu->flags & KGSL_FLAGS_STARTED) {
+ /* detach iommu attachment */
+ kgsl_detach_pagetable_iommu_domain(mmu);
+
+ mmu->flags &= ~KGSL_FLAGS_STARTED;
+ }
+
+ return 0;
+}
+
+static int kgsl_iommu_close(struct kgsl_device *device)
+{
+ struct kgsl_mmu *mmu = &device->mmu;
+ if (mmu->defaultpagetable)
+ kgsl_mmu_putpagetable(mmu->defaultpagetable);
+
+ return 0;
+}
+
+static unsigned int
+kgsl_iommu_get_current_ptbase(struct kgsl_device *device)
+{
+ /* Current base is always the hwpagetables domain as we
+ * do not use per process pagetables right not for iommu.
+ * This will change when we switch to per process pagetables.
+ */
+ return (unsigned int)device->mmu.hwpagetable->priv;
+}
+
+struct kgsl_mmu_ops iommu_ops = {
+ .mmu_init = kgsl_iommu_init,
+ .mmu_close = kgsl_iommu_close,
+ .mmu_start = kgsl_iommu_start,
+ .mmu_stop = kgsl_iommu_stop,
+ .mmu_setstate = kgsl_iommu_setstate,
+ .mmu_device_setstate = NULL,
+ .mmu_pagefault = NULL,
+ .mmu_get_current_ptbase = kgsl_iommu_get_current_ptbase,
+};
+
+struct kgsl_mmu_pt_ops iommu_pt_ops = {
+ .mmu_map = kgsl_iommu_map,
+ .mmu_unmap = kgsl_iommu_unmap,
+ .mmu_create_pagetable = kgsl_iommu_create_pagetable,
+ .mmu_destroy_pagetable = kgsl_iommu_destroy_pagetable,
+ .mmu_pt_equal = kgsl_iommu_pt_equal,
+ .mmu_pt_get_flags = NULL,
+};
diff --git a/drivers/gpu/msm/kgsl_mmu.c b/drivers/gpu/msm/kgsl_mmu.c
index 7471342..7eec9e5 100644
--- a/drivers/gpu/msm/kgsl_mmu.c
+++ b/drivers/gpu/msm/kgsl_mmu.c
@@ -16,6 +16,7 @@
#include <linux/genalloc.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/iommu.h>
#include "kgsl.h"
#include "kgsl_mmu.h"
@@ -25,313 +26,10 @@
#define KGSL_MMU_ALIGN_SHIFT 13
#define KGSL_MMU_ALIGN_MASK (~((1 << KGSL_MMU_ALIGN_SHIFT) - 1))
-#define GSL_PT_PAGE_BITS_MASK 0x00000007
-#define GSL_PT_PAGE_ADDR_MASK PAGE_MASK
+static enum kgsl_mmutype kgsl_mmu_type;
static void pagetable_remove_sysfs_objects(struct kgsl_pagetable *pagetable);
-static ssize_t
-sysfs_show_ptpool_entries(struct kobject *kobj,
- struct kobj_attribute *attr,
- char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.ptpool.entries);
-}
-
-static ssize_t
-sysfs_show_ptpool_min(struct kobject *kobj,
- struct kobj_attribute *attr,
- char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "%d\n",
- kgsl_driver.ptpool.static_entries);
-}
-
-static ssize_t
-sysfs_show_ptpool_chunks(struct kobject *kobj,
- struct kobj_attribute *attr,
- char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.ptpool.chunks);
-}
-
-static ssize_t
-sysfs_show_ptpool_ptsize(struct kobject *kobj,
- struct kobj_attribute *attr,
- char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.ptpool.ptsize);
-}
-
-static struct kobj_attribute attr_ptpool_entries = {
- .attr = { .name = "ptpool_entries", .mode = 0444 },
- .show = sysfs_show_ptpool_entries,
- .store = NULL,
-};
-
-static struct kobj_attribute attr_ptpool_min = {
- .attr = { .name = "ptpool_min", .mode = 0444 },
- .show = sysfs_show_ptpool_min,
- .store = NULL,
-};
-
-static struct kobj_attribute attr_ptpool_chunks = {
- .attr = { .name = "ptpool_chunks", .mode = 0444 },
- .show = sysfs_show_ptpool_chunks,
- .store = NULL,
-};
-
-static struct kobj_attribute attr_ptpool_ptsize = {
- .attr = { .name = "ptpool_ptsize", .mode = 0444 },
- .show = sysfs_show_ptpool_ptsize,
- .store = NULL,
-};
-
-static struct attribute *ptpool_attrs[] = {
- &attr_ptpool_entries.attr,
- &attr_ptpool_min.attr,
- &attr_ptpool_chunks.attr,
- &attr_ptpool_ptsize.attr,
- NULL,
-};
-
-static struct attribute_group ptpool_attr_group = {
- .attrs = ptpool_attrs,
-};
-
-static int
-_kgsl_ptpool_add_entries(struct kgsl_ptpool *pool, int count, int dynamic)
-{
- struct kgsl_ptpool_chunk *chunk;
- size_t size = ALIGN(count * pool->ptsize, PAGE_SIZE);
-
- BUG_ON(count == 0);
-
- if (get_order(size) >= MAX_ORDER) {
- KGSL_CORE_ERR("ptpool allocation is too big: %d\n", size);
- return -EINVAL;
- }
-
- chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
- if (chunk == NULL) {
- KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(*chunk));
- return -ENOMEM;
- }
-
- chunk->size = size;
- chunk->count = count;
- chunk->dynamic = dynamic;
-
- chunk->data = dma_alloc_coherent(NULL, size,
- &chunk->phys, GFP_KERNEL);
-
- if (chunk->data == NULL) {
- KGSL_CORE_ERR("dma_alloc_coherent(%d) failed\n", size);
- goto err;
- }
-
- chunk->bitmap = kzalloc(BITS_TO_LONGS(count) * 4, GFP_KERNEL);
-
- if (chunk->bitmap == NULL) {
- KGSL_CORE_ERR("kzalloc(%d) failed\n",
- BITS_TO_LONGS(count) * 4);
- goto err_dma;
- }
-
- list_add_tail(&chunk->list, &pool->list);
-
- pool->chunks++;
- pool->entries += count;
-
- if (!dynamic)
- pool->static_entries += count;
-
- return 0;
-
-err_dma:
- dma_free_coherent(NULL, chunk->size, chunk->data, chunk->phys);
-err:
- kfree(chunk);
- return -ENOMEM;
-}
-
-static void *
-_kgsl_ptpool_get_entry(struct kgsl_ptpool *pool, unsigned int *physaddr)
-{
- struct kgsl_ptpool_chunk *chunk;
-
- list_for_each_entry(chunk, &pool->list, list) {
- int bit = find_first_zero_bit(chunk->bitmap, chunk->count);
-
- if (bit >= chunk->count)
- continue;
-
- set_bit(bit, chunk->bitmap);
- *physaddr = chunk->phys + (bit * pool->ptsize);
-
- return chunk->data + (bit * pool->ptsize);
- }
-
- return NULL;
-}
-
-/**
- * kgsl_ptpool_add
- * @pool: A pointer to a ptpool structure
- * @entries: Number of entries to add
- *
- * Add static entries to the pagetable pool.
- */
-
-int
-kgsl_ptpool_add(struct kgsl_ptpool *pool, int count)
-{
- int ret = 0;
- BUG_ON(count == 0);
-
- mutex_lock(&pool->lock);
-
- /* Only 4MB can be allocated in one chunk, so larger allocations
- need to be split into multiple sections */
-
- while (count) {
- int entries = ((count * pool->ptsize) > SZ_4M) ?
- SZ_4M / pool->ptsize : count;
-
- /* Add the entries as static, i.e. they don't ever stand
- a chance of being removed */
-
- ret = _kgsl_ptpool_add_entries(pool, entries, 0);
- if (ret)
- break;
-
- count -= entries;
- }
-
- mutex_unlock(&pool->lock);
- return ret;
-}
-
-/**
- * kgsl_ptpool_alloc
- * @pool: A pointer to a ptpool structure
- * @addr: A pointer to store the physical address of the chunk
- *
- * Allocate a pagetable from the pool. Returns the virtual address
- * of the pagetable, the physical address is returned in physaddr
- */
-
-void *kgsl_ptpool_alloc(struct kgsl_ptpool *pool, unsigned int *physaddr)
-{
- void *addr = NULL;
- int ret;
-
- mutex_lock(&pool->lock);
- addr = _kgsl_ptpool_get_entry(pool, physaddr);
- if (addr)
- goto done;
-
- /* Add a chunk for 1 more pagetable and mark it as dynamic */
- ret = _kgsl_ptpool_add_entries(pool, 1, 1);
-
- if (ret)
- goto done;
-
- addr = _kgsl_ptpool_get_entry(pool, physaddr);
-done:
- mutex_unlock(&pool->lock);
- return addr;
-}
-
-static inline void _kgsl_ptpool_rm_chunk(struct kgsl_ptpool_chunk *chunk)
-{
- list_del(&chunk->list);
-
- if (chunk->data)
- dma_free_coherent(NULL, chunk->size, chunk->data,
- chunk->phys);
- kfree(chunk->bitmap);
- kfree(chunk);
-}
-
-/**
- * kgsl_ptpool_free
- * @pool: A pointer to a ptpool structure
- * @addr: A pointer to the virtual address to free
- *
- * Free a pagetable allocated from the pool
- */
-
-void kgsl_ptpool_free(struct kgsl_ptpool *pool, void *addr)
-{
- struct kgsl_ptpool_chunk *chunk, *tmp;
-
- if (pool == NULL || addr == NULL)
- return;
-
- mutex_lock(&pool->lock);
- list_for_each_entry_safe(chunk, tmp, &pool->list, list) {
- if (addr >= chunk->data &&
- addr < chunk->data + chunk->size) {
- int bit = ((unsigned long) (addr - chunk->data)) /
- pool->ptsize;
-
- clear_bit(bit, chunk->bitmap);
- memset(addr, 0, pool->ptsize);
-
- if (chunk->dynamic &&
- bitmap_empty(chunk->bitmap, chunk->count))
- _kgsl_ptpool_rm_chunk(chunk);
-
- break;
- }
- }
-
- mutex_unlock(&pool->lock);
-}
-
-void kgsl_ptpool_destroy(struct kgsl_ptpool *pool)
-{
- struct kgsl_ptpool_chunk *chunk, *tmp;
-
- if (pool == NULL)
- return;
-
- mutex_lock(&pool->lock);
- list_for_each_entry_safe(chunk, tmp, &pool->list, list)
- _kgsl_ptpool_rm_chunk(chunk);
- mutex_unlock(&pool->lock);
-
- memset(pool, 0, sizeof(*pool));
-}
-
-/**
- * kgsl_ptpool_init
- * @pool: A pointer to a ptpool structure to initialize
- * @ptsize: The size of each pagetable entry
- * @entries: The number of inital entries to add to the pool
- *
- * Initalize a pool and allocate an initial chunk of entries.
- */
-
-int kgsl_ptpool_init(struct kgsl_ptpool *pool, int ptsize, int entries)
-{
- int ret = 0;
- BUG_ON(ptsize == 0);
-
- pool->ptsize = ptsize;
- mutex_init(&pool->lock);
- INIT_LIST_HEAD(&pool->list);
-
- if (entries) {
- ret = kgsl_ptpool_add(pool, entries);
- if (ret)
- return ret;
- }
-
- return sysfs_create_group(kgsl_driver.ptkobj, &ptpool_attr_group);
-}
-
static int kgsl_cleanup_pt(struct kgsl_pagetable *pt)
{
int i;
@@ -357,14 +55,11 @@
kgsl_cleanup_pt(pagetable);
- kgsl_ptpool_free(&kgsl_driver.ptpool, pagetable->base.hostptr);
-
- kgsl_driver.stats.coherent -= KGSL_PAGETABLE_SIZE;
-
if (pagetable->pool)
gen_pool_destroy(pagetable->pool);
- kfree(pagetable->tlbflushfilter.base);
+ pagetable->pt_ops->mmu_destroy_pagetable(pagetable->priv);
+
kfree(pagetable);
}
@@ -452,7 +147,8 @@
pt = _get_pt_from_kobj(kobj);
if (pt)
- ret += snprintf(buf, PAGE_SIZE, "0x%x\n", pt->va_range);
+ ret += snprintf(buf, PAGE_SIZE, "0x%x\n",
+ CONFIG_MSM_KGSL_PAGE_TABLE_SIZE);
kgsl_put_pagetable(pt);
return ret;
@@ -570,37 +266,25 @@
return ret;
}
-static inline uint32_t
-kgsl_pt_entry_get(struct kgsl_pagetable *pt, uint32_t va)
+unsigned int kgsl_mmu_get_current_ptbase(struct kgsl_device *device)
{
- return (va - pt->va_base) >> PAGE_SHIFT;
+ struct kgsl_mmu *mmu = &device->mmu;
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
+ return 0;
+ else
+ return mmu->mmu_ops->mmu_get_current_ptbase(device);
}
-
-static inline void
-kgsl_pt_map_set(struct kgsl_pagetable *pt, uint32_t pte, uint32_t val)
-{
- uint32_t *baseptr = (uint32_t *)pt->base.hostptr;
-
- writel_relaxed(val, &baseptr[pte]);
-}
-
-static inline uint32_t
-kgsl_pt_map_getaddr(struct kgsl_pagetable *pt, uint32_t pte)
-{
- uint32_t *baseptr = (uint32_t *)pt->base.hostptr;
- uint32_t ret = readl_relaxed(&baseptr[pte]) & GSL_PT_PAGE_ADDR_MASK;
- return ret;
-}
+EXPORT_SYMBOL(kgsl_mmu_get_current_ptbase);
int
-kgsl_get_ptname_from_ptbase(unsigned int pt_base)
+kgsl_mmu_get_ptname_from_ptbase(unsigned int pt_base)
{
struct kgsl_pagetable *pt;
int ptid = -1;
spin_lock(&kgsl_driver.ptlock);
list_for_each_entry(pt, &kgsl_driver.pagetable_list, list) {
- if (pt_base == pt->base.gpuaddr) {
+ if (pt->pt_ops->mmu_pt_equal(pt, pt_base)) {
ptid = (int) pt->name;
break;
}
@@ -609,21 +293,52 @@
return ptid;
}
+EXPORT_SYMBOL(kgsl_mmu_get_ptname_from_ptbase);
-void kgsl_mmu_pagefault(struct kgsl_device *device)
+void kgsl_mmu_setstate(struct kgsl_device *device,
+ struct kgsl_pagetable *pagetable)
{
- unsigned int reg;
- unsigned int ptbase;
+ struct kgsl_mmu *mmu = &device->mmu;
- kgsl_regread(device, MH_MMU_PAGE_FAULT, ®);
- kgsl_regread(device, MH_MMU_PT_BASE, &ptbase);
-
- KGSL_MEM_CRIT(device,
- "mmu page fault: page=0x%lx pt=%d op=%s axi=%d\n",
- reg & ~(PAGE_SIZE - 1),
- kgsl_get_ptname_from_ptbase(ptbase),
- reg & 0x02 ? "WRITE" : "READ", (reg >> 4) & 0xF);
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
+ return;
+ else
+ mmu->mmu_ops->mmu_setstate(device,
+ pagetable);
}
+EXPORT_SYMBOL(kgsl_mmu_setstate);
+
+int kgsl_mmu_init(struct kgsl_device *device)
+{
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ mmu->device = device;
+
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type) {
+ dev_info(device->dev, "|%s| MMU type set for device is "
+ "NOMMU\n", __func__);
+ return 0;
+ } else if (KGSL_MMU_TYPE_GPU == kgsl_mmu_type)
+ mmu->mmu_ops = &gpummu_ops;
+ else if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_type)
+ mmu->mmu_ops = &iommu_ops;
+
+ return mmu->mmu_ops->mmu_init(device);
+}
+EXPORT_SYMBOL(kgsl_mmu_init);
+
+int kgsl_mmu_start(struct kgsl_device *device)
+{
+ struct kgsl_mmu *mmu = &device->mmu;
+
+ if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE) {
+ kgsl_regwrite(device, MH_MMU_CONFIG, 0);
+ return 0;
+ } else {
+ return mmu->mmu_ops->mmu_start(device);
+ }
+}
+EXPORT_SYMBOL(kgsl_mmu_start);
void kgsl_mh_intrcallback(struct kgsl_device *device)
{
@@ -638,7 +353,7 @@
if (status & MH_INTERRUPT_MASK__AXI_WRITE_ERROR)
KGSL_MEM_CRIT(device, "axi write error interrupt: %08x\n", reg);
if (status & MH_INTERRUPT_MASK__MMU_PAGE_FAULT)
- kgsl_mmu_pagefault(device);
+ device->mmu.mmu_ops->mmu_pagefault(device);
status &= KGSL_MMU_INT_MASK;
kgsl_regwrite(device, MH_INTERRUPT_CLEAR, status);
@@ -686,54 +401,34 @@
kref_init(&pagetable->refcount);
spin_lock_init(&pagetable->lock);
- pagetable->tlb_flags = 0;
pagetable->name = name;
- pagetable->va_base = KGSL_PAGETABLE_BASE;
- pagetable->va_range = CONFIG_MSM_KGSL_PAGE_TABLE_SIZE;
- pagetable->last_superpte = 0;
- pagetable->max_entries = KGSL_PAGETABLE_ENTRIES(pagetable->va_range);
-
- pagetable->tlbflushfilter.size = (pagetable->va_range /
- (PAGE_SIZE * GSL_PT_SUPER_PTE * 8)) + 1;
- pagetable->tlbflushfilter.base = (unsigned int *)
- kzalloc(pagetable->tlbflushfilter.size, GFP_KERNEL);
- if (!pagetable->tlbflushfilter.base) {
- KGSL_CORE_ERR("kzalloc(%d) failed\n",
- pagetable->tlbflushfilter.size);
- goto err_alloc;
- }
- GSL_TLBFLUSH_FILTER_RESET();
+ pagetable->max_entries = KGSL_PAGETABLE_ENTRIES(
+ CONFIG_MSM_KGSL_PAGE_TABLE_SIZE);
pagetable->pool = gen_pool_create(PAGE_SHIFT, -1);
if (pagetable->pool == NULL) {
KGSL_CORE_ERR("gen_pool_create(%d) failed\n", PAGE_SHIFT);
- goto err_flushfilter;
+ goto err_alloc;
}
- if (gen_pool_add(pagetable->pool, pagetable->va_base,
- pagetable->va_range, -1)) {
+ if (gen_pool_add(pagetable->pool, KGSL_PAGETABLE_BASE,
+ CONFIG_MSM_KGSL_PAGE_TABLE_SIZE, -1)) {
KGSL_CORE_ERR("gen_pool_add failed\n");
goto err_pool;
}
- pagetable->base.hostptr = kgsl_ptpool_alloc(&kgsl_driver.ptpool,
- &pagetable->base.physaddr);
+ if (KGSL_MMU_TYPE_GPU == kgsl_mmu_type)
+ pagetable->pt_ops = &gpummu_pt_ops;
+ else if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_type)
+ pagetable->pt_ops = &iommu_pt_ops;
- if (pagetable->base.hostptr == NULL)
+ pagetable->priv = pagetable->pt_ops->mmu_create_pagetable();
+ if (!pagetable->priv)
goto err_pool;
- /* ptpool allocations are from coherent memory, so update the
- device statistics acordingly */
-
- KGSL_STATS_ADD(KGSL_PAGETABLE_SIZE, kgsl_driver.stats.coherent,
- kgsl_driver.stats.coherent_max);
-
- pagetable->base.gpuaddr = pagetable->base.physaddr;
- pagetable->base.size = KGSL_PAGETABLE_SIZE;
-
status = kgsl_setup_pt(pagetable);
if (status)
- goto err_free_sharedmem;
+ goto err_mmu_create;
spin_lock_irqsave(&kgsl_driver.ptlock, flags);
list_add(&pagetable->list, &kgsl_driver.pagetable_list);
@@ -744,12 +439,10 @@
return pagetable;
-err_free_sharedmem:
- kgsl_ptpool_free(&kgsl_driver.ptpool, &pagetable->base.hostptr);
+err_mmu_create:
+ pagetable->pt_ops->mmu_destroy_pagetable(pagetable->priv);
err_pool:
gen_pool_destroy(pagetable->pool);
-err_flushfilter:
- kfree(pagetable->tlbflushfilter.base);
err_alloc:
kfree(pagetable);
@@ -760,6 +453,15 @@
{
struct kgsl_pagetable *pt;
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
+ return (void *)(-1);
+
+#ifdef CONFIG_KGSL_PER_PROCESS_PAGE_TABLE
+ if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_type)
+ name = KGSL_MMU_GLOBAL_PT;
+#else
+ name = KGSL_MMU_GLOBAL_PT;
+#endif
pt = kgsl_get_pagetable(name);
if (pt == NULL)
@@ -772,84 +474,29 @@
{
kgsl_put_pagetable(pagetable);
}
-
-void kgsl_default_setstate(struct kgsl_device *device, uint32_t flags)
-{
- if (!kgsl_mmu_enabled())
- return;
-
- if (flags & KGSL_MMUFLAGS_PTUPDATE) {
- kgsl_idle(device, KGSL_TIMEOUT_DEFAULT);
- kgsl_regwrite(device, MH_MMU_PT_BASE,
- device->mmu.hwpagetable->base.gpuaddr);
- }
-
- if (flags & KGSL_MMUFLAGS_TLBFLUSH) {
- /* Invalidate all and tc */
- kgsl_regwrite(device, MH_MMU_INVALIDATE, 0x00000003);
- }
-}
-EXPORT_SYMBOL(kgsl_default_setstate);
+EXPORT_SYMBOL(kgsl_mmu_putpagetable);
void kgsl_setstate(struct kgsl_device *device, uint32_t flags)
{
- if (device->ftbl->setstate)
+ struct kgsl_mmu *mmu = &device->mmu;
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
+ return;
+ else if (device->ftbl->setstate)
device->ftbl->setstate(device, flags);
+ else if (mmu->mmu_ops->mmu_device_setstate)
+ mmu->mmu_ops->mmu_device_setstate(device, flags);
}
EXPORT_SYMBOL(kgsl_setstate);
-void kgsl_mmu_setstate(struct kgsl_device *device,
- struct kgsl_pagetable *pagetable)
+void kgsl_mmu_device_setstate(struct kgsl_device *device, uint32_t flags)
{
struct kgsl_mmu *mmu = &device->mmu;
-
- if (mmu->flags & KGSL_FLAGS_STARTED) {
- /* page table not current, then setup mmu to use new
- * specified page table
- */
- if (mmu->hwpagetable != pagetable) {
- mmu->hwpagetable = pagetable;
- spin_lock(&mmu->hwpagetable->lock);
- mmu->hwpagetable->tlb_flags &= ~(1<<device->id);
- spin_unlock(&mmu->hwpagetable->lock);
-
- /* call device specific set page table */
- kgsl_setstate(mmu->device, KGSL_MMUFLAGS_TLBFLUSH |
- KGSL_MMUFLAGS_PTUPDATE);
- }
- }
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
+ return;
+ else if (mmu->mmu_ops->mmu_device_setstate)
+ mmu->mmu_ops->mmu_device_setstate(device, flags);
}
-EXPORT_SYMBOL(kgsl_mmu_setstate);
-
-int kgsl_mmu_init(struct kgsl_device *device)
-{
- /*
- * intialize device mmu
- *
- * call this with the global lock held
- */
- int status = 0;
- struct kgsl_mmu *mmu = &device->mmu;
-
- mmu->device = device;
-
- /* make sure aligned to pagesize */
- BUG_ON(device->mh.mpu_base & (PAGE_SIZE - 1));
- BUG_ON((device->mh.mpu_base + device->mh.mpu_range) & (PAGE_SIZE - 1));
-
- /* sub-client MMU lookups require address translation */
- if ((mmu->config & ~0x1) > 0) {
- /*make sure virtual address range is a multiple of 64Kb */
- BUG_ON(CONFIG_MSM_KGSL_PAGE_TABLE_SIZE & ((1 << 16) - 1));
-
- /* allocate memory used for completing r/w operations that
- * cannot be mapped by the MMU
- */
- status = kgsl_allocate_contiguous(&mmu->dummyspace, 64);
- }
-
- return status;
-}
+EXPORT_SYMBOL(kgsl_mmu_device_setstate);
void kgsl_mh_start(struct kgsl_device *device)
{
@@ -878,54 +525,6 @@
*/
}
-int kgsl_mmu_start(struct kgsl_device *device)
-{
- /*
- * intialize device mmu
- *
- * call this with the global lock held
- */
-
- struct kgsl_mmu *mmu = &device->mmu;
-
- if (mmu->flags & KGSL_FLAGS_STARTED)
- return 0;
- /* setup MMU and sub-client behavior */
- kgsl_regwrite(device, MH_MMU_CONFIG, mmu->config);
- kgsl_idle(device, KGSL_TIMEOUT_DEFAULT);
-
- kgsl_sharedmem_set(&mmu->dummyspace, 0, 0,
- mmu->dummyspace.size);
-
- /* TRAN_ERROR needs a 32 byte (32 byte aligned) chunk of memory
- * to complete transactions in case of an MMU fault. Note that
- * we'll leave the bottom 32 bytes of the dummyspace for other
- * purposes (e.g. use it when dummy read cycles are needed
- * for other blocks */
- kgsl_regwrite(device, MH_MMU_TRAN_ERROR,
- mmu->dummyspace.physaddr + 32);
-
- if (mmu->defaultpagetable == NULL)
- mmu->defaultpagetable =
- kgsl_mmu_getpagetable(KGSL_MMU_GLOBAL_PT);
-
- /* Return error if the default pagetable doesn't exist */
- if (mmu->defaultpagetable == NULL)
- return -ENOMEM;
-
- mmu->hwpagetable = mmu->defaultpagetable;
-
- kgsl_regwrite(device, MH_MMU_PT_BASE,
- mmu->hwpagetable->base.gpuaddr);
- kgsl_regwrite(device, MH_MMU_VA_RANGE,
- (mmu->hwpagetable->va_base |
- (mmu->hwpagetable->va_range >> 16)));
- kgsl_setstate(device, KGSL_MMUFLAGS_TLBFLUSH);
- mmu->flags |= KGSL_FLAGS_STARTED;
- return 0;
-}
-EXPORT_SYMBOL(kgsl_mmu_start);
-
unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr)
{
unsigned int physaddr = 0;
@@ -962,14 +561,12 @@
struct kgsl_memdesc *memdesc,
unsigned int protflags)
{
- int numpages;
- unsigned int pte, ptefirst, ptelast, physaddr;
- int flushtlb;
- unsigned int offset = 0;
+ int ret;
- BUG_ON(protflags & ~(GSL_PT_PAGE_RV | GSL_PT_PAGE_WV));
- BUG_ON(protflags == 0);
-
+ if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE) {
+ memdesc->gpuaddr = memdesc->physaddr;
+ return 0;
+ }
memdesc->gpuaddr = gen_pool_alloc_aligned(pagetable->pool,
memdesc->size, KGSL_MMU_ALIGN_SHIFT);
@@ -981,36 +578,11 @@
return -ENOMEM;
}
- numpages = (memdesc->size >> PAGE_SHIFT);
-
- ptefirst = kgsl_pt_entry_get(pagetable, memdesc->gpuaddr);
- ptelast = ptefirst + numpages;
-
- pte = ptefirst;
- flushtlb = 0;
-
- /* tlb needs to be flushed when the first and last pte are not at
- * superpte boundaries */
- if ((ptefirst & (GSL_PT_SUPER_PTE - 1)) != 0 ||
- ((ptelast + 1) & (GSL_PT_SUPER_PTE-1)) != 0)
- flushtlb = 1;
-
spin_lock(&pagetable->lock);
- for (pte = ptefirst; pte < ptelast; pte++, offset += PAGE_SIZE) {
-#ifdef VERBOSE_DEBUG
- /* check if PTE exists */
- uint32_t val = kgsl_pt_map_getaddr(pagetable, pte);
- BUG_ON(val != 0 && val != GSL_PT_PAGE_DIRTY);
-#endif
- if ((pte & (GSL_PT_SUPER_PTE-1)) == 0)
- if (GSL_TLBFLUSH_FILTER_ISDIRTY(pte / GSL_PT_SUPER_PTE))
- flushtlb = 1;
- /* mark pte as in use */
+ ret = pagetable->pt_ops->mmu_map(pagetable->priv, memdesc, protflags);
- physaddr = memdesc->ops->physaddr(memdesc, offset);
- BUG_ON(physaddr == 0);
- kgsl_pt_map_set(pagetable, pte, physaddr | protflags);
- }
+ if (ret)
+ goto err_free_gpuaddr;
/* Keep track of the statistics for the sysfs files */
@@ -1020,70 +592,40 @@
KGSL_STATS_ADD(memdesc->size, pagetable->stats.mapped,
pagetable->stats.max_mapped);
- /* Post all writes to the pagetable */
- wmb();
-
- /* Invalidate tlb only if current page table used by GPU is the
- * pagetable that we used to allocate */
- if (flushtlb) {
- /*set all devices as needing flushing*/
- pagetable->tlb_flags = UINT_MAX;
- GSL_TLBFLUSH_FILTER_RESET();
- }
spin_unlock(&pagetable->lock);
return 0;
+
+err_free_gpuaddr:
+ spin_unlock(&pagetable->lock);
+ gen_pool_free(pagetable->pool, memdesc->gpuaddr, memdesc->size);
+ memdesc->gpuaddr = 0;
+ return ret;
}
+EXPORT_SYMBOL(kgsl_mmu_map);
int
kgsl_mmu_unmap(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
- unsigned int numpages;
- unsigned int pte, ptefirst, ptelast, superpte;
- unsigned int range = memdesc->size;
-
- /* All GPU addresses as assigned are page aligned, but some
- functions purturb the gpuaddr with an offset, so apply the
- mask here to make sure we have the right address */
-
- unsigned int gpuaddr = memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK;
-
- if (range == 0 || gpuaddr == 0)
+ if (memdesc->size == 0 || memdesc->gpuaddr == 0)
return 0;
- numpages = (range >> PAGE_SHIFT);
- if (range & (PAGE_SIZE - 1))
- numpages++;
-
- ptefirst = kgsl_pt_entry_get(pagetable, gpuaddr);
- ptelast = ptefirst + numpages;
-
- spin_lock(&pagetable->lock);
- superpte = ptefirst - (ptefirst & (GSL_PT_SUPER_PTE-1));
- GSL_TLBFLUSH_FILTER_SETDIRTY(superpte / GSL_PT_SUPER_PTE);
- for (pte = ptefirst; pte < ptelast; pte++) {
-#ifdef VERBOSE_DEBUG
- /* check if PTE exists */
- BUG_ON(!kgsl_pt_map_getaddr(pagetable, pte));
-#endif
- kgsl_pt_map_set(pagetable, pte, GSL_PT_PAGE_DIRTY);
- superpte = pte - (pte & (GSL_PT_SUPER_PTE - 1));
- if (pte == superpte)
- GSL_TLBFLUSH_FILTER_SETDIRTY(superpte /
- GSL_PT_SUPER_PTE);
+ if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE) {
+ memdesc->gpuaddr = 0;
+ return 0;
}
-
+ spin_lock(&pagetable->lock);
+ pagetable->pt_ops->mmu_unmap(pagetable->priv, memdesc);
/* Remove the statistics */
pagetable->stats.entries--;
- pagetable->stats.mapped -= range;
-
- /* Post all writes to the pagetable */
- wmb();
+ pagetable->stats.mapped -= memdesc->size;
spin_unlock(&pagetable->lock);
- gen_pool_free(pagetable->pool, gpuaddr, range);
+ gen_pool_free(pagetable->pool,
+ memdesc->gpuaddr & KGSL_MMU_ALIGN_MASK,
+ memdesc->size);
return 0;
}
@@ -1099,6 +641,9 @@
KGSL_CORE_ERR("invalid memdesc\n");
goto error;
}
+ /* Not all global mappings are needed for all MMU types */
+ if (!memdesc->size)
+ return 0;
gpuaddr = memdesc->gpuaddr;
@@ -1124,26 +669,91 @@
int kgsl_mmu_stop(struct kgsl_device *device)
{
struct kgsl_mmu *mmu = &device->mmu;
- kgsl_regwrite(device, MH_MMU_CONFIG, 0x00000000);
- mmu->flags &= ~KGSL_FLAGS_STARTED;
- return 0;
+
+ if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE)
+ return 0;
+ else
+ return mmu->mmu_ops->mmu_stop(device);
}
EXPORT_SYMBOL(kgsl_mmu_stop);
int kgsl_mmu_close(struct kgsl_device *device)
{
- /*
- * close device mmu
- *
- * call this with the global lock held
- */
struct kgsl_mmu *mmu = &device->mmu;
- if (mmu->dummyspace.gpuaddr)
- kgsl_sharedmem_free(&mmu->dummyspace);
-
- if (mmu->defaultpagetable)
- kgsl_mmu_putpagetable(mmu->defaultpagetable);
-
- return 0;
+ if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE)
+ return 0;
+ else
+ return mmu->mmu_ops->mmu_close(device);
}
+EXPORT_SYMBOL(kgsl_mmu_close);
+
+int kgsl_mmu_pt_get_flags(struct kgsl_pagetable *pt,
+ enum kgsl_deviceid id)
+{
+ if (KGSL_MMU_TYPE_GPU == kgsl_mmu_type)
+ return pt->pt_ops->mmu_pt_get_flags(pt, id);
+ else
+ return 0;
+}
+EXPORT_SYMBOL(kgsl_mmu_pt_get_flags);
+
+void kgsl_mmu_ptpool_destroy(void *ptpool)
+{
+ if (KGSL_MMU_TYPE_GPU == kgsl_mmu_type)
+ kgsl_gpummu_ptpool_destroy(ptpool);
+ ptpool = 0;
+}
+EXPORT_SYMBOL(kgsl_mmu_ptpool_destroy);
+
+void *kgsl_mmu_ptpool_init(int ptsize, int entries)
+{
+ if (KGSL_MMU_TYPE_GPU == kgsl_mmu_type)
+ return kgsl_gpummu_ptpool_init(ptsize, entries);
+ else
+ return (void *)(-1);
+}
+EXPORT_SYMBOL(kgsl_mmu_ptpool_init);
+
+int kgsl_mmu_enabled(void)
+{
+ if (KGSL_MMU_TYPE_NONE != kgsl_mmu_type)
+ return 1;
+ else
+ return 0;
+}
+EXPORT_SYMBOL(kgsl_mmu_enabled);
+
+int kgsl_mmu_pt_equal(struct kgsl_pagetable *pt,
+ unsigned int pt_base)
+{
+ if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
+ return true;
+ else
+ return pt->pt_ops->mmu_pt_equal(pt, pt_base);
+}
+EXPORT_SYMBOL(kgsl_mmu_pt_equal);
+
+enum kgsl_mmutype kgsl_mmu_get_mmutype(void)
+{
+ return kgsl_mmu_type;
+}
+EXPORT_SYMBOL(kgsl_mmu_get_mmutype);
+
+void kgsl_mmu_set_mmutype(char *mmutype)
+{
+ kgsl_mmu_type = KGSL_MMU_TYPE_NONE;
+#ifdef CONFIG_MSM_KGSL_GPUMMU
+ kgsl_mmu_type = KGSL_MMU_TYPE_GPU;
+#elif defined(CONFIG_MSM_KGSL_IOMMU)
+ if (iommu_found())
+ kgsl_mmu_type = KGSL_MMU_TYPE_IOMMU;
+#endif
+ if (mmutype && !strncmp(mmutype, "gpummu", 6))
+ kgsl_mmu_type = KGSL_MMU_TYPE_GPU;
+ if (iommu_found() && mmutype && !strncmp(mmutype, "iommu", 5))
+ kgsl_mmu_type = KGSL_MMU_TYPE_IOMMU;
+ if (mmutype && !strncmp(mmutype, "nommu", 5))
+ kgsl_mmu_type = KGSL_MMU_TYPE_NONE;
+}
+EXPORT_SYMBOL(kgsl_mmu_set_mmutype);
diff --git a/drivers/gpu/msm/kgsl_mmu.h b/drivers/gpu/msm/kgsl_mmu.h
index 4e90b8f..4af073a 100644
--- a/drivers/gpu/msm/kgsl_mmu.h
+++ b/drivers/gpu/msm/kgsl_mmu.h
@@ -13,12 +13,17 @@
#ifndef __KGSL_MMU_H
#define __KGSL_MMU_H
+#define KGSL_MMU_ALIGN_SHIFT 13
+#define KGSL_MMU_ALIGN_MASK (~((1 << KGSL_MMU_ALIGN_SHIFT) - 1))
+
/* Identifier for the global page table */
/* Per process page tables will probably pass in the thread group
as an identifier */
#define KGSL_MMU_GLOBAL_PT 0
+struct kgsl_device;
+
#define GSL_PT_SUPER_PTE 8
#define GSL_PT_PAGE_WV 0x00000001
#define GSL_PT_PAGE_RV 0x00000002
@@ -82,43 +87,19 @@
MH_INTERRUPT_MASK__AXI_WRITE_ERROR)
#endif
-/* Macros to manage TLB flushing */
-#define GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS (sizeof(unsigned char) * 8)
-#define GSL_TLBFLUSH_FILTER_GET(superpte) \
- (*((unsigned char *) \
- (((unsigned int)pagetable->tlbflushfilter.base) \
- + (superpte / GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS))))
-#define GSL_TLBFLUSH_FILTER_SETDIRTY(superpte) \
- (GSL_TLBFLUSH_FILTER_GET((superpte)) |= 1 << \
- (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS))
-#define GSL_TLBFLUSH_FILTER_ISDIRTY(superpte) \
- (GSL_TLBFLUSH_FILTER_GET((superpte)) & \
- (1 << (superpte % GSL_TLBFLUSH_FILTER_ENTRY_NUMBITS)))
-#define GSL_TLBFLUSH_FILTER_RESET() memset(pagetable->tlbflushfilter.base,\
- 0, pagetable->tlbflushfilter.size)
-
-
-struct kgsl_device;
-
-struct kgsl_tlbflushfilter {
- unsigned int *base;
- unsigned int size;
+enum kgsl_mmutype {
+ KGSL_MMU_TYPE_GPU = 0,
+ KGSL_MMU_TYPE_IOMMU,
+ KGSL_MMU_TYPE_NONE
};
struct kgsl_pagetable {
spinlock_t lock;
struct kref refcount;
- struct kgsl_memdesc base;
- uint32_t va_base;
- unsigned int va_range;
- unsigned int last_superpte;
unsigned int max_entries;
struct gen_pool *pool;
struct list_head list;
unsigned int name;
- /* Maintain filter to manage tlb flushing */
- struct kgsl_tlbflushfilter tlbflushfilter;
- unsigned int tlb_flags;
struct kobject *kobj;
struct {
@@ -127,6 +108,36 @@
unsigned int max_mapped;
unsigned int max_entries;
} stats;
+ const struct kgsl_mmu_pt_ops *pt_ops;
+ void *priv;
+};
+
+struct kgsl_mmu_ops {
+ int (*mmu_init) (struct kgsl_device *device);
+ int (*mmu_close) (struct kgsl_device *device);
+ int (*mmu_start) (struct kgsl_device *device);
+ int (*mmu_stop) (struct kgsl_device *device);
+ void (*mmu_setstate) (struct kgsl_device *device,
+ struct kgsl_pagetable *pagetable);
+ void (*mmu_device_setstate) (struct kgsl_device *device,
+ uint32_t flags);
+ void (*mmu_pagefault) (struct kgsl_device *device);
+ unsigned int (*mmu_get_current_ptbase)
+ (struct kgsl_device *device);
+};
+
+struct kgsl_mmu_pt_ops {
+ int (*mmu_map) (void *mmu_pt,
+ struct kgsl_memdesc *memdesc,
+ unsigned int protflags);
+ int (*mmu_unmap) (void *mmu_pt,
+ struct kgsl_memdesc *memdesc);
+ void *(*mmu_create_pagetable) (void);
+ void (*mmu_destroy_pagetable) (void *pt);
+ int (*mmu_pt_equal) (struct kgsl_pagetable *pt,
+ unsigned int pt_base);
+ unsigned int (*mmu_pt_get_flags) (struct kgsl_pagetable *pt,
+ enum kgsl_deviceid id);
};
struct kgsl_mmu {
@@ -134,37 +145,27 @@
uint32_t flags;
struct kgsl_device *device;
unsigned int config;
- struct kgsl_memdesc dummyspace;
+ struct kgsl_memdesc setstate_memory;
/* current page table object being used by device mmu */
struct kgsl_pagetable *defaultpagetable;
struct kgsl_pagetable *hwpagetable;
+ const struct kgsl_mmu_ops *mmu_ops;
+ void *priv;
};
-struct kgsl_ptpool_chunk {
- size_t size;
- unsigned int count;
- int dynamic;
+#include "kgsl_gpummu.h"
- void *data;
- unsigned int phys;
-
- unsigned long *bitmap;
- struct list_head list;
-};
+extern struct kgsl_mmu_ops iommu_ops;
+extern struct kgsl_mmu_pt_ops iommu_pt_ops;
struct kgsl_pagetable *kgsl_mmu_getpagetable(unsigned long name);
-
+void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable);
void kgsl_mh_start(struct kgsl_device *device);
void kgsl_mh_intrcallback(struct kgsl_device *device);
-
-#ifdef CONFIG_MSM_KGSL_MMU
-
int kgsl_mmu_init(struct kgsl_device *device);
int kgsl_mmu_start(struct kgsl_device *device);
int kgsl_mmu_stop(struct kgsl_device *device);
int kgsl_mmu_close(struct kgsl_device *device);
-void kgsl_mmu_setstate(struct kgsl_device *device,
- struct kgsl_pagetable *pagetable);
int kgsl_mmu_map(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc,
unsigned int protflags);
@@ -172,112 +173,21 @@
struct kgsl_memdesc *memdesc, unsigned int protflags);
int kgsl_mmu_unmap(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc);
-void kgsl_ptpool_destroy(struct kgsl_ptpool *pool);
-int kgsl_ptpool_init(struct kgsl_ptpool *pool, int ptsize, int entries);
-void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable);
unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr);
void kgsl_setstate(struct kgsl_device *device, uint32_t flags);
-void kgsl_default_setstate(struct kgsl_device *device, uint32_t flags);
-int kgsl_get_ptname_from_ptbase(unsigned int pt_base);
+void kgsl_mmu_device_setstate(struct kgsl_device *device, uint32_t flags);
+void kgsl_mmu_setstate(struct kgsl_device *device,
+ struct kgsl_pagetable *pt);
+int kgsl_mmu_get_ptname_from_ptbase(unsigned int pt_base);
+int kgsl_mmu_pt_get_flags(struct kgsl_pagetable *pt,
+ enum kgsl_deviceid id);
-static inline int kgsl_mmu_enabled(void)
-{
- return 1;
-}
-
-#else
-
-static inline int kgsl_mmu_enabled(void)
-{
- return 0;
-}
-
-static inline int kgsl_mmu_init(struct kgsl_device *device)
-{
- return 0;
-}
-
-static inline int kgsl_mmu_start(struct kgsl_device *device)
-{
- return 0;
-}
-
-static inline int kgsl_mmu_stop(struct kgsl_device *device)
-{
- return 0;
-}
-
-static inline int kgsl_mmu_close(struct kgsl_device *device)
-{
- return 0;
-}
-
-static inline void kgsl_mmu_setstate(struct kgsl_device *device,
- struct kgsl_pagetable *pagetable) { }
-
-static inline int kgsl_mmu_map(struct kgsl_pagetable *pagetable,
- struct kgsl_memdesc *memdesc,
- unsigned int protflags)
-{
- memdesc->gpuaddr = memdesc->physaddr;
- return 0;
-}
-
-static inline int kgsl_mmu_unmap(struct kgsl_pagetable *pagetable,
- struct kgsl_memdesc *memdesc)
-{
- return 0;
-}
-
-static inline int kgsl_ptpool_init(struct kgsl_ptpool *pool, int ptsize,
- int entries)
-{
- return 0;
-}
-
-static inline int kgsl_mmu_map_global(struct kgsl_pagetable *pagetable,
- struct kgsl_memdesc *memdesc, unsigned int protflags)
-{
- memdesc->gpuaddr = memdesc->physaddr;
- return 0;
-}
-
-static inline void kgsl_ptpool_destroy(struct kgsl_ptpool *pool) { }
-
-static inline void kgsl_mh_intrcallback(struct kgsl_device *device) { }
-
-static inline void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable) { }
-
-static inline unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr)
-{
- return 0;
-}
-
-static inline void kgsl_setstate(struct kgsl_device *device, uint32_t flags)
-{ }
-
-static inline void kgsl_default_setstate(struct kgsl_device *device,
- uint32_t flags) { }
-
-static inline int kgsl_get_ptname_from_ptbase(unsigned int pt_base) { }
-
-#endif
-
-static inline unsigned int kgsl_pt_get_flags(struct kgsl_pagetable *pt,
- enum kgsl_deviceid id)
-{
- unsigned int result = 0;
-
- if (pt == NULL)
- return 0;
-
- spin_lock(&pt->lock);
- if (pt->tlb_flags && (1<<id)) {
- result = KGSL_MMUFLAGS_TLBFLUSH;
- pt->tlb_flags &= ~(1<<id);
- }
- spin_unlock(&pt->lock);
- return result;
-}
-
+void kgsl_mmu_ptpool_destroy(void *ptpool);
+void *kgsl_mmu_ptpool_init(int ptsize, int entries);
+int kgsl_mmu_enabled(void);
+int kgsl_mmu_pt_equal(struct kgsl_pagetable *pt,
+ unsigned int pt_base);
+void kgsl_mmu_set_mmutype(char *mmutype);
+unsigned int kgsl_mmu_get_current_ptbase(struct kgsl_device *device);
+enum kgsl_mmutype kgsl_mmu_get_mmutype(void);
#endif /* __KGSL_MMU_H */
diff --git a/drivers/gpu/msm/z180.c b/drivers/gpu/msm/z180.c
index a8aff37..37ba621 100644
--- a/drivers/gpu/msm/z180.c
+++ b/drivers/gpu/msm/z180.c
@@ -258,7 +258,7 @@
{
struct z180_device *z180_dev = Z180_DEVICE(device);
- kgsl_mmu_unmap(pagetable, &device->mmu.dummyspace);
+ kgsl_mmu_unmap(pagetable, &device->mmu.setstate_memory);
kgsl_mmu_unmap(pagetable, &device->memstore);
@@ -271,7 +271,7 @@
int result = 0;
struct z180_device *z180_dev = Z180_DEVICE(device);
- result = kgsl_mmu_map_global(pagetable, &device->mmu.dummyspace,
+ result = kgsl_mmu_map_global(pagetable, &device->mmu.setstate_memory,
GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
if (result)
@@ -290,7 +290,7 @@
return result;
error_unmap_dummy:
- kgsl_mmu_unmap(pagetable, &device->mmu.dummyspace);
+ kgsl_mmu_unmap(pagetable, &device->mmu.setstate_memory);
error_unmap_memstore:
kgsl_mmu_unmap(pagetable, &device->memstore);
@@ -387,11 +387,6 @@
return status;
}
-static void z180_setstate(struct kgsl_device *device, uint32_t flags)
-{
- kgsl_default_setstate(device, flags);
-}
-
int
z180_cmdstream_issueibcmds(struct kgsl_device_private *dev_priv,
struct kgsl_context *context,
@@ -439,7 +434,7 @@
cnt = PACKETSIZE_STATESTREAM;
ofs = 0;
}
- z180_setstate(device, kgsl_pt_get_flags(device->mmu.hwpagetable,
+ kgsl_setstate(device, kgsl_mmu_pt_get_flags(device->mmu.hwpagetable,
device->id));
result = wait_event_interruptible_timeout(device->wait_queue,
@@ -626,16 +621,12 @@
break;
case KGSL_PROP_MMU_ENABLE:
{
-#ifdef CONFIG_MSM_KGSL_MMU
- int mmuProp = 1;
-#else
- int mmuProp = 0;
-#endif
+ int mmu_prop = kgsl_mmu_enabled();
if (sizebytes != sizeof(int)) {
status = -EINVAL;
break;
}
- if (copy_to_user(value, &mmuProp, sizeof(mmuProp))) {
+ if (copy_to_user(value, &mmu_prop, sizeof(mmu_prop))) {
status = -EFAULT;
break;
}
@@ -910,7 +901,6 @@
.power_stats = z180_power_stats,
.irqctrl = z180_irqctrl,
/* Optional functions */
- .setstate = z180_setstate,
.drawctxt_create = NULL,
.drawctxt_destroy = z180_drawctxt_destroy,
.ioctl = NULL,