msm: kgsl: Synchronize access to IOMMU cfg port
Add a software based spinlock between CPU and GPU.
This spinlock is used to grant mutually exclusive access to
SMMU configuration between CPU and GPU. This mutual exclusion
is required to prevent deadlock in the system.
CRs-Fixed: 409198
Change-Id: Ic375beaaf4c5505b41d3fabc4adf15965d71b13a
Signed-off-by: Tarun Karra <tkarra@codeaurora.org>
Signed-off-by: Jordan Crouse <jcrouse@codeaurora.org>
Signed-off-by: Rajeev Kulkarnie <krajeev@codeaurora.org>
diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c
index e05dc28..7cdeb45 100644
--- a/drivers/gpu/msm/kgsl_iommu.c
+++ b/drivers/gpu/msm/kgsl_iommu.c
@@ -18,6 +18,9 @@
#include <linux/iommu.h>
#include <linux/msm_kgsl.h>
#include <mach/socinfo.h>
+#include <mach/msm_iomap.h>
+#include <mach/board.h>
+#include <stddef.h>
#include "kgsl.h"
#include "kgsl_device.h"
@@ -27,6 +30,8 @@
#include "adreno_pm4types.h"
#include "adreno.h"
#include "kgsl_trace.h"
+#include "z180.h"
+
static struct kgsl_iommu_register_list kgsl_iommuv1_reg[KGSL_IOMMU_REG_MAX] = {
{ 0, 0, 0 }, /* GLOBAL_BASE */
@@ -46,6 +51,8 @@
{ 0x008, 0, 0 } /* RESUME */
};
+struct remote_iommu_petersons_spinlock kgsl_iommu_sync_lock_vars;
+
static int get_iommu_unit(struct device *dev, struct kgsl_mmu **mmu_out,
struct kgsl_iommu_unit **iommu_unit_out)
{
@@ -546,6 +553,183 @@
}
/*
+ * kgsl_get_sync_lock - Init Sync Lock between GPU and CPU
+ * @mmu - Pointer to mmu device
+ *
+ * Return - 0 on success else error code
+ */
+static int kgsl_iommu_init_sync_lock(struct kgsl_mmu *mmu)
+{
+ struct kgsl_iommu *iommu = mmu->device->mmu.priv;
+ int status = 0;
+ struct kgsl_pagetable *pagetable = NULL;
+ uint32_t lock_gpu_addr = 0;
+ uint32_t lock_phy_addr = 0;
+ uint32_t page_offset = 0;
+
+ iommu->sync_lock_initialized = 0;
+
+ if (!(mmu->flags & KGSL_MMU_FLAGS_IOMMU_SYNC)) {
+ KGSL_DRV_ERR(mmu->device,
+ "The GPU microcode does not support IOMMUv1 sync opcodes\n");
+ return -ENXIO;
+ }
+
+ /* Get the physical address of the Lock variables */
+ lock_phy_addr = (msm_iommu_lock_initialize()
+ - MSM_SHARED_RAM_BASE + msm_shared_ram_phys);
+
+ if (!lock_phy_addr) {
+ KGSL_DRV_ERR(mmu->device,
+ "GPU CPU sync lock is not supported by kernel\n");
+ return -ENXIO;
+ }
+
+ /* Align the physical address to PAGE boundary and store the offset */
+ page_offset = (lock_phy_addr & (PAGE_SIZE - 1));
+ lock_phy_addr = (lock_phy_addr & ~(PAGE_SIZE - 1));
+ iommu->sync_lock_desc.physaddr = (unsigned int)lock_phy_addr;
+
+ iommu->sync_lock_desc.size =
+ PAGE_ALIGN(sizeof(kgsl_iommu_sync_lock_vars));
+ status = memdesc_sg_phys(&iommu->sync_lock_desc,
+ iommu->sync_lock_desc.physaddr,
+ iommu->sync_lock_desc.size);
+
+ if (status)
+ return status;
+
+ /* Map Lock variables to GPU pagetable */
+ iommu->sync_lock_desc.priv |= KGSL_MEMFLAGS_GLOBAL;
+
+ pagetable = mmu->priv_bank_table ? mmu->priv_bank_table :
+ mmu->defaultpagetable;
+
+ status = kgsl_mmu_map(pagetable, &iommu->sync_lock_desc,
+ GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
+
+ if (status) {
+ kgsl_mmu_unmap(pagetable, &iommu->sync_lock_desc);
+ iommu->sync_lock_desc.priv &= ~KGSL_MEMFLAGS_GLOBAL;
+ return status;
+ }
+
+ /* Store Lock variables GPU address */
+ lock_gpu_addr = (iommu->sync_lock_desc.gpuaddr + page_offset);
+
+ kgsl_iommu_sync_lock_vars.flag[PROC_APPS] = (lock_gpu_addr +
+ (offsetof(struct remote_iommu_petersons_spinlock,
+ flag[PROC_APPS])));
+ kgsl_iommu_sync_lock_vars.flag[PROC_GPU] = (lock_gpu_addr +
+ (offsetof(struct remote_iommu_petersons_spinlock,
+ flag[PROC_GPU])));
+ kgsl_iommu_sync_lock_vars.turn = (lock_gpu_addr +
+ (offsetof(struct remote_iommu_petersons_spinlock, turn)));
+
+ iommu->sync_lock_vars = &kgsl_iommu_sync_lock_vars;
+
+ /* Flag Sync Lock is Initialized */
+ iommu->sync_lock_initialized = 1;
+
+ return status;
+}
+
+/*
+ * kgsl_iommu_sync_lock - Acquire Sync Lock between GPU and CPU
+ * @mmu - Pointer to mmu device
+ * @cmds - Pointer to array of commands
+ *
+ * Return - int - number of commands.
+ */
+inline unsigned int kgsl_iommu_sync_lock(struct kgsl_mmu *mmu,
+ unsigned int *cmds)
+{
+ struct kgsl_device *device = mmu->device;
+ struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+ struct kgsl_iommu *iommu = mmu->device->mmu.priv;
+ struct remote_iommu_petersons_spinlock *lock_vars =
+ iommu->sync_lock_vars;
+ unsigned int *start = cmds;
+
+ if (!iommu->sync_lock_initialized)
+ return 0;
+
+ *cmds++ = cp_type3_packet(CP_MEM_WRITE, 2);
+ *cmds++ = lock_vars->flag[PROC_GPU];
+ *cmds++ = 1;
+
+ cmds += adreno_add_idle_cmds(adreno_dev, cmds);
+
+ *cmds++ = cp_type3_packet(CP_WAIT_REG_MEM, 5);
+ /* MEM SPACE = memory, FUNCTION = equals */
+ *cmds++ = 0x13;
+ *cmds++ = lock_vars->flag[PROC_GPU];
+ *cmds++ = 0x1;
+ *cmds++ = 0x1;
+ *cmds++ = 0x1;
+
+ *cmds++ = cp_type3_packet(CP_MEM_WRITE, 2);
+ *cmds++ = lock_vars->turn;
+ *cmds++ = 0;
+
+ cmds += adreno_add_idle_cmds(adreno_dev, cmds);
+
+ *cmds++ = cp_type3_packet(CP_WAIT_REG_MEM, 5);
+ /* MEM SPACE = memory, FUNCTION = equals */
+ *cmds++ = 0x13;
+ *cmds++ = lock_vars->flag[PROC_GPU];
+ *cmds++ = 0x1;
+ *cmds++ = 0x1;
+ *cmds++ = 0x1;
+
+ *cmds++ = cp_type3_packet(CP_TEST_TWO_MEMS, 3);
+ *cmds++ = lock_vars->flag[PROC_APPS];
+ *cmds++ = lock_vars->turn;
+ *cmds++ = 0;
+
+ cmds += adreno_add_idle_cmds(adreno_dev, cmds);
+
+ return cmds - start;
+}
+
+/*
+ * kgsl_iommu_sync_lock - Release Sync Lock between GPU and CPU
+ * @mmu - Pointer to mmu device
+ * @cmds - Pointer to array of commands
+ *
+ * Return - int - number of commands.
+ */
+inline unsigned int kgsl_iommu_sync_unlock(struct kgsl_mmu *mmu,
+ unsigned int *cmds)
+{
+ struct kgsl_device *device = mmu->device;
+ struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+ struct kgsl_iommu *iommu = mmu->device->mmu.priv;
+ struct remote_iommu_petersons_spinlock *lock_vars =
+ iommu->sync_lock_vars;
+ unsigned int *start = cmds;
+
+ if (!iommu->sync_lock_initialized)
+ return 0;
+
+ *cmds++ = cp_type3_packet(CP_MEM_WRITE, 2);
+ *cmds++ = lock_vars->flag[PROC_GPU];
+ *cmds++ = 0;
+
+ *cmds++ = cp_type3_packet(CP_WAIT_REG_MEM, 5);
+ /* MEM SPACE = memory, FUNCTION = equals */
+ *cmds++ = 0x13;
+ *cmds++ = lock_vars->flag[PROC_GPU];
+ *cmds++ = 0x0;
+ *cmds++ = 0x1;
+ *cmds++ = 0x1;
+
+ cmds += adreno_add_idle_cmds(adreno_dev, cmds);
+
+ return cmds - start;
+}
+
+/*
* kgsl_get_iommu_ctxt - Get device pointer to IOMMU contexts
* @mmu - Pointer to mmu device
*
@@ -823,6 +1007,7 @@
static int kgsl_iommu_start(struct kgsl_mmu *mmu)
{
+ struct kgsl_device *device = mmu->device;
int status;
struct kgsl_iommu *iommu = mmu->priv;
int i, j;
@@ -834,7 +1019,13 @@
status = kgsl_iommu_setup_defaultpagetable(mmu);
if (status)
return -ENOMEM;
+
+ /* Initialize the sync lock between GPU and CPU */
+ if (msm_soc_version_supports_iommu_v1() &&
+ (device->id == KGSL_DEVICE_3D0))
+ kgsl_iommu_init_sync_lock(mmu);
}
+
/* We use the GPU MMU to control access to IOMMU registers on 8960 with
* a225, hence we still keep the MMU active on 8960 */
if (cpu_is_msm8960()) {
@@ -1071,10 +1262,7 @@
pt_base &= (iommu->iommu_reg_list[KGSL_IOMMU_CTX_TTBR0].reg_mask <<
iommu->iommu_reg_list[KGSL_IOMMU_CTX_TTBR0].reg_shift);
- /* For v1 SMMU GPU needs to be idle for tlb invalidate as well */
- if (msm_soc_version_supports_iommu_v1())
- kgsl_idle(mmu->device);
-
+ //if (msm_soc_version_supports_iommu_v1())
/* Acquire GPU-CPU sync Lock here */
msm_iommu_lock();
@@ -1105,6 +1293,10 @@
mb();
}
}
+
+ /* Release GPU-CPU sync Lock here */
+ msm_iommu_unlock();
+
/* Disable smmu clock */
kgsl_iommu_disable_clk_on_ts(mmu, 0, false);
}
@@ -1154,6 +1346,8 @@
.mmu_get_num_iommu_units = kgsl_iommu_get_num_iommu_units,
.mmu_pt_equal = kgsl_iommu_pt_equal,
.mmu_get_pt_base_addr = kgsl_iommu_get_pt_base_addr,
+ .mmu_sync_lock = kgsl_iommu_sync_lock,
+ .mmu_sync_unlock = kgsl_iommu_sync_unlock,
};
struct kgsl_mmu_pt_ops iommu_pt_ops = {