msm: kgsl: Add per context timestamp
Add new ioctls for per context timestamps.
Timestamp functions (read/write/wait) will now be context
specific rather than only using the global timestamp.
Per context timestamps is a requirement for priority
based queueing.
Change-Id: I5fdfb816480241b9552ecf90ed1bb22db3a652b5
Signed-off-by: Carter Cooper <ccooper@codeaurora.org>
diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c
index 45bcf69..693aa74 100644
--- a/drivers/gpu/msm/kgsl.c
+++ b/drivers/gpu/msm/kgsl.c
@@ -59,22 +59,30 @@
* @returns - 0 on success or error code on failure
*/
-static int kgsl_add_event(struct kgsl_device *device, u32 ts,
- void (*cb)(struct kgsl_device *, void *, u32), void *priv,
+static int kgsl_add_event(struct kgsl_device *device, u32 id, u32 ts,
+ void (*cb)(struct kgsl_device *, void *, u32, u32), void *priv,
struct kgsl_device_private *owner)
{
struct kgsl_event *event;
struct list_head *n;
- unsigned int cur = device->ftbl->readtimestamp(device,
- KGSL_TIMESTAMP_RETIRED);
+ unsigned int cur_ts;
+ struct kgsl_context *context = NULL;
if (cb == NULL)
return -EINVAL;
+ if (id != KGSL_MEMSTORE_GLOBAL) {
+ context = idr_find(&device->context_idr, id);
+ if (context == NULL)
+ return -EINVAL;
+ }
+ cur_ts = device->ftbl->readtimestamp(device, context,
+ KGSL_TIMESTAMP_RETIRED);
+
/* Check to see if the requested timestamp has already fired */
- if (timestamp_cmp(cur, ts) >= 0) {
- cb(device, priv, cur);
+ if (timestamp_cmp(cur_ts, ts) >= 0) {
+ cb(device, priv, id, cur_ts);
return 0;
}
@@ -82,17 +90,24 @@
if (event == NULL)
return -ENOMEM;
+ event->context = context;
event->timestamp = ts;
event->priv = priv;
event->func = cb;
event->owner = owner;
- /* Add the event in order to the list */
+ /*
+ * Add the event in order to the list. Order is by context id
+ * first and then by timestamp for that context.
+ */
for (n = device->events.next ; n != &device->events; n = n->next) {
struct kgsl_event *e =
list_entry(n, struct kgsl_event, list);
+ if (e->context != context)
+ continue;
+
if (timestamp_cmp(e->timestamp, ts) > 0) {
list_add(&event->list, n->prev);
break;
@@ -116,12 +131,16 @@
struct kgsl_device_private *owner)
{
struct kgsl_event *event, *event_tmp;
- unsigned int cur = device->ftbl->readtimestamp(device,
- KGSL_TIMESTAMP_RETIRED);
+ unsigned int id, cur;
list_for_each_entry_safe(event, event_tmp, &device->events, list) {
if (event->owner != owner)
continue;
+
+ cur = device->ftbl->readtimestamp(device, event->context,
+ KGSL_TIMESTAMP_RETIRED);
+
+ id = event->context ? event->context->id : KGSL_MEMSTORE_GLOBAL;
/*
* "cancel" the events by calling their callback.
* Currently, events are used for lock and memory
@@ -129,7 +148,7 @@
* thing to do is release or free.
*/
if (event->func)
- event->func(device, event->priv, cur);
+ event->func(device, event->priv, id, cur);
list_del(&event->list);
kfree(event);
@@ -286,8 +305,8 @@
return NULL;
}
- ret = idr_get_new(&dev_priv->device->context_idr,
- context, &id);
+ ret = idr_get_new_above(&dev_priv->device->context_idr,
+ context, 1, &id);
if (ret != -EAGAIN)
break;
@@ -298,6 +317,16 @@
return NULL;
}
+ /* MAX - 1, there is one memdesc in memstore for device info */
+ if (id >= KGSL_MEMSTORE_MAX) {
+ KGSL_DRV_ERR(dev_priv->device, "cannot have more than %d "
+ "ctxts due to memstore limitation\n",
+ KGSL_MEMSTORE_MAX);
+ idr_remove(&dev_priv->device->context_idr, id);
+ kfree(context);
+ return NULL;
+ }
+
context->id = id;
context->dev_priv = dev_priv;
@@ -328,25 +357,28 @@
ts_expired_ws);
struct kgsl_event *event, *event_tmp;
uint32_t ts_processed;
+ unsigned int id;
mutex_lock(&device->mutex);
- /* get current EOP timestamp */
- ts_processed = device->ftbl->readtimestamp(device,
- KGSL_TIMESTAMP_RETIRED);
-
/* Process expired events */
list_for_each_entry_safe(event, event_tmp, &device->events, list) {
+ ts_processed = device->ftbl->readtimestamp(device,
+ event->context, KGSL_TIMESTAMP_RETIRED);
if (timestamp_cmp(ts_processed, event->timestamp) < 0)
- break;
+ continue;
+
+ id = event->context ? event->context->id : KGSL_MEMSTORE_GLOBAL;
if (event->func)
- event->func(device, event->priv, ts_processed);
+ event->func(device, event->priv, id, ts_processed);
list_del(&event->list);
kfree(event);
}
+ device->last_expired_ctxt_id = KGSL_CONTEXT_INVALID;
+
mutex_unlock(&device->mutex);
}
@@ -421,11 +453,15 @@
}
EXPORT_SYMBOL(kgsl_unregister_ts_notifier);
-int kgsl_check_timestamp(struct kgsl_device *device, unsigned int timestamp)
+int kgsl_check_timestamp(struct kgsl_device *device,
+ struct kgsl_context *context, unsigned int timestamp)
{
unsigned int ts_processed;
+ unsigned int global;
- ts_processed = device->ftbl->readtimestamp(device,
+ ts_processed = device->ftbl->readtimestamp(device, context,
+ KGSL_TIMESTAMP_RETIRED);
+ global = device->ftbl->readtimestamp(device, NULL,
KGSL_TIMESTAMP_RETIRED);
return (timestamp_cmp(ts_processed, timestamp) >= 0);
@@ -761,6 +797,9 @@
kgsl_check_suspended(device);
if (device->open_count == 0) {
+ kgsl_sharedmem_set(&device->memstore, 0, 0,
+ device->memstore.size);
+
result = device->ftbl->start(device, true);
if (result) {
@@ -910,23 +949,23 @@
return result;
}
-static long kgsl_ioctl_device_waittimestamp(struct kgsl_device_private
- *dev_priv, unsigned int cmd,
- void *data)
+static long _device_waittimestamp(struct kgsl_device_private *dev_priv,
+ struct kgsl_context *context,
+ unsigned int timestamp,
+ unsigned int timeout)
{
int result = 0;
- struct kgsl_device_waittimestamp *param = data;
- /* Set the active count so that suspend doesn't do the
- wrong thing */
+ /* Set the active count so that suspend doesn't do the wrong thing */
dev_priv->device->active_cnt++;
- trace_kgsl_waittimestamp_entry(dev_priv->device, param);
+ trace_kgsl_waittimestamp_entry(dev_priv->device,
+ context ? context->id : KGSL_MEMSTORE_GLOBAL,
+ timestamp, timeout);
result = dev_priv->device->ftbl->waittimestamp(dev_priv->device,
- param->timestamp,
- param->timeout);
+ context, timestamp, timeout);
trace_kgsl_waittimestamp_exit(dev_priv->device, result);
@@ -939,6 +978,34 @@
return result;
}
+static long kgsl_ioctl_device_waittimestamp(struct kgsl_device_private
+ *dev_priv, unsigned int cmd,
+ void *data)
+{
+ struct kgsl_device_waittimestamp *param = data;
+
+ return _device_waittimestamp(dev_priv, KGSL_MEMSTORE_GLOBAL,
+ param->timestamp, param->timeout);
+}
+
+static long kgsl_ioctl_device_waittimestamp_ctxtid(struct kgsl_device_private
+ *dev_priv, unsigned int cmd,
+ void *data)
+{
+ struct kgsl_device_waittimestamp_ctxtid *param = data;
+ struct kgsl_context *context;
+
+ context = kgsl_find_context(dev_priv, param->context_id);
+ if (context == NULL) {
+ KGSL_DRV_ERR(dev_priv->device, "invalid context_id %d\n",
+ param->context_id);
+ return -EINVAL;
+ }
+
+ return _device_waittimestamp(dev_priv, context,
+ param->timestamp, param->timeout);
+}
+
static long kgsl_ioctl_rb_issueibcmds(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
@@ -951,7 +1018,7 @@
if (context == NULL) {
result = -EINVAL;
KGSL_DRV_ERR(dev_priv->device,
- "invalid drawctxt drawctxt_id %d\n",
+ "invalid context_id %d\n",
param->drawctxt_id);
goto done;
}
@@ -1019,60 +1086,117 @@
return result;
}
+static long _cmdstream_readtimestamp(struct kgsl_device_private *dev_priv,
+ struct kgsl_context *context, unsigned int type,
+ unsigned int *timestamp)
+{
+ *timestamp = dev_priv->device->ftbl->readtimestamp(dev_priv->device,
+ context, type);
+
+ trace_kgsl_readtimestamp(dev_priv->device,
+ context ? context->id : KGSL_MEMSTORE_GLOBAL,
+ type, *timestamp);
+
+ return 0;
+}
+
static long kgsl_ioctl_cmdstream_readtimestamp(struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data)
{
struct kgsl_cmdstream_readtimestamp *param = data;
- param->timestamp =
- dev_priv->device->ftbl->readtimestamp(dev_priv->device,
- param->type);
+ return _cmdstream_readtimestamp(dev_priv, NULL,
+ param->type, ¶m->timestamp);
+}
- trace_kgsl_readtimestamp(dev_priv->device, param);
+static long kgsl_ioctl_cmdstream_readtimestamp_ctxtid(struct kgsl_device_private
+ *dev_priv, unsigned int cmd,
+ void *data)
+{
+ struct kgsl_cmdstream_readtimestamp_ctxtid *param = data;
+ struct kgsl_context *context;
- return 0;
+ context = kgsl_find_context(dev_priv, param->context_id);
+ if (context == NULL) {
+ KGSL_DRV_ERR(dev_priv->device, "invalid context_id %d\n",
+ param->context_id);
+ return -EINVAL;
+ }
+
+ return _cmdstream_readtimestamp(dev_priv, context,
+ param->type, ¶m->timestamp);
}
static void kgsl_freemem_event_cb(struct kgsl_device *device,
- void *priv, u32 timestamp)
+ void *priv, u32 id, u32 timestamp)
{
struct kgsl_mem_entry *entry = priv;
spin_lock(&entry->priv->mem_lock);
rb_erase(&entry->node, &entry->priv->mem_rb);
spin_unlock(&entry->priv->mem_lock);
- trace_kgsl_mem_timestamp_free(entry, timestamp);
+ trace_kgsl_mem_timestamp_free(entry, id, timestamp);
kgsl_mem_entry_detach_process(entry);
}
+static long _cmdstream_freememontimestamp(struct kgsl_device_private *dev_priv,
+ unsigned int gpuaddr, struct kgsl_context *context,
+ unsigned int timestamp, unsigned int type)
+{
+ int result = 0;
+ struct kgsl_mem_entry *entry = NULL;
+ struct kgsl_device *device = dev_priv->device;
+ unsigned int cur;
+ unsigned int context_id = context ? context->id : KGSL_MEMSTORE_GLOBAL;
+
+ spin_lock(&dev_priv->process_priv->mem_lock);
+ entry = kgsl_sharedmem_find(dev_priv->process_priv, gpuaddr);
+ spin_unlock(&dev_priv->process_priv->mem_lock);
+
+ if (entry) {
+ cur = device->ftbl->readtimestamp(device, context,
+ KGSL_TIMESTAMP_RETIRED);
+
+ trace_kgsl_mem_timestamp_queue(entry, context_id, cur);
+ result = kgsl_add_event(dev_priv->device, context_id,
+ timestamp, kgsl_freemem_event_cb,
+ entry, dev_priv);
+ } else {
+ KGSL_DRV_ERR(dev_priv->device,
+ "invalid gpuaddr %08x\n", gpuaddr);
+ result = -EINVAL;
+ }
+
+ return result;
+}
+
static long kgsl_ioctl_cmdstream_freememontimestamp(struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data)
{
- int result = 0;
struct kgsl_cmdstream_freememontimestamp *param = data;
- struct kgsl_mem_entry *entry = NULL;
- struct kgsl_device *device = dev_priv->device;
- unsigned int cur;
- spin_lock(&dev_priv->process_priv->mem_lock);
- entry = kgsl_sharedmem_find(dev_priv->process_priv, param->gpuaddr);
- spin_unlock(&dev_priv->process_priv->mem_lock);
+ return _cmdstream_freememontimestamp(dev_priv, param->gpuaddr,
+ NULL, param->timestamp, param->type);
+}
- if (entry) {
- cur = device->ftbl->readtimestamp(device,
- KGSL_TIMESTAMP_RETIRED);
+static long kgsl_ioctl_cmdstream_freememontimestamp_ctxtid(
+ struct kgsl_device_private
+ *dev_priv, unsigned int cmd,
+ void *data)
+{
+ struct kgsl_cmdstream_freememontimestamp_ctxtid *param = data;
+ struct kgsl_context *context;
- trace_kgsl_mem_timestamp_queue(entry, cur);
- result = kgsl_add_event(dev_priv->device, param->timestamp,
- kgsl_freemem_event_cb, entry, dev_priv);
- } else {
+ context = kgsl_find_context(dev_priv, param->context_id);
+ if (context == NULL) {
KGSL_DRV_ERR(dev_priv->device,
- "invalid gpuaddr %08x\n", param->gpuaddr);
- result = -EINVAL;
+ "invalid drawctxt context_id %d\n", param->context_id);
+ return -EINVAL;
}
- return result;
+ return _cmdstream_freememontimestamp(dev_priv, param->gpuaddr,
+ context, param->timestamp, param->type);
}
static long kgsl_ioctl_drawctxt_create(struct kgsl_device_private *dev_priv,
@@ -1095,7 +1219,6 @@
context, param->flags);
param->drawctxt_id = context->id;
-
done:
if (result && context)
kgsl_destroy_context(dev_priv, context);
@@ -1799,13 +1922,14 @@
* kgsl_genlock_event_cb - Event callback for a genlock timestamp event
* @device - The KGSL device that expired the timestamp
* @priv - private data for the event
+ * @context_id - the context id that goes with the timestamp
* @timestamp - the timestamp that triggered the event
*
* Release a genlock lock following the expiration of a timestamp
*/
static void kgsl_genlock_event_cb(struct kgsl_device *device,
- void *priv, u32 timestamp)
+ void *priv, u32 context_id, u32 timestamp)
{
struct kgsl_genlock_event_priv *ev = priv;
int ret;
@@ -1833,7 +1957,7 @@
*/
static int kgsl_add_genlock_event(struct kgsl_device *device,
- u32 timestamp, void __user *data, int len,
+ u32 context_id, u32 timestamp, void __user *data, int len,
struct kgsl_device_private *owner)
{
struct kgsl_genlock_event_priv *event;
@@ -1859,8 +1983,8 @@
return ret;
}
- ret = kgsl_add_event(device, timestamp, kgsl_genlock_event_cb, event,
- owner);
+ ret = kgsl_add_event(device, context_id, timestamp,
+ kgsl_genlock_event_cb, event, owner);
if (ret)
kfree(event);
@@ -1868,7 +1992,7 @@
}
#else
static long kgsl_add_genlock_event(struct kgsl_device *device,
- u32 timestamp, void __user *data, int len,
+ u32 context_id, u32 timestamp, void __user *data, int len,
struct kgsl_device_private *owner)
{
return -EINVAL;
@@ -1892,8 +2016,8 @@
switch (param->type) {
case KGSL_TIMESTAMP_EVENT_GENLOCK:
ret = kgsl_add_genlock_event(dev_priv->device,
- param->timestamp, param->priv, param->len,
- dev_priv);
+ param->context_id, param->timestamp, param->priv,
+ param->len, dev_priv);
break;
default:
ret = -EINVAL;
@@ -1917,12 +2041,18 @@
kgsl_ioctl_device_getproperty, 1),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP,
kgsl_ioctl_device_waittimestamp, 1),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP_CTXTID,
+ kgsl_ioctl_device_waittimestamp_ctxtid, 1),
KGSL_IOCTL_FUNC(IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS,
kgsl_ioctl_rb_issueibcmds, 1),
KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP,
kgsl_ioctl_cmdstream_readtimestamp, 1),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_CTXTID,
+ kgsl_ioctl_cmdstream_readtimestamp_ctxtid, 1),
KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP,
kgsl_ioctl_cmdstream_freememontimestamp, 1),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_CTXTID,
+ kgsl_ioctl_cmdstream_freememontimestamp_ctxtid, 1),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_CREATE,
kgsl_ioctl_drawctxt_create, 1),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_DESTROY,
@@ -2252,13 +2382,13 @@
INIT_LIST_HEAD(&device->events);
+ device->last_expired_ctxt_id = KGSL_CONTEXT_INVALID;
+
ret = kgsl_mmu_init(device);
if (ret != 0)
goto err_dest_work_q;
- ret = kgsl_allocate_contiguous(&device->memstore,
- sizeof(struct kgsl_devmemstore));
-
+ ret = kgsl_allocate_contiguous(&device->memstore, KGSL_MEMSTORE_SIZE);
if (ret != 0)
goto err_close_mmu;