blob: 65eed586aae0881326baf9f80afb43c7e3418cba [file] [log] [blame]
Duy Truonge833aca2013-02-12 13:35:08 -08001/* Copyright (c) 2008-2012, The Linux Foundation. All rights reserved.
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 */
Steve Mucklef132c6c2012-06-06 18:30:57 -070013#include <linux/module.h>
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070014#include <linux/fb.h>
15#include <linux/file.h>
16#include <linux/fs.h>
17#include <linux/debugfs.h>
18#include <linux/uaccess.h>
19#include <linux/interrupt.h>
20#include <linux/workqueue.h>
21#include <linux/android_pmem.h>
22#include <linux/vmalloc.h>
23#include <linux/pm_runtime.h>
Jordan Croused4bc9d22011-11-17 13:39:21 -070024#include <linux/genlock.h>
Jordan Crousec9559e42012-04-05 16:55:56 -060025#include <linux/rbtree.h>
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070026#include <linux/ashmem.h>
27#include <linux/major.h>
Mitchel Humpherys53044702012-09-06 10:36:51 -070028#include <linux/msm_ion.h>
Jeremy Gebben4204d0f2012-03-01 16:06:21 -070029#include <linux/io.h>
30#include <mach/socinfo.h>
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070031
32#include "kgsl.h"
33#include "kgsl_debugfs.h"
34#include "kgsl_cffdump.h"
35#include "kgsl_log.h"
36#include "kgsl_sharedmem.h"
37#include "kgsl_device.h"
Norman Geed7402ff2011-10-28 08:51:11 -060038#include "kgsl_trace.h"
Jeff Boodyfe6c39c2012-08-09 13:54:50 -060039#include "kgsl_sync.h"
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070040
41#undef MODULE_PARAM_PREFIX
42#define MODULE_PARAM_PREFIX "kgsl."
43
44static int kgsl_pagetable_count = KGSL_PAGETABLE_COUNT;
Shubhraprakash Das767fdda2011-08-15 15:49:45 -060045static char *ksgl_mmu_type;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070046module_param_named(ptcount, kgsl_pagetable_count, int, 0);
47MODULE_PARM_DESC(kgsl_pagetable_count,
48"Minimum number of pagetables for KGSL to allocate at initialization time");
Shubhraprakash Das767fdda2011-08-15 15:49:45 -060049module_param_named(mmutype, ksgl_mmu_type, charp, 0);
50MODULE_PARM_DESC(ksgl_mmu_type,
51"Type of MMU to be used for graphics. Valid values are 'iommu' or 'gpummu' or 'nommu'");
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070052
Jordan Crouse8eab35a2011-10-12 16:57:48 -060053static struct ion_client *kgsl_ion_client;
54
Jordan Crouse0fdf3a02012-03-16 14:53:41 -060055/* kgsl_get_mem_entry - get the mem_entry structure for the specified object
Shubhraprakash Das3cf33be2012-08-16 22:42:55 -070056 * @device - Pointer to the device structure
Jordan Crouse0fdf3a02012-03-16 14:53:41 -060057 * @ptbase - the pagetable base of the object
58 * @gpuaddr - the GPU address of the object
59 * @size - Size of the region to search
60 */
61
Shubhraprakash Das3cf33be2012-08-16 22:42:55 -070062struct kgsl_mem_entry *kgsl_get_mem_entry(struct kgsl_device *device,
63 unsigned int ptbase, unsigned int gpuaddr, unsigned int size)
Jordan Crouse0fdf3a02012-03-16 14:53:41 -060064{
65 struct kgsl_process_private *priv;
66 struct kgsl_mem_entry *entry;
67
68 mutex_lock(&kgsl_driver.process_mutex);
69
70 list_for_each_entry(priv, &kgsl_driver.process_list, list) {
Shubhraprakash Das3cf33be2012-08-16 22:42:55 -070071 if (!kgsl_mmu_pt_equal(&device->mmu, priv->pagetable, ptbase))
Jordan Crouse0fdf3a02012-03-16 14:53:41 -060072 continue;
73 spin_lock(&priv->mem_lock);
74 entry = kgsl_sharedmem_find_region(priv, gpuaddr, size);
75
76 if (entry) {
77 spin_unlock(&priv->mem_lock);
78 mutex_unlock(&kgsl_driver.process_mutex);
79 return entry;
80 }
81 spin_unlock(&priv->mem_lock);
82 }
83 mutex_unlock(&kgsl_driver.process_mutex);
84
85 return NULL;
86}
87EXPORT_SYMBOL(kgsl_get_mem_entry);
88
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070089static inline struct kgsl_mem_entry *
90kgsl_mem_entry_create(void)
91{
92 struct kgsl_mem_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
93
94 if (!entry)
95 KGSL_CORE_ERR("kzalloc(%d) failed\n", sizeof(*entry));
96 else
97 kref_init(&entry->refcount);
98
99 return entry;
100}
101
102void
103kgsl_mem_entry_destroy(struct kref *kref)
104{
105 struct kgsl_mem_entry *entry = container_of(kref,
106 struct kgsl_mem_entry,
107 refcount);
Jordan Crouse1b897cf2011-10-12 16:57:48 -0600108
Jordan Crouse1b897cf2011-10-12 16:57:48 -0600109 if (entry->memtype != KGSL_MEM_ENTRY_KERNEL)
110 kgsl_driver.stats.mapped -= entry->memdesc.size;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700111
Jordan Crouse8eab35a2011-10-12 16:57:48 -0600112 /*
Laura Abbottb14ed962012-01-30 14:18:08 -0800113 * Ion takes care of freeing the sglist for us so
114 * clear the sg before freeing the sharedmem so kgsl_sharedmem_free
Jordan Crouse8eab35a2011-10-12 16:57:48 -0600115 * doesn't try to free it again
116 */
117
118 if (entry->memtype == KGSL_MEM_ENTRY_ION) {
Jordan Crouse8eab35a2011-10-12 16:57:48 -0600119 entry->memdesc.sg = NULL;
120 }
121
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700122 kgsl_sharedmem_free(&entry->memdesc);
123
Jordan Crouse1b897cf2011-10-12 16:57:48 -0600124 switch (entry->memtype) {
125 case KGSL_MEM_ENTRY_PMEM:
126 case KGSL_MEM_ENTRY_ASHMEM:
127 if (entry->priv_data)
128 fput(entry->priv_data);
129 break;
Jordan Crouse8eab35a2011-10-12 16:57:48 -0600130 case KGSL_MEM_ENTRY_ION:
131 ion_free(kgsl_ion_client, entry->priv_data);
132 break;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700133 }
134
135 kfree(entry);
136}
137EXPORT_SYMBOL(kgsl_mem_entry_destroy);
138
139static
140void kgsl_mem_entry_attach_process(struct kgsl_mem_entry *entry,
141 struct kgsl_process_private *process)
142{
Jordan Crousec9559e42012-04-05 16:55:56 -0600143 struct rb_node **node;
144 struct rb_node *parent = NULL;
145
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700146 spin_lock(&process->mem_lock);
Jordan Crousec9559e42012-04-05 16:55:56 -0600147
148 node = &process->mem_rb.rb_node;
149
150 while (*node) {
151 struct kgsl_mem_entry *cur;
152
153 parent = *node;
154 cur = rb_entry(parent, struct kgsl_mem_entry, node);
155
156 if (entry->memdesc.gpuaddr < cur->memdesc.gpuaddr)
157 node = &parent->rb_left;
158 else
159 node = &parent->rb_right;
160 }
161
162 rb_link_node(&entry->node, parent, node);
163 rb_insert_color(&entry->node, &process->mem_rb);
164
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700165 spin_unlock(&process->mem_lock);
166
167 entry->priv = process;
168}
169
Jordan Crouse00714012012-03-16 14:53:40 -0600170/* Detach a memory entry from a process and unmap it from the MMU */
171
172static void kgsl_mem_entry_detach_process(struct kgsl_mem_entry *entry)
173{
174 if (entry == NULL)
175 return;
176
177 entry->priv->stats[entry->memtype].cur -= entry->memdesc.size;
178 entry->priv = NULL;
179
180 kgsl_mmu_unmap(entry->memdesc.pagetable, &entry->memdesc);
181
182 kgsl_mem_entry_put(entry);
183}
184
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700185/* Allocate a new context id */
186
187static struct kgsl_context *
188kgsl_create_context(struct kgsl_device_private *dev_priv)
189{
190 struct kgsl_context *context;
191 int ret, id;
192
193 context = kzalloc(sizeof(*context), GFP_KERNEL);
194
195 if (context == NULL)
196 return NULL;
197
198 while (1) {
199 if (idr_pre_get(&dev_priv->device->context_idr,
200 GFP_KERNEL) == 0) {
201 kfree(context);
202 return NULL;
203 }
204
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700205 ret = idr_get_new_above(&dev_priv->device->context_idr,
206 context, 1, &id);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700207
208 if (ret != -EAGAIN)
209 break;
210 }
211
212 if (ret) {
213 kfree(context);
214 return NULL;
215 }
216
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700217 /* MAX - 1, there is one memdesc in memstore for device info */
218 if (id >= KGSL_MEMSTORE_MAX) {
219 KGSL_DRV_ERR(dev_priv->device, "cannot have more than %d "
220 "ctxts due to memstore limitation\n",
221 KGSL_MEMSTORE_MAX);
222 idr_remove(&dev_priv->device->context_idr, id);
223 kfree(context);
224 return NULL;
225 }
226
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600227 kref_init(&context->refcount);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700228 context->id = id;
229 context->dev_priv = dev_priv;
230
Jeff Boodyfe6c39c2012-08-09 13:54:50 -0600231 if (kgsl_sync_timeline_create(context)) {
232 idr_remove(&dev_priv->device->context_idr, id);
Jordan Crouseef3456c2013-01-04 16:46:51 -0700233 goto func_end;
234 }
235
236 /* Initialize the pending event list */
237 INIT_LIST_HEAD(&context->events);
238
239 /*
240 * Initialize the node that is used to maintain the master list of
241 * contexts with pending events in the device structure. Normally we
242 * wouldn't take the time to initalize a node but at event add time we
243 * call list_empty() on the node as a quick way of determining if the
244 * context is already in the master list so it needs to always be either
245 * active or in an unused but initialized state
246 */
247
248 INIT_LIST_HEAD(&context->events_list);
249
250func_end:
251 if (ret) {
Jeff Boodyfe6c39c2012-08-09 13:54:50 -0600252 kfree(context);
253 return NULL;
254 }
255
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700256 return context;
257}
258
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600259/**
260 * kgsl_context_detach - Release the "master" context reference
261 * @context - The context that will be detached
262 *
263 * This is called when a context becomes unusable, because userspace
264 * has requested for it to be destroyed. The context itself may
265 * exist a bit longer until its reference count goes to zero.
266 * Other code referencing the context can detect that it has been
267 * detached because the context id will be set to KGSL_CONTEXT_INVALID.
268 */
269void
270kgsl_context_detach(struct kgsl_context *context)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700271{
272 int id;
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600273 struct kgsl_device *device;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700274 if (context == NULL)
275 return;
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600276 device = context->dev_priv->device;
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600277 trace_kgsl_context_detach(device, context);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700278 id = context->id;
Jeremy Gebben1384a6a2012-05-17 14:34:17 -0600279
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600280 if (device->ftbl->drawctxt_destroy)
281 device->ftbl->drawctxt_destroy(device, context);
282 /*device specific drawctxt_destroy MUST clean up devctxt */
283 BUG_ON(context->devctxt);
Jeremy Gebben1384a6a2012-05-17 14:34:17 -0600284 /*
285 * Cancel events after the device-specific context is
286 * destroyed, to avoid possibly freeing memory while
287 * it is still in use by the GPU.
288 */
289 kgsl_cancel_events_ctxt(device, context);
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600290 idr_remove(&device->context_idr, id);
291 context->id = KGSL_CONTEXT_INVALID;
292 kgsl_context_put(context);
293}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700294
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600295void
296kgsl_context_destroy(struct kref *kref)
297{
298 struct kgsl_context *context = container_of(kref, struct kgsl_context,
299 refcount);
Jeff Boodyfe6c39c2012-08-09 13:54:50 -0600300 kgsl_sync_timeline_destroy(context);
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600301 kfree(context);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700302}
303
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700304static void kgsl_check_idle_locked(struct kgsl_device *device)
305{
306 if (device->pwrctrl.nap_allowed == true &&
307 device->state == KGSL_STATE_ACTIVE &&
308 device->requested_state == KGSL_STATE_NONE) {
Jeremy Gebben388c2972011-12-16 09:05:07 -0700309 kgsl_pwrctrl_request_state(device, KGSL_STATE_NAP);
Lucille Sylvester721f7e72012-08-21 16:31:26 -0600310 kgsl_pwrscale_idle(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700311 if (kgsl_pwrctrl_sleep(device) != 0)
312 mod_timer(&device->idle_timer,
313 jiffies +
314 device->pwrctrl.interval_timeout);
315 }
316}
317
318static void kgsl_check_idle(struct kgsl_device *device)
319{
320 mutex_lock(&device->mutex);
321 kgsl_check_idle_locked(device);
322 mutex_unlock(&device->mutex);
323}
324
325struct kgsl_device *kgsl_get_device(int dev_idx)
326{
327 int i;
328 struct kgsl_device *ret = NULL;
329
330 mutex_lock(&kgsl_driver.devlock);
331
332 for (i = 0; i < KGSL_DEVICE_MAX; i++) {
333 if (kgsl_driver.devp[i] && kgsl_driver.devp[i]->id == dev_idx) {
334 ret = kgsl_driver.devp[i];
335 break;
336 }
337 }
338
339 mutex_unlock(&kgsl_driver.devlock);
340 return ret;
341}
342EXPORT_SYMBOL(kgsl_get_device);
343
344static struct kgsl_device *kgsl_get_minor(int minor)
345{
346 struct kgsl_device *ret = NULL;
347
348 if (minor < 0 || minor >= KGSL_DEVICE_MAX)
349 return NULL;
350
351 mutex_lock(&kgsl_driver.devlock);
352 ret = kgsl_driver.devp[minor];
353 mutex_unlock(&kgsl_driver.devlock);
354
355 return ret;
356}
357
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700358int kgsl_check_timestamp(struct kgsl_device *device,
359 struct kgsl_context *context, unsigned int timestamp)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700360{
361 unsigned int ts_processed;
362
Jeremy Gebben731dac52012-05-10 11:13:42 -0600363 ts_processed = kgsl_readtimestamp(device, context,
364 KGSL_TIMESTAMP_RETIRED);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700365
Jordan Crousee6239dd2011-11-17 13:39:21 -0700366 return (timestamp_cmp(ts_processed, timestamp) >= 0);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700367}
368EXPORT_SYMBOL(kgsl_check_timestamp);
369
370static int kgsl_suspend_device(struct kgsl_device *device, pm_message_t state)
371{
372 int status = -EINVAL;
373 unsigned int nap_allowed_saved;
374 struct kgsl_pwrscale_policy *policy_saved;
375
376 if (!device)
377 return -EINVAL;
378
379 KGSL_PWR_WARN(device, "suspend start\n");
380
381 mutex_lock(&device->mutex);
382 nap_allowed_saved = device->pwrctrl.nap_allowed;
383 device->pwrctrl.nap_allowed = false;
384 policy_saved = device->pwrscale.policy;
385 device->pwrscale.policy = NULL;
Jeremy Gebben388c2972011-12-16 09:05:07 -0700386 kgsl_pwrctrl_request_state(device, KGSL_STATE_SUSPEND);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700387 /* Make sure no user process is waiting for a timestamp *
388 * before supending */
Harsh Vardhan Dwivediae0e2ec2013-05-21 15:19:16 -0600389 if (device->state == KGSL_STATE_ACTIVE && device->active_cnt != 0) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700390 mutex_unlock(&device->mutex);
391 wait_for_completion(&device->suspend_gate);
392 mutex_lock(&device->mutex);
393 }
Suman Tatiraju4a32c652012-02-17 11:59:05 -0800394 /* Don't let the timer wake us during suspended sleep. */
395 del_timer_sync(&device->idle_timer);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700396 switch (device->state) {
397 case KGSL_STATE_INIT:
398 break;
399 case KGSL_STATE_ACTIVE:
400 /* Wait for the device to become idle */
Jordan Crousea29a2e02012-08-14 09:09:23 -0600401 device->ftbl->idle(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700402 case KGSL_STATE_NAP:
403 case KGSL_STATE_SLEEP:
404 /* Get the completion ready to be waited upon. */
405 INIT_COMPLETION(device->hwaccess_gate);
406 device->ftbl->suspend_context(device);
407 device->ftbl->stop(device);
Suman Tatiraju48e72762012-05-03 11:12:03 -0700408 pm_qos_update_request(&device->pm_qos_req_dma,
409 PM_QOS_DEFAULT_VALUE);
Jeremy Gebben388c2972011-12-16 09:05:07 -0700410 kgsl_pwrctrl_set_state(device, KGSL_STATE_SUSPEND);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700411 break;
Suman Tatiraju24569022011-10-27 11:11:12 -0700412 case KGSL_STATE_SLUMBER:
413 INIT_COMPLETION(device->hwaccess_gate);
Jeremy Gebben388c2972011-12-16 09:05:07 -0700414 kgsl_pwrctrl_set_state(device, KGSL_STATE_SUSPEND);
Suman Tatiraju24569022011-10-27 11:11:12 -0700415 break;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700416 default:
417 KGSL_PWR_ERR(device, "suspend fail, device %d\n",
418 device->id);
419 goto end;
420 }
Jeremy Gebben388c2972011-12-16 09:05:07 -0700421 kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700422 device->pwrctrl.nap_allowed = nap_allowed_saved;
423 device->pwrscale.policy = policy_saved;
424 status = 0;
425
426end:
427 mutex_unlock(&device->mutex);
428 KGSL_PWR_WARN(device, "suspend end\n");
429 return status;
430}
431
432static int kgsl_resume_device(struct kgsl_device *device)
433{
434 int status = -EINVAL;
435
436 if (!device)
437 return -EINVAL;
438
439 KGSL_PWR_WARN(device, "resume start\n");
440 mutex_lock(&device->mutex);
441 if (device->state == KGSL_STATE_SUSPEND) {
Jeremy Gebben388c2972011-12-16 09:05:07 -0700442 kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER);
Suman Tatiraju24569022011-10-27 11:11:12 -0700443 status = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700444 complete_all(&device->hwaccess_gate);
445 }
Jeremy Gebben388c2972011-12-16 09:05:07 -0700446 kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700447
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700448 mutex_unlock(&device->mutex);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700449 KGSL_PWR_WARN(device, "resume end\n");
450 return status;
451}
452
453static int kgsl_suspend(struct device *dev)
454{
455
456 pm_message_t arg = {0};
457 struct kgsl_device *device = dev_get_drvdata(dev);
458 return kgsl_suspend_device(device, arg);
459}
460
461static int kgsl_resume(struct device *dev)
462{
463 struct kgsl_device *device = dev_get_drvdata(dev);
464 return kgsl_resume_device(device);
465}
466
467static int kgsl_runtime_suspend(struct device *dev)
468{
469 return 0;
470}
471
472static int kgsl_runtime_resume(struct device *dev)
473{
474 return 0;
475}
476
477const struct dev_pm_ops kgsl_pm_ops = {
478 .suspend = kgsl_suspend,
479 .resume = kgsl_resume,
480 .runtime_suspend = kgsl_runtime_suspend,
481 .runtime_resume = kgsl_runtime_resume,
482};
483EXPORT_SYMBOL(kgsl_pm_ops);
484
485void kgsl_early_suspend_driver(struct early_suspend *h)
486{
487 struct kgsl_device *device = container_of(h,
488 struct kgsl_device, display_off);
Suman Tatiraju24569022011-10-27 11:11:12 -0700489 KGSL_PWR_WARN(device, "early suspend start\n");
Ranjhith Kalisamy8b636952011-09-03 14:48:31 +0530490 mutex_lock(&device->mutex);
Lucille Sylvester4c889b72012-08-03 11:25:25 -0600491 device->pwrctrl.restore_slumber = true;
Lucille Sylvester344e4622012-01-18 15:53:21 -0700492 kgsl_pwrctrl_request_state(device, KGSL_STATE_SLUMBER);
Suman Tatiraju24569022011-10-27 11:11:12 -0700493 kgsl_pwrctrl_sleep(device);
Ranjhith Kalisamy8b636952011-09-03 14:48:31 +0530494 mutex_unlock(&device->mutex);
Suman Tatiraju24569022011-10-27 11:11:12 -0700495 KGSL_PWR_WARN(device, "early suspend end\n");
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700496}
497EXPORT_SYMBOL(kgsl_early_suspend_driver);
498
499int kgsl_suspend_driver(struct platform_device *pdev,
500 pm_message_t state)
501{
502 struct kgsl_device *device = dev_get_drvdata(&pdev->dev);
503 return kgsl_suspend_device(device, state);
504}
505EXPORT_SYMBOL(kgsl_suspend_driver);
506
507int kgsl_resume_driver(struct platform_device *pdev)
508{
509 struct kgsl_device *device = dev_get_drvdata(&pdev->dev);
510 return kgsl_resume_device(device);
511}
512EXPORT_SYMBOL(kgsl_resume_driver);
513
514void kgsl_late_resume_driver(struct early_suspend *h)
515{
516 struct kgsl_device *device = container_of(h,
517 struct kgsl_device, display_off);
Suman Tatiraju24569022011-10-27 11:11:12 -0700518 KGSL_PWR_WARN(device, "late resume start\n");
Ranjhith Kalisamy8b636952011-09-03 14:48:31 +0530519 mutex_lock(&device->mutex);
Lucille Sylvester4c889b72012-08-03 11:25:25 -0600520 device->pwrctrl.restore_slumber = false;
Nilesh Shah94bdf2f2012-05-02 22:42:57 +0530521 if (device->pwrscale.policy == NULL)
522 kgsl_pwrctrl_pwrlevel_change(device, KGSL_PWRLEVEL_TURBO);
Suman Tatiraju3005cdd2012-03-19 14:38:11 -0700523 kgsl_pwrctrl_wake(device);
Ranjhith Kalisamy8b636952011-09-03 14:48:31 +0530524 mutex_unlock(&device->mutex);
Suman Tatiraju24569022011-10-27 11:11:12 -0700525 kgsl_check_idle(device);
526 KGSL_PWR_WARN(device, "late resume end\n");
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700527}
528EXPORT_SYMBOL(kgsl_late_resume_driver);
529
530/* file operations */
531static struct kgsl_process_private *
532kgsl_get_process_private(struct kgsl_device_private *cur_dev_priv)
533{
534 struct kgsl_process_private *private;
535
536 mutex_lock(&kgsl_driver.process_mutex);
537 list_for_each_entry(private, &kgsl_driver.process_list, list) {
538 if (private->pid == task_tgid_nr(current)) {
539 private->refcnt++;
540 goto out;
541 }
542 }
543
544 /* no existing process private found for this dev_priv, create one */
545 private = kzalloc(sizeof(struct kgsl_process_private), GFP_KERNEL);
546 if (private == NULL) {
547 KGSL_DRV_ERR(cur_dev_priv->device, "kzalloc(%d) failed\n",
548 sizeof(struct kgsl_process_private));
549 goto out;
550 }
551
552 spin_lock_init(&private->mem_lock);
553 private->refcnt = 1;
554 private->pid = task_tgid_nr(current);
Jordan Crousec9559e42012-04-05 16:55:56 -0600555 private->mem_rb = RB_ROOT;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700556
Shubhraprakash Das767fdda2011-08-15 15:49:45 -0600557 if (kgsl_mmu_enabled())
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700558 {
559 unsigned long pt_name;
560
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700561 pt_name = task_tgid_nr(current);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700562 private->pagetable = kgsl_mmu_getpagetable(pt_name);
563 if (private->pagetable == NULL) {
564 kfree(private);
565 private = NULL;
566 goto out;
567 }
568 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700569
570 list_add(&private->list, &kgsl_driver.process_list);
571
572 kgsl_process_init_sysfs(private);
Jeremy Gebbenddf93012012-09-25 10:57:38 -0600573 kgsl_process_init_debugfs(private);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700574
575out:
576 mutex_unlock(&kgsl_driver.process_mutex);
577 return private;
578}
579
580static void
581kgsl_put_process_private(struct kgsl_device *device,
582 struct kgsl_process_private *private)
583{
584 struct kgsl_mem_entry *entry = NULL;
Jordan Crousec9559e42012-04-05 16:55:56 -0600585 struct rb_node *node;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700586
587 if (!private)
588 return;
589
590 mutex_lock(&kgsl_driver.process_mutex);
591
592 if (--private->refcnt)
593 goto unlock;
594
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700595 kgsl_process_uninit_sysfs(private);
Jeremy Gebbenddf93012012-09-25 10:57:38 -0600596 debugfs_remove_recursive(private->debug_root);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700597
598 list_del(&private->list);
599
Jordan Crousec9559e42012-04-05 16:55:56 -0600600 for (node = rb_first(&private->mem_rb); node; ) {
601 entry = rb_entry(node, struct kgsl_mem_entry, node);
602 node = rb_next(&entry->node);
603
604 rb_erase(&entry->node, &private->mem_rb);
Jordan Crouse00714012012-03-16 14:53:40 -0600605 kgsl_mem_entry_detach_process(entry);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700606 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700607 kgsl_mmu_putpagetable(private->pagetable);
608 kfree(private);
609unlock:
610 mutex_unlock(&kgsl_driver.process_mutex);
611}
612
613static int kgsl_release(struct inode *inodep, struct file *filep)
614{
615 int result = 0;
Jordan Crouse2db0af92011-08-08 16:05:09 -0600616 struct kgsl_device_private *dev_priv = filep->private_data;
617 struct kgsl_process_private *private = dev_priv->process_priv;
618 struct kgsl_device *device = dev_priv->device;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700619 struct kgsl_context *context;
620 int next = 0;
621
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700622 filep->private_data = NULL;
623
624 mutex_lock(&device->mutex);
625 kgsl_check_suspended(device);
626
627 while (1) {
Jordan Crouse2db0af92011-08-08 16:05:09 -0600628 context = idr_get_next(&device->context_idr, &next);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700629 if (context == NULL)
630 break;
631
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600632 if (context->dev_priv == dev_priv)
633 kgsl_context_detach(context);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700634
635 next = next + 1;
636 }
Jeremy Gebben1384a6a2012-05-17 14:34:17 -0600637 /*
638 * Clean up any to-be-freed entries that belong to this
639 * process and this device. This is done after the context
640 * are destroyed to avoid possibly freeing memory while
641 * it is still in use by the GPU.
642 */
643 kgsl_cancel_events(device, dev_priv);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700644
645 device->open_count--;
646 if (device->open_count == 0) {
647 result = device->ftbl->stop(device);
Jeremy Gebben388c2972011-12-16 09:05:07 -0700648 kgsl_pwrctrl_set_state(device, KGSL_STATE_INIT);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700649 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700650
651 mutex_unlock(&device->mutex);
652 kfree(dev_priv);
653
654 kgsl_put_process_private(device, private);
655
656 pm_runtime_put(device->parentdev);
657 return result;
658}
659
660static int kgsl_open(struct inode *inodep, struct file *filep)
661{
662 int result;
663 struct kgsl_device_private *dev_priv;
664 struct kgsl_device *device;
665 unsigned int minor = iminor(inodep);
666
667 device = kgsl_get_minor(minor);
668 BUG_ON(device == NULL);
669
670 if (filep->f_flags & O_EXCL) {
671 KGSL_DRV_ERR(device, "O_EXCL not allowed\n");
672 return -EBUSY;
673 }
674
675 result = pm_runtime_get_sync(device->parentdev);
676 if (result < 0) {
677 KGSL_DRV_ERR(device,
678 "Runtime PM: Unable to wake up the device, rc = %d\n",
679 result);
680 return result;
681 }
682 result = 0;
683
684 dev_priv = kzalloc(sizeof(struct kgsl_device_private), GFP_KERNEL);
685 if (dev_priv == NULL) {
686 KGSL_DRV_ERR(device, "kzalloc failed(%d)\n",
687 sizeof(struct kgsl_device_private));
688 result = -ENOMEM;
689 goto err_pmruntime;
690 }
691
692 dev_priv->device = device;
693 filep->private_data = dev_priv;
694
695 /* Get file (per process) private struct */
696 dev_priv->process_priv = kgsl_get_process_private(dev_priv);
697 if (dev_priv->process_priv == NULL) {
698 result = -ENOMEM;
699 goto err_freedevpriv;
700 }
701
702 mutex_lock(&device->mutex);
703 kgsl_check_suspended(device);
704
705 if (device->open_count == 0) {
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700706 kgsl_sharedmem_set(&device->memstore, 0, 0,
707 device->memstore.size);
708
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700709 result = device->ftbl->start(device, true);
710
711 if (result) {
712 mutex_unlock(&device->mutex);
713 goto err_putprocess;
714 }
Jeremy Gebben388c2972011-12-16 09:05:07 -0700715 kgsl_pwrctrl_set_state(device, KGSL_STATE_ACTIVE);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700716 }
717 device->open_count++;
718 mutex_unlock(&device->mutex);
719
720 KGSL_DRV_INFO(device, "Initialized %s: mmu=%s pagetable_count=%d\n",
721 device->name, kgsl_mmu_enabled() ? "on" : "off",
722 kgsl_pagetable_count);
723
724 return result;
725
726err_putprocess:
727 kgsl_put_process_private(device, dev_priv->process_priv);
728err_freedevpriv:
729 filep->private_data = NULL;
730 kfree(dev_priv);
731err_pmruntime:
732 pm_runtime_put(device->parentdev);
733 return result;
734}
735
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700736/*call with private->mem_lock locked */
737struct kgsl_mem_entry *
738kgsl_sharedmem_find_region(struct kgsl_process_private *private,
Jordan Crousec9559e42012-04-05 16:55:56 -0600739 unsigned int gpuaddr, size_t size)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700740{
Jordan Crousec9559e42012-04-05 16:55:56 -0600741 struct rb_node *node = private->mem_rb.rb_node;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700742
Jordan Crousee22e21d2012-07-23 14:34:06 -0600743 if (!kgsl_mmu_gpuaddr_in_range(gpuaddr))
744 return NULL;
745
Jordan Crousec9559e42012-04-05 16:55:56 -0600746 while (node != NULL) {
747 struct kgsl_mem_entry *entry;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700748
Jordan Crousec9559e42012-04-05 16:55:56 -0600749 entry = rb_entry(node, struct kgsl_mem_entry, node);
750
751
752 if (kgsl_gpuaddr_in_memdesc(&entry->memdesc, gpuaddr, size))
753 return entry;
754
755 if (gpuaddr < entry->memdesc.gpuaddr)
756 node = node->rb_left;
757 else if (gpuaddr >=
758 (entry->memdesc.gpuaddr + entry->memdesc.size))
759 node = node->rb_right;
760 else {
761 return NULL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700762 }
763 }
764
Jordan Crousec9559e42012-04-05 16:55:56 -0600765 return NULL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700766}
767EXPORT_SYMBOL(kgsl_sharedmem_find_region);
768
Jordan Crousec9559e42012-04-05 16:55:56 -0600769/*call with private->mem_lock locked */
770static inline struct kgsl_mem_entry *
771kgsl_sharedmem_find(struct kgsl_process_private *private, unsigned int gpuaddr)
772{
773 return kgsl_sharedmem_find_region(private, gpuaddr, 1);
774}
775
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700776/*call all ioctl sub functions with driver locked*/
777static long kgsl_ioctl_device_getproperty(struct kgsl_device_private *dev_priv,
778 unsigned int cmd, void *data)
779{
780 int result = 0;
781 struct kgsl_device_getproperty *param = data;
782
783 switch (param->type) {
784 case KGSL_PROP_VERSION:
785 {
786 struct kgsl_version version;
787 if (param->sizebytes != sizeof(version)) {
788 result = -EINVAL;
789 break;
790 }
791
792 version.drv_major = KGSL_VERSION_MAJOR;
793 version.drv_minor = KGSL_VERSION_MINOR;
794 version.dev_major = dev_priv->device->ver_major;
795 version.dev_minor = dev_priv->device->ver_minor;
796
797 if (copy_to_user(param->value, &version, sizeof(version)))
798 result = -EFAULT;
799
800 break;
801 }
Shubhraprakash Das2dfe5dd2012-02-10 13:49:53 -0700802 case KGSL_PROP_GPU_RESET_STAT:
803 {
804 /* Return reset status of given context and clear it */
805 uint32_t id;
806 struct kgsl_context *context;
807
808 if (param->sizebytes != sizeof(unsigned int)) {
809 result = -EINVAL;
810 break;
811 }
812 /* We expect the value passed in to contain the context id */
813 if (copy_from_user(&id, param->value,
814 sizeof(unsigned int))) {
815 result = -EFAULT;
816 break;
817 }
818 context = kgsl_find_context(dev_priv, id);
819 if (!context) {
820 result = -EINVAL;
821 break;
822 }
823 /*
824 * Copy the reset status to value which also serves as
825 * the out parameter
826 */
827 if (copy_to_user(param->value, &(context->reset_status),
828 sizeof(unsigned int))) {
829 result = -EFAULT;
830 break;
831 }
832 /* Clear reset status once its been queried */
833 context->reset_status = KGSL_CTX_STAT_NO_ERROR;
834 break;
835 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700836 default:
837 result = dev_priv->device->ftbl->getproperty(
838 dev_priv->device, param->type,
839 param->value, param->sizebytes);
840 }
841
842
843 return result;
844}
845
Jordan Crouseed7dd7f2012-03-29 13:16:02 -0600846static long kgsl_ioctl_device_setproperty(struct kgsl_device_private *dev_priv,
847 unsigned int cmd, void *data)
848{
849 int result = 0;
850 /* The getproperty struct is reused for setproperty too */
851 struct kgsl_device_getproperty *param = data;
852
853 if (dev_priv->device->ftbl->setproperty)
854 result = dev_priv->device->ftbl->setproperty(
855 dev_priv->device, param->type,
856 param->value, param->sizebytes);
857
858 return result;
859}
860
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700861static long _device_waittimestamp(struct kgsl_device_private *dev_priv,
862 struct kgsl_context *context,
863 unsigned int timestamp,
864 unsigned int timeout)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700865{
866 int result = 0;
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600867 struct kgsl_device *device = dev_priv->device;
868 unsigned int context_id = context ? context->id : KGSL_MEMSTORE_GLOBAL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700869
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700870 /* Set the active count so that suspend doesn't do the wrong thing */
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700871
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600872 device->active_cnt++;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700873
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600874 trace_kgsl_waittimestamp_entry(device, context_id,
875 kgsl_readtimestamp(device, context,
876 KGSL_TIMESTAMP_RETIRED),
877 timestamp, timeout);
Norman Geed7402ff2011-10-28 08:51:11 -0600878
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600879 result = device->ftbl->waittimestamp(dev_priv->device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700880 context, timestamp, timeout);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700881
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600882 trace_kgsl_waittimestamp_exit(device,
883 kgsl_readtimestamp(device, context,
884 KGSL_TIMESTAMP_RETIRED),
885 result);
Norman Geed7402ff2011-10-28 08:51:11 -0600886
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700887 /* Fire off any pending suspend operations that are in flight */
Jordan Crouseab601992013-03-05 11:18:20 -0700888 kgsl_active_count_put(dev_priv->device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700889
890 return result;
891}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700892
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700893static long kgsl_ioctl_device_waittimestamp(struct kgsl_device_private
894 *dev_priv, unsigned int cmd,
895 void *data)
896{
897 struct kgsl_device_waittimestamp *param = data;
898
Jeremy Gebben4a3756c2012-05-08 16:51:43 -0600899 return _device_waittimestamp(dev_priv, NULL,
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700900 param->timestamp, param->timeout);
901}
902
903static long kgsl_ioctl_device_waittimestamp_ctxtid(struct kgsl_device_private
904 *dev_priv, unsigned int cmd,
905 void *data)
906{
907 struct kgsl_device_waittimestamp_ctxtid *param = data;
908 struct kgsl_context *context;
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600909 int result;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700910
911 context = kgsl_find_context(dev_priv, param->context_id);
Harsh Vardhan Dwivedifdd17d42013-02-28 16:59:57 -0700912 if (context == NULL)
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700913 return -EINVAL;
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600914 /*
915 * A reference count is needed here, because waittimestamp may
916 * block with the device mutex unlocked and userspace could
917 * request for the context to be destroyed during that time.
918 */
919 kgsl_context_get(context);
920 result = _device_waittimestamp(dev_priv, context,
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700921 param->timestamp, param->timeout);
Jeremy Gebben9ad86922012-05-08 15:33:23 -0600922 kgsl_context_put(context);
923 return result;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700924}
925
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700926static long kgsl_ioctl_rb_issueibcmds(struct kgsl_device_private *dev_priv,
927 unsigned int cmd, void *data)
928{
929 int result = 0;
930 struct kgsl_ringbuffer_issueibcmds *param = data;
931 struct kgsl_ibdesc *ibdesc;
932 struct kgsl_context *context;
933
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700934 context = kgsl_find_context(dev_priv, param->drawctxt_id);
935 if (context == NULL) {
936 result = -EINVAL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700937 goto done;
938 }
939
940 if (param->flags & KGSL_CONTEXT_SUBMIT_IB_LIST) {
941 KGSL_DRV_INFO(dev_priv->device,
942 "Using IB list mode for ib submission, numibs: %d\n",
943 param->numibs);
944 if (!param->numibs) {
945 KGSL_DRV_ERR(dev_priv->device,
946 "Invalid numibs as parameter: %d\n",
947 param->numibs);
948 result = -EINVAL;
949 goto done;
950 }
951
Jordan Crouse834c8592012-07-24 10:06:35 -0600952 /*
953 * Put a reasonable upper limit on the number of IBs that can be
954 * submitted
955 */
956
957 if (param->numibs > 10000) {
958 KGSL_DRV_ERR(dev_priv->device,
959 "Too many IBs submitted. count: %d max 10000\n",
960 param->numibs);
961 result = -EINVAL;
962 goto done;
963 }
964
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700965 ibdesc = kzalloc(sizeof(struct kgsl_ibdesc) * param->numibs,
966 GFP_KERNEL);
967 if (!ibdesc) {
968 KGSL_MEM_ERR(dev_priv->device,
969 "kzalloc(%d) failed\n",
970 sizeof(struct kgsl_ibdesc) * param->numibs);
971 result = -ENOMEM;
972 goto done;
973 }
974
975 if (copy_from_user(ibdesc, (void *)param->ibdesc_addr,
976 sizeof(struct kgsl_ibdesc) * param->numibs)) {
977 result = -EFAULT;
978 KGSL_DRV_ERR(dev_priv->device,
979 "copy_from_user failed\n");
980 goto free_ibdesc;
981 }
982 } else {
983 KGSL_DRV_INFO(dev_priv->device,
984 "Using single IB submission mode for ib submission\n");
985 /* If user space driver is still using the old mode of
986 * submitting single ib then we need to support that as well */
987 ibdesc = kzalloc(sizeof(struct kgsl_ibdesc), GFP_KERNEL);
988 if (!ibdesc) {
989 KGSL_MEM_ERR(dev_priv->device,
990 "kzalloc(%d) failed\n",
991 sizeof(struct kgsl_ibdesc));
992 result = -ENOMEM;
993 goto done;
994 }
995 ibdesc[0].gpuaddr = param->ibdesc_addr;
996 ibdesc[0].sizedwords = param->numibs;
997 param->numibs = 1;
998 }
999
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001000 result = dev_priv->device->ftbl->issueibcmds(dev_priv,
1001 context,
1002 ibdesc,
1003 param->numibs,
1004 &param->timestamp,
1005 param->flags);
1006
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001007 trace_kgsl_issueibcmds(dev_priv->device, param, ibdesc, result);
Wei Zouc8c01632012-03-24 17:27:26 -07001008
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001009free_ibdesc:
1010 kfree(ibdesc);
1011done:
1012
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001013 return result;
1014}
1015
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001016static long _cmdstream_readtimestamp(struct kgsl_device_private *dev_priv,
1017 struct kgsl_context *context, unsigned int type,
1018 unsigned int *timestamp)
1019{
Jeremy Gebben731dac52012-05-10 11:13:42 -06001020 *timestamp = kgsl_readtimestamp(dev_priv->device, context, type);
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001021
1022 trace_kgsl_readtimestamp(dev_priv->device,
1023 context ? context->id : KGSL_MEMSTORE_GLOBAL,
1024 type, *timestamp);
1025
1026 return 0;
1027}
1028
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001029static long kgsl_ioctl_cmdstream_readtimestamp(struct kgsl_device_private
1030 *dev_priv, unsigned int cmd,
1031 void *data)
1032{
1033 struct kgsl_cmdstream_readtimestamp *param = data;
1034
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001035 return _cmdstream_readtimestamp(dev_priv, NULL,
1036 param->type, &param->timestamp);
1037}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001038
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001039static long kgsl_ioctl_cmdstream_readtimestamp_ctxtid(struct kgsl_device_private
1040 *dev_priv, unsigned int cmd,
1041 void *data)
1042{
1043 struct kgsl_cmdstream_readtimestamp_ctxtid *param = data;
1044 struct kgsl_context *context;
Norman Geed7402ff2011-10-28 08:51:11 -06001045
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001046 context = kgsl_find_context(dev_priv, param->context_id);
Harsh Vardhan Dwivedifdd17d42013-02-28 16:59:57 -07001047 if (context == NULL)
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001048 return -EINVAL;
Harsh Vardhan Dwivedifdd17d42013-02-28 16:59:57 -07001049
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001050
1051 return _cmdstream_readtimestamp(dev_priv, context,
1052 param->type, &param->timestamp);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001053}
1054
Jeremy Gebbenc81a3c62012-02-07 16:10:23 -07001055static void kgsl_freemem_event_cb(struct kgsl_device *device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001056 void *priv, u32 id, u32 timestamp)
Jeremy Gebbenc81a3c62012-02-07 16:10:23 -07001057{
1058 struct kgsl_mem_entry *entry = priv;
1059 spin_lock(&entry->priv->mem_lock);
Jordan Crousec9559e42012-04-05 16:55:56 -06001060 rb_erase(&entry->node, &entry->priv->mem_rb);
Jeremy Gebbenc81a3c62012-02-07 16:10:23 -07001061 spin_unlock(&entry->priv->mem_lock);
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001062 trace_kgsl_mem_timestamp_free(device, entry, id, timestamp, 0);
Jordan Crouse00714012012-03-16 14:53:40 -06001063 kgsl_mem_entry_detach_process(entry);
Jeremy Gebbenc81a3c62012-02-07 16:10:23 -07001064}
1065
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001066static long _cmdstream_freememontimestamp(struct kgsl_device_private *dev_priv,
1067 unsigned int gpuaddr, struct kgsl_context *context,
1068 unsigned int timestamp, unsigned int type)
1069{
1070 int result = 0;
1071 struct kgsl_mem_entry *entry = NULL;
1072 struct kgsl_device *device = dev_priv->device;
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001073 unsigned int context_id = context ? context->id : KGSL_MEMSTORE_GLOBAL;
1074
1075 spin_lock(&dev_priv->process_priv->mem_lock);
1076 entry = kgsl_sharedmem_find(dev_priv->process_priv, gpuaddr);
1077 spin_unlock(&dev_priv->process_priv->mem_lock);
1078
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001079 if (!entry) {
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001080 KGSL_DRV_ERR(dev_priv->device,
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001081 "invalid gpuaddr %08x\n", gpuaddr);
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001082 result = -EINVAL;
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001083 goto done;
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001084 }
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001085 trace_kgsl_mem_timestamp_queue(device, entry, context_id,
1086 kgsl_readtimestamp(device, context,
1087 KGSL_TIMESTAMP_RETIRED),
1088 timestamp);
1089 result = kgsl_add_event(dev_priv->device, context_id, timestamp,
1090 kgsl_freemem_event_cb, entry, dev_priv);
1091done:
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001092 return result;
1093}
1094
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001095static long kgsl_ioctl_cmdstream_freememontimestamp(struct kgsl_device_private
1096 *dev_priv, unsigned int cmd,
1097 void *data)
1098{
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001099 struct kgsl_cmdstream_freememontimestamp *param = data;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001100
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001101 return _cmdstream_freememontimestamp(dev_priv, param->gpuaddr,
1102 NULL, param->timestamp, param->type);
1103}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001104
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001105static long kgsl_ioctl_cmdstream_freememontimestamp_ctxtid(
1106 struct kgsl_device_private
1107 *dev_priv, unsigned int cmd,
1108 void *data)
1109{
1110 struct kgsl_cmdstream_freememontimestamp_ctxtid *param = data;
1111 struct kgsl_context *context;
Jeremy Gebbena5859272012-03-01 12:46:28 -07001112
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001113 context = kgsl_find_context(dev_priv, param->context_id);
1114 if (context == NULL) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001115 KGSL_DRV_ERR(dev_priv->device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001116 "invalid drawctxt context_id %d\n", param->context_id);
1117 return -EINVAL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001118 }
1119
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001120 return _cmdstream_freememontimestamp(dev_priv, param->gpuaddr,
1121 context, param->timestamp, param->type);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001122}
1123
1124static long kgsl_ioctl_drawctxt_create(struct kgsl_device_private *dev_priv,
1125 unsigned int cmd, void *data)
1126{
1127 int result = 0;
1128 struct kgsl_drawctxt_create *param = data;
1129 struct kgsl_context *context = NULL;
1130
1131 context = kgsl_create_context(dev_priv);
1132
1133 if (context == NULL) {
1134 result = -ENOMEM;
1135 goto done;
1136 }
1137
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001138 if (dev_priv->device->ftbl->drawctxt_create) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001139 result = dev_priv->device->ftbl->drawctxt_create(
1140 dev_priv->device, dev_priv->process_priv->pagetable,
1141 context, param->flags);
Jeremy Gebben4a3756c2012-05-08 16:51:43 -06001142 if (result)
1143 goto done;
1144 }
1145 trace_kgsl_context_create(dev_priv->device, context, param->flags);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001146 param->drawctxt_id = context->id;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001147done:
1148 if (result && context)
Jeremy Gebben9ad86922012-05-08 15:33:23 -06001149 kgsl_context_detach(context);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001150
1151 return result;
1152}
1153
1154static long kgsl_ioctl_drawctxt_destroy(struct kgsl_device_private *dev_priv,
1155 unsigned int cmd, void *data)
1156{
1157 int result = 0;
1158 struct kgsl_drawctxt_destroy *param = data;
1159 struct kgsl_context *context;
1160
1161 context = kgsl_find_context(dev_priv, param->drawctxt_id);
1162
1163 if (context == NULL) {
1164 result = -EINVAL;
1165 goto done;
1166 }
1167
Jeremy Gebben9ad86922012-05-08 15:33:23 -06001168 kgsl_context_detach(context);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001169done:
1170 return result;
1171}
1172
1173static long kgsl_ioctl_sharedmem_free(struct kgsl_device_private *dev_priv,
1174 unsigned int cmd, void *data)
1175{
1176 int result = 0;
1177 struct kgsl_sharedmem_free *param = data;
1178 struct kgsl_process_private *private = dev_priv->process_priv;
1179 struct kgsl_mem_entry *entry = NULL;
1180
1181 spin_lock(&private->mem_lock);
1182 entry = kgsl_sharedmem_find(private, param->gpuaddr);
1183 if (entry)
Jordan Crousec9559e42012-04-05 16:55:56 -06001184 rb_erase(&entry->node, &private->mem_rb);
1185
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001186 spin_unlock(&private->mem_lock);
1187
1188 if (entry) {
Jeremy Gebbena5859272012-03-01 12:46:28 -07001189 trace_kgsl_mem_free(entry);
Jordan Crouse00714012012-03-16 14:53:40 -06001190 kgsl_mem_entry_detach_process(entry);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001191 } else {
1192 KGSL_CORE_ERR("invalid gpuaddr %08x\n", param->gpuaddr);
1193 result = -EINVAL;
1194 }
1195
1196 return result;
1197}
1198
1199static struct vm_area_struct *kgsl_get_vma_from_start_addr(unsigned int addr)
1200{
1201 struct vm_area_struct *vma;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001202
1203 down_read(&current->mm->mmap_sem);
1204 vma = find_vma(current->mm, addr);
1205 up_read(&current->mm->mmap_sem);
Jordan Crouse2c542b62011-07-26 08:30:20 -06001206 if (!vma)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001207 KGSL_CORE_ERR("find_vma(%x) failed\n", addr);
Jordan Crouse2c542b62011-07-26 08:30:20 -06001208
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001209 return vma;
1210}
1211
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001212static inline int _check_region(unsigned long start, unsigned long size,
1213 uint64_t len)
1214{
1215 uint64_t end = ((uint64_t) start) + size;
1216 return (end > len);
1217}
1218
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001219static int kgsl_get_phys_file(int fd, unsigned long *start, unsigned long *len,
1220 unsigned long *vstart, struct file **filep)
1221{
1222 struct file *fbfile;
1223 int ret = 0;
1224 dev_t rdev;
1225 struct fb_info *info;
1226
1227 *filep = NULL;
Jordan Crousefd978432011-09-02 14:34:32 -06001228#ifdef CONFIG_ANDROID_PMEM
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001229 if (!get_pmem_file(fd, start, vstart, len, filep))
1230 return 0;
Jordan Crousefd978432011-09-02 14:34:32 -06001231#endif
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001232
1233 fbfile = fget(fd);
1234 if (fbfile == NULL) {
1235 KGSL_CORE_ERR("fget_light failed\n");
1236 return -1;
1237 }
1238
1239 rdev = fbfile->f_dentry->d_inode->i_rdev;
1240 info = MAJOR(rdev) == FB_MAJOR ? registered_fb[MINOR(rdev)] : NULL;
1241 if (info) {
1242 *start = info->fix.smem_start;
1243 *len = info->fix.smem_len;
1244 *vstart = (unsigned long)__va(info->fix.smem_start);
1245 ret = 0;
1246 } else {
1247 KGSL_CORE_ERR("framebuffer minor %d not found\n",
1248 MINOR(rdev));
1249 ret = -1;
1250 }
1251
1252 fput(fbfile);
1253
1254 return ret;
1255}
1256
1257static int kgsl_setup_phys_file(struct kgsl_mem_entry *entry,
1258 struct kgsl_pagetable *pagetable,
1259 unsigned int fd, unsigned int offset,
1260 size_t size)
1261{
1262 int ret;
1263 unsigned long phys, virt, len;
1264 struct file *filep;
1265
1266 ret = kgsl_get_phys_file(fd, &phys, &len, &virt, &filep);
1267 if (ret)
1268 return ret;
1269
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001270 ret = -ERANGE;
1271
Wei Zou4061c0b2011-07-08 10:24:22 -07001272 if (phys == 0) {
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001273 KGSL_CORE_ERR("kgsl_get_phys_file returned phys=0\n");
Wei Zou4061c0b2011-07-08 10:24:22 -07001274 goto err;
1275 }
1276
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001277 /* Make sure the length of the region, the offset and the desired
1278 * size are all page aligned or bail
1279 */
1280 if ((len & ~PAGE_MASK) ||
1281 (offset & ~PAGE_MASK) ||
1282 (size & ~PAGE_MASK)) {
1283 KGSL_CORE_ERR("length %lu, offset %u or size %u "
1284 "is not page aligned\n",
1285 len, offset, size);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001286 goto err;
1287 }
1288
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001289 /* The size or offset can never be greater than the PMEM length */
1290 if (offset >= len || size > len) {
1291 KGSL_CORE_ERR("offset %u or size %u "
1292 "exceeds pmem length %lu\n",
1293 offset, size, len);
1294 goto err;
1295 }
1296
1297 /* If size is 0, then adjust it to default to the size of the region
1298 * minus the offset. If size isn't zero, then make sure that it will
1299 * fit inside of the region.
1300 */
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001301 if (size == 0)
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001302 size = len - offset;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001303
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001304 else if (_check_region(offset, size, len))
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001305 goto err;
1306
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001307 entry->priv_data = filep;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001308
1309 entry->memdesc.pagetable = pagetable;
1310 entry->memdesc.size = size;
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001311 entry->memdesc.physaddr = phys + offset;
1312 entry->memdesc.hostptr = (void *) (virt + offset);
Jordan Croused17e9aa2011-10-12 16:57:48 -06001313
Ajay Dudani62dd83e2012-09-11 16:38:13 -06001314 ret = memdesc_sg_phys(&entry->memdesc, phys + offset, size);
Jordan Croused17e9aa2011-10-12 16:57:48 -06001315 if (ret)
1316 goto err;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001317
1318 return 0;
1319err:
Jordan Crousefd978432011-09-02 14:34:32 -06001320#ifdef CONFIG_ANDROID_PMEM
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001321 put_pmem_file(filep);
Jordan Crousefd978432011-09-02 14:34:32 -06001322#endif
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001323 return ret;
1324}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001325
Jordan Croused17e9aa2011-10-12 16:57:48 -06001326static int memdesc_sg_virt(struct kgsl_memdesc *memdesc,
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001327 unsigned long paddr, int size)
Jordan Croused17e9aa2011-10-12 16:57:48 -06001328{
1329 int i;
1330 int sglen = PAGE_ALIGN(size) / PAGE_SIZE;
Jordan Croused17e9aa2011-10-12 16:57:48 -06001331
Jordan Crousea652a072012-04-06 16:26:33 -06001332 memdesc->sg = kgsl_sg_alloc(sglen);
1333
Jordan Croused17e9aa2011-10-12 16:57:48 -06001334 if (memdesc->sg == NULL)
1335 return -ENOMEM;
1336
1337 memdesc->sglen = sglen;
Rajeev Kulkarni8dfdc3362012-11-22 00:22:32 -08001338 memdesc->sglen_alloc = sglen;
1339
Jordan Croused17e9aa2011-10-12 16:57:48 -06001340 sg_init_table(memdesc->sg, sglen);
1341
1342 spin_lock(&current->mm->page_table_lock);
1343
1344 for (i = 0; i < sglen; i++, paddr += PAGE_SIZE) {
1345 struct page *page;
1346 pmd_t *ppmd;
1347 pte_t *ppte;
1348 pgd_t *ppgd = pgd_offset(current->mm, paddr);
1349
1350 if (pgd_none(*ppgd) || pgd_bad(*ppgd))
1351 goto err;
1352
Steve Mucklef132c6c2012-06-06 18:30:57 -07001353 ppmd = pmd_offset(pud_offset(ppgd, paddr), paddr);
Jordan Croused17e9aa2011-10-12 16:57:48 -06001354 if (pmd_none(*ppmd) || pmd_bad(*ppmd))
1355 goto err;
1356
1357 ppte = pte_offset_map(ppmd, paddr);
1358 if (ppte == NULL)
1359 goto err;
1360
1361 page = pfn_to_page(pte_pfn(*ppte));
1362 if (!page)
1363 goto err;
1364
1365 sg_set_page(&memdesc->sg[i], page, PAGE_SIZE, 0);
1366 pte_unmap(ppte);
1367 }
1368
1369 spin_unlock(&current->mm->page_table_lock);
1370
1371 return 0;
1372
1373err:
1374 spin_unlock(&current->mm->page_table_lock);
Jordan Crousea652a072012-04-06 16:26:33 -06001375 kgsl_sg_free(memdesc->sg, sglen);
Jordan Croused17e9aa2011-10-12 16:57:48 -06001376 memdesc->sg = NULL;
1377
1378 return -EINVAL;
1379}
1380
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001381static int kgsl_setup_useraddr(struct kgsl_mem_entry *entry,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001382 struct kgsl_pagetable *pagetable,
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001383 unsigned long useraddr, unsigned int offset,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001384 size_t size)
1385{
1386 struct vm_area_struct *vma;
1387 unsigned int len;
1388
1389 down_read(&current->mm->mmap_sem);
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001390 vma = find_vma(current->mm, useraddr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001391 up_read(&current->mm->mmap_sem);
1392
1393 if (!vma) {
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001394 KGSL_CORE_ERR("find_vma(%lx) failed\n", useraddr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001395 return -EINVAL;
1396 }
1397
1398 /* We don't necessarily start at vma->vm_start */
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001399 len = vma->vm_end - useraddr;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001400
1401 if (offset >= len)
1402 return -EINVAL;
1403
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001404 if (!KGSL_IS_PAGE_ALIGNED(useraddr) ||
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001405 !KGSL_IS_PAGE_ALIGNED(len)) {
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001406 KGSL_CORE_ERR("bad alignment: start(%lx) len(%u)\n",
1407 useraddr, len);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001408 return -EINVAL;
1409 }
1410
1411 if (size == 0)
1412 size = len;
1413
1414 /* Adjust the size of the region to account for the offset */
1415 size += offset & ~PAGE_MASK;
1416
1417 size = ALIGN(size, PAGE_SIZE);
1418
1419 if (_check_region(offset & PAGE_MASK, size, len)) {
1420 KGSL_CORE_ERR("Offset (%ld) + size (%d) is larger"
1421 "than region length %d\n",
1422 offset & PAGE_MASK, size, len);
1423 return -EINVAL;
1424 }
1425
1426 entry->memdesc.pagetable = pagetable;
1427 entry->memdesc.size = size;
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001428 entry->memdesc.useraddr = useraddr + (offset & PAGE_MASK);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001429
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001430 return memdesc_sg_virt(&entry->memdesc, entry->memdesc.useraddr,
1431 size);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001432}
1433
1434#ifdef CONFIG_ASHMEM
1435static int kgsl_setup_ashmem(struct kgsl_mem_entry *entry,
1436 struct kgsl_pagetable *pagetable,
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001437 int fd, unsigned long useraddr, size_t size)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001438{
1439 int ret;
1440 struct vm_area_struct *vma;
1441 struct file *filep, *vmfile;
1442 unsigned long len;
1443
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001444 vma = kgsl_get_vma_from_start_addr(useraddr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001445 if (vma == NULL)
1446 return -EINVAL;
1447
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001448 if (vma->vm_pgoff || vma->vm_start != useraddr) {
Jordan Crouse2c542b62011-07-26 08:30:20 -06001449 KGSL_CORE_ERR("Invalid vma region\n");
1450 return -EINVAL;
1451 }
1452
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001453 len = vma->vm_end - vma->vm_start;
1454
1455 if (size == 0)
1456 size = len;
1457
1458 if (size != len) {
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001459 KGSL_CORE_ERR("Invalid size %d for vma region %lx\n",
1460 size, useraddr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001461 return -EINVAL;
1462 }
1463
1464 ret = get_ashmem_file(fd, &filep, &vmfile, &len);
1465
1466 if (ret) {
1467 KGSL_CORE_ERR("get_ashmem_file failed\n");
1468 return ret;
1469 }
1470
1471 if (vmfile != vma->vm_file) {
1472 KGSL_CORE_ERR("ashmem shmem file does not match vma\n");
1473 ret = -EINVAL;
1474 goto err;
1475 }
1476
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001477 entry->priv_data = filep;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001478 entry->memdesc.pagetable = pagetable;
1479 entry->memdesc.size = ALIGN(size, PAGE_SIZE);
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001480 entry->memdesc.useraddr = useraddr;
Jordan Croused17e9aa2011-10-12 16:57:48 -06001481
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001482 ret = memdesc_sg_virt(&entry->memdesc, useraddr, size);
Jordan Croused17e9aa2011-10-12 16:57:48 -06001483 if (ret)
1484 goto err;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001485
1486 return 0;
1487
1488err:
1489 put_ashmem_file(filep);
1490 return ret;
1491}
1492#else
1493static int kgsl_setup_ashmem(struct kgsl_mem_entry *entry,
1494 struct kgsl_pagetable *pagetable,
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001495 int fd, unsigned long useraddr, size_t size)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001496{
1497 return -EINVAL;
1498}
1499#endif
1500
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001501static int kgsl_setup_ion(struct kgsl_mem_entry *entry,
1502 struct kgsl_pagetable *pagetable, int fd)
1503{
1504 struct ion_handle *handle;
1505 struct scatterlist *s;
Laura Abbottb14ed962012-01-30 14:18:08 -08001506 struct sg_table *sg_table;
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001507
Harsh Vardhan Dwivedif48af7f2012-04-13 12:50:44 -06001508 if (IS_ERR_OR_NULL(kgsl_ion_client))
1509 return -ENODEV;
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001510
Laura Abbottb14ed962012-01-30 14:18:08 -08001511 handle = ion_import_dma_buf(kgsl_ion_client, fd);
Ranjhith Kalisamy0d2e14f2012-08-14 19:49:39 +05301512 if (IS_ERR(handle))
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001513 return PTR_ERR(handle);
Ranjhith Kalisamy0d2e14f2012-08-14 19:49:39 +05301514 else if (!handle)
1515 return -EINVAL;
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001516
1517 entry->memtype = KGSL_MEM_ENTRY_ION;
1518 entry->priv_data = handle;
1519 entry->memdesc.pagetable = pagetable;
1520 entry->memdesc.size = 0;
1521
Laura Abbottb14ed962012-01-30 14:18:08 -08001522 sg_table = ion_sg_table(kgsl_ion_client, handle);
1523
1524 if (IS_ERR_OR_NULL(sg_table))
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001525 goto err;
1526
Laura Abbottb14ed962012-01-30 14:18:08 -08001527 entry->memdesc.sg = sg_table->sgl;
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001528
1529 /* Calculate the size of the memdesc from the sglist */
1530
1531 entry->memdesc.sglen = 0;
1532
1533 for (s = entry->memdesc.sg; s != NULL; s = sg_next(s)) {
1534 entry->memdesc.size += s->length;
1535 entry->memdesc.sglen++;
1536 }
1537
1538 return 0;
1539err:
1540 ion_free(kgsl_ion_client, handle);
1541 return -ENOMEM;
1542}
1543
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001544static long kgsl_ioctl_map_user_mem(struct kgsl_device_private *dev_priv,
1545 unsigned int cmd, void *data)
1546{
1547 int result = -EINVAL;
1548 struct kgsl_map_user_mem *param = data;
1549 struct kgsl_mem_entry *entry = NULL;
1550 struct kgsl_process_private *private = dev_priv->process_priv;
Jason848741a2011-07-12 10:24:25 -07001551 enum kgsl_user_mem_type memtype;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001552
1553 entry = kgsl_mem_entry_create();
1554
1555 if (entry == NULL)
1556 return -ENOMEM;
1557
Jason848741a2011-07-12 10:24:25 -07001558 if (_IOC_SIZE(cmd) == sizeof(struct kgsl_sharedmem_from_pmem))
1559 memtype = KGSL_USER_MEM_TYPE_PMEM;
1560 else
1561 memtype = param->memtype;
1562
Jordan Crousedc67dfb2012-10-25 09:41:46 -06001563 entry->memdesc.flags = param->flags;
1564
Jason848741a2011-07-12 10:24:25 -07001565 switch (memtype) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001566 case KGSL_USER_MEM_TYPE_PMEM:
1567 if (param->fd == 0 || param->len == 0)
1568 break;
1569
1570 result = kgsl_setup_phys_file(entry, private->pagetable,
1571 param->fd, param->offset,
1572 param->len);
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001573 entry->memtype = KGSL_MEM_ENTRY_PMEM;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001574 break;
1575
1576 case KGSL_USER_MEM_TYPE_ADDR:
Harsh Vardhan Dwivedia9eb7cb2012-03-26 15:21:38 -06001577 KGSL_DEV_ERR_ONCE(dev_priv->device, "User mem type "
1578 "KGSL_USER_MEM_TYPE_ADDR is deprecated\n");
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001579 if (!kgsl_mmu_enabled()) {
1580 KGSL_DRV_ERR(dev_priv->device,
1581 "Cannot map paged memory with the "
1582 "MMU disabled\n");
1583 break;
1584 }
1585
1586 if (param->hostptr == 0)
1587 break;
1588
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001589 result = kgsl_setup_useraddr(entry, private->pagetable,
1590 param->hostptr,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001591 param->offset, param->len);
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001592 entry->memtype = KGSL_MEM_ENTRY_USER;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001593 break;
1594
1595 case KGSL_USER_MEM_TYPE_ASHMEM:
1596 if (!kgsl_mmu_enabled()) {
1597 KGSL_DRV_ERR(dev_priv->device,
1598 "Cannot map paged memory with the "
1599 "MMU disabled\n");
1600 break;
1601 }
1602
1603 if (param->hostptr == 0)
1604 break;
1605
1606 result = kgsl_setup_ashmem(entry, private->pagetable,
Jeremy Gebbend1f8c902013-05-28 16:53:45 -06001607 param->fd, param->hostptr,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001608 param->len);
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001609
1610 entry->memtype = KGSL_MEM_ENTRY_ASHMEM;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001611 break;
Jordan Crouse8eab35a2011-10-12 16:57:48 -06001612 case KGSL_USER_MEM_TYPE_ION:
1613 result = kgsl_setup_ion(entry, private->pagetable,
1614 param->fd);
1615 break;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001616 default:
Jason848741a2011-07-12 10:24:25 -07001617 KGSL_CORE_ERR("Invalid memory type: %x\n", memtype);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001618 break;
1619 }
1620
1621 if (result)
1622 goto error;
1623
Rajeev Kulkarni8dfdc3362012-11-22 00:22:32 -08001624 if (entry->memdesc.size >= SZ_1M)
Jordan Crousedc67dfb2012-10-25 09:41:46 -06001625 kgsl_memdesc_set_align(&entry->memdesc, ilog2(SZ_1M));
Rajeev Kulkarni8dfdc3362012-11-22 00:22:32 -08001626 else if (entry->memdesc.size >= SZ_64K)
Jordan Crousedc67dfb2012-10-25 09:41:46 -06001627 kgsl_memdesc_set_align(&entry->memdesc, ilog2(SZ_64));
Rajeev Kulkarni8dfdc3362012-11-22 00:22:32 -08001628
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001629 result = kgsl_mmu_map(private->pagetable,
1630 &entry->memdesc,
1631 GSL_PT_PAGE_RV | GSL_PT_PAGE_WV);
1632
1633 if (result)
1634 goto error_put_file_ptr;
1635
1636 /* Adjust the returned value for a non 4k aligned offset */
1637 param->gpuaddr = entry->memdesc.gpuaddr + (param->offset & ~PAGE_MASK);
1638
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001639 KGSL_STATS_ADD(param->len, kgsl_driver.stats.mapped,
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001640 kgsl_driver.stats.mapped_max);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001641
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001642 kgsl_process_add_stats(private, entry->memtype, param->len);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001643
1644 kgsl_mem_entry_attach_process(entry, private);
Jeremy Gebbena5859272012-03-01 12:46:28 -07001645 trace_kgsl_mem_map(entry, param->fd);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001646
1647 kgsl_check_idle(dev_priv->device);
1648 return result;
1649
Jeremy Gebben53d4dd02012-05-07 15:42:00 -06001650error_put_file_ptr:
1651 switch (entry->memtype) {
1652 case KGSL_MEM_ENTRY_PMEM:
1653 case KGSL_MEM_ENTRY_ASHMEM:
1654 if (entry->priv_data)
1655 fput(entry->priv_data);
1656 break;
1657 case KGSL_MEM_ENTRY_ION:
Jeremy Gebben53d4dd02012-05-07 15:42:00 -06001658 ion_free(kgsl_ion_client, entry->priv_data);
1659 break;
1660 default:
1661 break;
1662 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001663error:
1664 kfree(entry);
1665 kgsl_check_idle(dev_priv->device);
1666 return result;
1667}
1668
1669/*This function flushes a graphics memory allocation from CPU cache
1670 *when caching is enabled with MMU*/
1671static long
1672kgsl_ioctl_sharedmem_flush_cache(struct kgsl_device_private *dev_priv,
1673 unsigned int cmd, void *data)
1674{
1675 int result = 0;
1676 struct kgsl_mem_entry *entry;
1677 struct kgsl_sharedmem_free *param = data;
1678 struct kgsl_process_private *private = dev_priv->process_priv;
1679
1680 spin_lock(&private->mem_lock);
1681 entry = kgsl_sharedmem_find(private, param->gpuaddr);
1682 if (!entry) {
1683 KGSL_CORE_ERR("invalid gpuaddr %08x\n", param->gpuaddr);
1684 result = -EINVAL;
Jeremy Gebben690f9d12011-08-08 16:33:49 -06001685 goto done;
1686 }
Jeremy Gebben690f9d12011-08-08 16:33:49 -06001687 if (!entry->memdesc.hostptr) {
1688 KGSL_CORE_ERR("invalid hostptr with gpuaddr %08x\n",
1689 param->gpuaddr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001690 goto done;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001691 }
Jeremy Gebben690f9d12011-08-08 16:33:49 -06001692
1693 kgsl_cache_range_op(&entry->memdesc, KGSL_CACHE_OP_CLEAN);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001694done:
Jeremy Gebben690f9d12011-08-08 16:33:49 -06001695 spin_unlock(&private->mem_lock);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001696 return result;
1697}
1698
1699static long
1700kgsl_ioctl_gpumem_alloc(struct kgsl_device_private *dev_priv,
1701 unsigned int cmd, void *data)
1702{
1703 struct kgsl_process_private *private = dev_priv->process_priv;
1704 struct kgsl_gpumem_alloc *param = data;
1705 struct kgsl_mem_entry *entry;
1706 int result;
1707
1708 entry = kgsl_mem_entry_create();
1709 if (entry == NULL)
1710 return -ENOMEM;
1711
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001712 result = kgsl_allocate_user(&entry->memdesc, private->pagetable,
1713 param->size, param->flags);
1714
1715 if (result == 0) {
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001716 entry->memtype = KGSL_MEM_ENTRY_KERNEL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001717 kgsl_mem_entry_attach_process(entry, private);
1718 param->gpuaddr = entry->memdesc.gpuaddr;
1719
Jordan Crouse1b897cf2011-10-12 16:57:48 -06001720 kgsl_process_add_stats(private, entry->memtype, param->size);
Jeremy Gebbena5859272012-03-01 12:46:28 -07001721 trace_kgsl_mem_alloc(entry);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001722 } else
1723 kfree(entry);
1724
1725 kgsl_check_idle(dev_priv->device);
1726 return result;
1727}
Jeremy Gebbena7423e42011-04-18 15:11:21 -06001728static long kgsl_ioctl_cff_syncmem(struct kgsl_device_private *dev_priv,
1729 unsigned int cmd, void *data)
1730{
1731 int result = 0;
1732 struct kgsl_cff_syncmem *param = data;
1733 struct kgsl_process_private *private = dev_priv->process_priv;
1734 struct kgsl_mem_entry *entry = NULL;
1735
1736 spin_lock(&private->mem_lock);
1737 entry = kgsl_sharedmem_find_region(private, param->gpuaddr, param->len);
1738 if (entry)
1739 kgsl_cffdump_syncmem(dev_priv, &entry->memdesc, param->gpuaddr,
1740 param->len, true);
1741 else
1742 result = -EINVAL;
1743 spin_unlock(&private->mem_lock);
1744 return result;
1745}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001746
Sushmita Susheelendra41f8fa32011-05-11 17:15:58 -06001747static long kgsl_ioctl_cff_user_event(struct kgsl_device_private *dev_priv,
1748 unsigned int cmd, void *data)
1749{
1750 int result = 0;
1751 struct kgsl_cff_user_event *param = data;
1752
1753 kgsl_cffdump_user_event(param->cff_opcode, param->op1, param->op2,
1754 param->op3, param->op4, param->op5);
1755
1756 return result;
1757}
1758
Jordan Croused4bc9d22011-11-17 13:39:21 -07001759#ifdef CONFIG_GENLOCK
1760struct kgsl_genlock_event_priv {
1761 struct genlock_handle *handle;
1762 struct genlock *lock;
1763};
1764
1765/**
1766 * kgsl_genlock_event_cb - Event callback for a genlock timestamp event
1767 * @device - The KGSL device that expired the timestamp
1768 * @priv - private data for the event
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001769 * @context_id - the context id that goes with the timestamp
Jordan Croused4bc9d22011-11-17 13:39:21 -07001770 * @timestamp - the timestamp that triggered the event
1771 *
1772 * Release a genlock lock following the expiration of a timestamp
1773 */
1774
1775static void kgsl_genlock_event_cb(struct kgsl_device *device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001776 void *priv, u32 context_id, u32 timestamp)
Jordan Croused4bc9d22011-11-17 13:39:21 -07001777{
1778 struct kgsl_genlock_event_priv *ev = priv;
1779 int ret;
1780
1781 ret = genlock_lock(ev->handle, GENLOCK_UNLOCK, 0, 0);
1782 if (ret)
1783 KGSL_CORE_ERR("Error while unlocking genlock: %d\n", ret);
1784
1785 genlock_put_handle(ev->handle);
1786
1787 kfree(ev);
1788}
1789
1790/**
1791 * kgsl_add_genlock-event - Create a new genlock event
1792 * @device - KGSL device to create the event on
1793 * @timestamp - Timestamp to trigger the event
1794 * @data - User space buffer containing struct kgsl_genlock_event_priv
1795 * @len - length of the userspace buffer
Jeremy Gebbenfd87f9a2012-02-10 07:06:09 -07001796 * @owner - driver instance that owns this event
Jordan Croused4bc9d22011-11-17 13:39:21 -07001797 * @returns 0 on success or error code on error
1798 *
1799 * Attack to a genlock handle and register an event to release the
1800 * genlock lock when the timestamp expires
1801 */
1802
1803static int kgsl_add_genlock_event(struct kgsl_device *device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001804 u32 context_id, u32 timestamp, void __user *data, int len,
Jeremy Gebbenfd87f9a2012-02-10 07:06:09 -07001805 struct kgsl_device_private *owner)
Jordan Croused4bc9d22011-11-17 13:39:21 -07001806{
1807 struct kgsl_genlock_event_priv *event;
1808 struct kgsl_timestamp_event_genlock priv;
1809 int ret;
1810
1811 if (len != sizeof(priv))
1812 return -EINVAL;
1813
1814 if (copy_from_user(&priv, data, sizeof(priv)))
1815 return -EFAULT;
1816
1817 event = kzalloc(sizeof(*event), GFP_KERNEL);
1818
1819 if (event == NULL)
1820 return -ENOMEM;
1821
1822 event->handle = genlock_get_handle_fd(priv.handle);
1823
1824 if (IS_ERR(event->handle)) {
1825 int ret = PTR_ERR(event->handle);
1826 kfree(event);
1827 return ret;
1828 }
1829
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001830 ret = kgsl_add_event(device, context_id, timestamp,
1831 kgsl_genlock_event_cb, event, owner);
Jordan Croused4bc9d22011-11-17 13:39:21 -07001832 if (ret)
1833 kfree(event);
1834
1835 return ret;
1836}
1837#else
1838static long kgsl_add_genlock_event(struct kgsl_device *device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001839 u32 context_id, u32 timestamp, void __user *data, int len,
Jeremy Gebbenfd87f9a2012-02-10 07:06:09 -07001840 struct kgsl_device_private *owner)
Jordan Croused4bc9d22011-11-17 13:39:21 -07001841{
1842 return -EINVAL;
1843}
1844#endif
1845
1846/**
1847 * kgsl_ioctl_timestamp_event - Register a new timestamp event from userspace
1848 * @dev_priv - pointer to the private device structure
1849 * @cmd - the ioctl cmd passed from kgsl_ioctl
1850 * @data - the user data buffer from kgsl_ioctl
1851 * @returns 0 on success or error code on failure
1852 */
1853
1854static long kgsl_ioctl_timestamp_event(struct kgsl_device_private *dev_priv,
1855 unsigned int cmd, void *data)
1856{
1857 struct kgsl_timestamp_event *param = data;
1858 int ret;
1859
1860 switch (param->type) {
1861 case KGSL_TIMESTAMP_EVENT_GENLOCK:
1862 ret = kgsl_add_genlock_event(dev_priv->device,
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001863 param->context_id, param->timestamp, param->priv,
1864 param->len, dev_priv);
Jordan Croused4bc9d22011-11-17 13:39:21 -07001865 break;
Jeff Boodyfe6c39c2012-08-09 13:54:50 -06001866 case KGSL_TIMESTAMP_EVENT_FENCE:
1867 ret = kgsl_add_fence_event(dev_priv->device,
1868 param->context_id, param->timestamp, param->priv,
1869 param->len, dev_priv);
1870 break;
Jordan Croused4bc9d22011-11-17 13:39:21 -07001871 default:
1872 ret = -EINVAL;
1873 }
1874
1875 return ret;
1876}
1877
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001878typedef long (*kgsl_ioctl_func_t)(struct kgsl_device_private *,
1879 unsigned int, void *);
1880
Vladimir Razgulin38345302013-01-22 18:41:59 -07001881#define KGSL_IOCTL_FUNC(_cmd, _func, _flags) \
1882 [_IOC_NR((_cmd))] = \
1883 { .cmd = (_cmd), .func = (_func), .flags = (_flags) }
1884
1885#define KGSL_IOCTL_LOCK BIT(0)
1886#define KGSL_IOCTL_WAKE BIT(1)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001887
1888static const struct {
1889 unsigned int cmd;
1890 kgsl_ioctl_func_t func;
Vladimir Razgulin38345302013-01-22 18:41:59 -07001891 int flags;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001892} kgsl_ioctl_funcs[] = {
1893 KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_GETPROPERTY,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001894 kgsl_ioctl_device_getproperty,
1895 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001896 KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001897 kgsl_ioctl_device_waittimestamp,
1898 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001899 KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP_CTXTID,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001900 kgsl_ioctl_device_waittimestamp_ctxtid,
1901 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001902 KGSL_IOCTL_FUNC(IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001903 kgsl_ioctl_rb_issueibcmds,
1904 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001905 KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001906 kgsl_ioctl_cmdstream_readtimestamp,
1907 KGSL_IOCTL_LOCK),
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001908 KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_CTXTID,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001909 kgsl_ioctl_cmdstream_readtimestamp_ctxtid,
1910 KGSL_IOCTL_LOCK),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001911 KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001912 kgsl_ioctl_cmdstream_freememontimestamp,
1913 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001914 KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_CTXTID,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001915 kgsl_ioctl_cmdstream_freememontimestamp_ctxtid,
1916 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001917 KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_CREATE,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001918 kgsl_ioctl_drawctxt_create,
1919 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001920 KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_DESTROY,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001921 kgsl_ioctl_drawctxt_destroy,
1922 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001923 KGSL_IOCTL_FUNC(IOCTL_KGSL_MAP_USER_MEM,
1924 kgsl_ioctl_map_user_mem, 0),
1925 KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FROM_PMEM,
1926 kgsl_ioctl_map_user_mem, 0),
1927 KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FREE,
1928 kgsl_ioctl_sharedmem_free, 0),
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001929 KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FLUSH_CACHE,
1930 kgsl_ioctl_sharedmem_flush_cache, 0),
1931 KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_ALLOC,
1932 kgsl_ioctl_gpumem_alloc, 0),
Jeremy Gebbena7423e42011-04-18 15:11:21 -06001933 KGSL_IOCTL_FUNC(IOCTL_KGSL_CFF_SYNCMEM,
1934 kgsl_ioctl_cff_syncmem, 0),
Sushmita Susheelendra41f8fa32011-05-11 17:15:58 -06001935 KGSL_IOCTL_FUNC(IOCTL_KGSL_CFF_USER_EVENT,
1936 kgsl_ioctl_cff_user_event, 0),
Jordan Croused4bc9d22011-11-17 13:39:21 -07001937 KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMESTAMP_EVENT,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001938 kgsl_ioctl_timestamp_event,
1939 KGSL_IOCTL_LOCK),
Jordan Crouseed7dd7f2012-03-29 13:16:02 -06001940 KGSL_IOCTL_FUNC(IOCTL_KGSL_SETPROPERTY,
Vladimir Razgulin38345302013-01-22 18:41:59 -07001941 kgsl_ioctl_device_setproperty,
1942 KGSL_IOCTL_LOCK | KGSL_IOCTL_WAKE)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001943};
1944
1945static long kgsl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1946{
1947 struct kgsl_device_private *dev_priv = filep->private_data;
Jordan Crouse1e76f612012-08-08 13:24:21 -06001948 unsigned int nr;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001949 kgsl_ioctl_func_t func;
Vladimir Razgulin38345302013-01-22 18:41:59 -07001950 int lock, ret, use_hw;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001951 char ustack[64];
1952 void *uptr = NULL;
1953
1954 BUG_ON(dev_priv == NULL);
1955
1956 /* Workaround for an previously incorrectly defined ioctl code.
1957 This helps ensure binary compatability */
1958
1959 if (cmd == IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_OLD)
1960 cmd = IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP;
Jason Varbedian80ba33d2011-07-11 17:29:05 -07001961 else if (cmd == IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_OLD)
1962 cmd = IOCTL_KGSL_CMDSTREAM_READTIMESTAMP;
Jeff Boodyfe6c39c2012-08-09 13:54:50 -06001963 else if (cmd == IOCTL_KGSL_TIMESTAMP_EVENT_OLD)
1964 cmd = IOCTL_KGSL_TIMESTAMP_EVENT;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001965
Jordan Crouse1e76f612012-08-08 13:24:21 -06001966 nr = _IOC_NR(cmd);
1967
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001968 if (cmd & (IOC_IN | IOC_OUT)) {
1969 if (_IOC_SIZE(cmd) < sizeof(ustack))
1970 uptr = ustack;
1971 else {
1972 uptr = kzalloc(_IOC_SIZE(cmd), GFP_KERNEL);
1973 if (uptr == NULL) {
1974 KGSL_MEM_ERR(dev_priv->device,
1975 "kzalloc(%d) failed\n", _IOC_SIZE(cmd));
1976 ret = -ENOMEM;
1977 goto done;
1978 }
1979 }
1980
1981 if (cmd & IOC_IN) {
1982 if (copy_from_user(uptr, (void __user *) arg,
1983 _IOC_SIZE(cmd))) {
1984 ret = -EFAULT;
1985 goto done;
1986 }
1987 } else
1988 memset(uptr, 0, _IOC_SIZE(cmd));
1989 }
1990
1991 if (nr < ARRAY_SIZE(kgsl_ioctl_funcs) &&
Jordan Crouse1e76f612012-08-08 13:24:21 -06001992 kgsl_ioctl_funcs[nr].func != NULL) {
1993
1994 /*
1995 * Make sure that nobody tried to send us a malformed ioctl code
1996 * with a valid NR but bogus flags
1997 */
1998
1999 if (kgsl_ioctl_funcs[nr].cmd != cmd) {
2000 KGSL_DRV_ERR(dev_priv->device,
2001 "Malformed ioctl code %08x\n", cmd);
2002 ret = -ENOIOCTLCMD;
2003 goto done;
2004 }
2005
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002006 func = kgsl_ioctl_funcs[nr].func;
Vladimir Razgulin38345302013-01-22 18:41:59 -07002007 lock = kgsl_ioctl_funcs[nr].flags & KGSL_IOCTL_LOCK;
2008 use_hw = kgsl_ioctl_funcs[nr].flags & KGSL_IOCTL_WAKE;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002009 } else {
2010 func = dev_priv->device->ftbl->ioctl;
2011 if (!func) {
2012 KGSL_DRV_INFO(dev_priv->device,
2013 "invalid ioctl code %08x\n", cmd);
Jeremy Gebbenc15b4612012-01-09 09:44:11 -07002014 ret = -ENOIOCTLCMD;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002015 goto done;
2016 }
2017 lock = 1;
Vladimir Razgulin38345302013-01-22 18:41:59 -07002018 use_hw = 1;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002019 }
2020
2021 if (lock) {
2022 mutex_lock(&dev_priv->device->mutex);
Vladimir Razgulin38345302013-01-22 18:41:59 -07002023 if (use_hw)
2024 kgsl_check_suspended(dev_priv->device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002025 }
2026
2027 ret = func(dev_priv, cmd, uptr);
2028
2029 if (lock) {
2030 kgsl_check_idle_locked(dev_priv->device);
2031 mutex_unlock(&dev_priv->device->mutex);
2032 }
2033
2034 if (ret == 0 && (cmd & IOC_OUT)) {
2035 if (copy_to_user((void __user *) arg, uptr, _IOC_SIZE(cmd)))
2036 ret = -EFAULT;
2037 }
2038
2039done:
2040 if (_IOC_SIZE(cmd) >= sizeof(ustack))
2041 kfree(uptr);
2042
2043 return ret;
2044}
2045
2046static int
2047kgsl_mmap_memstore(struct kgsl_device *device, struct vm_area_struct *vma)
2048{
2049 struct kgsl_memdesc *memdesc = &device->memstore;
2050 int result;
2051 unsigned int vma_size = vma->vm_end - vma->vm_start;
2052
2053 /* The memstore can only be mapped as read only */
2054
2055 if (vma->vm_flags & VM_WRITE)
2056 return -EPERM;
2057
2058 if (memdesc->size != vma_size) {
2059 KGSL_MEM_ERR(device, "memstore bad size: %d should be %d\n",
2060 vma_size, memdesc->size);
2061 return -EINVAL;
2062 }
2063
2064 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2065
Shubhraprakash Das87f68132012-07-30 23:25:13 -07002066 result = remap_pfn_range(vma, vma->vm_start,
2067 device->memstore.physaddr >> PAGE_SHIFT,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002068 vma_size, vma->vm_page_prot);
2069 if (result != 0)
2070 KGSL_MEM_ERR(device, "remap_pfn_range failed: %d\n",
2071 result);
2072
2073 return result;
2074}
2075
Jordan Crouse4283e172011-09-26 14:45:47 -06002076/*
2077 * kgsl_gpumem_vm_open is called whenever a vma region is copied or split.
2078 * Increase the refcount to make sure that the accounting stays correct
2079 */
2080
2081static void kgsl_gpumem_vm_open(struct vm_area_struct *vma)
2082{
2083 struct kgsl_mem_entry *entry = vma->vm_private_data;
2084 kgsl_mem_entry_get(entry);
2085}
2086
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002087static int
2088kgsl_gpumem_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2089{
2090 struct kgsl_mem_entry *entry = vma->vm_private_data;
2091
Jordan Croused17e9aa2011-10-12 16:57:48 -06002092 if (!entry->memdesc.ops || !entry->memdesc.ops->vmfault)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002093 return VM_FAULT_SIGBUS;
2094
2095 return entry->memdesc.ops->vmfault(&entry->memdesc, vma, vmf);
2096}
2097
2098static void
2099kgsl_gpumem_vm_close(struct vm_area_struct *vma)
2100{
2101 struct kgsl_mem_entry *entry = vma->vm_private_data;
Jeremy Gebbencc0c7092013-05-28 16:53:53 -06002102
2103 entry->memdesc.useraddr = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002104 kgsl_mem_entry_put(entry);
2105}
2106
2107static struct vm_operations_struct kgsl_gpumem_vm_ops = {
Jordan Crouse4283e172011-09-26 14:45:47 -06002108 .open = kgsl_gpumem_vm_open,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002109 .fault = kgsl_gpumem_vm_fault,
2110 .close = kgsl_gpumem_vm_close,
2111};
2112
2113static int kgsl_mmap(struct file *file, struct vm_area_struct *vma)
2114{
Jeremy Gebbencc0c7092013-05-28 16:53:53 -06002115 unsigned int ret;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002116 unsigned long vma_offset = vma->vm_pgoff << PAGE_SHIFT;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002117 struct kgsl_device_private *dev_priv = file->private_data;
2118 struct kgsl_process_private *private = dev_priv->process_priv;
Jordan Crousec9559e42012-04-05 16:55:56 -06002119 struct kgsl_mem_entry *entry = NULL;
Jordan Crouse2db0af92011-08-08 16:05:09 -06002120 struct kgsl_device *device = dev_priv->device;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002121
2122 /* Handle leagacy behavior for memstore */
2123
Shubhraprakash Das87f68132012-07-30 23:25:13 -07002124 if (vma_offset == device->memstore.gpuaddr)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002125 return kgsl_mmap_memstore(device, vma);
2126
2127 /* Find a chunk of GPU memory */
2128
2129 spin_lock(&private->mem_lock);
Jordan Crousec9559e42012-04-05 16:55:56 -06002130 entry = kgsl_sharedmem_find(private, vma_offset);
2131
2132 if (entry)
2133 kgsl_mem_entry_get(entry);
2134
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002135 spin_unlock(&private->mem_lock);
2136
2137 if (entry == NULL)
2138 return -EINVAL;
2139
Jordan Croused17e9aa2011-10-12 16:57:48 -06002140 if (!entry->memdesc.ops ||
2141 !entry->memdesc.ops->vmflags ||
Jeremy Gebbencc0c7092013-05-28 16:53:53 -06002142 !entry->memdesc.ops->vmfault) {
2143 ret = -EINVAL;
2144 goto err_put;
2145 }
2146
2147 if (entry->memdesc.useraddr != 0) {
2148 ret = -EBUSY;
2149 goto err_put;
2150 }
2151
2152 if (entry->memdesc.size != (vma->vm_end - vma->vm_start)) {
2153 ret = -ERANGE;
2154 goto err_put;
2155 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002156
2157 vma->vm_flags |= entry->memdesc.ops->vmflags(&entry->memdesc);
2158
2159 vma->vm_private_data = entry;
2160 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
2161 vma->vm_ops = &kgsl_gpumem_vm_ops;
2162 vma->vm_file = file;
2163
Jeremy Gebbencc0c7092013-05-28 16:53:53 -06002164 entry->memdesc.useraddr = vma->vm_start;
2165
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002166 return 0;
Jeremy Gebbencc0c7092013-05-28 16:53:53 -06002167err_put:
2168 kgsl_mem_entry_put(entry);
2169 return ret;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002170}
2171
Jordan Crouseb368e9b2012-04-27 14:01:59 -06002172static irqreturn_t kgsl_irq_handler(int irq, void *data)
2173{
2174 struct kgsl_device *device = data;
2175
2176 return device->ftbl->irq_handler(device);
2177
2178}
2179
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002180static const struct file_operations kgsl_fops = {
2181 .owner = THIS_MODULE,
2182 .release = kgsl_release,
2183 .open = kgsl_open,
2184 .mmap = kgsl_mmap,
2185 .unlocked_ioctl = kgsl_ioctl,
2186};
2187
2188struct kgsl_driver kgsl_driver = {
2189 .process_mutex = __MUTEX_INITIALIZER(kgsl_driver.process_mutex),
2190 .ptlock = __SPIN_LOCK_UNLOCKED(kgsl_driver.ptlock),
2191 .devlock = __MUTEX_INITIALIZER(kgsl_driver.devlock),
2192};
2193EXPORT_SYMBOL(kgsl_driver);
2194
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002195static void _unregister_device(struct kgsl_device *device)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002196{
2197 int minor;
2198
2199 mutex_lock(&kgsl_driver.devlock);
2200 for (minor = 0; minor < KGSL_DEVICE_MAX; minor++) {
2201 if (device == kgsl_driver.devp[minor])
2202 break;
2203 }
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002204 if (minor != KGSL_DEVICE_MAX) {
2205 device_destroy(kgsl_driver.class,
2206 MKDEV(MAJOR(kgsl_driver.major), minor));
2207 kgsl_driver.devp[minor] = NULL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002208 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002209 mutex_unlock(&kgsl_driver.devlock);
2210}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002211
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002212static int _register_device(struct kgsl_device *device)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002213{
2214 int minor, ret;
2215 dev_t dev;
2216
2217 /* Find a minor for the device */
2218
2219 mutex_lock(&kgsl_driver.devlock);
2220 for (minor = 0; minor < KGSL_DEVICE_MAX; minor++) {
2221 if (kgsl_driver.devp[minor] == NULL) {
2222 kgsl_driver.devp[minor] = device;
2223 break;
2224 }
2225 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002226 mutex_unlock(&kgsl_driver.devlock);
2227
2228 if (minor == KGSL_DEVICE_MAX) {
2229 KGSL_CORE_ERR("minor devices exhausted\n");
2230 return -ENODEV;
2231 }
2232
2233 /* Create the device */
2234 dev = MKDEV(MAJOR(kgsl_driver.major), minor);
2235 device->dev = device_create(kgsl_driver.class,
2236 device->parentdev,
2237 dev, device,
2238 device->name);
2239
2240 if (IS_ERR(device->dev)) {
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002241 mutex_lock(&kgsl_driver.devlock);
2242 kgsl_driver.devp[minor] = NULL;
2243 mutex_unlock(&kgsl_driver.devlock);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002244 ret = PTR_ERR(device->dev);
2245 KGSL_CORE_ERR("device_create(%s): %d\n", device->name, ret);
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002246 return ret;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002247 }
2248
2249 dev_set_drvdata(device->parentdev, device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002250 return 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002251}
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002252
Jordan Crouseb368e9b2012-04-27 14:01:59 -06002253int kgsl_device_platform_probe(struct kgsl_device *device)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002254{
Michael Street8bacdd02012-01-05 14:55:01 -08002255 int result;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002256 int status = -EINVAL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002257 struct resource *res;
2258 struct platform_device *pdev =
2259 container_of(device->parentdev, struct platform_device, dev);
2260
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002261 status = _register_device(device);
2262 if (status)
2263 return status;
2264
2265 /* Initialize logging first, so that failures below actually print. */
2266 kgsl_device_debugfs_init(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002267
2268 status = kgsl_pwrctrl_init(device);
2269 if (status)
2270 goto error;
2271
Harsh Vardhan Dwivedif48af7f2012-04-13 12:50:44 -06002272 kgsl_ion_client = msm_ion_client_create(UINT_MAX, KGSL_NAME);
2273
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002274 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2275 device->iomemname);
2276 if (res == NULL) {
2277 KGSL_DRV_ERR(device, "platform_get_resource_byname failed\n");
2278 status = -EINVAL;
2279 goto error_pwrctrl_close;
2280 }
2281 if (res->start == 0 || resource_size(res) == 0) {
Jordan Crouse7501d452012-04-19 08:58:44 -06002282 KGSL_DRV_ERR(device, "dev %d invalid register region\n",
2283 device->id);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002284 status = -EINVAL;
2285 goto error_pwrctrl_close;
2286 }
2287
Jordan Crouse7501d452012-04-19 08:58:44 -06002288 device->reg_phys = res->start;
2289 device->reg_len = resource_size(res);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002290
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002291 if (!devm_request_mem_region(device->dev, device->reg_phys,
2292 device->reg_len, device->name)) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002293 KGSL_DRV_ERR(device, "request_mem_region failed\n");
2294 status = -ENODEV;
2295 goto error_pwrctrl_close;
2296 }
2297
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002298 device->reg_virt = devm_ioremap(device->dev, device->reg_phys,
2299 device->reg_len);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002300
Jordan Crouse7501d452012-04-19 08:58:44 -06002301 if (device->reg_virt == NULL) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002302 KGSL_DRV_ERR(device, "ioremap failed\n");
2303 status = -ENODEV;
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002304 goto error_pwrctrl_close;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002305 }
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002306 /*acquire interrupt */
2307 device->pwrctrl.interrupt_num =
2308 platform_get_irq_byname(pdev, device->pwrctrl.irq_name);
2309
2310 if (device->pwrctrl.interrupt_num <= 0) {
2311 KGSL_DRV_ERR(device, "platform_get_irq_byname failed: %d\n",
2312 device->pwrctrl.interrupt_num);
2313 status = -EINVAL;
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002314 goto error_pwrctrl_close;
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002315 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002316
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002317 status = devm_request_irq(device->dev, device->pwrctrl.interrupt_num,
2318 kgsl_irq_handler, IRQF_TRIGGER_HIGH,
2319 device->name, device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002320 if (status) {
2321 KGSL_DRV_ERR(device, "request_irq(%d) failed: %d\n",
2322 device->pwrctrl.interrupt_num, status);
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002323 goto error_pwrctrl_close;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002324 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002325 disable_irq(device->pwrctrl.interrupt_num);
2326
2327 KGSL_DRV_INFO(device,
Jordan Crouse7501d452012-04-19 08:58:44 -06002328 "dev_id %d regs phys 0x%08lx size 0x%08x virt %p\n",
2329 device->id, device->reg_phys, device->reg_len,
2330 device->reg_virt);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002331
Michael Street8bacdd02012-01-05 14:55:01 -08002332 result = kgsl_drm_init(pdev);
2333 if (result)
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002334 goto error_pwrctrl_close;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002335
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002336 kgsl_cffdump_open(device->id);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002337
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002338 setup_timer(&device->idle_timer, kgsl_timer, (unsigned long) device);
2339 status = kgsl_create_device_workqueue(device);
2340 if (status)
Jeremy Gebben4204d0f2012-03-01 16:06:21 -07002341 goto error_pwrctrl_close;
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002342
2343 status = kgsl_mmu_init(device);
2344 if (status != 0) {
2345 KGSL_DRV_ERR(device, "kgsl_mmu_init failed %d\n", status);
2346 goto error_dest_work_q;
2347 }
2348
2349 status = kgsl_allocate_contiguous(&device->memstore,
Richard Ruigrok2ad5e9d2012-06-14 14:22:05 -07002350 KGSL_MEMSTORE_SIZE);
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002351
2352 if (status != 0) {
2353 KGSL_DRV_ERR(device, "kgsl_allocate_contiguous failed %d\n",
2354 status);
2355 goto error_close_mmu;
2356 }
2357
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002358 pm_qos_add_request(&device->pm_qos_req_dma, PM_QOS_CPU_DMA_LATENCY,
2359 PM_QOS_DEFAULT_VALUE);
2360
2361 /* Initalize the snapshot engine */
2362 kgsl_device_snapshot_init(device);
2363
2364 /* Initialize common sysfs entries */
2365 kgsl_pwrctrl_init_sysfs(device);
2366
2367 return 0;
2368
2369error_close_mmu:
2370 kgsl_mmu_close(device);
2371error_dest_work_q:
2372 destroy_workqueue(device->work_queue);
2373 device->work_queue = NULL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002374error_pwrctrl_close:
2375 kgsl_pwrctrl_close(device);
2376error:
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002377 _unregister_device(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002378 return status;
2379}
2380EXPORT_SYMBOL(kgsl_device_platform_probe);
2381
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002382int kgsl_postmortem_dump(struct kgsl_device *device, int manual)
2383{
2384 bool saved_nap;
2385 struct kgsl_pwrctrl *pwr = &device->pwrctrl;
2386
2387 BUG_ON(device == NULL);
2388
2389 kgsl_cffdump_hang(device->id);
2390
2391 /* For a manual dump, make sure that the system is idle */
2392
2393 if (manual) {
2394 if (device->active_cnt != 0) {
2395 mutex_unlock(&device->mutex);
2396 wait_for_completion(&device->suspend_gate);
2397 mutex_lock(&device->mutex);
2398 }
2399
2400 if (device->state == KGSL_STATE_ACTIVE)
Jordan Crousea29a2e02012-08-14 09:09:23 -06002401 kgsl_idle(device);
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002402
2403 }
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002404
Tarun Karra45a50d62013-01-28 21:47:37 -08002405 if (device->pm_dump_enable) {
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002406
Tarun Karra45a50d62013-01-28 21:47:37 -08002407 KGSL_LOG_DUMP(device,
2408 "POWER: FLAGS = %08lX | ACTIVE POWERLEVEL = %08X",
2409 pwr->power_flags, pwr->active_pwrlevel);
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002410
Tarun Karra45a50d62013-01-28 21:47:37 -08002411 KGSL_LOG_DUMP(device, "POWER: INTERVAL TIMEOUT = %08X ",
2412 pwr->interval_timeout);
2413
2414 }
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002415
2416 /* Disable the idle timer so we don't get interrupted */
2417 del_timer_sync(&device->idle_timer);
2418 mutex_unlock(&device->mutex);
2419 flush_workqueue(device->work_queue);
2420 mutex_lock(&device->mutex);
2421
2422 /* Turn off napping to make sure we have the clocks full
2423 attention through the following process */
2424 saved_nap = device->pwrctrl.nap_allowed;
2425 device->pwrctrl.nap_allowed = false;
2426
2427 /* Force on the clocks */
2428 kgsl_pwrctrl_wake(device);
2429
2430 /* Disable the irq */
2431 kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF);
2432
2433 /*Call the device specific postmortem dump function*/
2434 device->ftbl->postmortem_dump(device, manual);
2435
2436 /* Restore nap mode */
2437 device->pwrctrl.nap_allowed = saved_nap;
2438
2439 /* On a manual trigger, turn on the interrupts and put
2440 the clocks to sleep. They will recover themselves
2441 on the next event. For a hang, leave things as they
Tarun Karrad20d71a2013-01-25 15:38:57 -08002442 are until fault tolerance kicks in. */
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002443
2444 if (manual) {
2445 kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_ON);
2446
2447 /* try to go into a sleep mode until the next event */
2448 kgsl_pwrctrl_request_state(device, KGSL_STATE_SLEEP);
2449 kgsl_pwrctrl_sleep(device);
2450 }
2451
Harsh Vardhan Dwivedi715fb832012-05-18 00:24:18 -06002452 return 0;
2453}
2454EXPORT_SYMBOL(kgsl_postmortem_dump);
2455
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002456void kgsl_device_platform_remove(struct kgsl_device *device)
2457{
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002458 kgsl_device_snapshot_close(device);
2459
2460 kgsl_cffdump_close(device->id);
2461 kgsl_pwrctrl_uninit_sysfs(device);
2462
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002463 pm_qos_remove_request(&device->pm_qos_req_dma);
2464
2465 idr_destroy(&device->context_idr);
2466
2467 kgsl_sharedmem_free(&device->memstore);
2468
2469 kgsl_mmu_close(device);
2470
2471 if (device->work_queue) {
2472 destroy_workqueue(device->work_queue);
2473 device->work_queue = NULL;
2474 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002475 kgsl_pwrctrl_close(device);
2476
Jeremy Gebben4f5f0de2012-03-01 15:51:37 -07002477 _unregister_device(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002478}
2479EXPORT_SYMBOL(kgsl_device_platform_remove);
2480
2481static int __devinit
2482kgsl_ptdata_init(void)
2483{
Jordan Crouse6d76c4d2012-03-26 09:50:43 -06002484 kgsl_driver.ptpool = kgsl_mmu_ptpool_init(kgsl_pagetable_count);
2485
Shubhraprakash Das767fdda2011-08-15 15:49:45 -06002486 if (!kgsl_driver.ptpool)
2487 return -ENOMEM;
2488 return 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002489}
2490
2491static void kgsl_core_exit(void)
2492{
Ranjhith Kalisamy4ad59e92012-05-31 19:15:11 +05302493 kgsl_mmu_ptpool_destroy(kgsl_driver.ptpool);
Shubhraprakash Das767fdda2011-08-15 15:49:45 -06002494 kgsl_driver.ptpool = NULL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002495
Ranjhith Kalisamydad9df52012-06-01 17:05:13 +05302496 kgsl_drm_exit();
2497 kgsl_cffdump_destroy();
2498 kgsl_core_debugfs_close();
Ranjhith Kalisamydad9df52012-06-01 17:05:13 +05302499
Harsh Vardhan Dwivediefa6b012012-06-15 13:02:27 -06002500 /*
2501 * We call kgsl_sharedmem_uninit_sysfs() and device_unregister()
2502 * only if kgsl_driver.virtdev has been populated.
2503 * We check at least one member of kgsl_driver.virtdev to
2504 * see if it is not NULL (and thus, has been populated).
2505 */
2506 if (kgsl_driver.virtdev.class) {
2507 kgsl_sharedmem_uninit_sysfs();
2508 device_unregister(&kgsl_driver.virtdev);
2509 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002510
2511 if (kgsl_driver.class) {
2512 class_destroy(kgsl_driver.class);
2513 kgsl_driver.class = NULL;
2514 }
2515
Ranjhith Kalisamydad9df52012-06-01 17:05:13 +05302516 unregister_chrdev_region(kgsl_driver.major, KGSL_DEVICE_MAX);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002517}
2518
2519static int __init kgsl_core_init(void)
2520{
2521 int result = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002522 /* alloc major and minor device numbers */
2523 result = alloc_chrdev_region(&kgsl_driver.major, 0, KGSL_DEVICE_MAX,
2524 KGSL_NAME);
2525 if (result < 0) {
2526 KGSL_CORE_ERR("alloc_chrdev_region failed err = %d\n", result);
2527 goto err;
2528 }
2529
2530 cdev_init(&kgsl_driver.cdev, &kgsl_fops);
2531 kgsl_driver.cdev.owner = THIS_MODULE;
2532 kgsl_driver.cdev.ops = &kgsl_fops;
2533 result = cdev_add(&kgsl_driver.cdev, MKDEV(MAJOR(kgsl_driver.major), 0),
2534 KGSL_DEVICE_MAX);
2535
2536 if (result) {
2537 KGSL_CORE_ERR("kgsl: cdev_add() failed, dev_num= %d,"
2538 " result= %d\n", kgsl_driver.major, result);
2539 goto err;
2540 }
2541
2542 kgsl_driver.class = class_create(THIS_MODULE, KGSL_NAME);
2543
2544 if (IS_ERR(kgsl_driver.class)) {
2545 result = PTR_ERR(kgsl_driver.class);
2546 KGSL_CORE_ERR("failed to create class %s", KGSL_NAME);
2547 goto err;
2548 }
2549
2550 /* Make a virtual device for managing core related things
2551 in sysfs */
2552 kgsl_driver.virtdev.class = kgsl_driver.class;
2553 dev_set_name(&kgsl_driver.virtdev, "kgsl");
2554 result = device_register(&kgsl_driver.virtdev);
2555 if (result) {
2556 KGSL_CORE_ERR("driver_register failed\n");
2557 goto err;
2558 }
2559
2560 /* Make kobjects in the virtual device for storing statistics */
2561
2562 kgsl_driver.ptkobj =
2563 kobject_create_and_add("pagetables",
2564 &kgsl_driver.virtdev.kobj);
2565
2566 kgsl_driver.prockobj =
2567 kobject_create_and_add("proc",
2568 &kgsl_driver.virtdev.kobj);
2569
2570 kgsl_core_debugfs_init();
2571
2572 kgsl_sharedmem_init_sysfs();
2573 kgsl_cffdump_init();
2574
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002575 INIT_LIST_HEAD(&kgsl_driver.process_list);
2576
Shubhraprakash Das767fdda2011-08-15 15:49:45 -06002577 INIT_LIST_HEAD(&kgsl_driver.pagetable_list);
2578
2579 kgsl_mmu_set_mmutype(ksgl_mmu_type);
2580
2581 if (KGSL_MMU_TYPE_GPU == kgsl_mmu_get_mmutype()) {
2582 result = kgsl_ptdata_init();
2583 if (result)
2584 goto err;
2585 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002586
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002587 return 0;
2588
2589err:
2590 kgsl_core_exit();
2591 return result;
2592}
2593
2594module_init(kgsl_core_init);
2595module_exit(kgsl_core_exit);
2596
2597MODULE_AUTHOR("Qualcomm Innovation Center, Inc.");
2598MODULE_DESCRIPTION("MSM GPU driver");
2599MODULE_LICENSE("GPL");