blob: 5f16ff76fae1abddaea3958f9847bc04bea70e7f [file] [log] [blame]
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -07001/* Copyright (c) 2002,2007-2012, Code Aurora Forum. All rights reserved.
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 */
13#include <linux/firmware.h>
14#include <linux/slab.h>
15#include <linux/sched.h>
16#include <linux/log2.h>
17
18#include "kgsl.h"
19#include "kgsl_sharedmem.h"
20#include "kgsl_cffdump.h"
21
22#include "adreno.h"
23#include "adreno_pm4types.h"
24#include "adreno_ringbuffer.h"
Jeremy Gebbend0ab6ad2012-04-06 11:13:35 -060025#include "adreno_debugfs.h"
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070026
Jeremy Gebbeneebc4612011-08-31 10:15:21 -070027#include "a2xx_reg.h"
Jordan Crouseb4d31bd2012-02-01 22:11:12 -070028#include "a3xx_reg.h"
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070029
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070030#define GSL_RB_NOP_SIZEDWORDS 2
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070031
Jordan Crouseb4d31bd2012-02-01 22:11:12 -070032void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070033{
34 BUG_ON(rb->wptr == 0);
35
Lucille Sylvester958dc942011-09-06 18:19:49 -060036 /* Let the pwrscale policy know that new commands have
37 been submitted. */
38 kgsl_pwrscale_busy(rb->device);
39
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070040 /*synchronize memory before informing the hardware of the
41 *new commands.
42 */
43 mb();
44
45 adreno_regwrite(rb->device, REG_CP_RB_WPTR, rb->wptr);
46}
47
Carter Cooper6dd94c82011-10-13 14:43:53 -060048static void
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070049adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb, unsigned int numcmds,
50 int wptr_ahead)
51{
52 int nopcount;
53 unsigned int freecmds;
54 unsigned int *cmds;
55 uint cmds_gpu;
56
57 /* if wptr ahead, fill the remaining with NOPs */
58 if (wptr_ahead) {
59 /* -1 for header */
60 nopcount = rb->sizedwords - rb->wptr - 1;
61
62 cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
63 cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr;
64
Jordan Crouse084427d2011-07-28 08:37:58 -060065 GSL_RB_WRITE(cmds, cmds_gpu, cp_nop_packet(nopcount));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070066
67 /* Make sure that rptr is not 0 before submitting
68 * commands at the end of ringbuffer. We do not
69 * want the rptr and wptr to become equal when
70 * the ringbuffer is not empty */
71 do {
72 GSL_RB_GET_READPTR(rb, &rb->rptr);
73 } while (!rb->rptr);
74
75 rb->wptr++;
76
77 adreno_ringbuffer_submit(rb);
78
79 rb->wptr = 0;
80 }
81
82 /* wait for space in ringbuffer */
83 do {
84 GSL_RB_GET_READPTR(rb, &rb->rptr);
85
86 freecmds = rb->rptr - rb->wptr;
87
88 } while ((freecmds != 0) && (freecmds <= numcmds));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070089}
90
Jordan Crouseb4d31bd2012-02-01 22:11:12 -070091unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070092 unsigned int numcmds)
93{
94 unsigned int *ptr = NULL;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070095
96 BUG_ON(numcmds >= rb->sizedwords);
97
98 GSL_RB_GET_READPTR(rb, &rb->rptr);
99 /* check for available space */
100 if (rb->wptr >= rb->rptr) {
101 /* wptr ahead or equal to rptr */
102 /* reserve dwords for nop packet */
103 if ((rb->wptr + numcmds) > (rb->sizedwords -
104 GSL_RB_NOP_SIZEDWORDS))
Carter Cooper6dd94c82011-10-13 14:43:53 -0600105 adreno_ringbuffer_waitspace(rb, numcmds, 1);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700106 } else {
107 /* wptr behind rptr */
108 if ((rb->wptr + numcmds) >= rb->rptr)
Carter Cooper6dd94c82011-10-13 14:43:53 -0600109 adreno_ringbuffer_waitspace(rb, numcmds, 0);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700110 /* check for remaining space */
111 /* reserve dwords for nop packet */
112 if ((rb->wptr + numcmds) > (rb->sizedwords -
113 GSL_RB_NOP_SIZEDWORDS))
Carter Cooper6dd94c82011-10-13 14:43:53 -0600114 adreno_ringbuffer_waitspace(rb, numcmds, 1);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700115 }
116
Carter Cooper6dd94c82011-10-13 14:43:53 -0600117 ptr = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
118 rb->wptr += numcmds;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700119
120 return ptr;
121}
122
123static int _load_firmware(struct kgsl_device *device, const char *fwfile,
124 void **data, int *len)
125{
126 const struct firmware *fw = NULL;
127 int ret;
128
129 ret = request_firmware(&fw, fwfile, device->dev);
130
131 if (ret) {
132 KGSL_DRV_ERR(device, "request_firmware(%s) failed: %d\n",
133 fwfile, ret);
134 return ret;
135 }
136
137 *data = kmalloc(fw->size, GFP_KERNEL);
138
139 if (*data) {
140 memcpy(*data, fw->data, fw->size);
141 *len = fw->size;
142 } else
143 KGSL_MEM_ERR(device, "kmalloc(%d) failed\n", fw->size);
144
145 release_firmware(fw);
146 return (*data != NULL) ? 0 : -ENOMEM;
147}
148
149static int adreno_ringbuffer_load_pm4_ucode(struct kgsl_device *device)
150{
151 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700152 int i, ret = 0;
153
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700154 if (adreno_dev->pm4_fw == NULL) {
155 int len;
Jordan Crouse505df9c2011-07-28 08:37:59 -0600156 void *ptr;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700157
Jordan Crouse505df9c2011-07-28 08:37:59 -0600158 ret = _load_firmware(device, adreno_dev->pm4_fwfile,
159 &ptr, &len);
160
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700161 if (ret)
162 goto err;
163
164 /* PM4 size is 3 dword aligned plus 1 dword of version */
165 if (len % ((sizeof(uint32_t) * 3)) != sizeof(uint32_t)) {
166 KGSL_DRV_ERR(device, "Bad firmware size: %d\n", len);
167 ret = -EINVAL;
Jeremy Gebben79acee62011-08-08 16:44:07 -0600168 kfree(ptr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700169 goto err;
170 }
171
172 adreno_dev->pm4_fw_size = len / sizeof(uint32_t);
173 adreno_dev->pm4_fw = ptr;
174 }
175
176 KGSL_DRV_INFO(device, "loading pm4 ucode version: %d\n",
177 adreno_dev->pm4_fw[0]);
178
179 adreno_regwrite(device, REG_CP_DEBUG, 0x02000000);
180 adreno_regwrite(device, REG_CP_ME_RAM_WADDR, 0);
181 for (i = 1; i < adreno_dev->pm4_fw_size; i++)
182 adreno_regwrite(device, REG_CP_ME_RAM_DATA,
183 adreno_dev->pm4_fw[i]);
184err:
185 return ret;
186}
187
188static int adreno_ringbuffer_load_pfp_ucode(struct kgsl_device *device)
189{
190 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700191 int i, ret = 0;
192
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700193 if (adreno_dev->pfp_fw == NULL) {
194 int len;
Jordan Crouse505df9c2011-07-28 08:37:59 -0600195 void *ptr;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700196
Jordan Crouse505df9c2011-07-28 08:37:59 -0600197 ret = _load_firmware(device, adreno_dev->pfp_fwfile,
198 &ptr, &len);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700199 if (ret)
200 goto err;
201
202 /* PFP size shold be dword aligned */
203 if (len % sizeof(uint32_t) != 0) {
204 KGSL_DRV_ERR(device, "Bad firmware size: %d\n", len);
205 ret = -EINVAL;
Jeremy Gebben79acee62011-08-08 16:44:07 -0600206 kfree(ptr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700207 goto err;
208 }
209
210 adreno_dev->pfp_fw_size = len / sizeof(uint32_t);
211 adreno_dev->pfp_fw = ptr;
212 }
213
214 KGSL_DRV_INFO(device, "loading pfp ucode version: %d\n",
215 adreno_dev->pfp_fw[0]);
216
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700217 adreno_regwrite(device, adreno_dev->gpudev->reg_cp_pfp_ucode_addr, 0);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700218 for (i = 1; i < adreno_dev->pfp_fw_size; i++)
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700219 adreno_regwrite(device,
220 adreno_dev->gpudev->reg_cp_pfp_ucode_data,
221 adreno_dev->pfp_fw[i]);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700222err:
223 return ret;
224}
225
226int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
227{
228 int status;
229 /*cp_rb_cntl_u cp_rb_cntl; */
230 union reg_cp_rb_cntl cp_rb_cntl;
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700231 unsigned int rb_cntl;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700232 struct kgsl_device *device = rb->device;
Jeremy Gebbenddf6b572011-09-09 13:39:49 -0700233 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700234
235 if (rb->flags & KGSL_FLAGS_STARTED)
236 return 0;
237
238 if (init_ram) {
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700239 rb->timestamp[KGSL_MEMSTORE_GLOBAL] = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700240 GSL_RB_INIT_TIMESTAMP(rb);
241 }
242
243 kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
244 sizeof(struct kgsl_rbmemptrs));
245
246 kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
247 (rb->sizedwords << 2));
248
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700249 if (adreno_is_a2xx(adreno_dev)) {
250 adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
251 (rb->memptrs_desc.gpuaddr
252 + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700253
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700254 /* setup WPTR delay */
255 adreno_regwrite(device, REG_CP_RB_WPTR_DELAY,
256 0 /*0x70000010 */);
257 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700258
259 /*setup REG_CP_RB_CNTL */
260 adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
261 cp_rb_cntl.val = rb_cntl;
262
263 /*
264 * The size of the ringbuffer in the hardware is the log2
265 * representation of the size in quadwords (sizedwords / 2)
266 */
267 cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
268
269 /*
270 * Specify the quadwords to read before updating mem RPTR.
271 * Like above, pass the log2 representation of the blocksize
272 * in quadwords.
273 */
274 cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
275
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700276 if (adreno_is_a2xx(adreno_dev)) {
277 /* WPTR polling */
278 cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
279 }
280
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700281 /* mem RPTR writebacks */
282 cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
283
284 adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
285
286 adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
287
288 adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
289 rb->memptrs_desc.gpuaddr +
290 GSL_RB_MEMPTRS_RPTR_OFFSET);
291
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700292 if (adreno_is_a3xx(adreno_dev)) {
293 /* enable access protection to privileged registers */
294 adreno_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007);
295
296 /* RBBM registers */
297 adreno_regwrite(device, A3XX_CP_PROTECT_REG_0, 0x63000040);
298 adreno_regwrite(device, A3XX_CP_PROTECT_REG_1, 0x62000080);
299 adreno_regwrite(device, A3XX_CP_PROTECT_REG_2, 0x600000CC);
300 adreno_regwrite(device, A3XX_CP_PROTECT_REG_3, 0x60000108);
301 adreno_regwrite(device, A3XX_CP_PROTECT_REG_4, 0x64000140);
302 adreno_regwrite(device, A3XX_CP_PROTECT_REG_5, 0x66000400);
303
304 /* CP registers */
305 adreno_regwrite(device, A3XX_CP_PROTECT_REG_6, 0x65000700);
306 adreno_regwrite(device, A3XX_CP_PROTECT_REG_7, 0x610007D8);
307 adreno_regwrite(device, A3XX_CP_PROTECT_REG_8, 0x620007E0);
308 adreno_regwrite(device, A3XX_CP_PROTECT_REG_9, 0x61001178);
309 adreno_regwrite(device, A3XX_CP_PROTECT_REG_A, 0x64001180);
310
311 /* RB registers */
312 adreno_regwrite(device, A3XX_CP_PROTECT_REG_B, 0x60003300);
313
314 /* VBIF registers */
315 adreno_regwrite(device, A3XX_CP_PROTECT_REG_C, 0x6B00C000);
316 }
317
318 if (adreno_is_a2xx(adreno_dev)) {
319 /* explicitly clear all cp interrupts */
320 adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
321 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700322
323 /* setup scratch/timestamp */
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700324 adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr +
325 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
326 soptimestamp));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700327
328 adreno_regwrite(device, REG_SCRATCH_UMSK,
329 GSL_RB_MEMPTRS_SCRATCH_MASK);
330
331 /* load the CP ucode */
332
333 status = adreno_ringbuffer_load_pm4_ucode(device);
334 if (status != 0)
335 return status;
336
337 /* load the prefetch parser ucode */
338 status = adreno_ringbuffer_load_pfp_ucode(device);
339 if (status != 0)
340 return status;
341
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700342
343 rb->rptr = 0;
344 rb->wptr = 0;
345
346 /* clear ME_HALT to start micro engine */
347 adreno_regwrite(device, REG_CP_ME_CNTL, 0);
348
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700349 /* ME init is GPU specific, so jump into the sub-function */
350 adreno_dev->gpudev->rb_init(adreno_dev, rb);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700351
352 /* idle device to validate ME INIT */
353 status = adreno_idle(device, KGSL_TIMEOUT_DEFAULT);
354
355 if (status == 0)
356 rb->flags |= KGSL_FLAGS_STARTED;
357
358 return status;
359}
360
Carter Cooper6dd94c82011-10-13 14:43:53 -0600361void adreno_ringbuffer_stop(struct adreno_ringbuffer *rb)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700362{
363 if (rb->flags & KGSL_FLAGS_STARTED) {
364 /* ME_HALT */
365 adreno_regwrite(rb->device, REG_CP_ME_CNTL, 0x10000000);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700366 rb->flags &= ~KGSL_FLAGS_STARTED;
367 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700368}
369
370int adreno_ringbuffer_init(struct kgsl_device *device)
371{
372 int status;
373 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
374 struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
375
376 rb->device = device;
377 /*
378 * It is silly to convert this to words and then back to bytes
379 * immediately below, but most of the rest of the code deals
380 * in words, so we might as well only do the math once
381 */
382 rb->sizedwords = KGSL_RB_SIZE >> 2;
383
384 /* allocate memory for ringbuffer */
385 status = kgsl_allocate_contiguous(&rb->buffer_desc,
386 (rb->sizedwords << 2));
387
388 if (status != 0) {
389 adreno_ringbuffer_close(rb);
390 return status;
391 }
392
393 /* allocate memory for polling and timestamps */
394 /* This really can be at 4 byte alignment boundry but for using MMU
395 * we need to make it at page boundary */
396 status = kgsl_allocate_contiguous(&rb->memptrs_desc,
397 sizeof(struct kgsl_rbmemptrs));
398
399 if (status != 0) {
400 adreno_ringbuffer_close(rb);
401 return status;
402 }
403
404 /* overlay structure on memptrs memory */
405 rb->memptrs = (struct kgsl_rbmemptrs *) rb->memptrs_desc.hostptr;
406
407 return 0;
408}
409
Carter Cooper6dd94c82011-10-13 14:43:53 -0600410void adreno_ringbuffer_close(struct adreno_ringbuffer *rb)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700411{
412 struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
413
414 kgsl_sharedmem_free(&rb->buffer_desc);
415 kgsl_sharedmem_free(&rb->memptrs_desc);
416
417 kfree(adreno_dev->pfp_fw);
418 kfree(adreno_dev->pm4_fw);
419
420 adreno_dev->pfp_fw = NULL;
421 adreno_dev->pm4_fw = NULL;
422
423 memset(rb, 0, sizeof(struct adreno_ringbuffer));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700424}
425
426static uint32_t
427adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700428 struct adreno_context *context,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700429 unsigned int flags, unsigned int *cmds,
430 int sizedwords)
431{
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700432 struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700433 unsigned int *ringcmds;
434 unsigned int timestamp;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700435 unsigned int total_sizedwords = sizedwords;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700436 unsigned int i;
437 unsigned int rcmd_gpu;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700438 unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
439 unsigned int gpuaddr = rb->device->memstore.gpuaddr;
440
441 if (context != NULL) {
442 /*
443 * if the context was not created with per context timestamp
444 * support, we must use the global timestamp since issueibcmds
445 * will be returning that one.
446 */
447 if (context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
448 context_id = context->id;
449 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700450
451 /* reserve space to temporarily turn off protected mode
452 * error checking if needed
453 */
454 total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
455 total_sizedwords += !(flags & KGSL_CMD_FLAGS_NO_TS_CMP) ? 7 : 0;
456 total_sizedwords += !(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD) ? 2 : 0;
457
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700458 if (adreno_is_a3xx(adreno_dev))
459 total_sizedwords += 7;
460
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700461 total_sizedwords += 2; /* scratchpad ts for recovery */
462 if (context) {
463 total_sizedwords += 3; /* sop timestamp */
464 total_sizedwords += 4; /* eop timestamp */
465 }
466 total_sizedwords += 4; /* global timestamp for recovery*/
467
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700468 ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
469 rcmd_gpu = rb->buffer_desc.gpuaddr
470 + sizeof(uint)*(rb->wptr-total_sizedwords);
471
472 if (!(flags & KGSL_CMD_FLAGS_NOT_KERNEL_CMD)) {
Jordan Crouse084427d2011-07-28 08:37:58 -0600473 GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700474 GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
475 }
476 if (flags & KGSL_CMD_FLAGS_PMODE) {
477 /* disable protected mode error checking */
478 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600479 cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700480 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
481 }
482
483 for (i = 0; i < sizedwords; i++) {
484 GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
485 cmds++;
486 }
487
488 if (flags & KGSL_CMD_FLAGS_PMODE) {
489 /* re-enable protected mode error checking */
490 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600491 cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700492 GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
493 }
494
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700495 /* always increment the global timestamp. once. */
496 rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
497 if (context) {
498 if (context_id == KGSL_MEMSTORE_GLOBAL)
499 rb->timestamp[context_id] =
500 rb->timestamp[KGSL_MEMSTORE_GLOBAL];
501 else
502 rb->timestamp[context_id]++;
503 }
504 timestamp = rb->timestamp[context_id];
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700505
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700506 /* scratchpad ts for recovery */
Jordan Crouse084427d2011-07-28 08:37:58 -0600507 GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700508 GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700509
510 if (adreno_is_a3xx(adreno_dev)) {
511 /*
512 * FLush HLSQ lazy updates to make sure there are no
513 * rsources pending for indirect loads after the timestamp
514 */
515
516 GSL_RB_WRITE(ringcmds, rcmd_gpu,
517 cp_type3_packet(CP_EVENT_WRITE, 1));
518 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */
519 GSL_RB_WRITE(ringcmds, rcmd_gpu,
520 cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
521 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
522 }
523
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700524 if (context) {
525 /* start-of-pipeline timestamp */
526 GSL_RB_WRITE(ringcmds, rcmd_gpu,
527 cp_type3_packet(CP_MEM_WRITE, 2));
528 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
529 KGSL_MEMSTORE_OFFSET(context->id, soptimestamp)));
530 GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
531
532 /* end-of-pipeline timestamp */
533 GSL_RB_WRITE(ringcmds, rcmd_gpu,
534 cp_type3_packet(CP_EVENT_WRITE, 3));
535 GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
536 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
537 KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp)));
538 GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
539 }
540
Jordan Crouse084427d2011-07-28 08:37:58 -0600541 GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type3_packet(CP_EVENT_WRITE, 3));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700542 GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700543 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
544 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
545 eoptimestamp)));
546 GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700547
548 if (!(flags & KGSL_CMD_FLAGS_NO_TS_CMP)) {
549 /* Conditional execution based on memory values */
550 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600551 cp_type3_packet(CP_COND_EXEC, 4));
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700552 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
553 KGSL_MEMSTORE_OFFSET(
554 context_id, ts_cmp_enable)) >> 2);
555 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
556 KGSL_MEMSTORE_OFFSET(
557 context_id, ref_wait_ts)) >> 2);
558 GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700559 /* # of conditional command DWORDs */
560 GSL_RB_WRITE(ringcmds, rcmd_gpu, 2);
561 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600562 cp_type3_packet(CP_INTERRUPT, 1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700563 GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
564 }
565
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700566 if (adreno_is_a3xx(adreno_dev)) {
567 /* Dummy set-constant to trigger context rollover */
568 GSL_RB_WRITE(ringcmds, rcmd_gpu,
569 cp_type3_packet(CP_SET_CONSTANT, 2));
570 GSL_RB_WRITE(ringcmds, rcmd_gpu,
571 (0x4<<16)|(A3XX_HLSQ_CL_KERNEL_GROUP_X_REG - 0x2000));
572 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
573 }
574
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700575 adreno_ringbuffer_submit(rb);
576
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700577 return timestamp;
578}
579
580void
581adreno_ringbuffer_issuecmds(struct kgsl_device *device,
582 unsigned int flags,
583 unsigned int *cmds,
584 int sizedwords)
585{
586 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
587 struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
588
589 if (device->state & KGSL_STATE_HUNG)
590 return;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700591 adreno_ringbuffer_addcmds(rb, NULL, flags, cmds, sizedwords);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700592}
593
Jeremy Gebbend0ab6ad2012-04-06 11:13:35 -0600594static bool _parse_ibs(struct kgsl_device_private *dev_priv, uint gpuaddr,
595 int sizedwords);
596
597static bool
598_handle_type3(struct kgsl_device_private *dev_priv, uint *hostaddr)
599{
600 unsigned int opcode = cp_type3_opcode(*hostaddr);
601 switch (opcode) {
602 case CP_INDIRECT_BUFFER_PFD:
603 case CP_INDIRECT_BUFFER_PFE:
604 case CP_COND_INDIRECT_BUFFER_PFE:
605 case CP_COND_INDIRECT_BUFFER_PFD:
606 return _parse_ibs(dev_priv, hostaddr[1], hostaddr[2]);
607 case CP_NOP:
608 case CP_WAIT_FOR_IDLE:
609 case CP_WAIT_REG_MEM:
610 case CP_WAIT_REG_EQ:
611 case CP_WAT_REG_GTE:
612 case CP_WAIT_UNTIL_READ:
613 case CP_WAIT_IB_PFD_COMPLETE:
614 case CP_REG_RMW:
615 case CP_REG_TO_MEM:
616 case CP_MEM_WRITE:
617 case CP_MEM_WRITE_CNTR:
618 case CP_COND_EXEC:
619 case CP_COND_WRITE:
620 case CP_EVENT_WRITE:
621 case CP_EVENT_WRITE_SHD:
622 case CP_EVENT_WRITE_CFL:
623 case CP_EVENT_WRITE_ZPD:
624 case CP_DRAW_INDX:
625 case CP_DRAW_INDX_2:
626 case CP_DRAW_INDX_BIN:
627 case CP_DRAW_INDX_2_BIN:
628 case CP_VIZ_QUERY:
629 case CP_SET_STATE:
630 case CP_SET_CONSTANT:
631 case CP_IM_LOAD:
632 case CP_IM_LOAD_IMMEDIATE:
633 case CP_LOAD_CONSTANT_CONTEXT:
634 case CP_INVALIDATE_STATE:
635 case CP_SET_SHADER_BASES:
636 case CP_SET_BIN_MASK:
637 case CP_SET_BIN_SELECT:
638 case CP_SET_BIN_BASE_OFFSET:
639 case CP_SET_BIN_DATA:
640 case CP_CONTEXT_UPDATE:
641 case CP_INTERRUPT:
642 case CP_IM_STORE:
643 case CP_LOAD_STATE:
644 break;
645 /* these shouldn't come from userspace */
646 case CP_ME_INIT:
647 case CP_SET_PROTECTED_MODE:
648 default:
649 KGSL_CMD_ERR(dev_priv->device, "bad CP opcode %0x\n", opcode);
650 return false;
651 break;
652 }
653
654 return true;
655}
656
657static bool
658_handle_type0(struct kgsl_device_private *dev_priv, uint *hostaddr)
659{
660 unsigned int reg = type0_pkt_offset(*hostaddr);
661 unsigned int cnt = type0_pkt_size(*hostaddr);
662 if (reg < 0x0192 || (reg + cnt) >= 0x8000) {
663 KGSL_CMD_ERR(dev_priv->device, "bad type0 reg: 0x%0x cnt: %d\n",
664 reg, cnt);
665 return false;
666 }
667 return true;
668}
669
670/*
671 * Traverse IBs and dump them to test vector. Detect swap by inspecting
672 * register writes, keeping note of the current state, and dump
673 * framebuffer config to test vector
674 */
675static bool _parse_ibs(struct kgsl_device_private *dev_priv,
676 uint gpuaddr, int sizedwords)
677{
678 static uint level; /* recursion level */
679 bool ret = false;
680 uint *hostaddr, *hoststart;
681 int dwords_left = sizedwords; /* dwords left in the current command
682 buffer */
683 struct kgsl_mem_entry *entry;
684
685 spin_lock(&dev_priv->process_priv->mem_lock);
686 entry = kgsl_sharedmem_find_region(dev_priv->process_priv,
687 gpuaddr, sizedwords * sizeof(uint));
688 spin_unlock(&dev_priv->process_priv->mem_lock);
689 if (entry == NULL) {
690 KGSL_CMD_ERR(dev_priv->device,
691 "no mapping for gpuaddr: 0x%08x\n", gpuaddr);
692 return false;
693 }
694
695 hostaddr = (uint *)kgsl_gpuaddr_to_vaddr(&entry->memdesc, gpuaddr);
696 if (hostaddr == NULL) {
697 KGSL_CMD_ERR(dev_priv->device,
698 "no mapping for gpuaddr: 0x%08x\n", gpuaddr);
699 return false;
700 }
701
702 hoststart = hostaddr;
703
704 level++;
705
706 KGSL_CMD_INFO(dev_priv->device, "ib: gpuaddr:0x%08x, wc:%d, hptr:%p\n",
707 gpuaddr, sizedwords, hostaddr);
708
709 mb();
710 while (dwords_left > 0) {
711 bool cur_ret = true;
712 int count = 0; /* dword count including packet header */
713
714 switch (*hostaddr >> 30) {
715 case 0x0: /* type-0 */
716 count = (*hostaddr >> 16)+2;
717 cur_ret = _handle_type0(dev_priv, hostaddr);
718 break;
719 case 0x1: /* type-1 */
720 count = 2;
721 break;
722 case 0x3: /* type-3 */
723 count = ((*hostaddr >> 16) & 0x3fff) + 2;
724 cur_ret = _handle_type3(dev_priv, hostaddr);
725 break;
726 default:
727 KGSL_CMD_ERR(dev_priv->device, "unexpected type: "
728 "type:%d, word:0x%08x @ 0x%p, gpu:0x%08x\n",
729 *hostaddr >> 30, *hostaddr, hostaddr,
730 gpuaddr+4*(sizedwords-dwords_left));
731 cur_ret = false;
732 count = dwords_left;
733 break;
734 }
735
736 if (!cur_ret) {
737 KGSL_CMD_ERR(dev_priv->device,
738 "bad sub-type: #:%d/%d, v:0x%08x"
739 " @ 0x%p[gb:0x%08x], level:%d\n",
740 sizedwords-dwords_left, sizedwords, *hostaddr,
741 hostaddr, gpuaddr+4*(sizedwords-dwords_left),
742 level);
743
744 if (ADRENO_DEVICE(dev_priv->device)->ib_check_level
745 >= 2)
746 print_hex_dump(KERN_ERR,
747 level == 1 ? "IB1:" : "IB2:",
748 DUMP_PREFIX_OFFSET, 32, 4, hoststart,
749 sizedwords*4, 0);
750 goto done;
751 }
752
753 /* jump to next packet */
754 dwords_left -= count;
755 hostaddr += count;
756 if (dwords_left < 0) {
757 KGSL_CMD_ERR(dev_priv->device,
758 "bad count: c:%d, #:%d/%d, "
759 "v:0x%08x @ 0x%p[gb:0x%08x], level:%d\n",
760 count, sizedwords-(dwords_left+count),
761 sizedwords, *(hostaddr-count), hostaddr-count,
762 gpuaddr+4*(sizedwords-(dwords_left+count)),
763 level);
764 if (ADRENO_DEVICE(dev_priv->device)->ib_check_level
765 >= 2)
766 print_hex_dump(KERN_ERR,
767 level == 1 ? "IB1:" : "IB2:",
768 DUMP_PREFIX_OFFSET, 32, 4, hoststart,
769 sizedwords*4, 0);
770 goto done;
771 }
772 }
773
774 ret = true;
775done:
776 if (!ret)
777 KGSL_DRV_ERR(dev_priv->device,
778 "parsing failed: gpuaddr:0x%08x, "
779 "host:0x%p, wc:%d\n", gpuaddr, hoststart, sizedwords);
780
781 level--;
782
783 return ret;
784}
785
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700786int
787adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv,
788 struct kgsl_context *context,
789 struct kgsl_ibdesc *ibdesc,
790 unsigned int numibs,
791 uint32_t *timestamp,
792 unsigned int flags)
793{
794 struct kgsl_device *device = dev_priv->device;
795 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
796 unsigned int *link;
797 unsigned int *cmds;
798 unsigned int i;
Jeremy Gebben3c127f52011-08-08 17:04:11 -0600799 struct adreno_context *drawctxt;
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -0700800 unsigned int start_index = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700801
802 if (device->state & KGSL_STATE_HUNG)
803 return -EBUSY;
804 if (!(adreno_dev->ringbuffer.flags & KGSL_FLAGS_STARTED) ||
Jeremy Gebben3c127f52011-08-08 17:04:11 -0600805 context == NULL || ibdesc == 0 || numibs == 0)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700806 return -EINVAL;
807
Jeremy Gebben3c127f52011-08-08 17:04:11 -0600808 drawctxt = context->devctxt;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700809
810 if (drawctxt->flags & CTXT_FLAGS_GPU_HANG) {
811 KGSL_CTXT_WARN(device, "Context %p caused a gpu hang.."
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700812 " will not accept commands for context %d\n",
813 drawctxt, drawctxt->id);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700814 return -EDEADLK;
815 }
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600816
817 cmds = link = kzalloc(sizeof(unsigned int) * (numibs * 3 + 4),
818 GFP_KERNEL);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700819 if (!link) {
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600820 KGSL_CORE_ERR("kzalloc(%d) failed\n",
821 sizeof(unsigned int) * (numibs * 3 + 4));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700822 return -ENOMEM;
823 }
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -0700824
825 /*When preamble is enabled, the preamble buffer with state restoration
826 commands are stored in the first node of the IB chain. We can skip that
827 if a context switch hasn't occured */
828
829 if (drawctxt->flags & CTXT_FLAGS_PREAMBLE &&
830 adreno_dev->drawctxt_active == drawctxt)
831 start_index = 1;
832
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600833 if (!start_index) {
834 *cmds++ = cp_nop_packet(1);
835 *cmds++ = KGSL_START_OF_IB_IDENTIFIER;
836 } else {
837 *cmds++ = cp_nop_packet(4);
838 *cmds++ = KGSL_START_OF_IB_IDENTIFIER;
839 *cmds++ = CP_HDR_INDIRECT_BUFFER_PFD;
840 *cmds++ = ibdesc[0].gpuaddr;
841 *cmds++ = ibdesc[0].sizedwords;
842 }
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -0700843 for (i = start_index; i < numibs; i++) {
Jeremy Gebbend0ab6ad2012-04-06 11:13:35 -0600844 if (unlikely(adreno_dev->ib_check_level >= 1 &&
845 !_parse_ibs(dev_priv, ibdesc[i].gpuaddr,
846 ibdesc[i].sizedwords))) {
847 kfree(link);
848 return -EINVAL;
849 }
Jordan Crouse084427d2011-07-28 08:37:58 -0600850 *cmds++ = CP_HDR_INDIRECT_BUFFER_PFD;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700851 *cmds++ = ibdesc[i].gpuaddr;
852 *cmds++ = ibdesc[i].sizedwords;
853 }
854
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600855 *cmds++ = cp_nop_packet(1);
856 *cmds++ = KGSL_END_OF_IB_IDENTIFIER;
857
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700858 kgsl_setstate(device,
Shubhraprakash Das767fdda2011-08-15 15:49:45 -0600859 kgsl_mmu_pt_get_flags(device->mmu.hwpagetable,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700860 device->id));
861
862 adreno_drawctxt_switch(adreno_dev, drawctxt, flags);
863
864 *timestamp = adreno_ringbuffer_addcmds(&adreno_dev->ringbuffer,
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700865 drawctxt,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700866 KGSL_CMD_FLAGS_NOT_KERNEL_CMD,
867 &link[0], (cmds - link));
868
869 KGSL_CMD_INFO(device, "ctxt %d g %08x numibs %d ts %d\n",
870 context->id, (unsigned int)ibdesc, numibs, *timestamp);
871
872 kfree(link);
873
874#ifdef CONFIG_MSM_KGSL_CFF_DUMP
875 /*
876 * insert wait for idle after every IB1
877 * this is conservative but works reliably and is ok
878 * even for performance simulations
879 */
880 adreno_idle(device, KGSL_TIMEOUT_DEFAULT);
881#endif
882
883 return 0;
884}
885
886int adreno_ringbuffer_extract(struct adreno_ringbuffer *rb,
887 unsigned int *temp_rb_buffer,
888 int *rb_size)
889{
890 struct kgsl_device *device = rb->device;
891 unsigned int rb_rptr;
892 unsigned int retired_timestamp;
893 unsigned int temp_idx = 0;
894 unsigned int value;
895 unsigned int val1;
896 unsigned int val2;
897 unsigned int val3;
898 unsigned int copy_rb_contents = 0;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700899 struct kgsl_context *context;
900 unsigned int context_id;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700901
902 GSL_RB_GET_READPTR(rb, &rb->rptr);
903
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700904 /* current_context is the context that is presently active in the
905 * GPU, i.e the context in which the hang is caused */
906 kgsl_sharedmem_readl(&device->memstore, &context_id,
907 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
908 current_context));
909 KGSL_DRV_ERR(device, "Last context id: %d\n", context_id);
910 context = idr_find(&device->context_idr, context_id);
911 if (context == NULL) {
912 KGSL_DRV_ERR(device,
913 "GPU recovery from hang not possible because last"
914 " context id is invalid.\n");
915 return -EINVAL;
916 }
917 retired_timestamp = device->ftbl->readtimestamp(device, context,
918 KGSL_TIMESTAMP_RETIRED);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700919 KGSL_DRV_ERR(device, "GPU successfully executed till ts: %x\n",
920 retired_timestamp);
921 /*
922 * We need to go back in history by 4 dwords from the current location
923 * of read pointer as 4 dwords are read to match the end of a command.
924 * Also, take care of wrap around when moving back
925 */
926 if (rb->rptr >= 4)
927 rb_rptr = (rb->rptr - 4) * sizeof(unsigned int);
928 else
929 rb_rptr = rb->buffer_desc.size -
930 ((4 - rb->rptr) * sizeof(unsigned int));
931 /* Read the rb contents going backwards to locate end of last
932 * sucessfully executed command */
933 while ((rb_rptr / sizeof(unsigned int)) != rb->wptr) {
934 kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr);
935 if (value == retired_timestamp) {
936 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
937 rb->buffer_desc.size);
938 kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr);
939 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
940 rb->buffer_desc.size);
941 kgsl_sharedmem_readl(&rb->buffer_desc, &val2, rb_rptr);
942 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
943 rb->buffer_desc.size);
944 kgsl_sharedmem_readl(&rb->buffer_desc, &val3, rb_rptr);
945 /* match the pattern found at the end of a command */
946 if ((val1 == 2 &&
Jordan Crouse084427d2011-07-28 08:37:58 -0600947 val2 == cp_type3_packet(CP_INTERRUPT, 1)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700948 && val3 == CP_INT_CNTL__RB_INT_MASK) ||
Jordan Crouse084427d2011-07-28 08:37:58 -0600949 (val1 == cp_type3_packet(CP_EVENT_WRITE, 3)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700950 && val2 == CACHE_FLUSH_TS &&
951 val3 == (rb->device->memstore.gpuaddr +
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700952 KGSL_MEMSTORE_OFFSET(context_id,
953 eoptimestamp)))) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700954 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
955 rb->buffer_desc.size);
956 KGSL_DRV_ERR(device,
957 "Found end of last executed "
958 "command at offset: %x\n",
959 rb_rptr / sizeof(unsigned int));
960 break;
961 } else {
962 if (rb_rptr < (3 * sizeof(unsigned int)))
963 rb_rptr = rb->buffer_desc.size -
964 (3 * sizeof(unsigned int))
965 + rb_rptr;
966 else
967 rb_rptr -= (3 * sizeof(unsigned int));
968 }
969 }
970
971 if (rb_rptr == 0)
972 rb_rptr = rb->buffer_desc.size - sizeof(unsigned int);
973 else
974 rb_rptr -= sizeof(unsigned int);
975 }
976
977 if ((rb_rptr / sizeof(unsigned int)) == rb->wptr) {
978 KGSL_DRV_ERR(device,
979 "GPU recovery from hang not possible because last"
980 " successful timestamp is overwritten\n");
981 return -EINVAL;
982 }
983 /* rb_rptr is now pointing to the first dword of the command following
984 * the last sucessfully executed command sequence. Assumption is that
985 * GPU is hung in the command sequence pointed by rb_rptr */
986 /* make sure the GPU is not hung in a command submitted by kgsl
987 * itself */
988 kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr);
989 kgsl_sharedmem_readl(&rb->buffer_desc, &val2,
990 adreno_ringbuffer_inc_wrapped(rb_rptr,
991 rb->buffer_desc.size));
Jordan Crouse084427d2011-07-28 08:37:58 -0600992 if (val1 == cp_nop_packet(1) && val2 == KGSL_CMD_IDENTIFIER) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700993 KGSL_DRV_ERR(device,
994 "GPU recovery from hang not possible because "
995 "of hang in kgsl command\n");
996 return -EINVAL;
997 }
998
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700999 while ((rb_rptr / sizeof(unsigned int)) != rb->wptr) {
1000 kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr);
1001 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
1002 rb->buffer_desc.size);
1003 /* check for context switch indicator */
1004 if (value == KGSL_CONTEXT_TO_MEM_IDENTIFIER) {
1005 kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr);
1006 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
1007 rb->buffer_desc.size);
Jordan Crouse084427d2011-07-28 08:37:58 -06001008 BUG_ON(value != cp_type3_packet(CP_MEM_WRITE, 2));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001009 kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr);
1010 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
1011 rb->buffer_desc.size);
1012 BUG_ON(val1 != (device->memstore.gpuaddr +
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001013 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
1014 current_context)));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001015 kgsl_sharedmem_readl(&rb->buffer_desc, &value, rb_rptr);
1016 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr,
1017 rb->buffer_desc.size);
Jordan Crousea400d8d2012-03-16 14:53:39 -06001018
1019 /*
1020 * If other context switches were already lost and
1021 * and the current context is the one that is hanging,
1022 * then we cannot recover. Print an error message
1023 * and leave.
1024 */
1025
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001026 if ((copy_rb_contents == 0) && (value == context_id)) {
Jordan Crousea400d8d2012-03-16 14:53:39 -06001027 KGSL_DRV_ERR(device, "GPU recovery could not "
1028 "find the previous context\n");
1029 return -EINVAL;
1030 }
1031
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001032 /*
1033 * If we were copying the commands and got to this point
1034 * then we need to remove the 3 commands that appear
1035 * before KGSL_CONTEXT_TO_MEM_IDENTIFIER
1036 */
1037 if (temp_idx)
1038 temp_idx -= 3;
1039 /* if context switches to a context that did not cause
1040 * hang then start saving the rb contents as those
1041 * commands can be executed */
Carter Cooper7e7f02e2012-02-15 09:36:31 -07001042 if (value != context_id) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001043 copy_rb_contents = 1;
Jordan Crouse084427d2011-07-28 08:37:58 -06001044 temp_rb_buffer[temp_idx++] = cp_nop_packet(1);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001045 temp_rb_buffer[temp_idx++] =
1046 KGSL_CMD_IDENTIFIER;
Jordan Crouse084427d2011-07-28 08:37:58 -06001047 temp_rb_buffer[temp_idx++] = cp_nop_packet(1);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001048 temp_rb_buffer[temp_idx++] =
1049 KGSL_CONTEXT_TO_MEM_IDENTIFIER;
1050 temp_rb_buffer[temp_idx++] =
Jordan Crouse084427d2011-07-28 08:37:58 -06001051 cp_type3_packet(CP_MEM_WRITE, 2);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001052 temp_rb_buffer[temp_idx++] = val1;
1053 temp_rb_buffer[temp_idx++] = value;
1054 } else {
1055 copy_rb_contents = 0;
1056 }
1057 } else if (copy_rb_contents)
1058 temp_rb_buffer[temp_idx++] = value;
1059 }
1060
1061 *rb_size = temp_idx;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001062 return 0;
1063}
1064
1065void
1066adreno_ringbuffer_restore(struct adreno_ringbuffer *rb, unsigned int *rb_buff,
1067 int num_rb_contents)
1068{
1069 int i;
1070 unsigned int *ringcmds;
1071 unsigned int rcmd_gpu;
1072
1073 if (!num_rb_contents)
1074 return;
1075
1076 if (num_rb_contents > (rb->buffer_desc.size - rb->wptr)) {
1077 adreno_regwrite(rb->device, REG_CP_RB_RPTR, 0);
1078 rb->rptr = 0;
1079 BUG_ON(num_rb_contents > rb->buffer_desc.size);
1080 }
1081 ringcmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
1082 rcmd_gpu = rb->buffer_desc.gpuaddr + sizeof(unsigned int) * rb->wptr;
1083 for (i = 0; i < num_rb_contents; i++)
1084 GSL_RB_WRITE(ringcmds, rcmd_gpu, rb_buff[i]);
1085 rb->wptr += num_rb_contents;
1086 adreno_ringbuffer_submit(rb);
1087}