blob: 179027c3b89a042e0d367e90b8676c204c27f98f [file] [log] [blame]
Carter Cooper740f6742013-01-03 16:19:23 -07001/* Copyright (c) 2002,2007-2013, The Linux Foundation. All rights reserved.
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07002 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 */
13#include <linux/firmware.h>
14#include <linux/slab.h>
15#include <linux/sched.h>
16#include <linux/log2.h>
17
18#include "kgsl.h"
19#include "kgsl_sharedmem.h"
20#include "kgsl_cffdump.h"
21
22#include "adreno.h"
23#include "adreno_pm4types.h"
24#include "adreno_ringbuffer.h"
25
Jeremy Gebbeneebc4612011-08-31 10:15:21 -070026#include "a2xx_reg.h"
Jordan Crouseb4d31bd2012-02-01 22:11:12 -070027#include "a3xx_reg.h"
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070028
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070029#define GSL_RB_NOP_SIZEDWORDS 2
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070030
Jordan Crousef50bfdc2012-11-01 13:48:35 -060031/*
32 * CP DEBUG settings for all cores:
33 * DYNAMIC_CLK_DISABLE [27] - turn off the dynamic clock control
34 * PROG_END_PTR_ENABLE [25] - Allow 128 bit writes to the VBIF
35 */
36
37#define CP_DEBUG_DEFAULT ((1 << 27) | (1 << 25))
38
Jordan Crouseb4d31bd2012-02-01 22:11:12 -070039void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070040{
41 BUG_ON(rb->wptr == 0);
42
Lucille Sylvester958dc942011-09-06 18:19:49 -060043 /* Let the pwrscale policy know that new commands have
44 been submitted. */
45 kgsl_pwrscale_busy(rb->device);
46
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070047 /*synchronize memory before informing the hardware of the
48 *new commands.
49 */
50 mb();
51
52 adreno_regwrite(rb->device, REG_CP_RB_WPTR, rb->wptr);
53}
54
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -070055static int
56adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb,
57 struct adreno_context *context,
58 unsigned int numcmds, int wptr_ahead)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070059{
60 int nopcount;
61 unsigned int freecmds;
62 unsigned int *cmds;
63 uint cmds_gpu;
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -060064 unsigned long wait_time;
Jordan Crouse21f75a02012-08-09 15:08:59 -060065 unsigned long wait_timeout = msecs_to_jiffies(ADRENO_IDLE_TIMEOUT);
Tarun Karra3335f142012-06-19 14:11:48 -070066 unsigned long wait_time_part;
Tarun Karra696f89e2013-01-27 21:31:40 -080067 unsigned int prev_reg_val[ft_detect_regs_count];
Tarun Karra3335f142012-06-19 14:11:48 -070068
69 memset(prev_reg_val, 0, sizeof(prev_reg_val));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070070
71 /* if wptr ahead, fill the remaining with NOPs */
72 if (wptr_ahead) {
73 /* -1 for header */
74 nopcount = rb->sizedwords - rb->wptr - 1;
75
76 cmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
77 cmds_gpu = rb->buffer_desc.gpuaddr + sizeof(uint)*rb->wptr;
78
Jordan Crouse084427d2011-07-28 08:37:58 -060079 GSL_RB_WRITE(cmds, cmds_gpu, cp_nop_packet(nopcount));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070080
81 /* Make sure that rptr is not 0 before submitting
82 * commands at the end of ringbuffer. We do not
83 * want the rptr and wptr to become equal when
84 * the ringbuffer is not empty */
85 do {
86 GSL_RB_GET_READPTR(rb, &rb->rptr);
87 } while (!rb->rptr);
88
89 rb->wptr++;
90
91 adreno_ringbuffer_submit(rb);
92
93 rb->wptr = 0;
94 }
95
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -060096 wait_time = jiffies + wait_timeout;
Jordan Crouse21f75a02012-08-09 15:08:59 -060097 wait_time_part = jiffies + msecs_to_jiffies(KGSL_TIMEOUT_PART);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -070098 /* wait for space in ringbuffer */
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -060099 while (1) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700100 GSL_RB_GET_READPTR(rb, &rb->rptr);
101
102 freecmds = rb->rptr - rb->wptr;
103
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600104 if (freecmds == 0 || freecmds > numcmds)
105 break;
106
Tarun Karra3335f142012-06-19 14:11:48 -0700107 /* Dont wait for timeout, detect hang faster.
108 */
109 if (time_after(jiffies, wait_time_part)) {
110 wait_time_part = jiffies +
Jordan Crouse21f75a02012-08-09 15:08:59 -0600111 msecs_to_jiffies(KGSL_TIMEOUT_PART);
Tarun Karra696f89e2013-01-27 21:31:40 -0800112 if ((adreno_ft_detect(rb->device,
Tarun Karra3335f142012-06-19 14:11:48 -0700113 prev_reg_val))){
114 KGSL_DRV_ERR(rb->device,
115 "Hang detected while waiting for freespace in"
116 "ringbuffer rptr: 0x%x, wptr: 0x%x\n",
117 rb->rptr, rb->wptr);
118 goto err;
119 }
120 }
121
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600122 if (time_after(jiffies, wait_time)) {
123 KGSL_DRV_ERR(rb->device,
124 "Timed out while waiting for freespace in ringbuffer "
125 "rptr: 0x%x, wptr: 0x%x\n", rb->rptr, rb->wptr);
Tarun Karra3335f142012-06-19 14:11:48 -0700126 goto err;
127 }
128
Wei Zou50ec3372012-07-17 15:46:52 -0700129 continue;
130
Tarun Karra3335f142012-06-19 14:11:48 -0700131err:
Tarun Karrad20d71a2013-01-25 15:38:57 -0800132 if (!adreno_dump_and_exec_ft(rb->device)) {
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700133 if (context && context->flags & CTXT_FLAGS_GPU_HANG) {
134 KGSL_CTXT_WARN(rb->device,
135 "Context %p caused a gpu hang. Will not accept commands for context %d\n",
136 context, context->id);
137 return -EDEADLK;
138 }
139 wait_time = jiffies + wait_timeout;
140 } else {
Tarun Karrad20d71a2013-01-25 15:38:57 -0800141 /* GPU is hung and fault tolerance failed */
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700142 BUG();
143 }
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600144 }
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700145 return 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700146}
147
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700148unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb,
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700149 struct adreno_context *context,
150 unsigned int numcmds)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700151{
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700152 unsigned int *ptr = NULL;
153 int ret = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700154 BUG_ON(numcmds >= rb->sizedwords);
155
156 GSL_RB_GET_READPTR(rb, &rb->rptr);
157 /* check for available space */
158 if (rb->wptr >= rb->rptr) {
159 /* wptr ahead or equal to rptr */
160 /* reserve dwords for nop packet */
161 if ((rb->wptr + numcmds) > (rb->sizedwords -
162 GSL_RB_NOP_SIZEDWORDS))
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700163 ret = adreno_ringbuffer_waitspace(rb, context,
164 numcmds, 1);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700165 } else {
166 /* wptr behind rptr */
167 if ((rb->wptr + numcmds) >= rb->rptr)
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700168 ret = adreno_ringbuffer_waitspace(rb, context,
169 numcmds, 0);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700170 /* check for remaining space */
171 /* reserve dwords for nop packet */
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700172 if (!ret && (rb->wptr + numcmds) > (rb->sizedwords -
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700173 GSL_RB_NOP_SIZEDWORDS))
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700174 ret = adreno_ringbuffer_waitspace(rb, context,
175 numcmds, 1);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700176 }
177
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700178 if (!ret) {
179 ptr = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
180 rb->wptr += numcmds;
181 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700182
183 return ptr;
184}
185
186static int _load_firmware(struct kgsl_device *device, const char *fwfile,
187 void **data, int *len)
188{
189 const struct firmware *fw = NULL;
190 int ret;
191
192 ret = request_firmware(&fw, fwfile, device->dev);
193
194 if (ret) {
195 KGSL_DRV_ERR(device, "request_firmware(%s) failed: %d\n",
196 fwfile, ret);
197 return ret;
198 }
199
200 *data = kmalloc(fw->size, GFP_KERNEL);
201
202 if (*data) {
203 memcpy(*data, fw->data, fw->size);
204 *len = fw->size;
205 } else
206 KGSL_MEM_ERR(device, "kmalloc(%d) failed\n", fw->size);
207
208 release_firmware(fw);
209 return (*data != NULL) ? 0 : -ENOMEM;
210}
211
Tarun Karra9c070822012-11-27 16:43:51 -0700212int adreno_ringbuffer_read_pm4_ucode(struct kgsl_device *device)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700213{
214 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
Tarun Karra9c070822012-11-27 16:43:51 -0700215 int ret = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700216
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700217 if (adreno_dev->pm4_fw == NULL) {
218 int len;
Jordan Crouse505df9c2011-07-28 08:37:59 -0600219 void *ptr;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700220
Jordan Crouse505df9c2011-07-28 08:37:59 -0600221 ret = _load_firmware(device, adreno_dev->pm4_fwfile,
222 &ptr, &len);
223
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700224 if (ret)
225 goto err;
226
227 /* PM4 size is 3 dword aligned plus 1 dword of version */
228 if (len % ((sizeof(uint32_t) * 3)) != sizeof(uint32_t)) {
229 KGSL_DRV_ERR(device, "Bad firmware size: %d\n", len);
230 ret = -EINVAL;
Jeremy Gebben79acee62011-08-08 16:44:07 -0600231 kfree(ptr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700232 goto err;
233 }
234
235 adreno_dev->pm4_fw_size = len / sizeof(uint32_t);
236 adreno_dev->pm4_fw = ptr;
Tarun Karra9c070822012-11-27 16:43:51 -0700237 adreno_dev->pm4_fw_version = adreno_dev->pm4_fw[1];
238 }
239
240err:
241 return ret;
242}
243
244
245int adreno_ringbuffer_load_pm4_ucode(struct kgsl_device *device)
246{
247 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
248 int i;
249
250 if (adreno_dev->pm4_fw == NULL) {
251 int ret = adreno_ringbuffer_read_pm4_ucode(device);
252 if (ret)
253 return ret;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700254 }
255
256 KGSL_DRV_INFO(device, "loading pm4 ucode version: %d\n",
Tarun Karra9c070822012-11-27 16:43:51 -0700257 adreno_dev->pm4_fw_version);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700258
Jordan Crousef50bfdc2012-11-01 13:48:35 -0600259 adreno_regwrite(device, REG_CP_DEBUG, CP_DEBUG_DEFAULT);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700260 adreno_regwrite(device, REG_CP_ME_RAM_WADDR, 0);
261 for (i = 1; i < adreno_dev->pm4_fw_size; i++)
262 adreno_regwrite(device, REG_CP_ME_RAM_DATA,
Tarun Karra9c070822012-11-27 16:43:51 -0700263 adreno_dev->pm4_fw[i]);
264
265 return 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700266}
267
Tarun Karra9c070822012-11-27 16:43:51 -0700268int adreno_ringbuffer_read_pfp_ucode(struct kgsl_device *device)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700269{
270 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
Tarun Karra9c070822012-11-27 16:43:51 -0700271 int ret = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700272
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700273 if (adreno_dev->pfp_fw == NULL) {
274 int len;
Jordan Crouse505df9c2011-07-28 08:37:59 -0600275 void *ptr;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700276
Jordan Crouse505df9c2011-07-28 08:37:59 -0600277 ret = _load_firmware(device, adreno_dev->pfp_fwfile,
278 &ptr, &len);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700279 if (ret)
280 goto err;
281
282 /* PFP size shold be dword aligned */
283 if (len % sizeof(uint32_t) != 0) {
284 KGSL_DRV_ERR(device, "Bad firmware size: %d\n", len);
285 ret = -EINVAL;
Jeremy Gebben79acee62011-08-08 16:44:07 -0600286 kfree(ptr);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700287 goto err;
288 }
289
290 adreno_dev->pfp_fw_size = len / sizeof(uint32_t);
291 adreno_dev->pfp_fw = ptr;
Tarun Karra9c070822012-11-27 16:43:51 -0700292 adreno_dev->pfp_fw_version = adreno_dev->pfp_fw[5];
293 }
294
295err:
296 return ret;
297}
298
299int adreno_ringbuffer_load_pfp_ucode(struct kgsl_device *device)
300{
301 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
302 int i;
303
304 if (adreno_dev->pfp_fw == NULL) {
305 int ret = adreno_ringbuffer_read_pfp_ucode(device);
306 if (ret)
307 return ret;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700308 }
309
310 KGSL_DRV_INFO(device, "loading pfp ucode version: %d\n",
Tarun Karra9c070822012-11-27 16:43:51 -0700311 adreno_dev->pfp_fw_version);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700312
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700313 adreno_regwrite(device, adreno_dev->gpudev->reg_cp_pfp_ucode_addr, 0);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700314 for (i = 1; i < adreno_dev->pfp_fw_size; i++)
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700315 adreno_regwrite(device,
Tarun Karra9c070822012-11-27 16:43:51 -0700316 adreno_dev->gpudev->reg_cp_pfp_ucode_data,
317 adreno_dev->pfp_fw[i]);
318
319 return 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700320}
321
322int adreno_ringbuffer_start(struct adreno_ringbuffer *rb, unsigned int init_ram)
323{
324 int status;
325 /*cp_rb_cntl_u cp_rb_cntl; */
326 union reg_cp_rb_cntl cp_rb_cntl;
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700327 unsigned int rb_cntl;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700328 struct kgsl_device *device = rb->device;
Jeremy Gebbenddf6b572011-09-09 13:39:49 -0700329 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700330
331 if (rb->flags & KGSL_FLAGS_STARTED)
332 return 0;
333
Carter Coopercb3e8eb2012-04-11 09:39:40 -0600334 if (init_ram)
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700335 rb->timestamp[KGSL_MEMSTORE_GLOBAL] = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700336
337 kgsl_sharedmem_set(&rb->memptrs_desc, 0, 0,
338 sizeof(struct kgsl_rbmemptrs));
339
340 kgsl_sharedmem_set(&rb->buffer_desc, 0, 0xAA,
341 (rb->sizedwords << 2));
342
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700343 if (adreno_is_a2xx(adreno_dev)) {
344 adreno_regwrite(device, REG_CP_RB_WPTR_BASE,
345 (rb->memptrs_desc.gpuaddr
346 + GSL_RB_MEMPTRS_WPTRPOLL_OFFSET));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700347
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700348 /* setup WPTR delay */
349 adreno_regwrite(device, REG_CP_RB_WPTR_DELAY,
350 0 /*0x70000010 */);
351 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700352
353 /*setup REG_CP_RB_CNTL */
354 adreno_regread(device, REG_CP_RB_CNTL, &rb_cntl);
355 cp_rb_cntl.val = rb_cntl;
356
357 /*
358 * The size of the ringbuffer in the hardware is the log2
359 * representation of the size in quadwords (sizedwords / 2)
360 */
361 cp_rb_cntl.f.rb_bufsz = ilog2(rb->sizedwords >> 1);
362
363 /*
364 * Specify the quadwords to read before updating mem RPTR.
365 * Like above, pass the log2 representation of the blocksize
366 * in quadwords.
367 */
368 cp_rb_cntl.f.rb_blksz = ilog2(KGSL_RB_BLKSIZE >> 3);
369
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700370 if (adreno_is_a2xx(adreno_dev)) {
371 /* WPTR polling */
372 cp_rb_cntl.f.rb_poll_en = GSL_RB_CNTL_POLL_EN;
373 }
374
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700375 /* mem RPTR writebacks */
376 cp_rb_cntl.f.rb_no_update = GSL_RB_CNTL_NO_UPDATE;
377
378 adreno_regwrite(device, REG_CP_RB_CNTL, cp_rb_cntl.val);
379
380 adreno_regwrite(device, REG_CP_RB_BASE, rb->buffer_desc.gpuaddr);
381
382 adreno_regwrite(device, REG_CP_RB_RPTR_ADDR,
383 rb->memptrs_desc.gpuaddr +
384 GSL_RB_MEMPTRS_RPTR_OFFSET);
385
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700386 if (adreno_is_a3xx(adreno_dev)) {
387 /* enable access protection to privileged registers */
388 adreno_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007);
389
390 /* RBBM registers */
391 adreno_regwrite(device, A3XX_CP_PROTECT_REG_0, 0x63000040);
392 adreno_regwrite(device, A3XX_CP_PROTECT_REG_1, 0x62000080);
393 adreno_regwrite(device, A3XX_CP_PROTECT_REG_2, 0x600000CC);
394 adreno_regwrite(device, A3XX_CP_PROTECT_REG_3, 0x60000108);
395 adreno_regwrite(device, A3XX_CP_PROTECT_REG_4, 0x64000140);
396 adreno_regwrite(device, A3XX_CP_PROTECT_REG_5, 0x66000400);
397
398 /* CP registers */
399 adreno_regwrite(device, A3XX_CP_PROTECT_REG_6, 0x65000700);
400 adreno_regwrite(device, A3XX_CP_PROTECT_REG_7, 0x610007D8);
401 adreno_regwrite(device, A3XX_CP_PROTECT_REG_8, 0x620007E0);
402 adreno_regwrite(device, A3XX_CP_PROTECT_REG_9, 0x61001178);
403 adreno_regwrite(device, A3XX_CP_PROTECT_REG_A, 0x64001180);
404
405 /* RB registers */
406 adreno_regwrite(device, A3XX_CP_PROTECT_REG_B, 0x60003300);
407
408 /* VBIF registers */
409 adreno_regwrite(device, A3XX_CP_PROTECT_REG_C, 0x6B00C000);
410 }
411
412 if (adreno_is_a2xx(adreno_dev)) {
413 /* explicitly clear all cp interrupts */
414 adreno_regwrite(device, REG_CP_INT_ACK, 0xFFFFFFFF);
415 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700416
417 /* setup scratch/timestamp */
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700418 adreno_regwrite(device, REG_SCRATCH_ADDR, device->memstore.gpuaddr +
419 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
420 soptimestamp));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700421
422 adreno_regwrite(device, REG_SCRATCH_UMSK,
423 GSL_RB_MEMPTRS_SCRATCH_MASK);
424
425 /* load the CP ucode */
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700426 status = adreno_ringbuffer_load_pm4_ucode(device);
427 if (status != 0)
428 return status;
429
430 /* load the prefetch parser ucode */
431 status = adreno_ringbuffer_load_pfp_ucode(device);
432 if (status != 0)
433 return status;
434
Kevin Matlageff806df2012-05-07 18:13:21 -0600435 /* CP ROQ queue sizes (bytes) - RB:16, ST:16, IB1:32, IB2:64 */
Kevin Matlagee8d35862012-04-26 12:58:15 -0600436 if (adreno_is_a305(adreno_dev) || adreno_is_a320(adreno_dev))
Kevin Matlageff806df2012-05-07 18:13:21 -0600437 adreno_regwrite(device, REG_CP_QUEUE_THRESHOLDS, 0x000E0602);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700438
439 rb->rptr = 0;
440 rb->wptr = 0;
441
442 /* clear ME_HALT to start micro engine */
443 adreno_regwrite(device, REG_CP_ME_CNTL, 0);
444
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700445 /* ME init is GPU specific, so jump into the sub-function */
446 adreno_dev->gpudev->rb_init(adreno_dev, rb);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700447
448 /* idle device to validate ME INIT */
Jordan Crousea29a2e02012-08-14 09:09:23 -0600449 status = adreno_idle(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700450
451 if (status == 0)
452 rb->flags |= KGSL_FLAGS_STARTED;
453
454 return status;
455}
456
Carter Cooper6dd94c82011-10-13 14:43:53 -0600457void adreno_ringbuffer_stop(struct adreno_ringbuffer *rb)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700458{
Rajeev Kulkarnibf7a3822012-08-14 21:21:14 +0530459 struct kgsl_device *device = rb->device;
460 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
461
462 if (rb->flags & KGSL_FLAGS_STARTED) {
463 if (adreno_is_a200(adreno_dev))
464 adreno_regwrite(rb->device, REG_CP_ME_CNTL, 0x10000000);
465
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700466 rb->flags &= ~KGSL_FLAGS_STARTED;
Rajeev Kulkarnibf7a3822012-08-14 21:21:14 +0530467 }
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700468}
469
470int adreno_ringbuffer_init(struct kgsl_device *device)
471{
472 int status;
473 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
474 struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
475
476 rb->device = device;
477 /*
478 * It is silly to convert this to words and then back to bytes
479 * immediately below, but most of the rest of the code deals
480 * in words, so we might as well only do the math once
481 */
482 rb->sizedwords = KGSL_RB_SIZE >> 2;
483
484 /* allocate memory for ringbuffer */
485 status = kgsl_allocate_contiguous(&rb->buffer_desc,
486 (rb->sizedwords << 2));
487
488 if (status != 0) {
489 adreno_ringbuffer_close(rb);
490 return status;
491 }
492
493 /* allocate memory for polling and timestamps */
494 /* This really can be at 4 byte alignment boundry but for using MMU
495 * we need to make it at page boundary */
496 status = kgsl_allocate_contiguous(&rb->memptrs_desc,
497 sizeof(struct kgsl_rbmemptrs));
498
499 if (status != 0) {
500 adreno_ringbuffer_close(rb);
501 return status;
502 }
503
504 /* overlay structure on memptrs memory */
505 rb->memptrs = (struct kgsl_rbmemptrs *) rb->memptrs_desc.hostptr;
506
507 return 0;
508}
509
Carter Cooper6dd94c82011-10-13 14:43:53 -0600510void adreno_ringbuffer_close(struct adreno_ringbuffer *rb)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700511{
512 struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
513
514 kgsl_sharedmem_free(&rb->buffer_desc);
515 kgsl_sharedmem_free(&rb->memptrs_desc);
516
517 kfree(adreno_dev->pfp_fw);
518 kfree(adreno_dev->pm4_fw);
519
520 adreno_dev->pfp_fw = NULL;
521 adreno_dev->pm4_fw = NULL;
522
523 memset(rb, 0, sizeof(struct adreno_ringbuffer));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700524}
525
526static uint32_t
527adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700528 struct adreno_context *context,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700529 unsigned int flags, unsigned int *cmds,
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700530 int sizedwords, uint32_t timestamp)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700531{
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700532 struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700533 unsigned int *ringcmds;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700534 unsigned int total_sizedwords = sizedwords;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700535 unsigned int i;
536 unsigned int rcmd_gpu;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700537 unsigned int context_id = KGSL_MEMSTORE_GLOBAL;
538 unsigned int gpuaddr = rb->device->memstore.gpuaddr;
539
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600540 /*
541 * if the context was not created with per context timestamp
542 * support, we must use the global timestamp since issueibcmds
543 * will be returning that one.
544 */
Carter Cooperedbe4032012-11-20 11:09:38 -0700545 if (context && context->flags & CTXT_FLAGS_PER_CONTEXT_TS)
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600546 context_id = context->id;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700547
Carter Cooper4e8b4022012-11-30 11:34:18 -0700548 if ((context && context->flags & CTXT_FLAGS_USER_GENERATED_TS) &&
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700549 (!(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))) {
550 if (timestamp_cmp(rb->timestamp[context_id],
551 timestamp) >= 0) {
552 KGSL_DRV_ERR(rb->device,
553 "Invalid user generated ts <%d:0x%x>, "
554 "less than last issued ts <%d:0x%x>\n",
555 context_id, timestamp, context_id,
556 rb->timestamp[context_id]);
557 return -ERANGE;
558 }
559 }
560
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700561 /* reserve space to temporarily turn off protected mode
562 * error checking if needed
563 */
564 total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
Shubhraprakash Dasc3ad5802012-05-30 18:10:06 -0600565 /* 2 dwords to store the start of command sequence */
566 total_sizedwords += 2;
Jordan Crouseef02fc02013-03-05 11:19:31 -0700567
568 /* Add CP_COND_EXEC commands to generate CP_INTERRUPT */
569 total_sizedwords += context ? 13 : 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700570
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700571 if (adreno_is_a3xx(adreno_dev))
572 total_sizedwords += 7;
573
Tarun Karrad20d71a2013-01-25 15:38:57 -0800574 total_sizedwords += 2; /* scratchpad ts for fault tolerance */
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700575 if (context && context->flags & CTXT_FLAGS_PER_CONTEXT_TS &&
576 !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700577 total_sizedwords += 3; /* sop timestamp */
578 total_sizedwords += 4; /* eop timestamp */
Rajesh Kemisettic5699302012-04-21 21:09:05 +0530579 total_sizedwords += 3; /* global timestamp without cache
580 * flush for non-zero context */
581 } else {
Tarun Karrad20d71a2013-01-25 15:38:57 -0800582 total_sizedwords += 4; /* global timestamp for fault tolerance*/
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700583 }
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700584
Shubhraprakash Dasd316ff82012-08-02 12:43:48 -0700585 ringcmds = adreno_ringbuffer_allocspace(rb, context, total_sizedwords);
586 if (!ringcmds) {
587 /*
588 * We could not allocate space in ringbuffer, just return the
589 * last timestamp
590 */
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600591 return rb->timestamp[context_id];
592 }
593
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700594 rcmd_gpu = rb->buffer_desc.gpuaddr
595 + sizeof(uint)*(rb->wptr-total_sizedwords);
596
Shubhraprakash Dasc3ad5802012-05-30 18:10:06 -0600597 GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
598 GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_CMD_IDENTIFIER);
599
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700600 if (flags & KGSL_CMD_FLAGS_PMODE) {
601 /* disable protected mode error checking */
602 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600603 cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700604 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
605 }
606
607 for (i = 0; i < sizedwords; i++) {
608 GSL_RB_WRITE(ringcmds, rcmd_gpu, *cmds);
609 cmds++;
610 }
611
612 if (flags & KGSL_CMD_FLAGS_PMODE) {
613 /* re-enable protected mode error checking */
614 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600615 cp_type3_packet(CP_SET_PROTECTED_MODE, 1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700616 GSL_RB_WRITE(ringcmds, rcmd_gpu, 1);
617 }
618
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700619 /* always increment the global timestamp. once. */
620 rb->timestamp[KGSL_MEMSTORE_GLOBAL]++;
Carter Cooper7ffaba62012-05-24 13:59:53 -0600621
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700622 /* Do not update context's timestamp for internal submissions */
623 if (context && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700624 if (context_id == KGSL_MEMSTORE_GLOBAL)
Carter Cooper7ffaba62012-05-24 13:59:53 -0600625 rb->timestamp[context->id] =
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700626 rb->timestamp[KGSL_MEMSTORE_GLOBAL];
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700627 else if (context->flags & CTXT_FLAGS_USER_GENERATED_TS)
628 rb->timestamp[context_id] = timestamp;
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700629 else
630 rb->timestamp[context_id]++;
631 }
632 timestamp = rb->timestamp[context_id];
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700633
Tarun Karrad20d71a2013-01-25 15:38:57 -0800634 /* scratchpad ts for fault tolerance */
Jordan Crouse084427d2011-07-28 08:37:58 -0600635 GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_type0_packet(REG_CP_TIMESTAMP, 1));
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700636 GSL_RB_WRITE(ringcmds, rcmd_gpu, rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700637
638 if (adreno_is_a3xx(adreno_dev)) {
639 /*
640 * FLush HLSQ lazy updates to make sure there are no
641 * rsources pending for indirect loads after the timestamp
642 */
643
644 GSL_RB_WRITE(ringcmds, rcmd_gpu,
645 cp_type3_packet(CP_EVENT_WRITE, 1));
646 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x07); /* HLSQ_FLUSH */
647 GSL_RB_WRITE(ringcmds, rcmd_gpu,
648 cp_type3_packet(CP_WAIT_FOR_IDLE, 1));
649 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x00);
650 }
651
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700652 if (context && context->flags & CTXT_FLAGS_PER_CONTEXT_TS
653 && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700654 /* start-of-pipeline timestamp */
655 GSL_RB_WRITE(ringcmds, rcmd_gpu,
656 cp_type3_packet(CP_MEM_WRITE, 2));
657 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
Carter Cooper7ffaba62012-05-24 13:59:53 -0600658 KGSL_MEMSTORE_OFFSET(context_id, soptimestamp)));
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700659 GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
660
661 /* end-of-pipeline timestamp */
662 GSL_RB_WRITE(ringcmds, rcmd_gpu,
663 cp_type3_packet(CP_EVENT_WRITE, 3));
664 GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
665 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
Carter Cooper7ffaba62012-05-24 13:59:53 -0600666 KGSL_MEMSTORE_OFFSET(context_id, eoptimestamp)));
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700667 GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700668
Rajesh Kemisettic5699302012-04-21 21:09:05 +0530669 GSL_RB_WRITE(ringcmds, rcmd_gpu,
670 cp_type3_packet(CP_MEM_WRITE, 2));
671 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
Carter Cooper7ffaba62012-05-24 13:59:53 -0600672 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
673 eoptimestamp)));
Rajesh Kemisettic5699302012-04-21 21:09:05 +0530674 GSL_RB_WRITE(ringcmds, rcmd_gpu,
675 rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
676 } else {
677 GSL_RB_WRITE(ringcmds, rcmd_gpu,
678 cp_type3_packet(CP_EVENT_WRITE, 3));
679 GSL_RB_WRITE(ringcmds, rcmd_gpu, CACHE_FLUSH_TS);
680 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700681 KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
682 eoptimestamp)));
683 GSL_RB_WRITE(ringcmds, rcmd_gpu,
684 rb->timestamp[KGSL_MEMSTORE_GLOBAL]);
Rajesh Kemisettic5699302012-04-21 21:09:05 +0530685 }
Rajeev Kulkarnid98d6562013-01-02 16:10:56 -0800686 if (context) {
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700687 /* Conditional execution based on memory values */
688 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600689 cp_type3_packet(CP_COND_EXEC, 4));
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700690 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
691 KGSL_MEMSTORE_OFFSET(
692 context_id, ts_cmp_enable)) >> 2);
693 GSL_RB_WRITE(ringcmds, rcmd_gpu, (gpuaddr +
694 KGSL_MEMSTORE_OFFSET(
695 context_id, ref_wait_ts)) >> 2);
696 GSL_RB_WRITE(ringcmds, rcmd_gpu, timestamp);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700697 /* # of conditional command DWORDs */
Jordan Crouseef02fc02013-03-05 11:19:31 -0700698 GSL_RB_WRITE(ringcmds, rcmd_gpu, 8);
699
700 /* Clear the ts_cmp_enable for the context */
701 GSL_RB_WRITE(ringcmds, rcmd_gpu,
702 cp_type3_packet(CP_MEM_WRITE, 2));
703 GSL_RB_WRITE(ringcmds, rcmd_gpu, gpuaddr +
704 KGSL_MEMSTORE_OFFSET(
705 context_id, ts_cmp_enable));
706 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x0);
707
708 /* Clear the ts_cmp_enable for the global timestamp */
709 GSL_RB_WRITE(ringcmds, rcmd_gpu,
710 cp_type3_packet(CP_MEM_WRITE, 2));
711 GSL_RB_WRITE(ringcmds, rcmd_gpu, gpuaddr +
712 KGSL_MEMSTORE_OFFSET(
713 KGSL_MEMSTORE_GLOBAL, ts_cmp_enable));
714 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0x0);
715
716 /* Trigger the interrupt */
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700717 GSL_RB_WRITE(ringcmds, rcmd_gpu,
Jordan Crouse084427d2011-07-28 08:37:58 -0600718 cp_type3_packet(CP_INTERRUPT, 1));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700719 GSL_RB_WRITE(ringcmds, rcmd_gpu, CP_INT_CNTL__RB_INT_MASK);
720 }
721
Jordan Crouseb4d31bd2012-02-01 22:11:12 -0700722 if (adreno_is_a3xx(adreno_dev)) {
723 /* Dummy set-constant to trigger context rollover */
724 GSL_RB_WRITE(ringcmds, rcmd_gpu,
725 cp_type3_packet(CP_SET_CONSTANT, 2));
726 GSL_RB_WRITE(ringcmds, rcmd_gpu,
727 (0x4<<16)|(A3XX_HLSQ_CL_KERNEL_GROUP_X_REG - 0x2000));
728 GSL_RB_WRITE(ringcmds, rcmd_gpu, 0);
729 }
730
Tarun Karradeeecc02013-01-21 23:42:17 -0800731 if (flags & KGSL_CMD_FLAGS_EOF) {
732 GSL_RB_WRITE(ringcmds, rcmd_gpu, cp_nop_packet(1));
733 GSL_RB_WRITE(ringcmds, rcmd_gpu, KGSL_END_OF_FRAME_IDENTIFIER);
734 }
735
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700736 adreno_ringbuffer_submit(rb);
737
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700738 return timestamp;
739}
740
Shubhraprakash Dascb068072012-06-07 17:52:41 -0600741unsigned int
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700742adreno_ringbuffer_issuecmds(struct kgsl_device *device,
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -0600743 struct adreno_context *drawctxt,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700744 unsigned int flags,
745 unsigned int *cmds,
746 int sizedwords)
747{
748 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
749 struct adreno_ringbuffer *rb = &adreno_dev->ringbuffer;
750
751 if (device->state & KGSL_STATE_HUNG)
Shubhraprakash Dascb068072012-06-07 17:52:41 -0600752 return kgsl_readtimestamp(device, KGSL_MEMSTORE_GLOBAL,
753 KGSL_TIMESTAMP_RETIRED);
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -0700754
755 flags |= KGSL_CMD_FLAGS_INTERNAL_ISSUE;
756
757 return adreno_ringbuffer_addcmds(rb, drawctxt, flags, cmds,
758 sizedwords, 0);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700759}
760
Jeremy Gebbend0ab6ad2012-04-06 11:13:35 -0600761static bool _parse_ibs(struct kgsl_device_private *dev_priv, uint gpuaddr,
762 int sizedwords);
763
764static bool
765_handle_type3(struct kgsl_device_private *dev_priv, uint *hostaddr)
766{
767 unsigned int opcode = cp_type3_opcode(*hostaddr);
768 switch (opcode) {
769 case CP_INDIRECT_BUFFER_PFD:
770 case CP_INDIRECT_BUFFER_PFE:
771 case CP_COND_INDIRECT_BUFFER_PFE:
772 case CP_COND_INDIRECT_BUFFER_PFD:
773 return _parse_ibs(dev_priv, hostaddr[1], hostaddr[2]);
774 case CP_NOP:
775 case CP_WAIT_FOR_IDLE:
776 case CP_WAIT_REG_MEM:
777 case CP_WAIT_REG_EQ:
778 case CP_WAT_REG_GTE:
779 case CP_WAIT_UNTIL_READ:
780 case CP_WAIT_IB_PFD_COMPLETE:
781 case CP_REG_RMW:
782 case CP_REG_TO_MEM:
783 case CP_MEM_WRITE:
784 case CP_MEM_WRITE_CNTR:
785 case CP_COND_EXEC:
786 case CP_COND_WRITE:
787 case CP_EVENT_WRITE:
788 case CP_EVENT_WRITE_SHD:
789 case CP_EVENT_WRITE_CFL:
790 case CP_EVENT_WRITE_ZPD:
791 case CP_DRAW_INDX:
792 case CP_DRAW_INDX_2:
793 case CP_DRAW_INDX_BIN:
794 case CP_DRAW_INDX_2_BIN:
795 case CP_VIZ_QUERY:
796 case CP_SET_STATE:
797 case CP_SET_CONSTANT:
798 case CP_IM_LOAD:
799 case CP_IM_LOAD_IMMEDIATE:
800 case CP_LOAD_CONSTANT_CONTEXT:
801 case CP_INVALIDATE_STATE:
802 case CP_SET_SHADER_BASES:
803 case CP_SET_BIN_MASK:
804 case CP_SET_BIN_SELECT:
805 case CP_SET_BIN_BASE_OFFSET:
806 case CP_SET_BIN_DATA:
807 case CP_CONTEXT_UPDATE:
808 case CP_INTERRUPT:
809 case CP_IM_STORE:
810 case CP_LOAD_STATE:
811 break;
812 /* these shouldn't come from userspace */
813 case CP_ME_INIT:
814 case CP_SET_PROTECTED_MODE:
815 default:
816 KGSL_CMD_ERR(dev_priv->device, "bad CP opcode %0x\n", opcode);
817 return false;
818 break;
819 }
820
821 return true;
822}
823
824static bool
825_handle_type0(struct kgsl_device_private *dev_priv, uint *hostaddr)
826{
827 unsigned int reg = type0_pkt_offset(*hostaddr);
828 unsigned int cnt = type0_pkt_size(*hostaddr);
829 if (reg < 0x0192 || (reg + cnt) >= 0x8000) {
830 KGSL_CMD_ERR(dev_priv->device, "bad type0 reg: 0x%0x cnt: %d\n",
831 reg, cnt);
832 return false;
833 }
834 return true;
835}
836
837/*
838 * Traverse IBs and dump them to test vector. Detect swap by inspecting
839 * register writes, keeping note of the current state, and dump
840 * framebuffer config to test vector
841 */
842static bool _parse_ibs(struct kgsl_device_private *dev_priv,
843 uint gpuaddr, int sizedwords)
844{
845 static uint level; /* recursion level */
846 bool ret = false;
847 uint *hostaddr, *hoststart;
848 int dwords_left = sizedwords; /* dwords left in the current command
849 buffer */
850 struct kgsl_mem_entry *entry;
851
852 spin_lock(&dev_priv->process_priv->mem_lock);
853 entry = kgsl_sharedmem_find_region(dev_priv->process_priv,
854 gpuaddr, sizedwords * sizeof(uint));
855 spin_unlock(&dev_priv->process_priv->mem_lock);
856 if (entry == NULL) {
857 KGSL_CMD_ERR(dev_priv->device,
858 "no mapping for gpuaddr: 0x%08x\n", gpuaddr);
859 return false;
860 }
861
862 hostaddr = (uint *)kgsl_gpuaddr_to_vaddr(&entry->memdesc, gpuaddr);
863 if (hostaddr == NULL) {
864 KGSL_CMD_ERR(dev_priv->device,
865 "no mapping for gpuaddr: 0x%08x\n", gpuaddr);
866 return false;
867 }
868
869 hoststart = hostaddr;
870
871 level++;
872
873 KGSL_CMD_INFO(dev_priv->device, "ib: gpuaddr:0x%08x, wc:%d, hptr:%p\n",
874 gpuaddr, sizedwords, hostaddr);
875
876 mb();
877 while (dwords_left > 0) {
878 bool cur_ret = true;
879 int count = 0; /* dword count including packet header */
880
881 switch (*hostaddr >> 30) {
882 case 0x0: /* type-0 */
883 count = (*hostaddr >> 16)+2;
884 cur_ret = _handle_type0(dev_priv, hostaddr);
885 break;
886 case 0x1: /* type-1 */
887 count = 2;
888 break;
889 case 0x3: /* type-3 */
890 count = ((*hostaddr >> 16) & 0x3fff) + 2;
891 cur_ret = _handle_type3(dev_priv, hostaddr);
892 break;
893 default:
894 KGSL_CMD_ERR(dev_priv->device, "unexpected type: "
895 "type:%d, word:0x%08x @ 0x%p, gpu:0x%08x\n",
896 *hostaddr >> 30, *hostaddr, hostaddr,
897 gpuaddr+4*(sizedwords-dwords_left));
898 cur_ret = false;
899 count = dwords_left;
900 break;
901 }
902
903 if (!cur_ret) {
904 KGSL_CMD_ERR(dev_priv->device,
905 "bad sub-type: #:%d/%d, v:0x%08x"
906 " @ 0x%p[gb:0x%08x], level:%d\n",
907 sizedwords-dwords_left, sizedwords, *hostaddr,
908 hostaddr, gpuaddr+4*(sizedwords-dwords_left),
909 level);
910
911 if (ADRENO_DEVICE(dev_priv->device)->ib_check_level
912 >= 2)
913 print_hex_dump(KERN_ERR,
914 level == 1 ? "IB1:" : "IB2:",
915 DUMP_PREFIX_OFFSET, 32, 4, hoststart,
916 sizedwords*4, 0);
917 goto done;
918 }
919
920 /* jump to next packet */
921 dwords_left -= count;
922 hostaddr += count;
923 if (dwords_left < 0) {
924 KGSL_CMD_ERR(dev_priv->device,
925 "bad count: c:%d, #:%d/%d, "
926 "v:0x%08x @ 0x%p[gb:0x%08x], level:%d\n",
927 count, sizedwords-(dwords_left+count),
928 sizedwords, *(hostaddr-count), hostaddr-count,
929 gpuaddr+4*(sizedwords-(dwords_left+count)),
930 level);
931 if (ADRENO_DEVICE(dev_priv->device)->ib_check_level
932 >= 2)
933 print_hex_dump(KERN_ERR,
934 level == 1 ? "IB1:" : "IB2:",
935 DUMP_PREFIX_OFFSET, 32, 4, hoststart,
936 sizedwords*4, 0);
937 goto done;
938 }
939 }
940
941 ret = true;
942done:
943 if (!ret)
944 KGSL_DRV_ERR(dev_priv->device,
945 "parsing failed: gpuaddr:0x%08x, "
946 "host:0x%p, wc:%d\n", gpuaddr, hoststart, sizedwords);
947
948 level--;
949
950 return ret;
951}
952
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700953int
954adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv,
955 struct kgsl_context *context,
956 struct kgsl_ibdesc *ibdesc,
957 unsigned int numibs,
958 uint32_t *timestamp,
959 unsigned int flags)
960{
961 struct kgsl_device *device = dev_priv->device;
962 struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
963 unsigned int *link;
964 unsigned int *cmds;
965 unsigned int i;
Jeremy Gebben3c127f52011-08-08 17:04:11 -0600966 struct adreno_context *drawctxt;
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -0700967 unsigned int start_index = 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700968
969 if (device->state & KGSL_STATE_HUNG)
970 return -EBUSY;
971 if (!(adreno_dev->ringbuffer.flags & KGSL_FLAGS_STARTED) ||
Jeremy Gebben3c127f52011-08-08 17:04:11 -0600972 context == NULL || ibdesc == 0 || numibs == 0)
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700973 return -EINVAL;
974
Jeremy Gebben3c127f52011-08-08 17:04:11 -0600975 drawctxt = context->devctxt;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700976
977 if (drawctxt->flags & CTXT_FLAGS_GPU_HANG) {
Tarun Karra696f89e2013-01-27 21:31:40 -0800978 KGSL_CTXT_ERR(device, "proc %s failed fault tolerance"
Carter Cooper7e7f02e2012-02-15 09:36:31 -0700979 " will not accept commands for context %d\n",
Tarun Karra696f89e2013-01-27 21:31:40 -0800980 drawctxt->pid_name, drawctxt->id);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700981 return -EDEADLK;
982 }
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600983
Tarun Karradeeecc02013-01-21 23:42:17 -0800984 if (drawctxt->flags & CTXT_FLAGS_SKIP_EOF) {
985 KGSL_CTXT_ERR(device,
Tarun Karra696f89e2013-01-27 21:31:40 -0800986 "proc %s triggered fault tolerance"
Tarun Karradeeecc02013-01-21 23:42:17 -0800987 " skipping commands for context till EOF %d\n",
Tarun Karra696f89e2013-01-27 21:31:40 -0800988 drawctxt->pid_name, drawctxt->id);
Tarun Karradeeecc02013-01-21 23:42:17 -0800989 if (flags & KGSL_CMD_FLAGS_EOF)
990 drawctxt->flags &= ~CTXT_FLAGS_SKIP_EOF;
991 numibs = 0;
992 }
993
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600994 cmds = link = kzalloc(sizeof(unsigned int) * (numibs * 3 + 4),
995 GFP_KERNEL);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700996 if (!link) {
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -0600997 KGSL_CORE_ERR("kzalloc(%d) failed\n",
998 sizeof(unsigned int) * (numibs * 3 + 4));
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -0700999 return -ENOMEM;
1000 }
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -07001001
1002 /*When preamble is enabled, the preamble buffer with state restoration
1003 commands are stored in the first node of the IB chain. We can skip that
1004 if a context switch hasn't occured */
1005
1006 if (drawctxt->flags & CTXT_FLAGS_PREAMBLE &&
1007 adreno_dev->drawctxt_active == drawctxt)
1008 start_index = 1;
1009
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -06001010 if (!start_index) {
1011 *cmds++ = cp_nop_packet(1);
1012 *cmds++ = KGSL_START_OF_IB_IDENTIFIER;
1013 } else {
1014 *cmds++ = cp_nop_packet(4);
1015 *cmds++ = KGSL_START_OF_IB_IDENTIFIER;
1016 *cmds++ = CP_HDR_INDIRECT_BUFFER_PFD;
1017 *cmds++ = ibdesc[0].gpuaddr;
1018 *cmds++ = ibdesc[0].sizedwords;
1019 }
Vijay Krishnamoorthybef66932012-01-24 09:32:05 -07001020 for (i = start_index; i < numibs; i++) {
Jeremy Gebbend0ab6ad2012-04-06 11:13:35 -06001021 if (unlikely(adreno_dev->ib_check_level >= 1 &&
1022 !_parse_ibs(dev_priv, ibdesc[i].gpuaddr,
1023 ibdesc[i].sizedwords))) {
1024 kfree(link);
1025 return -EINVAL;
1026 }
Jordan Crouse084427d2011-07-28 08:37:58 -06001027 *cmds++ = CP_HDR_INDIRECT_BUFFER_PFD;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001028 *cmds++ = ibdesc[i].gpuaddr;
1029 *cmds++ = ibdesc[i].sizedwords;
1030 }
1031
Shubhraprakash Dasd23ff4b2012-04-05 16:55:54 -06001032 *cmds++ = cp_nop_packet(1);
1033 *cmds++ = KGSL_END_OF_IB_IDENTIFIER;
1034
Shubhraprakash Dasb2abc452012-06-08 16:33:03 -06001035 kgsl_setstate(&device->mmu, context->id,
Shubhraprakash Das767fdda2011-08-15 15:49:45 -06001036 kgsl_mmu_pt_get_flags(device->mmu.hwpagetable,
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001037 device->id));
1038
1039 adreno_drawctxt_switch(adreno_dev, drawctxt, flags);
1040
1041 *timestamp = adreno_ringbuffer_addcmds(&adreno_dev->ringbuffer,
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -07001042 drawctxt,
Tarun Karradeeecc02013-01-21 23:42:17 -08001043 (flags & KGSL_CMD_FLAGS_EOF),
Vijay Krishnamoorthye80c3462012-08-27 14:07:32 -07001044 &link[0], (cmds - link), *timestamp);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001045
1046 KGSL_CMD_INFO(device, "ctxt %d g %08x numibs %d ts %d\n",
1047 context->id, (unsigned int)ibdesc, numibs, *timestamp);
1048
1049 kfree(link);
1050
1051#ifdef CONFIG_MSM_KGSL_CFF_DUMP
1052 /*
1053 * insert wait for idle after every IB1
1054 * this is conservative but works reliably and is ok
1055 * even for performance simulations
1056 */
Jordan Crousea29a2e02012-08-14 09:09:23 -06001057 adreno_idle(device);
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001058#endif
Tarun Karradeeecc02013-01-21 23:42:17 -08001059
Tarun Karrad20d71a2013-01-25 15:38:57 -08001060 /*
1061 * If context hung and recovered then return error so that the
1062 * application may handle it
1063 */
1064 if (drawctxt->flags & CTXT_FLAGS_GPU_HANG_FT) {
1065 drawctxt->flags &= ~CTXT_FLAGS_GPU_HANG_FT;
1066 return -EPROTO;
1067 } else
Shubhraprakash Das32240ef2012-06-06 20:27:46 -06001068 return 0;
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001069}
1070
Shubhraprakash Das2a85f1f2012-06-04 17:01:39 -06001071static void _turn_preamble_on_for_ib_seq(struct adreno_ringbuffer *rb,
1072 unsigned int rb_rptr)
1073{
1074 unsigned int temp_rb_rptr = rb_rptr;
1075 unsigned int size = rb->buffer_desc.size;
1076 unsigned int val[2];
1077 int i = 0;
1078 bool check = false;
1079 bool cmd_start = false;
1080
1081 /* Go till the start of the ib sequence and turn on preamble */
1082 while (temp_rb_rptr / sizeof(unsigned int) != rb->wptr) {
1083 kgsl_sharedmem_readl(&rb->buffer_desc, &val[i], temp_rb_rptr);
1084 if (check && KGSL_START_OF_IB_IDENTIFIER == val[i]) {
1085 /* decrement i */
1086 i = (i + 1) % 2;
1087 if (val[i] == cp_nop_packet(4)) {
1088 temp_rb_rptr = adreno_ringbuffer_dec_wrapped(
1089 temp_rb_rptr, size);
1090 kgsl_sharedmem_writel(&rb->buffer_desc,
1091 temp_rb_rptr, cp_nop_packet(1));
1092 }
Tarun Karrad20d71a2013-01-25 15:38:57 -08001093 KGSL_FT_INFO(rb->device,
Shubhraprakash Das2a85f1f2012-06-04 17:01:39 -06001094 "Turned preamble on at offset 0x%x\n",
1095 temp_rb_rptr / 4);
1096 break;
1097 }
1098 /* If you reach beginning of next command sequence then exit
1099 * First command encountered is the current one so don't break
1100 * on that. */
1101 if (KGSL_CMD_IDENTIFIER == val[i]) {
1102 if (cmd_start)
1103 break;
1104 cmd_start = true;
1105 }
1106
1107 i = (i + 1) % 2;
1108 if (1 == i)
1109 check = true;
1110 temp_rb_rptr = adreno_ringbuffer_inc_wrapped(temp_rb_rptr,
1111 size);
1112 }
1113}
1114
Shubhraprakash Das460cc762013-01-16 16:57:46 -08001115void adreno_ringbuffer_extract(struct adreno_ringbuffer *rb,
Tarun Karrad20d71a2013-01-25 15:38:57 -08001116 struct adreno_ft_data *ft_data)
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001117{
Shubhraprakash Das460cc762013-01-16 16:57:46 -08001118 struct kgsl_device *device = rb->device;
Tarun Karrad20d71a2013-01-25 15:38:57 -08001119 unsigned int rb_rptr = ft_data->start_of_replay_cmds;
Tarun Karradeeecc02013-01-21 23:42:17 -08001120 unsigned int good_rb_idx = 0, bad_rb_idx = 0, temp_rb_idx = 0;
1121 unsigned int last_good_cmd_end_idx = 0, last_bad_cmd_end_idx = 0;
1122 unsigned int cmd_start_idx = 0;
1123 unsigned int val1 = 0;
1124 int copy_rb_contents = 0;
1125 unsigned int temp_rb_rptr;
1126 struct kgsl_context *k_ctxt;
1127 struct adreno_context *a_ctxt;
1128 unsigned int size = rb->buffer_desc.size;
Tarun Karrad20d71a2013-01-25 15:38:57 -08001129 unsigned int *temp_rb_buffer = ft_data->rb_buffer;
1130 int *rb_size = &ft_data->rb_size;
1131 unsigned int *bad_rb_buffer = ft_data->bad_rb_buffer;
1132 int *bad_rb_size = &ft_data->bad_rb_size;
1133 unsigned int *good_rb_buffer = ft_data->good_rb_buffer;
1134 int *good_rb_size = &ft_data->good_rb_size;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001135
Shubhraprakash Das460cc762013-01-16 16:57:46 -08001136 /*
1137 * If the start index from where commands need to be copied is invalid
1138 * then no need to save off any commands
1139 */
Tarun Karrad20d71a2013-01-25 15:38:57 -08001140 if (0xFFFFFFFF == ft_data->start_of_replay_cmds)
Shubhraprakash Das460cc762013-01-16 16:57:46 -08001141 return;
1142
Tarun Karrad20d71a2013-01-25 15:38:57 -08001143 k_ctxt = idr_find(&device->context_idr, ft_data->context_id);
Shubhraprakash Das460cc762013-01-16 16:57:46 -08001144 if (k_ctxt) {
1145 a_ctxt = k_ctxt->devctxt;
1146 if (a_ctxt->flags & CTXT_FLAGS_PREAMBLE)
1147 _turn_preamble_on_for_ib_seq(rb, rb_rptr);
1148 }
1149 k_ctxt = NULL;
1150
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001151 /* Walk the rb from the context switch. Omit any commands
1152 * for an invalid context. */
1153 while ((rb_rptr / sizeof(unsigned int)) != rb->wptr) {
1154 kgsl_sharedmem_readl(&rb->buffer_desc, &val1, rb_rptr);
1155
1156 if (KGSL_CMD_IDENTIFIER == val1) {
1157 /* Start is the NOP dword that comes before
1158 * KGSL_CMD_IDENTIFIER */
Tarun Karradeeecc02013-01-21 23:42:17 -08001159 cmd_start_idx = temp_rb_idx - 1;
1160 if ((copy_rb_contents) && (good_rb_idx))
1161 last_good_cmd_end_idx = good_rb_idx - 1;
1162 if ((!copy_rb_contents) && (bad_rb_idx))
1163 last_bad_cmd_end_idx = bad_rb_idx - 1;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001164 }
1165
1166 /* check for context switch indicator */
1167 if (val1 == KGSL_CONTEXT_TO_MEM_IDENTIFIER) {
1168 unsigned int temp_idx, val2;
1169 /* increment by 3 to get to the context_id */
1170 temp_rb_rptr = rb_rptr + (3 * sizeof(unsigned int)) %
1171 size;
1172 kgsl_sharedmem_readl(&rb->buffer_desc, &val2,
1173 temp_rb_rptr);
1174
1175 /* if context switches to a context that did not cause
1176 * hang then start saving the rb contents as those
1177 * commands can be executed */
1178 k_ctxt = idr_find(&rb->device->context_idr, val2);
1179 if (k_ctxt) {
1180 a_ctxt = k_ctxt->devctxt;
1181
1182 /* If we are changing to a good context and were not
1183 * copying commands then copy over commands to the good
1184 * context */
1185 if (!copy_rb_contents && ((k_ctxt &&
1186 !(a_ctxt->flags & CTXT_FLAGS_GPU_HANG)) ||
1187 !k_ctxt)) {
1188 for (temp_idx = cmd_start_idx;
Tarun Karradeeecc02013-01-21 23:42:17 -08001189 temp_idx < temp_rb_idx;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001190 temp_idx++)
Tarun Karradeeecc02013-01-21 23:42:17 -08001191 good_rb_buffer[good_rb_idx++] =
1192 temp_rb_buffer[temp_idx];
Tarun Karrad20d71a2013-01-25 15:38:57 -08001193 ft_data->last_valid_ctx_id = val2;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001194 copy_rb_contents = 1;
Tarun Karradeeecc02013-01-21 23:42:17 -08001195 /* remove the good commands from bad buffer */
1196 bad_rb_idx = last_bad_cmd_end_idx;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001197 } else if (copy_rb_contents && k_ctxt &&
1198 (a_ctxt->flags & CTXT_FLAGS_GPU_HANG)) {
Tarun Karradeeecc02013-01-21 23:42:17 -08001199
1200 /* If we are changing back to a bad context
1201 * from good ctxt and were not copying commands
1202 * to bad ctxt then copy over commands to
1203 * the bad context */
1204 for (temp_idx = cmd_start_idx;
1205 temp_idx < temp_rb_idx;
1206 temp_idx++)
1207 bad_rb_buffer[bad_rb_idx++] =
1208 temp_rb_buffer[temp_idx];
1209 /* If we are changing to bad context then
1210 * remove the dwords we copied for this
1211 * sequence from the good buffer */
1212 good_rb_idx = last_good_cmd_end_idx;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001213 copy_rb_contents = 0;
1214 }
1215 }
1216 }
1217
1218 if (copy_rb_contents)
Tarun Karradeeecc02013-01-21 23:42:17 -08001219 good_rb_buffer[good_rb_idx++] = val1;
1220 else
1221 bad_rb_buffer[bad_rb_idx++] = val1;
1222
1223 /* Copy both good and bad commands to temp buffer */
1224 temp_rb_buffer[temp_rb_idx++] = val1;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001225
1226 rb_rptr = adreno_ringbuffer_inc_wrapped(rb_rptr, size);
1227 }
Tarun Karradeeecc02013-01-21 23:42:17 -08001228 *good_rb_size = good_rb_idx;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001229 *bad_rb_size = bad_rb_idx;
Tarun Karradeeecc02013-01-21 23:42:17 -08001230 *rb_size = temp_rb_idx;
Shubhraprakash Das1d577fe2012-05-31 18:28:22 -06001231}
1232
Bryan Huntsman3f2bc4d2011-08-16 17:27:22 -07001233void
1234adreno_ringbuffer_restore(struct adreno_ringbuffer *rb, unsigned int *rb_buff,
1235 int num_rb_contents)
1236{
1237 int i;
1238 unsigned int *ringcmds;
1239 unsigned int rcmd_gpu;
1240
1241 if (!num_rb_contents)
1242 return;
1243
1244 if (num_rb_contents > (rb->buffer_desc.size - rb->wptr)) {
1245 adreno_regwrite(rb->device, REG_CP_RB_RPTR, 0);
1246 rb->rptr = 0;
1247 BUG_ON(num_rb_contents > rb->buffer_desc.size);
1248 }
1249 ringcmds = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
1250 rcmd_gpu = rb->buffer_desc.gpuaddr + sizeof(unsigned int) * rb->wptr;
1251 for (i = 0; i < num_rb_contents; i++)
1252 GSL_RB_WRITE(ringcmds, rcmd_gpu, rb_buff[i]);
1253 rb->wptr += num_rb_contents;
1254 adreno_ringbuffer_submit(rb);
1255}