1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/interrupt.h>
5#include <linux/iopoll.h>
6#include <linux/device.h>
7#include <linux/slab.h>
8
9#include <drm/lima_drm.h>
10
11#include "lima_device.h"
12#include "lima_gp.h"
13#include "lima_regs.h"
14#include "lima_gem.h"
15#include "lima_vm.h"
16
17#define gp_write(reg, data) writel(data, ip->iomem + reg)
18#define gp_read(reg) readl(ip->iomem + reg)
19
20static irqreturn_t lima_gp_irq_handler(int irq, void *data)
21{
22 struct lima_ip *ip = data;
23 struct lima_device *dev = ip->dev;
24 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
25 struct lima_sched_task *task = pipe->current_task;
26 u32 state = gp_read(LIMA_GP_INT_STAT);
27 u32 status = gp_read(LIMA_GP_STATUS);
28 bool done = false;
29
30 /* for shared irq case */
31 if (!state)
32 return IRQ_NONE;
33
34 if (state & LIMA_GP_IRQ_MASK_ERROR) {
35 if ((state & LIMA_GP_IRQ_MASK_ERROR) ==
36 LIMA_GP_IRQ_PLBU_OUT_OF_MEM) {
37 dev_dbg(dev->dev, "%s out of heap irq status=%x\n",
38 lima_ip_name(ip), status);
39 } else {
40 dev_err(dev->dev, "%s error irq state=%x status=%x\n",
41 lima_ip_name(ip), state, status);
42 if (task)
43 task->recoverable = false;
44 }
45
46 /* mask all interrupts before hard reset */
47 gp_write(LIMA_GP_INT_MASK, 0);
48
49 pipe->error = true;
50 done = true;
51 } else {
52 bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST |
53 LIMA_GP_IRQ_PLBU_END_CMD_LST);
54 bool active = status & (LIMA_GP_STATUS_VS_ACTIVE |
55 LIMA_GP_STATUS_PLBU_ACTIVE);
56 done = valid && !active;
57 pipe->error = false;
58 }
59
60 gp_write(LIMA_GP_INT_CLEAR, state);
61
62 if (done)
63 lima_sched_pipe_task_done(pipe);
64
65 return IRQ_HANDLED;
66}
67
68static void lima_gp_soft_reset_async(struct lima_ip *ip)
69{
70 if (ip->data.async_reset)
71 return;
72
73 gp_write(LIMA_GP_INT_MASK, 0);
74 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED);
75 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET);
76 ip->data.async_reset = true;
77}
78
79static int lima_gp_soft_reset_async_wait(struct lima_ip *ip)
80{
81 struct lima_device *dev = ip->dev;
82 int err;
83 u32 v;
84
85 if (!ip->data.async_reset)
86 return 0;
87
88 err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v,
89 v & LIMA_GP_IRQ_RESET_COMPLETED,
90 0, 100);
91 if (err) {
92 dev_err(dev->dev, "%s soft reset time out\n",
93 lima_ip_name(ip));
94 return err;
95 }
96
97 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);
98 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
99
100 ip->data.async_reset = false;
101 return 0;
102}
103
104static int lima_gp_task_validate(struct lima_sched_pipe *pipe,
105 struct lima_sched_task *task)
106{
107 struct drm_lima_gp_frame *frame = task->frame;
108 u32 *f = frame->frame;
109 (void)pipe;
110
111 if (f[LIMA_GP_VSCL_START_ADDR >> 2] >
112 f[LIMA_GP_VSCL_END_ADDR >> 2] ||
113 f[LIMA_GP_PLBUCL_START_ADDR >> 2] >
114 f[LIMA_GP_PLBUCL_END_ADDR >> 2] ||
115 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] >
116 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2])
117 return -EINVAL;
118
119 if (f[LIMA_GP_VSCL_START_ADDR >> 2] ==
120 f[LIMA_GP_VSCL_END_ADDR >> 2] &&
121 f[LIMA_GP_PLBUCL_START_ADDR >> 2] ==
122 f[LIMA_GP_PLBUCL_END_ADDR >> 2])
123 return -EINVAL;
124
125 return 0;
126}
127
128static void lima_gp_task_run(struct lima_sched_pipe *pipe,
129 struct lima_sched_task *task)
130{
131 struct lima_ip *ip = pipe->processor[0];
132 struct drm_lima_gp_frame *frame = task->frame;
133 u32 *f = frame->frame;
134 u32 cmd = 0;
135 int i;
136
137 /* update real heap buffer size for GP */
138 for (i = 0; i < task->num_bos; i++) {
139 struct lima_bo *bo = task->bos[i];
140
141 if (bo->heap_size &&
142 lima_vm_get_va(vm: task->vm, bo) ==
143 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]) {
144 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] =
145 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] +
146 bo->heap_size;
147 task->recoverable = true;
148 task->heap = bo;
149 break;
150 }
151 }
152
153 if (f[LIMA_GP_VSCL_START_ADDR >> 2] !=
154 f[LIMA_GP_VSCL_END_ADDR >> 2])
155 cmd |= LIMA_GP_CMD_START_VS;
156 if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] !=
157 f[LIMA_GP_PLBUCL_END_ADDR >> 2])
158 cmd |= LIMA_GP_CMD_START_PLBU;
159
160 /* before any hw ops, wait last success task async soft reset */
161 lima_gp_soft_reset_async_wait(ip);
162
163 for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++)
164 writel(val: f[i], addr: ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4);
165
166 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);
167 gp_write(LIMA_GP_CMD, cmd);
168}
169
170static int lima_gp_bus_stop_poll(struct lima_ip *ip)
171{
172 return !!(gp_read(LIMA_GP_STATUS) & LIMA_GP_STATUS_BUS_STOPPED);
173}
174
175static int lima_gp_hard_reset_poll(struct lima_ip *ip)
176{
177 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000);
178 return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000;
179}
180
181static int lima_gp_hard_reset(struct lima_ip *ip)
182{
183 struct lima_device *dev = ip->dev;
184 int ret;
185
186 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000);
187 gp_write(LIMA_GP_INT_MASK, 0);
188
189 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_STOP_BUS);
190 ret = lima_poll_timeout(ip, func: lima_gp_bus_stop_poll, sleep_us: 10, timeout_us: 100);
191 if (ret) {
192 dev_err(dev->dev, "%s bus stop timeout\n", lima_ip_name(ip));
193 return ret;
194 }
195 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET);
196 ret = lima_poll_timeout(ip, func: lima_gp_hard_reset_poll, sleep_us: 10, timeout_us: 100);
197 if (ret) {
198 dev_err(dev->dev, "%s hard reset timeout\n", lima_ip_name(ip));
199 return ret;
200 }
201
202 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0);
203 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);
204 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
205
206 /*
207 * if there was an async soft reset queued,
208 * don't wait for it in the next job
209 */
210 ip->data.async_reset = false;
211
212 return 0;
213}
214
215static void lima_gp_task_fini(struct lima_sched_pipe *pipe)
216{
217 lima_gp_soft_reset_async(ip: pipe->processor[0]);
218}
219
220static void lima_gp_task_error(struct lima_sched_pipe *pipe)
221{
222 struct lima_ip *ip = pipe->processor[0];
223
224 dev_err(ip->dev->dev, "%s task error int_state=%x status=%x\n",
225 lima_ip_name(ip), gp_read(LIMA_GP_INT_STAT),
226 gp_read(LIMA_GP_STATUS));
227
228 lima_gp_hard_reset(ip);
229}
230
231static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe)
232{
233 lima_sched_pipe_task_done(pipe);
234}
235
236static void lima_gp_task_mask_irq(struct lima_sched_pipe *pipe)
237{
238 struct lima_ip *ip = pipe->processor[0];
239
240 gp_write(LIMA_GP_INT_MASK, 0);
241}
242
243static int lima_gp_task_recover(struct lima_sched_pipe *pipe)
244{
245 struct lima_ip *ip = pipe->processor[0];
246 struct lima_sched_task *task = pipe->current_task;
247 struct drm_lima_gp_frame *frame = task->frame;
248 u32 *f = frame->frame;
249 size_t fail_size =
250 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] -
251 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2];
252
253 if (fail_size == task->heap->heap_size) {
254 int ret;
255
256 ret = lima_heap_alloc(bo: task->heap, vm: task->vm);
257 if (ret < 0)
258 return ret;
259 }
260
261 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
262 /* Resume from where we stopped, i.e. new start is old end */
263 gp_write(LIMA_GP_PLBU_ALLOC_START_ADDR,
264 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]);
265 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] =
266 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + task->heap->heap_size;
267 gp_write(LIMA_GP_PLBU_ALLOC_END_ADDR,
268 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]);
269 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);
270 return 0;
271}
272
273static void lima_gp_print_version(struct lima_ip *ip)
274{
275 u32 version, major, minor;
276 char *name;
277
278 version = gp_read(LIMA_GP_VERSION);
279 major = (version >> 8) & 0xFF;
280 minor = version & 0xFF;
281 switch (version >> 16) {
282 case 0xA07:
283 name = "mali200";
284 break;
285 case 0xC07:
286 name = "mali300";
287 break;
288 case 0xB07:
289 name = "mali400";
290 break;
291 case 0xD07:
292 name = "mali450";
293 break;
294 default:
295 name = "unknown";
296 break;
297 }
298 dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",
299 lima_ip_name(ip), name, major, minor);
300}
301
302static struct kmem_cache *lima_gp_task_slab;
303static int lima_gp_task_slab_refcnt;
304
305static int lima_gp_hw_init(struct lima_ip *ip)
306{
307 ip->data.async_reset = false;
308 lima_gp_soft_reset_async(ip);
309 return lima_gp_soft_reset_async_wait(ip);
310}
311
312int lima_gp_resume(struct lima_ip *ip)
313{
314 return lima_gp_hw_init(ip);
315}
316
317void lima_gp_suspend(struct lima_ip *ip)
318{
319
320}
321
322int lima_gp_init(struct lima_ip *ip)
323{
324 struct lima_device *dev = ip->dev;
325 int err;
326
327 lima_gp_print_version(ip);
328
329 err = lima_gp_hw_init(ip);
330 if (err)
331 return err;
332
333 err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_gp_irq_handler,
334 IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip);
335 if (err) {
336 dev_err(dev->dev, "%s fail to request irq\n",
337 lima_ip_name(ip));
338 return err;
339 }
340
341 dev->gp_version = gp_read(LIMA_GP_VERSION);
342
343 return 0;
344}
345
346void lima_gp_fini(struct lima_ip *ip)
347{
348 struct lima_device *dev = ip->dev;
349
350 devm_free_irq(dev: dev->dev, irq: ip->irq, dev_id: ip);
351}
352
353int lima_gp_pipe_init(struct lima_device *dev)
354{
355 int frame_size = sizeof(struct drm_lima_gp_frame);
356 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
357
358 if (!lima_gp_task_slab) {
359 lima_gp_task_slab = kmem_cache_create_usercopy(
360 name: "lima_gp_task", size: sizeof(struct lima_sched_task) + frame_size,
361 align: 0, SLAB_HWCACHE_ALIGN, useroffset: sizeof(struct lima_sched_task),
362 usersize: frame_size, NULL);
363 if (!lima_gp_task_slab)
364 return -ENOMEM;
365 }
366 lima_gp_task_slab_refcnt++;
367
368 pipe->frame_size = frame_size;
369 pipe->task_slab = lima_gp_task_slab;
370
371 pipe->task_validate = lima_gp_task_validate;
372 pipe->task_run = lima_gp_task_run;
373 pipe->task_fini = lima_gp_task_fini;
374 pipe->task_error = lima_gp_task_error;
375 pipe->task_mmu_error = lima_gp_task_mmu_error;
376 pipe->task_recover = lima_gp_task_recover;
377 pipe->task_mask_irq = lima_gp_task_mask_irq;
378
379 return 0;
380}
381
382void lima_gp_pipe_fini(struct lima_device *dev)
383{
384 if (!--lima_gp_task_slab_refcnt) {
385 kmem_cache_destroy(s: lima_gp_task_slab);
386 lima_gp_task_slab = NULL;
387 }
388}
389

source code of linux/drivers/gpu/drm/lima/lima_gp.c