1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 *
5 */
6
7#include <linux/iopoll.h>
8
9#include <drm/drm_print.h>
10#include <drm/drm_vblank.h>
11
12#include "intel_crtc.h"
13#include "intel_de.h"
14#include "intel_display_regs.h"
15#include "intel_display_rpm.h"
16#include "intel_display_types.h"
17#include "intel_dsb.h"
18#include "intel_dsb_buffer.h"
19#include "intel_dsb_regs.h"
20#include "intel_vblank.h"
21#include "intel_vrr.h"
22#include "skl_watermark.h"
23
24#define CACHELINE_BYTES 64
25
26struct intel_dsb {
27 enum intel_dsb_id id;
28
29 struct intel_dsb_buffer dsb_buf;
30 struct intel_crtc *crtc;
31
32 /*
33 * maximum number of dwords the buffer will hold.
34 */
35 unsigned int size;
36
37 /*
38 * free_pos will point the first free dword and
39 * help in calculating tail of command buffer.
40 */
41 unsigned int free_pos;
42
43 /*
44 * Previously emitted DSB instruction. Used to
45 * identify/adjust the instruction for indexed
46 * register writes.
47 */
48 u32 ins[2];
49
50 /*
51 * Start of the previously emitted DSB instruction.
52 * Used to adjust the instruction for indexed
53 * register writes.
54 */
55 unsigned int ins_start_offset;
56
57 u32 chicken;
58 int hw_dewake_scanline;
59};
60
61/**
62 * DOC: DSB
63 *
64 * A DSB (Display State Buffer) is a queue of MMIO instructions in the memory
65 * which can be offloaded to DSB HW in Display Controller. DSB HW is a DMA
66 * engine that can be programmed to download the DSB from memory.
67 * It allows driver to batch submit display HW programming. This helps to
68 * reduce loading time and CPU activity, thereby making the context switch
69 * faster. DSB Support added from Gen12 Intel graphics based platform.
70 *
71 * DSB's can access only the pipe, plane, and transcoder Data Island Packet
72 * registers.
73 *
74 * DSB HW can support only register writes (both indexed and direct MMIO
75 * writes). There are no registers reads possible with DSB HW engine.
76 */
77
78/* DSB opcodes. */
79#define DSB_OPCODE_SHIFT 24
80#define DSB_OPCODE_NOOP 0x0
81#define DSB_OPCODE_MMIO_WRITE 0x1
82#define DSB_BYTE_EN 0xf
83#define DSB_BYTE_EN_SHIFT 20
84#define DSB_REG_VALUE_MASK 0xfffff
85#define DSB_OPCODE_WAIT_USEC 0x2
86#define DSB_OPCODE_WAIT_SCANLINE 0x3
87#define DSB_OPCODE_WAIT_VBLANKS 0x4
88#define DSB_OPCODE_WAIT_DSL_IN 0x5
89#define DSB_OPCODE_WAIT_DSL_OUT 0x6
90#define DSB_SCANLINE_UPPER_SHIFT 20
91#define DSB_SCANLINE_LOWER_SHIFT 0
92#define DSB_OPCODE_INTERRUPT 0x7
93#define DSB_OPCODE_INDEXED_WRITE 0x9
94/* see DSB_REG_VALUE_MASK */
95#define DSB_OPCODE_POLL 0xA
96/* see DSB_REG_VALUE_MASK */
97#define DSB_OPCODE_GOSUB 0xC /* ptl+ */
98#define DSB_GOSUB_HEAD_SHIFT 26
99#define DSB_GOSUB_TAIL_SHIFT 0
100#define DSB_GOSUB_CONVERT_ADDR(x) ((x) >> 6)
101
102static bool pre_commit_is_vrr_active(struct intel_atomic_state *state,
103 struct intel_crtc *crtc)
104{
105 const struct intel_crtc_state *old_crtc_state =
106 intel_atomic_get_old_crtc_state(state, crtc);
107 const struct intel_crtc_state *new_crtc_state =
108 intel_atomic_get_new_crtc_state(state, crtc);
109
110 /* VRR will be enabled afterwards, if necessary */
111 if (intel_crtc_needs_modeset(crtc_state: new_crtc_state))
112 return false;
113
114 /* VRR will have been disabled during intel_pre_plane_update() */
115 return old_crtc_state->vrr.enable && !intel_crtc_vrr_disabling(state, crtc);
116}
117
118static int dsb_vtotal(struct intel_atomic_state *state,
119 struct intel_crtc *crtc)
120{
121 const struct intel_crtc_state *crtc_state =
122 intel_pre_commit_crtc_state(state, crtc);
123
124 if (pre_commit_is_vrr_active(state, crtc))
125 return intel_vrr_vmax_vtotal(crtc_state);
126 else
127 return intel_mode_vtotal(mode: &crtc_state->hw.adjusted_mode);
128}
129
130static int dsb_dewake_scanline_start(struct intel_atomic_state *state,
131 struct intel_crtc *crtc)
132{
133 struct intel_display *display = to_intel_display(state);
134 const struct intel_crtc_state *crtc_state =
135 intel_pre_commit_crtc_state(state, crtc);
136 unsigned int latency = skl_watermark_max_latency(display, initial_wm_level: 0);
137
138 return intel_mode_vdisplay(mode: &crtc_state->hw.adjusted_mode) -
139 intel_usecs_to_scanlines(adjusted_mode: &crtc_state->hw.adjusted_mode, usecs: latency);
140}
141
142static int dsb_dewake_scanline_end(struct intel_atomic_state *state,
143 struct intel_crtc *crtc)
144{
145 const struct intel_crtc_state *crtc_state =
146 intel_pre_commit_crtc_state(state, crtc);
147
148 return intel_mode_vdisplay(mode: &crtc_state->hw.adjusted_mode);
149}
150
151static int dsb_scanline_to_hw(struct intel_atomic_state *state,
152 struct intel_crtc *crtc, int scanline)
153{
154 const struct intel_crtc_state *crtc_state =
155 intel_pre_commit_crtc_state(state, crtc);
156 int vtotal = dsb_vtotal(state, crtc);
157
158 return (scanline + vtotal - intel_crtc_scanline_offset(crtc_state)) % vtotal;
159}
160
161/*
162 * Bspec suggests that we should always set DSB_SKIP_WAITS_EN. We have approach
163 * different from what is explained in Bspec on how flip is considered being
164 * complete. We are waiting for vblank in DSB and generate interrupt when it
165 * happens and this interrupt is considered as indication of completion -> we
166 * definitely do not want to skip vblank wait. We also have concern what comes
167 * to skipping vblank evasion. I.e. arming registers are latched before we have
168 * managed writing them. Due to these reasons we are not setting
169 * DSB_SKIP_WAITS_EN.
170 */
171static u32 dsb_chicken(struct intel_atomic_state *state,
172 struct intel_crtc *crtc)
173{
174 if (pre_commit_is_vrr_active(state, crtc))
175 return DSB_CTRL_WAIT_SAFE_WINDOW |
176 DSB_CTRL_NO_WAIT_VBLANK |
177 DSB_INST_WAIT_SAFE_WINDOW |
178 DSB_INST_NO_WAIT_VBLANK;
179 else
180 return 0;
181}
182
183static bool assert_dsb_has_room(struct intel_dsb *dsb)
184{
185 struct intel_crtc *crtc = dsb->crtc;
186 struct intel_display *display = to_intel_display(crtc->base.dev);
187
188 /* each instruction is 2 dwords */
189 return !drm_WARN(display->drm, dsb->free_pos > dsb->size - 2,
190 "[CRTC:%d:%s] DSB %d buffer overflow\n",
191 crtc->base.base.id, crtc->base.name, dsb->id);
192}
193
194static bool assert_dsb_tail_is_aligned(struct intel_dsb *dsb)
195{
196 struct intel_crtc *crtc = dsb->crtc;
197 struct intel_display *display = to_intel_display(crtc->base.dev);
198
199 return !drm_WARN_ON(display->drm,
200 !IS_ALIGNED(dsb->free_pos * 4, CACHELINE_BYTES));
201}
202
203static void intel_dsb_dump(struct intel_dsb *dsb)
204{
205 struct intel_crtc *crtc = dsb->crtc;
206 struct intel_display *display = to_intel_display(crtc->base.dev);
207 int i;
208
209 drm_dbg_kms(display->drm, "[CRTC:%d:%s] DSB %d commands {\n",
210 crtc->base.base.id, crtc->base.name, dsb->id);
211 for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
212 drm_dbg_kms(display->drm,
213 " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
214 intel_dsb_buffer_read(&dsb->dsb_buf, i),
215 intel_dsb_buffer_read(&dsb->dsb_buf, i + 1),
216 intel_dsb_buffer_read(&dsb->dsb_buf, i + 2),
217 intel_dsb_buffer_read(&dsb->dsb_buf, i + 3));
218 drm_dbg_kms(display->drm, "}\n");
219}
220
221static bool is_dsb_busy(struct intel_display *display, enum pipe pipe,
222 enum intel_dsb_id dsb_id)
223{
224 return intel_de_read_fw(display, DSB_CTRL(pipe, dsb_id)) & DSB_STATUS_BUSY;
225}
226
227unsigned int intel_dsb_size(struct intel_dsb *dsb)
228{
229 return dsb->free_pos * 4;
230}
231
232unsigned int intel_dsb_head(struct intel_dsb *dsb)
233{
234 return intel_dsb_buffer_ggtt_offset(dsb_buf: &dsb->dsb_buf);
235}
236
237static unsigned int intel_dsb_tail(struct intel_dsb *dsb)
238{
239 return intel_dsb_buffer_ggtt_offset(dsb_buf: &dsb->dsb_buf) + intel_dsb_size(dsb);
240}
241
242static void intel_dsb_ins_align(struct intel_dsb *dsb)
243{
244 /*
245 * Every instruction should be 8 byte aligned.
246 *
247 * The only way to get unaligned free_pos is via
248 * intel_dsb_reg_write_indexed() which already
249 * makes sure the next dword is zeroed, so no need
250 * to clear it here.
251 */
252 dsb->free_pos = ALIGN(dsb->free_pos, 2);
253}
254
255static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
256{
257 if (!assert_dsb_has_room(dsb))
258 return;
259
260 intel_dsb_ins_align(dsb);
261
262 dsb->ins_start_offset = dsb->free_pos;
263 dsb->ins[0] = ldw;
264 dsb->ins[1] = udw;
265
266 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos++, val: dsb->ins[0]);
267 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos++, val: dsb->ins[1]);
268}
269
270static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
271 u32 opcode, i915_reg_t reg)
272{
273 u32 prev_opcode, prev_reg;
274
275 /*
276 * Nothing emitted yet? Must check before looking
277 * at the actual data since i915_gem_object_create_internal()
278 * does *not* give you zeroed memory!
279 */
280 if (dsb->free_pos == 0)
281 return false;
282
283 prev_opcode = dsb->ins[1] & ~DSB_REG_VALUE_MASK;
284 prev_reg = dsb->ins[1] & DSB_REG_VALUE_MASK;
285
286 return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
287}
288
289static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
290{
291 return intel_dsb_prev_ins_is_write(dsb,
292 DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT,
293 reg);
294}
295
296/**
297 * intel_dsb_reg_write_indexed() - Emit indexed register write to the DSB context
298 * @dsb: DSB context
299 * @reg: register address.
300 * @val: value.
301 *
302 * This function is used for writing register-value pair in command
303 * buffer of DSB.
304 *
305 * Note that indexed writes are slower than normal MMIO writes
306 * for a small number (less than 5 or so) of writes to the same
307 * register.
308 */
309void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
310 i915_reg_t reg, u32 val)
311{
312 /*
313 * For example the buffer will look like below for 3 dwords for auto
314 * increment register:
315 * +--------------------------------------------------------+
316 * | size = 3 | offset &| value1 | value2 | value3 | zero |
317 * | | opcode | | | | |
318 * +--------------------------------------------------------+
319 * + + + + + + +
320 * 0 4 8 12 16 20 24
321 * Byte
322 *
323 * As every instruction is 8 byte aligned the index of dsb instruction
324 * will start always from even number while dealing with u32 array. If
325 * we are writing odd no of dwords, Zeros will be added in the end for
326 * padding.
327 */
328 if (!intel_dsb_prev_ins_is_indexed_write(dsb, reg))
329 intel_dsb_emit(dsb, ldw: 0, /* count */
330 udw: (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
331 i915_mmio_reg_offset(reg));
332
333 if (!assert_dsb_has_room(dsb))
334 return;
335
336 /* Update the count */
337 dsb->ins[0]++;
338 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->ins_start_offset + 0,
339 val: dsb->ins[0]);
340
341 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos++, val);
342 /* if number of data words is odd, then the last dword should be 0.*/
343 if (dsb->free_pos & 0x1)
344 intel_dsb_buffer_write(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos, val: 0);
345}
346
347void intel_dsb_reg_write(struct intel_dsb *dsb,
348 i915_reg_t reg, u32 val)
349{
350 intel_dsb_emit(dsb, ldw: val,
351 udw: (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
352 (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
353 i915_mmio_reg_offset(reg));
354}
355
356static u32 intel_dsb_mask_to_byte_en(u32 mask)
357{
358 return (!!(mask & 0xff000000) << 3 |
359 !!(mask & 0x00ff0000) << 2 |
360 !!(mask & 0x0000ff00) << 1 |
361 !!(mask & 0x000000ff) << 0);
362}
363
364/* Note: mask implemented via byte enables! */
365void intel_dsb_reg_write_masked(struct intel_dsb *dsb,
366 i915_reg_t reg, u32 mask, u32 val)
367{
368 intel_dsb_emit(dsb, ldw: val,
369 udw: (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
370 (intel_dsb_mask_to_byte_en(mask) << DSB_BYTE_EN_SHIFT) |
371 i915_mmio_reg_offset(reg));
372}
373
374void intel_dsb_noop(struct intel_dsb *dsb, int count)
375{
376 int i;
377
378 for (i = 0; i < count; i++)
379 intel_dsb_emit(dsb, ldw: 0,
380 DSB_OPCODE_NOOP << DSB_OPCODE_SHIFT);
381}
382
383void intel_dsb_nonpost_start(struct intel_dsb *dsb)
384{
385 struct intel_crtc *crtc = dsb->crtc;
386 enum pipe pipe = crtc->pipe;
387
388 intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
389 DSB_NON_POSTED, DSB_NON_POSTED);
390 intel_dsb_noop(dsb, count: 4);
391}
392
393void intel_dsb_nonpost_end(struct intel_dsb *dsb)
394{
395 struct intel_crtc *crtc = dsb->crtc;
396 enum pipe pipe = crtc->pipe;
397
398 intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
399 DSB_NON_POSTED, val: 0);
400 intel_dsb_noop(dsb, count: 4);
401}
402
403void intel_dsb_interrupt(struct intel_dsb *dsb)
404{
405 intel_dsb_emit(dsb, ldw: 0,
406 DSB_OPCODE_INTERRUPT << DSB_OPCODE_SHIFT);
407}
408
409void intel_dsb_wait_usec(struct intel_dsb *dsb, int count)
410{
411 /* +1 to make sure we never wait less time than asked for */
412 intel_dsb_emit(dsb, ldw: count + 1,
413 DSB_OPCODE_WAIT_USEC << DSB_OPCODE_SHIFT);
414}
415
416void intel_dsb_wait_vblanks(struct intel_dsb *dsb, int count)
417{
418 intel_dsb_emit(dsb, ldw: count,
419 DSB_OPCODE_WAIT_VBLANKS << DSB_OPCODE_SHIFT);
420}
421
422static void intel_dsb_emit_wait_dsl(struct intel_dsb *dsb,
423 u32 opcode, int lower, int upper)
424{
425 u64 window = ((u64)upper << DSB_SCANLINE_UPPER_SHIFT) |
426 ((u64)lower << DSB_SCANLINE_LOWER_SHIFT);
427
428 intel_dsb_emit(dsb, lower_32_bits(window),
429 udw: (opcode << DSB_OPCODE_SHIFT) |
430 upper_32_bits(window));
431}
432
433static void intel_dsb_wait_dsl(struct intel_atomic_state *state,
434 struct intel_dsb *dsb,
435 int lower_in, int upper_in,
436 int lower_out, int upper_out)
437{
438 struct intel_crtc *crtc = dsb->crtc;
439
440 lower_in = dsb_scanline_to_hw(state, crtc, scanline: lower_in);
441 upper_in = dsb_scanline_to_hw(state, crtc, scanline: upper_in);
442
443 lower_out = dsb_scanline_to_hw(state, crtc, scanline: lower_out);
444 upper_out = dsb_scanline_to_hw(state, crtc, scanline: upper_out);
445
446 if (upper_in >= lower_in)
447 intel_dsb_emit_wait_dsl(dsb, DSB_OPCODE_WAIT_DSL_IN,
448 lower: lower_in, upper: upper_in);
449 else if (upper_out >= lower_out)
450 intel_dsb_emit_wait_dsl(dsb, DSB_OPCODE_WAIT_DSL_OUT,
451 lower: lower_out, upper: upper_out);
452 else
453 drm_WARN_ON(crtc->base.dev, 1); /* assert_dsl_ok() should have caught it already */
454}
455
456static void assert_dsl_ok(struct intel_atomic_state *state,
457 struct intel_dsb *dsb,
458 int start, int end)
459{
460 struct intel_crtc *crtc = dsb->crtc;
461 int vtotal = dsb_vtotal(state, crtc);
462
463 /*
464 * Waiting for the entire frame doesn't make sense,
465 * (IN==don't wait, OUT=wait forever).
466 */
467 drm_WARN(crtc->base.dev, (end - start + vtotal) % vtotal == vtotal - 1,
468 "[CRTC:%d:%s] DSB %d bad scanline window wait: %d-%d (vt=%d)\n",
469 crtc->base.base.id, crtc->base.name, dsb->id,
470 start, end, vtotal);
471}
472
473void intel_dsb_wait_scanline_in(struct intel_atomic_state *state,
474 struct intel_dsb *dsb,
475 int start, int end)
476{
477 assert_dsl_ok(state, dsb, start, end);
478
479 intel_dsb_wait_dsl(state, dsb,
480 lower_in: start, upper_in: end,
481 lower_out: end + 1, upper_out: start - 1);
482}
483
484void intel_dsb_wait_scanline_out(struct intel_atomic_state *state,
485 struct intel_dsb *dsb,
486 int start, int end)
487{
488 assert_dsl_ok(state, dsb, start, end);
489
490 intel_dsb_wait_dsl(state, dsb,
491 lower_in: end + 1, upper_in: start - 1,
492 lower_out: start, upper_out: end);
493}
494
495void intel_dsb_poll(struct intel_dsb *dsb,
496 i915_reg_t reg, u32 mask, u32 val,
497 int wait_us, int count)
498{
499 struct intel_crtc *crtc = dsb->crtc;
500 enum pipe pipe = crtc->pipe;
501
502 intel_dsb_reg_write(dsb, DSB_POLLMASK(pipe, dsb->id), val: mask);
503 intel_dsb_reg_write(dsb, DSB_POLLFUNC(pipe, dsb->id),
504 DSB_POLL_ENABLE |
505 DSB_POLL_WAIT(wait_us) | DSB_POLL_COUNT(count));
506
507 intel_dsb_noop(dsb, count: 5);
508
509 intel_dsb_emit(dsb, ldw: val,
510 udw: (DSB_OPCODE_POLL << DSB_OPCODE_SHIFT) |
511 i915_mmio_reg_offset(reg));
512}
513
514static void intel_dsb_align_tail(struct intel_dsb *dsb)
515{
516 u32 aligned_tail, tail;
517
518 intel_dsb_ins_align(dsb);
519
520 tail = dsb->free_pos * 4;
521 aligned_tail = ALIGN(tail, CACHELINE_BYTES);
522
523 if (aligned_tail > tail)
524 intel_dsb_buffer_memset(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos, val: 0,
525 size: aligned_tail - tail);
526
527 dsb->free_pos = aligned_tail / 4;
528}
529
530static void intel_dsb_gosub_align(struct intel_dsb *dsb)
531{
532 u32 aligned_tail, tail;
533
534 intel_dsb_ins_align(dsb);
535
536 tail = dsb->free_pos * 4;
537 aligned_tail = ALIGN(tail, CACHELINE_BYTES);
538
539 /*
540 * Wa_16024917128
541 * "Ensure GOSUB is not placed in cacheline QW slot 6 or 7 (numbered 0-7)"
542 */
543 if (aligned_tail - tail <= 2 * 8)
544 intel_dsb_buffer_memset(dsb_buf: &dsb->dsb_buf, idx: dsb->free_pos, val: 0,
545 size: aligned_tail - tail);
546
547 dsb->free_pos = aligned_tail / 4;
548}
549
550void intel_dsb_gosub(struct intel_dsb *dsb,
551 struct intel_dsb *sub_dsb)
552{
553 struct intel_crtc *crtc = dsb->crtc;
554 struct intel_display *display = to_intel_display(crtc->base.dev);
555 unsigned int head, tail;
556 u64 head_tail;
557
558 if (drm_WARN_ON(display->drm, dsb->id != sub_dsb->id))
559 return;
560
561 if (!assert_dsb_tail_is_aligned(dsb: sub_dsb))
562 return;
563
564 intel_dsb_gosub_align(dsb);
565
566 head = intel_dsb_head(dsb: sub_dsb);
567 tail = intel_dsb_tail(dsb: sub_dsb);
568
569 /*
570 * The GOSUB instruction has the following memory layout.
571 *
572 * +------------------------------------------------------------+
573 * | Opcode | Rsvd | Head Ptr | Tail Ptr |
574 * | 0x0c | | | |
575 * +------------------------------------------------------------+
576 * |<- 8bits->|<- 4bits ->|<-- 26bits -->|<-- 26bits -->|
577 *
578 * We have only 26 bits each to represent the head and tail
579 * pointers even though the addresses itself are of 32 bit. However, this
580 * is not a problem because the addresses are 64 bit aligned and therefore
581 * the last 6 bits are always Zero's. Therefore, we right shift the address
582 * by 6 before embedding it into the GOSUB instruction.
583 */
584
585 head_tail = ((u64)(DSB_GOSUB_CONVERT_ADDR(head)) << DSB_GOSUB_HEAD_SHIFT) |
586 ((u64)(DSB_GOSUB_CONVERT_ADDR(tail)) << DSB_GOSUB_TAIL_SHIFT);
587
588 intel_dsb_emit(dsb, lower_32_bits(head_tail),
589 udw: (DSB_OPCODE_GOSUB << DSB_OPCODE_SHIFT) |
590 upper_32_bits(head_tail));
591
592 /*
593 * "NOTE: the instructions within the cacheline
594 * FOLLOWING the GOSUB instruction must be NOPs."
595 */
596 intel_dsb_align_tail(dsb);
597}
598
599void intel_dsb_gosub_finish(struct intel_dsb *dsb)
600{
601 intel_dsb_align_tail(dsb);
602
603 /*
604 * Wa_16024917128
605 * "Ensure that all subroutines called by GOSUB end with a cacheline of NOPs"
606 */
607 intel_dsb_noop(dsb, count: 8);
608
609 intel_dsb_buffer_flush_map(dsb_buf: &dsb->dsb_buf);
610}
611
612void intel_dsb_finish(struct intel_dsb *dsb)
613{
614 intel_dsb_align_tail(dsb);
615
616 intel_dsb_buffer_flush_map(dsb_buf: &dsb->dsb_buf);
617}
618
619static u32 dsb_error_int_status(struct intel_display *display)
620{
621 u32 errors;
622
623 errors = DSB_GTT_FAULT_INT_STATUS |
624 DSB_RSPTIMEOUT_INT_STATUS |
625 DSB_POLL_ERR_INT_STATUS;
626
627 /*
628 * All the non-existing status bits operate as
629 * normal r/w bits, so any attempt to clear them
630 * will just end up setting them. Never do that so
631 * we won't mistake them for actual error interrupts.
632 */
633 if (DISPLAY_VER(display) >= 14)
634 errors |= DSB_ATS_FAULT_INT_STATUS;
635
636 if (DISPLAY_VER(display) >= 30)
637 errors |= DSB_GOSUB_INT_STATUS;
638
639 return errors;
640}
641
642static u32 dsb_error_int_en(struct intel_display *display)
643{
644 u32 errors;
645
646 errors = DSB_GTT_FAULT_INT_EN |
647 DSB_RSPTIMEOUT_INT_EN |
648 DSB_POLL_ERR_INT_EN;
649
650 if (DISPLAY_VER(display) >= 14)
651 errors |= DSB_ATS_FAULT_INT_EN;
652
653 /*
654 * Wa_16024917128
655 * "Disable nested GOSUB interrupt (DSB_INTERRUPT bit 21)"
656 */
657 if (0 && DISPLAY_VER(display) >= 30)
658 errors |= DSB_GOSUB_INT_EN;
659
660 return errors;
661}
662
663/*
664 * FIXME calibrate these sensibly, ideally compute based on
665 * the number of regisetrs to be written. But that requires
666 * measuring the actual DSB execution speed on each platform
667 * (and the speed also depends on CDCLK and memory clock)...
668 */
669static int intel_dsb_noarm_exec_time_us(void)
670{
671 return 80;
672}
673
674static int intel_dsb_arm_exec_time_us(void)
675{
676 return 20;
677}
678
679int intel_dsb_exec_time_us(void)
680{
681 return intel_dsb_noarm_exec_time_us() +
682 intel_dsb_arm_exec_time_us();
683}
684
685void intel_dsb_vblank_evade(struct intel_atomic_state *state,
686 struct intel_dsb *dsb)
687{
688 struct intel_crtc *crtc = dsb->crtc;
689 const struct intel_crtc_state *crtc_state =
690 intel_pre_commit_crtc_state(state, crtc);
691 int latency = intel_usecs_to_scanlines(adjusted_mode: &crtc_state->hw.adjusted_mode,
692 usecs: intel_dsb_arm_exec_time_us());
693 int start, end;
694
695 /*
696 * PIPEDSL is reading as 0 when in SRDENT(PSR1) or DEEP_SLEEP(PSR2). On
697 * wake-up scanline counting starts from vblank_start - 1. We don't know
698 * if wake-up is already ongoing when evasion starts. In worst case
699 * PIPEDSL could start reading valid value right after checking the
700 * scanline. In this scenario we wouldn't have enough time to write all
701 * registers. To tackle this evade scanline 0 as well. As a drawback we
702 * have 1 frame delay in flip when waking up.
703 */
704 if (crtc_state->has_psr)
705 intel_dsb_emit_wait_dsl(dsb, DSB_OPCODE_WAIT_DSL_OUT, lower: 0, upper: 0);
706
707 if (pre_commit_is_vrr_active(state, crtc)) {
708 int vblank_delay = crtc_state->set_context_latency;
709
710 end = intel_vrr_vmin_vblank_start(crtc_state);
711 start = end - vblank_delay - latency;
712 intel_dsb_wait_scanline_out(state, dsb, start, end);
713
714 end = intel_vrr_vmax_vblank_start(crtc_state);
715 start = end - vblank_delay - latency;
716 intel_dsb_wait_scanline_out(state, dsb, start, end);
717 } else {
718 int vblank_delay = intel_mode_vblank_delay(mode: &crtc_state->hw.adjusted_mode);
719
720 end = intel_mode_vblank_start(mode: &crtc_state->hw.adjusted_mode);
721 start = end - vblank_delay - latency;
722 intel_dsb_wait_scanline_out(state, dsb, start, end);
723 }
724}
725
726static void _intel_dsb_chain(struct intel_atomic_state *state,
727 struct intel_dsb *dsb,
728 struct intel_dsb *chained_dsb,
729 u32 ctrl)
730{
731 struct intel_display *display = to_intel_display(state->base.dev);
732 struct intel_crtc *crtc = dsb->crtc;
733 enum pipe pipe = crtc->pipe;
734
735 if (drm_WARN_ON(display->drm, dsb->id == chained_dsb->id))
736 return;
737
738 if (!assert_dsb_tail_is_aligned(dsb: chained_dsb))
739 return;
740
741 intel_dsb_reg_write(dsb, DSB_CTRL(pipe, chained_dsb->id),
742 val: ctrl | DSB_ENABLE);
743
744 intel_dsb_reg_write(dsb, DSB_CHICKEN(pipe, chained_dsb->id),
745 val: dsb_chicken(state, crtc));
746
747 intel_dsb_reg_write(dsb, DSB_INTERRUPT(pipe, chained_dsb->id),
748 val: dsb_error_int_status(display) | DSB_PROG_INT_STATUS |
749 dsb_error_int_en(display) | DSB_PROG_INT_EN);
750
751 if (ctrl & DSB_WAIT_FOR_VBLANK) {
752 int dewake_scanline = dsb_dewake_scanline_start(state, crtc);
753 int hw_dewake_scanline = dsb_scanline_to_hw(state, crtc, scanline: dewake_scanline);
754
755 intel_dsb_reg_write(dsb, DSB_PMCTRL(pipe, chained_dsb->id),
756 DSB_ENABLE_DEWAKE |
757 DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline));
758 } else {
759 intel_dsb_reg_write(dsb, DSB_PMCTRL(pipe, chained_dsb->id), val: 0);
760 }
761
762 intel_dsb_reg_write(dsb, DSB_HEAD(pipe, chained_dsb->id),
763 val: intel_dsb_head(dsb: chained_dsb));
764
765 intel_dsb_reg_write(dsb, DSB_TAIL(pipe, chained_dsb->id),
766 val: intel_dsb_tail(dsb: chained_dsb));
767
768 if (ctrl & DSB_WAIT_FOR_VBLANK) {
769 /*
770 * Keep DEwake alive via the first DSB, in
771 * case we're already past dewake_scanline,
772 * and thus DSB_ENABLE_DEWAKE on the second
773 * DSB won't do its job.
774 */
775 intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(pipe, dsb->id),
776 DSB_FORCE_DEWAKE, DSB_FORCE_DEWAKE);
777
778 intel_dsb_wait_scanline_out(state, dsb,
779 start: dsb_dewake_scanline_start(state, crtc),
780 end: dsb_dewake_scanline_end(state, crtc));
781
782 /*
783 * DSB_FORCE_DEWAKE remains active even after DSB is
784 * disabled, so make sure to clear it.
785 */
786 intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(crtc->pipe, dsb->id),
787 DSB_FORCE_DEWAKE, val: 0);
788 }
789}
790
791void intel_dsb_chain(struct intel_atomic_state *state,
792 struct intel_dsb *dsb,
793 struct intel_dsb *chained_dsb,
794 bool wait_for_vblank)
795{
796 _intel_dsb_chain(state, dsb, chained_dsb,
797 ctrl: wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0);
798}
799
800void intel_dsb_wait_for_delayed_vblank(struct intel_atomic_state *state,
801 struct intel_dsb *dsb)
802{
803 struct intel_crtc *crtc = dsb->crtc;
804 const struct intel_crtc_state *crtc_state =
805 intel_pre_commit_crtc_state(state, crtc);
806 const struct drm_display_mode *adjusted_mode =
807 &crtc_state->hw.adjusted_mode;
808 int wait_scanlines;
809
810 if (pre_commit_is_vrr_active(state, crtc)) {
811 /*
812 * If the push happened before the vmin decision boundary
813 * we don't know how far we are from the undelayed vblank.
814 * Wait until we're past the vmin safe window, at which
815 * point we're SCL lines away from the delayed vblank.
816 *
817 * If the push happened after the vmin decision boundary
818 * the hardware itself guarantees that we're SCL lines
819 * away from the delayed vblank, and we won't be inside
820 * the vmin safe window so this extra wait does nothing.
821 */
822 intel_dsb_wait_scanline_out(state, dsb,
823 start: intel_vrr_safe_window_start(crtc_state),
824 end: intel_vrr_vmin_safe_window_end(crtc_state));
825 /*
826 * When the push is sent during vblank it will trigger
827 * on the next scanline, hence we have up to one extra
828 * scanline until the delayed vblank occurs after
829 * TRANS_PUSH has been written.
830 */
831 wait_scanlines = crtc_state->set_context_latency + 1;
832 } else {
833 wait_scanlines = intel_mode_vblank_delay(mode: adjusted_mode);
834 }
835
836 intel_dsb_wait_usec(dsb, count: intel_scanlines_to_usecs(adjusted_mode, scanlines: wait_scanlines));
837}
838
839/**
840 * intel_dsb_commit() - Trigger workload execution of DSB.
841 * @dsb: DSB context
842 *
843 * This function is used to do actual write to hardware using DSB.
844 */
845void intel_dsb_commit(struct intel_dsb *dsb)
846{
847 struct intel_crtc *crtc = dsb->crtc;
848 struct intel_display *display = to_intel_display(crtc->base.dev);
849 enum pipe pipe = crtc->pipe;
850
851 if (!assert_dsb_tail_is_aligned(dsb))
852 return;
853
854 if (is_dsb_busy(display, pipe, dsb_id: dsb->id)) {
855 drm_err(display->drm, "[CRTC:%d:%s] DSB %d is busy\n",
856 crtc->base.base.id, crtc->base.name, dsb->id);
857 return;
858 }
859
860 intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id),
861 DSB_ENABLE);
862
863 intel_de_write_fw(display, DSB_CHICKEN(pipe, dsb->id),
864 val: dsb->chicken);
865
866 intel_de_write_fw(display, DSB_INTERRUPT(pipe, dsb->id),
867 val: dsb_error_int_status(display) | DSB_PROG_INT_STATUS |
868 dsb_error_int_en(display) | DSB_PROG_INT_EN);
869
870 intel_de_write_fw(display, DSB_PMCTRL(pipe, dsb->id), val: 0);
871
872 intel_de_write_fw(display, DSB_HEAD(pipe, dsb->id),
873 val: intel_dsb_head(dsb));
874
875 intel_de_write_fw(display, DSB_TAIL(pipe, dsb->id),
876 val: intel_dsb_tail(dsb));
877}
878
879void intel_dsb_wait(struct intel_dsb *dsb)
880{
881 struct intel_crtc *crtc = dsb->crtc;
882 struct intel_display *display = to_intel_display(crtc->base.dev);
883 enum pipe pipe = crtc->pipe;
884 bool is_busy;
885 int ret;
886
887 ret = poll_timeout_us(is_busy = is_dsb_busy(display, pipe, dsb->id),
888 !is_busy,
889 100, 1000, false);
890 if (ret) {
891 u32 offset = intel_dsb_buffer_ggtt_offset(dsb_buf: &dsb->dsb_buf);
892
893 intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id),
894 DSB_ENABLE | DSB_HALT);
895
896 drm_err(display->drm,
897 "[CRTC:%d:%s] DSB %d timed out waiting for idle (current head=0x%x, head=0x%x, tail=0x%x)\n",
898 crtc->base.base.id, crtc->base.name, dsb->id,
899 intel_de_read_fw(display, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset,
900 intel_de_read_fw(display, DSB_HEAD(pipe, dsb->id)) - offset,
901 intel_de_read_fw(display, DSB_TAIL(pipe, dsb->id)) - offset);
902
903 intel_dsb_dump(dsb);
904 }
905
906 /* Attempt to reset it */
907 dsb->free_pos = 0;
908 dsb->ins_start_offset = 0;
909 dsb->ins[0] = 0;
910 dsb->ins[1] = 0;
911
912 intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id), val: 0);
913
914 intel_de_write_fw(display, DSB_INTERRUPT(pipe, dsb->id),
915 val: dsb_error_int_status(display) | DSB_PROG_INT_STATUS);
916}
917
918/**
919 * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer.
920 * @state: the atomic state
921 * @crtc: the CRTC
922 * @dsb_id: the DSB engine to use
923 * @max_cmds: number of commands we need to fit into command buffer
924 *
925 * This function prepare the command buffer which is used to store dsb
926 * instructions with data.
927 *
928 * Returns:
929 * DSB context, NULL on failure
930 */
931struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
932 struct intel_crtc *crtc,
933 enum intel_dsb_id dsb_id,
934 unsigned int max_cmds)
935{
936 struct intel_display *display = to_intel_display(state);
937 struct ref_tracker *wakeref;
938 struct intel_dsb *dsb;
939 unsigned int size;
940
941 if (!HAS_DSB(display))
942 return NULL;
943
944 if (!display->params.enable_dsb)
945 return NULL;
946
947 dsb = kzalloc(sizeof(*dsb), GFP_KERNEL);
948 if (!dsb)
949 goto out;
950
951 wakeref = intel_display_rpm_get(display);
952
953 /* ~1 qword per instruction, full cachelines */
954 size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
955
956 if (!intel_dsb_buffer_create(crtc, dsb_buf: &dsb->dsb_buf, size))
957 goto out_put_rpm;
958
959 intel_display_rpm_put(display, wakeref);
960
961 dsb->id = dsb_id;
962 dsb->crtc = crtc;
963 dsb->size = size / 4; /* in dwords */
964
965 dsb->chicken = dsb_chicken(state, crtc);
966 dsb->hw_dewake_scanline =
967 dsb_scanline_to_hw(state, crtc, scanline: dsb_dewake_scanline_start(state, crtc));
968
969 return dsb;
970
971out_put_rpm:
972 intel_display_rpm_put(display, wakeref);
973 kfree(objp: dsb);
974out:
975 drm_info_once(display->drm,
976 "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
977 crtc->base.base.id, crtc->base.name, dsb_id);
978
979 return NULL;
980}
981
982/**
983 * intel_dsb_cleanup() - To cleanup DSB context.
984 * @dsb: DSB context
985 *
986 * This function cleanup the DSB context by unpinning and releasing
987 * the VMA object associated with it.
988 */
989void intel_dsb_cleanup(struct intel_dsb *dsb)
990{
991 intel_dsb_buffer_cleanup(dsb_buf: &dsb->dsb_buf);
992 kfree(objp: dsb);
993}
994
995void intel_dsb_irq_handler(struct intel_display *display,
996 enum pipe pipe, enum intel_dsb_id dsb_id)
997{
998 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
999 u32 tmp, errors;
1000
1001 tmp = intel_de_read_fw(display, DSB_INTERRUPT(pipe, dsb_id));
1002 intel_de_write_fw(display, DSB_INTERRUPT(pipe, dsb_id), val: tmp);
1003
1004 if (tmp & DSB_PROG_INT_STATUS) {
1005 spin_lock(lock: &display->drm->event_lock);
1006
1007 if (crtc->dsb_event) {
1008 /*
1009 * Update vblank counter/timestamp in case it
1010 * hasn't been done yet for this frame.
1011 */
1012 drm_crtc_accurate_vblank_count(crtc: &crtc->base);
1013
1014 drm_crtc_send_vblank_event(crtc: &crtc->base, e: crtc->dsb_event);
1015 crtc->dsb_event = NULL;
1016 }
1017
1018 spin_unlock(lock: &display->drm->event_lock);
1019 }
1020
1021 errors = tmp & dsb_error_int_status(display);
1022 if (errors & DSB_ATS_FAULT_INT_STATUS)
1023 drm_err(display->drm, "[CRTC:%d:%s] DSB %d ATS fault\n",
1024 crtc->base.base.id, crtc->base.name, dsb_id);
1025 if (errors & DSB_GTT_FAULT_INT_STATUS)
1026 drm_err(display->drm, "[CRTC:%d:%s] DSB %d GTT fault\n",
1027 crtc->base.base.id, crtc->base.name, dsb_id);
1028 if (errors & DSB_RSPTIMEOUT_INT_STATUS)
1029 drm_err(display->drm, "[CRTC:%d:%s] DSB %d response timeout\n",
1030 crtc->base.base.id, crtc->base.name, dsb_id);
1031 if (errors & DSB_POLL_ERR_INT_STATUS)
1032 drm_err(display->drm, "[CRTC:%d:%s] DSB %d poll error\n",
1033 crtc->base.base.id, crtc->base.name, dsb_id);
1034 if (errors & DSB_GOSUB_INT_STATUS)
1035 drm_err(display->drm, "[CRTC:%d:%s] DSB %d GOSUB programming error\n",
1036 crtc->base.base.id, crtc->base.name, dsb_id);
1037}
1038

source code of linux/drivers/gpu/drm/i915/display/intel_dsb.c