1/*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24/**
25 * DOC: Frame Buffer Compression (FBC)
26 *
27 * FBC tries to save memory bandwidth (and so power consumption) by
28 * compressing the amount of memory used by the display. It is total
29 * transparent to user space and completely handled in the kernel.
30 *
31 * The benefits of FBC are mostly visible with solid backgrounds and
32 * variation-less patterns. It comes from keeping the memory footprint small
33 * and having fewer memory pages opened and accessed for refreshing the display.
34 *
35 * i915 is responsible to reserve stolen memory for FBC and configure its
36 * offset on proper registers. The hardware takes care of all
37 * compress/decompress. However there are many known cases where we have to
38 * forcibly disable it to allow proper screen updates.
39 */
40
41#include <linux/debugfs.h>
42#include <linux/string_helpers.h>
43
44#include <drm/drm_blend.h>
45#include <drm/drm_fourcc.h>
46#include <drm/drm_print.h>
47
48#include "gem/i915_gem_stolen.h"
49
50#include "gt/intel_gt_types.h"
51
52#include "i915_drv.h"
53#include "i915_vgpu.h"
54#include "i915_vma.h"
55#include "i9xx_plane_regs.h"
56#include "intel_de.h"
57#include "intel_display_device.h"
58#include "intel_display_regs.h"
59#include "intel_display_rpm.h"
60#include "intel_display_trace.h"
61#include "intel_display_types.h"
62#include "intel_display_utils.h"
63#include "intel_display_wa.h"
64#include "intel_fbc.h"
65#include "intel_fbc_regs.h"
66#include "intel_frontbuffer.h"
67
68#define for_each_fbc_id(__display, __fbc_id) \
69 for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
70 for_each_if(DISPLAY_RUNTIME_INFO(__display)->fbc_mask & BIT(__fbc_id))
71
72#define for_each_intel_fbc(__display, __fbc, __fbc_id) \
73 for_each_fbc_id((__display), (__fbc_id)) \
74 for_each_if((__fbc) = (__display)->fbc[(__fbc_id)])
75
76struct intel_fbc_funcs {
77 void (*activate)(struct intel_fbc *fbc);
78 void (*deactivate)(struct intel_fbc *fbc);
79 bool (*is_active)(struct intel_fbc *fbc);
80 bool (*is_compressing)(struct intel_fbc *fbc);
81 void (*nuke)(struct intel_fbc *fbc);
82 void (*program_cfb)(struct intel_fbc *fbc);
83 void (*set_false_color)(struct intel_fbc *fbc, bool enable);
84};
85
86struct intel_fbc_state {
87 struct intel_plane *plane;
88 unsigned int cfb_stride;
89 unsigned int cfb_size;
90 unsigned int fence_y_offset;
91 u16 override_cfb_stride;
92 u16 interval;
93 s8 fence_id;
94 struct drm_rect dirty_rect;
95};
96
97struct intel_fbc {
98 struct intel_display *display;
99 const struct intel_fbc_funcs *funcs;
100
101 /* This is always the outer lock when overlapping with stolen_lock */
102 struct mutex lock;
103 unsigned int busy_bits;
104
105 struct intel_stolen_node *compressed_fb;
106 struct intel_stolen_node *compressed_llb;
107
108 enum intel_fbc_id id;
109
110 u8 limit;
111
112 bool false_color;
113
114 bool active;
115 bool activated;
116 bool flip_pending;
117
118 bool underrun_detected;
119 struct work_struct underrun_work;
120
121 /*
122 * This structure contains everything that's relevant to program the
123 * hardware registers. When we want to figure out if we need to disable
124 * and re-enable FBC for a new configuration we just check if there's
125 * something different in the struct. The genx_fbc_activate functions
126 * are supposed to read from it in order to program the registers.
127 */
128 struct intel_fbc_state state;
129 const char *no_fbc_reason;
130};
131
132/* plane stride in pixels */
133static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
134{
135 const struct drm_framebuffer *fb = plane_state->hw.fb;
136 unsigned int stride;
137
138 stride = plane_state->view.color_plane[0].mapping_stride;
139 if (!drm_rotation_90_or_270(rotation: plane_state->hw.rotation))
140 stride /= fb->format->cpp[0];
141
142 return stride;
143}
144
145static unsigned int intel_fbc_cfb_cpp(const struct intel_plane_state *plane_state)
146{
147 const struct drm_framebuffer *fb = plane_state->hw.fb;
148 unsigned int cpp = fb->format->cpp[0];
149
150 return max(cpp, 4);
151}
152
153/* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
154static unsigned int intel_fbc_plane_cfb_stride(const struct intel_plane_state *plane_state)
155{
156 unsigned int cpp = intel_fbc_cfb_cpp(plane_state);
157
158 return intel_fbc_plane_stride(plane_state) * cpp;
159}
160
161/* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
162static unsigned int skl_fbc_min_cfb_stride(struct intel_display *display,
163 unsigned int cpp, unsigned int width)
164{
165 unsigned int limit = 4; /* 1:4 compression limit is the worst case */
166 unsigned int height = 4; /* FBC segment is 4 lines */
167 unsigned int stride;
168
169 /* minimum segment stride we can use */
170 stride = width * cpp * height / limit;
171
172 /*
173 * Wa_16011863758: icl+
174 * Avoid some hardware segment address miscalculation.
175 */
176 if (DISPLAY_VER(display) >= 11)
177 stride += 64;
178
179 /*
180 * At least some of the platforms require each 4 line segment to
181 * be 512 byte aligned. Just do it always for simplicity.
182 */
183 stride = ALIGN(stride, 512);
184
185 /* convert back to single line equivalent with 1:1 compression limit */
186 return stride * limit / height;
187}
188
189/* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
190static unsigned int _intel_fbc_cfb_stride(struct intel_display *display,
191 unsigned int cpp, unsigned int width,
192 unsigned int stride)
193{
194 /*
195 * At least some of the platforms require each 4 line segment to
196 * be 512 byte aligned. Aligning each line to 512 bytes guarantees
197 * that regardless of the compression limit we choose later.
198 */
199 if (DISPLAY_VER(display) >= 9)
200 return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(display, cpp, width));
201 else
202 return stride;
203}
204
205static unsigned int intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
206{
207 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
208 unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
209 unsigned int width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
210 unsigned int cpp = intel_fbc_cfb_cpp(plane_state);
211
212 return _intel_fbc_cfb_stride(display, cpp, width, stride);
213}
214
215/*
216 * Maximum height the hardware will compress, on HSW+
217 * additional lines (up to the actual plane height) will
218 * remain uncompressed.
219 */
220static unsigned int intel_fbc_max_cfb_height(struct intel_display *display)
221{
222 if (DISPLAY_VER(display) >= 8)
223 return 2560;
224 else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
225 return 2048;
226 else
227 return 1536;
228}
229
230static unsigned int _intel_fbc_cfb_size(struct intel_display *display,
231 unsigned int height, unsigned int stride)
232{
233 return min(height, intel_fbc_max_cfb_height(display)) * stride;
234}
235
236static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_state)
237{
238 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
239 unsigned int height = drm_rect_height(r: &plane_state->uapi.src) >> 16;
240
241 return _intel_fbc_cfb_size(display, height, stride: intel_fbc_cfb_stride(plane_state));
242}
243
244static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_state)
245{
246 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
247 unsigned int stride_aligned = intel_fbc_cfb_stride(plane_state);
248 unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
249 const struct drm_framebuffer *fb = plane_state->hw.fb;
250
251 /*
252 * Override stride in 64 byte units per 4 line segment.
253 *
254 * Gen9 hw miscalculates cfb stride for linear as
255 * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
256 * we always need to use the override there.
257 *
258 * wa_14022269668 For bmg, always program the FBC_STRIDE before fbc enable
259 */
260 if (stride != stride_aligned ||
261 (DISPLAY_VER(display) == 9 && fb->modifier == DRM_FORMAT_MOD_LINEAR) ||
262 display->platform.battlemage)
263 return stride_aligned * 4 / 64;
264
265 return 0;
266}
267
268static bool intel_fbc_has_fences(struct intel_display *display)
269{
270 struct drm_i915_private __maybe_unused *i915 = to_i915(dev: display->drm);
271
272 return intel_gt_support_legacy_fencing(to_gt(i915));
273}
274
275static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)
276{
277 struct intel_display *display = fbc->display;
278 const struct intel_fbc_state *fbc_state = &fbc->state;
279 unsigned int cfb_stride;
280 u32 fbc_ctl;
281
282 cfb_stride = fbc_state->cfb_stride / fbc->limit;
283
284 /* FBC_CTL wants 32B or 64B units */
285 if (DISPLAY_VER(display) == 2)
286 cfb_stride = (cfb_stride / 32) - 1;
287 else
288 cfb_stride = (cfb_stride / 64) - 1;
289
290 fbc_ctl = FBC_CTL_PERIODIC |
291 FBC_CTL_INTERVAL(fbc_state->interval) |
292 FBC_CTL_STRIDE(cfb_stride);
293
294 if (display->platform.i945gm)
295 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
296
297 if (fbc_state->fence_id >= 0)
298 fbc_ctl |= FBC_CTL_FENCENO(fbc_state->fence_id);
299
300 return fbc_ctl;
301}
302
303static u32 i965_fbc_ctl2(struct intel_fbc *fbc)
304{
305 const struct intel_fbc_state *fbc_state = &fbc->state;
306 u32 fbc_ctl2;
307
308 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM |
309 FBC_CTL_PLANE(fbc_state->plane->i9xx_plane);
310
311 if (fbc_state->fence_id >= 0)
312 fbc_ctl2 |= FBC_CTL_CPU_FENCE_EN;
313
314 return fbc_ctl2;
315}
316
317static void i8xx_fbc_deactivate(struct intel_fbc *fbc)
318{
319 struct intel_display *display = fbc->display;
320 u32 fbc_ctl;
321
322 /* Disable compression */
323 fbc_ctl = intel_de_read(display, FBC_CONTROL);
324 if ((fbc_ctl & FBC_CTL_EN) == 0)
325 return;
326
327 fbc_ctl &= ~FBC_CTL_EN;
328 intel_de_write(display, FBC_CONTROL, val: fbc_ctl);
329
330 /* Wait for compressing bit to clear */
331 if (intel_de_wait_for_clear_ms(display, FBC_STATUS,
332 FBC_STAT_COMPRESSING, timeout_ms: 10)) {
333 drm_dbg_kms(display->drm, "FBC idle timed out\n");
334 return;
335 }
336}
337
338static void i8xx_fbc_activate(struct intel_fbc *fbc)
339{
340 struct intel_display *display = fbc->display;
341 const struct intel_fbc_state *fbc_state = &fbc->state;
342 int i;
343
344 /* Clear old tags */
345 for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
346 intel_de_write(display, FBC_TAG(i), val: 0);
347
348 if (DISPLAY_VER(display) == 4) {
349 intel_de_write(display, FBC_CONTROL2,
350 val: i965_fbc_ctl2(fbc));
351 intel_de_write(display, FBC_FENCE_OFF,
352 val: fbc_state->fence_y_offset);
353 }
354
355 intel_de_write(display, FBC_CONTROL,
356 FBC_CTL_EN | i8xx_fbc_ctl(fbc));
357}
358
359static bool i8xx_fbc_is_active(struct intel_fbc *fbc)
360{
361 return intel_de_read(display: fbc->display, FBC_CONTROL) & FBC_CTL_EN;
362}
363
364static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)
365{
366 return intel_de_read(display: fbc->display, FBC_STATUS) &
367 (FBC_STAT_COMPRESSING | FBC_STAT_COMPRESSED);
368}
369
370static void i8xx_fbc_nuke(struct intel_fbc *fbc)
371{
372 struct intel_display *display = fbc->display;
373 struct intel_fbc_state *fbc_state = &fbc->state;
374 enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
375
376 intel_de_write_fw(display, DSPADDR(display, i9xx_plane),
377 val: intel_de_read_fw(display, DSPADDR(display, i9xx_plane)));
378}
379
380static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)
381{
382 struct intel_display *display = fbc->display;
383
384 drm_WARN_ON(display->drm,
385 range_end_overflows_t(u64, i915_gem_stolen_area_address(display->drm),
386 i915_gem_stolen_node_offset(fbc->compressed_fb),
387 U32_MAX));
388 drm_WARN_ON(display->drm,
389 range_end_overflows_t(u64, i915_gem_stolen_area_address(display->drm),
390 i915_gem_stolen_node_offset(fbc->compressed_llb),
391 U32_MAX));
392 intel_de_write(display, FBC_CFB_BASE,
393 val: i915_gem_stolen_node_address(node: fbc->compressed_fb));
394 intel_de_write(display, FBC_LL_BASE,
395 val: i915_gem_stolen_node_address(node: fbc->compressed_llb));
396}
397
398static const struct intel_fbc_funcs i8xx_fbc_funcs = {
399 .activate = i8xx_fbc_activate,
400 .deactivate = i8xx_fbc_deactivate,
401 .is_active = i8xx_fbc_is_active,
402 .is_compressing = i8xx_fbc_is_compressing,
403 .nuke = i8xx_fbc_nuke,
404 .program_cfb = i8xx_fbc_program_cfb,
405};
406
407static void i965_fbc_nuke(struct intel_fbc *fbc)
408{
409 struct intel_display *display = fbc->display;
410 struct intel_fbc_state *fbc_state = &fbc->state;
411 enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
412
413 intel_de_write_fw(display, DSPSURF(display, i9xx_plane),
414 val: intel_de_read_fw(display, DSPSURF(display, i9xx_plane)));
415}
416
417static const struct intel_fbc_funcs i965_fbc_funcs = {
418 .activate = i8xx_fbc_activate,
419 .deactivate = i8xx_fbc_deactivate,
420 .is_active = i8xx_fbc_is_active,
421 .is_compressing = i8xx_fbc_is_compressing,
422 .nuke = i965_fbc_nuke,
423 .program_cfb = i8xx_fbc_program_cfb,
424};
425
426static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)
427{
428 switch (fbc->limit) {
429 default:
430 MISSING_CASE(fbc->limit);
431 fallthrough;
432 case 1:
433 return DPFC_CTL_LIMIT_1X;
434 case 2:
435 return DPFC_CTL_LIMIT_2X;
436 case 4:
437 return DPFC_CTL_LIMIT_4X;
438 }
439}
440
441static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)
442{
443 struct intel_display *display = fbc->display;
444 const struct intel_fbc_state *fbc_state = &fbc->state;
445 u32 dpfc_ctl;
446
447 dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |
448 DPFC_CTL_PLANE_G4X(fbc_state->plane->i9xx_plane);
449
450 if (display->platform.g4x)
451 dpfc_ctl |= DPFC_CTL_SR_EN;
452
453 if (fbc_state->fence_id >= 0) {
454 dpfc_ctl |= DPFC_CTL_FENCE_EN_G4X;
455
456 if (DISPLAY_VER(display) < 6)
457 dpfc_ctl |= DPFC_CTL_FENCENO(fbc_state->fence_id);
458 }
459
460 return dpfc_ctl;
461}
462
463static void g4x_fbc_activate(struct intel_fbc *fbc)
464{
465 struct intel_display *display = fbc->display;
466 const struct intel_fbc_state *fbc_state = &fbc->state;
467
468 intel_de_write(display, DPFC_FENCE_YOFF,
469 val: fbc_state->fence_y_offset);
470
471 intel_de_write(display, DPFC_CONTROL,
472 DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
473}
474
475static void g4x_fbc_deactivate(struct intel_fbc *fbc)
476{
477 struct intel_display *display = fbc->display;
478 u32 dpfc_ctl;
479
480 /* Disable compression */
481 dpfc_ctl = intel_de_read(display, DPFC_CONTROL);
482 if (dpfc_ctl & DPFC_CTL_EN) {
483 dpfc_ctl &= ~DPFC_CTL_EN;
484 intel_de_write(display, DPFC_CONTROL, val: dpfc_ctl);
485 }
486}
487
488static bool g4x_fbc_is_active(struct intel_fbc *fbc)
489{
490 return intel_de_read(display: fbc->display, DPFC_CONTROL) & DPFC_CTL_EN;
491}
492
493static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)
494{
495 return intel_de_read(display: fbc->display, DPFC_STATUS) & DPFC_COMP_SEG_MASK;
496}
497
498static void g4x_fbc_program_cfb(struct intel_fbc *fbc)
499{
500 struct intel_display *display = fbc->display;
501
502 intel_de_write(display, DPFC_CB_BASE,
503 val: i915_gem_stolen_node_offset(node: fbc->compressed_fb));
504}
505
506static const struct intel_fbc_funcs g4x_fbc_funcs = {
507 .activate = g4x_fbc_activate,
508 .deactivate = g4x_fbc_deactivate,
509 .is_active = g4x_fbc_is_active,
510 .is_compressing = g4x_fbc_is_compressing,
511 .nuke = i965_fbc_nuke,
512 .program_cfb = g4x_fbc_program_cfb,
513};
514
515static void ilk_fbc_activate(struct intel_fbc *fbc)
516{
517 struct intel_display *display = fbc->display;
518 struct intel_fbc_state *fbc_state = &fbc->state;
519
520 intel_de_write(display, ILK_DPFC_FENCE_YOFF(fbc->id),
521 val: fbc_state->fence_y_offset);
522
523 intel_de_write(display, ILK_DPFC_CONTROL(fbc->id),
524 DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
525}
526
527static void fbc_compressor_clkgate_disable_wa(struct intel_fbc *fbc,
528 bool disable)
529{
530 struct intel_display *display = fbc->display;
531
532 if (display->platform.dg2)
533 intel_de_rmw(display, GEN9_CLKGATE_DIS_4, DG2_DPFC_GATING_DIS,
534 set: disable ? DG2_DPFC_GATING_DIS : 0);
535 else if (DISPLAY_VER(display) >= 14)
536 intel_de_rmw(display, MTL_PIPE_CLKGATE_DIS2(fbc->id),
537 MTL_DPFC_GATING_DIS,
538 set: disable ? MTL_DPFC_GATING_DIS : 0);
539}
540
541static void ilk_fbc_deactivate(struct intel_fbc *fbc)
542{
543 struct intel_display *display = fbc->display;
544 u32 dpfc_ctl;
545
546 if (HAS_FBC_DIRTY_RECT(display))
547 intel_de_write(display, XE3_FBC_DIRTY_CTL(fbc->id), val: 0);
548
549 /* Disable compression */
550 dpfc_ctl = intel_de_read(display, ILK_DPFC_CONTROL(fbc->id));
551 if (dpfc_ctl & DPFC_CTL_EN) {
552 dpfc_ctl &= ~DPFC_CTL_EN;
553 intel_de_write(display, ILK_DPFC_CONTROL(fbc->id), val: dpfc_ctl);
554 }
555}
556
557static bool ilk_fbc_is_active(struct intel_fbc *fbc)
558{
559 return intel_de_read(display: fbc->display, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;
560}
561
562static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)
563{
564 return intel_de_read(display: fbc->display, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;
565}
566
567static void ilk_fbc_program_cfb(struct intel_fbc *fbc)
568{
569 struct intel_display *display = fbc->display;
570
571 intel_de_write(display, ILK_DPFC_CB_BASE(fbc->id),
572 val: i915_gem_stolen_node_offset(node: fbc->compressed_fb));
573}
574
575static const struct intel_fbc_funcs ilk_fbc_funcs = {
576 .activate = ilk_fbc_activate,
577 .deactivate = ilk_fbc_deactivate,
578 .is_active = ilk_fbc_is_active,
579 .is_compressing = ilk_fbc_is_compressing,
580 .nuke = i965_fbc_nuke,
581 .program_cfb = ilk_fbc_program_cfb,
582};
583
584static void snb_fbc_program_fence(struct intel_fbc *fbc)
585{
586 struct intel_display *display = fbc->display;
587 const struct intel_fbc_state *fbc_state = &fbc->state;
588 u32 ctl = 0;
589
590 if (fbc_state->fence_id >= 0)
591 ctl = SNB_DPFC_FENCE_EN | SNB_DPFC_FENCENO(fbc_state->fence_id);
592
593 intel_de_write(display, SNB_DPFC_CTL_SA, val: ctl);
594 intel_de_write(display, SNB_DPFC_CPU_FENCE_OFFSET, val: fbc_state->fence_y_offset);
595}
596
597static void snb_fbc_activate(struct intel_fbc *fbc)
598{
599 snb_fbc_program_fence(fbc);
600
601 ilk_fbc_activate(fbc);
602}
603
604static void snb_fbc_nuke(struct intel_fbc *fbc)
605{
606 struct intel_display *display = fbc->display;
607
608 intel_de_write(display, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);
609 intel_de_posting_read(display, MSG_FBC_REND_STATE(fbc->id));
610}
611
612static const struct intel_fbc_funcs snb_fbc_funcs = {
613 .activate = snb_fbc_activate,
614 .deactivate = ilk_fbc_deactivate,
615 .is_active = ilk_fbc_is_active,
616 .is_compressing = ilk_fbc_is_compressing,
617 .nuke = snb_fbc_nuke,
618 .program_cfb = ilk_fbc_program_cfb,
619};
620
621static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)
622{
623 struct intel_display *display = fbc->display;
624 const struct intel_fbc_state *fbc_state = &fbc->state;
625 u32 val = 0;
626
627 if (fbc_state->override_cfb_stride)
628 val |= FBC_STRIDE_OVERRIDE |
629 FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
630
631 intel_de_write(display, GLK_FBC_STRIDE(fbc->id), val);
632}
633
634static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)
635{
636 struct intel_display *display = fbc->display;
637 const struct intel_fbc_state *fbc_state = &fbc->state;
638 u32 val = 0;
639
640 /* Display WA #0529: skl, kbl, bxt. */
641 if (fbc_state->override_cfb_stride)
642 val |= CHICKEN_FBC_STRIDE_OVERRIDE |
643 CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
644
645 intel_de_rmw(display, CHICKEN_MISC_4,
646 CHICKEN_FBC_STRIDE_OVERRIDE |
647 CHICKEN_FBC_STRIDE_MASK, set: val);
648}
649
650static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)
651{
652 struct intel_display *display = fbc->display;
653 const struct intel_fbc_state *fbc_state = &fbc->state;
654 u32 dpfc_ctl;
655
656 dpfc_ctl = g4x_dpfc_ctl_limit(fbc);
657
658 if (display->platform.ivybridge)
659 dpfc_ctl |= DPFC_CTL_PLANE_IVB(fbc_state->plane->i9xx_plane);
660
661 if (DISPLAY_VER(display) >= 20)
662 dpfc_ctl |= DPFC_CTL_PLANE_BINDING(fbc_state->plane->id);
663
664 if (fbc_state->fence_id >= 0)
665 dpfc_ctl |= DPFC_CTL_FENCE_EN_IVB;
666
667 if (fbc->false_color)
668 dpfc_ctl |= DPFC_CTL_FALSE_COLOR;
669
670 return dpfc_ctl;
671}
672
673static void ivb_fbc_activate(struct intel_fbc *fbc)
674{
675 struct intel_display *display = fbc->display;
676 u32 dpfc_ctl;
677
678 if (DISPLAY_VER(display) >= 10)
679 glk_fbc_program_cfb_stride(fbc);
680 else if (DISPLAY_VER(display) == 9)
681 skl_fbc_program_cfb_stride(fbc);
682
683 if (intel_fbc_has_fences(display))
684 snb_fbc_program_fence(fbc);
685
686 /* wa_14019417088 Alternative WA*/
687 dpfc_ctl = ivb_dpfc_ctl(fbc);
688 if (DISPLAY_VER(display) >= 20)
689 intel_de_write(display, ILK_DPFC_CONTROL(fbc->id), val: dpfc_ctl);
690
691 if (HAS_FBC_DIRTY_RECT(display))
692 intel_de_write(display, XE3_FBC_DIRTY_CTL(fbc->id),
693 FBC_DIRTY_RECT_EN);
694
695 intel_de_write(display, ILK_DPFC_CONTROL(fbc->id),
696 DPFC_CTL_EN | dpfc_ctl);
697}
698
699static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)
700{
701 return intel_de_read(display: fbc->display, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;
702}
703
704static void ivb_fbc_set_false_color(struct intel_fbc *fbc,
705 bool enable)
706{
707 intel_de_rmw(display: fbc->display, ILK_DPFC_CONTROL(fbc->id),
708 DPFC_CTL_FALSE_COLOR, set: enable ? DPFC_CTL_FALSE_COLOR : 0);
709}
710
711static const struct intel_fbc_funcs ivb_fbc_funcs = {
712 .activate = ivb_fbc_activate,
713 .deactivate = ilk_fbc_deactivate,
714 .is_active = ilk_fbc_is_active,
715 .is_compressing = ivb_fbc_is_compressing,
716 .nuke = snb_fbc_nuke,
717 .program_cfb = ilk_fbc_program_cfb,
718 .set_false_color = ivb_fbc_set_false_color,
719};
720
721static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)
722{
723 return fbc->funcs->is_active(fbc);
724}
725
726static void intel_fbc_hw_activate(struct intel_fbc *fbc)
727{
728 trace_intel_fbc_activate(plane: fbc->state.plane);
729
730 fbc->active = true;
731 fbc->activated = true;
732
733 fbc->funcs->activate(fbc);
734}
735
736static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)
737{
738 trace_intel_fbc_deactivate(plane: fbc->state.plane);
739
740 fbc->active = false;
741
742 fbc->funcs->deactivate(fbc);
743}
744
745static bool intel_fbc_is_compressing(struct intel_fbc *fbc)
746{
747 return fbc->funcs->is_compressing(fbc);
748}
749
750static void intel_fbc_nuke(struct intel_fbc *fbc)
751{
752 struct intel_display *display = fbc->display;
753
754 lockdep_assert_held(&fbc->lock);
755 drm_WARN_ON(display->drm, fbc->flip_pending);
756
757 trace_intel_fbc_nuke(plane: fbc->state.plane);
758
759 fbc->funcs->nuke(fbc);
760}
761
762static void intel_fbc_activate(struct intel_fbc *fbc)
763{
764 struct intel_display *display = fbc->display;
765
766 lockdep_assert_held(&fbc->lock);
767
768 /* only the fence can change for a flip nuke */
769 if (fbc->active && !intel_fbc_has_fences(display))
770 return;
771 /*
772 * In case of FBC dirt rect, any updates to the FBC registers will
773 * trigger the nuke.
774 */
775 drm_WARN_ON(display->drm, fbc->active && HAS_FBC_DIRTY_RECT(display));
776
777 intel_fbc_hw_activate(fbc);
778 intel_fbc_nuke(fbc);
779
780 fbc->no_fbc_reason = NULL;
781}
782
783static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)
784{
785 lockdep_assert_held(&fbc->lock);
786
787 if (fbc->active)
788 intel_fbc_hw_deactivate(fbc);
789
790 fbc->no_fbc_reason = reason;
791}
792
793static u64 intel_fbc_cfb_base_max(struct intel_display *display)
794{
795 if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
796 return BIT_ULL(28);
797 else
798 return BIT_ULL(32);
799}
800
801static u64 intel_fbc_stolen_end(struct intel_display *display)
802{
803 u64 end;
804
805 /* The FBC hardware for BDW/SKL doesn't have access to the stolen
806 * reserved range size, so it always assumes the maximum (8mb) is used.
807 * If we enable FBC using a CFB on that memory range we'll get FIFO
808 * underruns, even if that range is not reserved by the BIOS. */
809 if (display->platform.broadwell ||
810 (DISPLAY_VER(display) == 9 && !display->platform.broxton))
811 end = i915_gem_stolen_area_size(drm: display->drm) - 8 * 1024 * 1024;
812 else
813 end = U64_MAX;
814
815 return min(end, intel_fbc_cfb_base_max(display));
816}
817
818static int intel_fbc_min_limit(const struct intel_plane_state *plane_state)
819{
820 return plane_state->hw.fb->format->cpp[0] == 2 ? 2 : 1;
821}
822
823static int intel_fbc_max_limit(struct intel_display *display)
824{
825 /* WaFbcOnly1to1Ratio:ctg */
826 if (display->platform.g4x)
827 return 1;
828
829 /*
830 * FBC2 can only do 1:1, 1:2, 1:4, we limit
831 * FBC1 to the same out of convenience.
832 */
833 return 4;
834}
835
836static int find_compression_limit(struct intel_fbc *fbc,
837 unsigned int size, int min_limit)
838{
839 struct intel_display *display = fbc->display;
840 u64 end = intel_fbc_stolen_end(display);
841 int ret, limit = min_limit;
842
843 size /= limit;
844
845 /* Try to over-allocate to reduce reallocations and fragmentation. */
846 ret = i915_gem_stolen_insert_node_in_range(node: fbc->compressed_fb,
847 size: size <<= 1, alignment: 4096, start: 0, end);
848 if (ret == 0)
849 return limit;
850
851 for (; limit <= intel_fbc_max_limit(display); limit <<= 1) {
852 ret = i915_gem_stolen_insert_node_in_range(node: fbc->compressed_fb,
853 size: size >>= 1, alignment: 4096, start: 0, end);
854 if (ret == 0)
855 return limit;
856 }
857
858 return 0;
859}
860
861static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
862 unsigned int size, int min_limit)
863{
864 struct intel_display *display = fbc->display;
865 int ret;
866
867 drm_WARN_ON(display->drm,
868 i915_gem_stolen_node_allocated(fbc->compressed_fb));
869 drm_WARN_ON(display->drm,
870 i915_gem_stolen_node_allocated(fbc->compressed_llb));
871
872 if (DISPLAY_VER(display) < 5 && !display->platform.g4x) {
873 ret = i915_gem_stolen_insert_node(node: fbc->compressed_llb, size: 4096, alignment: 4096);
874 if (ret)
875 goto err;
876 }
877
878 ret = find_compression_limit(fbc, size, min_limit);
879 if (!ret)
880 goto err_llb;
881 else if (ret > min_limit)
882 drm_info_once(display->drm,
883 "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
884
885 fbc->limit = ret;
886
887 drm_dbg_kms(display->drm,
888 "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
889 i915_gem_stolen_node_size(fbc->compressed_fb), fbc->limit);
890 return 0;
891
892err_llb:
893 if (i915_gem_stolen_node_allocated(node: fbc->compressed_llb))
894 i915_gem_stolen_remove_node(node: fbc->compressed_llb);
895err:
896 if (i915_gem_stolen_initialized(drm: display->drm))
897 drm_info_once(display->drm,
898 "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
899 return -ENOSPC;
900}
901
902static void intel_fbc_program_cfb(struct intel_fbc *fbc)
903{
904 fbc->funcs->program_cfb(fbc);
905}
906
907static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
908{
909 struct intel_display *display = fbc->display;
910
911 if (display->platform.skylake || display->platform.broxton) {
912 /*
913 * WaFbcHighMemBwCorruptionAvoidance:skl,bxt
914 * Display WA #0883: skl,bxt
915 */
916 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
917 clear: 0, DPFC_DISABLE_DUMMY0);
918 }
919
920 if (display->platform.skylake || display->platform.kabylake ||
921 display->platform.coffeelake || display->platform.cometlake) {
922 /*
923 * WaFbcNukeOnHostModify:skl,kbl,cfl
924 * Display WA #0873: skl,kbl,cfl
925 */
926 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
927 clear: 0, DPFC_NUKE_ON_ANY_MODIFICATION);
928 }
929
930 /* Wa_1409120013:icl,jsl,tgl,dg1 */
931 if (IS_DISPLAY_VER(display, 11, 12))
932 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
933 clear: 0, DPFC_CHICKEN_COMP_DUMMY_PIXEL);
934 /*
935 * Wa_22014263786
936 * Fixes: Screen flicker with FBC and Package C state enabled
937 * Workaround: Forced SLB invalidation before start of new frame.
938 */
939 if (intel_display_wa(display, 22014263786))
940 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
941 clear: 0, DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
942
943 /* wa_18038517565 Disable DPFC clock gating before FBC enable */
944 if (display->platform.dg2 || DISPLAY_VER(display) >= 14)
945 fbc_compressor_clkgate_disable_wa(fbc, disable: true);
946}
947
948static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
949{
950 if (WARN_ON(intel_fbc_hw_is_active(fbc)))
951 return;
952
953 if (i915_gem_stolen_node_allocated(node: fbc->compressed_llb))
954 i915_gem_stolen_remove_node(node: fbc->compressed_llb);
955 if (i915_gem_stolen_node_allocated(node: fbc->compressed_fb))
956 i915_gem_stolen_remove_node(node: fbc->compressed_fb);
957}
958
959void intel_fbc_cleanup(struct intel_display *display)
960{
961 struct intel_fbc *fbc;
962 enum intel_fbc_id fbc_id;
963
964 for_each_intel_fbc(display, fbc, fbc_id) {
965 mutex_lock(&fbc->lock);
966 __intel_fbc_cleanup_cfb(fbc);
967 mutex_unlock(lock: &fbc->lock);
968
969 i915_gem_stolen_node_free(node: fbc->compressed_fb);
970 i915_gem_stolen_node_free(node: fbc->compressed_llb);
971
972 kfree(objp: fbc);
973 }
974}
975
976static bool i8xx_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
977{
978 const struct drm_framebuffer *fb = plane_state->hw.fb;
979 unsigned int stride = intel_fbc_plane_stride(plane_state) *
980 fb->format->cpp[0];
981
982 return stride == 4096 || stride == 8192;
983}
984
985static bool i965_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
986{
987 const struct drm_framebuffer *fb = plane_state->hw.fb;
988 unsigned int stride = intel_fbc_plane_stride(plane_state) *
989 fb->format->cpp[0];
990
991 return stride >= 2048 && stride <= 16384;
992}
993
994static bool g4x_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
995{
996 return true;
997}
998
999static bool skl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
1000{
1001 const struct drm_framebuffer *fb = plane_state->hw.fb;
1002 unsigned int stride = intel_fbc_plane_stride(plane_state) *
1003 fb->format->cpp[0];
1004
1005 /* Display WA #1105: skl,bxt,kbl,cfl,glk */
1006 if (fb->modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
1007 return false;
1008
1009 return true;
1010}
1011
1012static bool icl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
1013{
1014 return true;
1015}
1016
1017static bool stride_is_valid(const struct intel_plane_state *plane_state)
1018{
1019 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1020
1021 if (DISPLAY_VER(display) >= 11)
1022 return icl_fbc_stride_is_valid(plane_state);
1023 else if (DISPLAY_VER(display) >= 9)
1024 return skl_fbc_stride_is_valid(plane_state);
1025 else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
1026 return g4x_fbc_stride_is_valid(plane_state);
1027 else if (DISPLAY_VER(display) == 4)
1028 return i965_fbc_stride_is_valid(plane_state);
1029 else
1030 return i8xx_fbc_stride_is_valid(plane_state);
1031}
1032
1033static bool i8xx_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1034{
1035 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1036 const struct drm_framebuffer *fb = plane_state->hw.fb;
1037
1038 switch (fb->format->format) {
1039 case DRM_FORMAT_XRGB8888:
1040 case DRM_FORMAT_XBGR8888:
1041 return true;
1042 case DRM_FORMAT_XRGB1555:
1043 case DRM_FORMAT_RGB565:
1044 /* 16bpp not supported on gen2 */
1045 if (DISPLAY_VER(display) == 2)
1046 return false;
1047 return true;
1048 default:
1049 return false;
1050 }
1051}
1052
1053static bool g4x_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1054{
1055 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1056 const struct drm_framebuffer *fb = plane_state->hw.fb;
1057
1058 switch (fb->format->format) {
1059 case DRM_FORMAT_XRGB8888:
1060 case DRM_FORMAT_XBGR8888:
1061 return true;
1062 case DRM_FORMAT_RGB565:
1063 /* WaFbcOnly1to1Ratio:ctg */
1064 if (display->platform.g4x)
1065 return false;
1066 return true;
1067 default:
1068 return false;
1069 }
1070}
1071
1072static bool lnl_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1073{
1074 const struct drm_framebuffer *fb = plane_state->hw.fb;
1075
1076 switch (fb->format->format) {
1077 case DRM_FORMAT_XRGB8888:
1078 case DRM_FORMAT_XBGR8888:
1079 case DRM_FORMAT_ARGB8888:
1080 case DRM_FORMAT_ABGR8888:
1081 case DRM_FORMAT_RGB565:
1082 return true;
1083 default:
1084 return false;
1085 }
1086}
1087
1088static bool
1089xe3p_lpd_fbc_fp16_format_is_valid(const struct intel_plane_state *plane_state)
1090{
1091 const struct drm_framebuffer *fb = plane_state->hw.fb;
1092
1093 switch (fb->format->format) {
1094 case DRM_FORMAT_ARGB16161616F:
1095 case DRM_FORMAT_ABGR16161616F:
1096 return true;
1097 default:
1098 return false;
1099 }
1100}
1101
1102static bool xe3p_lpd_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1103{
1104 const struct drm_framebuffer *fb = plane_state->hw.fb;
1105
1106 if (lnl_fbc_pixel_format_is_valid(plane_state))
1107 return true;
1108
1109 if (xe3p_lpd_fbc_fp16_format_is_valid(plane_state))
1110 return true;
1111
1112 switch (fb->format->format) {
1113 case DRM_FORMAT_XRGB16161616:
1114 case DRM_FORMAT_XBGR16161616:
1115 case DRM_FORMAT_ARGB16161616:
1116 case DRM_FORMAT_ABGR16161616:
1117 return true;
1118 default:
1119 return false;
1120 }
1121}
1122
1123bool
1124intel_fbc_is_enable_pixel_normalizer(const struct intel_plane_state *plane_state)
1125{
1126 struct intel_display *display = to_intel_display(plane_state);
1127
1128 return DISPLAY_VER(display) >= 35 &&
1129 xe3p_lpd_fbc_fp16_format_is_valid(plane_state);
1130}
1131
1132static bool pixel_format_is_valid(const struct intel_plane_state *plane_state)
1133{
1134 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1135
1136 if (DISPLAY_VER(display) >= 35)
1137 return xe3p_lpd_fbc_pixel_format_is_valid(plane_state);
1138 else if (DISPLAY_VER(display) >= 20)
1139 return lnl_fbc_pixel_format_is_valid(plane_state);
1140 else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
1141 return g4x_fbc_pixel_format_is_valid(plane_state);
1142 else
1143 return i8xx_fbc_pixel_format_is_valid(plane_state);
1144}
1145
1146static bool i8xx_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1147{
1148 return plane_state->hw.rotation == DRM_MODE_ROTATE_0;
1149}
1150
1151static bool g4x_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1152{
1153 return true;
1154}
1155
1156static bool skl_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1157{
1158 const struct drm_framebuffer *fb = plane_state->hw.fb;
1159 unsigned int rotation = plane_state->hw.rotation;
1160
1161 if (fb->format->format == DRM_FORMAT_RGB565 &&
1162 drm_rotation_90_or_270(rotation))
1163 return false;
1164
1165 return true;
1166}
1167
1168static bool rotation_is_valid(const struct intel_plane_state *plane_state)
1169{
1170 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1171
1172 if (DISPLAY_VER(display) >= 9)
1173 return skl_fbc_rotation_is_valid(plane_state);
1174 else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
1175 return g4x_fbc_rotation_is_valid(plane_state);
1176 else
1177 return i8xx_fbc_rotation_is_valid(plane_state);
1178}
1179
1180static void intel_fbc_max_surface_size(struct intel_display *display,
1181 unsigned int *w, unsigned int *h)
1182{
1183 if (DISPLAY_VER(display) >= 11) {
1184 *w = 8192;
1185 *h = 4096;
1186 } else if (DISPLAY_VER(display) >= 10) {
1187 *w = 5120;
1188 *h = 4096;
1189 } else if (DISPLAY_VER(display) >= 7) {
1190 *w = 4096;
1191 *h = 4096;
1192 } else if (DISPLAY_VER(display) >= 5 || display->platform.g4x) {
1193 *w = 4096;
1194 *h = 2048;
1195 } else {
1196 *w = 2048;
1197 *h = 1536;
1198 }
1199}
1200
1201/*
1202 * For some reason, the hardware tracking starts looking at whatever we
1203 * programmed as the display plane base address register. It does not look at
1204 * the X and Y offset registers. That's why we include the src x/y offsets
1205 * instead of just looking at the plane size.
1206 */
1207static bool intel_fbc_surface_size_ok(const struct intel_plane_state *plane_state)
1208{
1209 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1210 unsigned int effective_w, effective_h, max_w, max_h;
1211
1212 intel_fbc_max_surface_size(display, w: &max_w, h: &max_h);
1213
1214 effective_w = plane_state->view.color_plane[0].x +
1215 (drm_rect_width(r: &plane_state->uapi.src) >> 16);
1216 effective_h = plane_state->view.color_plane[0].y +
1217 (drm_rect_height(r: &plane_state->uapi.src) >> 16);
1218
1219 return effective_w <= max_w && effective_h <= max_h;
1220}
1221
1222static void intel_fbc_max_plane_size(struct intel_display *display,
1223 unsigned int *w, unsigned int *h)
1224{
1225 if (DISPLAY_VER(display) >= 10) {
1226 *w = 5120;
1227 *h = 4096;
1228 } else if (DISPLAY_VER(display) >= 8 || display->platform.haswell) {
1229 *w = 4096;
1230 *h = 4096;
1231 } else if (DISPLAY_VER(display) >= 5 || display->platform.g4x) {
1232 *w = 4096;
1233 *h = 2048;
1234 } else {
1235 *w = 2048;
1236 *h = 1536;
1237 }
1238}
1239
1240static bool intel_fbc_plane_size_valid(const struct intel_plane_state *plane_state)
1241{
1242 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1243 unsigned int w, h, max_w, max_h;
1244
1245 intel_fbc_max_plane_size(display, w: &max_w, h: &max_h);
1246
1247 w = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1248 h = drm_rect_height(r: &plane_state->uapi.src) >> 16;
1249
1250 return w <= max_w && h <= max_h;
1251}
1252
1253static bool i8xx_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1254{
1255 const struct drm_framebuffer *fb = plane_state->hw.fb;
1256
1257 return fb->modifier == I915_FORMAT_MOD_X_TILED;
1258}
1259
1260static bool skl_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1261{
1262 return true;
1263}
1264
1265static bool tiling_is_valid(const struct intel_plane_state *plane_state)
1266{
1267 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1268
1269 if (DISPLAY_VER(display) >= 9)
1270 return skl_fbc_tiling_valid(plane_state);
1271 else
1272 return i8xx_fbc_tiling_valid(plane_state);
1273}
1274
1275static void
1276intel_fbc_invalidate_dirty_rect(struct intel_fbc *fbc)
1277{
1278 lockdep_assert_held(&fbc->lock);
1279
1280 fbc->state.dirty_rect = DRM_RECT_INIT(0, 0, 0, 0);
1281}
1282
1283static void
1284intel_fbc_program_dirty_rect(struct intel_dsb *dsb, struct intel_fbc *fbc,
1285 const struct drm_rect *fbc_dirty_rect)
1286{
1287 struct intel_display *display = fbc->display;
1288
1289 drm_WARN_ON(display->drm, fbc_dirty_rect->y2 == 0);
1290
1291 intel_de_write_dsb(display, dsb, XE3_FBC_DIRTY_RECT(fbc->id),
1292 FBC_DIRTY_RECT_START_LINE(fbc_dirty_rect->y1) |
1293 FBC_DIRTY_RECT_END_LINE(fbc_dirty_rect->y2 - 1));
1294}
1295
1296static void
1297intel_fbc_dirty_rect_update(struct intel_dsb *dsb, struct intel_fbc *fbc)
1298{
1299 const struct drm_rect *fbc_dirty_rect = &fbc->state.dirty_rect;
1300
1301 lockdep_assert_held(&fbc->lock);
1302
1303 if (!drm_rect_visible(r: fbc_dirty_rect))
1304 return;
1305
1306 intel_fbc_program_dirty_rect(dsb, fbc, fbc_dirty_rect);
1307}
1308
1309void
1310intel_fbc_dirty_rect_update_noarm(struct intel_dsb *dsb,
1311 struct intel_plane *plane)
1312{
1313 struct intel_display *display = to_intel_display(plane);
1314 struct intel_fbc *fbc = plane->fbc;
1315
1316 if (!HAS_FBC_DIRTY_RECT(display))
1317 return;
1318
1319 mutex_lock(&fbc->lock);
1320
1321 if (fbc->state.plane == plane)
1322 intel_fbc_dirty_rect_update(dsb, fbc);
1323
1324 mutex_unlock(lock: &fbc->lock);
1325}
1326
1327static void
1328intel_fbc_hw_intialize_dirty_rect(struct intel_fbc *fbc,
1329 const struct intel_plane_state *plane_state)
1330{
1331 struct drm_rect src;
1332
1333 /*
1334 * Initializing the FBC HW with the whole plane area as the dirty rect.
1335 * This is to ensure that we have valid coords be written to the
1336 * HW as dirty rect.
1337 */
1338 drm_rect_fp_to_int(dst: &src, src: &plane_state->uapi.src);
1339
1340 intel_fbc_program_dirty_rect(NULL, fbc, fbc_dirty_rect: &src);
1341}
1342
1343static void intel_fbc_update_state(struct intel_atomic_state *state,
1344 struct intel_crtc *crtc,
1345 struct intel_plane *plane)
1346{
1347 struct intel_display *display = to_intel_display(state->base.dev);
1348 const struct intel_crtc_state *crtc_state =
1349 intel_atomic_get_new_crtc_state(state, crtc);
1350 const struct intel_plane_state *plane_state =
1351 intel_atomic_get_new_plane_state(state, plane);
1352 struct intel_fbc *fbc = plane->fbc;
1353 struct intel_fbc_state *fbc_state = &fbc->state;
1354
1355 WARN_ON(plane_state->no_fbc_reason);
1356 WARN_ON(fbc_state->plane && fbc_state->plane != plane);
1357
1358 fbc_state->plane = plane;
1359
1360 /* FBC1 compression interval: arbitrary choice of 1 second */
1361 fbc_state->interval = drm_mode_vrefresh(mode: &crtc_state->hw.adjusted_mode);
1362
1363 fbc_state->fence_y_offset = intel_plane_fence_y_offset(plane_state);
1364
1365 drm_WARN_ON(display->drm, plane_state->flags & PLANE_HAS_FENCE &&
1366 !intel_fbc_has_fences(display));
1367
1368 if (plane_state->flags & PLANE_HAS_FENCE)
1369 fbc_state->fence_id = i915_vma_fence_id(vma: plane_state->ggtt_vma);
1370 else
1371 fbc_state->fence_id = -1;
1372
1373 fbc_state->cfb_stride = intel_fbc_cfb_stride(plane_state);
1374 fbc_state->cfb_size = intel_fbc_cfb_size(plane_state);
1375 fbc_state->override_cfb_stride = intel_fbc_override_cfb_stride(plane_state);
1376}
1377
1378static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
1379{
1380 struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1381
1382 /*
1383 * The use of a CPU fence is one of two ways to detect writes by the
1384 * CPU to the scanout and trigger updates to the FBC.
1385 *
1386 * The other method is by software tracking (see
1387 * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
1388 * the current compressed buffer and recompress it.
1389 *
1390 * Note that is possible for a tiled surface to be unmappable (and
1391 * so have no fence associated with it) due to aperture constraints
1392 * at the time of pinning.
1393 */
1394 return DISPLAY_VER(display) >= 9 ||
1395 (plane_state->flags & PLANE_HAS_FENCE &&
1396 i915_vma_fence_id(vma: plane_state->ggtt_vma) != -1);
1397}
1398
1399static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
1400{
1401 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1402 struct intel_fbc *fbc = plane->fbc;
1403
1404 return intel_fbc_min_limit(plane_state) <= fbc->limit &&
1405 intel_fbc_cfb_size(plane_state) <= fbc->limit *
1406 i915_gem_stolen_node_size(node: fbc->compressed_fb);
1407}
1408
1409static bool intel_fbc_is_ok(const struct intel_plane_state *plane_state)
1410{
1411 return !plane_state->no_fbc_reason &&
1412 intel_fbc_is_fence_ok(plane_state) &&
1413 intel_fbc_is_cfb_ok(plane_state);
1414}
1415
1416static void
1417__intel_fbc_prepare_dirty_rect(const struct intel_plane_state *plane_state,
1418 const struct intel_crtc_state *crtc_state)
1419{
1420 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1421 struct intel_fbc *fbc = plane->fbc;
1422 struct drm_rect *fbc_dirty_rect = &fbc->state.dirty_rect;
1423 int width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1424 const struct drm_rect *damage = &plane_state->damage;
1425 int y_offset = plane_state->view.color_plane[0].y;
1426
1427 lockdep_assert_held(&fbc->lock);
1428
1429 if (intel_crtc_needs_modeset(crtc_state) ||
1430 !intel_fbc_is_ok(plane_state)) {
1431 intel_fbc_invalidate_dirty_rect(fbc);
1432 return;
1433 }
1434
1435 if (drm_rect_visible(r: damage))
1436 *fbc_dirty_rect = *damage;
1437 else
1438 /* dirty rect must cover at least one line */
1439 *fbc_dirty_rect = DRM_RECT_INIT(0, y_offset, width, 1);
1440}
1441
1442void
1443intel_fbc_prepare_dirty_rect(struct intel_atomic_state *state,
1444 struct intel_crtc *crtc)
1445{
1446 struct intel_display *display = to_intel_display(state);
1447 const struct intel_crtc_state *crtc_state =
1448 intel_atomic_get_new_crtc_state(state, crtc);
1449 struct intel_plane_state *plane_state;
1450 struct intel_plane *plane;
1451 int i;
1452
1453 if (!HAS_FBC_DIRTY_RECT(display))
1454 return;
1455
1456 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1457 struct intel_fbc *fbc = plane->fbc;
1458
1459 if (!fbc || plane->pipe != crtc->pipe)
1460 continue;
1461
1462 mutex_lock(&fbc->lock);
1463
1464 if (fbc->state.plane == plane)
1465 __intel_fbc_prepare_dirty_rect(plane_state,
1466 crtc_state);
1467
1468 mutex_unlock(lock: &fbc->lock);
1469 }
1470}
1471
1472static int _intel_fbc_min_cdclk(const struct intel_crtc_state *crtc_state)
1473{
1474 struct intel_display *display = to_intel_display(crtc_state);
1475
1476 /* WaFbcExceedCdClockThreshold:hsw,bdw */
1477 if (display->platform.haswell || display->platform.broadwell)
1478 return DIV_ROUND_UP(crtc_state->pixel_rate * 100, 95);
1479
1480 /* no FBC specific limits to worry about */
1481 return 0;
1482}
1483
1484static int intel_fbc_check_plane(struct intel_atomic_state *state,
1485 struct intel_plane *plane)
1486{
1487 struct intel_display *display = to_intel_display(state->base.dev);
1488 struct drm_i915_private *i915 = to_i915(dev: display->drm);
1489 struct intel_plane_state *plane_state =
1490 intel_atomic_get_new_plane_state(state, plane);
1491 const struct drm_framebuffer *fb = plane_state->hw.fb;
1492 struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1493 const struct intel_crtc_state *crtc_state;
1494 struct intel_fbc *fbc = plane->fbc;
1495
1496 if (!fbc)
1497 return 0;
1498
1499 if (!i915_gem_stolen_initialized(drm: display->drm)) {
1500 plane_state->no_fbc_reason = "stolen memory not initialised";
1501 return 0;
1502 }
1503
1504 if (intel_vgpu_active(i915)) {
1505 plane_state->no_fbc_reason = "VGPU active";
1506 return 0;
1507 }
1508
1509 if (!display->params.enable_fbc) {
1510 plane_state->no_fbc_reason = "disabled per module param or by default";
1511 return 0;
1512 }
1513
1514 if (!plane_state->uapi.visible) {
1515 plane_state->no_fbc_reason = "plane not visible";
1516 return 0;
1517 }
1518
1519 if (intel_display_wa(display, 16023588340)) {
1520 plane_state->no_fbc_reason = "Wa_16023588340";
1521 return 0;
1522 }
1523
1524 /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1525 if (intel_display_vtd_active(display) &&
1526 (display->platform.skylake || display->platform.broxton)) {
1527 plane_state->no_fbc_reason = "VT-d enabled";
1528 return 0;
1529 }
1530
1531 crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1532
1533 if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
1534 plane_state->no_fbc_reason = "interlaced mode not supported";
1535 return 0;
1536 }
1537
1538 if (crtc_state->double_wide) {
1539 plane_state->no_fbc_reason = "double wide pipe not supported";
1540 return 0;
1541 }
1542
1543 /*
1544 * Display 12+ is not supporting FBC with PSR2.
1545 * Recommendation is to keep this combination disabled
1546 * Bspec: 50422 HSD: 14010260002
1547 *
1548 * TODO: Implement a logic to select between PSR2 selective fetch and
1549 * FBC based on Bspec: 68881 in xe2lpd onwards.
1550 *
1551 * As we still see some strange underruns in those platforms while
1552 * disabling PSR2, keep FBC disabled in case of selective update is on
1553 * until the selection logic is implemented.
1554 */
1555 if (DISPLAY_VER(display) >= 12 && crtc_state->has_sel_update) {
1556 plane_state->no_fbc_reason = "Selective update enabled";
1557 return 0;
1558 }
1559
1560 /* Wa_14016291713 */
1561 if ((IS_DISPLAY_VER(display, 12, 13) ||
1562 IS_DISPLAY_VERx100_STEP(display, 1400, STEP_A0, STEP_C0)) &&
1563 crtc_state->has_psr && !crtc_state->has_panel_replay) {
1564 plane_state->no_fbc_reason = "PSR1 enabled (Wa_14016291713)";
1565 return 0;
1566 }
1567
1568 if (!pixel_format_is_valid(plane_state)) {
1569 plane_state->no_fbc_reason = "pixel format not supported";
1570 return 0;
1571 }
1572
1573 if (!tiling_is_valid(plane_state)) {
1574 plane_state->no_fbc_reason = "tiling not supported";
1575 return 0;
1576 }
1577
1578 if (!rotation_is_valid(plane_state)) {
1579 plane_state->no_fbc_reason = "rotation not supported";
1580 return 0;
1581 }
1582
1583 if (!stride_is_valid(plane_state)) {
1584 plane_state->no_fbc_reason = "stride not supported";
1585 return 0;
1586 }
1587
1588 if (DISPLAY_VER(display) < 20 &&
1589 plane_state->hw.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
1590 fb->format->has_alpha) {
1591 plane_state->no_fbc_reason = "per-pixel alpha not supported";
1592 return 0;
1593 }
1594
1595 if (!intel_fbc_plane_size_valid(plane_state)) {
1596 plane_state->no_fbc_reason = "plane size too big";
1597 return 0;
1598 }
1599
1600 if (!intel_fbc_surface_size_ok(plane_state)) {
1601 plane_state->no_fbc_reason = "surface size too big";
1602 return 0;
1603 }
1604
1605 /*
1606 * Work around a problem on GEN9+ HW, where enabling FBC on a plane
1607 * having a Y offset that isn't divisible by 4 causes FIFO underrun
1608 * and screen flicker.
1609 */
1610 if (IS_DISPLAY_VER(display, 9, 12) &&
1611 plane_state->view.color_plane[0].y & 3) {
1612 plane_state->no_fbc_reason = "plane start Y offset misaligned";
1613 return 0;
1614 }
1615
1616 /* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
1617 if (IS_DISPLAY_VER(display, 9, 12) &&
1618 (plane_state->view.color_plane[0].y +
1619 (drm_rect_height(r: &plane_state->uapi.src) >> 16)) & 3) {
1620 plane_state->no_fbc_reason = "plane end Y offset misaligned";
1621 return 0;
1622 }
1623
1624 if (_intel_fbc_min_cdclk(crtc_state) > display->cdclk.max_cdclk_freq) {
1625 plane_state->no_fbc_reason = "pixel rate too high";
1626 return 0;
1627 }
1628
1629 plane_state->no_fbc_reason = NULL;
1630
1631 return 0;
1632}
1633
1634int intel_fbc_min_cdclk(const struct intel_crtc_state *crtc_state)
1635{
1636 struct intel_display *display = to_intel_display(crtc_state);
1637 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1638 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1639 int min_cdclk;
1640
1641 if (!plane->fbc)
1642 return 0;
1643
1644 min_cdclk = _intel_fbc_min_cdclk(crtc_state);
1645
1646 /*
1647 * Do not ask for more than the max CDCLK frequency,
1648 * if that is not enough FBC will simply not be used.
1649 */
1650 if (min_cdclk > display->cdclk.max_cdclk_freq)
1651 return 0;
1652
1653 return min_cdclk;
1654}
1655
1656static bool intel_fbc_can_flip_nuke(struct intel_atomic_state *state,
1657 struct intel_crtc *crtc,
1658 struct intel_plane *plane)
1659{
1660 const struct intel_crtc_state *new_crtc_state =
1661 intel_atomic_get_new_crtc_state(state, crtc);
1662 const struct intel_plane_state *old_plane_state =
1663 intel_atomic_get_old_plane_state(state, plane);
1664 const struct intel_plane_state *new_plane_state =
1665 intel_atomic_get_new_plane_state(state, plane);
1666 const struct drm_framebuffer *old_fb = old_plane_state->hw.fb;
1667 const struct drm_framebuffer *new_fb = new_plane_state->hw.fb;
1668
1669 if (intel_crtc_needs_modeset(crtc_state: new_crtc_state))
1670 return false;
1671
1672 if (!intel_fbc_is_ok(plane_state: old_plane_state) ||
1673 !intel_fbc_is_ok(plane_state: new_plane_state))
1674 return false;
1675
1676 if (old_fb->format->format != new_fb->format->format)
1677 return false;
1678
1679 if (old_fb->modifier != new_fb->modifier)
1680 return false;
1681
1682 if (intel_fbc_plane_stride(plane_state: old_plane_state) !=
1683 intel_fbc_plane_stride(plane_state: new_plane_state))
1684 return false;
1685
1686 if (intel_fbc_cfb_stride(plane_state: old_plane_state) !=
1687 intel_fbc_cfb_stride(plane_state: new_plane_state))
1688 return false;
1689
1690 if (intel_fbc_cfb_size(plane_state: old_plane_state) !=
1691 intel_fbc_cfb_size(plane_state: new_plane_state))
1692 return false;
1693
1694 if (intel_fbc_override_cfb_stride(plane_state: old_plane_state) !=
1695 intel_fbc_override_cfb_stride(plane_state: new_plane_state))
1696 return false;
1697
1698 return true;
1699}
1700
1701static bool __intel_fbc_pre_update(struct intel_atomic_state *state,
1702 struct intel_crtc *crtc,
1703 struct intel_plane *plane)
1704{
1705 struct intel_display *display = to_intel_display(state->base.dev);
1706 struct intel_fbc *fbc = plane->fbc;
1707 bool need_vblank_wait = false;
1708
1709 lockdep_assert_held(&fbc->lock);
1710
1711 fbc->flip_pending = true;
1712
1713 if (intel_fbc_can_flip_nuke(state, crtc, plane))
1714 return need_vblank_wait;
1715
1716 intel_fbc_deactivate(fbc, reason: "update pending");
1717
1718 /*
1719 * Display WA #1198: glk+
1720 * Need an extra vblank wait between FBC disable and most plane
1721 * updates. Bspec says this is only needed for plane disable, but
1722 * that is not true. Touching most plane registers will cause the
1723 * corruption to appear. Also SKL/derivatives do not seem to be
1724 * affected.
1725 *
1726 * TODO: could optimize this a bit by sampling the frame
1727 * counter when we disable FBC (if it was already done earlier)
1728 * and skipping the extra vblank wait before the plane update
1729 * if at least one frame has already passed.
1730 */
1731 if (fbc->activated && DISPLAY_VER(display) >= 10)
1732 need_vblank_wait = true;
1733 fbc->activated = false;
1734
1735 return need_vblank_wait;
1736}
1737
1738bool intel_fbc_pre_update(struct intel_atomic_state *state,
1739 struct intel_crtc *crtc)
1740{
1741 const struct intel_plane_state __maybe_unused *plane_state;
1742 bool need_vblank_wait = false;
1743 struct intel_plane *plane;
1744 int i;
1745
1746 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1747 struct intel_fbc *fbc = plane->fbc;
1748
1749 if (!fbc || plane->pipe != crtc->pipe)
1750 continue;
1751
1752 mutex_lock(&fbc->lock);
1753
1754 if (fbc->state.plane == plane)
1755 need_vblank_wait |= __intel_fbc_pre_update(state, crtc, plane);
1756
1757 mutex_unlock(lock: &fbc->lock);
1758 }
1759
1760 return need_vblank_wait;
1761}
1762
1763static void __intel_fbc_disable(struct intel_fbc *fbc)
1764{
1765 struct intel_display *display = fbc->display;
1766 struct intel_plane *plane = fbc->state.plane;
1767
1768 lockdep_assert_held(&fbc->lock);
1769 drm_WARN_ON(display->drm, fbc->active);
1770
1771 drm_dbg_kms(display->drm, "Disabling FBC on [PLANE:%d:%s]\n",
1772 plane->base.base.id, plane->base.name);
1773
1774 intel_fbc_invalidate_dirty_rect(fbc);
1775
1776 __intel_fbc_cleanup_cfb(fbc);
1777
1778 /* wa_18038517565 Enable DPFC clock gating after FBC disable */
1779 if (display->platform.dg2 || DISPLAY_VER(display) >= 14)
1780 fbc_compressor_clkgate_disable_wa(fbc, disable: false);
1781
1782 fbc->state.plane = NULL;
1783 fbc->flip_pending = false;
1784 fbc->busy_bits = 0;
1785}
1786
1787static void __intel_fbc_post_update(struct intel_fbc *fbc)
1788{
1789 lockdep_assert_held(&fbc->lock);
1790
1791 fbc->flip_pending = false;
1792 fbc->busy_bits = 0;
1793
1794 intel_fbc_activate(fbc);
1795}
1796
1797void intel_fbc_post_update(struct intel_atomic_state *state,
1798 struct intel_crtc *crtc)
1799{
1800 const struct intel_plane_state __maybe_unused *plane_state;
1801 struct intel_plane *plane;
1802 int i;
1803
1804 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1805 struct intel_fbc *fbc = plane->fbc;
1806
1807 if (!fbc || plane->pipe != crtc->pipe)
1808 continue;
1809
1810 mutex_lock(&fbc->lock);
1811
1812 if (fbc->state.plane == plane)
1813 __intel_fbc_post_update(fbc);
1814
1815 mutex_unlock(lock: &fbc->lock);
1816 }
1817}
1818
1819static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1820{
1821 if (fbc->state.plane)
1822 return fbc->state.plane->frontbuffer_bit;
1823 else
1824 return 0;
1825}
1826
1827static void __intel_fbc_invalidate(struct intel_fbc *fbc,
1828 unsigned int frontbuffer_bits,
1829 enum fb_op_origin origin)
1830{
1831 if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1832 return;
1833
1834 mutex_lock(&fbc->lock);
1835
1836 frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1837 if (!frontbuffer_bits)
1838 goto out;
1839
1840 fbc->busy_bits |= frontbuffer_bits;
1841 intel_fbc_deactivate(fbc, reason: "frontbuffer write");
1842
1843out:
1844 mutex_unlock(lock: &fbc->lock);
1845}
1846
1847void intel_fbc_invalidate(struct intel_display *display,
1848 unsigned int frontbuffer_bits,
1849 enum fb_op_origin origin)
1850{
1851 struct intel_fbc *fbc;
1852 enum intel_fbc_id fbc_id;
1853
1854 for_each_intel_fbc(display, fbc, fbc_id)
1855 __intel_fbc_invalidate(fbc, frontbuffer_bits, origin);
1856
1857}
1858
1859static void __intel_fbc_flush(struct intel_fbc *fbc,
1860 unsigned int frontbuffer_bits,
1861 enum fb_op_origin origin)
1862{
1863 mutex_lock(&fbc->lock);
1864
1865 frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1866 if (!frontbuffer_bits)
1867 goto out;
1868
1869 fbc->busy_bits &= ~frontbuffer_bits;
1870
1871 if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1872 goto out;
1873
1874 if (fbc->busy_bits || fbc->flip_pending)
1875 goto out;
1876
1877 if (fbc->active)
1878 intel_fbc_nuke(fbc);
1879 else
1880 intel_fbc_activate(fbc);
1881
1882out:
1883 mutex_unlock(lock: &fbc->lock);
1884}
1885
1886void intel_fbc_flush(struct intel_display *display,
1887 unsigned int frontbuffer_bits,
1888 enum fb_op_origin origin)
1889{
1890 struct intel_fbc *fbc;
1891 enum intel_fbc_id fbc_id;
1892
1893 for_each_intel_fbc(display, fbc, fbc_id)
1894 __intel_fbc_flush(fbc, frontbuffer_bits, origin);
1895}
1896
1897int intel_fbc_atomic_check(struct intel_atomic_state *state)
1898{
1899 struct intel_plane_state __maybe_unused *plane_state;
1900 struct intel_plane *plane;
1901 int i;
1902
1903 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1904 int ret;
1905
1906 ret = intel_fbc_check_plane(state, plane);
1907 if (ret)
1908 return ret;
1909 }
1910
1911 return 0;
1912}
1913
1914static void __intel_fbc_enable(struct intel_atomic_state *state,
1915 struct intel_crtc *crtc,
1916 struct intel_plane *plane)
1917{
1918 struct intel_display *display = to_intel_display(state->base.dev);
1919 const struct intel_plane_state *plane_state =
1920 intel_atomic_get_new_plane_state(state, plane);
1921 struct intel_fbc *fbc = plane->fbc;
1922
1923 lockdep_assert_held(&fbc->lock);
1924
1925 if (fbc->state.plane) {
1926 if (fbc->state.plane != plane)
1927 return;
1928
1929 if (intel_fbc_is_ok(plane_state)) {
1930 intel_fbc_update_state(state, crtc, plane);
1931 return;
1932 }
1933
1934 __intel_fbc_disable(fbc);
1935 }
1936
1937 drm_WARN_ON(display->drm, fbc->active);
1938
1939 fbc->no_fbc_reason = plane_state->no_fbc_reason;
1940 if (fbc->no_fbc_reason)
1941 return;
1942
1943 if (!intel_fbc_is_fence_ok(plane_state)) {
1944 fbc->no_fbc_reason = "framebuffer not fenced";
1945 return;
1946 }
1947
1948 if (fbc->underrun_detected) {
1949 fbc->no_fbc_reason = "FIFO underrun";
1950 return;
1951 }
1952
1953 if (intel_fbc_alloc_cfb(fbc, size: intel_fbc_cfb_size(plane_state),
1954 min_limit: intel_fbc_min_limit(plane_state))) {
1955 fbc->no_fbc_reason = "not enough stolen memory";
1956 return;
1957 }
1958
1959 drm_dbg_kms(display->drm, "Enabling FBC on [PLANE:%d:%s]\n",
1960 plane->base.base.id, plane->base.name);
1961 fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1962
1963 intel_fbc_update_state(state, crtc, plane);
1964
1965 if (HAS_FBC_DIRTY_RECT(display))
1966 intel_fbc_hw_intialize_dirty_rect(fbc, plane_state);
1967
1968 intel_fbc_program_workarounds(fbc);
1969 intel_fbc_program_cfb(fbc);
1970}
1971
1972/**
1973 * intel_fbc_disable - disable FBC if it's associated with crtc
1974 * @crtc: the CRTC
1975 *
1976 * This function disables FBC if it's associated with the provided CRTC.
1977 */
1978void intel_fbc_disable(struct intel_crtc *crtc)
1979{
1980 struct intel_display *display = to_intel_display(crtc->base.dev);
1981 struct intel_plane *plane;
1982
1983 for_each_intel_plane(display->drm, plane) {
1984 struct intel_fbc *fbc = plane->fbc;
1985
1986 if (!fbc || plane->pipe != crtc->pipe)
1987 continue;
1988
1989 mutex_lock(&fbc->lock);
1990 if (fbc->state.plane == plane)
1991 __intel_fbc_disable(fbc);
1992 mutex_unlock(lock: &fbc->lock);
1993 }
1994}
1995
1996void intel_fbc_update(struct intel_atomic_state *state,
1997 struct intel_crtc *crtc)
1998{
1999 const struct intel_crtc_state *crtc_state =
2000 intel_atomic_get_new_crtc_state(state, crtc);
2001 const struct intel_plane_state *plane_state;
2002 struct intel_plane *plane;
2003 int i;
2004
2005 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2006 struct intel_fbc *fbc = plane->fbc;
2007
2008 if (!fbc || plane->pipe != crtc->pipe)
2009 continue;
2010
2011 mutex_lock(&fbc->lock);
2012
2013 if (intel_crtc_needs_fastset(crtc_state) &&
2014 plane_state->no_fbc_reason) {
2015 if (fbc->state.plane == plane)
2016 __intel_fbc_disable(fbc);
2017 } else {
2018 __intel_fbc_enable(state, crtc, plane);
2019 }
2020
2021 mutex_unlock(lock: &fbc->lock);
2022 }
2023}
2024
2025static void intel_fbc_underrun_work_fn(struct work_struct *work)
2026{
2027 struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);
2028 struct intel_display *display = fbc->display;
2029
2030 mutex_lock(&fbc->lock);
2031
2032 /* Maybe we were scheduled twice. */
2033 if (fbc->underrun_detected || !fbc->state.plane)
2034 goto out;
2035
2036 drm_dbg_kms(display->drm, "Disabling FBC due to FIFO underrun.\n");
2037 fbc->underrun_detected = true;
2038
2039 intel_fbc_deactivate(fbc, reason: "FIFO underrun");
2040 if (!fbc->flip_pending)
2041 intel_crtc_wait_for_next_vblank(crtc: intel_crtc_for_pipe(display, pipe: fbc->state.plane->pipe));
2042 __intel_fbc_disable(fbc);
2043out:
2044 mutex_unlock(lock: &fbc->lock);
2045}
2046
2047static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)
2048{
2049 struct intel_display *display = fbc->display;
2050
2051 cancel_work_sync(work: &fbc->underrun_work);
2052
2053 mutex_lock(&fbc->lock);
2054
2055 if (fbc->underrun_detected) {
2056 drm_dbg_kms(display->drm,
2057 "Re-allowing FBC after fifo underrun\n");
2058 fbc->no_fbc_reason = "FIFO underrun cleared";
2059 }
2060
2061 fbc->underrun_detected = false;
2062 mutex_unlock(lock: &fbc->lock);
2063}
2064
2065/*
2066 * intel_fbc_reset_underrun - reset FBC fifo underrun status.
2067 * @display: display
2068 *
2069 * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
2070 * want to re-enable FBC after an underrun to increase test coverage.
2071 */
2072void intel_fbc_reset_underrun(struct intel_display *display)
2073{
2074 struct intel_fbc *fbc;
2075 enum intel_fbc_id fbc_id;
2076
2077 for_each_intel_fbc(display, fbc, fbc_id)
2078 __intel_fbc_reset_underrun(fbc);
2079}
2080
2081static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)
2082{
2083 struct intel_display *display = fbc->display;
2084
2085 /*
2086 * There's no guarantee that underrun_detected won't be set to true
2087 * right after this check and before the work is scheduled, but that's
2088 * not a problem since we'll check it again under the work function
2089 * while FBC is locked. This check here is just to prevent us from
2090 * unnecessarily scheduling the work, and it relies on the fact that we
2091 * never switch underrun_detect back to false after it's true.
2092 */
2093 if (READ_ONCE(fbc->underrun_detected))
2094 return;
2095
2096 queue_work(wq: display->wq.unordered, work: &fbc->underrun_work);
2097}
2098
2099/**
2100 * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
2101 * @display: display
2102 *
2103 * Without FBC, most underruns are harmless and don't really cause too many
2104 * problems, except for an annoying message on dmesg. With FBC, underruns can
2105 * become black screens or even worse, especially when paired with bad
2106 * watermarks. So in order for us to be on the safe side, completely disable FBC
2107 * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
2108 * already suggests that watermarks may be bad, so try to be as safe as
2109 * possible.
2110 *
2111 * This function is called from the IRQ handler.
2112 */
2113void intel_fbc_handle_fifo_underrun_irq(struct intel_display *display)
2114{
2115 struct intel_fbc *fbc;
2116 enum intel_fbc_id fbc_id;
2117
2118 for_each_intel_fbc(display, fbc, fbc_id)
2119 __intel_fbc_handle_fifo_underrun_irq(fbc);
2120}
2121
2122/*
2123 * The DDX driver changes its behavior depending on the value it reads from
2124 * i915.enable_fbc, so sanitize it by translating the default value into either
2125 * 0 or 1 in order to allow it to know what's going on.
2126 *
2127 * Notice that this is done at driver initialization and we still allow user
2128 * space to change the value during runtime without sanitizing it again. IGT
2129 * relies on being able to change i915.enable_fbc at runtime.
2130 */
2131static int intel_sanitize_fbc_option(struct intel_display *display)
2132{
2133 if (display->params.enable_fbc >= 0)
2134 return !!display->params.enable_fbc;
2135
2136 if (!HAS_FBC(display))
2137 return 0;
2138
2139 if (display->platform.broadwell || DISPLAY_VER(display) >= 9)
2140 return 1;
2141
2142 return 0;
2143}
2144
2145void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)
2146{
2147 plane->fbc = fbc;
2148}
2149
2150static struct intel_fbc *intel_fbc_create(struct intel_display *display,
2151 enum intel_fbc_id fbc_id)
2152{
2153 struct intel_fbc *fbc;
2154
2155 fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);
2156 if (!fbc)
2157 return NULL;
2158
2159 fbc->compressed_fb = i915_gem_stolen_node_alloc(drm: display->drm);
2160 if (!fbc->compressed_fb)
2161 goto err;
2162 fbc->compressed_llb = i915_gem_stolen_node_alloc(drm: display->drm);
2163 if (!fbc->compressed_llb)
2164 goto err;
2165
2166 fbc->id = fbc_id;
2167 fbc->display = display;
2168 INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
2169 mutex_init(&fbc->lock);
2170
2171 if (DISPLAY_VER(display) >= 7)
2172 fbc->funcs = &ivb_fbc_funcs;
2173 else if (DISPLAY_VER(display) == 6)
2174 fbc->funcs = &snb_fbc_funcs;
2175 else if (DISPLAY_VER(display) == 5)
2176 fbc->funcs = &ilk_fbc_funcs;
2177 else if (display->platform.g4x)
2178 fbc->funcs = &g4x_fbc_funcs;
2179 else if (DISPLAY_VER(display) == 4)
2180 fbc->funcs = &i965_fbc_funcs;
2181 else
2182 fbc->funcs = &i8xx_fbc_funcs;
2183
2184 return fbc;
2185
2186err:
2187 i915_gem_stolen_node_free(node: fbc->compressed_llb);
2188 i915_gem_stolen_node_free(node: fbc->compressed_fb);
2189 kfree(objp: fbc);
2190
2191 return NULL;
2192}
2193
2194/**
2195 * intel_fbc_init - Initialize FBC
2196 * @display: display
2197 *
2198 * This function might be called during PM init process.
2199 */
2200void intel_fbc_init(struct intel_display *display)
2201{
2202 enum intel_fbc_id fbc_id;
2203
2204 display->params.enable_fbc = intel_sanitize_fbc_option(display);
2205 drm_dbg_kms(display->drm, "Sanitized enable_fbc value: %d\n",
2206 display->params.enable_fbc);
2207
2208 for_each_fbc_id(display, fbc_id)
2209 display->fbc[fbc_id] = intel_fbc_create(display, fbc_id);
2210}
2211
2212/**
2213 * intel_fbc_sanitize - Sanitize FBC
2214 * @display: display
2215 *
2216 * Make sure FBC is initially disabled since we have no
2217 * idea eg. into which parts of stolen it might be scribbling
2218 * into.
2219 */
2220void intel_fbc_sanitize(struct intel_display *display)
2221{
2222 struct intel_fbc *fbc;
2223 enum intel_fbc_id fbc_id;
2224
2225 for_each_intel_fbc(display, fbc, fbc_id) {
2226 if (intel_fbc_hw_is_active(fbc))
2227 intel_fbc_hw_deactivate(fbc);
2228 }
2229}
2230
2231static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
2232{
2233 struct intel_fbc *fbc = m->private;
2234 struct intel_display *display = fbc->display;
2235 struct intel_plane *plane;
2236 struct ref_tracker *wakeref;
2237
2238 drm_modeset_lock_all(dev: display->drm);
2239
2240 wakeref = intel_display_rpm_get(display);
2241 mutex_lock(&fbc->lock);
2242
2243 if (fbc->active) {
2244 seq_puts(m, s: "FBC enabled\n");
2245 seq_printf(m, fmt: "Compressing: %s\n",
2246 str_yes_no(v: intel_fbc_is_compressing(fbc)));
2247 } else {
2248 seq_printf(m, fmt: "FBC disabled: %s\n", fbc->no_fbc_reason);
2249 }
2250
2251 for_each_intel_plane(display->drm, plane) {
2252 const struct intel_plane_state *plane_state =
2253 to_intel_plane_state(plane->base.state);
2254
2255 if (plane->fbc != fbc)
2256 continue;
2257
2258 seq_printf(m, fmt: "%c [PLANE:%d:%s]: %s\n",
2259 fbc->state.plane == plane ? '*' : ' ',
2260 plane->base.base.id, plane->base.name,
2261 plane_state->no_fbc_reason ?: "FBC possible");
2262 }
2263
2264 mutex_unlock(lock: &fbc->lock);
2265 intel_display_rpm_put(display, wakeref);
2266
2267 drm_modeset_unlock_all(dev: display->drm);
2268
2269 return 0;
2270}
2271
2272DEFINE_SHOW_ATTRIBUTE(intel_fbc_debugfs_status);
2273
2274static int intel_fbc_debugfs_false_color_get(void *data, u64 *val)
2275{
2276 struct intel_fbc *fbc = data;
2277
2278 *val = fbc->false_color;
2279
2280 return 0;
2281}
2282
2283static int intel_fbc_debugfs_false_color_set(void *data, u64 val)
2284{
2285 struct intel_fbc *fbc = data;
2286
2287 mutex_lock(&fbc->lock);
2288
2289 fbc->false_color = val;
2290
2291 if (fbc->active)
2292 fbc->funcs->set_false_color(fbc, fbc->false_color);
2293
2294 mutex_unlock(lock: &fbc->lock);
2295
2296 return 0;
2297}
2298
2299DEFINE_DEBUGFS_ATTRIBUTE(intel_fbc_debugfs_false_color_fops,
2300 intel_fbc_debugfs_false_color_get,
2301 intel_fbc_debugfs_false_color_set,
2302 "%llu\n");
2303
2304static void intel_fbc_debugfs_add(struct intel_fbc *fbc,
2305 struct dentry *parent)
2306{
2307 debugfs_create_file("i915_fbc_status", 0444, parent,
2308 fbc, &intel_fbc_debugfs_status_fops);
2309
2310 if (fbc->funcs->set_false_color)
2311 debugfs_create_file_unsafe(name: "i915_fbc_false_color", mode: 0644, parent,
2312 data: fbc, fops: &intel_fbc_debugfs_false_color_fops);
2313}
2314
2315void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc)
2316{
2317 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
2318
2319 if (plane->fbc)
2320 intel_fbc_debugfs_add(fbc: plane->fbc, parent: crtc->base.debugfs_entry);
2321}
2322
2323/* FIXME: remove this once igt is on board with per-crtc stuff */
2324void intel_fbc_debugfs_register(struct intel_display *display)
2325{
2326 struct intel_fbc *fbc;
2327
2328 fbc = display->fbc[INTEL_FBC_A];
2329 if (fbc)
2330 intel_fbc_debugfs_add(fbc, parent: display->drm->debugfs_root);
2331}
2332

source code of linux/drivers/gpu/drm/i915/display/intel_fbc.c