1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include <linux/debugfs.h>
7
8	#include <drm/drm_blend.h>
9	#include <drm/drm_print.h>
10
11	#include "soc/intel_dram.h"
12	#include "i915_reg.h"
13	#include "i9xx_wm.h"
14	#include "intel_atomic.h"
15	#include "intel_bw.h"
16	#include "intel_cdclk.h"
17	#include "intel_crtc.h"
18	#include "intel_cursor_regs.h"
19	#include "intel_de.h"
20	#include "intel_display.h"
21	#include "intel_display_power.h"
22	#include "intel_display_regs.h"
23	#include "intel_display_rpm.h"
24	#include "intel_display_types.h"
25	#include "intel_display_utils.h"
26	#include "intel_fb.h"
27	#include "intel_fixed.h"
28	#include "intel_flipq.h"
29	#include "intel_pcode.h"
30	#include "intel_plane.h"
31	#include "intel_vblank.h"
32	#include "intel_wm.h"
33	#include "skl_prefill.h"
34	#include "skl_scaler.h"
35	#include "skl_universal_plane_regs.h"
36	#include "skl_watermark.h"
37	#include "skl_watermark_regs.h"
38
39	struct intel_dbuf_state {
40	struct intel_global_state base;
41
42	struct skl_ddb_entry ddb[I915_MAX_PIPES];
43	unsigned int weight[I915_MAX_PIPES];
44	u8 slices[I915_MAX_PIPES];
45	u8 enabled_slices;
46	u8 active_pipes;
47	u8 mdclk_cdclk_ratio;
48	bool joined_mbus;
49	};
50
51	#define to_intel_dbuf_state(global_state) \
52	container_of_const((global_state), struct intel_dbuf_state, base)
53
54	#define intel_atomic_get_old_dbuf_state(state) \
55	to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_intel_display(state)->dbuf.obj))
56	#define intel_atomic_get_new_dbuf_state(state) \
57	to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_intel_display(state)->dbuf.obj))
58
59	static void skl_sagv_disable(struct intel_display *display);
60
61	/* Stores plane specific WM parameters */
62	struct skl_wm_params {
63	bool x_tiled, y_tiled;
64	bool rc_surface;
65	bool is_planar;
66	u32 width;
67	u8 cpp;
68	u32 plane_pixel_rate;
69	u32 y_min_scanlines;
70	u32 plane_bytes_per_line;
71	uint_fixed_16_16_t plane_blocks_per_line;
72	uint_fixed_16_16_t y_tile_minimum;
73	u32 linetime_us;
74	u32 dbuf_block_size;
75	};
76
77	u8 intel_enabled_dbuf_slices_mask(struct intel_display *display)
78	{
79	u8 enabled_slices = 0;
80	enum dbuf_slice slice;
81
82	for_each_dbuf_slice(display, slice) {
83	if (intel_de_read(display, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
84	enabled_slices |= BIT(slice);
85	}
86
87	return enabled_slices;
88	}
89
90	/*
91	* FIXME: We still don't have the proper code detect if we need to apply the WA,
92	* so assume we'll always need it in order to avoid underruns.
93	*/
94	static bool skl_needs_memory_bw_wa(struct intel_display *display)
95	{
96	return DISPLAY_VER(display) == 9;
97	}
98
99	bool
100	intel_has_sagv(struct intel_display *display)
101	{
102	return HAS_SAGV(display) && display->sagv.status != I915_SAGV_NOT_CONTROLLED;
103	}
104
105	static u32
106	intel_sagv_block_time(struct intel_display *display)
107	{
108	if (DISPLAY_VER(display) >= 14) {
109	u32 val;
110
111	val = intel_de_read(display, MTL_LATENCY_SAGV);
112
113	return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
114	} else if (DISPLAY_VER(display) >= 12) {
115	u32 val = 0;
116	int ret;
117
118	ret = intel_pcode_read(drm: display->drm,
119	GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
120	val: &val, NULL);
121	if (ret) {
122	drm_dbg_kms(display->drm, "Couldn't read SAGV block time!\n");
123	return 0;
124	}
125
126	return val;
127	} else if (DISPLAY_VER(display) == 11) {
128	return 10;
129	} else if (HAS_SAGV(display)) {
130	return 30;
131	} else {
132	return 0;
133	}
134	}
135
136	static void intel_sagv_init(struct intel_display *display)
137	{
138	if (!HAS_SAGV(display))
139	display->sagv.status = I915_SAGV_NOT_CONTROLLED;
140
141	/*
142	* Probe to see if we have working SAGV control.
143	* For icl+ this was already determined by intel_bw_init_hw().
144	*/
145	if (DISPLAY_VER(display) < 11)
146	skl_sagv_disable(display);
147
148	drm_WARN_ON(display->drm, display->sagv.status == I915_SAGV_UNKNOWN);
149
150	display->sagv.block_time_us = intel_sagv_block_time(display);
151
152	drm_dbg_kms(display->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
153	str_yes_no(intel_has_sagv(display)), display->sagv.block_time_us);
154
155	/* avoid overflow when adding with wm0 latency/etc. */
156	if (drm_WARN(display->drm, display->sagv.block_time_us > U16_MAX,
157	"Excessive SAGV block time %u, ignoring\n",
158	display->sagv.block_time_us))
159	display->sagv.block_time_us = 0;
160
161	if (!intel_has_sagv(display))
162	display->sagv.block_time_us = 0;
163	}
164
165	/*
166	* SAGV dynamically adjusts the system agent voltage and clock frequencies
167	* depending on power and performance requirements. The display engine access
168	* to system memory is blocked during the adjustment time. Because of the
169	* blocking time, having this enabled can cause full system hangs and/or pipe
170	* underruns if we don't meet all of the following requirements:
171	*
172	* - <= 1 pipe enabled
173	* - All planes can enable watermarks for latencies >= SAGV engine block time
174	* - We're not using an interlaced display configuration
175	*/
176	static void skl_sagv_enable(struct intel_display *display)
177	{
178	int ret;
179
180	if (!intel_has_sagv(display))
181	return;
182
183	if (display->sagv.status == I915_SAGV_ENABLED)
184	return;
185
186	drm_dbg_kms(display->drm, "Enabling SAGV\n");
187	ret = intel_pcode_write(display->drm, GEN9_PCODE_SAGV_CONTROL,
188	GEN9_SAGV_ENABLE);
189
190	/* We don't need to wait for SAGV when enabling */
191
192	/*
193	* Some skl systems, pre-release machines in particular,
194	* don't actually have SAGV.
195	*/
196	if (display->platform.skylake && ret == -ENXIO) {
197	drm_dbg(display->drm, "No SAGV found on system, ignoring\n");
198	display->sagv.status = I915_SAGV_NOT_CONTROLLED;
199	return;
200	} else if (ret < 0) {
201	drm_err(display->drm, "Failed to enable SAGV\n");
202	return;
203	}
204
205	display->sagv.status = I915_SAGV_ENABLED;
206	}
207
208	static void skl_sagv_disable(struct intel_display *display)
209	{
210	int ret;
211
212	if (!intel_has_sagv(display))
213	return;
214
215	if (display->sagv.status == I915_SAGV_DISABLED)
216	return;
217
218	drm_dbg_kms(display->drm, "Disabling SAGV\n");
219	/* bspec says to keep retrying for at least 1 ms */
220	ret = intel_pcode_request(drm: display->drm, GEN9_PCODE_SAGV_CONTROL,
221	GEN9_SAGV_DISABLE,
222	GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, timeout_base_ms: 1);
223	/*
224	* Some skl systems, pre-release machines in particular,
225	* don't actually have SAGV.
226	*/
227	if (display->platform.skylake && ret == -ENXIO) {
228	drm_dbg(display->drm, "No SAGV found on system, ignoring\n");
229	display->sagv.status = I915_SAGV_NOT_CONTROLLED;
230	return;
231	} else if (ret < 0) {
232	drm_err(display->drm, "Failed to disable SAGV (%d)\n", ret);
233	return;
234	}
235
236	display->sagv.status = I915_SAGV_DISABLED;
237	}
238
239	static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
240	{
241	struct intel_display *display = to_intel_display(state);
242	const struct intel_bw_state *new_bw_state =
243	intel_atomic_get_new_bw_state(state);
244
245	if (!new_bw_state)
246	return;
247
248	if (!intel_bw_can_enable_sagv(display, bw_state: new_bw_state))
249	skl_sagv_disable(display);
250	}
251
252	static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
253	{
254	struct intel_display *display = to_intel_display(state);
255	const struct intel_bw_state *new_bw_state =
256	intel_atomic_get_new_bw_state(state);
257
258	if (!new_bw_state)
259	return;
260
261	if (intel_bw_can_enable_sagv(display, bw_state: new_bw_state))
262	skl_sagv_enable(display);
263	}
264
265	void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
266	{
267	struct intel_display *display = to_intel_display(state);
268
269	/*
270	* Just return if we can't control SAGV or don't have it.
271	* This is different from situation when we have SAGV but just can't
272	* afford it due to DBuf limitation - in case if SAGV is completely
273	* disabled in a BIOS, we are not even allowed to send a PCode request,
274	* as it will throw an error. So have to check it here.
275	*/
276	if (!intel_has_sagv(display))
277	return;
278
279	if (DISPLAY_VER(display) >= 11)
280	icl_sagv_pre_plane_update(state);
281	else
282	skl_sagv_pre_plane_update(state);
283	}
284
285	void intel_sagv_post_plane_update(struct intel_atomic_state *state)
286	{
287	struct intel_display *display = to_intel_display(state);
288
289	/*
290	* Just return if we can't control SAGV or don't have it.
291	* This is different from situation when we have SAGV but just can't
292	* afford it due to DBuf limitation - in case if SAGV is completely
293	* disabled in a BIOS, we are not even allowed to send a PCode request,
294	* as it will throw an error. So have to check it here.
295	*/
296	if (!intel_has_sagv(display))
297	return;
298
299	if (DISPLAY_VER(display) >= 11)
300	icl_sagv_post_plane_update(state);
301	else
302	skl_sagv_post_plane_update(state);
303	}
304
305	static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
306	{
307	struct intel_display *display = to_intel_display(crtc_state);
308	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
309	enum plane_id plane_id;
310	int max_level = INT_MAX;
311
312	if (!intel_has_sagv(display))
313	return false;
314
315	if (!crtc_state->hw.active)
316	return true;
317
318	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
319	return false;
320
321	for_each_plane_id_on_crtc(crtc, plane_id) {
322	const struct skl_plane_wm *wm =
323	&crtc_state->wm.skl.optimal.planes[plane_id];
324	int level;
325
326	/* Skip this plane if it's not enabled */
327	if (!wm->wm[0].enable)
328	continue;
329
330	/* Find the highest enabled wm level for this plane */
331	for (level = display->wm.num_levels - 1;
332	!wm->wm[level].enable; --level)
333	{ }
334
335	/* Highest common enabled wm level for all planes */
336	max_level = min(level, max_level);
337	}
338
339	/* No enabled planes? */
340	if (max_level == INT_MAX)
341	return true;
342
343	for_each_plane_id_on_crtc(crtc, plane_id) {
344	const struct skl_plane_wm *wm =
345	&crtc_state->wm.skl.optimal.planes[plane_id];
346
347	/*
348	* All enabled planes must have enabled a common wm level that
349	* can tolerate memory latencies higher than sagv_block_time_us
350	*/
351	if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
352	return false;
353	}
354
355	return true;
356	}
357
358	static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
359	{
360	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
361	enum plane_id plane_id;
362
363	if (!crtc_state->hw.active)
364	return true;
365
366	for_each_plane_id_on_crtc(crtc, plane_id) {
367	const struct skl_plane_wm *wm =
368	&crtc_state->wm.skl.optimal.planes[plane_id];
369
370	if (wm->wm[0].enable && !wm->sagv.wm0.enable)
371	return false;
372	}
373
374	return true;
375	}
376
377	bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
378	{
379	struct intel_display *display = to_intel_display(crtc_state);
380
381	if (!display->params.enable_sagv)
382	return false;
383
384	/*
385	* SAGV is initially forced off because its current
386	* state can't be queried from pcode. Allow SAGV to
387	* be enabled upon the first real commit.
388	*/
389	if (crtc_state->inherited)
390	return false;
391
392	if (DISPLAY_VER(display) >= 12)
393	return tgl_crtc_can_enable_sagv(crtc_state);
394	else
395	return skl_crtc_can_enable_sagv(crtc_state);
396	}
397
398	static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
399	u16 start, u16 end)
400	{
401	entry->start = start;
402	entry->end = end;
403
404	return end;
405	}
406
407	static int intel_dbuf_slice_size(struct intel_display *display)
408	{
409	return DISPLAY_INFO(display)->dbuf.size /
410	hweight8(DISPLAY_INFO(display)->dbuf.slice_mask);
411	}
412
413	static void
414	skl_ddb_entry_for_slices(struct intel_display *display, u8 slice_mask,
415	struct skl_ddb_entry *ddb)
416	{
417	int slice_size = intel_dbuf_slice_size(display);
418
419	if (!slice_mask) {
420	ddb->start = 0;
421	ddb->end = 0;
422	return;
423	}
424
425	ddb->start = (ffs(slice_mask) - 1) * slice_size;
426	ddb->end = fls(x: slice_mask) * slice_size;
427
428	WARN_ON(ddb->start >= ddb->end);
429	WARN_ON(ddb->end > DISPLAY_INFO(display)->dbuf.size);
430	}
431
432	static unsigned int mbus_ddb_offset(struct intel_display *display, u8 slice_mask)
433	{
434	struct skl_ddb_entry ddb;
435
436	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
437	slice_mask = BIT(DBUF_S1);
438	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
439	slice_mask = BIT(DBUF_S3);
440
441	skl_ddb_entry_for_slices(display, slice_mask, ddb: &ddb);
442
443	return ddb.start;
444	}
445
446	u32 skl_ddb_dbuf_slice_mask(struct intel_display *display,
447	const struct skl_ddb_entry *entry)
448	{
449	int slice_size = intel_dbuf_slice_size(display);
450	enum dbuf_slice start_slice, end_slice;
451	u8 slice_mask = 0;
452
453	if (!skl_ddb_entry_size(entry))
454	return 0;
455
456	start_slice = entry->start / slice_size;
457	end_slice = (entry->end - 1) / slice_size;
458
459	/*
460	* Per plane DDB entry can in a really worst case be on multiple slices
461	* but single entry is anyway contiguous.
462	*/
463	while (start_slice <= end_slice) {
464	slice_mask |= BIT(start_slice);
465	start_slice++;
466	}
467
468	return slice_mask;
469	}
470
471	static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
472	{
473	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
474	int hdisplay, vdisplay;
475
476	if (!crtc_state->hw.active)
477	return 0;
478
479	/*
480	* Watermark/ddb requirement highly depends upon width of the
481	* framebuffer, So instead of allocating DDB equally among pipes
482	* distribute DDB based on resolution/width of the display.
483	*/
484	drm_mode_get_hv_timing(mode: pipe_mode, hdisplay: &hdisplay, vdisplay: &vdisplay);
485
486	return hdisplay;
487	}
488
489	static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
490	enum pipe for_pipe,
491	unsigned int *weight_start,
492	unsigned int *weight_end,
493	unsigned int *weight_total)
494	{
495	struct intel_display *display = to_intel_display(dbuf_state->base.state->base.dev);
496	enum pipe pipe;
497
498	*weight_start = 0;
499	*weight_end = 0;
500	*weight_total = 0;
501
502	for_each_pipe(display, pipe) {
503	int weight = dbuf_state->weight[pipe];
504
505	/*
506	* Do not account pipes using other slice sets
507	* luckily as of current BSpec slice sets do not partially
508	* intersect(pipes share either same one slice or same slice set
509	* i.e no partial intersection), so it is enough to check for
510	* equality for now.
511	*/
512	if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
513	continue;
514
515	*weight_total += weight;
516	if (pipe < for_pipe) {
517	*weight_start += weight;
518	*weight_end += weight;
519	} else if (pipe == for_pipe) {
520	*weight_end += weight;
521	}
522	}
523	}
524
525	static int
526	skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
527	{
528	struct intel_display *display = to_intel_display(crtc);
529	unsigned int weight_total, weight_start, weight_end;
530	const struct intel_dbuf_state *old_dbuf_state =
531	intel_atomic_get_old_dbuf_state(state);
532	struct intel_dbuf_state *new_dbuf_state =
533	intel_atomic_get_new_dbuf_state(state);
534	struct intel_crtc_state *crtc_state;
535	struct skl_ddb_entry ddb_slices;
536	enum pipe pipe = crtc->pipe;
537	unsigned int mbus_offset = 0;
538	u32 ddb_range_size;
539	u32 dbuf_slice_mask;
540	u32 start, end;
541	int ret;
542
543	if (new_dbuf_state->weight[pipe] == 0) {
544	skl_ddb_entry_init(entry: &new_dbuf_state->ddb[pipe], start: 0, end: 0);
545	goto out;
546	}
547
548	dbuf_slice_mask = new_dbuf_state->slices[pipe];
549
550	skl_ddb_entry_for_slices(display, slice_mask: dbuf_slice_mask, ddb: &ddb_slices);
551	mbus_offset = mbus_ddb_offset(display, slice_mask: dbuf_slice_mask);
552	ddb_range_size = skl_ddb_entry_size(entry: &ddb_slices);
553
554	intel_crtc_dbuf_weights(dbuf_state: new_dbuf_state, for_pipe: pipe,
555	weight_start: &weight_start, weight_end: &weight_end, weight_total: &weight_total);
556
557	start = ddb_range_size * weight_start / weight_total;
558	end = ddb_range_size * weight_end / weight_total;
559
560	skl_ddb_entry_init(entry: &new_dbuf_state->ddb[pipe],
561	start: ddb_slices.start - mbus_offset + start,
562	end: ddb_slices.start - mbus_offset + end);
563
564	out:
565	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
566	skl_ddb_entry_equal(e1: &old_dbuf_state->ddb[pipe],
567	e2: &new_dbuf_state->ddb[pipe]))
568	return 0;
569
570	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
571	if (ret)
572	return ret;
573
574	crtc_state = intel_atomic_get_crtc_state(state: &state->base, crtc);
575	if (IS_ERR(ptr: crtc_state))
576	return PTR_ERR(ptr: crtc_state);
577
578	/*
579	* Used for checking overlaps, so we need absolute
580	* offsets instead of MBUS relative offsets.
581	*/
582	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
583	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
584
585	drm_dbg_kms(display->drm,
586	"[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
587	crtc->base.base.id, crtc->base.name,
588	old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
589	old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
590	new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
591	old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
592
593	return 0;
594	}
595
596	static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
597	int width, const struct drm_format_info *format,
598	u64 modifier, unsigned int rotation,
599	u32 plane_pixel_rate, struct skl_wm_params *wp,
600	int color_plane, unsigned int pan_x);
601
602	static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
603	struct intel_plane *plane,
604	int level,
605	unsigned int latency,
606	const struct skl_wm_params *wp,
607	const struct skl_wm_level *result_prev,
608	struct skl_wm_level *result /* out */);
609
610	static unsigned int skl_wm_latency(struct intel_display *display, int level,
611	const struct skl_wm_params *wp)
612	{
613	unsigned int latency = display->wm.skl_latency[level];
614
615	if (latency == 0)
616	return 0;
617
618	/*
619	* WaIncreaseLatencyIPCEnabled: kbl,cfl
620	* Display WA #1141: kbl,cfl
621	*/
622	if ((display->platform.kabylake || display->platform.coffeelake ||
623	display->platform.cometlake) && skl_watermark_ipc_enabled(display))
624	latency += 4;
625
626	if (skl_needs_memory_bw_wa(display) && wp && wp->x_tiled)
627	latency += 15;
628
629	return latency;
630	}
631
632	static unsigned int
633	skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
634	int num_active)
635	{
636	struct intel_display *display = to_intel_display(crtc_state);
637	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
638	const struct drm_mode_config *mode_config = &display->drm->mode_config;
639	const struct drm_format_info *info;
640	struct skl_wm_level wm = {};
641	int ret, min_ddb_alloc = 0;
642	struct skl_wm_params wp;
643	u64 modifier;
644	u32 format;
645	int level;
646
647	format = DRM_FORMAT_ARGB8888;
648	modifier = DRM_FORMAT_MOD_LINEAR;
649
650	info = drm_get_format_info(dev: display->drm, pixel_format: format, modifier);
651
652	ret = skl_compute_wm_params(crtc_state, width: mode_config->cursor_width,
653	format: info, modifier, DRM_MODE_ROTATE_0,
654	plane_pixel_rate: crtc_state->pixel_rate, wp: &wp, color_plane: 0, pan_x: 0);
655	drm_WARN_ON(display->drm, ret);
656
657	for (level = 0; level < display->wm.num_levels; level++) {
658	unsigned int latency = skl_wm_latency(display, level, wp: &wp);
659
660	skl_compute_plane_wm(crtc_state, plane, level, latency, wp: &wp, result_prev: &wm, result: &wm);
661	if (wm.min_ddb_alloc == U16_MAX)
662	break;
663
664	min_ddb_alloc = wm.min_ddb_alloc;
665	}
666
667	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
668	}
669
670	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
671	{
672	skl_ddb_entry_init(entry,
673	REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
674	REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
675	if (entry->end)
676	entry->end++;
677	}
678
679	static void
680	skl_ddb_get_hw_plane_state(struct intel_display *display,
681	const enum pipe pipe,
682	const enum plane_id plane_id,
683	struct skl_ddb_entry *ddb,
684	struct skl_ddb_entry *ddb_y,
685	u16 *min_ddb, u16 *interim_ddb)
686	{
687	u32 val;
688
689	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
690	if (plane_id == PLANE_CURSOR) {
691	val = intel_de_read(display, CUR_BUF_CFG(pipe));
692	skl_ddb_entry_init_from_hw(entry: ddb, reg: val);
693	return;
694	}
695
696	val = intel_de_read(display, PLANE_BUF_CFG(pipe, plane_id));
697	skl_ddb_entry_init_from_hw(entry: ddb, reg: val);
698
699	if (DISPLAY_VER(display) >= 30) {
700	val = intel_de_read(display, PLANE_MIN_BUF_CFG(pipe, plane_id));
701
702	*min_ddb = REG_FIELD_GET(PLANE_MIN_DBUF_BLOCKS_MASK, val);
703	*interim_ddb = REG_FIELD_GET(PLANE_INTERIM_DBUF_BLOCKS_MASK, val);
704	}
705
706	if (DISPLAY_VER(display) >= 11)
707	return;
708
709	val = intel_de_read(display, PLANE_NV12_BUF_CFG(pipe, plane_id));
710	skl_ddb_entry_init_from_hw(entry: ddb_y, reg: val);
711	}
712
713	static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
714	struct skl_ddb_entry *ddb,
715	struct skl_ddb_entry *ddb_y,
716	u16 *min_ddb, u16 *interim_ddb)
717	{
718	struct intel_display *display = to_intel_display(crtc);
719	enum intel_display_power_domain power_domain;
720	enum pipe pipe = crtc->pipe;
721	intel_wakeref_t wakeref;
722	enum plane_id plane_id;
723
724	power_domain = POWER_DOMAIN_PIPE(pipe);
725	wakeref = intel_display_power_get_if_enabled(display, domain: power_domain);
726	if (!wakeref)
727	return;
728
729	for_each_plane_id_on_crtc(crtc, plane_id)
730	skl_ddb_get_hw_plane_state(display, pipe,
731	plane_id,
732	ddb: &ddb[plane_id],
733	ddb_y: &ddb_y[plane_id],
734	min_ddb: &min_ddb[plane_id],
735	interim_ddb: &interim_ddb[plane_id]);
736
737	intel_display_power_put(display, domain: power_domain, wakeref);
738	}
739
740	struct dbuf_slice_conf_entry {
741	u8 active_pipes;
742	u8 dbuf_mask[I915_MAX_PIPES];
743	bool join_mbus;
744	};
745
746	/*
747	* Table taken from Bspec 12716
748	* Pipes do have some preferred DBuf slice affinity,
749	* plus there are some hardcoded requirements on how
750	* those should be distributed for multipipe scenarios.
751	* For more DBuf slices algorithm can get even more messy
752	* and less readable, so decided to use a table almost
753	* as is from BSpec itself - that way it is at least easier
754	* to compare, change and check.
755	*/
756	static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
757	/* Autogenerated with igt/tools/intel_dbuf_map tool: */
758	{
759	{
760	.active_pipes = BIT(PIPE_A),
761	.dbuf_mask = {
762	[PIPE_A] = BIT(DBUF_S1),
763	},
764	},
765	{
766	.active_pipes = BIT(PIPE_B),
767	.dbuf_mask = {
768	[PIPE_B] = BIT(DBUF_S1),
769	},
770	},
771	{
772	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
773	.dbuf_mask = {
774	[PIPE_A] = BIT(DBUF_S1),
775	[PIPE_B] = BIT(DBUF_S2),
776	},
777	},
778	{
779	.active_pipes = BIT(PIPE_C),
780	.dbuf_mask = {
781	[PIPE_C] = BIT(DBUF_S2),
782	},
783	},
784	{
785	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
786	.dbuf_mask = {
787	[PIPE_A] = BIT(DBUF_S1),
788	[PIPE_C] = BIT(DBUF_S2),
789	},
790	},
791	{
792	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
793	.dbuf_mask = {
794	[PIPE_B] = BIT(DBUF_S1),
795	[PIPE_C] = BIT(DBUF_S2),
796	},
797	},
798	{
799	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
800	.dbuf_mask = {
801	[PIPE_A] = BIT(DBUF_S1),
802	[PIPE_B] = BIT(DBUF_S1),
803	[PIPE_C] = BIT(DBUF_S2),
804	},
805	},
806	{}
807	};
808
809	/*
810	* Table taken from Bspec 49255
811	* Pipes do have some preferred DBuf slice affinity,
812	* plus there are some hardcoded requirements on how
813	* those should be distributed for multipipe scenarios.
814	* For more DBuf slices algorithm can get even more messy
815	* and less readable, so decided to use a table almost
816	* as is from BSpec itself - that way it is at least easier
817	* to compare, change and check.
818	*/
819	static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
820	/* Autogenerated with igt/tools/intel_dbuf_map tool: */
821	{
822	{
823	.active_pipes = BIT(PIPE_A),
824	.dbuf_mask = {
825	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
826	},
827	},
828	{
829	.active_pipes = BIT(PIPE_B),
830	.dbuf_mask = {
831	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
832	},
833	},
834	{
835	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
836	.dbuf_mask = {
837	[PIPE_A] = BIT(DBUF_S2),
838	[PIPE_B] = BIT(DBUF_S1),
839	},
840	},
841	{
842	.active_pipes = BIT(PIPE_C),
843	.dbuf_mask = {
844	[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
845	},
846	},
847	{
848	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
849	.dbuf_mask = {
850	[PIPE_A] = BIT(DBUF_S1),
851	[PIPE_C] = BIT(DBUF_S2),
852	},
853	},
854	{
855	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
856	.dbuf_mask = {
857	[PIPE_B] = BIT(DBUF_S1),
858	[PIPE_C] = BIT(DBUF_S2),
859	},
860	},
861	{
862	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
863	.dbuf_mask = {
864	[PIPE_A] = BIT(DBUF_S1),
865	[PIPE_B] = BIT(DBUF_S1),
866	[PIPE_C] = BIT(DBUF_S2),
867	},
868	},
869	{
870	.active_pipes = BIT(PIPE_D),
871	.dbuf_mask = {
872	[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
873	},
874	},
875	{
876	.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
877	.dbuf_mask = {
878	[PIPE_A] = BIT(DBUF_S1),
879	[PIPE_D] = BIT(DBUF_S2),
880	},
881	},
882	{
883	.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
884	.dbuf_mask = {
885	[PIPE_B] = BIT(DBUF_S1),
886	[PIPE_D] = BIT(DBUF_S2),
887	},
888	},
889	{
890	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
891	.dbuf_mask = {
892	[PIPE_A] = BIT(DBUF_S1),
893	[PIPE_B] = BIT(DBUF_S1),
894	[PIPE_D] = BIT(DBUF_S2),
895	},
896	},
897	{
898	.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
899	.dbuf_mask = {
900	[PIPE_C] = BIT(DBUF_S1),
901	[PIPE_D] = BIT(DBUF_S2),
902	},
903	},
904	{
905	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
906	.dbuf_mask = {
907	[PIPE_A] = BIT(DBUF_S1),
908	[PIPE_C] = BIT(DBUF_S2),
909	[PIPE_D] = BIT(DBUF_S2),
910	},
911	},
912	{
913	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
914	.dbuf_mask = {
915	[PIPE_B] = BIT(DBUF_S1),
916	[PIPE_C] = BIT(DBUF_S2),
917	[PIPE_D] = BIT(DBUF_S2),
918	},
919	},
920	{
921	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
922	.dbuf_mask = {
923	[PIPE_A] = BIT(DBUF_S1),
924	[PIPE_B] = BIT(DBUF_S1),
925	[PIPE_C] = BIT(DBUF_S2),
926	[PIPE_D] = BIT(DBUF_S2),
927	},
928	},
929	{}
930	};
931
932	static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
933	{
934	.active_pipes = BIT(PIPE_A),
935	.dbuf_mask = {
936	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
937	},
938	},
939	{
940	.active_pipes = BIT(PIPE_B),
941	.dbuf_mask = {
942	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
943	},
944	},
945	{
946	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
947	.dbuf_mask = {
948	[PIPE_A] = BIT(DBUF_S1),
949	[PIPE_B] = BIT(DBUF_S2),
950	},
951	},
952	{
953	.active_pipes = BIT(PIPE_C),
954	.dbuf_mask = {
955	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
956	},
957	},
958	{
959	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
960	.dbuf_mask = {
961	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
962	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
963	},
964	},
965	{
966	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
967	.dbuf_mask = {
968	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
969	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
970	},
971	},
972	{
973	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
974	.dbuf_mask = {
975	[PIPE_A] = BIT(DBUF_S1),
976	[PIPE_B] = BIT(DBUF_S2),
977	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
978	},
979	},
980	{
981	.active_pipes = BIT(PIPE_D),
982	.dbuf_mask = {
983	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
984	},
985	},
986	{
987	.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
988	.dbuf_mask = {
989	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
990	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
991	},
992	},
993	{
994	.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
995	.dbuf_mask = {
996	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
997	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
998	},
999	},
1000	{
1001	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1002	.dbuf_mask = {
1003	[PIPE_A] = BIT(DBUF_S1),
1004	[PIPE_B] = BIT(DBUF_S2),
1005	[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1006	},
1007	},
1008	{
1009	.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1010	.dbuf_mask = {
1011	[PIPE_C] = BIT(DBUF_S3),
1012	[PIPE_D] = BIT(DBUF_S4),
1013	},
1014	},
1015	{
1016	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1017	.dbuf_mask = {
1018	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1019	[PIPE_C] = BIT(DBUF_S3),
1020	[PIPE_D] = BIT(DBUF_S4),
1021	},
1022	},
1023	{
1024	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1025	.dbuf_mask = {
1026	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1027	[PIPE_C] = BIT(DBUF_S3),
1028	[PIPE_D] = BIT(DBUF_S4),
1029	},
1030	},
1031	{
1032	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1033	.dbuf_mask = {
1034	[PIPE_A] = BIT(DBUF_S1),
1035	[PIPE_B] = BIT(DBUF_S2),
1036	[PIPE_C] = BIT(DBUF_S3),
1037	[PIPE_D] = BIT(DBUF_S4),
1038	},
1039	},
1040	{}
1041	};
1042
1043	static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1044	/*
1045	* Keep the join_mbus cases first so check_mbus_joined()
1046	* will prefer them over the !join_mbus cases.
1047	*/
1048	{
1049	.active_pipes = BIT(PIPE_A),
1050	.dbuf_mask = {
1051	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1052	},
1053	.join_mbus = true,
1054	},
1055	{
1056	.active_pipes = BIT(PIPE_B),
1057	.dbuf_mask = {
1058	[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1059	},
1060	.join_mbus = true,
1061	},
1062	{
1063	.active_pipes = BIT(PIPE_A),
1064	.dbuf_mask = {
1065	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1066	},
1067	.join_mbus = false,
1068	},
1069	{
1070	.active_pipes = BIT(PIPE_B),
1071	.dbuf_mask = {
1072	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1073	},
1074	.join_mbus = false,
1075	},
1076	{
1077	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1078	.dbuf_mask = {
1079	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1080	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1081	},
1082	},
1083	{
1084	.active_pipes = BIT(PIPE_C),
1085	.dbuf_mask = {
1086	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1087	},
1088	},
1089	{
1090	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1091	.dbuf_mask = {
1092	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1093	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1094	},
1095	},
1096	{
1097	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1098	.dbuf_mask = {
1099	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1100	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1101	},
1102	},
1103	{
1104	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1105	.dbuf_mask = {
1106	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1107	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1108	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1109	},
1110	},
1111	{
1112	.active_pipes = BIT(PIPE_D),
1113	.dbuf_mask = {
1114	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1115	},
1116	},
1117	{
1118	.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1119	.dbuf_mask = {
1120	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1121	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1122	},
1123	},
1124	{
1125	.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1126	.dbuf_mask = {
1127	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1128	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1129	},
1130	},
1131	{
1132	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1133	.dbuf_mask = {
1134	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1135	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1136	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1137	},
1138	},
1139	{
1140	.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1141	.dbuf_mask = {
1142	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1143	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1144	},
1145	},
1146	{
1147	.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1148	.dbuf_mask = {
1149	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1150	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1151	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1152	},
1153	},
1154	{
1155	.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1156	.dbuf_mask = {
1157	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1158	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1159	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1160	},
1161	},
1162	{
1163	.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1164	.dbuf_mask = {
1165	[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1166	[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1167	[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1168	[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1169	},
1170	},
1171	{}
1172
1173	};
1174
1175	static bool check_mbus_joined(u8 active_pipes,
1176	const struct dbuf_slice_conf_entry *dbuf_slices)
1177	{
1178	int i;
1179
1180	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1181	if (dbuf_slices[i].active_pipes == active_pipes)
1182	return dbuf_slices[i].join_mbus;
1183	}
1184	return false;
1185	}
1186
1187	static bool adlp_check_mbus_joined(u8 active_pipes)
1188	{
1189	return check_mbus_joined(active_pipes, dbuf_slices: adlp_allowed_dbufs);
1190	}
1191
1192	static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1193	const struct dbuf_slice_conf_entry *dbuf_slices)
1194	{
1195	int i;
1196
1197	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1198	if (dbuf_slices[i].active_pipes == active_pipes &&
1199	dbuf_slices[i].join_mbus == join_mbus)
1200	return dbuf_slices[i].dbuf_mask[pipe];
1201	}
1202	return 0;
1203	}
1204
1205	/*
1206	* This function finds an entry with same enabled pipe configuration and
1207	* returns correspondent DBuf slice mask as stated in BSpec for particular
1208	* platform.
1209	*/
1210	static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1211	{
1212	/*
1213	* FIXME: For ICL this is still a bit unclear as prev BSpec revision
1214	* required calculating "pipe ratio" in order to determine
1215	* if one or two slices can be used for single pipe configurations
1216	* as additional constraint to the existing table.
1217	* However based on recent info, it should be not "pipe ratio"
1218	* but rather ratio between pixel_rate and cdclk with additional
1219	* constants, so for now we are using only table until this is
1220	* clarified. Also this is the reason why crtc_state param is
1221	* still here - we will need it once those additional constraints
1222	* pop up.
1223	*/
1224	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1225	dbuf_slices: icl_allowed_dbufs);
1226	}
1227
1228	static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1229	{
1230	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1231	dbuf_slices: tgl_allowed_dbufs);
1232	}
1233
1234	static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1235	{
1236	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1237	dbuf_slices: adlp_allowed_dbufs);
1238	}
1239
1240	static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1241	{
1242	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1243	dbuf_slices: dg2_allowed_dbufs);
1244	}
1245
1246	static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1247	{
1248	struct intel_display *display = to_intel_display(crtc);
1249	enum pipe pipe = crtc->pipe;
1250
1251	if (display->platform.dg2)
1252	return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1253	else if (DISPLAY_VER(display) >= 13)
1254	return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1255	else if (DISPLAY_VER(display) == 12)
1256	return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1257	else if (DISPLAY_VER(display) == 11)
1258	return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1259	/*
1260	* For anything else just return one slice yet.
1261	* Should be extended for other platforms.
1262	*/
1263	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1264	}
1265
1266	static bool
1267	use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1268	struct intel_plane *plane)
1269	{
1270	struct intel_display *display = to_intel_display(plane);
1271
1272	/* Xe3+ are auto minimum DDB capble. So don't force minimal wm0 */
1273	return IS_DISPLAY_VER(display, 13, 20) &&
1274	crtc_state->uapi.async_flip &&
1275	plane->async_flip;
1276	}
1277
1278	unsigned int
1279	skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
1280	struct intel_plane *plane, int width, int height,
1281	int cpp)
1282	{
1283	/*
1284	* We calculate extra ddb based on ratio plane rate/total data rate
1285	* in case, in some cases we should not allocate extra ddb for the plane,
1286	* so do not count its data rate, if this is the case.
1287	*/
1288	if (use_minimal_wm0_only(crtc_state, plane))
1289	return 0;
1290
1291	return width * height * cpp;
1292	}
1293
1294	static u64
1295	skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1296	{
1297	struct intel_display *display = to_intel_display(crtc_state);
1298	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1299	enum plane_id plane_id;
1300	u64 data_rate = 0;
1301
1302	for_each_plane_id_on_crtc(crtc, plane_id) {
1303	if (plane_id == PLANE_CURSOR)
1304	continue;
1305
1306	data_rate += crtc_state->rel_data_rate[plane_id];
1307
1308	if (DISPLAY_VER(display) < 11)
1309	data_rate += crtc_state->rel_data_rate_y[plane_id];
1310	}
1311
1312	return data_rate;
1313	}
1314
1315	const struct skl_wm_level *
1316	skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1317	enum plane_id plane_id,
1318	int level)
1319	{
1320	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1321
1322	if (level == 0 && pipe_wm->use_sagv_wm)
1323	return &wm->sagv.wm0;
1324
1325	return &wm->wm[level];
1326	}
1327
1328	const struct skl_wm_level *
1329	skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1330	enum plane_id plane_id)
1331	{
1332	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1333
1334	if (pipe_wm->use_sagv_wm)
1335	return &wm->sagv.trans_wm;
1336
1337	return &wm->trans_wm;
1338	}
1339
1340	/*
1341	* We only disable the watermarks for each plane if
1342	* they exceed the ddb allocation of said plane. This
1343	* is done so that we don't end up touching cursor
1344	* watermarks needlessly when some other plane reduces
1345	* our max possible watermark level.
1346	*
1347	* Bspec has this to say about the PLANE_WM enable bit:
1348	* "All the watermarks at this level for all enabled
1349	* planes must be enabled before the level will be used."
1350	* So this is actually safe to do.
1351	*/
1352	static void
1353	skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1354	{
1355	if (wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb))
1356	memset(wm, 0, sizeof(*wm));
1357	}
1358
1359	static void
1360	skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1361	const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1362	{
1363	if (wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb_y) ||
1364	uv_wm->min_ddb_alloc > skl_ddb_entry_size(entry: ddb)) {
1365	memset(wm, 0, sizeof(*wm));
1366	memset(uv_wm, 0, sizeof(*uv_wm));
1367	}
1368	}
1369
1370	static bool skl_need_wm_copy_wa(struct intel_display *display, int level,
1371	const struct skl_plane_wm *wm)
1372	{
1373	/*
1374	* Wa_1408961008:icl, ehl
1375	* Wa_14012656716:tgl, adl
1376	* Wa_14017887344:icl
1377	* Wa_14017868169:adl, tgl
1378	* Due to some power saving optimizations, different subsystems
1379	* like PSR, might still use even disabled wm level registers,
1380	* for "reference", so lets keep at least the values sane.
1381	* Considering amount of WA requiring us to do similar things, was
1382	* decided to simply do it for all of the platforms, as those wm
1383	* levels are disabled, this isn't going to do harm anyway.
1384	*/
1385	return level > 0 && !wm->wm[level].enable;
1386	}
1387
1388	struct skl_plane_ddb_iter {
1389	u64 data_rate;
1390	u16 start, size;
1391	};
1392
1393	static void
1394	skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1395	struct skl_ddb_entry *ddb,
1396	const struct skl_wm_level *wm,
1397	u64 data_rate)
1398	{
1399	u16 size, extra = 0;
1400
1401	if (data_rate && iter->data_rate) {
1402	extra = min_t(u16, iter->size,
1403	DIV64_U64_ROUND_UP(iter->size * data_rate,
1404	iter->data_rate));
1405	iter->size -= extra;
1406	iter->data_rate -= data_rate;
1407	}
1408
1409	/*
1410	* Keep ddb entry of all disabled planes explicitly zeroed
1411	* to avoid skl_ddb_add_affected_planes() adding them to
1412	* the state when other planes change their allocations.
1413	*/
1414	size = wm->min_ddb_alloc + extra;
1415	if (size)
1416	iter->start = skl_ddb_entry_init(entry: ddb, start: iter->start,
1417	end: iter->start + size);
1418	}
1419
1420	static int
1421	skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1422	struct intel_crtc *crtc)
1423	{
1424	struct intel_crtc_state *crtc_state =
1425	intel_atomic_get_new_crtc_state(state, crtc);
1426	const struct intel_dbuf_state *dbuf_state =
1427	intel_atomic_get_new_dbuf_state(state);
1428	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1429	struct intel_display *display = to_intel_display(state);
1430	int num_active = hweight8(dbuf_state->active_pipes);
1431	struct skl_plane_ddb_iter iter;
1432	enum plane_id plane_id;
1433	u16 cursor_size;
1434	u32 blocks;
1435	int level;
1436
1437	/* Clear the partitioning for disabled planes. */
1438	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1439	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1440	memset(crtc_state->wm.skl.plane_min_ddb, 0,
1441	sizeof(crtc_state->wm.skl.plane_min_ddb));
1442	memset(crtc_state->wm.skl.plane_interim_ddb, 0,
1443	sizeof(crtc_state->wm.skl.plane_interim_ddb));
1444
1445	if (!crtc_state->hw.active)
1446	return 0;
1447
1448	iter.start = alloc->start;
1449	iter.size = skl_ddb_entry_size(entry: alloc);
1450	if (iter.size == 0)
1451	return 0;
1452
1453	/* Allocate fixed number of blocks for cursor. */
1454	cursor_size = skl_cursor_allocation(crtc_state, num_active);
1455	iter.size -= cursor_size;
1456	skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1457	start: alloc->end - cursor_size, end: alloc->end);
1458
1459	iter.data_rate = skl_total_relative_data_rate(crtc_state);
1460
1461	/*
1462	* Find the highest watermark level for which we can satisfy the block
1463	* requirement of active planes.
1464	*/
1465	for (level = display->wm.num_levels - 1; level >= 0; level--) {
1466	blocks = 0;
1467	for_each_plane_id_on_crtc(crtc, plane_id) {
1468	const struct skl_plane_wm *wm =
1469	&crtc_state->wm.skl.optimal.planes[plane_id];
1470
1471	if (plane_id == PLANE_CURSOR) {
1472	const struct skl_ddb_entry *ddb =
1473	&crtc_state->wm.skl.plane_ddb[plane_id];
1474
1475	if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(entry: ddb)) {
1476	drm_WARN_ON(display->drm,
1477	wm->wm[level].min_ddb_alloc != U16_MAX);
1478	blocks = U32_MAX;
1479	break;
1480	}
1481	continue;
1482	}
1483
1484	blocks += wm->wm[level].min_ddb_alloc;
1485	blocks += wm->uv_wm[level].min_ddb_alloc;
1486	}
1487
1488	if (blocks <= iter.size) {
1489	iter.size -= blocks;
1490	break;
1491	}
1492	}
1493
1494	if (level < 0) {
1495	drm_dbg_kms(display->drm,
1496	"Requested display configuration exceeds system DDB limitations");
1497	drm_dbg_kms(display->drm, "minimum required %d/%d\n",
1498	blocks, iter.size);
1499	return -EINVAL;
1500	}
1501
1502	/* avoid the WARN later when we don't allocate any extra DDB */
1503	if (iter.data_rate == 0)
1504	iter.size = 0;
1505
1506	/*
1507	* Grant each plane the blocks it requires at the highest achievable
1508	* watermark level, plus an extra share of the leftover blocks
1509	* proportional to its relative data rate.
1510	*/
1511	for_each_plane_id_on_crtc(crtc, plane_id) {
1512	struct skl_ddb_entry *ddb =
1513	&crtc_state->wm.skl.plane_ddb[plane_id];
1514	struct skl_ddb_entry *ddb_y =
1515	&crtc_state->wm.skl.plane_ddb_y[plane_id];
1516	u16 *min_ddb = &crtc_state->wm.skl.plane_min_ddb[plane_id];
1517	u16 *interim_ddb =
1518	&crtc_state->wm.skl.plane_interim_ddb[plane_id];
1519	const struct skl_plane_wm *wm =
1520	&crtc_state->wm.skl.optimal.planes[plane_id];
1521
1522	if (plane_id == PLANE_CURSOR)
1523	continue;
1524
1525	if (DISPLAY_VER(display) < 11 &&
1526	crtc_state->nv12_planes & BIT(plane_id)) {
1527	skl_allocate_plane_ddb(iter: &iter, ddb: ddb_y, wm: &wm->wm[level],
1528	data_rate: crtc_state->rel_data_rate_y[plane_id]);
1529	skl_allocate_plane_ddb(iter: &iter, ddb, wm: &wm->uv_wm[level],
1530	data_rate: crtc_state->rel_data_rate[plane_id]);
1531	} else {
1532	skl_allocate_plane_ddb(iter: &iter, ddb, wm: &wm->wm[level],
1533	data_rate: crtc_state->rel_data_rate[plane_id]);
1534	}
1535
1536	if (DISPLAY_VER(display) >= 30) {
1537	*min_ddb = wm->wm[0].min_ddb_alloc;
1538	*interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1539	}
1540	}
1541	drm_WARN_ON(display->drm, iter.size != 0 || iter.data_rate != 0);
1542
1543	/*
1544	* When we calculated watermark values we didn't know how high
1545	* of a level we'd actually be able to hit, so we just marked
1546	* all levels as "enabled." Go back now and disable the ones
1547	* that aren't actually possible.
1548	*/
1549	for (level++; level < display->wm.num_levels; level++) {
1550	for_each_plane_id_on_crtc(crtc, plane_id) {
1551	const struct skl_ddb_entry *ddb =
1552	&crtc_state->wm.skl.plane_ddb[plane_id];
1553	const struct skl_ddb_entry *ddb_y =
1554	&crtc_state->wm.skl.plane_ddb_y[plane_id];
1555	struct skl_plane_wm *wm =
1556	&crtc_state->wm.skl.optimal.planes[plane_id];
1557
1558	if (DISPLAY_VER(display) < 11 &&
1559	crtc_state->nv12_planes & BIT(plane_id))
1560	skl_check_nv12_wm_level(wm: &wm->wm[level],
1561	uv_wm: &wm->uv_wm[level],
1562	ddb_y, ddb);
1563	else
1564	skl_check_wm_level(wm: &wm->wm[level], ddb);
1565
1566	if (skl_need_wm_copy_wa(display, level, wm)) {
1567	wm->wm[level].blocks = wm->wm[level - 1].blocks;
1568	wm->wm[level].lines = wm->wm[level - 1].lines;
1569	wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1570	}
1571	}
1572	}
1573
1574	/*
1575	* Go back and disable the transition and SAGV watermarks
1576	* if it turns out we don't have enough DDB blocks for them.
1577	*/
1578	for_each_plane_id_on_crtc(crtc, plane_id) {
1579	const struct skl_ddb_entry *ddb =
1580	&crtc_state->wm.skl.plane_ddb[plane_id];
1581	const struct skl_ddb_entry *ddb_y =
1582	&crtc_state->wm.skl.plane_ddb_y[plane_id];
1583	u16 *interim_ddb =
1584	&crtc_state->wm.skl.plane_interim_ddb[plane_id];
1585	struct skl_plane_wm *wm =
1586	&crtc_state->wm.skl.optimal.planes[plane_id];
1587
1588	if (DISPLAY_VER(display) < 11 &&
1589	crtc_state->nv12_planes & BIT(plane_id)) {
1590	skl_check_wm_level(wm: &wm->trans_wm, ddb: ddb_y);
1591	} else {
1592	WARN_ON(skl_ddb_entry_size(ddb_y));
1593
1594	skl_check_wm_level(wm: &wm->trans_wm, ddb);
1595	}
1596
1597	skl_check_wm_level(wm: &wm->sagv.wm0, ddb);
1598	if (DISPLAY_VER(display) >= 30)
1599	*interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1600
1601	skl_check_wm_level(wm: &wm->sagv.trans_wm, ddb);
1602	}
1603
1604	return 0;
1605	}
1606
1607	/*
1608	* The max latency should be 257 (max the punit can code is 255 and we add 2us
1609	* for the read latency) and cpp should always be <= 8, so that
1610	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
1611	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1612	*/
1613	static uint_fixed_16_16_t
1614	skl_wm_method1(struct intel_display *display, u32 pixel_rate,
1615	u8 cpp, u32 latency, u32 dbuf_block_size)
1616	{
1617	u32 wm_intermediate_val;
1618	uint_fixed_16_16_t ret;
1619
1620	if (latency == 0)
1621	return FP_16_16_MAX;
1622
1623	wm_intermediate_val = latency * pixel_rate * cpp;
1624	ret = div_fixed16(val: wm_intermediate_val, d: 1000 * dbuf_block_size);
1625
1626	if (DISPLAY_VER(display) >= 10)
1627	ret = add_fixed16_u32(add1: ret, add2: 1);
1628
1629	return ret;
1630	}
1631
1632	static uint_fixed_16_16_t
1633	skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1634	uint_fixed_16_16_t plane_blocks_per_line)
1635	{
1636	u32 wm_intermediate_val;
1637	uint_fixed_16_16_t ret;
1638
1639	if (latency == 0)
1640	return FP_16_16_MAX;
1641
1642	wm_intermediate_val = latency * pixel_rate;
1643	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1644	pipe_htotal * 1000);
1645	ret = mul_u32_fixed16(val: wm_intermediate_val, mul: plane_blocks_per_line);
1646	return ret;
1647	}
1648
1649	static int skl_wm_linetime_us(const struct intel_crtc_state *crtc_state,
1650	int pixel_rate)
1651	{
1652	return DIV_ROUND_UP(crtc_state->hw.pipe_mode.crtc_htotal * 1000,
1653	pixel_rate);
1654	}
1655
1656	static int
1657	skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1658	int width, const struct drm_format_info *format,
1659	u64 modifier, unsigned int rotation,
1660	u32 plane_pixel_rate, struct skl_wm_params *wp,
1661	int color_plane, unsigned int pan_x)
1662	{
1663	struct intel_display *display = to_intel_display(crtc_state);
1664	u32 interm_pbpl;
1665
1666	/* only planar format has two planes */
1667	if (color_plane == 1 &&
1668	!intel_format_info_is_yuv_semiplanar(info: format, modifier)) {
1669	drm_dbg_kms(display->drm,
1670	"Non planar format have single plane\n");
1671	return -EINVAL;
1672	}
1673
1674	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1675	wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1676	intel_fb_is_tiled_modifier(modifier);
1677	wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1678	wp->is_planar = intel_format_info_is_yuv_semiplanar(info: format, modifier);
1679
1680	wp->width = width;
1681	if (color_plane == 1 && wp->is_planar)
1682	wp->width /= 2;
1683
1684	wp->cpp = format->cpp[color_plane];
1685	wp->plane_pixel_rate = plane_pixel_rate;
1686
1687	if (DISPLAY_VER(display) >= 11 &&
1688	modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
1689	wp->dbuf_block_size = 256;
1690	else
1691	wp->dbuf_block_size = 512;
1692
1693	if (drm_rotation_90_or_270(rotation)) {
1694	switch (wp->cpp) {
1695	case 1:
1696	wp->y_min_scanlines = 16;
1697	break;
1698	case 2:
1699	wp->y_min_scanlines = 8;
1700	break;
1701	case 4:
1702	wp->y_min_scanlines = 4;
1703	break;
1704	default:
1705	MISSING_CASE(wp->cpp);
1706	return -EINVAL;
1707	}
1708	} else {
1709	wp->y_min_scanlines = 4;
1710	}
1711
1712	if (skl_needs_memory_bw_wa(display))
1713	wp->y_min_scanlines *= 2;
1714
1715	wp->plane_bytes_per_line = wp->width * wp->cpp;
1716	if (wp->y_tiled) {
1717	interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1718	wp->y_min_scanlines,
1719	wp->dbuf_block_size);
1720
1721	if (DISPLAY_VER(display) >= 30)
1722	interm_pbpl += (pan_x != 0);
1723	else if (DISPLAY_VER(display) >= 10)
1724	interm_pbpl++;
1725
1726	wp->plane_blocks_per_line = div_fixed16(val: interm_pbpl,
1727	d: wp->y_min_scanlines);
1728	} else {
1729	interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1730	wp->dbuf_block_size);
1731
1732	if (!wp->x_tiled || DISPLAY_VER(display) >= 10)
1733	interm_pbpl++;
1734
1735	wp->plane_blocks_per_line = u32_to_fixed16(val: interm_pbpl);
1736	}
1737
1738	wp->y_tile_minimum = mul_u32_fixed16(val: wp->y_min_scanlines,
1739	mul: wp->plane_blocks_per_line);
1740
1741	wp->linetime_us = skl_wm_linetime_us(crtc_state, pixel_rate: plane_pixel_rate);
1742
1743	return 0;
1744	}
1745
1746	static int
1747	skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1748	const struct intel_plane_state *plane_state,
1749	struct skl_wm_params *wp, int color_plane)
1750	{
1751	const struct drm_framebuffer *fb = plane_state->hw.fb;
1752	int width;
1753
1754	/*
1755	* Src coordinates are already rotated by 270 degrees for
1756	* the 90/270 degree plane rotation cases (to match the
1757	* GTT mapping), hence no need to account for rotation here.
1758	*/
1759	width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1760
1761	return skl_compute_wm_params(crtc_state, width,
1762	format: fb->format, modifier: fb->modifier,
1763	rotation: plane_state->hw.rotation,
1764	plane_pixel_rate: intel_plane_pixel_rate(crtc_state, plane_state),
1765	wp, color_plane,
1766	pan_x: plane_state->uapi.src.x1);
1767	}
1768
1769	static bool skl_wm_has_lines(struct intel_display *display, int level)
1770	{
1771	if (DISPLAY_VER(display) >= 10)
1772	return true;
1773
1774	/* The number of lines are ignored for the level 0 watermark. */
1775	return level > 0;
1776	}
1777
1778	static int skl_wm_max_lines(struct intel_display *display)
1779	{
1780	if (DISPLAY_VER(display) >= 13)
1781	return 255;
1782	else
1783	return 31;
1784	}
1785
1786	static bool xe3_auto_min_alloc_capable(struct intel_plane *plane, int level)
1787	{
1788	struct intel_display *display = to_intel_display(plane);
1789
1790	return DISPLAY_VER(display) >= 30 && level == 0 && plane->id != PLANE_CURSOR;
1791	}
1792
1793	static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1794	struct intel_plane *plane,
1795	int level,
1796	unsigned int latency,
1797	const struct skl_wm_params *wp,
1798	const struct skl_wm_level *result_prev,
1799	struct skl_wm_level *result /* out */)
1800	{
1801	struct intel_display *display = to_intel_display(crtc_state);
1802	uint_fixed_16_16_t method1, method2;
1803	uint_fixed_16_16_t selected_result;
1804	u32 blocks, lines, min_ddb_alloc = 0;
1805
1806	if (latency == 0 ||
1807	(use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1808	/* reject it */
1809	result->min_ddb_alloc = U16_MAX;
1810	return;
1811	}
1812
1813	method1 = skl_wm_method1(display, pixel_rate: wp->plane_pixel_rate,
1814	cpp: wp->cpp, latency, dbuf_block_size: wp->dbuf_block_size);
1815	method2 = skl_wm_method2(pixel_rate: wp->plane_pixel_rate,
1816	pipe_htotal: crtc_state->hw.pipe_mode.crtc_htotal,
1817	latency,
1818	plane_blocks_per_line: wp->plane_blocks_per_line);
1819
1820	if (wp->y_tiled) {
1821	selected_result = max_fixed16(max1: method2, max2: wp->y_tile_minimum);
1822	} else if (DISPLAY_VER(display) >= 35) {
1823	selected_result = method2;
1824	} else {
1825	if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1826	wp->dbuf_block_size < 1) &&
1827	(wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1828	selected_result = method2;
1829	} else if (latency >= wp->linetime_us) {
1830	if (DISPLAY_VER(display) == 9)
1831	selected_result = min_fixed16(min1: method1, min2: method2);
1832	else
1833	selected_result = method2;
1834	} else {
1835	selected_result = method1;
1836	}
1837	}
1838
1839	blocks = fixed16_to_u32_round_up(fp: selected_result);
1840	if (DISPLAY_VER(display) < 30)
1841	blocks++;
1842
1843	/*
1844	* Lets have blocks at minimum equivalent to plane_blocks_per_line
1845	* as there will be at minimum one line for lines configuration. This
1846	* is a work around for FIFO underruns observed with resolutions like
1847	* 4k 60 Hz in single channel DRAM configurations.
1848	*
1849	* As per the Bspec 49325, if the ddb allocation can hold at least
1850	* one plane_blocks_per_line, we should have selected method2 in
1851	* the above logic. Assuming that modern versions have enough dbuf
1852	* and method2 guarantees blocks equivalent to at least 1 line,
1853	* select the blocks as plane_blocks_per_line.
1854	*
1855	* TODO: Revisit the logic when we have better understanding on DRAM
1856	* channels' impact on the level 0 memory latency and the relevant
1857	* wm calculations.
1858	*/
1859	if (skl_wm_has_lines(display, level))
1860	blocks = max(blocks,
1861	fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1862	lines = div_round_up_fixed16(val: selected_result,
1863	d: wp->plane_blocks_per_line);
1864
1865	if (DISPLAY_VER(display) == 9) {
1866	/* Display WA #1125: skl,bxt,kbl */
1867	if (level == 0 && wp->rc_surface)
1868	blocks += fixed16_to_u32_round_up(fp: wp->y_tile_minimum);
1869
1870	/* Display WA #1126: skl,bxt,kbl */
1871	if (level >= 1 && level <= 7) {
1872	if (wp->y_tiled) {
1873	blocks += fixed16_to_u32_round_up(fp: wp->y_tile_minimum);
1874	lines += wp->y_min_scanlines;
1875	} else {
1876	blocks++;
1877	}
1878	}
1879	}
1880
1881	/*
1882	* Make sure result blocks for higher latency levels are
1883	* at least as high as level below the current level.
1884	* Assumption in DDB algorithm optimization for special
1885	* cases. Also covers Display WA #1125 for RC.
1886	*
1887	* Let's always do this as the algorithm can give non
1888	* monotonic results on any platform.
1889	*/
1890	blocks = max_t(u32, blocks, result_prev->blocks);
1891	lines = max_t(u32, lines, result_prev->lines);
1892
1893	if (DISPLAY_VER(display) >= 11) {
1894	if (wp->y_tiled) {
1895	int extra_lines;
1896
1897	if (lines % wp->y_min_scanlines == 0)
1898	extra_lines = wp->y_min_scanlines;
1899	else
1900	extra_lines = wp->y_min_scanlines * 2 -
1901	lines % wp->y_min_scanlines;
1902
1903	min_ddb_alloc = mul_round_up_u32_fixed16(val: lines + extra_lines,
1904	mul: wp->plane_blocks_per_line);
1905	} else {
1906	min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1907	}
1908	}
1909
1910	if (!skl_wm_has_lines(display, level))
1911	lines = 0;
1912
1913	if (lines > skl_wm_max_lines(display)) {
1914	/* reject it */
1915	result->min_ddb_alloc = U16_MAX;
1916	return;
1917	}
1918
1919	/*
1920	* If lines is valid, assume we can use this watermark level
1921	* for now. We'll come back and disable it after we calculate the
1922	* DDB allocation if it turns out we don't actually have enough
1923	* blocks to satisfy it.
1924	*/
1925	result->blocks = blocks;
1926	result->lines = lines;
1927	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1928	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1929	result->enable = true;
1930	result->auto_min_alloc_wm_enable = xe3_auto_min_alloc_capable(plane, level);
1931
1932	if (DISPLAY_VER(display) < 12 && display->sagv.block_time_us)
1933	result->can_sagv = latency >= display->sagv.block_time_us;
1934	}
1935
1936	static void
1937	skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1938	struct intel_plane *plane,
1939	const struct skl_wm_params *wm_params,
1940	struct skl_wm_level *levels)
1941	{
1942	struct intel_display *display = to_intel_display(crtc_state);
1943	struct skl_wm_level *result_prev = &levels[0];
1944	int level;
1945
1946	for (level = 0; level < display->wm.num_levels; level++) {
1947	struct skl_wm_level *result = &levels[level];
1948	unsigned int latency = skl_wm_latency(display, level, wp: wm_params);
1949
1950	skl_compute_plane_wm(crtc_state, plane, level, latency,
1951	wp: wm_params, result_prev, result);
1952
1953	result_prev = result;
1954	}
1955	}
1956
1957	static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
1958	struct intel_plane *plane,
1959	const struct skl_wm_params *wm_params,
1960	struct skl_plane_wm *plane_wm)
1961	{
1962	struct intel_display *display = to_intel_display(crtc_state);
1963	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
1964	struct skl_wm_level *levels = plane_wm->wm;
1965	unsigned int latency = 0;
1966
1967	if (display->sagv.block_time_us)
1968	latency = display->sagv.block_time_us +
1969	skl_wm_latency(display, level: 0, wp: wm_params);
1970
1971	skl_compute_plane_wm(crtc_state, plane, level: 0, latency,
1972	wp: wm_params, result_prev: &levels[0],
1973	result: sagv_wm);
1974	}
1975
1976	static void skl_compute_transition_wm(struct intel_display *display,
1977	struct skl_wm_level *trans_wm,
1978	const struct skl_wm_level *wm0,
1979	const struct skl_wm_params *wp)
1980	{
1981	u16 trans_min, trans_amount, trans_y_tile_min;
1982	u16 wm0_blocks, trans_offset, blocks;
1983
1984	/* Transition WM don't make any sense if ipc is disabled */
1985	if (!skl_watermark_ipc_enabled(display))
1986	return;
1987
1988	/*
1989	* WaDisableTWM:skl,kbl,cfl,bxt
1990	* Transition WM are not recommended by HW team for GEN9
1991	*/
1992	if (DISPLAY_VER(display) == 9)
1993	return;
1994
1995	if (DISPLAY_VER(display) >= 11)
1996	trans_min = 4;
1997	else
1998	trans_min = 14;
1999
2000	/* Display WA #1140: glk,cnl */
2001	if (DISPLAY_VER(display) == 10)
2002	trans_amount = 0;
2003	else
2004	trans_amount = 10; /* This is configurable amount */
2005
2006	trans_offset = trans_min + trans_amount;
2007
2008	/*
2009	* The spec asks for Selected Result Blocks for wm0 (the real value),
2010	* not Result Blocks (the integer value). Pay attention to the capital
2011	* letters. The value wm_l0->blocks is actually Result Blocks, but
2012	* since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2013	* and since we later will have to get the ceiling of the sum in the
2014	* transition watermarks calculation, we can just pretend Selected
2015	* Result Blocks is Result Blocks minus 1 and it should work for the
2016	* current platforms.
2017	*/
2018	wm0_blocks = wm0->blocks - 1;
2019
2020	if (wp->y_tiled) {
2021	trans_y_tile_min =
2022	(u16)mul_round_up_u32_fixed16(val: 2, mul: wp->y_tile_minimum);
2023	blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2024	} else {
2025	blocks = wm0_blocks + trans_offset;
2026	}
2027	blocks++;
2028
2029	/*
2030	* Just assume we can enable the transition watermark. After
2031	* computing the DDB we'll come back and disable it if that
2032	* assumption turns out to be false.
2033	*/
2034	trans_wm->blocks = blocks;
2035	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2036	trans_wm->enable = true;
2037	}
2038
2039	static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2040	const struct intel_plane_state *plane_state,
2041	struct intel_plane *plane, int color_plane)
2042	{
2043	struct intel_display *display = to_intel_display(crtc_state);
2044	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2045	struct skl_wm_params wm_params;
2046	int ret;
2047
2048	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2049	wp: &wm_params, color_plane);
2050	if (ret)
2051	return ret;
2052
2053	skl_compute_wm_levels(crtc_state, plane, wm_params: &wm_params, levels: wm->wm);
2054
2055	skl_compute_transition_wm(display, trans_wm: &wm->trans_wm,
2056	wm0: &wm->wm[0], wp: &wm_params);
2057
2058	if (DISPLAY_VER(display) >= 12) {
2059	tgl_compute_sagv_wm(crtc_state, plane, wm_params: &wm_params, plane_wm: wm);
2060
2061	skl_compute_transition_wm(display, trans_wm: &wm->sagv.trans_wm,
2062	wm0: &wm->sagv.wm0, wp: &wm_params);
2063	}
2064
2065	return 0;
2066	}
2067
2068	static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2069	const struct intel_plane_state *plane_state,
2070	struct intel_plane *plane)
2071	{
2072	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2073	struct skl_wm_params wm_params;
2074	int ret;
2075
2076	wm->is_planar = true;
2077
2078	/* uv plane watermarks must also be validated for NV12/Planar */
2079	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2080	wp: &wm_params, color_plane: 1);
2081	if (ret)
2082	return ret;
2083
2084	skl_compute_wm_levels(crtc_state, plane, wm_params: &wm_params, levels: wm->uv_wm);
2085
2086	return 0;
2087	}
2088
2089	static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2090	const struct intel_plane_state *plane_state)
2091	{
2092	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2093	enum plane_id plane_id = plane->id;
2094	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2095	const struct drm_framebuffer *fb = plane_state->hw.fb;
2096	int ret;
2097
2098	memset(wm, 0, sizeof(*wm));
2099
2100	if (!intel_wm_plane_visible(crtc_state, plane_state))
2101	return 0;
2102
2103	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2104	plane, color_plane: 0);
2105	if (ret)
2106	return ret;
2107
2108	if (fb->format->is_yuv && fb->format->num_planes > 1) {
2109	ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2110	plane);
2111	if (ret)
2112	return ret;
2113	}
2114
2115	return 0;
2116	}
2117
2118	static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2119	const struct intel_plane_state *plane_state)
2120	{
2121	struct intel_display *display = to_intel_display(plane_state);
2122	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2123	enum plane_id plane_id = plane->id;
2124	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2125	int ret;
2126
2127	/* Watermarks calculated on UV plane */
2128	if (plane_state->is_y_plane)
2129	return 0;
2130
2131	memset(wm, 0, sizeof(*wm));
2132
2133	if (plane_state->planar_linked_plane) {
2134	const struct drm_framebuffer *fb = plane_state->hw.fb;
2135
2136	drm_WARN_ON(display->drm,
2137	!intel_wm_plane_visible(crtc_state, plane_state));
2138	drm_WARN_ON(display->drm, !fb->format->is_yuv ||
2139	fb->format->num_planes == 1);
2140
2141	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2142	plane: plane_state->planar_linked_plane, color_plane: 0);
2143	if (ret)
2144	return ret;
2145
2146	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2147	plane, color_plane: 1);
2148	if (ret)
2149	return ret;
2150	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2151	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2152	plane, color_plane: 0);
2153	if (ret)
2154	return ret;
2155	}
2156
2157	return 0;
2158	}
2159
2160	unsigned int skl_wm0_prefill_lines_worst(const struct intel_crtc_state *crtc_state)
2161	{
2162	struct intel_display *display = to_intel_display(crtc_state);
2163	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->primary);
2164	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
2165	int ret, pixel_rate, width, level = 0;
2166	const struct drm_format_info *info;
2167	struct skl_wm_level wm = {};
2168	struct skl_wm_params wp;
2169	unsigned int latency;
2170	u64 modifier;
2171	u32 format;
2172
2173	/* only expected to be used for VRR guardband calculation */
2174	drm_WARN_ON(display->drm, !HAS_VRR(display));
2175
2176	/* FIXME rather ugly to pick this by hand but maybe no better way? */
2177	format = DRM_FORMAT_XBGR16161616F;
2178	if (HAS_4TILE(display))
2179	modifier = I915_FORMAT_MOD_4_TILED;
2180	else
2181	modifier = I915_FORMAT_MOD_Y_TILED;
2182
2183	info = drm_get_format_info(dev: display->drm, pixel_format: format, modifier);
2184
2185	pixel_rate = DIV_ROUND_UP_ULL(mul_u32_u32(skl_scaler_max_total_scale(crtc_state),
2186	pipe_mode->crtc_clock),
2187	0x10000);
2188
2189	/* FIXME limit to max plane width? */
2190	width = DIV_ROUND_UP_ULL(mul_u32_u32(skl_scaler_max_hscale(crtc_state),
2191	pipe_mode->crtc_hdisplay),
2192	0x10000);
2193
2194	/* FIXME is 90/270 rotation worse than 0/180? */
2195	ret = skl_compute_wm_params(crtc_state, width, format: info,
2196	modifier, DRM_MODE_ROTATE_0,
2197	plane_pixel_rate: pixel_rate, wp: &wp, color_plane: 0, pan_x: 1);
2198	drm_WARN_ON(display->drm, ret);
2199
2200	latency = skl_wm_latency(display, level, wp: &wp);
2201
2202	skl_compute_plane_wm(crtc_state, plane, level, latency, wp: &wp, result_prev: &wm, result: &wm);
2203
2204	/* FIXME is this sane? */
2205	if (wm.min_ddb_alloc == U16_MAX)
2206	wm.lines = skl_wm_max_lines(display);
2207
2208	return wm.lines << 16;
2209	}
2210
2211	static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2212	{
2213	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2214	enum plane_id plane_id;
2215	int wm0_lines = 0;
2216
2217	for_each_plane_id_on_crtc(crtc, plane_id) {
2218	const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2219
2220	/* FIXME what about !skl_wm_has_lines() platforms? */
2221	wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2222	}
2223
2224	return wm0_lines;
2225	}
2226
2227	unsigned int skl_wm0_prefill_lines(const struct intel_crtc_state *crtc_state)
2228	{
2229	return skl_max_wm0_lines(crtc_state) << 16;
2230	}
2231
2232	/*
2233	* TODO: In case we use PKG_C_LATENCY to allow C-states when the delayed vblank
2234	* size is too small for the package C exit latency we need to notify PSR about
2235	* the scenario to apply Wa_16025596647.
2236	*/
2237	static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2238	const struct skl_prefill_ctx *ctx)
2239	{
2240	struct intel_display *display = to_intel_display(crtc_state);
2241	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2242	int level;
2243
2244	for (level = display->wm.num_levels - 1; level >= 0; level--) {
2245	int latency;
2246
2247	/* FIXME should we care about the latency w/a's? */
2248	latency = skl_wm_latency(display, level, NULL);
2249	if (latency == 0)
2250	continue;
2251
2252	/* FIXME is it correct to use 0 latency for wm0 here? */
2253	if (level == 0)
2254	latency = 0;
2255
2256	if (!skl_prefill_vblank_too_short(ctx, crtc_state, latency_us: latency))
2257	return level;
2258	}
2259
2260	drm_dbg_kms(display->drm, "[CRTC:%d:%s] Not enough time in vblank for prefill\n",
2261	crtc->base.base.id, crtc->base.name);
2262
2263	return -EINVAL;
2264	}
2265
2266	static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2267	{
2268	struct intel_display *display = to_intel_display(crtc_state);
2269	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2270	struct skl_prefill_ctx ctx;
2271	int level;
2272
2273	if (!crtc_state->hw.active)
2274	return 0;
2275
2276	skl_prefill_init(ctx: &ctx, crtc_state);
2277
2278	level = skl_max_wm_level_for_vblank(crtc_state, ctx: &ctx);
2279	if (level < 0)
2280	return level;
2281
2282	/*
2283	* PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2284	* based on whether we're limited by the vblank duration.
2285	*/
2286	crtc_state->wm_level_disabled = level < display->wm.num_levels - 1;
2287
2288	/*
2289	* TODO: assert that we are in fact using the maximum guardband
2290	* if we end up disabling any WM levels here. Otherwise we clearly
2291	* failed in using a realistic worst case prefill estimate when
2292	* determining the guardband size.
2293	*/
2294
2295	for (level++; level < display->wm.num_levels; level++) {
2296	enum plane_id plane_id;
2297
2298	for_each_plane_id_on_crtc(crtc, plane_id) {
2299	struct skl_plane_wm *wm =
2300	&crtc_state->wm.skl.optimal.planes[plane_id];
2301
2302	/*
2303	* FIXME just clear enable or flag the entire
2304	* thing as bad via min_ddb_alloc=U16_MAX?
2305	*/
2306	wm->wm[level].enable = false;
2307	wm->uv_wm[level].enable = false;
2308	}
2309	}
2310
2311	if (DISPLAY_VER(display) >= 12 &&
2312	display->sagv.block_time_us &&
2313	skl_prefill_vblank_too_short(ctx: &ctx, crtc_state,
2314	latency_us: display->sagv.block_time_us)) {
2315	enum plane_id plane_id;
2316
2317	for_each_plane_id_on_crtc(crtc, plane_id) {
2318	struct skl_plane_wm *wm =
2319	&crtc_state->wm.skl.optimal.planes[plane_id];
2320
2321	wm->sagv.wm0.enable = false;
2322	wm->sagv.trans_wm.enable = false;
2323	}
2324	}
2325
2326	return 0;
2327	}
2328
2329	static int skl_build_pipe_wm(struct intel_atomic_state *state,
2330	struct intel_crtc *crtc)
2331	{
2332	struct intel_display *display = to_intel_display(crtc);
2333	struct intel_crtc_state *crtc_state =
2334	intel_atomic_get_new_crtc_state(state, crtc);
2335	const struct intel_plane_state *plane_state;
2336	struct intel_plane *plane;
2337	int ret, i;
2338
2339	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2340	/*
2341	* FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2342	* instead but we don't populate that correctly for NV12 Y
2343	* planes so for now hack this.
2344	*/
2345	if (plane->pipe != crtc->pipe)
2346	continue;
2347
2348	if (DISPLAY_VER(display) >= 11)
2349	ret = icl_build_plane_wm(crtc_state, plane_state);
2350	else
2351	ret = skl_build_plane_wm(crtc_state, plane_state);
2352	if (ret)
2353	return ret;
2354	}
2355
2356	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2357
2358	return skl_wm_check_vblank(crtc_state);
2359	}
2360
2361	static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2362	const struct skl_wm_level *l2)
2363	{
2364	return l1->enable == l2->enable &&
2365	l1->ignore_lines == l2->ignore_lines &&
2366	l1->lines == l2->lines &&
2367	l1->blocks == l2->blocks &&
2368	l1->auto_min_alloc_wm_enable == l2->auto_min_alloc_wm_enable;
2369	}
2370
2371	static bool skl_plane_wm_equals(struct intel_display *display,
2372	const struct skl_plane_wm *wm1,
2373	const struct skl_plane_wm *wm2)
2374	{
2375	int level;
2376
2377	for (level = 0; level < display->wm.num_levels; level++) {
2378	/*
2379	* We don't check uv_wm as the hardware doesn't actually
2380	* use it. It only gets used for calculating the required
2381	* ddb allocation.
2382	*/
2383	if (!skl_wm_level_equals(l1: &wm1->wm[level], l2: &wm2->wm[level]))
2384	return false;
2385	}
2386
2387	return skl_wm_level_equals(l1: &wm1->trans_wm, l2: &wm2->trans_wm) &&
2388	skl_wm_level_equals(l1: &wm1->sagv.wm0, l2: &wm2->sagv.wm0) &&
2389	skl_wm_level_equals(l1: &wm1->sagv.trans_wm, l2: &wm2->sagv.trans_wm);
2390	}
2391
2392	static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2393	const struct skl_ddb_entry *b)
2394	{
2395	return a->start < b->end && b->start < a->end;
2396	}
2397
2398	static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2399	const struct skl_ddb_entry *b)
2400	{
2401	if (a->end && b->end) {
2402	a->start = min(a->start, b->start);
2403	a->end = max(a->end, b->end);
2404	} else if (b->end) {
2405	a->start = b->start;
2406	a->end = b->end;
2407	}
2408	}
2409
2410	bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2411	const struct skl_ddb_entry *entries,
2412	int num_entries, int ignore_idx)
2413	{
2414	int i;
2415
2416	for (i = 0; i < num_entries; i++) {
2417	if (i != ignore_idx &&
2418	skl_ddb_entries_overlap(a: ddb, b: &entries[i]))
2419	return true;
2420	}
2421
2422	return false;
2423	}
2424
2425	static int
2426	skl_ddb_add_affected_planes(struct intel_atomic_state *state,
2427	struct intel_crtc *crtc)
2428	{
2429	struct intel_display *display = to_intel_display(state);
2430	const struct intel_crtc_state *old_crtc_state =
2431	intel_atomic_get_old_crtc_state(state, crtc);
2432	struct intel_crtc_state *new_crtc_state =
2433	intel_atomic_get_new_crtc_state(state, crtc);
2434	struct intel_plane *plane;
2435
2436	for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2437	struct intel_plane_state *plane_state;
2438	enum plane_id plane_id = plane->id;
2439
2440	if (skl_ddb_entry_equal(e1: &old_crtc_state->wm.skl.plane_ddb[plane_id],
2441	e2: &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2442	skl_ddb_entry_equal(e1: &old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2443	e2: &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2444	continue;
2445
2446	if (new_crtc_state->do_async_flip) {
2447	drm_dbg_kms(display->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n",
2448	plane->base.base.id, plane->base.name);
2449	return -EINVAL;
2450	}
2451
2452	plane_state = intel_atomic_get_plane_state(state, plane);
2453	if (IS_ERR(ptr: plane_state))
2454	return PTR_ERR(ptr: plane_state);
2455
2456	new_crtc_state->update_planes |= BIT(plane_id);
2457	new_crtc_state->async_flip_planes = 0;
2458	new_crtc_state->do_async_flip = false;
2459	}
2460
2461	return 0;
2462	}
2463
2464	static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2465	{
2466	struct intel_display *display = to_intel_display(dbuf_state->base.state->base.dev);
2467	u8 enabled_slices;
2468	enum pipe pipe;
2469
2470	/*
2471	* FIXME: For now we always enable slice S1 as per
2472	* the Bspec display initialization sequence.
2473	*/
2474	enabled_slices = BIT(DBUF_S1);
2475
2476	for_each_pipe(display, pipe)
2477	enabled_slices |= dbuf_state->slices[pipe];
2478
2479	return enabled_slices;
2480	}
2481
2482	static int
2483	skl_compute_ddb(struct intel_atomic_state *state)
2484	{
2485	struct intel_display *display = to_intel_display(state);
2486	const struct intel_dbuf_state *old_dbuf_state;
2487	struct intel_dbuf_state *new_dbuf_state = NULL;
2488	struct intel_crtc_state *new_crtc_state;
2489	struct intel_crtc *crtc;
2490	int ret, i;
2491
2492	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2493	new_dbuf_state = intel_atomic_get_dbuf_state(state);
2494	if (IS_ERR(ptr: new_dbuf_state))
2495	return PTR_ERR(ptr: new_dbuf_state);
2496
2497	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2498	break;
2499	}
2500
2501	if (!new_dbuf_state)
2502	return 0;
2503
2504	new_dbuf_state->active_pipes =
2505	intel_calc_active_pipes(state, active_pipes: old_dbuf_state->active_pipes);
2506
2507	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2508	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2509	if (ret)
2510	return ret;
2511	}
2512
2513	if (HAS_MBUS_JOINING(display)) {
2514	new_dbuf_state->joined_mbus =
2515	adlp_check_mbus_joined(active_pipes: new_dbuf_state->active_pipes);
2516
2517	if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2518	ret = intel_cdclk_state_set_joined_mbus(state, joined_mbus: new_dbuf_state->joined_mbus);
2519	if (ret)
2520	return ret;
2521	}
2522	}
2523
2524	for_each_intel_crtc(display->drm, crtc) {
2525	enum pipe pipe = crtc->pipe;
2526
2527	new_dbuf_state->slices[pipe] =
2528	skl_compute_dbuf_slices(crtc, active_pipes: new_dbuf_state->active_pipes,
2529	join_mbus: new_dbuf_state->joined_mbus);
2530
2531	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2532	continue;
2533
2534	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2535	if (ret)
2536	return ret;
2537	}
2538
2539	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(dbuf_state: new_dbuf_state);
2540
2541	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2542	old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2543	ret = intel_atomic_serialize_global_state(obj_state: &new_dbuf_state->base);
2544	if (ret)
2545	return ret;
2546
2547	drm_dbg_kms(display->drm,
2548	"Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2549	old_dbuf_state->enabled_slices,
2550	new_dbuf_state->enabled_slices,
2551	DISPLAY_INFO(display)->dbuf.slice_mask,
2552	str_yes_no(old_dbuf_state->joined_mbus),
2553	str_yes_no(new_dbuf_state->joined_mbus));
2554	}
2555
2556	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2557	enum pipe pipe = crtc->pipe;
2558
2559	new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state: new_crtc_state);
2560
2561	if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2562	continue;
2563
2564	ret = intel_atomic_lock_global_state(obj_state: &new_dbuf_state->base);
2565	if (ret)
2566	return ret;
2567	}
2568
2569	for_each_intel_crtc(display->drm, crtc) {
2570	ret = skl_crtc_allocate_ddb(state, crtc);
2571	if (ret)
2572	return ret;
2573	}
2574
2575	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2576	ret = skl_crtc_allocate_plane_ddb(state, crtc);
2577	if (ret)
2578	return ret;
2579
2580	ret = skl_ddb_add_affected_planes(state, crtc);
2581	if (ret)
2582	return ret;
2583	}
2584
2585	return 0;
2586	}
2587
2588	static char enast(bool enable)
2589	{
2590	return enable ? '*' : ' ';
2591	}
2592
2593	static noinline_for_stack void
2594	skl_print_plane_changes(struct intel_display *display,
2595	struct intel_plane *plane,
2596	const struct skl_plane_wm *old_wm,
2597	const struct skl_plane_wm *new_wm)
2598	{
2599	drm_dbg_kms(display->drm,
2600	"[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2601	" -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2602	plane->base.base.id, plane->base.name,
2603	enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2604	enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2605	enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2606	enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2607	enast(old_wm->trans_wm.enable),
2608	enast(old_wm->sagv.wm0.enable),
2609	enast(old_wm->sagv.trans_wm.enable),
2610	enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2611	enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2612	enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2613	enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2614	enast(new_wm->trans_wm.enable),
2615	enast(new_wm->sagv.wm0.enable),
2616	enast(new_wm->sagv.trans_wm.enable));
2617
2618	drm_dbg_kms(display->drm,
2619	"[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2620	" -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2621	plane->base.base.id, plane->base.name,
2622	enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2623	enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2624	enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2625	enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2626	enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2627	enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2628	enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2629	enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2630	enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2631	enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2632	enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2633	enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2634	enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2635	enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2636	enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2637	enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2638	enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2639	enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2640	enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2641	enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2642	enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2643	enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2644
2645	drm_dbg_kms(display->drm,
2646	"[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2647	" -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2648	plane->base.base.id, plane->base.name,
2649	old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2650	old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2651	old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2652	old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2653	old_wm->trans_wm.blocks,
2654	old_wm->sagv.wm0.blocks,
2655	old_wm->sagv.trans_wm.blocks,
2656	new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2657	new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2658	new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2659	new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2660	new_wm->trans_wm.blocks,
2661	new_wm->sagv.wm0.blocks,
2662	new_wm->sagv.trans_wm.blocks);
2663
2664	drm_dbg_kms(display->drm,
2665	"[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2666	" -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2667	plane->base.base.id, plane->base.name,
2668	old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2669	old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2670	old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2671	old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2672	old_wm->trans_wm.min_ddb_alloc,
2673	old_wm->sagv.wm0.min_ddb_alloc,
2674	old_wm->sagv.trans_wm.min_ddb_alloc,
2675	new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2676	new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2677	new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2678	new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2679	new_wm->trans_wm.min_ddb_alloc,
2680	new_wm->sagv.wm0.min_ddb_alloc,
2681	new_wm->sagv.trans_wm.min_ddb_alloc);
2682	}
2683
2684	static void
2685	skl_print_wm_changes(struct intel_atomic_state *state)
2686	{
2687	struct intel_display *display = to_intel_display(state);
2688	const struct intel_crtc_state *old_crtc_state;
2689	const struct intel_crtc_state *new_crtc_state;
2690	struct intel_plane *plane;
2691	struct intel_crtc *crtc;
2692	int i;
2693
2694	if (!drm_debug_enabled(DRM_UT_KMS))
2695	return;
2696
2697	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2698	new_crtc_state, i) {
2699	const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2700
2701	old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2702	new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2703
2704	for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2705	enum plane_id plane_id = plane->id;
2706	const struct skl_ddb_entry *old, *new;
2707
2708	old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2709	new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2710
2711	if (skl_ddb_entry_equal(e1: old, e2: new))
2712	continue;
2713	drm_dbg_kms(display->drm,
2714	"[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2715	plane->base.base.id, plane->base.name,
2716	old->start, old->end, new->start, new->end,
2717	skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2718	}
2719
2720	for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2721	enum plane_id plane_id = plane->id;
2722	const struct skl_plane_wm *old_wm, *new_wm;
2723
2724	old_wm = &old_pipe_wm->planes[plane_id];
2725	new_wm = &new_pipe_wm->planes[plane_id];
2726
2727	if (skl_plane_wm_equals(display, wm1: old_wm, wm2: new_wm))
2728	continue;
2729
2730	skl_print_plane_changes(display, plane, old_wm, new_wm);
2731	}
2732	}
2733	}
2734
2735	static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2736	const struct skl_pipe_wm *old_pipe_wm,
2737	const struct skl_pipe_wm *new_pipe_wm)
2738	{
2739	struct intel_display *display = to_intel_display(plane);
2740	int level;
2741
2742	for (level = 0; level < display->wm.num_levels; level++) {
2743	/*
2744	* We don't check uv_wm as the hardware doesn't actually
2745	* use it. It only gets used for calculating the required
2746	* ddb allocation.
2747	*/
2748	if (!skl_wm_level_equals(l1: skl_plane_wm_level(pipe_wm: old_pipe_wm, plane_id: plane->id, level),
2749	l2: skl_plane_wm_level(pipe_wm: new_pipe_wm, plane_id: plane->id, level)))
2750	return false;
2751	}
2752
2753	if (HAS_HW_SAGV_WM(display)) {
2754	const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2755	const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2756
2757	if (!skl_wm_level_equals(l1: &old_wm->sagv.wm0, l2: &new_wm->sagv.wm0) ||
2758	!skl_wm_level_equals(l1: &old_wm->sagv.trans_wm, l2: &new_wm->sagv.trans_wm))
2759	return false;
2760	}
2761
2762	return skl_wm_level_equals(l1: skl_plane_trans_wm(pipe_wm: old_pipe_wm, plane_id: plane->id),
2763	l2: skl_plane_trans_wm(pipe_wm: new_pipe_wm, plane_id: plane->id));
2764	}
2765
2766	/*
2767	* To make sure the cursor watermark registers are always consistent
2768	* with our computed state the following scenario needs special
2769	* treatment:
2770	*
2771	* 1. enable cursor
2772	* 2. move cursor entirely offscreen
2773	* 3. disable cursor
2774	*
2775	* Step 2. does call .disable_plane() but does not zero the watermarks
2776	* (since we consider an offscreen cursor still active for the purposes
2777	* of watermarks). Step 3. would not normally call .disable_plane()
2778	* because the actual plane visibility isn't changing, and we don't
2779	* deallocate the cursor ddb until the pipe gets disabled. So we must
2780	* force step 3. to call .disable_plane() to update the watermark
2781	* registers properly.
2782	*
2783	* Other planes do not suffer from this issues as their watermarks are
2784	* calculated based on the actual plane visibility. The only time this
2785	* can trigger for the other planes is during the initial readout as the
2786	* default value of the watermarks registers is not zero.
2787	*/
2788	static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2789	struct intel_crtc *crtc)
2790	{
2791	struct intel_display *display = to_intel_display(state);
2792	const struct intel_crtc_state *old_crtc_state =
2793	intel_atomic_get_old_crtc_state(state, crtc);
2794	struct intel_crtc_state *new_crtc_state =
2795	intel_atomic_get_new_crtc_state(state, crtc);
2796	struct intel_plane *plane;
2797
2798	for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
2799	struct intel_plane_state *plane_state;
2800	enum plane_id plane_id = plane->id;
2801
2802	/*
2803	* Force a full wm update for every plane on modeset.
2804	* Required because the reset value of the wm registers
2805	* is non-zero, whereas we want all disabled planes to
2806	* have zero watermarks. So if we turn off the relevant
2807	* power well the hardware state will go out of sync
2808	* with the software state.
2809	*/
2810	if (!intel_crtc_needs_modeset(crtc_state: new_crtc_state) &&
2811	skl_plane_selected_wm_equals(plane,
2812	old_pipe_wm: &old_crtc_state->wm.skl.optimal,
2813	new_pipe_wm: &new_crtc_state->wm.skl.optimal))
2814	continue;
2815
2816	if (new_crtc_state->do_async_flip) {
2817	drm_dbg_kms(display->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n",
2818	plane->base.base.id, plane->base.name);
2819	return -EINVAL;
2820	}
2821
2822	plane_state = intel_atomic_get_plane_state(state, plane);
2823	if (IS_ERR(ptr: plane_state))
2824	return PTR_ERR(ptr: plane_state);
2825
2826	new_crtc_state->update_planes |= BIT(plane_id);
2827	new_crtc_state->async_flip_planes = 0;
2828	new_crtc_state->do_async_flip = false;
2829	}
2830
2831	return 0;
2832	}
2833
2834	static int pkgc_max_linetime(struct intel_atomic_state *state)
2835	{
2836	struct intel_display *display = to_intel_display(state);
2837	const struct intel_crtc_state *crtc_state;
2838	struct intel_crtc *crtc;
2839	int i, max_linetime;
2840
2841	/*
2842	* Apparenty the hardware uses WM_LINETIME internally for
2843	* this stuff, compute everything based on that.
2844	*/
2845	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2846	display->pkgc.disable[crtc->pipe] = crtc_state->vrr.enable;
2847	display->pkgc.linetime[crtc->pipe] = DIV_ROUND_UP(crtc_state->linetime, 8);
2848	}
2849
2850	max_linetime = 0;
2851	for_each_intel_crtc(display->drm, crtc) {
2852	if (display->pkgc.disable[crtc->pipe])
2853	return 0;
2854
2855	max_linetime = max(display->pkgc.linetime[crtc->pipe], max_linetime);
2856	}
2857
2858	return max_linetime;
2859	}
2860
2861	void
2862	intel_program_dpkgc_latency(struct intel_atomic_state *state)
2863	{
2864	struct intel_display *display = to_intel_display(state);
2865	int max_linetime, latency, added_wake_time = 0;
2866
2867	if (DISPLAY_VER(display) < 20)
2868	return;
2869
2870	mutex_lock(&display->wm.wm_mutex);
2871
2872	latency = skl_watermark_max_latency(display, initial_wm_level: 1);
2873
2874	/* FIXME runtime changes to enable_flipq are racy */
2875	if (display->params.enable_flipq)
2876	added_wake_time = intel_flipq_exec_time_us(display);
2877
2878	/*
2879	* Wa_22020432604
2880	* "PKG_C_LATENCY Added Wake Time field is not working"
2881	*/
2882	if (latency && IS_DISPLAY_VER(display, 20, 30)) {
2883	latency += added_wake_time;
2884	added_wake_time = 0;
2885	}
2886
2887	max_linetime = pkgc_max_linetime(state);
2888
2889	if (max_linetime == 0 || latency == 0) {
2890	latency = REG_FIELD_GET(LNL_PKG_C_LATENCY_MASK,
2891	LNL_PKG_C_LATENCY_MASK);
2892	added_wake_time = 0;
2893	} else {
2894	/*
2895	* Wa_22020299601
2896	* "Increase the latency programmed in PKG_C_LATENCY Pkg C Latency to be a
2897	* multiple of the pipeline time from WM_LINETIME"
2898	*/
2899	latency = roundup(latency, max_linetime);
2900	}
2901
2902	intel_de_write(display, LNL_PKG_C_LATENCY,
2903	REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time) |
2904	REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, latency));
2905
2906	mutex_unlock(lock: &display->wm.wm_mutex);
2907	}
2908
2909	static int
2910	skl_compute_wm(struct intel_atomic_state *state)
2911	{
2912	struct intel_display *display = to_intel_display(state);
2913	struct intel_crtc *crtc;
2914	struct intel_crtc_state __maybe_unused *new_crtc_state;
2915	int ret, i;
2916
2917	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2918	ret = skl_build_pipe_wm(state, crtc);
2919	if (ret)
2920	return ret;
2921	}
2922
2923	ret = skl_compute_ddb(state);
2924	if (ret)
2925	return ret;
2926
2927	/*
2928	* skl_compute_ddb() will have adjusted the final watermarks
2929	* based on how much ddb is available. Now we can actually
2930	* check if the final watermarks changed.
2931	*/
2932	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2933	struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
2934
2935	/*
2936	* We store use_sagv_wm in the crtc state rather than relying on
2937	* that bw state since we have no convenient way to get at the
2938	* latter from the plane commit hooks (especially in the legacy
2939	* cursor case).
2940	*
2941	* drm_atomic_check_only() gets upset if we pull more crtcs
2942	* into the state, so we have to calculate this based on the
2943	* individual intel_crtc_can_enable_sagv() rather than
2944	* the overall intel_bw_can_enable_sagv(). Otherwise the
2945	* crtcs not included in the commit would not switch to the
2946	* SAGV watermarks when we are about to enable SAGV, and that
2947	* would lead to underruns. This does mean extra power draw
2948	* when only a subset of the crtcs are blocking SAGV as the
2949	* other crtcs can't be allowed to use the more optimal
2950	* normal (ie. non-SAGV) watermarks.
2951	*/
2952	pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(display) &&
2953	DISPLAY_VER(display) >= 12 &&
2954	intel_crtc_can_enable_sagv(crtc_state: new_crtc_state);
2955
2956	ret = skl_wm_add_affected_planes(state, crtc);
2957	if (ret)
2958	return ret;
2959	}
2960
2961	skl_print_wm_changes(state);
2962
2963	return 0;
2964	}
2965
2966	static void skl_wm_level_from_reg_val(struct intel_display *display,
2967	u32 val, struct skl_wm_level *level)
2968	{
2969	level->enable = val & PLANE_WM_EN;
2970	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2971	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2972	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2973	level->auto_min_alloc_wm_enable = DISPLAY_VER(display) >= 30 ?
2974	val & PLANE_WM_AUTO_MIN_ALLOC_EN : 0;
2975	}
2976
2977	static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2978	struct skl_pipe_wm *out)
2979	{
2980	struct intel_display *display = to_intel_display(crtc);
2981	enum pipe pipe = crtc->pipe;
2982	enum plane_id plane_id;
2983	int level;
2984	u32 val;
2985
2986	for_each_plane_id_on_crtc(crtc, plane_id) {
2987	struct skl_plane_wm *wm = &out->planes[plane_id];
2988
2989	for (level = 0; level < display->wm.num_levels; level++) {
2990	if (plane_id != PLANE_CURSOR)
2991	val = intel_de_read(display, PLANE_WM(pipe, plane_id, level));
2992	else
2993	val = intel_de_read(display, CUR_WM(pipe, level));
2994
2995	skl_wm_level_from_reg_val(display, val, level: &wm->wm[level]);
2996	}
2997
2998	if (plane_id != PLANE_CURSOR)
2999	val = intel_de_read(display, PLANE_WM_TRANS(pipe, plane_id));
3000	else
3001	val = intel_de_read(display, CUR_WM_TRANS(pipe));
3002
3003	skl_wm_level_from_reg_val(display, val, level: &wm->trans_wm);
3004
3005	if (HAS_HW_SAGV_WM(display)) {
3006	if (plane_id != PLANE_CURSOR)
3007	val = intel_de_read(display, PLANE_WM_SAGV(pipe, plane_id));
3008	else
3009	val = intel_de_read(display, CUR_WM_SAGV(pipe));
3010
3011	skl_wm_level_from_reg_val(display, val, level: &wm->sagv.wm0);
3012
3013	if (plane_id != PLANE_CURSOR)
3014	val = intel_de_read(display, PLANE_WM_SAGV_TRANS(pipe, plane_id));
3015	else
3016	val = intel_de_read(display, CUR_WM_SAGV_TRANS(pipe));
3017
3018	skl_wm_level_from_reg_val(display, val, level: &wm->sagv.trans_wm);
3019	} else if (DISPLAY_VER(display) >= 12) {
3020	wm->sagv.wm0 = wm->wm[0];
3021	wm->sagv.trans_wm = wm->trans_wm;
3022	}
3023	}
3024	}
3025
3026	static void skl_wm_get_hw_state(struct intel_display *display)
3027	{
3028	struct intel_dbuf_state *dbuf_state =
3029	to_intel_dbuf_state(display->dbuf.obj.state);
3030	struct intel_crtc *crtc;
3031
3032	if (HAS_MBUS_JOINING(display))
3033	dbuf_state->joined_mbus = intel_de_read(display, MBUS_CTL) & MBUS_JOIN;
3034
3035	dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, cdclk_config: &display->cdclk.hw);
3036	dbuf_state->active_pipes = 0;
3037
3038	for_each_intel_crtc(display->drm, crtc) {
3039	struct intel_crtc_state *crtc_state =
3040	to_intel_crtc_state(crtc->base.state);
3041	enum pipe pipe = crtc->pipe;
3042	unsigned int mbus_offset;
3043	enum plane_id plane_id;
3044	u8 slices;
3045
3046	memset(&crtc_state->wm.skl.optimal, 0,
3047	sizeof(crtc_state->wm.skl.optimal));
3048	if (crtc_state->hw.active) {
3049	skl_pipe_wm_get_hw_state(crtc, out: &crtc_state->wm.skl.optimal);
3050	dbuf_state->active_pipes |= BIT(pipe);
3051	}
3052	crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3053
3054	memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3055
3056	for_each_plane_id_on_crtc(crtc, plane_id) {
3057	struct skl_ddb_entry *ddb =
3058	&crtc_state->wm.skl.plane_ddb[plane_id];
3059	struct skl_ddb_entry *ddb_y =
3060	&crtc_state->wm.skl.plane_ddb_y[plane_id];
3061	u16 *min_ddb =
3062	&crtc_state->wm.skl.plane_min_ddb[plane_id];
3063	u16 *interim_ddb =
3064	&crtc_state->wm.skl.plane_interim_ddb[plane_id];
3065
3066	if (!crtc_state->hw.active)
3067	continue;
3068
3069	skl_ddb_get_hw_plane_state(display, pipe: crtc->pipe,
3070	plane_id, ddb, ddb_y,
3071	min_ddb, interim_ddb);
3072
3073	skl_ddb_entry_union(a: &dbuf_state->ddb[pipe], b: ddb);
3074	skl_ddb_entry_union(a: &dbuf_state->ddb[pipe], b: ddb_y);
3075	}
3076
3077	dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3078
3079	/*
3080	* Used for checking overlaps, so we need absolute
3081	* offsets instead of MBUS relative offsets.
3082	*/
3083	slices = skl_compute_dbuf_slices(crtc, active_pipes: dbuf_state->active_pipes,
3084	join_mbus: dbuf_state->joined_mbus);
3085	mbus_offset = mbus_ddb_offset(display, slice_mask: slices);
3086	crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3087	crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3088
3089	/* The slices actually used by the planes on the pipe */
3090	dbuf_state->slices[pipe] =
3091	skl_ddb_dbuf_slice_mask(display, entry: &crtc_state->wm.skl.ddb);
3092
3093	drm_dbg_kms(display->drm,
3094	"[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3095	crtc->base.base.id, crtc->base.name,
3096	dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3097	dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3098	str_yes_no(dbuf_state->joined_mbus));
3099	}
3100
3101	dbuf_state->enabled_slices = display->dbuf.enabled_slices;
3102	}
3103
3104	bool skl_watermark_ipc_enabled(struct intel_display *display)
3105	{
3106	return display->wm.ipc_enabled;
3107	}
3108
3109	void skl_watermark_ipc_update(struct intel_display *display)
3110	{
3111	if (!HAS_IPC(display))
3112	return;
3113
3114	intel_de_rmw(display, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3115	set: skl_watermark_ipc_enabled(display) ? DISP_IPC_ENABLE : 0);
3116	}
3117
3118	static bool skl_watermark_ipc_can_enable(struct intel_display *display)
3119	{
3120	/* Display WA #0477 WaDisableIPC: skl */
3121	if (display->platform.skylake)
3122	return false;
3123
3124	/* Display WA #1141: SKL:all KBL:all CFL */
3125	if (display->platform.kabylake ||
3126	display->platform.coffeelake ||
3127	display->platform.cometlake) {
3128	const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
3129
3130	return dram_info->symmetric_memory;
3131	}
3132
3133	return true;
3134	}
3135
3136	void skl_watermark_ipc_init(struct intel_display *display)
3137	{
3138	if (!HAS_IPC(display))
3139	return;
3140
3141	display->wm.ipc_enabled = skl_watermark_ipc_can_enable(display);
3142
3143	skl_watermark_ipc_update(display);
3144	}
3145
3146	static void multiply_wm_latency(struct intel_display *display, int mult)
3147	{
3148	u16 *wm = display->wm.skl_latency;
3149	int level, num_levels = display->wm.num_levels;
3150
3151	for (level = 0; level < num_levels; level++)
3152	wm[level] *= mult;
3153	}
3154
3155	static void increase_wm_latency(struct intel_display *display, int inc)
3156	{
3157	u16 *wm = display->wm.skl_latency;
3158	int level, num_levels = display->wm.num_levels;
3159
3160	wm[0] += inc;
3161
3162	for (level = 1; level < num_levels; level++) {
3163	if (wm[level] == 0)
3164	break;
3165
3166	wm[level] += inc;
3167	}
3168	}
3169
3170	static bool need_16gb_dimm_wa(struct intel_display *display)
3171	{
3172	const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
3173
3174	return (display->platform.skylake || display->platform.kabylake ||
3175	display->platform.coffeelake || display->platform.cometlake ||
3176	DISPLAY_VER(display) == 11) && dram_info->has_16gb_dimms;
3177	}
3178
3179	static int wm_read_latency(struct intel_display *display)
3180	{
3181	if (DISPLAY_VER(display) >= 14)
3182	return 6;
3183	else if (DISPLAY_VER(display) >= 12)
3184	return 3;
3185	else
3186	return 2;
3187	}
3188
3189	static void sanitize_wm_latency(struct intel_display *display)
3190	{
3191	u16 *wm = display->wm.skl_latency;
3192	int level, num_levels = display->wm.num_levels;
3193
3194	/*
3195	* Xe3p and beyond should ignore level 0's reported latency and
3196	* always apply WaWmMemoryReadLatency logic.
3197	*/
3198	if (DISPLAY_VER(display) >= 35)
3199	wm[0] = 0;
3200
3201	/*
3202	* If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3203	* need to be disabled. We make sure to sanitize the values out
3204	* of the punit to satisfy this requirement.
3205	*/
3206	for (level = 1; level < num_levels; level++) {
3207	if (wm[level] == 0)
3208	break;
3209	}
3210
3211	for (level = level + 1; level < num_levels; level++)
3212	wm[level] = 0;
3213	}
3214
3215	static void make_wm_latency_monotonic(struct intel_display *display)
3216	{
3217	u16 *wm = display->wm.skl_latency;
3218	int level, num_levels = display->wm.num_levels;
3219
3220	for (level = 1; level < num_levels; level++) {
3221	if (wm[level] == 0)
3222	break;
3223
3224	wm[level] = max(wm[level], wm[level-1]);
3225	}
3226	}
3227
3228	static void
3229	adjust_wm_latency(struct intel_display *display)
3230	{
3231	u16 *wm = display->wm.skl_latency;
3232
3233	if (display->platform.dg2)
3234	multiply_wm_latency(display, mult: 2);
3235
3236	sanitize_wm_latency(display);
3237
3238	make_wm_latency_monotonic(display);
3239
3240	/*
3241	* WaWmMemoryReadLatency
3242	*
3243	* punit doesn't take into account the read latency so we need
3244	* to add proper adjustment to each valid level we retrieve
3245	* from the punit when level 0 response data is 0us.
3246	*/
3247	if (wm[0] == 0)
3248	increase_wm_latency(display, inc: wm_read_latency(display));
3249
3250	/*
3251	* WA Level-0 adjustment for 16Gb+ DIMMs: SKL+
3252	* If we could not get dimm info enable this WA to prevent from
3253	* any underrun. If not able to get DIMM info assume 16Gb+ DIMM
3254	* to avoid any underrun.
3255	*/
3256	if (need_16gb_dimm_wa(display))
3257	increase_wm_latency(display, inc: 1);
3258	}
3259
3260	static void mtl_read_wm_latency(struct intel_display *display)
3261	{
3262	u16 *wm = display->wm.skl_latency;
3263	u32 val;
3264
3265	val = intel_de_read(display, MTL_LATENCY_LP0_LP1);
3266	wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3267	wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3268
3269	val = intel_de_read(display, MTL_LATENCY_LP2_LP3);
3270	wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3271	wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3272
3273	val = intel_de_read(display, MTL_LATENCY_LP4_LP5);
3274	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3275	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3276	}
3277
3278	static void skl_read_wm_latency(struct intel_display *display)
3279	{
3280	u16 *wm = display->wm.skl_latency;
3281	u32 val;
3282	int ret;
3283
3284	/* read the first set of memory latencies[0:3] */
3285	val = 0; /* data0 to be programmed to 0 for first set */
3286	ret = intel_pcode_read(drm: display->drm, GEN9_PCODE_READ_MEM_LATENCY, val: &val, NULL);
3287	if (ret) {
3288	drm_err(display->drm, "SKL Mailbox read error = %d\n", ret);
3289	return;
3290	}
3291
3292	wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val);
3293	wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val);
3294	wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val);
3295	wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val);
3296
3297	/* read the second set of memory latencies[4:7] */
3298	val = 1; /* data0 to be programmed to 1 for second set */
3299	ret = intel_pcode_read(drm: display->drm, GEN9_PCODE_READ_MEM_LATENCY, val: &val, NULL);
3300	if (ret) {
3301	drm_err(display->drm, "SKL Mailbox read error = %d\n", ret);
3302	return;
3303	}
3304
3305	wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val);
3306	wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val);
3307	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val);
3308	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val);
3309	}
3310
3311	static void skl_setup_wm_latency(struct intel_display *display)
3312	{
3313	if (HAS_HW_SAGV_WM(display))
3314	display->wm.num_levels = 6;
3315	else
3316	display->wm.num_levels = 8;
3317
3318	if (DISPLAY_VER(display) >= 14)
3319	mtl_read_wm_latency(display);
3320	else
3321	skl_read_wm_latency(display);
3322
3323	intel_print_wm_latency(display, name: "original", wm: display->wm.skl_latency);
3324
3325	adjust_wm_latency(display);
3326
3327	intel_print_wm_latency(display, name: "adjusted", wm: display->wm.skl_latency);
3328	}
3329
3330	static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3331	{
3332	struct intel_dbuf_state *dbuf_state;
3333
3334	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3335	if (!dbuf_state)
3336	return NULL;
3337
3338	return &dbuf_state->base;
3339	}
3340
3341	static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3342	struct intel_global_state *state)
3343	{
3344	kfree(objp: state);
3345	}
3346
3347	static const struct intel_global_state_funcs intel_dbuf_funcs = {
3348	.atomic_duplicate_state = intel_dbuf_duplicate_state,
3349	.atomic_destroy_state = intel_dbuf_destroy_state,
3350	};
3351
3352	struct intel_dbuf_state *
3353	intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3354	{
3355	struct intel_display *display = to_intel_display(state);
3356	struct intel_global_state *dbuf_state;
3357
3358	dbuf_state = intel_atomic_get_global_obj_state(state, obj: &display->dbuf.obj);
3359	if (IS_ERR(ptr: dbuf_state))
3360	return ERR_CAST(ptr: dbuf_state);
3361
3362	return to_intel_dbuf_state(dbuf_state);
3363	}
3364
3365	int intel_dbuf_init(struct intel_display *display)
3366	{
3367	struct intel_dbuf_state *dbuf_state;
3368
3369	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3370	if (!dbuf_state)
3371	return -ENOMEM;
3372
3373	intel_atomic_global_obj_init(display, obj: &display->dbuf.obj,
3374	state: &dbuf_state->base, funcs: &intel_dbuf_funcs);
3375
3376	return 0;
3377	}
3378
3379	static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3380	{
3381	switch (pipe) {
3382	case PIPE_A:
3383	case PIPE_D:
3384	active_pipes &= BIT(PIPE_A) | BIT(PIPE_D);
3385	break;
3386	case PIPE_B:
3387	case PIPE_C:
3388	active_pipes &= BIT(PIPE_B) | BIT(PIPE_C);
3389	break;
3390	default: /* to suppress compiler warning */
3391	MISSING_CASE(pipe);
3392	return false;
3393	}
3394
3395	return is_power_of_2(n: active_pipes);
3396	}
3397
3398	static u32 pipe_mbus_dbox_ctl(const struct intel_crtc *crtc,
3399	const struct intel_dbuf_state *dbuf_state)
3400	{
3401	struct intel_display *display = to_intel_display(crtc);
3402	u32 val = 0;
3403
3404	if (DISPLAY_VER(display) >= 14)
3405	val |= MBUS_DBOX_I_CREDIT(2);
3406
3407	if (DISPLAY_VER(display) >= 12) {
3408	val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3409	val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3410	val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3411	}
3412
3413	if (DISPLAY_VER(display) >= 14)
3414	val |= dbuf_state->joined_mbus ?
3415	MBUS_DBOX_A_CREDIT(12) : MBUS_DBOX_A_CREDIT(8);
3416	else if (display->platform.alderlake_p)
3417	/* Wa_22010947358:adl-p */
3418	val |= dbuf_state->joined_mbus ?
3419	MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
3420	else
3421	val |= MBUS_DBOX_A_CREDIT(2);
3422
3423	if (DISPLAY_VER(display) >= 14) {
3424	val |= MBUS_DBOX_B_CREDIT(0xA);
3425	} else if (display->platform.alderlake_p) {
3426	val |= MBUS_DBOX_BW_CREDIT(2);
3427	val |= MBUS_DBOX_B_CREDIT(8);
3428	} else if (DISPLAY_VER(display) >= 12) {
3429	val |= MBUS_DBOX_BW_CREDIT(2);
3430	val |= MBUS_DBOX_B_CREDIT(12);
3431	} else {
3432	val |= MBUS_DBOX_BW_CREDIT(1);
3433	val |= MBUS_DBOX_B_CREDIT(8);
3434	}
3435
3436	if (DISPLAY_VERx100(display) == 1400) {
3437	if (xelpdp_is_only_pipe_per_dbuf_bank(pipe: crtc->pipe, active_pipes: dbuf_state->active_pipes))
3438	val |= MBUS_DBOX_BW_8CREDITS_MTL;
3439	else
3440	val |= MBUS_DBOX_BW_4CREDITS_MTL;
3441	}
3442
3443	return val;
3444	}
3445
3446	static void pipe_mbus_dbox_ctl_update(struct intel_display *display,
3447	const struct intel_dbuf_state *dbuf_state)
3448	{
3449	struct intel_crtc *crtc;
3450
3451	for_each_intel_crtc_in_pipe_mask(display->drm, crtc, dbuf_state->active_pipes)
3452	intel_de_write(display, PIPE_MBUS_DBOX_CTL(crtc->pipe),
3453	val: pipe_mbus_dbox_ctl(crtc, dbuf_state));
3454	}
3455
3456	static void intel_mbus_dbox_update(struct intel_atomic_state *state)
3457	{
3458	struct intel_display *display = to_intel_display(state);
3459	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3460
3461	if (DISPLAY_VER(display) < 11)
3462	return;
3463
3464	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3465	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3466	if (!new_dbuf_state ||
3467	(new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3468	new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3469	return;
3470
3471	pipe_mbus_dbox_ctl_update(display, dbuf_state: new_dbuf_state);
3472	}
3473
3474	int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
3475	int ratio)
3476	{
3477	struct intel_dbuf_state *dbuf_state;
3478
3479	dbuf_state = intel_atomic_get_dbuf_state(state);
3480	if (IS_ERR(ptr: dbuf_state))
3481	return PTR_ERR(ptr: dbuf_state);
3482
3483	dbuf_state->mdclk_cdclk_ratio = ratio;
3484
3485	return intel_atomic_lock_global_state(obj_state: &dbuf_state->base);
3486	}
3487
3488	void intel_dbuf_mdclk_cdclk_ratio_update(struct intel_display *display,
3489	int ratio, bool joined_mbus)
3490	{
3491	enum dbuf_slice slice;
3492
3493	if (!HAS_MBUS_JOINING(display))
3494	return;
3495
3496	if (DISPLAY_VER(display) >= 35)
3497	intel_de_rmw(display, MBUS_CTL, XE3P_MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3498	XE3P_MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3499	else if (DISPLAY_VER(display) >= 20)
3500	intel_de_rmw(display, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3501	MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3502
3503	if (joined_mbus)
3504	ratio *= 2;
3505
3506	drm_dbg_kms(display->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
3507	ratio, str_yes_no(joined_mbus));
3508
3509	for_each_dbuf_slice(display, slice)
3510	if (DISPLAY_VER(display) >= 35)
3511	intel_de_rmw(display, DBUF_CTL_S(slice),
3512	XE3P_DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3513	XE3P_DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3514	else
3515	intel_de_rmw(display, DBUF_CTL_S(slice),
3516	DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3517	DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3518	}
3519
3520	static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
3521	{
3522	struct intel_display *display = to_intel_display(state);
3523	const struct intel_dbuf_state *old_dbuf_state =
3524	intel_atomic_get_old_dbuf_state(state);
3525	const struct intel_dbuf_state *new_dbuf_state =
3526	intel_atomic_get_new_dbuf_state(state);
3527	int mdclk_cdclk_ratio;
3528
3529	if (intel_cdclk_is_decreasing_later(state)) {
3530	/* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
3531	mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
3532	} else {
3533	/* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
3534	mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
3535	}
3536
3537	intel_dbuf_mdclk_cdclk_ratio_update(display, ratio: mdclk_cdclk_ratio,
3538	joined_mbus: new_dbuf_state->joined_mbus);
3539	}
3540
3541	static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
3542	const struct intel_dbuf_state *dbuf_state)
3543	{
3544	struct intel_display *display = to_intel_display(state);
3545	enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
3546	const struct intel_crtc_state *new_crtc_state;
3547	struct intel_crtc *crtc;
3548
3549	drm_WARN_ON(display->drm, !dbuf_state->joined_mbus);
3550	drm_WARN_ON(display->drm, !is_power_of_2(dbuf_state->active_pipes));
3551
3552	crtc = intel_crtc_for_pipe(display, pipe);
3553	new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
3554
3555	if (new_crtc_state && !intel_crtc_needs_modeset(crtc_state: new_crtc_state))
3556	return pipe;
3557	else
3558	return INVALID_PIPE;
3559	}
3560
3561	static void mbus_ctl_join_update(struct intel_display *display,
3562	const struct intel_dbuf_state *dbuf_state,
3563	enum pipe pipe)
3564	{
3565	u32 mbus_ctl;
3566
3567	if (dbuf_state->joined_mbus)
3568	mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
3569	else
3570	mbus_ctl = MBUS_HASHING_MODE_2x2;
3571
3572	if (pipe != INVALID_PIPE)
3573	mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
3574	else
3575	mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
3576
3577	intel_de_rmw(display, MBUS_CTL,
3578	MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3579	MBUS_JOIN_PIPE_SELECT_MASK, set: mbus_ctl);
3580	}
3581
3582	static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
3583	enum pipe pipe)
3584	{
3585	struct intel_display *display = to_intel_display(state);
3586	const struct intel_dbuf_state *old_dbuf_state =
3587	intel_atomic_get_old_dbuf_state(state);
3588	const struct intel_dbuf_state *new_dbuf_state =
3589	intel_atomic_get_new_dbuf_state(state);
3590
3591	drm_dbg_kms(display->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
3592	str_yes_no(old_dbuf_state->joined_mbus),
3593	str_yes_no(new_dbuf_state->joined_mbus),
3594	pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
3595
3596	mbus_ctl_join_update(display, dbuf_state: new_dbuf_state, pipe);
3597	}
3598
3599	void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
3600	{
3601	const struct intel_dbuf_state *new_dbuf_state =
3602	intel_atomic_get_new_dbuf_state(state);
3603	const struct intel_dbuf_state *old_dbuf_state =
3604	intel_atomic_get_old_dbuf_state(state);
3605
3606	if (!new_dbuf_state)
3607	return;
3608
3609	if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
3610	enum pipe pipe = intel_mbus_joined_pipe(state, dbuf_state: new_dbuf_state);
3611
3612	WARN_ON(!new_dbuf_state->base.changed);
3613
3614	intel_dbuf_mbus_join_update(state, pipe);
3615	intel_mbus_dbox_update(state);
3616	intel_dbuf_mdclk_min_tracker_update(state);
3617	}
3618	}
3619
3620	void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
3621	{
3622	struct intel_display *display = to_intel_display(state);
3623	const struct intel_dbuf_state *new_dbuf_state =
3624	intel_atomic_get_new_dbuf_state(state);
3625	const struct intel_dbuf_state *old_dbuf_state =
3626	intel_atomic_get_old_dbuf_state(state);
3627
3628	if (!new_dbuf_state)
3629	return;
3630
3631	if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
3632	enum pipe pipe = intel_mbus_joined_pipe(state, dbuf_state: old_dbuf_state);
3633
3634	WARN_ON(!new_dbuf_state->base.changed);
3635
3636	intel_dbuf_mdclk_min_tracker_update(state);
3637	intel_mbus_dbox_update(state);
3638	intel_dbuf_mbus_join_update(state, pipe);
3639
3640	if (pipe != INVALID_PIPE) {
3641	struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
3642
3643	intel_crtc_wait_for_next_vblank(crtc);
3644	}
3645	} else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
3646	old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
3647	WARN_ON(!new_dbuf_state->base.changed);
3648
3649	intel_dbuf_mdclk_min_tracker_update(state);
3650	intel_mbus_dbox_update(state);
3651	}
3652
3653	}
3654
3655	void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3656	{
3657	struct intel_display *display = to_intel_display(state);
3658	const struct intel_dbuf_state *new_dbuf_state =
3659	intel_atomic_get_new_dbuf_state(state);
3660	const struct intel_dbuf_state *old_dbuf_state =
3661	intel_atomic_get_old_dbuf_state(state);
3662	u8 old_slices, new_slices;
3663
3664	if (!new_dbuf_state)
3665	return;
3666
3667	old_slices = old_dbuf_state->enabled_slices;
3668	new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3669
3670	if (old_slices == new_slices)
3671	return;
3672
3673	WARN_ON(!new_dbuf_state->base.changed);
3674
3675	gen9_dbuf_slices_update(display, req_slices: new_slices);
3676	}
3677
3678	void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3679	{
3680	struct intel_display *display = to_intel_display(state);
3681	const struct intel_dbuf_state *new_dbuf_state =
3682	intel_atomic_get_new_dbuf_state(state);
3683	const struct intel_dbuf_state *old_dbuf_state =
3684	intel_atomic_get_old_dbuf_state(state);
3685	u8 old_slices, new_slices;
3686
3687	if (!new_dbuf_state)
3688	return;
3689
3690	old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3691	new_slices = new_dbuf_state->enabled_slices;
3692
3693	if (old_slices == new_slices)
3694	return;
3695
3696	WARN_ON(!new_dbuf_state->base.changed);
3697
3698	gen9_dbuf_slices_update(display, req_slices: new_slices);
3699	}
3700
3701	int intel_dbuf_num_enabled_slices(const struct intel_dbuf_state *dbuf_state)
3702	{
3703	return hweight8(dbuf_state->enabled_slices);
3704	}
3705
3706	int intel_dbuf_num_active_pipes(const struct intel_dbuf_state *dbuf_state)
3707	{
3708	return hweight8(dbuf_state->active_pipes);
3709	}
3710
3711	bool intel_dbuf_pmdemand_needs_update(struct intel_atomic_state *state)
3712	{
3713	struct intel_display *display = to_intel_display(state);
3714	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3715
3716	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3717	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3718
3719	if (new_dbuf_state &&
3720	new_dbuf_state->active_pipes != old_dbuf_state->active_pipes)
3721	return true;
3722
3723	if (DISPLAY_VER(display) < 30) {
3724	if (new_dbuf_state &&
3725	new_dbuf_state->enabled_slices !=
3726	old_dbuf_state->enabled_slices)
3727	return true;
3728	}
3729
3730	return false;
3731	}
3732
3733	static void skl_mbus_sanitize(struct intel_display *display)
3734	{
3735	struct intel_dbuf_state *dbuf_state =
3736	to_intel_dbuf_state(display->dbuf.obj.state);
3737
3738	if (!HAS_MBUS_JOINING(display))
3739	return;
3740
3741	if (!dbuf_state->joined_mbus ||
3742	adlp_check_mbus_joined(active_pipes: dbuf_state->active_pipes))
3743	return;
3744
3745	drm_dbg_kms(display->drm, "Disabling redundant MBUS joining (active pipes 0x%x)\n",
3746	dbuf_state->active_pipes);
3747
3748	dbuf_state->joined_mbus = false;
3749	intel_dbuf_mdclk_cdclk_ratio_update(display,
3750	ratio: dbuf_state->mdclk_cdclk_ratio,
3751	joined_mbus: dbuf_state->joined_mbus);
3752	pipe_mbus_dbox_ctl_update(display, dbuf_state);
3753	mbus_ctl_join_update(display, dbuf_state, pipe: INVALID_PIPE);
3754	}
3755
3756	static bool skl_dbuf_is_misconfigured(struct intel_display *display)
3757	{
3758	const struct intel_dbuf_state *dbuf_state =
3759	to_intel_dbuf_state(display->dbuf.obj.state);
3760	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3761	struct intel_crtc *crtc;
3762
3763	for_each_intel_crtc(display->drm, crtc) {
3764	const struct intel_crtc_state *crtc_state =
3765	to_intel_crtc_state(crtc->base.state);
3766
3767	entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3768	}
3769
3770	for_each_intel_crtc(display->drm, crtc) {
3771	const struct intel_crtc_state *crtc_state =
3772	to_intel_crtc_state(crtc->base.state);
3773	u8 slices;
3774
3775	slices = skl_compute_dbuf_slices(crtc, active_pipes: dbuf_state->active_pipes,
3776	join_mbus: dbuf_state->joined_mbus);
3777	if (dbuf_state->slices[crtc->pipe] & ~slices)
3778	return true;
3779
3780	if (skl_ddb_allocation_overlaps(ddb: &crtc_state->wm.skl.ddb, entries,
3781	num_entries: I915_MAX_PIPES, ignore_idx: crtc->pipe))
3782	return true;
3783	}
3784
3785	return false;
3786	}
3787
3788	static void skl_dbuf_sanitize(struct intel_display *display)
3789	{
3790	struct intel_crtc *crtc;
3791
3792	/*
3793	* On TGL/RKL (at least) the BIOS likes to assign the planes
3794	* to the wrong DBUF slices. This will cause an infinite loop
3795	* in skl_commit_modeset_enables() as it can't find a way to
3796	* transition between the old bogus DBUF layout to the new
3797	* proper DBUF layout without DBUF allocation overlaps between
3798	* the planes (which cannot be allowed or else the hardware
3799	* may hang). If we detect a bogus DBUF layout just turn off
3800	* all the planes so that skl_commit_modeset_enables() can
3801	* simply ignore them.
3802	*/
3803	if (!skl_dbuf_is_misconfigured(display))
3804	return;
3805
3806	drm_dbg_kms(display->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3807
3808	for_each_intel_crtc(display->drm, crtc) {
3809	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3810	const struct intel_plane_state *plane_state =
3811	to_intel_plane_state(plane->base.state);
3812	struct intel_crtc_state *crtc_state =
3813	to_intel_crtc_state(crtc->base.state);
3814
3815	if (plane_state->uapi.visible)
3816	intel_plane_disable_noatomic(crtc, plane);
3817
3818	drm_WARN_ON(display->drm, crtc_state->active_planes != 0);
3819
3820	memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3821	}
3822	}
3823
3824	static void skl_wm_sanitize(struct intel_display *display)
3825	{
3826	skl_mbus_sanitize(display);
3827	skl_dbuf_sanitize(display);
3828	}
3829
3830	void skl_wm_crtc_disable_noatomic(struct intel_crtc *crtc)
3831	{
3832	struct intel_display *display = to_intel_display(crtc);
3833	struct intel_crtc_state *crtc_state =
3834	to_intel_crtc_state(crtc->base.state);
3835	struct intel_dbuf_state *dbuf_state =
3836	to_intel_dbuf_state(display->dbuf.obj.state);
3837	enum pipe pipe = crtc->pipe;
3838
3839	if (DISPLAY_VER(display) < 9)
3840	return;
3841
3842	dbuf_state->active_pipes &= ~BIT(pipe);
3843
3844	dbuf_state->weight[pipe] = 0;
3845	dbuf_state->slices[pipe] = 0;
3846
3847	memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3848
3849	memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3850	}
3851
3852	void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc,
3853	struct intel_plane *plane)
3854	{
3855	struct intel_display *display = to_intel_display(crtc);
3856	struct intel_crtc_state *crtc_state =
3857	to_intel_crtc_state(crtc->base.state);
3858
3859	if (DISPLAY_VER(display) < 9)
3860	return;
3861
3862	skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[plane->id], start: 0, end: 0);
3863	skl_ddb_entry_init(entry: &crtc_state->wm.skl.plane_ddb[plane->id], start: 0, end: 0);
3864
3865	crtc_state->wm.skl.plane_min_ddb[plane->id] = 0;
3866	crtc_state->wm.skl.plane_interim_ddb[plane->id] = 0;
3867
3868	memset(&crtc_state->wm.skl.raw.planes[plane->id], 0,
3869	sizeof(crtc_state->wm.skl.raw.planes[plane->id]));
3870	memset(&crtc_state->wm.skl.optimal.planes[plane->id], 0,
3871	sizeof(crtc_state->wm.skl.optimal.planes[plane->id]));
3872	}
3873
3874	void intel_wm_state_verify(struct intel_atomic_state *state,
3875	struct intel_crtc *crtc)
3876	{
3877	struct intel_display *display = to_intel_display(state);
3878	const struct intel_crtc_state *new_crtc_state =
3879	intel_atomic_get_new_crtc_state(state, crtc);
3880	struct skl_hw_state {
3881	struct skl_ddb_entry ddb[I915_MAX_PLANES];
3882	struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3883	u16 min_ddb[I915_MAX_PLANES];
3884	u16 interim_ddb[I915_MAX_PLANES];
3885	struct skl_pipe_wm wm;
3886	} *hw;
3887	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3888	struct intel_plane *plane;
3889	u8 hw_enabled_slices;
3890	int level;
3891
3892	if (DISPLAY_VER(display) < 9 || !new_crtc_state->hw.active)
3893	return;
3894
3895	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3896	if (!hw)
3897	return;
3898
3899	skl_pipe_wm_get_hw_state(crtc, out: &hw->wm);
3900
3901	skl_pipe_ddb_get_hw_state(crtc, ddb: hw->ddb, ddb_y: hw->ddb_y, min_ddb: hw->min_ddb, interim_ddb: hw->interim_ddb);
3902
3903	hw_enabled_slices = intel_enabled_dbuf_slices_mask(display);
3904
3905	if (DISPLAY_VER(display) >= 11 &&
3906	hw_enabled_slices != display->dbuf.enabled_slices)
3907	drm_err(display->drm,
3908	"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3909	display->dbuf.enabled_slices,
3910	hw_enabled_slices);
3911
3912	for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
3913	const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3914	const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3915
3916	/* Watermarks */
3917	for (level = 0; level < display->wm.num_levels; level++) {
3918	hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3919	sw_wm_level = skl_plane_wm_level(pipe_wm: sw_wm, plane_id: plane->id, level);
3920
3921	if (skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level))
3922	continue;
3923
3924	drm_err(display->drm,
3925	"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3926	plane->base.base.id, plane->base.name, level,
3927	sw_wm_level->enable,
3928	sw_wm_level->blocks,
3929	sw_wm_level->lines,
3930	hw_wm_level->enable,
3931	hw_wm_level->blocks,
3932	hw_wm_level->lines);
3933	}
3934
3935	hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3936	sw_wm_level = skl_plane_trans_wm(pipe_wm: sw_wm, plane_id: plane->id);
3937
3938	if (!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3939	drm_err(display->drm,
3940	"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3941	plane->base.base.id, plane->base.name,
3942	sw_wm_level->enable,
3943	sw_wm_level->blocks,
3944	sw_wm_level->lines,
3945	hw_wm_level->enable,
3946	hw_wm_level->blocks,
3947	hw_wm_level->lines);
3948	}
3949
3950	hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3951	sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3952
3953	if (HAS_HW_SAGV_WM(display) &&
3954	!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3955	drm_err(display->drm,
3956	"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3957	plane->base.base.id, plane->base.name,
3958	sw_wm_level->enable,
3959	sw_wm_level->blocks,
3960	sw_wm_level->lines,
3961	hw_wm_level->enable,
3962	hw_wm_level->blocks,
3963	hw_wm_level->lines);
3964	}
3965
3966	hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3967	sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3968
3969	if (HAS_HW_SAGV_WM(display) &&
3970	!skl_wm_level_equals(l1: hw_wm_level, l2: sw_wm_level)) {
3971	drm_err(display->drm,
3972	"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3973	plane->base.base.id, plane->base.name,
3974	sw_wm_level->enable,
3975	sw_wm_level->blocks,
3976	sw_wm_level->lines,
3977	hw_wm_level->enable,
3978	hw_wm_level->blocks,
3979	hw_wm_level->lines);
3980	}
3981
3982	/* DDB */
3983	hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3984	sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3985
3986	if (!skl_ddb_entry_equal(e1: hw_ddb_entry, e2: sw_ddb_entry)) {
3987	drm_err(display->drm,
3988	"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3989	plane->base.base.id, plane->base.name,
3990	sw_ddb_entry->start, sw_ddb_entry->end,
3991	hw_ddb_entry->start, hw_ddb_entry->end);
3992	}
3993	}
3994
3995	kfree(objp: hw);
3996	}
3997
3998	static const struct intel_wm_funcs skl_wm_funcs = {
3999	.compute_global_watermarks = skl_compute_wm,
4000	.get_hw_state = skl_wm_get_hw_state,
4001	.sanitize = skl_wm_sanitize,
4002	};
4003
4004	void skl_wm_init(struct intel_display *display)
4005	{
4006	intel_sagv_init(display);
4007
4008	skl_setup_wm_latency(display);
4009
4010	display->funcs.wm = &skl_wm_funcs;
4011	}
4012
4013	static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
4014	{
4015	struct intel_display *display = m->private;
4016
4017	seq_printf(m, fmt: "Isochronous Priority Control: %s\n",
4018	str_yes_no(v: skl_watermark_ipc_enabled(display)));
4019	return 0;
4020	}
4021
4022	static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
4023	{
4024	struct intel_display *display = inode->i_private;
4025
4026	return single_open(file, skl_watermark_ipc_status_show, display);
4027	}
4028
4029	static ssize_t skl_watermark_ipc_status_write(struct file *file,
4030	const char __user *ubuf,
4031	size_t len, loff_t *offp)
4032	{
4033	struct seq_file *m = file->private_data;
4034	struct intel_display *display = m->private;
4035	bool enable;
4036	int ret;
4037
4038	ret = kstrtobool_from_user(s: ubuf, count: len, res: &enable);
4039	if (ret < 0)
4040	return ret;
4041
4042	with_intel_display_rpm(display) {
4043	if (!skl_watermark_ipc_enabled(display) && enable)
4044	drm_info(display->drm,
4045	"Enabling IPC: WM will be proper only after next commit\n");
4046	display->wm.ipc_enabled = enable;
4047	skl_watermark_ipc_update(display);
4048	}
4049
4050	return len;
4051	}
4052
4053	static const struct file_operations skl_watermark_ipc_status_fops = {
4054	.owner = THIS_MODULE,
4055	.open = skl_watermark_ipc_status_open,
4056	.read = seq_read,
4057	.llseek = seq_lseek,
4058	.release = single_release,
4059	.write = skl_watermark_ipc_status_write
4060	};
4061
4062	static int intel_sagv_status_show(struct seq_file *m, void *unused)
4063	{
4064	struct intel_display *display = m->private;
4065	static const char * const sagv_status[] = {
4066	[I915_SAGV_UNKNOWN] = "unknown",
4067	[I915_SAGV_DISABLED] = "disabled",
4068	[I915_SAGV_ENABLED] = "enabled",
4069	[I915_SAGV_NOT_CONTROLLED] = "not controlled",
4070	};
4071
4072	seq_printf(m, fmt: "SAGV available: %s\n", str_yes_no(v: intel_has_sagv(display)));
4073	seq_printf(m, fmt: "SAGV modparam: %s\n",
4074	str_enabled_disabled(v: display->params.enable_sagv));
4075	seq_printf(m, fmt: "SAGV status: %s\n", sagv_status[display->sagv.status]);
4076	seq_printf(m, fmt: "SAGV block time: %d usec\n", display->sagv.block_time_us);
4077
4078	return 0;
4079	}
4080
4081	DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
4082
4083	void skl_watermark_debugfs_register(struct intel_display *display)
4084	{
4085	struct dentry *debugfs_root = display->drm->debugfs_root;
4086
4087	if (HAS_IPC(display))
4088	debugfs_create_file("i915_ipc_status", 0644, debugfs_root,
4089	display, &skl_watermark_ipc_status_fops);
4090
4091	if (HAS_SAGV(display))
4092	debugfs_create_file("i915_sagv_status", 0444, debugfs_root,
4093	display, &intel_sagv_status_fops);
4094	}
4095
4096	unsigned int skl_watermark_max_latency(struct intel_display *display, int initial_wm_level)
4097	{
4098	int level;
4099
4100	for (level = display->wm.num_levels - 1; level >= initial_wm_level; level--) {
4101	unsigned int latency = skl_wm_latency(display, level, NULL);
4102
4103	if (latency)
4104	return latency;
4105	}
4106
4107	return 0;
4108	}
4109