1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2023 Intel Corporation
4	*/
5
6	#include <linux/iopoll.h>
7
8	#include <drm/drm_print.h>
9
10	#include "soc/intel_dram.h"
11
12	#include "i915_drv.h"
13	#include "i915_reg.h"
14	#include "i9xx_wm.h"
15	#include "i9xx_wm_regs.h"
16	#include "intel_atomic.h"
17	#include "intel_bo.h"
18	#include "intel_de.h"
19	#include "intel_display.h"
20	#include "intel_display_regs.h"
21	#include "intel_display_trace.h"
22	#include "intel_fb.h"
23	#include "intel_mchbar_regs.h"
24	#include "intel_wm.h"
25	#include "skl_watermark.h"
26	#include "vlv_sideband.h"
27
28	struct intel_watermark_params {
29	u16 fifo_size;
30	u16 max_wm;
31	u8 default_wm;
32	u8 guard_size;
33	u8 cacheline_size;
34	};
35
36	/* used in computing the new watermarks state */
37	struct intel_wm_config {
38	unsigned int num_pipes_active;
39	bool sprites_enabled;
40	bool sprites_scaled;
41	};
42
43	struct cxsr_latency {
44	bool is_desktop : 1;
45	bool is_ddr3 : 1;
46	u16 fsb_freq;
47	u16 mem_freq;
48	u16 display_sr;
49	u16 display_hpll_disable;
50	u16 cursor_sr;
51	u16 cursor_hpll_disable;
52	};
53
54	static const struct cxsr_latency cxsr_latency_table[] = {
55	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
56	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
57	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
58	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
59	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
60
61	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
62	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
63	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
64	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
65	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
66
67	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
68	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
69	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
70	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
71	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
72
73	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
74	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
75	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
76	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
77	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
78
79	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
80	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
81	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
82	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
83	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
84
85	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
86	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
87	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
88	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
89	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
90	};
91
92	static const struct cxsr_latency *pnv_get_cxsr_latency(struct intel_display *display)
93	{
94	const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
95	bool is_ddr3 = dram_info->type == INTEL_DRAM_DDR3;
96	int i;
97
98	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
99	const struct cxsr_latency *latency = &cxsr_latency_table[i];
100	bool is_desktop = !display->platform.mobile;
101
102	if (is_desktop == latency->is_desktop &&
103	is_ddr3 == latency->is_ddr3 &&
104	DIV_ROUND_CLOSEST(dram_info->fsb_freq, 1000) == latency->fsb_freq &&
105	DIV_ROUND_CLOSEST(dram_info->mem_freq, 1000) == latency->mem_freq)
106	return latency;
107	}
108
109	drm_dbg_kms(display->drm,
110	"Could not find CxSR latency for %s, FSB %u kHz, MEM %u kHz\n",
111	intel_dram_type_str(dram_info->type),
112	dram_info->fsb_freq, dram_info->mem_freq);
113
114	return NULL;
115	}
116
117	static void chv_set_memory_dvfs(struct intel_display *display, bool enable)
118	{
119	u32 val;
120	int ret;
121
122	vlv_punit_get(drm: display->drm);
123
124	val = vlv_punit_read(drm: display->drm, PUNIT_REG_DDR_SETUP2);
125	if (enable)
126	val &= ~FORCE_DDR_HIGH_FREQ;
127	else
128	val |= FORCE_DDR_HIGH_FREQ;
129	val &= ~FORCE_DDR_LOW_FREQ;
130	val |= FORCE_DDR_FREQ_REQ_ACK;
131	vlv_punit_write(drm: display->drm, PUNIT_REG_DDR_SETUP2, val);
132
133	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
134	(val & FORCE_DDR_FREQ_REQ_ACK) == 0,
135	500, 3000, false);
136	if (ret)
137	drm_err(display->drm,
138	"timed out waiting for Punit DDR DVFS request\n");
139
140	vlv_punit_put(drm: display->drm);
141	}
142
143	static void chv_set_memory_pm5(struct intel_display *display, bool enable)
144	{
145	u32 val;
146
147	vlv_punit_get(drm: display->drm);
148
149	val = vlv_punit_read(drm: display->drm, PUNIT_REG_DSPSSPM);
150	if (enable)
151	val |= DSP_MAXFIFO_PM5_ENABLE;
152	else
153	val &= ~DSP_MAXFIFO_PM5_ENABLE;
154	vlv_punit_write(drm: display->drm, PUNIT_REG_DSPSSPM, val);
155
156	vlv_punit_put(drm: display->drm);
157	}
158
159	#define FW_WM(value, plane) \
160	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
161
162	static bool _intel_set_memory_cxsr(struct intel_display *display, bool enable)
163	{
164	bool was_enabled;
165	u32 val;
166
167	if (display->platform.valleyview || display->platform.cherryview) {
168	was_enabled = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
169	intel_de_write(display, FW_BLC_SELF_VLV, val: enable ? FW_CSPWRDWNEN : 0);
170	intel_de_posting_read(display, FW_BLC_SELF_VLV);
171	} else if (display->platform.g4x || display->platform.i965gm) {
172	was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
173	intel_de_write(display, FW_BLC_SELF, val: enable ? FW_BLC_SELF_EN : 0);
174	intel_de_posting_read(display, FW_BLC_SELF);
175	} else if (display->platform.pineview) {
176	val = intel_de_read(display, DSPFW3(display));
177	was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
178	if (enable)
179	val |= PINEVIEW_SELF_REFRESH_EN;
180	else
181	val &= ~PINEVIEW_SELF_REFRESH_EN;
182	intel_de_write(display, DSPFW3(display), val);
183	intel_de_posting_read(display, DSPFW3(display));
184	} else if (display->platform.i945g || display->platform.i945gm) {
185	was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
186	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
187	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
188	intel_de_write(display, FW_BLC_SELF, val);
189	intel_de_posting_read(display, FW_BLC_SELF);
190	} else if (display->platform.i915gm) {
191	/*
192	* FIXME can't find a bit like this for 915G, and
193	* yet it does have the related watermark in
194	* FW_BLC_SELF. What's going on?
195	*/
196	was_enabled = intel_de_read(display, INSTPM) & INSTPM_SELF_EN;
197	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
198	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
199	intel_de_write(display, INSTPM, val);
200	intel_de_posting_read(display, INSTPM);
201	} else {
202	return false;
203	}
204
205	trace_intel_memory_cxsr(display, old: was_enabled, new: enable);
206
207	drm_dbg_kms(display->drm, "memory self-refresh is %s (was %s)\n",
208	str_enabled_disabled(enable),
209	str_enabled_disabled(was_enabled));
210
211	return was_enabled;
212	}
213
214	/**
215	* intel_set_memory_cxsr - Configure CxSR state
216	* @display: display device
217	* @enable: Allow vs. disallow CxSR
218	*
219	* Allow or disallow the system to enter a special CxSR
220	* (C-state self refresh) state. What typically happens in CxSR mode
221	* is that several display FIFOs may get combined into a single larger
222	* FIFO for a particular plane (so called max FIFO mode) to allow the
223	* system to defer memory fetches longer, and the memory will enter
224	* self refresh.
225	*
226	* Note that enabling CxSR does not guarantee that the system enter
227	* this special mode, nor does it guarantee that the system stays
228	* in that mode once entered. So this just allows/disallows the system
229	* to autonomously utilize the CxSR mode. Other factors such as core
230	* C-states will affect when/if the system actually enters/exits the
231	* CxSR mode.
232	*
233	* Note that on VLV/CHV this actually only controls the max FIFO mode,
234	* and the system is free to enter/exit memory self refresh at any time
235	* even when the use of CxSR has been disallowed.
236	*
237	* While the system is actually in the CxSR/max FIFO mode, some plane
238	* control registers will not get latched on vblank. Thus in order to
239	* guarantee the system will respond to changes in the plane registers
240	* we must always disallow CxSR prior to making changes to those registers.
241	* Unfortunately the system will re-evaluate the CxSR conditions at
242	* frame start which happens after vblank start (which is when the plane
243	* registers would get latched), so we can't proceed with the plane update
244	* during the same frame where we disallowed CxSR.
245	*
246	* Certain platforms also have a deeper HPLL SR mode. Fortunately the
247	* HPLL SR mode depends on CxSR itself, so we don't have to hand hold
248	* the hardware w.r.t. HPLL SR when writing to plane registers.
249	* Disallowing just CxSR is sufficient.
250	*/
251	bool intel_set_memory_cxsr(struct intel_display *display, bool enable)
252	{
253	bool ret;
254
255	mutex_lock(&display->wm.wm_mutex);
256	ret = _intel_set_memory_cxsr(display, enable);
257	if (display->platform.valleyview || display->platform.cherryview)
258	display->wm.vlv.cxsr = enable;
259	else if (display->platform.g4x)
260	display->wm.g4x.cxsr = enable;
261	mutex_unlock(lock: &display->wm.wm_mutex);
262
263	return ret;
264	}
265
266	/*
267	* Latency for FIFO fetches is dependent on several factors:
268	* - memory configuration (speed, channels)
269	* - chipset
270	* - current MCH state
271	* It can be fairly high in some situations, so here we assume a fairly
272	* pessimal value. It's a tradeoff between extra memory fetches (if we
273	* set this value too high, the FIFO will fetch frequently to stay full)
274	* and power consumption (set it too low to save power and we might see
275	* FIFO underruns and display "flicker").
276	*
277	* A value of 5us seems to be a good balance; safe for very low end
278	* platforms but not overly aggressive on lower latency configs.
279	*/
280	static const int pessimal_latency_ns = 5000;
281
282	#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
283	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
284
285	static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
286	{
287	struct intel_display *display = to_intel_display(crtc_state);
288	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
289	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
290	enum pipe pipe = crtc->pipe;
291	int sprite0_start, sprite1_start;
292	u32 dsparb, dsparb2, dsparb3;
293
294	switch (pipe) {
295	case PIPE_A:
296	dsparb = intel_de_read(display, DSPARB(display));
297	dsparb2 = intel_de_read(display, DSPARB2);
298	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
299	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
300	break;
301	case PIPE_B:
302	dsparb = intel_de_read(display, DSPARB(display));
303	dsparb2 = intel_de_read(display, DSPARB2);
304	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
305	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
306	break;
307	case PIPE_C:
308	dsparb2 = intel_de_read(display, DSPARB2);
309	dsparb3 = intel_de_read(display, DSPARB3);
310	sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
311	sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
312	break;
313	default:
314	MISSING_CASE(pipe);
315	return;
316	}
317
318	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
319	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
320	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
321	fifo_state->plane[PLANE_CURSOR] = 63;
322	}
323
324	static int i9xx_get_fifo_size(struct intel_display *display,
325	enum i9xx_plane_id i9xx_plane)
326	{
327	u32 dsparb = intel_de_read(display, DSPARB(display));
328	int size;
329
330	size = dsparb & 0x7f;
331	if (i9xx_plane == PLANE_B)
332	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
333
334	drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
335	dsparb, plane_name(i9xx_plane), size);
336
337	return size;
338	}
339
340	static int i830_get_fifo_size(struct intel_display *display,
341	enum i9xx_plane_id i9xx_plane)
342	{
343	u32 dsparb = intel_de_read(display, DSPARB(display));
344	int size;
345
346	size = dsparb & 0x1ff;
347	if (i9xx_plane == PLANE_B)
348	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
349	size >>= 1; /* Convert to cachelines */
350
351	drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
352	dsparb, plane_name(i9xx_plane), size);
353
354	return size;
355	}
356
357	static int i845_get_fifo_size(struct intel_display *display,
358	enum i9xx_plane_id i9xx_plane)
359	{
360	u32 dsparb = intel_de_read(display, DSPARB(display));
361	int size;
362
363	size = dsparb & 0x7f;
364	size >>= 2; /* Convert to cachelines */
365
366	drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
367	dsparb, plane_name(i9xx_plane), size);
368
369	return size;
370	}
371
372	/* Pineview has different values for various configs */
373	static const struct intel_watermark_params pnv_display_wm = {
374	.fifo_size = PINEVIEW_DISPLAY_FIFO,
375	.max_wm = PINEVIEW_MAX_WM,
376	.default_wm = PINEVIEW_DFT_WM,
377	.guard_size = PINEVIEW_GUARD_WM,
378	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
379	};
380
381	static const struct intel_watermark_params pnv_display_hplloff_wm = {
382	.fifo_size = PINEVIEW_DISPLAY_FIFO,
383	.max_wm = PINEVIEW_MAX_WM,
384	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
385	.guard_size = PINEVIEW_GUARD_WM,
386	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
387	};
388
389	static const struct intel_watermark_params pnv_cursor_wm = {
390	.fifo_size = PINEVIEW_CURSOR_FIFO,
391	.max_wm = PINEVIEW_CURSOR_MAX_WM,
392	.default_wm = PINEVIEW_CURSOR_DFT_WM,
393	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
394	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
395	};
396
397	static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
398	.fifo_size = PINEVIEW_CURSOR_FIFO,
399	.max_wm = PINEVIEW_CURSOR_MAX_WM,
400	.default_wm = PINEVIEW_CURSOR_DFT_WM,
401	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
402	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
403	};
404
405	static const struct intel_watermark_params i965_cursor_wm_info = {
406	.fifo_size = I965_CURSOR_FIFO,
407	.max_wm = I965_CURSOR_MAX_WM,
408	.default_wm = I965_CURSOR_DFT_WM,
409	.guard_size = 2,
410	.cacheline_size = I915_FIFO_LINE_SIZE,
411	};
412
413	static const struct intel_watermark_params i945_wm_info = {
414	.fifo_size = I945_FIFO_SIZE,
415	.max_wm = I915_MAX_WM,
416	.default_wm = 1,
417	.guard_size = 2,
418	.cacheline_size = I915_FIFO_LINE_SIZE,
419	};
420
421	static const struct intel_watermark_params i915_wm_info = {
422	.fifo_size = I915_FIFO_SIZE,
423	.max_wm = I915_MAX_WM,
424	.default_wm = 1,
425	.guard_size = 2,
426	.cacheline_size = I915_FIFO_LINE_SIZE,
427	};
428
429	static const struct intel_watermark_params i830_a_wm_info = {
430	.fifo_size = I855GM_FIFO_SIZE,
431	.max_wm = I915_MAX_WM,
432	.default_wm = 1,
433	.guard_size = 2,
434	.cacheline_size = I830_FIFO_LINE_SIZE,
435	};
436
437	static const struct intel_watermark_params i830_bc_wm_info = {
438	.fifo_size = I855GM_FIFO_SIZE,
439	.max_wm = I915_MAX_WM / 2,
440	.default_wm = 1,
441	.guard_size = 2,
442	.cacheline_size = I830_FIFO_LINE_SIZE,
443	};
444
445	static const struct intel_watermark_params i845_wm_info = {
446	.fifo_size = I830_FIFO_SIZE,
447	.max_wm = I915_MAX_WM,
448	.default_wm = 1,
449	.guard_size = 2,
450	.cacheline_size = I830_FIFO_LINE_SIZE,
451	};
452
453	/**
454	* intel_wm_method1 - Method 1 / "small buffer" watermark formula
455	* @pixel_rate: Pipe pixel rate in kHz
456	* @cpp: Plane bytes per pixel
457	* @latency: Memory wakeup latency in 0.1us units
458	*
459	* Compute the watermark using the method 1 or "small buffer"
460	* formula. The caller may additionally add extra cachelines
461	* to account for TLB misses and clock crossings.
462	*
463	* This method is concerned with the short term drain rate
464	* of the FIFO, ie. it does not account for blanking periods
465	* which would effectively reduce the average drain rate across
466	* a longer period. The name "small" refers to the fact the
467	* FIFO is relatively small compared to the amount of data
468	* fetched.
469	*
470	* The FIFO level vs. time graph might look something like:
471	*
472	* |\ |\
473	* | \ | \
474	* __---__---__ (- plane active, _ blanking)
475	* -> time
476	*
477	* or perhaps like this:
478	*
479	* |\|\ |\|\
480	* __----__----__ (- plane active, _ blanking)
481	* -> time
482	*
483	* Returns:
484	* The watermark in bytes
485	*/
486	static unsigned int intel_wm_method1(unsigned int pixel_rate,
487	unsigned int cpp,
488	unsigned int latency)
489	{
490	u64 ret;
491
492	ret = mul_u32_u32(a: pixel_rate, b: cpp * latency);
493	ret = DIV_ROUND_UP_ULL(ret, 10000);
494
495	return ret;
496	}
497
498	/**
499	* intel_wm_method2 - Method 2 / "large buffer" watermark formula
500	* @pixel_rate: Pipe pixel rate in kHz
501	* @htotal: Pipe horizontal total
502	* @width: Plane width in pixels
503	* @cpp: Plane bytes per pixel
504	* @latency: Memory wakeup latency in 0.1us units
505	*
506	* Compute the watermark using the method 2 or "large buffer"
507	* formula. The caller may additionally add extra cachelines
508	* to account for TLB misses and clock crossings.
509	*
510	* This method is concerned with the long term drain rate
511	* of the FIFO, ie. it does account for blanking periods
512	* which effectively reduce the average drain rate across
513	* a longer period. The name "large" refers to the fact the
514	* FIFO is relatively large compared to the amount of data
515	* fetched.
516	*
517	* The FIFO level vs. time graph might look something like:
518	*
519	* |\___ |\___
520	* | \___ | \___
521	* | \ | \
522	* __ --__--__--__--__--__--__ (- plane active, _ blanking)
523	* -> time
524	*
525	* Returns:
526	* The watermark in bytes
527	*/
528	static unsigned int intel_wm_method2(unsigned int pixel_rate,
529	unsigned int htotal,
530	unsigned int width,
531	unsigned int cpp,
532	unsigned int latency)
533	{
534	unsigned int ret;
535
536	/*
537	* FIXME remove once all users are computing
538	* watermarks in the correct place.
539	*/
540	if (WARN_ON_ONCE(htotal == 0))
541	htotal = 1;
542
543	ret = (latency * pixel_rate) / (htotal * 10000);
544	ret = (ret + 1) * width * cpp;
545
546	return ret;
547	}
548
549	/**
550	* intel_calculate_wm - calculate watermark level
551	* @display: display device
552	* @pixel_rate: pixel clock
553	* @wm: chip FIFO params
554	* @fifo_size: size of the FIFO buffer
555	* @cpp: bytes per pixel
556	* @latency_ns: memory latency for the platform
557	*
558	* Calculate the watermark level (the level at which the display plane will
559	* start fetching from memory again). Each chip has a different display
560	* FIFO size and allocation, so the caller needs to figure that out and pass
561	* in the correct intel_watermark_params structure.
562	*
563	* As the pixel clock runs, the FIFO will be drained at a rate that depends
564	* on the pixel size. When it reaches the watermark level, it'll start
565	* fetching FIFO line sized based chunks from memory until the FIFO fills
566	* past the watermark point. If the FIFO drains completely, a FIFO underrun
567	* will occur, and a display engine hang could result.
568	*/
569	static unsigned int intel_calculate_wm(struct intel_display *display,
570	int pixel_rate,
571	const struct intel_watermark_params *wm,
572	int fifo_size, int cpp,
573	unsigned int latency_ns)
574	{
575	int entries, wm_size;
576
577	/*
578	* Note: we need to make sure we don't overflow for various clock &
579	* latency values.
580	* clocks go from a few thousand to several hundred thousand.
581	* latency is usually a few thousand
582	*/
583	entries = intel_wm_method1(pixel_rate, cpp,
584	latency: latency_ns / 100);
585	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
586	wm->guard_size;
587	drm_dbg_kms(display->drm, "FIFO entries required for mode: %d\n", entries);
588
589	wm_size = fifo_size - entries;
590	drm_dbg_kms(display->drm, "FIFO watermark level: %d\n", wm_size);
591
592	/* Don't promote wm_size to unsigned... */
593	if (wm_size > wm->max_wm)
594	wm_size = wm->max_wm;
595	if (wm_size <= 0)
596	wm_size = wm->default_wm;
597
598	/*
599	* Bspec seems to indicate that the value shouldn't be lower than
600	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
601	* Lets go for 8 which is the burst size since certain platforms
602	* already use a hardcoded 8 (which is what the spec says should be
603	* done).
604	*/
605	if (wm_size <= 8)
606	wm_size = 8;
607
608	return wm_size;
609	}
610
611	static bool is_disabling(int old, int new, int threshold)
612	{
613	return old >= threshold && new < threshold;
614	}
615
616	static bool is_enabling(int old, int new, int threshold)
617	{
618	return old < threshold && new >= threshold;
619	}
620
621	static bool intel_crtc_active(struct intel_crtc *crtc)
622	{
623	/* Be paranoid as we can arrive here with only partial
624	* state retrieved from the hardware during setup.
625	*
626	* We can ditch the adjusted_mode.crtc_clock check as soon
627	* as Haswell has gained clock readout/fastboot support.
628	*
629	* We can ditch the crtc->primary->state->fb check as soon as we can
630	* properly reconstruct framebuffers.
631	*
632	* FIXME: The intel_crtc->active here should be switched to
633	* crtc->state->active once we have proper CRTC states wired up
634	* for atomic.
635	*/
636	return crtc->active && crtc->base.primary->state->fb &&
637	crtc->config->hw.adjusted_mode.crtc_clock;
638	}
639
640	static struct intel_crtc *single_enabled_crtc(struct intel_display *display)
641	{
642	struct intel_crtc *crtc, *enabled = NULL;
643
644	for_each_intel_crtc(display->drm, crtc) {
645	if (intel_crtc_active(crtc)) {
646	if (enabled)
647	return NULL;
648	enabled = crtc;
649	}
650	}
651
652	return enabled;
653	}
654
655	static void pnv_update_wm(struct intel_display *display)
656	{
657	struct intel_crtc *crtc;
658	const struct cxsr_latency *latency;
659	u32 reg;
660	unsigned int wm;
661
662	latency = pnv_get_cxsr_latency(display);
663	if (!latency) {
664	drm_dbg_kms(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
665	intel_set_memory_cxsr(display, enable: false);
666	return;
667	}
668
669	crtc = single_enabled_crtc(display);
670	if (crtc) {
671	const struct drm_framebuffer *fb =
672	crtc->base.primary->state->fb;
673	int pixel_rate = crtc->config->pixel_rate;
674	int cpp = fb->format->cpp[0];
675
676	/* Display SR */
677	wm = intel_calculate_wm(display, pixel_rate,
678	wm: &pnv_display_wm,
679	fifo_size: pnv_display_wm.fifo_size,
680	cpp, latency_ns: latency->display_sr);
681	reg = intel_de_read(display, DSPFW1(display));
682	reg &= ~DSPFW_SR_MASK;
683	reg |= FW_WM(wm, SR);
684	intel_de_write(display, DSPFW1(display), val: reg);
685	drm_dbg_kms(display->drm, "DSPFW1 register is %x\n", reg);
686
687	/* cursor SR */
688	wm = intel_calculate_wm(display, pixel_rate,
689	wm: &pnv_cursor_wm,
690	fifo_size: pnv_display_wm.fifo_size,
691	cpp: 4, latency_ns: latency->cursor_sr);
692	intel_de_rmw(display, DSPFW3(display),
693	DSPFW_CURSOR_SR_MASK, FW_WM(wm, CURSOR_SR));
694
695	/* Display HPLL off SR */
696	wm = intel_calculate_wm(display, pixel_rate,
697	wm: &pnv_display_hplloff_wm,
698	fifo_size: pnv_display_hplloff_wm.fifo_size,
699	cpp, latency_ns: latency->display_hpll_disable);
700	intel_de_rmw(display, DSPFW3(display),
701	DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
702
703	/* cursor HPLL off SR */
704	wm = intel_calculate_wm(display, pixel_rate,
705	wm: &pnv_cursor_hplloff_wm,
706	fifo_size: pnv_display_hplloff_wm.fifo_size,
707	cpp: 4, latency_ns: latency->cursor_hpll_disable);
708	reg = intel_de_read(display, DSPFW3(display));
709	reg &= ~DSPFW_HPLL_CURSOR_MASK;
710	reg |= FW_WM(wm, HPLL_CURSOR);
711	intel_de_write(display, DSPFW3(display), val: reg);
712	drm_dbg_kms(display->drm, "DSPFW3 register is %x\n", reg);
713
714	intel_set_memory_cxsr(display, enable: true);
715	} else {
716	intel_set_memory_cxsr(display, enable: false);
717	}
718	}
719
720	static bool i9xx_wm_need_update(const struct intel_plane_state *old_plane_state,
721	const struct intel_plane_state *new_plane_state)
722	{
723	/* Update watermarks on tiling or size changes. */
724	if (old_plane_state->uapi.visible != new_plane_state->uapi.visible)
725	return true;
726
727	if (!old_plane_state->hw.fb || !new_plane_state->hw.fb)
728	return false;
729
730	if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier ||
731	old_plane_state->hw.rotation != new_plane_state->hw.rotation ||
732	drm_rect_width(r: &old_plane_state->uapi.src) != drm_rect_width(r: &new_plane_state->uapi.src) ||
733	drm_rect_height(r: &old_plane_state->uapi.src) != drm_rect_height(r: &new_plane_state->uapi.src) ||
734	drm_rect_width(r: &old_plane_state->uapi.dst) != drm_rect_width(r: &new_plane_state->uapi.dst) ||
735	drm_rect_height(r: &old_plane_state->uapi.dst) != drm_rect_height(r: &new_plane_state->uapi.dst))
736	return true;
737
738	return false;
739	}
740
741	static void i9xx_wm_compute(struct intel_crtc_state *new_crtc_state,
742	const struct intel_plane_state *old_plane_state,
743	const struct intel_plane_state *new_plane_state)
744	{
745	bool turn_off, turn_on, visible, was_visible, mode_changed;
746
747	mode_changed = intel_crtc_needs_modeset(crtc_state: new_crtc_state);
748	was_visible = old_plane_state->uapi.visible;
749	visible = new_plane_state->uapi.visible;
750
751	if (!was_visible && !visible)
752	return;
753
754	turn_off = was_visible && (!visible || mode_changed);
755	turn_on = visible && (!was_visible || mode_changed);
756
757	/* FIXME nuke when all wm code is atomic */
758	if (turn_on) {
759	new_crtc_state->update_wm_pre = true;
760	} else if (turn_off) {
761	new_crtc_state->update_wm_post = true;
762	} else if (i9xx_wm_need_update(old_plane_state, new_plane_state)) {
763	/* FIXME bollocks */
764	new_crtc_state->update_wm_pre = true;
765	new_crtc_state->update_wm_post = true;
766	}
767	}
768
769	static int i9xx_compute_watermarks(struct intel_atomic_state *state,
770	struct intel_crtc *crtc)
771	{
772	struct intel_crtc_state *new_crtc_state =
773	intel_atomic_get_new_crtc_state(state, crtc);
774	const struct intel_plane_state *old_plane_state;
775	const struct intel_plane_state *new_plane_state;
776	struct intel_plane *plane;
777	int i;
778
779	for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
780	new_plane_state, i) {
781	if (plane->pipe != crtc->pipe)
782	continue;
783
784	i9xx_wm_compute(new_crtc_state, old_plane_state, new_plane_state);
785	}
786
787	return 0;
788	}
789
790	/*
791	* Documentation says:
792	* "If the line size is small, the TLB fetches can get in the way of the
793	* data fetches, causing some lag in the pixel data return which is not
794	* accounted for in the above formulas. The following adjustment only
795	* needs to be applied if eight whole lines fit in the buffer at once.
796	* The WM is adjusted upwards by the difference between the FIFO size
797	* and the size of 8 whole lines. This adjustment is always performed
798	* in the actual pixel depth regardless of whether FBC is enabled or not."
799	*/
800	static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
801	{
802	int tlb_miss = fifo_size * 64 - width * cpp * 8;
803
804	return max(0, tlb_miss);
805	}
806
807	static void g4x_write_wm_values(struct intel_display *display,
808	const struct g4x_wm_values *wm)
809	{
810	enum pipe pipe;
811
812	for_each_pipe(display, pipe)
813	trace_g4x_wm(crtc: intel_crtc_for_pipe(display, pipe), wm);
814
815	intel_de_write(display, DSPFW1(display),
816	FW_WM(wm->sr.plane, SR) |
817	FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
818	FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
819	FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
820	intel_de_write(display, DSPFW2(display),
821	val: (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
822	FW_WM(wm->sr.fbc, FBC_SR) |
823	FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
824	FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
825	FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
826	FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
827	intel_de_write(display, DSPFW3(display),
828	val: (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
829	FW_WM(wm->sr.cursor, CURSOR_SR) |
830	FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
831	FW_WM(wm->hpll.plane, HPLL_SR));
832
833	intel_de_posting_read(display, DSPFW1(display));
834	}
835
836	#define FW_WM_VLV(value, plane) \
837	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
838
839	static void vlv_write_wm_values(struct intel_display *display,
840	const struct vlv_wm_values *wm)
841	{
842	enum pipe pipe;
843
844	for_each_pipe(display, pipe) {
845	trace_vlv_wm(crtc: intel_crtc_for_pipe(display, pipe), wm);
846
847	intel_de_write(display, VLV_DDL(pipe),
848	val: (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
849	(wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
850	(wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
851	(wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
852	}
853
854	/*
855	* Zero the (unused) WM1 watermarks, and also clear all the
856	* high order bits so that there are no out of bounds values
857	* present in the registers during the reprogramming.
858	*/
859	intel_de_write(display, DSPHOWM, val: 0);
860	intel_de_write(display, DSPHOWM1, val: 0);
861	intel_de_write(display, DSPFW4, val: 0);
862	intel_de_write(display, DSPFW5, val: 0);
863	intel_de_write(display, DSPFW6, val: 0);
864
865	intel_de_write(display, DSPFW1(display),
866	FW_WM(wm->sr.plane, SR) |
867	FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
868	FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
869	FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
870	intel_de_write(display, DSPFW2(display),
871	FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
872	FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
873	FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
874	intel_de_write(display, DSPFW3(display),
875	FW_WM(wm->sr.cursor, CURSOR_SR));
876
877	if (display->platform.cherryview) {
878	intel_de_write(display, DSPFW7_CHV,
879	FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
880	FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
881	intel_de_write(display, DSPFW8_CHV,
882	FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
883	FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
884	intel_de_write(display, DSPFW9_CHV,
885	FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
886	FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
887	intel_de_write(display, DSPHOWM,
888	FW_WM(wm->sr.plane >> 9, SR_HI) |
889	FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
890	FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
891	FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
892	FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
893	FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
894	FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
895	FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
896	FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
897	FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
898	} else {
899	intel_de_write(display, DSPFW7,
900	FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
901	FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
902	intel_de_write(display, DSPHOWM,
903	FW_WM(wm->sr.plane >> 9, SR_HI) |
904	FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
905	FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
906	FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
907	FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
908	FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
909	FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
910	}
911
912	intel_de_posting_read(display, DSPFW1(display));
913	}
914
915	#undef FW_WM_VLV
916
917	static void g4x_setup_wm_latency(struct intel_display *display)
918	{
919	/* all latencies in usec */
920	display->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
921	display->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
922	display->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
923
924	display->wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
925	}
926
927	static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
928	{
929	/*
930	* DSPCNTR[13] supposedly controls whether the
931	* primary plane can use the FIFO space otherwise
932	* reserved for the sprite plane. It's not 100% clear
933	* what the actual FIFO size is, but it looks like we
934	* can happily set both primary and sprite watermarks
935	* up to 127 cachelines. So that would seem to mean
936	* that either DSPCNTR[13] doesn't do anything, or that
937	* the total FIFO is >= 256 cachelines in size. Either
938	* way, we don't seem to have to worry about this
939	* repartitioning as the maximum watermark value the
940	* register can hold for each plane is lower than the
941	* minimum FIFO size.
942	*/
943	switch (plane_id) {
944	case PLANE_CURSOR:
945	return 63;
946	case PLANE_PRIMARY:
947	return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
948	case PLANE_SPRITE0:
949	return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
950	default:
951	MISSING_CASE(plane_id);
952	return 0;
953	}
954	}
955
956	static int g4x_fbc_fifo_size(int level)
957	{
958	switch (level) {
959	case G4X_WM_LEVEL_SR:
960	return 7;
961	case G4X_WM_LEVEL_HPLL:
962	return 15;
963	default:
964	MISSING_CASE(level);
965	return 0;
966	}
967	}
968
969	static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
970	const struct intel_plane_state *plane_state,
971	int level)
972	{
973	struct intel_display *display = to_intel_display(plane_state);
974	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
975	const struct drm_display_mode *pipe_mode =
976	&crtc_state->hw.pipe_mode;
977	unsigned int latency = display->wm.pri_latency[level] * 10;
978	unsigned int pixel_rate, htotal, cpp, width, wm;
979
980	if (latency == 0)
981	return USHRT_MAX;
982
983	if (!intel_wm_plane_visible(crtc_state, plane_state))
984	return 0;
985
986	cpp = plane_state->hw.fb->format->cpp[0];
987
988	/*
989	* WaUse32BppForSRWM:ctg,elk
990	*
991	* The spec fails to list this restriction for the
992	* HPLL watermark, which seems a little strange.
993	* Let's use 32bpp for the HPLL watermark as well.
994	*/
995	if (plane->id == PLANE_PRIMARY &&
996	level != G4X_WM_LEVEL_NORMAL)
997	cpp = max(cpp, 4u);
998
999	pixel_rate = crtc_state->pixel_rate;
1000	htotal = pipe_mode->crtc_htotal;
1001	width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1002
1003	if (plane->id == PLANE_CURSOR) {
1004	wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1005	} else if (plane->id == PLANE_PRIMARY &&
1006	level == G4X_WM_LEVEL_NORMAL) {
1007	wm = intel_wm_method1(pixel_rate, cpp, latency);
1008	} else {
1009	unsigned int small, large;
1010
1011	small = intel_wm_method1(pixel_rate, cpp, latency);
1012	large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1013
1014	wm = min(small, large);
1015	}
1016
1017	wm += g4x_tlb_miss_wa(fifo_size: g4x_plane_fifo_size(plane_id: plane->id, level),
1018	width, cpp);
1019
1020	wm = DIV_ROUND_UP(wm, 64) + 2;
1021
1022	return min_t(unsigned int, wm, USHRT_MAX);
1023	}
1024
1025	static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1026	int level, enum plane_id plane_id, u16 value)
1027	{
1028	struct intel_display *display = to_intel_display(crtc_state);
1029	bool dirty = false;
1030
1031	for (; level < display->wm.num_levels; level++) {
1032	struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1033
1034	dirty |= raw->plane[plane_id] != value;
1035	raw->plane[plane_id] = value;
1036	}
1037
1038	return dirty;
1039	}
1040
1041	static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1042	int level, u16 value)
1043	{
1044	struct intel_display *display = to_intel_display(crtc_state);
1045	bool dirty = false;
1046
1047	/* NORMAL level doesn't have an FBC watermark */
1048	level = max(level, G4X_WM_LEVEL_SR);
1049
1050	for (; level < display->wm.num_levels; level++) {
1051	struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1052
1053	dirty |= raw->fbc != value;
1054	raw->fbc = value;
1055	}
1056
1057	return dirty;
1058	}
1059
1060	static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1061	const struct intel_plane_state *plane_state,
1062	u32 pri_val);
1063
1064	static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1065	const struct intel_plane_state *plane_state)
1066	{
1067	struct intel_display *display = to_intel_display(crtc_state);
1068	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1069	enum plane_id plane_id = plane->id;
1070	bool dirty = false;
1071	int level;
1072
1073	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1074	dirty |= g4x_raw_plane_wm_set(crtc_state, level: 0, plane_id, value: 0);
1075	if (plane_id == PLANE_PRIMARY)
1076	dirty |= g4x_raw_fbc_wm_set(crtc_state, level: 0, value: 0);
1077	goto out;
1078	}
1079
1080	for (level = 0; level < display->wm.num_levels; level++) {
1081	struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1082	int wm, max_wm;
1083
1084	wm = g4x_compute_wm(crtc_state, plane_state, level);
1085	max_wm = g4x_plane_fifo_size(plane_id, level);
1086
1087	if (wm > max_wm)
1088	break;
1089
1090	dirty |= raw->plane[plane_id] != wm;
1091	raw->plane[plane_id] = wm;
1092
1093	if (plane_id != PLANE_PRIMARY ||
1094	level == G4X_WM_LEVEL_NORMAL)
1095	continue;
1096
1097	wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1098	pri_val: raw->plane[plane_id]);
1099	max_wm = g4x_fbc_fifo_size(level);
1100
1101	/*
1102	* FBC wm is not mandatory as we
1103	* can always just disable its use.
1104	*/
1105	if (wm > max_wm)
1106	wm = USHRT_MAX;
1107
1108	dirty |= raw->fbc != wm;
1109	raw->fbc = wm;
1110	}
1111
1112	/* mark watermarks as invalid */
1113	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1114
1115	if (plane_id == PLANE_PRIMARY)
1116	dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1117
1118	out:
1119	if (dirty) {
1120	drm_dbg_kms(display->drm,
1121	"%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1122	plane->base.name,
1123	crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1124	crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1125	crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1126
1127	if (plane_id == PLANE_PRIMARY)
1128	drm_dbg_kms(display->drm,
1129	"FBC watermarks: SR=%d, HPLL=%d\n",
1130	crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1131	crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1132	}
1133
1134	return dirty;
1135	}
1136
1137	static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1138	enum plane_id plane_id, int level)
1139	{
1140	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1141
1142	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1143	}
1144
1145	static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1146	int level)
1147	{
1148	struct intel_display *display = to_intel_display(crtc_state);
1149
1150	if (level >= display->wm.num_levels)
1151	return false;
1152
1153	return g4x_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_PRIMARY, level) &&
1154	g4x_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_SPRITE0, level) &&
1155	g4x_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_CURSOR, level);
1156	}
1157
1158	/* mark all levels starting from 'level' as invalid */
1159	static void g4x_invalidate_wms(struct intel_crtc *crtc,
1160	struct g4x_wm_state *wm_state, int level)
1161	{
1162	if (level <= G4X_WM_LEVEL_NORMAL) {
1163	enum plane_id plane_id;
1164
1165	for_each_plane_id_on_crtc(crtc, plane_id)
1166	wm_state->wm.plane[plane_id] = USHRT_MAX;
1167	}
1168
1169	if (level <= G4X_WM_LEVEL_SR) {
1170	wm_state->cxsr = false;
1171	wm_state->sr.cursor = USHRT_MAX;
1172	wm_state->sr.plane = USHRT_MAX;
1173	wm_state->sr.fbc = USHRT_MAX;
1174	}
1175
1176	if (level <= G4X_WM_LEVEL_HPLL) {
1177	wm_state->hpll_en = false;
1178	wm_state->hpll.cursor = USHRT_MAX;
1179	wm_state->hpll.plane = USHRT_MAX;
1180	wm_state->hpll.fbc = USHRT_MAX;
1181	}
1182	}
1183
1184	static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1185	int level)
1186	{
1187	if (level < G4X_WM_LEVEL_SR)
1188	return false;
1189
1190	if (level >= G4X_WM_LEVEL_SR &&
1191	wm_state->sr.fbc > g4x_fbc_fifo_size(level: G4X_WM_LEVEL_SR))
1192	return false;
1193
1194	if (level >= G4X_WM_LEVEL_HPLL &&
1195	wm_state->hpll.fbc > g4x_fbc_fifo_size(level: G4X_WM_LEVEL_HPLL))
1196	return false;
1197
1198	return true;
1199	}
1200
1201	static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1202	{
1203	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1204	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1205	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1206	const struct g4x_pipe_wm *raw;
1207	enum plane_id plane_id;
1208	int level;
1209
1210	level = G4X_WM_LEVEL_NORMAL;
1211	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1212	goto out;
1213
1214	raw = &crtc_state->wm.g4x.raw[level];
1215	for_each_plane_id_on_crtc(crtc, plane_id)
1216	wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1217
1218	level = G4X_WM_LEVEL_SR;
1219	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1220	goto out;
1221
1222	raw = &crtc_state->wm.g4x.raw[level];
1223	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1224	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1225	wm_state->sr.fbc = raw->fbc;
1226
1227	wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1228
1229	level = G4X_WM_LEVEL_HPLL;
1230	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1231	goto out;
1232
1233	raw = &crtc_state->wm.g4x.raw[level];
1234	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1235	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1236	wm_state->hpll.fbc = raw->fbc;
1237
1238	wm_state->hpll_en = wm_state->cxsr;
1239
1240	level++;
1241
1242	out:
1243	if (level == G4X_WM_LEVEL_NORMAL)
1244	return -EINVAL;
1245
1246	/* invalidate the higher levels */
1247	g4x_invalidate_wms(crtc, wm_state, level);
1248
1249	/*
1250	* Determine if the FBC watermark(s) can be used. IF
1251	* this isn't the case we prefer to disable the FBC
1252	* watermark(s) rather than disable the SR/HPLL
1253	* level(s) entirely. 'level-1' is the highest valid
1254	* level here.
1255	*/
1256	wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level: level - 1);
1257
1258	return 0;
1259	}
1260
1261	static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1262	struct intel_crtc *crtc)
1263	{
1264	struct intel_crtc_state *crtc_state =
1265	intel_atomic_get_new_crtc_state(state, crtc);
1266	const struct intel_plane_state *old_plane_state;
1267	const struct intel_plane_state *new_plane_state;
1268	struct intel_plane *plane;
1269	unsigned int dirty = 0;
1270	int i;
1271
1272	for_each_oldnew_intel_plane_in_state(state, plane,
1273	old_plane_state,
1274	new_plane_state, i) {
1275	if (new_plane_state->hw.crtc != &crtc->base &&
1276	old_plane_state->hw.crtc != &crtc->base)
1277	continue;
1278
1279	if (g4x_raw_plane_wm_compute(crtc_state, plane_state: new_plane_state))
1280	dirty |= BIT(plane->id);
1281	}
1282
1283	if (!dirty)
1284	return 0;
1285
1286	return _g4x_compute_pipe_wm(crtc_state);
1287	}
1288
1289	static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1290	struct intel_crtc *crtc)
1291	{
1292	struct intel_display *display = to_intel_display(state);
1293	struct intel_crtc_state *new_crtc_state =
1294	intel_atomic_get_new_crtc_state(state, crtc);
1295	const struct intel_crtc_state *old_crtc_state =
1296	intel_atomic_get_old_crtc_state(state, crtc);
1297	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1298	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1299	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1300	enum plane_id plane_id;
1301
1302	if (!new_crtc_state->hw.active ||
1303	intel_crtc_needs_modeset(crtc_state: new_crtc_state)) {
1304	*intermediate = *optimal;
1305
1306	intermediate->cxsr = false;
1307	intermediate->hpll_en = false;
1308	goto out;
1309	}
1310
1311	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1312	!new_crtc_state->disable_cxsr;
1313	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1314	!new_crtc_state->disable_cxsr;
1315	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1316
1317	for_each_plane_id_on_crtc(crtc, plane_id) {
1318	intermediate->wm.plane[plane_id] =
1319	max(optimal->wm.plane[plane_id],
1320	active->wm.plane[plane_id]);
1321
1322	drm_WARN_ON(display->drm, intermediate->wm.plane[plane_id] >
1323	g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1324	}
1325
1326	intermediate->sr.plane = max(optimal->sr.plane,
1327	active->sr.plane);
1328	intermediate->sr.cursor = max(optimal->sr.cursor,
1329	active->sr.cursor);
1330	intermediate->sr.fbc = max(optimal->sr.fbc,
1331	active->sr.fbc);
1332
1333	intermediate->hpll.plane = max(optimal->hpll.plane,
1334	active->hpll.plane);
1335	intermediate->hpll.cursor = max(optimal->hpll.cursor,
1336	active->hpll.cursor);
1337	intermediate->hpll.fbc = max(optimal->hpll.fbc,
1338	active->hpll.fbc);
1339
1340	drm_WARN_ON(display->drm,
1341	(intermediate->sr.plane >
1342	g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1343	intermediate->sr.cursor >
1344	g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1345	intermediate->cxsr);
1346	drm_WARN_ON(display->drm,
1347	(intermediate->sr.plane >
1348	g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1349	intermediate->sr.cursor >
1350	g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1351	intermediate->hpll_en);
1352
1353	drm_WARN_ON(display->drm,
1354	intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1355	intermediate->fbc_en && intermediate->cxsr);
1356	drm_WARN_ON(display->drm,
1357	intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1358	intermediate->fbc_en && intermediate->hpll_en);
1359
1360	out:
1361	/*
1362	* If our intermediate WM are identical to the final WM, then we can
1363	* omit the post-vblank programming; only update if it's different.
1364	*/
1365	if (memcmp(p: intermediate, q: optimal, size: sizeof(*intermediate)) != 0)
1366	new_crtc_state->wm.need_postvbl_update = true;
1367
1368	return 0;
1369	}
1370
1371	static int g4x_compute_watermarks(struct intel_atomic_state *state,
1372	struct intel_crtc *crtc)
1373	{
1374	int ret;
1375
1376	ret = g4x_compute_pipe_wm(state, crtc);
1377	if (ret)
1378	return ret;
1379
1380	ret = g4x_compute_intermediate_wm(state, crtc);
1381	if (ret)
1382	return ret;
1383
1384	return 0;
1385	}
1386
1387	static void g4x_merge_wm(struct intel_display *display,
1388	struct g4x_wm_values *wm)
1389	{
1390	struct intel_crtc *crtc;
1391	int num_active_pipes = 0;
1392
1393	wm->cxsr = true;
1394	wm->hpll_en = true;
1395	wm->fbc_en = true;
1396
1397	for_each_intel_crtc(display->drm, crtc) {
1398	const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1399
1400	if (!crtc->active)
1401	continue;
1402
1403	if (!wm_state->cxsr)
1404	wm->cxsr = false;
1405	if (!wm_state->hpll_en)
1406	wm->hpll_en = false;
1407	if (!wm_state->fbc_en)
1408	wm->fbc_en = false;
1409
1410	num_active_pipes++;
1411	}
1412
1413	if (num_active_pipes != 1) {
1414	wm->cxsr = false;
1415	wm->hpll_en = false;
1416	wm->fbc_en = false;
1417	}
1418
1419	for_each_intel_crtc(display->drm, crtc) {
1420	const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1421	enum pipe pipe = crtc->pipe;
1422
1423	wm->pipe[pipe] = wm_state->wm;
1424	if (crtc->active && wm->cxsr)
1425	wm->sr = wm_state->sr;
1426	if (crtc->active && wm->hpll_en)
1427	wm->hpll = wm_state->hpll;
1428	}
1429	}
1430
1431	static void g4x_program_watermarks(struct intel_display *display)
1432	{
1433	struct g4x_wm_values *old_wm = &display->wm.g4x;
1434	struct g4x_wm_values new_wm = {};
1435
1436	g4x_merge_wm(display, wm: &new_wm);
1437
1438	if (memcmp(p: old_wm, q: &new_wm, size: sizeof(new_wm)) == 0)
1439	return;
1440
1441	if (is_disabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
1442	_intel_set_memory_cxsr(display, enable: false);
1443
1444	g4x_write_wm_values(display, wm: &new_wm);
1445
1446	if (is_enabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
1447	_intel_set_memory_cxsr(display, enable: true);
1448
1449	*old_wm = new_wm;
1450	}
1451
1452	static void g4x_initial_watermarks(struct intel_atomic_state *state,
1453	struct intel_crtc *crtc)
1454	{
1455	struct intel_display *display = to_intel_display(crtc);
1456	const struct intel_crtc_state *crtc_state =
1457	intel_atomic_get_new_crtc_state(state, crtc);
1458
1459	mutex_lock(&display->wm.wm_mutex);
1460	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1461	g4x_program_watermarks(display);
1462	mutex_unlock(lock: &display->wm.wm_mutex);
1463	}
1464
1465	static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1466	struct intel_crtc *crtc)
1467	{
1468	struct intel_display *display = to_intel_display(crtc);
1469	const struct intel_crtc_state *crtc_state =
1470	intel_atomic_get_new_crtc_state(state, crtc);
1471
1472	if (!crtc_state->wm.need_postvbl_update)
1473	return;
1474
1475	mutex_lock(&display->wm.wm_mutex);
1476	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1477	g4x_program_watermarks(display);
1478	mutex_unlock(lock: &display->wm.wm_mutex);
1479	}
1480
1481	/* latency must be in 0.1us units. */
1482	static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1483	unsigned int htotal,
1484	unsigned int width,
1485	unsigned int cpp,
1486	unsigned int latency)
1487	{
1488	unsigned int ret;
1489
1490	ret = intel_wm_method2(pixel_rate, htotal,
1491	width, cpp, latency);
1492	ret = DIV_ROUND_UP(ret, 64);
1493
1494	return ret;
1495	}
1496
1497	static void vlv_setup_wm_latency(struct intel_display *display)
1498	{
1499	/* all latencies in usec */
1500	display->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1501
1502	display->wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1503
1504	if (display->platform.cherryview) {
1505	display->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1506	display->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1507
1508	display->wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1509	}
1510	}
1511
1512	static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1513	const struct intel_plane_state *plane_state,
1514	int level)
1515	{
1516	struct intel_display *display = to_intel_display(plane_state);
1517	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1518	const struct drm_display_mode *pipe_mode =
1519	&crtc_state->hw.pipe_mode;
1520	unsigned int pixel_rate, htotal, cpp, width, wm;
1521
1522	if (display->wm.pri_latency[level] == 0)
1523	return USHRT_MAX;
1524
1525	if (!intel_wm_plane_visible(crtc_state, plane_state))
1526	return 0;
1527
1528	cpp = plane_state->hw.fb->format->cpp[0];
1529	pixel_rate = crtc_state->pixel_rate;
1530	htotal = pipe_mode->crtc_htotal;
1531	width = drm_rect_width(r: &plane_state->uapi.src) >> 16;
1532
1533	if (plane->id == PLANE_CURSOR) {
1534	/*
1535	* FIXME the formula gives values that are
1536	* too big for the cursor FIFO, and hence we
1537	* would never be able to use cursors. For
1538	* now just hardcode the watermark.
1539	*/
1540	wm = 63;
1541	} else {
1542	wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1543	latency: display->wm.pri_latency[level] * 10);
1544	}
1545
1546	return min_t(unsigned int, wm, USHRT_MAX);
1547	}
1548
1549	static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1550	{
1551	return (active_planes & (BIT(PLANE_SPRITE0) |
1552	BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1553	}
1554
1555	static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1556	{
1557	struct intel_display *display = to_intel_display(crtc_state);
1558	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1559	const struct g4x_pipe_wm *raw =
1560	&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1561	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1562	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1563	int num_active_planes = hweight8(active_planes);
1564	const int fifo_size = 511;
1565	int fifo_extra, fifo_left = fifo_size;
1566	int sprite0_fifo_extra = 0;
1567	unsigned int total_rate;
1568	enum plane_id plane_id;
1569
1570	/*
1571	* When enabling sprite0 after sprite1 has already been enabled
1572	* we tend to get an underrun unless sprite0 already has some
1573	* FIFO space allocated. Hence we always allocate at least one
1574	* cacheline for sprite0 whenever sprite1 is enabled.
1575	*
1576	* All other plane enable sequences appear immune to this problem.
1577	*/
1578	if (vlv_need_sprite0_fifo_workaround(active_planes))
1579	sprite0_fifo_extra = 1;
1580
1581	total_rate = raw->plane[PLANE_PRIMARY] +
1582	raw->plane[PLANE_SPRITE0] +
1583	raw->plane[PLANE_SPRITE1] +
1584	sprite0_fifo_extra;
1585
1586	if (total_rate > fifo_size)
1587	return -EINVAL;
1588
1589	if (total_rate == 0)
1590	total_rate = 1;
1591
1592	for_each_plane_id_on_crtc(crtc, plane_id) {
1593	unsigned int rate;
1594
1595	if ((active_planes & BIT(plane_id)) == 0) {
1596	fifo_state->plane[plane_id] = 0;
1597	continue;
1598	}
1599
1600	rate = raw->plane[plane_id];
1601	fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1602	fifo_left -= fifo_state->plane[plane_id];
1603	}
1604
1605	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1606	fifo_left -= sprite0_fifo_extra;
1607
1608	fifo_state->plane[PLANE_CURSOR] = 63;
1609
1610	fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1611
1612	/* spread the remainder evenly */
1613	for_each_plane_id_on_crtc(crtc, plane_id) {
1614	int plane_extra;
1615
1616	if (fifo_left == 0)
1617	break;
1618
1619	if ((active_planes & BIT(plane_id)) == 0)
1620	continue;
1621
1622	plane_extra = min(fifo_extra, fifo_left);
1623	fifo_state->plane[plane_id] += plane_extra;
1624	fifo_left -= plane_extra;
1625	}
1626
1627	drm_WARN_ON(display->drm, active_planes != 0 && fifo_left != 0);
1628
1629	/* give it all to the first plane if none are active */
1630	if (active_planes == 0) {
1631	drm_WARN_ON(display->drm, fifo_left != fifo_size);
1632	fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1633	}
1634
1635	return 0;
1636	}
1637
1638	/* mark all levels starting from 'level' as invalid */
1639	static void vlv_invalidate_wms(struct intel_crtc *crtc,
1640	struct vlv_wm_state *wm_state, int level)
1641	{
1642	struct intel_display *display = to_intel_display(crtc);
1643
1644	for (; level < display->wm.num_levels; level++) {
1645	enum plane_id plane_id;
1646
1647	for_each_plane_id_on_crtc(crtc, plane_id)
1648	wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1649
1650	wm_state->sr[level].cursor = USHRT_MAX;
1651	wm_state->sr[level].plane = USHRT_MAX;
1652	}
1653	}
1654
1655	static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1656	{
1657	if (wm > fifo_size)
1658	return USHRT_MAX;
1659	else
1660	return fifo_size - wm;
1661	}
1662
1663	/*
1664	* Starting from 'level' set all higher
1665	* levels to 'value' in the "raw" watermarks.
1666	*/
1667	static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1668	int level, enum plane_id plane_id, u16 value)
1669	{
1670	struct intel_display *display = to_intel_display(crtc_state);
1671	bool dirty = false;
1672
1673	for (; level < display->wm.num_levels; level++) {
1674	struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1675
1676	dirty |= raw->plane[plane_id] != value;
1677	raw->plane[plane_id] = value;
1678	}
1679
1680	return dirty;
1681	}
1682
1683	static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1684	const struct intel_plane_state *plane_state)
1685	{
1686	struct intel_display *display = to_intel_display(crtc_state);
1687	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1688	enum plane_id plane_id = plane->id;
1689	int level;
1690	bool dirty = false;
1691
1692	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1693	dirty |= vlv_raw_plane_wm_set(crtc_state, level: 0, plane_id, value: 0);
1694	goto out;
1695	}
1696
1697	for (level = 0; level < display->wm.num_levels; level++) {
1698	struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1699	int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1700	int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1701
1702	if (wm > max_wm)
1703	break;
1704
1705	dirty |= raw->plane[plane_id] != wm;
1706	raw->plane[plane_id] = wm;
1707	}
1708
1709	/* mark all higher levels as invalid */
1710	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1711
1712	out:
1713	if (dirty)
1714	drm_dbg_kms(display->drm,
1715	"%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1716	plane->base.name,
1717	crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1718	crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1719	crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1720
1721	return dirty;
1722	}
1723
1724	static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1725	enum plane_id plane_id, int level)
1726	{
1727	const struct g4x_pipe_wm *raw =
1728	&crtc_state->wm.vlv.raw[level];
1729	const struct vlv_fifo_state *fifo_state =
1730	&crtc_state->wm.vlv.fifo_state;
1731
1732	return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1733	}
1734
1735	static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1736	{
1737	return vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_PRIMARY, level) &&
1738	vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_SPRITE0, level) &&
1739	vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_SPRITE1, level) &&
1740	vlv_raw_plane_wm_is_valid(crtc_state, plane_id: PLANE_CURSOR, level);
1741	}
1742
1743	static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1744	{
1745	struct intel_display *display = to_intel_display(crtc_state);
1746	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1747	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1748	const struct vlv_fifo_state *fifo_state =
1749	&crtc_state->wm.vlv.fifo_state;
1750	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1751	int num_active_planes = hweight8(active_planes);
1752	enum plane_id plane_id;
1753	int level;
1754
1755	/* initially allow all levels */
1756	wm_state->num_levels = display->wm.num_levels;
1757	/*
1758	* Note that enabling cxsr with no primary/sprite planes
1759	* enabled can wedge the pipe. Hence we only allow cxsr
1760	* with exactly one enabled primary/sprite plane.
1761	*/
1762	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1763
1764	for (level = 0; level < wm_state->num_levels; level++) {
1765	const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1766	const int sr_fifo_size = INTEL_NUM_PIPES(display) * 512 - 1;
1767
1768	if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1769	break;
1770
1771	for_each_plane_id_on_crtc(crtc, plane_id) {
1772	wm_state->wm[level].plane[plane_id] =
1773	vlv_invert_wm_value(wm: raw->plane[plane_id],
1774	fifo_size: fifo_state->plane[plane_id]);
1775	}
1776
1777	wm_state->sr[level].plane =
1778	vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1779	raw->plane[PLANE_SPRITE0],
1780	raw->plane[PLANE_SPRITE1]),
1781	fifo_size: sr_fifo_size);
1782
1783	wm_state->sr[level].cursor =
1784	vlv_invert_wm_value(wm: raw->plane[PLANE_CURSOR],
1785	fifo_size: 63);
1786	}
1787
1788	if (level == 0)
1789	return -EINVAL;
1790
1791	/* limit to only levels we can actually handle */
1792	wm_state->num_levels = level;
1793
1794	/* invalidate the higher levels */
1795	vlv_invalidate_wms(crtc, wm_state, level);
1796
1797	return 0;
1798	}
1799
1800	static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1801	struct intel_crtc *crtc)
1802	{
1803	struct intel_crtc_state *crtc_state =
1804	intel_atomic_get_new_crtc_state(state, crtc);
1805	const struct intel_plane_state *old_plane_state;
1806	const struct intel_plane_state *new_plane_state;
1807	struct intel_plane *plane;
1808	unsigned int dirty = 0;
1809	int i;
1810
1811	for_each_oldnew_intel_plane_in_state(state, plane,
1812	old_plane_state,
1813	new_plane_state, i) {
1814	if (new_plane_state->hw.crtc != &crtc->base &&
1815	old_plane_state->hw.crtc != &crtc->base)
1816	continue;
1817
1818	if (vlv_raw_plane_wm_compute(crtc_state, plane_state: new_plane_state))
1819	dirty |= BIT(plane->id);
1820	}
1821
1822	/*
1823	* DSPARB registers may have been reset due to the
1824	* power well being turned off. Make sure we restore
1825	* them to a consistent state even if no primary/sprite
1826	* planes are initially active. We also force a FIFO
1827	* recomputation so that we are sure to sanitize the
1828	* FIFO setting we took over from the BIOS even if there
1829	* are no active planes on the crtc.
1830	*/
1831	if (intel_crtc_needs_modeset(crtc_state))
1832	dirty = ~0;
1833
1834	if (!dirty)
1835	return 0;
1836
1837	/* cursor changes don't warrant a FIFO recompute */
1838	if (dirty & ~BIT(PLANE_CURSOR)) {
1839	const struct intel_crtc_state *old_crtc_state =
1840	intel_atomic_get_old_crtc_state(state, crtc);
1841	const struct vlv_fifo_state *old_fifo_state =
1842	&old_crtc_state->wm.vlv.fifo_state;
1843	const struct vlv_fifo_state *new_fifo_state =
1844	&crtc_state->wm.vlv.fifo_state;
1845	int ret;
1846
1847	ret = vlv_compute_fifo(crtc_state);
1848	if (ret)
1849	return ret;
1850
1851	if (intel_crtc_needs_modeset(crtc_state) ||
1852	memcmp(p: old_fifo_state, q: new_fifo_state,
1853	size: sizeof(*new_fifo_state)) != 0)
1854	crtc_state->fifo_changed = true;
1855	}
1856
1857	return _vlv_compute_pipe_wm(crtc_state);
1858	}
1859
1860	#define VLV_FIFO(plane, value) \
1861	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1862
1863	static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1864	struct intel_crtc *crtc)
1865	{
1866	struct intel_display *display = to_intel_display(crtc);
1867	struct drm_i915_private *dev_priv = to_i915(dev: crtc->base.dev);
1868	struct intel_uncore *uncore = &dev_priv->uncore;
1869	const struct intel_crtc_state *crtc_state =
1870	intel_atomic_get_new_crtc_state(state, crtc);
1871	const struct vlv_fifo_state *fifo_state =
1872	&crtc_state->wm.vlv.fifo_state;
1873	int sprite0_start, sprite1_start, fifo_size;
1874	u32 dsparb, dsparb2, dsparb3;
1875
1876	if (!crtc_state->fifo_changed)
1877	return;
1878
1879	sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1880	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1881	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1882
1883	drm_WARN_ON(display->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1884	drm_WARN_ON(display->drm, fifo_size != 511);
1885
1886	trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1887
1888	/*
1889	* uncore.lock serves a double purpose here. It allows us to
1890	* use the less expensive I915_{READ,WRITE}_FW() functions, and
1891	* it protects the DSPARB registers from getting clobbered by
1892	* parallel updates from multiple pipes.
1893	*
1894	* intel_pipe_update_start() has already disabled interrupts
1895	* for us, so a plain spin_lock() is sufficient here.
1896	*/
1897	spin_lock(lock: &uncore->lock);
1898
1899	switch (crtc->pipe) {
1900	case PIPE_A:
1901	dsparb = intel_de_read_fw(display, DSPARB(display));
1902	dsparb2 = intel_de_read_fw(display, DSPARB2);
1903
1904	dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1905	VLV_FIFO(SPRITEB, 0xff));
1906	dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1907	VLV_FIFO(SPRITEB, sprite1_start));
1908
1909	dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1910	VLV_FIFO(SPRITEB_HI, 0x1));
1911	dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1912	VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1913
1914	intel_de_write_fw(display, DSPARB(display), val: dsparb);
1915	intel_de_write_fw(display, DSPARB2, val: dsparb2);
1916	break;
1917	case PIPE_B:
1918	dsparb = intel_de_read_fw(display, DSPARB(display));
1919	dsparb2 = intel_de_read_fw(display, DSPARB2);
1920
1921	dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1922	VLV_FIFO(SPRITED, 0xff));
1923	dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1924	VLV_FIFO(SPRITED, sprite1_start));
1925
1926	dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1927	VLV_FIFO(SPRITED_HI, 0xff));
1928	dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1929	VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1930
1931	intel_de_write_fw(display, DSPARB(display), val: dsparb);
1932	intel_de_write_fw(display, DSPARB2, val: dsparb2);
1933	break;
1934	case PIPE_C:
1935	dsparb3 = intel_de_read_fw(display, DSPARB3);
1936	dsparb2 = intel_de_read_fw(display, DSPARB2);
1937
1938	dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1939	VLV_FIFO(SPRITEF, 0xff));
1940	dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1941	VLV_FIFO(SPRITEF, sprite1_start));
1942
1943	dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1944	VLV_FIFO(SPRITEF_HI, 0xff));
1945	dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1946	VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1947
1948	intel_de_write_fw(display, DSPARB3, val: dsparb3);
1949	intel_de_write_fw(display, DSPARB2, val: dsparb2);
1950	break;
1951	default:
1952	break;
1953	}
1954
1955	intel_de_read_fw(display, DSPARB(display));
1956
1957	spin_unlock(lock: &uncore->lock);
1958	}
1959
1960	#undef VLV_FIFO
1961
1962	static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1963	struct intel_crtc *crtc)
1964	{
1965	struct intel_crtc_state *new_crtc_state =
1966	intel_atomic_get_new_crtc_state(state, crtc);
1967	const struct intel_crtc_state *old_crtc_state =
1968	intel_atomic_get_old_crtc_state(state, crtc);
1969	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1970	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1971	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1972	int level;
1973
1974	if (!new_crtc_state->hw.active ||
1975	intel_crtc_needs_modeset(crtc_state: new_crtc_state)) {
1976	*intermediate = *optimal;
1977
1978	intermediate->cxsr = false;
1979	goto out;
1980	}
1981
1982	intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1983	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1984	!new_crtc_state->disable_cxsr;
1985
1986	for (level = 0; level < intermediate->num_levels; level++) {
1987	enum plane_id plane_id;
1988
1989	for_each_plane_id_on_crtc(crtc, plane_id) {
1990	intermediate->wm[level].plane[plane_id] =
1991	min(optimal->wm[level].plane[plane_id],
1992	active->wm[level].plane[plane_id]);
1993	}
1994
1995	intermediate->sr[level].plane = min(optimal->sr[level].plane,
1996	active->sr[level].plane);
1997	intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1998	active->sr[level].cursor);
1999	}
2000
2001	vlv_invalidate_wms(crtc, wm_state: intermediate, level);
2002
2003	out:
2004	/*
2005	* If our intermediate WM are identical to the final WM, then we can
2006	* omit the post-vblank programming; only update if it's different.
2007	*/
2008	if (memcmp(p: intermediate, q: optimal, size: sizeof(*intermediate)) != 0)
2009	new_crtc_state->wm.need_postvbl_update = true;
2010
2011	return 0;
2012	}
2013
2014	static int vlv_compute_watermarks(struct intel_atomic_state *state,
2015	struct intel_crtc *crtc)
2016	{
2017	int ret;
2018
2019	ret = vlv_compute_pipe_wm(state, crtc);
2020	if (ret)
2021	return ret;
2022
2023	ret = vlv_compute_intermediate_wm(state, crtc);
2024	if (ret)
2025	return ret;
2026
2027	return 0;
2028	}
2029
2030	static void vlv_merge_wm(struct intel_display *display,
2031	struct vlv_wm_values *wm)
2032	{
2033	struct intel_crtc *crtc;
2034	int num_active_pipes = 0;
2035
2036	wm->level = display->wm.num_levels - 1;
2037	wm->cxsr = true;
2038
2039	for_each_intel_crtc(display->drm, crtc) {
2040	const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2041
2042	if (!crtc->active)
2043	continue;
2044
2045	if (!wm_state->cxsr)
2046	wm->cxsr = false;
2047
2048	num_active_pipes++;
2049	wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2050	}
2051
2052	if (num_active_pipes != 1)
2053	wm->cxsr = false;
2054
2055	if (num_active_pipes > 1)
2056	wm->level = VLV_WM_LEVEL_PM2;
2057
2058	for_each_intel_crtc(display->drm, crtc) {
2059	const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2060	enum pipe pipe = crtc->pipe;
2061
2062	wm->pipe[pipe] = wm_state->wm[wm->level];
2063	if (crtc->active && wm->cxsr)
2064	wm->sr = wm_state->sr[wm->level];
2065
2066	wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2067	wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2068	wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2069	wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2070	}
2071	}
2072
2073	static void vlv_program_watermarks(struct intel_display *display)
2074	{
2075	struct vlv_wm_values *old_wm = &display->wm.vlv;
2076	struct vlv_wm_values new_wm = {};
2077
2078	vlv_merge_wm(display, wm: &new_wm);
2079
2080	if (memcmp(p: old_wm, q: &new_wm, size: sizeof(new_wm)) == 0)
2081	return;
2082
2083	if (is_disabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_DDR_DVFS))
2084	chv_set_memory_dvfs(display, enable: false);
2085
2086	if (is_disabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_PM5))
2087	chv_set_memory_pm5(display, enable: false);
2088
2089	if (is_disabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
2090	_intel_set_memory_cxsr(display, enable: false);
2091
2092	vlv_write_wm_values(display, wm: &new_wm);
2093
2094	if (is_enabling(old: old_wm->cxsr, new: new_wm.cxsr, threshold: true))
2095	_intel_set_memory_cxsr(display, enable: true);
2096
2097	if (is_enabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_PM5))
2098	chv_set_memory_pm5(display, enable: true);
2099
2100	if (is_enabling(old: old_wm->level, new: new_wm.level, threshold: VLV_WM_LEVEL_DDR_DVFS))
2101	chv_set_memory_dvfs(display, enable: true);
2102
2103	*old_wm = new_wm;
2104	}
2105
2106	static void vlv_initial_watermarks(struct intel_atomic_state *state,
2107	struct intel_crtc *crtc)
2108	{
2109	struct intel_display *display = to_intel_display(crtc);
2110	const struct intel_crtc_state *crtc_state =
2111	intel_atomic_get_new_crtc_state(state, crtc);
2112
2113	mutex_lock(&display->wm.wm_mutex);
2114	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2115	vlv_program_watermarks(display);
2116	mutex_unlock(lock: &display->wm.wm_mutex);
2117	}
2118
2119	static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2120	struct intel_crtc *crtc)
2121	{
2122	struct intel_display *display = to_intel_display(crtc);
2123	const struct intel_crtc_state *crtc_state =
2124	intel_atomic_get_new_crtc_state(state, crtc);
2125
2126	if (!crtc_state->wm.need_postvbl_update)
2127	return;
2128
2129	mutex_lock(&display->wm.wm_mutex);
2130	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2131	vlv_program_watermarks(display);
2132	mutex_unlock(lock: &display->wm.wm_mutex);
2133	}
2134
2135	static void i965_update_wm(struct intel_display *display)
2136	{
2137	struct intel_crtc *crtc;
2138	int srwm = 1;
2139	int cursor_sr = 16;
2140	bool cxsr_enabled;
2141
2142	/* Calc sr entries for one plane configs */
2143	crtc = single_enabled_crtc(display);
2144	if (crtc) {
2145	/* self-refresh has much higher latency */
2146	static const int sr_latency_ns = 12000;
2147	const struct drm_display_mode *pipe_mode =
2148	&crtc->config->hw.pipe_mode;
2149	const struct drm_framebuffer *fb =
2150	crtc->base.primary->state->fb;
2151	int pixel_rate = crtc->config->pixel_rate;
2152	int htotal = pipe_mode->crtc_htotal;
2153	int width = drm_rect_width(r: &crtc->base.primary->state->src) >> 16;
2154	int cpp = fb->format->cpp[0];
2155	int entries;
2156
2157	entries = intel_wm_method2(pixel_rate, htotal,
2158	width, cpp, latency: sr_latency_ns / 100);
2159	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2160	srwm = I965_FIFO_SIZE - entries;
2161	if (srwm < 0)
2162	srwm = 1;
2163	srwm &= 0x1ff;
2164	drm_dbg_kms(display->drm,
2165	"self-refresh entries: %d, wm: %d\n",
2166	entries, srwm);
2167
2168	entries = intel_wm_method2(pixel_rate, htotal,
2169	width: crtc->base.cursor->state->crtc_w, cpp: 4,
2170	latency: sr_latency_ns / 100);
2171	entries = DIV_ROUND_UP(entries,
2172	i965_cursor_wm_info.cacheline_size) +
2173	i965_cursor_wm_info.guard_size;
2174
2175	cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2176	if (cursor_sr > i965_cursor_wm_info.max_wm)
2177	cursor_sr = i965_cursor_wm_info.max_wm;
2178
2179	drm_dbg_kms(display->drm,
2180	"self-refresh watermark: display plane %d "
2181	"cursor %d\n", srwm, cursor_sr);
2182
2183	cxsr_enabled = true;
2184	} else {
2185	cxsr_enabled = false;
2186	/* Turn off self refresh if both pipes are enabled */
2187	intel_set_memory_cxsr(display, enable: false);
2188	}
2189
2190	drm_dbg_kms(display->drm,
2191	"Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2192	srwm);
2193
2194	/* 965 has limitations... */
2195	intel_de_write(display, DSPFW1(display),
2196	FW_WM(srwm, SR) |
2197	FW_WM(8, CURSORB) |
2198	FW_WM(8, PLANEB) |
2199	FW_WM(8, PLANEA));
2200	intel_de_write(display, DSPFW2(display),
2201	FW_WM(8, CURSORA) |
2202	FW_WM(8, PLANEC_OLD));
2203	/* update cursor SR watermark */
2204	intel_de_write(display, DSPFW3(display),
2205	FW_WM(cursor_sr, CURSOR_SR));
2206
2207	if (cxsr_enabled)
2208	intel_set_memory_cxsr(display, enable: true);
2209	}
2210
2211	#undef FW_WM
2212
2213	static struct intel_crtc *intel_crtc_for_plane(struct intel_display *display,
2214	enum i9xx_plane_id i9xx_plane)
2215	{
2216	struct intel_plane *plane;
2217
2218	for_each_intel_plane(display->drm, plane) {
2219	if (plane->id == PLANE_PRIMARY &&
2220	plane->i9xx_plane == i9xx_plane)
2221	return intel_crtc_for_pipe(display, pipe: plane->pipe);
2222	}
2223
2224	return NULL;
2225	}
2226
2227	static void i9xx_update_wm(struct intel_display *display)
2228	{
2229	const struct intel_watermark_params *wm_info;
2230	u32 fwater_lo;
2231	u32 fwater_hi;
2232	int cwm, srwm = 1;
2233	int fifo_size;
2234	int planea_wm, planeb_wm;
2235	struct intel_crtc *crtc;
2236
2237	if (display->platform.i945gm)
2238	wm_info = &i945_wm_info;
2239	else if (DISPLAY_VER(display) != 2)
2240	wm_info = &i915_wm_info;
2241	else
2242	wm_info = &i830_a_wm_info;
2243
2244	if (DISPLAY_VER(display) == 2)
2245	fifo_size = i830_get_fifo_size(display, i9xx_plane: PLANE_A);
2246	else
2247	fifo_size = i9xx_get_fifo_size(display, i9xx_plane: PLANE_A);
2248	crtc = intel_crtc_for_plane(display, i9xx_plane: PLANE_A);
2249	if (intel_crtc_active(crtc)) {
2250	const struct drm_framebuffer *fb =
2251	crtc->base.primary->state->fb;
2252	int cpp;
2253
2254	if (DISPLAY_VER(display) == 2)
2255	cpp = 4;
2256	else
2257	cpp = fb->format->cpp[0];
2258
2259	planea_wm = intel_calculate_wm(display, pixel_rate: crtc->config->pixel_rate,
2260	wm: wm_info, fifo_size, cpp,
2261	latency_ns: pessimal_latency_ns);
2262	} else {
2263	planea_wm = fifo_size - wm_info->guard_size;
2264	if (planea_wm > (long)wm_info->max_wm)
2265	planea_wm = wm_info->max_wm;
2266	}
2267
2268	if (DISPLAY_VER(display) == 2)
2269	wm_info = &i830_bc_wm_info;
2270
2271	if (DISPLAY_VER(display) == 2)
2272	fifo_size = i830_get_fifo_size(display, i9xx_plane: PLANE_B);
2273	else
2274	fifo_size = i9xx_get_fifo_size(display, i9xx_plane: PLANE_B);
2275	crtc = intel_crtc_for_plane(display, i9xx_plane: PLANE_B);
2276	if (intel_crtc_active(crtc)) {
2277	const struct drm_framebuffer *fb =
2278	crtc->base.primary->state->fb;
2279	int cpp;
2280
2281	if (DISPLAY_VER(display) == 2)
2282	cpp = 4;
2283	else
2284	cpp = fb->format->cpp[0];
2285
2286	planeb_wm = intel_calculate_wm(display, pixel_rate: crtc->config->pixel_rate,
2287	wm: wm_info, fifo_size, cpp,
2288	latency_ns: pessimal_latency_ns);
2289	} else {
2290	planeb_wm = fifo_size - wm_info->guard_size;
2291	if (planeb_wm > (long)wm_info->max_wm)
2292	planeb_wm = wm_info->max_wm;
2293	}
2294
2295	drm_dbg_kms(display->drm,
2296	"FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2297
2298	crtc = single_enabled_crtc(display);
2299	if (display->platform.i915gm && crtc) {
2300	const struct drm_framebuffer *fb =
2301	crtc->base.primary->state->fb;
2302
2303	/* self-refresh seems busted with untiled */
2304	if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
2305	crtc = NULL;
2306	}
2307
2308	/*
2309	* Overlay gets an aggressive default since video jitter is bad.
2310	*/
2311	cwm = 2;
2312
2313	/* Play safe and disable self-refresh before adjusting watermarks. */
2314	intel_set_memory_cxsr(display, enable: false);
2315
2316	/* Calc sr entries for one plane configs */
2317	if (HAS_FW_BLC(display) && crtc) {
2318	/* self-refresh has much higher latency */
2319	static const int sr_latency_ns = 6000;
2320	const struct drm_display_mode *pipe_mode =
2321	&crtc->config->hw.pipe_mode;
2322	const struct drm_framebuffer *fb =
2323	crtc->base.primary->state->fb;
2324	int pixel_rate = crtc->config->pixel_rate;
2325	int htotal = pipe_mode->crtc_htotal;
2326	int width = drm_rect_width(r: &crtc->base.primary->state->src) >> 16;
2327	int cpp;
2328	int entries;
2329
2330	if (display->platform.i915gm || display->platform.i945gm)
2331	cpp = 4;
2332	else
2333	cpp = fb->format->cpp[0];
2334
2335	entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2336	latency: sr_latency_ns / 100);
2337	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2338	drm_dbg_kms(display->drm,
2339	"self-refresh entries: %d\n", entries);
2340	srwm = wm_info->fifo_size - entries;
2341	if (srwm < 0)
2342	srwm = 1;
2343
2344	if (display->platform.i945g || display->platform.i945gm)
2345	intel_de_write(display, FW_BLC_SELF,
2346	FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2347	else
2348	intel_de_write(display, FW_BLC_SELF, val: srwm & 0x3f);
2349	}
2350
2351	drm_dbg_kms(display->drm,
2352	"Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2353	planea_wm, planeb_wm, cwm, srwm);
2354
2355	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2356	fwater_hi = (cwm & 0x1f);
2357
2358	/* Set request length to 8 cachelines per fetch */
2359	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2360	fwater_hi = fwater_hi | (1 << 8);
2361
2362	intel_de_write(display, FW_BLC, val: fwater_lo);
2363	intel_de_write(display, FW_BLC2, val: fwater_hi);
2364
2365	if (crtc)
2366	intel_set_memory_cxsr(display, enable: true);
2367	}
2368
2369	static void i845_update_wm(struct intel_display *display)
2370	{
2371	struct intel_crtc *crtc;
2372	u32 fwater_lo;
2373	int planea_wm;
2374
2375	crtc = single_enabled_crtc(display);
2376	if (crtc == NULL)
2377	return;
2378
2379	planea_wm = intel_calculate_wm(display, pixel_rate: crtc->config->pixel_rate,
2380	wm: &i845_wm_info,
2381	fifo_size: i845_get_fifo_size(display, i9xx_plane: PLANE_A),
2382	cpp: 4, latency_ns: pessimal_latency_ns);
2383	fwater_lo = intel_de_read(display, FW_BLC) & ~0xfff;
2384	fwater_lo |= (3<<8) | planea_wm;
2385
2386	drm_dbg_kms(display->drm,
2387	"Setting FIFO watermarks - A: %d\n", planea_wm);
2388
2389	intel_de_write(display, FW_BLC, val: fwater_lo);
2390	}
2391
2392	/* latency must be in 0.1us units. */
2393	static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2394	unsigned int cpp,
2395	unsigned int latency)
2396	{
2397	unsigned int ret;
2398
2399	ret = intel_wm_method1(pixel_rate, cpp, latency);
2400	ret = DIV_ROUND_UP(ret, 64) + 2;
2401
2402	return ret;
2403	}
2404
2405	/* latency must be in 0.1us units. */
2406	static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2407	unsigned int htotal,
2408	unsigned int width,
2409	unsigned int cpp,
2410	unsigned int latency)
2411	{
2412	unsigned int ret;
2413
2414	ret = intel_wm_method2(pixel_rate, htotal,
2415	width, cpp, latency);
2416	ret = DIV_ROUND_UP(ret, 64) + 2;
2417
2418	return ret;
2419	}
2420
2421	static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2422	{
2423	/*
2424	* Neither of these should be possible since this function shouldn't be
2425	* called if the CRTC is off or the plane is invisible. But let's be
2426	* extra paranoid to avoid a potential divide-by-zero if we screw up
2427	* elsewhere in the driver.
2428	*/
2429	if (WARN_ON(!cpp))
2430	return 0;
2431	if (WARN_ON(!horiz_pixels))
2432	return 0;
2433
2434	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2435	}
2436
2437	struct ilk_wm_maximums {
2438	u16 pri;
2439	u16 spr;
2440	u16 cur;
2441	u16 fbc;
2442	};
2443
2444	/*
2445	* For both WM_PIPE and WM_LP.
2446	* mem_value must be in 0.1us units.
2447	*/
2448	static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2449	const struct intel_plane_state *plane_state,
2450	u32 mem_value, bool is_lp)
2451	{
2452	u32 method1, method2;
2453	int cpp;
2454
2455	if (mem_value == 0)
2456	return U32_MAX;
2457
2458	if (!intel_wm_plane_visible(crtc_state, plane_state))
2459	return 0;
2460
2461	cpp = plane_state->hw.fb->format->cpp[0];
2462
2463	method1 = ilk_wm_method1(pixel_rate: crtc_state->pixel_rate, cpp, latency: mem_value);
2464
2465	if (!is_lp)
2466	return method1;
2467
2468	method2 = ilk_wm_method2(pixel_rate: crtc_state->pixel_rate,
2469	htotal: crtc_state->hw.pipe_mode.crtc_htotal,
2470	width: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2471	cpp, latency: mem_value);
2472
2473	return min(method1, method2);
2474	}
2475
2476	/*
2477	* For both WM_PIPE and WM_LP.
2478	* mem_value must be in 0.1us units.
2479	*/
2480	static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2481	const struct intel_plane_state *plane_state,
2482	u32 mem_value)
2483	{
2484	u32 method1, method2;
2485	int cpp;
2486
2487	if (mem_value == 0)
2488	return U32_MAX;
2489
2490	if (!intel_wm_plane_visible(crtc_state, plane_state))
2491	return 0;
2492
2493	cpp = plane_state->hw.fb->format->cpp[0];
2494
2495	method1 = ilk_wm_method1(pixel_rate: crtc_state->pixel_rate, cpp, latency: mem_value);
2496	method2 = ilk_wm_method2(pixel_rate: crtc_state->pixel_rate,
2497	htotal: crtc_state->hw.pipe_mode.crtc_htotal,
2498	width: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2499	cpp, latency: mem_value);
2500	return min(method1, method2);
2501	}
2502
2503	/*
2504	* For both WM_PIPE and WM_LP.
2505	* mem_value must be in 0.1us units.
2506	*/
2507	static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2508	const struct intel_plane_state *plane_state,
2509	u32 mem_value)
2510	{
2511	int cpp;
2512
2513	if (mem_value == 0)
2514	return U32_MAX;
2515
2516	if (!intel_wm_plane_visible(crtc_state, plane_state))
2517	return 0;
2518
2519	cpp = plane_state->hw.fb->format->cpp[0];
2520
2521	return ilk_wm_method2(pixel_rate: crtc_state->pixel_rate,
2522	htotal: crtc_state->hw.pipe_mode.crtc_htotal,
2523	width: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2524	cpp, latency: mem_value);
2525	}
2526
2527	/* Only for WM_LP. */
2528	static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2529	const struct intel_plane_state *plane_state,
2530	u32 pri_val)
2531	{
2532	int cpp;
2533
2534	if (!intel_wm_plane_visible(crtc_state, plane_state))
2535	return 0;
2536
2537	cpp = plane_state->hw.fb->format->cpp[0];
2538
2539	return ilk_wm_fbc(pri_val, horiz_pixels: drm_rect_width(r: &plane_state->uapi.src) >> 16,
2540	cpp);
2541	}
2542
2543	static unsigned int
2544	ilk_display_fifo_size(struct intel_display *display)
2545	{
2546	if (DISPLAY_VER(display) >= 8)
2547	return 3072;
2548	else if (DISPLAY_VER(display) >= 7)
2549	return 768;
2550	else
2551	return 512;
2552	}
2553
2554	static unsigned int
2555	ilk_plane_wm_reg_max(struct intel_display *display,
2556	int level, bool is_sprite)
2557	{
2558	if (DISPLAY_VER(display) >= 8)
2559	/* BDW primary/sprite plane watermarks */
2560	return level == 0 ? 255 : 2047;
2561	else if (DISPLAY_VER(display) >= 7)
2562	/* IVB/HSW primary/sprite plane watermarks */
2563	return level == 0 ? 127 : 1023;
2564	else if (!is_sprite)
2565	/* ILK/SNB primary plane watermarks */
2566	return level == 0 ? 127 : 511;
2567	else
2568	/* ILK/SNB sprite plane watermarks */
2569	return level == 0 ? 63 : 255;
2570	}
2571
2572	static unsigned int
2573	ilk_cursor_wm_reg_max(struct intel_display *display, int level)
2574	{
2575	if (DISPLAY_VER(display) >= 7)
2576	return level == 0 ? 63 : 255;
2577	else
2578	return level == 0 ? 31 : 63;
2579	}
2580
2581	static unsigned int ilk_fbc_wm_reg_max(struct intel_display *display)
2582	{
2583	if (DISPLAY_VER(display) >= 8)
2584	return 31;
2585	else
2586	return 15;
2587	}
2588
2589	/* Calculate the maximum primary/sprite plane watermark */
2590	static unsigned int ilk_plane_wm_max(struct intel_display *display,
2591	int level,
2592	const struct intel_wm_config *config,
2593	enum intel_ddb_partitioning ddb_partitioning,
2594	bool is_sprite)
2595	{
2596	unsigned int fifo_size = ilk_display_fifo_size(display);
2597
2598	/* if sprites aren't enabled, sprites get nothing */
2599	if (is_sprite && !config->sprites_enabled)
2600	return 0;
2601
2602	/* HSW allows LP1+ watermarks even with multiple pipes */
2603	if (level == 0 || config->num_pipes_active > 1) {
2604	fifo_size /= INTEL_NUM_PIPES(display);
2605
2606	/*
2607	* For some reason the non self refresh
2608	* FIFO size is only half of the self
2609	* refresh FIFO size on ILK/SNB.
2610	*/
2611	if (DISPLAY_VER(display) < 7)
2612	fifo_size /= 2;
2613	}
2614
2615	if (config->sprites_enabled) {
2616	/* level 0 is always calculated with 1:1 split */
2617	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2618	if (is_sprite)
2619	fifo_size *= 5;
2620	fifo_size /= 6;
2621	} else {
2622	fifo_size /= 2;
2623	}
2624	}
2625
2626	/* clamp to max that the registers can hold */
2627	return min(fifo_size, ilk_plane_wm_reg_max(display, level, is_sprite));
2628	}
2629
2630	/* Calculate the maximum cursor plane watermark */
2631	static unsigned int ilk_cursor_wm_max(struct intel_display *display,
2632	int level,
2633	const struct intel_wm_config *config)
2634	{
2635	/* HSW LP1+ watermarks w/ multiple pipes */
2636	if (level > 0 && config->num_pipes_active > 1)
2637	return 64;
2638
2639	/* otherwise just report max that registers can hold */
2640	return ilk_cursor_wm_reg_max(display, level);
2641	}
2642
2643	static void ilk_compute_wm_maximums(struct intel_display *display,
2644	int level,
2645	const struct intel_wm_config *config,
2646	enum intel_ddb_partitioning ddb_partitioning,
2647	struct ilk_wm_maximums *max)
2648	{
2649	max->pri = ilk_plane_wm_max(display, level, config, ddb_partitioning, is_sprite: false);
2650	max->spr = ilk_plane_wm_max(display, level, config, ddb_partitioning, is_sprite: true);
2651	max->cur = ilk_cursor_wm_max(display, level, config);
2652	max->fbc = ilk_fbc_wm_reg_max(display);
2653	}
2654
2655	static void ilk_compute_wm_reg_maximums(struct intel_display *display,
2656	int level,
2657	struct ilk_wm_maximums *max)
2658	{
2659	max->pri = ilk_plane_wm_reg_max(display, level, is_sprite: false);
2660	max->spr = ilk_plane_wm_reg_max(display, level, is_sprite: true);
2661	max->cur = ilk_cursor_wm_reg_max(display, level);
2662	max->fbc = ilk_fbc_wm_reg_max(display);
2663	}
2664
2665	static bool ilk_validate_wm_level(struct intel_display *display,
2666	int level,
2667	const struct ilk_wm_maximums *max,
2668	struct intel_wm_level *result)
2669	{
2670	bool ret;
2671
2672	/* already determined to be invalid? */
2673	if (!result->enable)
2674	return false;
2675
2676	result->enable = result->pri_val <= max->pri &&
2677	result->spr_val <= max->spr &&
2678	result->cur_val <= max->cur;
2679
2680	ret = result->enable;
2681
2682	/*
2683	* HACK until we can pre-compute everything,
2684	* and thus fail gracefully if LP0 watermarks
2685	* are exceeded...
2686	*/
2687	if (level == 0 && !result->enable) {
2688	if (result->pri_val > max->pri)
2689	drm_dbg_kms(display->drm,
2690	"Primary WM%d too large %u (max %u)\n",
2691	level, result->pri_val, max->pri);
2692	if (result->spr_val > max->spr)
2693	drm_dbg_kms(display->drm,
2694	"Sprite WM%d too large %u (max %u)\n",
2695	level, result->spr_val, max->spr);
2696	if (result->cur_val > max->cur)
2697	drm_dbg_kms(display->drm,
2698	"Cursor WM%d too large %u (max %u)\n",
2699	level, result->cur_val, max->cur);
2700
2701	result->pri_val = min_t(u32, result->pri_val, max->pri);
2702	result->spr_val = min_t(u32, result->spr_val, max->spr);
2703	result->cur_val = min_t(u32, result->cur_val, max->cur);
2704	result->enable = true;
2705	}
2706
2707	return ret;
2708	}
2709
2710	static void ilk_compute_wm_level(struct intel_display *display,
2711	const struct intel_crtc *crtc,
2712	int level,
2713	struct intel_crtc_state *crtc_state,
2714	const struct intel_plane_state *pristate,
2715	const struct intel_plane_state *sprstate,
2716	const struct intel_plane_state *curstate,
2717	struct intel_wm_level *result)
2718	{
2719	u16 pri_latency = display->wm.pri_latency[level];
2720	u16 spr_latency = display->wm.spr_latency[level];
2721	u16 cur_latency = display->wm.cur_latency[level];
2722
2723	/* WM1+ latency values stored in 0.5us units */
2724	if (level > 0) {
2725	pri_latency *= 5;
2726	spr_latency *= 5;
2727	cur_latency *= 5;
2728	}
2729
2730	if (pristate) {
2731	result->pri_val = ilk_compute_pri_wm(crtc_state, plane_state: pristate,
2732	mem_value: pri_latency, is_lp: level);
2733	result->fbc_val = ilk_compute_fbc_wm(crtc_state, plane_state: pristate, pri_val: result->pri_val);
2734	}
2735
2736	if (sprstate)
2737	result->spr_val = ilk_compute_spr_wm(crtc_state, plane_state: sprstate, mem_value: spr_latency);
2738
2739	if (curstate)
2740	result->cur_val = ilk_compute_cur_wm(crtc_state, plane_state: curstate, mem_value: cur_latency);
2741
2742	result->enable = true;
2743	}
2744
2745	static void hsw_read_wm_latency(struct intel_display *display, u16 wm[])
2746	{
2747	struct drm_i915_private *i915 = to_i915(dev: display->drm);
2748	u64 sskpd;
2749
2750	display->wm.num_levels = 5;
2751
2752	sskpd = intel_uncore_read64(uncore: &i915->uncore, MCH_SSKPD);
2753
2754	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2755	if (wm[0] == 0)
2756	wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2757	wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2758	wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2759	wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2760	wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2761	}
2762
2763	static void snb_read_wm_latency(struct intel_display *display, u16 wm[])
2764	{
2765	struct drm_i915_private *i915 = to_i915(dev: display->drm);
2766	u32 sskpd;
2767
2768	display->wm.num_levels = 4;
2769
2770	sskpd = intel_uncore_read(uncore: &i915->uncore, MCH_SSKPD);
2771
2772	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2773	wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2774	wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2775	wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2776	}
2777
2778	static void ilk_read_wm_latency(struct intel_display *display, u16 wm[])
2779	{
2780	struct drm_i915_private *i915 = to_i915(dev: display->drm);
2781	u32 mltr;
2782
2783	display->wm.num_levels = 3;
2784
2785	mltr = intel_uncore_read(uncore: &i915->uncore, MLTR_ILK);
2786
2787	/* ILK primary LP0 latency is 700 ns */
2788	wm[0] = 7;
2789	wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2790	wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2791	}
2792
2793	static void intel_fixup_spr_wm_latency(struct intel_display *display, u16 wm[5])
2794	{
2795	/* ILK sprite LP0 latency is 1300 ns */
2796	if (DISPLAY_VER(display) == 5)
2797	wm[0] = 13;
2798	}
2799
2800	static void intel_fixup_cur_wm_latency(struct intel_display *display, u16 wm[5])
2801	{
2802	/* ILK cursor LP0 latency is 1300 ns */
2803	if (DISPLAY_VER(display) == 5)
2804	wm[0] = 13;
2805	}
2806
2807	static bool ilk_increase_wm_latency(struct intel_display *display, u16 wm[5], u16 min)
2808	{
2809	int level;
2810
2811	if (wm[0] >= min)
2812	return false;
2813
2814	wm[0] = max(wm[0], min);
2815	for (level = 1; level < display->wm.num_levels; level++)
2816	wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2817
2818	return true;
2819	}
2820
2821	static void snb_wm_latency_quirk(struct intel_display *display)
2822	{
2823	bool changed;
2824
2825	/*
2826	* The BIOS provided WM memory latency values are often
2827	* inadequate for high resolution displays. Adjust them.
2828	*/
2829	changed = ilk_increase_wm_latency(display, wm: display->wm.pri_latency, min: 12);
2830	changed |= ilk_increase_wm_latency(display, wm: display->wm.spr_latency, min: 12);
2831	changed |= ilk_increase_wm_latency(display, wm: display->wm.cur_latency, min: 12);
2832
2833	if (!changed)
2834	return;
2835
2836	drm_dbg_kms(display->drm,
2837	"WM latency values increased to avoid potential underruns\n");
2838	intel_print_wm_latency(display, name: "Primary", wm: display->wm.pri_latency);
2839	intel_print_wm_latency(display, name: "Sprite", wm: display->wm.spr_latency);
2840	intel_print_wm_latency(display, name: "Cursor", wm: display->wm.cur_latency);
2841	}
2842
2843	static void snb_wm_lp3_irq_quirk(struct intel_display *display)
2844	{
2845	/*
2846	* On some SNB machines (Thinkpad X220 Tablet at least)
2847	* LP3 usage can cause vblank interrupts to be lost.
2848	* The DEIIR bit will go high but it looks like the CPU
2849	* never gets interrupted.
2850	*
2851	* It's not clear whether other interrupt source could
2852	* be affected or if this is somehow limited to vblank
2853	* interrupts only. To play it safe we disable LP3
2854	* watermarks entirely.
2855	*/
2856	if (display->wm.pri_latency[3] == 0 &&
2857	display->wm.spr_latency[3] == 0 &&
2858	display->wm.cur_latency[3] == 0)
2859	return;
2860
2861	display->wm.pri_latency[3] = 0;
2862	display->wm.spr_latency[3] = 0;
2863	display->wm.cur_latency[3] = 0;
2864
2865	drm_dbg_kms(display->drm,
2866	"LP3 watermarks disabled due to potential for lost interrupts\n");
2867	intel_print_wm_latency(display, name: "Primary", wm: display->wm.pri_latency);
2868	intel_print_wm_latency(display, name: "Sprite", wm: display->wm.spr_latency);
2869	intel_print_wm_latency(display, name: "Cursor", wm: display->wm.cur_latency);
2870	}
2871
2872	static void ilk_setup_wm_latency(struct intel_display *display)
2873	{
2874	if (display->platform.broadwell || display->platform.haswell)
2875	hsw_read_wm_latency(display, wm: display->wm.pri_latency);
2876	else if (DISPLAY_VER(display) >= 6)
2877	snb_read_wm_latency(display, wm: display->wm.pri_latency);
2878	else
2879	ilk_read_wm_latency(display, wm: display->wm.pri_latency);
2880
2881	memcpy(display->wm.spr_latency, display->wm.pri_latency,
2882	sizeof(display->wm.pri_latency));
2883	memcpy(display->wm.cur_latency, display->wm.pri_latency,
2884	sizeof(display->wm.pri_latency));
2885
2886	intel_fixup_spr_wm_latency(display, wm: display->wm.spr_latency);
2887	intel_fixup_cur_wm_latency(display, wm: display->wm.cur_latency);
2888
2889	intel_print_wm_latency(display, name: "Primary", wm: display->wm.pri_latency);
2890	intel_print_wm_latency(display, name: "Sprite", wm: display->wm.spr_latency);
2891	intel_print_wm_latency(display, name: "Cursor", wm: display->wm.cur_latency);
2892
2893	if (DISPLAY_VER(display) == 6) {
2894	snb_wm_latency_quirk(display);
2895	snb_wm_lp3_irq_quirk(display);
2896	}
2897	}
2898
2899	static bool ilk_validate_pipe_wm(struct intel_display *display,
2900	struct intel_pipe_wm *pipe_wm)
2901	{
2902	/* LP0 watermark maximums depend on this pipe alone */
2903	const struct intel_wm_config config = {
2904	.num_pipes_active = 1,
2905	.sprites_enabled = pipe_wm->sprites_enabled,
2906	.sprites_scaled = pipe_wm->sprites_scaled,
2907	};
2908	struct ilk_wm_maximums max;
2909
2910	/* LP0 watermarks always use 1/2 DDB partitioning */
2911	ilk_compute_wm_maximums(display, level: 0, config: &config, ddb_partitioning: INTEL_DDB_PART_1_2, max: &max);
2912
2913	/* At least LP0 must be valid */
2914	if (!ilk_validate_wm_level(display, level: 0, max: &max, result: &pipe_wm->wm[0])) {
2915	drm_dbg_kms(display->drm, "LP0 watermark invalid\n");
2916	return false;
2917	}
2918
2919	return true;
2920	}
2921
2922	/* Compute new watermarks for the pipe */
2923	static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2924	struct intel_crtc *crtc)
2925	{
2926	struct intel_display *display = to_intel_display(state);
2927	struct intel_crtc_state *crtc_state =
2928	intel_atomic_get_new_crtc_state(state, crtc);
2929	struct intel_pipe_wm *pipe_wm;
2930	struct intel_plane *plane;
2931	const struct intel_plane_state *plane_state;
2932	const struct intel_plane_state *pristate = NULL;
2933	const struct intel_plane_state *sprstate = NULL;
2934	const struct intel_plane_state *curstate = NULL;
2935	struct ilk_wm_maximums max;
2936	int level, usable_level;
2937
2938	pipe_wm = &crtc_state->wm.ilk.optimal;
2939
2940	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2941	if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2942	pristate = plane_state;
2943	else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2944	sprstate = plane_state;
2945	else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2946	curstate = plane_state;
2947	}
2948
2949	pipe_wm->pipe_enabled = crtc_state->hw.active;
2950	pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2951	pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2952
2953	usable_level = display->wm.num_levels - 1;
2954
2955	/* ILK/SNB: LP2+ watermarks only w/o sprites */
2956	if (DISPLAY_VER(display) < 7 && pipe_wm->sprites_enabled)
2957	usable_level = 1;
2958
2959	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2960	if (pipe_wm->sprites_scaled)
2961	usable_level = 0;
2962
2963	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2964	ilk_compute_wm_level(display, crtc, level: 0, crtc_state,
2965	pristate, sprstate, curstate, result: &pipe_wm->wm[0]);
2966
2967	if (!ilk_validate_pipe_wm(display, pipe_wm))
2968	return -EINVAL;
2969
2970	ilk_compute_wm_reg_maximums(display, level: 1, max: &max);
2971
2972	for (level = 1; level <= usable_level; level++) {
2973	struct intel_wm_level *wm = &pipe_wm->wm[level];
2974
2975	ilk_compute_wm_level(display, crtc, level, crtc_state,
2976	pristate, sprstate, curstate, result: wm);
2977
2978	/*
2979	* Disable any watermark level that exceeds the
2980	* register maximums since such watermarks are
2981	* always invalid.
2982	*/
2983	if (!ilk_validate_wm_level(display, level, max: &max, result: wm)) {
2984	memset(wm, 0, sizeof(*wm));
2985	break;
2986	}
2987	}
2988
2989	return 0;
2990	}
2991
2992	/*
2993	* Build a set of 'intermediate' watermark values that satisfy both the old
2994	* state and the new state. These can be programmed to the hardware
2995	* immediately.
2996	*/
2997	static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2998	struct intel_crtc *crtc)
2999	{
3000	struct intel_display *display = to_intel_display(crtc);
3001	struct intel_crtc_state *new_crtc_state =
3002	intel_atomic_get_new_crtc_state(state, crtc);
3003	const struct intel_crtc_state *old_crtc_state =
3004	intel_atomic_get_old_crtc_state(state, crtc);
3005	struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate;
3006	const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal;
3007	const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal;
3008	int level;
3009
3010	/*
3011	* Start with the final, target watermarks, then combine with the
3012	* currently active watermarks to get values that are safe both before
3013	* and after the vblank.
3014	*/
3015	*intermediate = *optimal;
3016	if (!new_crtc_state->hw.active ||
3017	intel_crtc_needs_modeset(crtc_state: new_crtc_state) ||
3018	state->skip_intermediate_wm)
3019	return 0;
3020
3021	intermediate->pipe_enabled |= active->pipe_enabled;
3022	intermediate->sprites_enabled |= active->sprites_enabled;
3023	intermediate->sprites_scaled |= active->sprites_scaled;
3024
3025	for (level = 0; level < display->wm.num_levels; level++) {
3026	struct intel_wm_level *intermediate_wm = &intermediate->wm[level];
3027	const struct intel_wm_level *active_wm = &active->wm[level];
3028
3029	intermediate_wm->enable &= active_wm->enable;
3030	intermediate_wm->pri_val = max(intermediate_wm->pri_val,
3031	active_wm->pri_val);
3032	intermediate_wm->spr_val = max(intermediate_wm->spr_val,
3033	active_wm->spr_val);
3034	intermediate_wm->cur_val = max(intermediate_wm->cur_val,
3035	active_wm->cur_val);
3036	intermediate_wm->fbc_val = max(intermediate_wm->fbc_val,
3037	active_wm->fbc_val);
3038	}
3039
3040	/*
3041	* We need to make sure that these merged watermark values are
3042	* actually a valid configuration themselves. If they're not,
3043	* there's no safe way to transition from the old state to
3044	* the new state, so we need to fail the atomic transaction.
3045	*/
3046	if (!ilk_validate_pipe_wm(display, pipe_wm: intermediate))
3047	return -EINVAL;
3048
3049	/*
3050	* If our intermediate WM are identical to the final WM, then we can
3051	* omit the post-vblank programming; only update if it's different.
3052	*/
3053	if (memcmp(p: intermediate, q: optimal, size: sizeof(*intermediate)) != 0)
3054	new_crtc_state->wm.need_postvbl_update = true;
3055
3056	return 0;
3057	}
3058
3059	static int ilk_compute_watermarks(struct intel_atomic_state *state,
3060	struct intel_crtc *crtc)
3061	{
3062	int ret;
3063
3064	ret = ilk_compute_pipe_wm(state, crtc);
3065	if (ret)
3066	return ret;
3067
3068	ret = ilk_compute_intermediate_wm(state, crtc);
3069	if (ret)
3070	return ret;
3071
3072	return 0;
3073	}
3074
3075	/*
3076	* Merge the watermarks from all active pipes for a specific level.
3077	*/
3078	static void ilk_merge_wm_level(struct intel_display *display,
3079	int level,
3080	struct intel_wm_level *ret_wm)
3081	{
3082	const struct intel_crtc *crtc;
3083
3084	ret_wm->enable = true;
3085
3086	for_each_intel_crtc(display->drm, crtc) {
3087	const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3088	const struct intel_wm_level *wm = &active->wm[level];
3089
3090	if (!active->pipe_enabled)
3091	continue;
3092
3093	/*
3094	* The watermark values may have been used in the past,
3095	* so we must maintain them in the registers for some
3096	* time even if the level is now disabled.
3097	*/
3098	if (!wm->enable)
3099	ret_wm->enable = false;
3100
3101	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3102	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3103	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3104	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3105	}
3106	}
3107
3108	/*
3109	* Merge all low power watermarks for all active pipes.
3110	*/
3111	static void ilk_wm_merge(struct intel_display *display,
3112	const struct intel_wm_config *config,
3113	const struct ilk_wm_maximums *max,
3114	struct intel_pipe_wm *merged)
3115	{
3116	int level, num_levels = display->wm.num_levels;
3117	int last_enabled_level = num_levels - 1;
3118
3119	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3120	if ((DISPLAY_VER(display) < 7 || display->platform.ivybridge) &&
3121	config->num_pipes_active > 1)
3122	last_enabled_level = 0;
3123
3124	/* ILK: FBC WM must be disabled always */
3125	merged->fbc_wm_enabled = DISPLAY_VER(display) >= 6;
3126
3127	/* merge each WM1+ level */
3128	for (level = 1; level < num_levels; level++) {
3129	struct intel_wm_level *wm = &merged->wm[level];
3130
3131	ilk_merge_wm_level(display, level, ret_wm: wm);
3132
3133	if (level > last_enabled_level)
3134	wm->enable = false;
3135	else if (!ilk_validate_wm_level(display, level, max, result: wm))
3136	/* make sure all following levels get disabled */
3137	last_enabled_level = level - 1;
3138
3139	/*
3140	* The spec says it is preferred to disable
3141	* FBC WMs instead of disabling a WM level.
3142	*/
3143	if (wm->fbc_val > max->fbc) {
3144	if (wm->enable)
3145	merged->fbc_wm_enabled = false;
3146	wm->fbc_val = 0;
3147	}
3148	}
3149
3150	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3151	if (DISPLAY_VER(display) == 5 && HAS_FBC(display) &&
3152	display->params.enable_fbc && !merged->fbc_wm_enabled) {
3153	for (level = 2; level < num_levels; level++) {
3154	struct intel_wm_level *wm = &merged->wm[level];
3155
3156	wm->enable = false;
3157	}
3158	}
3159	}
3160
3161	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3162	{
3163	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3164	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3165	}
3166
3167	/* The value we need to program into the WM_LPx latency field */
3168	static unsigned int ilk_wm_lp_latency(struct intel_display *display,
3169	int level)
3170	{
3171	if (display->platform.haswell || display->platform.broadwell)
3172	return 2 * level;
3173	else
3174	return display->wm.pri_latency[level];
3175	}
3176
3177	static void ilk_compute_wm_results(struct intel_display *display,
3178	const struct intel_pipe_wm *merged,
3179	enum intel_ddb_partitioning partitioning,
3180	struct ilk_wm_values *results)
3181	{
3182	struct intel_crtc *crtc;
3183	int level, wm_lp;
3184
3185	results->enable_fbc_wm = merged->fbc_wm_enabled;
3186	results->partitioning = partitioning;
3187
3188	/* LP1+ register values */
3189	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3190	const struct intel_wm_level *r;
3191
3192	level = ilk_wm_lp_to_level(wm_lp, pipe_wm: merged);
3193
3194	r = &merged->wm[level];
3195
3196	/*
3197	* Maintain the watermark values even if the level is
3198	* disabled. Doing otherwise could cause underruns.
3199	*/
3200	results->wm_lp[wm_lp - 1] =
3201	WM_LP_LATENCY(ilk_wm_lp_latency(display, level)) |
3202	WM_LP_PRIMARY(r->pri_val) |
3203	WM_LP_CURSOR(r->cur_val);
3204
3205	if (r->enable)
3206	results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3207
3208	if (DISPLAY_VER(display) >= 8)
3209	results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3210	else
3211	results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3212
3213	results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3214
3215	/*
3216	* Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3217	* level is disabled. Doing otherwise could cause underruns.
3218	*/
3219	if (DISPLAY_VER(display) < 7 && r->spr_val) {
3220	drm_WARN_ON(display->drm, wm_lp != 1);
3221	results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3222	}
3223	}
3224
3225	/* LP0 register values */
3226	for_each_intel_crtc(display->drm, crtc) {
3227	enum pipe pipe = crtc->pipe;
3228	const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3229	const struct intel_wm_level *r = &pipe_wm->wm[0];
3230
3231	if (drm_WARN_ON(display->drm, !r->enable))
3232	continue;
3233
3234	results->wm_pipe[pipe] =
3235	WM0_PIPE_PRIMARY(r->pri_val) |
3236	WM0_PIPE_SPRITE(r->spr_val) |
3237	WM0_PIPE_CURSOR(r->cur_val);
3238	}
3239	}
3240
3241	/*
3242	* Find the result with the highest level enabled. Check for enable_fbc_wm in
3243	* case both are at the same level. Prefer r1 in case they're the same.
3244	*/
3245	static struct intel_pipe_wm *
3246	ilk_find_best_result(struct intel_display *display,
3247	struct intel_pipe_wm *r1,
3248	struct intel_pipe_wm *r2)
3249	{
3250	int level, level1 = 0, level2 = 0;
3251
3252	for (level = 1; level < display->wm.num_levels; level++) {
3253	if (r1->wm[level].enable)
3254	level1 = level;
3255	if (r2->wm[level].enable)
3256	level2 = level;
3257	}
3258
3259	if (level1 == level2) {
3260	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3261	return r2;
3262	else
3263	return r1;
3264	} else if (level1 > level2) {
3265	return r1;
3266	} else {
3267	return r2;
3268	}
3269	}
3270
3271	/* dirty bits used to track which watermarks need changes */
3272	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3273	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3274	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3275	#define WM_DIRTY_FBC (1 << 24)
3276	#define WM_DIRTY_DDB (1 << 25)
3277
3278	static unsigned int ilk_compute_wm_dirty(struct intel_display *display,
3279	const struct ilk_wm_values *old,
3280	const struct ilk_wm_values *new)
3281	{
3282	unsigned int dirty = 0;
3283	enum pipe pipe;
3284	int wm_lp;
3285
3286	for_each_pipe(display, pipe) {
3287	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3288	dirty |= WM_DIRTY_PIPE(pipe);
3289	/* Must disable LP1+ watermarks too */
3290	dirty |= WM_DIRTY_LP_ALL;
3291	}
3292	}
3293
3294	if (old->enable_fbc_wm != new->enable_fbc_wm) {
3295	dirty |= WM_DIRTY_FBC;
3296	/* Must disable LP1+ watermarks too */
3297	dirty |= WM_DIRTY_LP_ALL;
3298	}
3299
3300	if (old->partitioning != new->partitioning) {
3301	dirty |= WM_DIRTY_DDB;
3302	/* Must disable LP1+ watermarks too */
3303	dirty |= WM_DIRTY_LP_ALL;
3304	}
3305
3306	/* LP1+ watermarks already deemed dirty, no need to continue */
3307	if (dirty & WM_DIRTY_LP_ALL)
3308	return dirty;
3309
3310	/* Find the lowest numbered LP1+ watermark in need of an update... */
3311	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3312	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3313	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3314	break;
3315	}
3316
3317	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3318	for (; wm_lp <= 3; wm_lp++)
3319	dirty |= WM_DIRTY_LP(wm_lp);
3320
3321	return dirty;
3322	}
3323
3324	static bool _ilk_disable_lp_wm(struct intel_display *display,
3325	unsigned int dirty)
3326	{
3327	struct ilk_wm_values *previous = &display->wm.hw;
3328	bool changed = false;
3329
3330	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3331	previous->wm_lp[2] &= ~WM_LP_ENABLE;
3332	intel_de_write(display, WM3_LP_ILK, val: previous->wm_lp[2]);
3333	changed = true;
3334	}
3335	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3336	previous->wm_lp[1] &= ~WM_LP_ENABLE;
3337	intel_de_write(display, WM2_LP_ILK, val: previous->wm_lp[1]);
3338	changed = true;
3339	}
3340	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3341	previous->wm_lp[0] &= ~WM_LP_ENABLE;
3342	intel_de_write(display, WM1_LP_ILK, val: previous->wm_lp[0]);
3343	changed = true;
3344	}
3345
3346	/*
3347	* Don't touch WM_LP_SPRITE_ENABLE here.
3348	* Doing so could cause underruns.
3349	*/
3350
3351	return changed;
3352	}
3353
3354	/*
3355	* The spec says we shouldn't write when we don't need, because every write
3356	* causes WMs to be re-evaluated, expending some power.
3357	*/
3358	static void ilk_write_wm_values(struct intel_display *display,
3359	struct ilk_wm_values *results)
3360	{
3361	struct ilk_wm_values *previous = &display->wm.hw;
3362	unsigned int dirty;
3363
3364	dirty = ilk_compute_wm_dirty(display, old: previous, new: results);
3365	if (!dirty)
3366	return;
3367
3368	_ilk_disable_lp_wm(display, dirty);
3369
3370	if (dirty & WM_DIRTY_PIPE(PIPE_A))
3371	intel_de_write(display, WM0_PIPE_ILK(PIPE_A), val: results->wm_pipe[0]);
3372	if (dirty & WM_DIRTY_PIPE(PIPE_B))
3373	intel_de_write(display, WM0_PIPE_ILK(PIPE_B), val: results->wm_pipe[1]);
3374	if (dirty & WM_DIRTY_PIPE(PIPE_C))
3375	intel_de_write(display, WM0_PIPE_ILK(PIPE_C), val: results->wm_pipe[2]);
3376
3377	if (dirty & WM_DIRTY_DDB) {
3378	if (display->platform.haswell || display->platform.broadwell)
3379	intel_de_rmw(display, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3380	set: results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3381	WM_MISC_DATA_PARTITION_5_6);
3382	else
3383	intel_de_rmw(display, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3384	set: results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3385	DISP_DATA_PARTITION_5_6);
3386	}
3387
3388	if (dirty & WM_DIRTY_FBC)
3389	intel_de_rmw(display, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3390	set: results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3391
3392	if (dirty & WM_DIRTY_LP(1) &&
3393	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3394	intel_de_write(display, WM1S_LP_ILK, val: results->wm_lp_spr[0]);
3395
3396	if (DISPLAY_VER(display) >= 7) {
3397	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3398	intel_de_write(display, WM2S_LP_IVB, val: results->wm_lp_spr[1]);
3399	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3400	intel_de_write(display, WM3S_LP_IVB, val: results->wm_lp_spr[2]);
3401	}
3402
3403	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3404	intel_de_write(display, WM1_LP_ILK, val: results->wm_lp[0]);
3405	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3406	intel_de_write(display, WM2_LP_ILK, val: results->wm_lp[1]);
3407	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3408	intel_de_write(display, WM3_LP_ILK, val: results->wm_lp[2]);
3409
3410	display->wm.hw = *results;
3411	}
3412
3413	bool ilk_disable_cxsr(struct intel_display *display)
3414	{
3415	return _ilk_disable_lp_wm(display, WM_DIRTY_LP_ALL);
3416	}
3417
3418	static void ilk_compute_wm_config(struct intel_display *display,
3419	struct intel_wm_config *config)
3420	{
3421	struct intel_crtc *crtc;
3422
3423	/* Compute the currently _active_ config */
3424	for_each_intel_crtc(display->drm, crtc) {
3425	const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3426
3427	if (!wm->pipe_enabled)
3428	continue;
3429
3430	config->sprites_enabled |= wm->sprites_enabled;
3431	config->sprites_scaled |= wm->sprites_scaled;
3432	config->num_pipes_active++;
3433	}
3434	}
3435
3436	static void ilk_program_watermarks(struct intel_display *display)
3437	{
3438	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3439	struct ilk_wm_maximums max;
3440	struct intel_wm_config config = {};
3441	struct ilk_wm_values results = {};
3442	enum intel_ddb_partitioning partitioning;
3443
3444	ilk_compute_wm_config(display, config: &config);
3445
3446	ilk_compute_wm_maximums(display, level: 1, config: &config, ddb_partitioning: INTEL_DDB_PART_1_2, max: &max);
3447	ilk_wm_merge(display, config: &config, max: &max, merged: &lp_wm_1_2);
3448
3449	/* 5/6 split only in single pipe config on IVB+ */
3450	if (DISPLAY_VER(display) >= 7 &&
3451	config.num_pipes_active == 1 && config.sprites_enabled) {
3452	ilk_compute_wm_maximums(display, level: 1, config: &config, ddb_partitioning: INTEL_DDB_PART_5_6, max: &max);
3453	ilk_wm_merge(display, config: &config, max: &max, merged: &lp_wm_5_6);
3454
3455	best_lp_wm = ilk_find_best_result(display, r1: &lp_wm_1_2, r2: &lp_wm_5_6);
3456	} else {
3457	best_lp_wm = &lp_wm_1_2;
3458	}
3459
3460	partitioning = (best_lp_wm == &lp_wm_1_2) ?
3461	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3462
3463	ilk_compute_wm_results(display, merged: best_lp_wm, partitioning, results: &results);
3464
3465	ilk_write_wm_values(display, results: &results);
3466	}
3467
3468	static void ilk_initial_watermarks(struct intel_atomic_state *state,
3469	struct intel_crtc *crtc)
3470	{
3471	struct intel_display *display = to_intel_display(crtc);
3472	const struct intel_crtc_state *crtc_state =
3473	intel_atomic_get_new_crtc_state(state, crtc);
3474
3475	mutex_lock(&display->wm.wm_mutex);
3476	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3477	ilk_program_watermarks(display);
3478	mutex_unlock(lock: &display->wm.wm_mutex);
3479	}
3480
3481	static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3482	struct intel_crtc *crtc)
3483	{
3484	struct intel_display *display = to_intel_display(crtc);
3485	const struct intel_crtc_state *crtc_state =
3486	intel_atomic_get_new_crtc_state(state, crtc);
3487
3488	if (!crtc_state->wm.need_postvbl_update)
3489	return;
3490
3491	mutex_lock(&display->wm.wm_mutex);
3492	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3493	ilk_program_watermarks(display);
3494	mutex_unlock(lock: &display->wm.wm_mutex);
3495	}
3496
3497	static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3498	{
3499	struct intel_display *display = to_intel_display(crtc);
3500	struct ilk_wm_values *hw = &display->wm.hw;
3501	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3502	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3503	enum pipe pipe = crtc->pipe;
3504
3505	hw->wm_pipe[pipe] = intel_de_read(display, WM0_PIPE_ILK(pipe));
3506
3507	memset(active, 0, sizeof(*active));
3508
3509	active->pipe_enabled = crtc->active;
3510
3511	if (active->pipe_enabled) {
3512	u32 tmp = hw->wm_pipe[pipe];
3513
3514	/*
3515	* For active pipes LP0 watermark is marked as
3516	* enabled, and LP1+ watermaks as disabled since
3517	* we can't really reverse compute them in case
3518	* multiple pipes are active.
3519	*/
3520	active->wm[0].enable = true;
3521	active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3522	active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3523	active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3524	} else {
3525	int level;
3526
3527	/*
3528	* For inactive pipes, all watermark levels
3529	* should be marked as enabled but zeroed,
3530	* which is what we'd compute them to.
3531	*/
3532	for (level = 0; level < display->wm.num_levels; level++)
3533	active->wm[level].enable = true;
3534	}
3535
3536	crtc->wm.active.ilk = *active;
3537	}
3538
3539	static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3540	{
3541	struct drm_plane *plane;
3542	struct intel_crtc *crtc;
3543
3544	for_each_intel_crtc(state->dev, crtc) {
3545	struct intel_crtc_state *crtc_state;
3546
3547	crtc_state = intel_atomic_get_crtc_state(state, crtc);
3548	if (IS_ERR(ptr: crtc_state))
3549	return PTR_ERR(ptr: crtc_state);
3550
3551	if (crtc_state->hw.active) {
3552	/*
3553	* Preserve the inherited flag to avoid
3554	* taking the full modeset path.
3555	*/
3556	crtc_state->inherited = true;
3557	}
3558	}
3559
3560	drm_for_each_plane(plane, state->dev) {
3561	struct drm_plane_state *plane_state;
3562
3563	plane_state = drm_atomic_get_plane_state(state, plane);
3564	if (IS_ERR(ptr: plane_state))
3565	return PTR_ERR(ptr: plane_state);
3566	}
3567
3568	return 0;
3569	}
3570
3571	/*
3572	* Calculate what we think the watermarks should be for the state we've read
3573	* out of the hardware and then immediately program those watermarks so that
3574	* we ensure the hardware settings match our internal state.
3575	*
3576	* We can calculate what we think WM's should be by creating a duplicate of the
3577	* current state (which was constructed during hardware readout) and running it
3578	* through the atomic check code to calculate new watermark values in the
3579	* state object.
3580	*/
3581	void ilk_wm_sanitize(struct intel_display *display)
3582	{
3583	struct drm_atomic_state *state;
3584	struct intel_atomic_state *intel_state;
3585	struct intel_crtc *crtc;
3586	struct intel_crtc_state *crtc_state;
3587	struct drm_modeset_acquire_ctx ctx;
3588	int ret;
3589	int i;
3590
3591	/* Only supported on platforms that use atomic watermark design */
3592	if (!display->funcs.wm->optimize_watermarks)
3593	return;
3594
3595	if (drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 9))
3596	return;
3597
3598	state = drm_atomic_state_alloc(dev: display->drm);
3599	if (drm_WARN_ON(display->drm, !state))
3600	return;
3601
3602	intel_state = to_intel_atomic_state(state);
3603
3604	drm_modeset_acquire_init(ctx: &ctx, flags: 0);
3605
3606	state->acquire_ctx = &ctx;
3607	to_intel_atomic_state(state)->internal = true;
3608
3609	retry:
3610	/*
3611	* Hardware readout is the only time we don't want to calculate
3612	* intermediate watermarks (since we don't trust the current
3613	* watermarks).
3614	*/
3615	if (!HAS_GMCH(display))
3616	intel_state->skip_intermediate_wm = true;
3617
3618	ret = ilk_sanitize_watermarks_add_affected(state);
3619	if (ret)
3620	goto fail;
3621
3622	ret = intel_atomic_check(dev: display->drm, state);
3623	if (ret)
3624	goto fail;
3625
3626	/* Write calculated watermark values back */
3627	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3628	crtc_state->wm.need_postvbl_update = true;
3629	intel_optimize_watermarks(state: intel_state, crtc);
3630
3631	to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3632	}
3633
3634	fail:
3635	if (ret == -EDEADLK) {
3636	drm_atomic_state_clear(state);
3637	drm_modeset_backoff(ctx: &ctx);
3638	goto retry;
3639	}
3640
3641	/*
3642	* If we fail here, it means that the hardware appears to be
3643	* programmed in a way that shouldn't be possible, given our
3644	* understanding of watermark requirements. This might mean a
3645	* mistake in the hardware readout code or a mistake in the
3646	* watermark calculations for a given platform. Raise a WARN
3647	* so that this is noticeable.
3648	*
3649	* If this actually happens, we'll have to just leave the
3650	* BIOS-programmed watermarks untouched and hope for the best.
3651	*/
3652	drm_WARN(display->drm, ret,
3653	"Could not determine valid watermarks for inherited state\n");
3654
3655	drm_atomic_state_put(state);
3656
3657	drm_modeset_drop_locks(ctx: &ctx);
3658	drm_modeset_acquire_fini(ctx: &ctx);
3659	}
3660
3661	#define _FW_WM(value, plane) \
3662	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3663	#define _FW_WM_VLV(value, plane) \
3664	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3665
3666	static void g4x_read_wm_values(struct intel_display *display,
3667	struct g4x_wm_values *wm)
3668	{
3669	u32 tmp;
3670
3671	tmp = intel_de_read(display, DSPFW1(display));
3672	wm->sr.plane = _FW_WM(tmp, SR);
3673	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3674	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3675	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3676
3677	tmp = intel_de_read(display, DSPFW2(display));
3678	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3679	wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3680	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3681	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3682	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3683	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3684
3685	tmp = intel_de_read(display, DSPFW3(display));
3686	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3687	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3688	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3689	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3690	}
3691
3692	static void vlv_read_wm_values(struct intel_display *display,
3693	struct vlv_wm_values *wm)
3694	{
3695	enum pipe pipe;
3696	u32 tmp;
3697
3698	for_each_pipe(display, pipe) {
3699	tmp = intel_de_read(display, VLV_DDL(pipe));
3700
3701	wm->ddl[pipe].plane[PLANE_PRIMARY] =
3702	(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3703	wm->ddl[pipe].plane[PLANE_CURSOR] =
3704	(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3705	wm->ddl[pipe].plane[PLANE_SPRITE0] =
3706	(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3707	wm->ddl[pipe].plane[PLANE_SPRITE1] =
3708	(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3709	}
3710
3711	tmp = intel_de_read(display, DSPFW1(display));
3712	wm->sr.plane = _FW_WM(tmp, SR);
3713	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3714	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3715	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3716
3717	tmp = intel_de_read(display, DSPFW2(display));
3718	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3719	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3720	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3721
3722	tmp = intel_de_read(display, DSPFW3(display));
3723	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3724
3725	if (display->platform.cherryview) {
3726	tmp = intel_de_read(display, DSPFW7_CHV);
3727	wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3728	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3729
3730	tmp = intel_de_read(display, DSPFW8_CHV);
3731	wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3732	wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3733
3734	tmp = intel_de_read(display, DSPFW9_CHV);
3735	wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3736	wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3737
3738	tmp = intel_de_read(display, DSPHOWM);
3739	wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3740	wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3741	wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3742	wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3743	wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3744	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3745	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3746	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3747	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3748	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3749	} else {
3750	tmp = intel_de_read(display, DSPFW7);
3751	wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3752	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3753
3754	tmp = intel_de_read(display, DSPHOWM);
3755	wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3756	wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3757	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3758	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3759	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3760	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3761	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3762	}
3763	}
3764
3765	#undef _FW_WM
3766	#undef _FW_WM_VLV
3767
3768	static void g4x_wm_get_hw_state(struct intel_display *display)
3769	{
3770	struct g4x_wm_values *wm = &display->wm.g4x;
3771	struct intel_crtc *crtc;
3772
3773	g4x_read_wm_values(display, wm);
3774
3775	wm->cxsr = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
3776
3777	for_each_intel_crtc(display->drm, crtc) {
3778	struct intel_crtc_state *crtc_state =
3779	to_intel_crtc_state(crtc->base.state);
3780	struct g4x_wm_state *active = &crtc->wm.active.g4x;
3781	struct g4x_pipe_wm *raw;
3782	enum pipe pipe = crtc->pipe;
3783	enum plane_id plane_id;
3784	int level, max_level;
3785
3786	active->cxsr = wm->cxsr;
3787	active->hpll_en = wm->hpll_en;
3788	active->fbc_en = wm->fbc_en;
3789
3790	active->sr = wm->sr;
3791	active->hpll = wm->hpll;
3792
3793	for_each_plane_id_on_crtc(crtc, plane_id) {
3794	active->wm.plane[plane_id] =
3795	wm->pipe[pipe].plane[plane_id];
3796	}
3797
3798	if (wm->cxsr && wm->hpll_en)
3799	max_level = G4X_WM_LEVEL_HPLL;
3800	else if (wm->cxsr)
3801	max_level = G4X_WM_LEVEL_SR;
3802	else
3803	max_level = G4X_WM_LEVEL_NORMAL;
3804
3805	level = G4X_WM_LEVEL_NORMAL;
3806	raw = &crtc_state->wm.g4x.raw[level];
3807	for_each_plane_id_on_crtc(crtc, plane_id)
3808	raw->plane[plane_id] = active->wm.plane[plane_id];
3809
3810	level = G4X_WM_LEVEL_SR;
3811	if (level > max_level)
3812	goto out;
3813
3814	raw = &crtc_state->wm.g4x.raw[level];
3815	raw->plane[PLANE_PRIMARY] = active->sr.plane;
3816	raw->plane[PLANE_CURSOR] = active->sr.cursor;
3817	raw->plane[PLANE_SPRITE0] = 0;
3818	raw->fbc = active->sr.fbc;
3819
3820	level = G4X_WM_LEVEL_HPLL;
3821	if (level > max_level)
3822	goto out;
3823
3824	raw = &crtc_state->wm.g4x.raw[level];
3825	raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3826	raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3827	raw->plane[PLANE_SPRITE0] = 0;
3828	raw->fbc = active->hpll.fbc;
3829
3830	level++;
3831	out:
3832	for_each_plane_id_on_crtc(crtc, plane_id)
3833	g4x_raw_plane_wm_set(crtc_state, level,
3834	plane_id, USHRT_MAX);
3835	g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3836
3837	g4x_invalidate_wms(crtc, wm_state: active, level);
3838
3839	crtc_state->wm.g4x.optimal = *active;
3840	crtc_state->wm.g4x.intermediate = *active;
3841
3842	drm_dbg_kms(display->drm,
3843	"Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3844	pipe_name(pipe),
3845	wm->pipe[pipe].plane[PLANE_PRIMARY],
3846	wm->pipe[pipe].plane[PLANE_CURSOR],
3847	wm->pipe[pipe].plane[PLANE_SPRITE0]);
3848	}
3849
3850	drm_dbg_kms(display->drm,
3851	"Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3852	wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3853	drm_dbg_kms(display->drm,
3854	"Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3855	wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3856	drm_dbg_kms(display->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3857	str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3858	str_yes_no(wm->fbc_en));
3859	}
3860
3861	static void g4x_wm_sanitize(struct intel_display *display)
3862	{
3863	struct intel_plane *plane;
3864	struct intel_crtc *crtc;
3865
3866	mutex_lock(&display->wm.wm_mutex);
3867
3868	for_each_intel_plane(display->drm, plane) {
3869	struct intel_crtc *crtc =
3870	intel_crtc_for_pipe(display, pipe: plane->pipe);
3871	struct intel_crtc_state *crtc_state =
3872	to_intel_crtc_state(crtc->base.state);
3873	struct intel_plane_state *plane_state =
3874	to_intel_plane_state(plane->base.state);
3875	enum plane_id plane_id = plane->id;
3876	int level;
3877
3878	if (plane_state->uapi.visible)
3879	continue;
3880
3881	for (level = 0; level < display->wm.num_levels; level++) {
3882	struct g4x_pipe_wm *raw =
3883	&crtc_state->wm.g4x.raw[level];
3884
3885	raw->plane[plane_id] = 0;
3886
3887	if (plane_id == PLANE_PRIMARY)
3888	raw->fbc = 0;
3889	}
3890	}
3891
3892	for_each_intel_crtc(display->drm, crtc) {
3893	struct intel_crtc_state *crtc_state =
3894	to_intel_crtc_state(crtc->base.state);
3895	int ret;
3896
3897	ret = _g4x_compute_pipe_wm(crtc_state);
3898	drm_WARN_ON(display->drm, ret);
3899
3900	crtc_state->wm.g4x.intermediate =
3901	crtc_state->wm.g4x.optimal;
3902	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3903	}
3904
3905	g4x_program_watermarks(display);
3906
3907	mutex_unlock(lock: &display->wm.wm_mutex);
3908	}
3909
3910	static void vlv_wm_get_hw_state(struct intel_display *display)
3911	{
3912	struct vlv_wm_values *wm = &display->wm.vlv;
3913	struct intel_crtc *crtc;
3914	u32 val;
3915	int ret;
3916
3917	vlv_read_wm_values(display, wm);
3918
3919	wm->cxsr = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3920	wm->level = VLV_WM_LEVEL_PM2;
3921
3922	if (display->platform.cherryview) {
3923	vlv_punit_get(drm: display->drm);
3924
3925	val = vlv_punit_read(drm: display->drm, PUNIT_REG_DSPSSPM);
3926	if (val & DSP_MAXFIFO_PM5_ENABLE)
3927	wm->level = VLV_WM_LEVEL_PM5;
3928
3929	/*
3930	* If DDR DVFS is disabled in the BIOS, Punit
3931	* will never ack the request. So if that happens
3932	* assume we don't have to enable/disable DDR DVFS
3933	* dynamically. To test that just set the REQ_ACK
3934	* bit to poke the Punit, but don't change the
3935	* HIGH/LOW bits so that we don't actually change
3936	* the current state.
3937	*/
3938	val = vlv_punit_read(drm: display->drm, PUNIT_REG_DDR_SETUP2);
3939	val |= FORCE_DDR_FREQ_REQ_ACK;
3940	vlv_punit_write(drm: display->drm, PUNIT_REG_DDR_SETUP2, val);
3941
3942	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
3943	(val & FORCE_DDR_FREQ_REQ_ACK) == 0,
3944	500, 3000, false);
3945	if (ret) {
3946	drm_dbg_kms(display->drm,
3947	"Punit not acking DDR DVFS request, "
3948	"assuming DDR DVFS is disabled\n");
3949	display->wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3950	} else {
3951	val = vlv_punit_read(drm: display->drm, PUNIT_REG_DDR_SETUP2);
3952	if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3953	wm->level = VLV_WM_LEVEL_DDR_DVFS;
3954	}
3955
3956	vlv_punit_put(drm: display->drm);
3957	}
3958
3959	for_each_intel_crtc(display->drm, crtc) {
3960	struct intel_crtc_state *crtc_state =
3961	to_intel_crtc_state(crtc->base.state);
3962	struct vlv_wm_state *active = &crtc->wm.active.vlv;
3963	const struct vlv_fifo_state *fifo_state =
3964	&crtc_state->wm.vlv.fifo_state;
3965	enum pipe pipe = crtc->pipe;
3966	enum plane_id plane_id;
3967	int level;
3968
3969	vlv_get_fifo_size(crtc_state);
3970
3971	active->num_levels = wm->level + 1;
3972	active->cxsr = wm->cxsr;
3973
3974	for (level = 0; level < active->num_levels; level++) {
3975	struct g4x_pipe_wm *raw =
3976	&crtc_state->wm.vlv.raw[level];
3977
3978	active->sr[level].plane = wm->sr.plane;
3979	active->sr[level].cursor = wm->sr.cursor;
3980
3981	for_each_plane_id_on_crtc(crtc, plane_id) {
3982	active->wm[level].plane[plane_id] =
3983	wm->pipe[pipe].plane[plane_id];
3984
3985	raw->plane[plane_id] =
3986	vlv_invert_wm_value(wm: active->wm[level].plane[plane_id],
3987	fifo_size: fifo_state->plane[plane_id]);
3988	}
3989	}
3990
3991	for_each_plane_id_on_crtc(crtc, plane_id)
3992	vlv_raw_plane_wm_set(crtc_state, level,
3993	plane_id, USHRT_MAX);
3994	vlv_invalidate_wms(crtc, wm_state: active, level);
3995
3996	crtc_state->wm.vlv.optimal = *active;
3997	crtc_state->wm.vlv.intermediate = *active;
3998
3999	drm_dbg_kms(display->drm,
4000	"Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
4001	pipe_name(pipe),
4002	wm->pipe[pipe].plane[PLANE_PRIMARY],
4003	wm->pipe[pipe].plane[PLANE_CURSOR],
4004	wm->pipe[pipe].plane[PLANE_SPRITE0],
4005	wm->pipe[pipe].plane[PLANE_SPRITE1]);
4006	}
4007
4008	drm_dbg_kms(display->drm,
4009	"Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4010	wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4011	}
4012
4013	static void vlv_wm_sanitize(struct intel_display *display)
4014	{
4015	struct intel_plane *plane;
4016	struct intel_crtc *crtc;
4017
4018	mutex_lock(&display->wm.wm_mutex);
4019
4020	for_each_intel_plane(display->drm, plane) {
4021	struct intel_crtc *crtc =
4022	intel_crtc_for_pipe(display, pipe: plane->pipe);
4023	struct intel_crtc_state *crtc_state =
4024	to_intel_crtc_state(crtc->base.state);
4025	struct intel_plane_state *plane_state =
4026	to_intel_plane_state(plane->base.state);
4027	enum plane_id plane_id = plane->id;
4028	int level;
4029
4030	if (plane_state->uapi.visible)
4031	continue;
4032
4033	for (level = 0; level < display->wm.num_levels; level++) {
4034	struct g4x_pipe_wm *raw =
4035	&crtc_state->wm.vlv.raw[level];
4036
4037	raw->plane[plane_id] = 0;
4038	}
4039	}
4040
4041	for_each_intel_crtc(display->drm, crtc) {
4042	struct intel_crtc_state *crtc_state =
4043	to_intel_crtc_state(crtc->base.state);
4044	int ret;
4045
4046	ret = _vlv_compute_pipe_wm(crtc_state);
4047	drm_WARN_ON(display->drm, ret);
4048
4049	crtc_state->wm.vlv.intermediate =
4050	crtc_state->wm.vlv.optimal;
4051	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4052	}
4053
4054	vlv_program_watermarks(display);
4055
4056	mutex_unlock(lock: &display->wm.wm_mutex);
4057	}
4058
4059	/*
4060	* FIXME should probably kill this and improve
4061	* the real watermark readout/sanitation instead
4062	*/
4063	static void ilk_init_lp_watermarks(struct intel_display *display)
4064	{
4065	intel_de_rmw(display, WM3_LP_ILK, WM_LP_ENABLE, set: 0);
4066	intel_de_rmw(display, WM2_LP_ILK, WM_LP_ENABLE, set: 0);
4067	intel_de_rmw(display, WM1_LP_ILK, WM_LP_ENABLE, set: 0);
4068
4069	/*
4070	* Don't touch WM_LP_SPRITE_ENABLE here.
4071	* Doing so could cause underruns.
4072	*/
4073	}
4074
4075	static void ilk_wm_get_hw_state(struct intel_display *display)
4076	{
4077	struct ilk_wm_values *hw = &display->wm.hw;
4078	struct intel_crtc *crtc;
4079
4080	ilk_init_lp_watermarks(display);
4081
4082	for_each_intel_crtc(display->drm, crtc)
4083	ilk_pipe_wm_get_hw_state(crtc);
4084
4085	hw->wm_lp[0] = intel_de_read(display, WM1_LP_ILK);
4086	hw->wm_lp[1] = intel_de_read(display, WM2_LP_ILK);
4087	hw->wm_lp[2] = intel_de_read(display, WM3_LP_ILK);
4088
4089	hw->wm_lp_spr[0] = intel_de_read(display, WM1S_LP_ILK);
4090	if (DISPLAY_VER(display) >= 7) {
4091	hw->wm_lp_spr[1] = intel_de_read(display, WM2S_LP_IVB);
4092	hw->wm_lp_spr[2] = intel_de_read(display, WM3S_LP_IVB);
4093	}
4094
4095	if (display->platform.haswell || display->platform.broadwell)
4096	hw->partitioning = (intel_de_read(display, WM_MISC) &
4097	WM_MISC_DATA_PARTITION_5_6) ?
4098	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4099	else if (display->platform.ivybridge)
4100	hw->partitioning = (intel_de_read(display, DISP_ARB_CTL2) &
4101	DISP_DATA_PARTITION_5_6) ?
4102	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4103
4104	hw->enable_fbc_wm =
4105	!(intel_de_read(display, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4106	}
4107
4108	static const struct intel_wm_funcs ilk_wm_funcs = {
4109	.compute_watermarks = ilk_compute_watermarks,
4110	.initial_watermarks = ilk_initial_watermarks,
4111	.optimize_watermarks = ilk_optimize_watermarks,
4112	.get_hw_state = ilk_wm_get_hw_state,
4113	};
4114
4115	static const struct intel_wm_funcs vlv_wm_funcs = {
4116	.compute_watermarks = vlv_compute_watermarks,
4117	.initial_watermarks = vlv_initial_watermarks,
4118	.optimize_watermarks = vlv_optimize_watermarks,
4119	.atomic_update_watermarks = vlv_atomic_update_fifo,
4120	.get_hw_state = vlv_wm_get_hw_state,
4121	.sanitize = vlv_wm_sanitize,
4122	};
4123
4124	static const struct intel_wm_funcs g4x_wm_funcs = {
4125	.compute_watermarks = g4x_compute_watermarks,
4126	.initial_watermarks = g4x_initial_watermarks,
4127	.optimize_watermarks = g4x_optimize_watermarks,
4128	.get_hw_state = g4x_wm_get_hw_state,
4129	.sanitize = g4x_wm_sanitize,
4130	};
4131
4132	static const struct intel_wm_funcs pnv_wm_funcs = {
4133	.compute_watermarks = i9xx_compute_watermarks,
4134	.update_wm = pnv_update_wm,
4135	};
4136
4137	static const struct intel_wm_funcs i965_wm_funcs = {
4138	.compute_watermarks = i9xx_compute_watermarks,
4139	.update_wm = i965_update_wm,
4140	};
4141
4142	static const struct intel_wm_funcs i9xx_wm_funcs = {
4143	.compute_watermarks = i9xx_compute_watermarks,
4144	.update_wm = i9xx_update_wm,
4145	};
4146
4147	static const struct intel_wm_funcs i845_wm_funcs = {
4148	.compute_watermarks = i9xx_compute_watermarks,
4149	.update_wm = i845_update_wm,
4150	};
4151
4152	static const struct intel_wm_funcs nop_funcs = {
4153	};
4154
4155	void i9xx_wm_init(struct intel_display *display)
4156	{
4157	/* For FIFO watermark updates */
4158	if (HAS_PCH_SPLIT(display)) {
4159	ilk_setup_wm_latency(display);
4160	display->funcs.wm = &ilk_wm_funcs;
4161	} else if (display->platform.valleyview || display->platform.cherryview) {
4162	vlv_setup_wm_latency(display);
4163	display->funcs.wm = &vlv_wm_funcs;
4164	} else if (display->platform.g4x) {
4165	g4x_setup_wm_latency(display);
4166	display->funcs.wm = &g4x_wm_funcs;
4167	} else if (display->platform.pineview) {
4168	if (!pnv_get_cxsr_latency(display)) {
4169	drm_info(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
4170	/* Disable CxSR and never update its watermark again */
4171	intel_set_memory_cxsr(display, enable: false);
4172	display->funcs.wm = &nop_funcs;
4173	} else {
4174	display->funcs.wm = &pnv_wm_funcs;
4175	}
4176	} else if (DISPLAY_VER(display) == 4) {
4177	display->funcs.wm = &i965_wm_funcs;
4178	} else if (DISPLAY_VER(display) == 3) {
4179	display->funcs.wm = &i9xx_wm_funcs;
4180	} else if (DISPLAY_VER(display) == 2) {
4181	if (INTEL_NUM_PIPES(display) == 1)
4182	display->funcs.wm = &i845_wm_funcs;
4183	else
4184	display->funcs.wm = &i9xx_wm_funcs;
4185	} else {
4186	drm_err(display->drm,
4187	"unexpected fall-through in %s\n", __func__);
4188	display->funcs.wm = &nop_funcs;
4189	}
4190	}
4191