1	/* SPDX-License-Identifier: MIT */
2	/*
3	* Copyright © 2019 Intel Corporation
4	*/
5
6	#include <linux/iopoll.h>
7	#include <linux/string_helpers.h>
8
9	#include <drm/drm_print.h>
10
11	#include "soc/intel_dram.h"
12
13	#include "i915_drv.h"
14	#include "i915_irq.h"
15	#include "i915_reg.h"
16	#include "intel_backlight_regs.h"
17	#include "intel_cdclk.h"
18	#include "intel_clock_gating.h"
19	#include "intel_combo_phy.h"
20	#include "intel_de.h"
21	#include "intel_display_power.h"
22	#include "intel_display_power_map.h"
23	#include "intel_display_power_well.h"
24	#include "intel_display_regs.h"
25	#include "intel_display_rpm.h"
26	#include "intel_display_types.h"
27	#include "intel_display_utils.h"
28	#include "intel_dmc.h"
29	#include "intel_mchbar_regs.h"
30	#include "intel_pch_refclk.h"
31	#include "intel_pcode.h"
32	#include "intel_pmdemand.h"
33	#include "intel_pps_regs.h"
34	#include "intel_snps_phy.h"
35	#include "skl_watermark.h"
36	#include "skl_watermark_regs.h"
37	#include "vlv_sideband.h"
38
39	#define for_each_power_domain_well(__display, __power_well, __domain) \
40	for_each_power_well((__display), __power_well) \
41	for_each_if(test_bit((__domain), (__power_well)->domains.bits))
42
43	#define for_each_power_domain_well_reverse(__display, __power_well, __domain) \
44	for_each_power_well_reverse((__display), __power_well) \
45	for_each_if(test_bit((__domain), (__power_well)->domains.bits))
46
47	static const char *
48	intel_display_power_domain_str(enum intel_display_power_domain domain)
49	{
50	switch (domain) {
51	case POWER_DOMAIN_DISPLAY_CORE:
52	return "DISPLAY_CORE";
53	case POWER_DOMAIN_PIPE_A:
54	return "PIPE_A";
55	case POWER_DOMAIN_PIPE_B:
56	return "PIPE_B";
57	case POWER_DOMAIN_PIPE_C:
58	return "PIPE_C";
59	case POWER_DOMAIN_PIPE_D:
60	return "PIPE_D";
61	case POWER_DOMAIN_PIPE_PANEL_FITTER_A:
62	return "PIPE_PANEL_FITTER_A";
63	case POWER_DOMAIN_PIPE_PANEL_FITTER_B:
64	return "PIPE_PANEL_FITTER_B";
65	case POWER_DOMAIN_PIPE_PANEL_FITTER_C:
66	return "PIPE_PANEL_FITTER_C";
67	case POWER_DOMAIN_PIPE_PANEL_FITTER_D:
68	return "PIPE_PANEL_FITTER_D";
69	case POWER_DOMAIN_TRANSCODER_A:
70	return "TRANSCODER_A";
71	case POWER_DOMAIN_TRANSCODER_B:
72	return "TRANSCODER_B";
73	case POWER_DOMAIN_TRANSCODER_C:
74	return "TRANSCODER_C";
75	case POWER_DOMAIN_TRANSCODER_D:
76	return "TRANSCODER_D";
77	case POWER_DOMAIN_TRANSCODER_EDP:
78	return "TRANSCODER_EDP";
79	case POWER_DOMAIN_TRANSCODER_DSI_A:
80	return "TRANSCODER_DSI_A";
81	case POWER_DOMAIN_TRANSCODER_DSI_C:
82	return "TRANSCODER_DSI_C";
83	case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
84	return "TRANSCODER_VDSC_PW2";
85	case POWER_DOMAIN_PORT_DDI_LANES_A:
86	return "PORT_DDI_LANES_A";
87	case POWER_DOMAIN_PORT_DDI_LANES_B:
88	return "PORT_DDI_LANES_B";
89	case POWER_DOMAIN_PORT_DDI_LANES_C:
90	return "PORT_DDI_LANES_C";
91	case POWER_DOMAIN_PORT_DDI_LANES_D:
92	return "PORT_DDI_LANES_D";
93	case POWER_DOMAIN_PORT_DDI_LANES_E:
94	return "PORT_DDI_LANES_E";
95	case POWER_DOMAIN_PORT_DDI_LANES_F:
96	return "PORT_DDI_LANES_F";
97	case POWER_DOMAIN_PORT_DDI_LANES_TC1:
98	return "PORT_DDI_LANES_TC1";
99	case POWER_DOMAIN_PORT_DDI_LANES_TC2:
100	return "PORT_DDI_LANES_TC2";
101	case POWER_DOMAIN_PORT_DDI_LANES_TC3:
102	return "PORT_DDI_LANES_TC3";
103	case POWER_DOMAIN_PORT_DDI_LANES_TC4:
104	return "PORT_DDI_LANES_TC4";
105	case POWER_DOMAIN_PORT_DDI_LANES_TC5:
106	return "PORT_DDI_LANES_TC5";
107	case POWER_DOMAIN_PORT_DDI_LANES_TC6:
108	return "PORT_DDI_LANES_TC6";
109	case POWER_DOMAIN_PORT_DDI_IO_A:
110	return "PORT_DDI_IO_A";
111	case POWER_DOMAIN_PORT_DDI_IO_B:
112	return "PORT_DDI_IO_B";
113	case POWER_DOMAIN_PORT_DDI_IO_C:
114	return "PORT_DDI_IO_C";
115	case POWER_DOMAIN_PORT_DDI_IO_D:
116	return "PORT_DDI_IO_D";
117	case POWER_DOMAIN_PORT_DDI_IO_E:
118	return "PORT_DDI_IO_E";
119	case POWER_DOMAIN_PORT_DDI_IO_F:
120	return "PORT_DDI_IO_F";
121	case POWER_DOMAIN_PORT_DDI_IO_TC1:
122	return "PORT_DDI_IO_TC1";
123	case POWER_DOMAIN_PORT_DDI_IO_TC2:
124	return "PORT_DDI_IO_TC2";
125	case POWER_DOMAIN_PORT_DDI_IO_TC3:
126	return "PORT_DDI_IO_TC3";
127	case POWER_DOMAIN_PORT_DDI_IO_TC4:
128	return "PORT_DDI_IO_TC4";
129	case POWER_DOMAIN_PORT_DDI_IO_TC5:
130	return "PORT_DDI_IO_TC5";
131	case POWER_DOMAIN_PORT_DDI_IO_TC6:
132	return "PORT_DDI_IO_TC6";
133	case POWER_DOMAIN_PORT_DSI:
134	return "PORT_DSI";
135	case POWER_DOMAIN_PORT_CRT:
136	return "PORT_CRT";
137	case POWER_DOMAIN_PORT_OTHER:
138	return "PORT_OTHER";
139	case POWER_DOMAIN_VGA:
140	return "VGA";
141	case POWER_DOMAIN_AUDIO_MMIO:
142	return "AUDIO_MMIO";
143	case POWER_DOMAIN_AUDIO_PLAYBACK:
144	return "AUDIO_PLAYBACK";
145	case POWER_DOMAIN_AUX_IO_A:
146	return "AUX_IO_A";
147	case POWER_DOMAIN_AUX_IO_B:
148	return "AUX_IO_B";
149	case POWER_DOMAIN_AUX_IO_C:
150	return "AUX_IO_C";
151	case POWER_DOMAIN_AUX_IO_D:
152	return "AUX_IO_D";
153	case POWER_DOMAIN_AUX_IO_E:
154	return "AUX_IO_E";
155	case POWER_DOMAIN_AUX_IO_F:
156	return "AUX_IO_F";
157	case POWER_DOMAIN_AUX_A:
158	return "AUX_A";
159	case POWER_DOMAIN_AUX_B:
160	return "AUX_B";
161	case POWER_DOMAIN_AUX_C:
162	return "AUX_C";
163	case POWER_DOMAIN_AUX_D:
164	return "AUX_D";
165	case POWER_DOMAIN_AUX_E:
166	return "AUX_E";
167	case POWER_DOMAIN_AUX_F:
168	return "AUX_F";
169	case POWER_DOMAIN_AUX_USBC1:
170	return "AUX_USBC1";
171	case POWER_DOMAIN_AUX_USBC2:
172	return "AUX_USBC2";
173	case POWER_DOMAIN_AUX_USBC3:
174	return "AUX_USBC3";
175	case POWER_DOMAIN_AUX_USBC4:
176	return "AUX_USBC4";
177	case POWER_DOMAIN_AUX_USBC5:
178	return "AUX_USBC5";
179	case POWER_DOMAIN_AUX_USBC6:
180	return "AUX_USBC6";
181	case POWER_DOMAIN_AUX_TBT1:
182	return "AUX_TBT1";
183	case POWER_DOMAIN_AUX_TBT2:
184	return "AUX_TBT2";
185	case POWER_DOMAIN_AUX_TBT3:
186	return "AUX_TBT3";
187	case POWER_DOMAIN_AUX_TBT4:
188	return "AUX_TBT4";
189	case POWER_DOMAIN_AUX_TBT5:
190	return "AUX_TBT5";
191	case POWER_DOMAIN_AUX_TBT6:
192	return "AUX_TBT6";
193	case POWER_DOMAIN_GMBUS:
194	return "GMBUS";
195	case POWER_DOMAIN_INIT:
196	return "INIT";
197	case POWER_DOMAIN_GT_IRQ:
198	return "GT_IRQ";
199	case POWER_DOMAIN_DC_OFF:
200	return "DC_OFF";
201	case POWER_DOMAIN_TC_COLD_OFF:
202	return "TC_COLD_OFF";
203	default:
204	MISSING_CASE(domain);
205	return "?";
206	}
207	}
208
209	static bool __intel_display_power_is_enabled(struct intel_display *display,
210	enum intel_display_power_domain domain)
211	{
212	struct i915_power_well *power_well;
213	bool is_enabled;
214
215	if (intel_display_rpm_suspended(display))
216	return false;
217
218	is_enabled = true;
219
220	for_each_power_domain_well_reverse(display, power_well, domain) {
221	if (intel_power_well_is_always_on(power_well))
222	continue;
223
224	if (!intel_power_well_is_enabled_cached(power_well)) {
225	is_enabled = false;
226	break;
227	}
228	}
229
230	return is_enabled;
231	}
232
233	/**
234	* intel_display_power_is_enabled - check for a power domain
235	* @display: display device instance
236	* @domain: power domain to check
237	*
238	* This function can be used to check the hw power domain state. It is mostly
239	* used in hardware state readout functions. Everywhere else code should rely
240	* upon explicit power domain reference counting to ensure that the hardware
241	* block is powered up before accessing it.
242	*
243	* Callers must hold the relevant modesetting locks to ensure that concurrent
244	* threads can't disable the power well while the caller tries to read a few
245	* registers.
246	*
247	* Returns:
248	* True when the power domain is enabled, false otherwise.
249	*/
250	bool intel_display_power_is_enabled(struct intel_display *display,
251	enum intel_display_power_domain domain)
252	{
253	struct i915_power_domains *power_domains = &display->power.domains;
254	bool ret;
255
256	mutex_lock(&power_domains->lock);
257	ret = __intel_display_power_is_enabled(display, domain);
258	mutex_unlock(lock: &power_domains->lock);
259
260	return ret;
261	}
262
263	static u32
264	sanitize_target_dc_state(struct intel_display *display,
265	u32 target_dc_state)
266	{
267	struct i915_power_domains *power_domains = &display->power.domains;
268	static const u32 states[] = {
269	DC_STATE_EN_UPTO_DC6,
270	DC_STATE_EN_UPTO_DC5,
271	DC_STATE_EN_DC3CO,
272	DC_STATE_DISABLE,
273	};
274	int i;
275
276	for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
277	if (target_dc_state != states[i])
278	continue;
279
280	if (power_domains->allowed_dc_mask & target_dc_state)
281	break;
282
283	target_dc_state = states[i + 1];
284	}
285
286	return target_dc_state;
287	}
288
289	/**
290	* intel_display_power_set_target_dc_state - Set target dc state.
291	* @display: display device
292	* @state: state which needs to be set as target_dc_state.
293	*
294	* This function set the "DC off" power well target_dc_state,
295	* based upon this target_dc_stste, "DC off" power well will
296	* enable desired DC state.
297	*/
298	void intel_display_power_set_target_dc_state(struct intel_display *display,
299	u32 state)
300	{
301	struct i915_power_well *power_well;
302	bool dc_off_enabled;
303	struct i915_power_domains *power_domains = &display->power.domains;
304
305	mutex_lock(&power_domains->lock);
306	power_well = lookup_power_well(display, id: SKL_DISP_DC_OFF);
307
308	if (drm_WARN_ON(display->drm, !power_well))
309	goto unlock;
310
311	state = sanitize_target_dc_state(display, target_dc_state: state);
312
313	if (state == power_domains->target_dc_state)
314	goto unlock;
315
316	dc_off_enabled = intel_power_well_is_enabled(display, power_well);
317	/*
318	* If DC off power well is disabled, need to enable and disable the
319	* DC off power well to effect target DC state.
320	*/
321	if (!dc_off_enabled)
322	intel_power_well_enable(display, power_well);
323
324	power_domains->target_dc_state = state;
325
326	if (!dc_off_enabled)
327	intel_power_well_disable(display, power_well);
328
329	unlock:
330	mutex_unlock(lock: &power_domains->lock);
331	}
332
333	/**
334	* intel_display_power_get_current_dc_state - Set target dc state.
335	* @display: display device
336	*
337	* This function set the "DC off" power well target_dc_state,
338	* based upon this target_dc_stste, "DC off" power well will
339	* enable desired DC state.
340	*/
341	u32 intel_display_power_get_current_dc_state(struct intel_display *display)
342	{
343	struct i915_power_well *power_well;
344	struct i915_power_domains *power_domains = &display->power.domains;
345	u32 current_dc_state = DC_STATE_DISABLE;
346
347	mutex_lock(&power_domains->lock);
348	power_well = lookup_power_well(display, id: SKL_DISP_DC_OFF);
349
350	if (drm_WARN_ON(display->drm, !power_well))
351	goto unlock;
352
353	current_dc_state = intel_power_well_is_enabled(display, power_well) ?
354	DC_STATE_DISABLE : power_domains->target_dc_state;
355
356	unlock:
357	mutex_unlock(lock: &power_domains->lock);
358
359	return current_dc_state;
360	}
361
362	static void __async_put_domains_mask(struct i915_power_domains *power_domains,
363	struct intel_power_domain_mask *mask)
364	{
365	bitmap_or(dst: mask->bits,
366	src1: power_domains->async_put_domains[0].bits,
367	src2: power_domains->async_put_domains[1].bits,
368	nbits: POWER_DOMAIN_NUM);
369	}
370
371	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
372
373	static bool
374	assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
375	{
376	struct intel_display *display = container_of(power_domains,
377	struct intel_display,
378	power.domains);
379
380	return !drm_WARN_ON(display->drm,
381	bitmap_intersects(power_domains->async_put_domains[0].bits,
382	power_domains->async_put_domains[1].bits,
383	POWER_DOMAIN_NUM));
384	}
385
386	static bool
387	__async_put_domains_state_ok(struct i915_power_domains *power_domains)
388	{
389	struct intel_display *display = container_of(power_domains,
390	struct intel_display,
391	power.domains);
392	struct intel_power_domain_mask async_put_mask;
393	enum intel_display_power_domain domain;
394	bool err = false;
395
396	err |= !assert_async_put_domain_masks_disjoint(power_domains);
397	__async_put_domains_mask(power_domains, mask: &async_put_mask);
398	err |= drm_WARN_ON(display->drm,
399	!!power_domains->async_put_wakeref !=
400	!bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM));
401
402	for_each_power_domain(domain, &async_put_mask)
403	err |= drm_WARN_ON(display->drm,
404	power_domains->domain_use_count[domain] != 1);
405
406	return !err;
407	}
408
409	static void print_power_domains(struct i915_power_domains *power_domains,
410	const char *prefix, struct intel_power_domain_mask *mask)
411	{
412	struct intel_display *display = container_of(power_domains,
413	struct intel_display,
414	power.domains);
415	enum intel_display_power_domain domain;
416
417	drm_dbg_kms(display->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
418	for_each_power_domain(domain, mask)
419	drm_dbg_kms(display->drm, "%s use_count %d\n",
420	intel_display_power_domain_str(domain),
421	power_domains->domain_use_count[domain]);
422	}
423
424	static void
425	print_async_put_domains_state(struct i915_power_domains *power_domains)
426	{
427	struct intel_display *display = container_of(power_domains,
428	struct intel_display,
429	power.domains);
430
431	drm_dbg_kms(display->drm, "async_put_wakeref: %s\n",
432	str_yes_no(power_domains->async_put_wakeref));
433
434	print_power_domains(power_domains, prefix: "async_put_domains[0]",
435	mask: &power_domains->async_put_domains[0]);
436	print_power_domains(power_domains, prefix: "async_put_domains[1]",
437	mask: &power_domains->async_put_domains[1]);
438	}
439
440	static void
441	verify_async_put_domains_state(struct i915_power_domains *power_domains)
442	{
443	if (!__async_put_domains_state_ok(power_domains))
444	print_async_put_domains_state(power_domains);
445	}
446
447	#else
448
449	static void
450	assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
451	{
452	}
453
454	static void
455	verify_async_put_domains_state(struct i915_power_domains *power_domains)
456	{
457	}
458
459	#endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
460
461	static void async_put_domains_mask(struct i915_power_domains *power_domains,
462	struct intel_power_domain_mask *mask)
463
464	{
465	assert_async_put_domain_masks_disjoint(power_domains);
466
467	__async_put_domains_mask(power_domains, mask);
468	}
469
470	static void
471	async_put_domains_clear_domain(struct i915_power_domains *power_domains,
472	enum intel_display_power_domain domain)
473	{
474	assert_async_put_domain_masks_disjoint(power_domains);
475
476	clear_bit(nr: domain, addr: power_domains->async_put_domains[0].bits);
477	clear_bit(nr: domain, addr: power_domains->async_put_domains[1].bits);
478	}
479
480	static void
481	cancel_async_put_work(struct i915_power_domains *power_domains, bool sync)
482	{
483	if (sync)
484	cancel_delayed_work_sync(dwork: &power_domains->async_put_work);
485	else
486	cancel_delayed_work(dwork: &power_domains->async_put_work);
487
488	power_domains->async_put_next_delay = 0;
489	}
490
491	static bool
492	intel_display_power_grab_async_put_ref(struct intel_display *display,
493	enum intel_display_power_domain domain)
494	{
495	struct i915_power_domains *power_domains = &display->power.domains;
496	struct intel_power_domain_mask async_put_mask;
497	bool ret = false;
498
499	async_put_domains_mask(power_domains, mask: &async_put_mask);
500	if (!test_bit(domain, async_put_mask.bits))
501	goto out_verify;
502
503	async_put_domains_clear_domain(power_domains, domain);
504
505	ret = true;
506
507	async_put_domains_mask(power_domains, mask: &async_put_mask);
508	if (!bitmap_empty(src: async_put_mask.bits, nbits: POWER_DOMAIN_NUM))
509	goto out_verify;
510
511	cancel_async_put_work(power_domains, sync: false);
512	intel_display_rpm_put_raw(display,
513	fetch_and_zero(&power_domains->async_put_wakeref));
514	out_verify:
515	verify_async_put_domains_state(power_domains);
516
517	return ret;
518	}
519
520	static void
521	__intel_display_power_get_domain(struct intel_display *display,
522	enum intel_display_power_domain domain)
523	{
524	struct i915_power_domains *power_domains = &display->power.domains;
525	struct i915_power_well *power_well;
526
527	if (intel_display_power_grab_async_put_ref(display, domain))
528	return;
529
530	for_each_power_domain_well(display, power_well, domain)
531	intel_power_well_get(display, power_well);
532
533	power_domains->domain_use_count[domain]++;
534	}
535
536	/**
537	* intel_display_power_get - grab a power domain reference
538	* @display: display device instance
539	* @domain: power domain to reference
540	*
541	* This function grabs a power domain reference for @domain and ensures that the
542	* power domain and all its parents are powered up. Therefore users should only
543	* grab a reference to the innermost power domain they need.
544	*
545	* Any power domain reference obtained by this function must have a symmetric
546	* call to intel_display_power_put() to release the reference again.
547	*/
548	intel_wakeref_t intel_display_power_get(struct intel_display *display,
549	enum intel_display_power_domain domain)
550	{
551	struct i915_power_domains *power_domains = &display->power.domains;
552	struct ref_tracker *wakeref;
553
554	wakeref = intel_display_rpm_get(display);
555
556	mutex_lock(&power_domains->lock);
557	__intel_display_power_get_domain(display, domain);
558	mutex_unlock(lock: &power_domains->lock);
559
560	return wakeref;
561	}
562
563	/**
564	* intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
565	* @display: display device instance
566	* @domain: power domain to reference
567	*
568	* This function grabs a power domain reference for @domain and ensures that the
569	* power domain and all its parents are powered up. Therefore users should only
570	* grab a reference to the innermost power domain they need.
571	*
572	* Any power domain reference obtained by this function must have a symmetric
573	* call to intel_display_power_put() to release the reference again.
574	*/
575	intel_wakeref_t
576	intel_display_power_get_if_enabled(struct intel_display *display,
577	enum intel_display_power_domain domain)
578	{
579	struct i915_power_domains *power_domains = &display->power.domains;
580	struct ref_tracker *wakeref;
581	bool is_enabled;
582
583	wakeref = intel_display_rpm_get_if_in_use(display);
584	if (!wakeref)
585	return NULL;
586
587	mutex_lock(&power_domains->lock);
588
589	if (__intel_display_power_is_enabled(display, domain)) {
590	__intel_display_power_get_domain(display, domain);
591	is_enabled = true;
592	} else {
593	is_enabled = false;
594	}
595
596	mutex_unlock(lock: &power_domains->lock);
597
598	if (!is_enabled) {
599	intel_display_rpm_put(display, wakeref);
600	wakeref = NULL;
601	}
602
603	return wakeref;
604	}
605
606	static void
607	__intel_display_power_put_domain(struct intel_display *display,
608	enum intel_display_power_domain domain)
609	{
610	struct i915_power_domains *power_domains = &display->power.domains;
611	struct i915_power_well *power_well;
612	const char *name = intel_display_power_domain_str(domain);
613	struct intel_power_domain_mask async_put_mask;
614
615	drm_WARN(display->drm, !power_domains->domain_use_count[domain],
616	"Use count on domain %s is already zero\n",
617	name);
618	async_put_domains_mask(power_domains, mask: &async_put_mask);
619	drm_WARN(display->drm,
620	test_bit(domain, async_put_mask.bits),
621	"Async disabling of domain %s is pending\n",
622	name);
623
624	power_domains->domain_use_count[domain]--;
625
626	for_each_power_domain_well_reverse(display, power_well, domain)
627	intel_power_well_put(display, power_well);
628	}
629
630	static void __intel_display_power_put(struct intel_display *display,
631	enum intel_display_power_domain domain)
632	{
633	struct i915_power_domains *power_domains = &display->power.domains;
634
635	mutex_lock(&power_domains->lock);
636	__intel_display_power_put_domain(display, domain);
637	mutex_unlock(lock: &power_domains->lock);
638	}
639
640	static void
641	queue_async_put_domains_work(struct i915_power_domains *power_domains,
642	intel_wakeref_t wakeref,
643	int delay_ms)
644	{
645	struct intel_display *display = container_of(power_domains,
646	struct intel_display,
647	power.domains);
648	drm_WARN_ON(display->drm, power_domains->async_put_wakeref);
649	power_domains->async_put_wakeref = wakeref;
650	drm_WARN_ON(display->drm, !queue_delayed_work(system_unbound_wq,
651	&power_domains->async_put_work,
652	msecs_to_jiffies(delay_ms)));
653	}
654
655	static void
656	release_async_put_domains(struct i915_power_domains *power_domains,
657	struct intel_power_domain_mask *mask)
658	{
659	struct intel_display *display = container_of(power_domains,
660	struct intel_display,
661	power.domains);
662	enum intel_display_power_domain domain;
663	struct ref_tracker *wakeref;
664
665	wakeref = intel_display_rpm_get_noresume(display);
666
667	for_each_power_domain(domain, mask) {
668	/* Clear before put, so put's sanity check is happy. */
669	async_put_domains_clear_domain(power_domains, domain);
670	__intel_display_power_put_domain(display, domain);
671	}
672
673	intel_display_rpm_put(display, wakeref);
674	}
675
676	static void
677	intel_display_power_put_async_work(struct work_struct *work)
678	{
679	struct intel_display *display = container_of(work, struct intel_display,
680	power.domains.async_put_work.work);
681	struct i915_power_domains *power_domains = &display->power.domains;
682	struct ref_tracker *new_work_wakeref, *old_work_wakeref = NULL;
683
684	new_work_wakeref = intel_display_rpm_get_raw(display);
685
686	mutex_lock(&power_domains->lock);
687
688	/*
689	* Bail out if all the domain refs pending to be released were grabbed
690	* by subsequent gets or a flush_work.
691	*/
692	old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
693	if (!old_work_wakeref)
694	goto out_verify;
695
696	release_async_put_domains(power_domains,
697	mask: &power_domains->async_put_domains[0]);
698
699	/*
700	* Cancel the work that got queued after this one got dequeued,
701	* since here we released the corresponding async-put reference.
702	*/
703	cancel_async_put_work(power_domains, sync: false);
704
705	/* Requeue the work if more domains were async put meanwhile. */
706	if (!bitmap_empty(src: power_domains->async_put_domains[1].bits, nbits: POWER_DOMAIN_NUM)) {
707	bitmap_copy(dst: power_domains->async_put_domains[0].bits,
708	src: power_domains->async_put_domains[1].bits,
709	nbits: POWER_DOMAIN_NUM);
710	bitmap_zero(dst: power_domains->async_put_domains[1].bits,
711	nbits: POWER_DOMAIN_NUM);
712	queue_async_put_domains_work(power_domains,
713	fetch_and_zero(&new_work_wakeref),
714	delay_ms: power_domains->async_put_next_delay);
715	power_domains->async_put_next_delay = 0;
716	}
717
718	out_verify:
719	verify_async_put_domains_state(power_domains);
720
721	mutex_unlock(lock: &power_domains->lock);
722
723	if (old_work_wakeref)
724	intel_display_rpm_put_raw(display, wakeref: old_work_wakeref);
725	if (new_work_wakeref)
726	intel_display_rpm_put_raw(display, wakeref: new_work_wakeref);
727	}
728
729	/**
730	* __intel_display_power_put_async - release a power domain reference asynchronously
731	* @display: display device instance
732	* @domain: power domain to reference
733	* @wakeref: wakeref acquired for the reference that is being released
734	* @delay_ms: delay of powering down the power domain
735	*
736	* This function drops the power domain reference obtained by
737	* intel_display_power_get*() and schedules a work to power down the
738	* corresponding hardware block if this is the last reference.
739	* The power down is delayed by @delay_ms if this is >= 0, or by a default
740	* 100 ms otherwise.
741	*/
742	void __intel_display_power_put_async(struct intel_display *display,
743	enum intel_display_power_domain domain,
744	intel_wakeref_t wakeref,
745	int delay_ms)
746	{
747	struct i915_power_domains *power_domains = &display->power.domains;
748	struct ref_tracker *work_wakeref;
749
750	work_wakeref = intel_display_rpm_get_raw(display);
751
752	delay_ms = delay_ms >= 0 ? delay_ms : 100;
753
754	mutex_lock(&power_domains->lock);
755
756	if (power_domains->domain_use_count[domain] > 1) {
757	__intel_display_power_put_domain(display, domain);
758
759	goto out_verify;
760	}
761
762	drm_WARN_ON(display->drm, power_domains->domain_use_count[domain] != 1);
763
764	/* Let a pending work requeue itself or queue a new one. */
765	if (power_domains->async_put_wakeref) {
766	set_bit(nr: domain, addr: power_domains->async_put_domains[1].bits);
767	power_domains->async_put_next_delay = max(power_domains->async_put_next_delay,
768	delay_ms);
769	} else {
770	set_bit(nr: domain, addr: power_domains->async_put_domains[0].bits);
771	queue_async_put_domains_work(power_domains,
772	fetch_and_zero(&work_wakeref),
773	delay_ms);
774	}
775
776	out_verify:
777	verify_async_put_domains_state(power_domains);
778
779	mutex_unlock(lock: &power_domains->lock);
780
781	if (work_wakeref)
782	intel_display_rpm_put_raw(display, wakeref: work_wakeref);
783
784	intel_display_rpm_put(display, wakeref);
785	}
786
787	/**
788	* intel_display_power_flush_work - flushes the async display power disabling work
789	* @display: display device instance
790	*
791	* Flushes any pending work that was scheduled by a preceding
792	* intel_display_power_put_async() call, completing the disabling of the
793	* corresponding power domains.
794	*
795	* Note that the work handler function may still be running after this
796	* function returns; to ensure that the work handler isn't running use
797	* intel_display_power_flush_work_sync() instead.
798	*/
799	void intel_display_power_flush_work(struct intel_display *display)
800	{
801	struct i915_power_domains *power_domains = &display->power.domains;
802	struct intel_power_domain_mask async_put_mask;
803	intel_wakeref_t work_wakeref;
804
805	mutex_lock(&power_domains->lock);
806
807	work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
808	if (!work_wakeref)
809	goto out_verify;
810
811	async_put_domains_mask(power_domains, mask: &async_put_mask);
812	release_async_put_domains(power_domains, mask: &async_put_mask);
813	cancel_async_put_work(power_domains, sync: false);
814
815	out_verify:
816	verify_async_put_domains_state(power_domains);
817
818	mutex_unlock(lock: &power_domains->lock);
819
820	if (work_wakeref)
821	intel_display_rpm_put_raw(display, wakeref: work_wakeref);
822	}
823
824	/**
825	* intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
826	* @display: display device instance
827	*
828	* Like intel_display_power_flush_work(), but also ensure that the work
829	* handler function is not running any more when this function returns.
830	*/
831	static void
832	intel_display_power_flush_work_sync(struct intel_display *display)
833	{
834	struct i915_power_domains *power_domains = &display->power.domains;
835
836	intel_display_power_flush_work(display);
837	cancel_async_put_work(power_domains, sync: true);
838
839	verify_async_put_domains_state(power_domains);
840
841	drm_WARN_ON(display->drm, power_domains->async_put_wakeref);
842	}
843
844	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
845	/**
846	* intel_display_power_put - release a power domain reference
847	* @display: display device instance
848	* @domain: power domain to reference
849	* @wakeref: wakeref acquired for the reference that is being released
850	*
851	* This function drops the power domain reference obtained by
852	* intel_display_power_get() and might power down the corresponding hardware
853	* block right away if this is the last reference.
854	*/
855	void intel_display_power_put(struct intel_display *display,
856	enum intel_display_power_domain domain,
857	intel_wakeref_t wakeref)
858	{
859	__intel_display_power_put(display, domain);
860	intel_display_rpm_put(display, wakeref);
861	}
862	#else
863	/**
864	* intel_display_power_put_unchecked - release an unchecked power domain reference
865	* @display: display device instance
866	* @domain: power domain to reference
867	*
868	* This function drops the power domain reference obtained by
869	* intel_display_power_get() and might power down the corresponding hardware
870	* block right away if this is the last reference.
871	*
872	* This function is only for the power domain code's internal use to suppress wakeref
873	* tracking when the corresponding debug kconfig option is disabled, should not
874	* be used otherwise.
875	*/
876	void intel_display_power_put_unchecked(struct intel_display *display,
877	enum intel_display_power_domain domain)
878	{
879	__intel_display_power_put(display, domain);
880	intel_display_rpm_put_unchecked(display);
881	}
882	#endif
883
884	void
885	intel_display_power_get_in_set(struct intel_display *display,
886	struct intel_display_power_domain_set *power_domain_set,
887	enum intel_display_power_domain domain)
888	{
889	intel_wakeref_t __maybe_unused wf;
890
891	drm_WARN_ON(display->drm, test_bit(domain, power_domain_set->mask.bits));
892
893	wf = intel_display_power_get(display, domain);
894	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
895	power_domain_set->wakerefs[domain] = wf;
896	#endif
897	set_bit(nr: domain, addr: power_domain_set->mask.bits);
898	}
899
900	bool
901	intel_display_power_get_in_set_if_enabled(struct intel_display *display,
902	struct intel_display_power_domain_set *power_domain_set,
903	enum intel_display_power_domain domain)
904	{
905	intel_wakeref_t wf;
906
907	drm_WARN_ON(display->drm, test_bit(domain, power_domain_set->mask.bits));
908
909	wf = intel_display_power_get_if_enabled(display, domain);
910	if (!wf)
911	return false;
912
913	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
914	power_domain_set->wakerefs[domain] = wf;
915	#endif
916	set_bit(nr: domain, addr: power_domain_set->mask.bits);
917
918	return true;
919	}
920
921	void
922	intel_display_power_put_mask_in_set(struct intel_display *display,
923	struct intel_display_power_domain_set *power_domain_set,
924	struct intel_power_domain_mask *mask)
925	{
926	enum intel_display_power_domain domain;
927
928	drm_WARN_ON(display->drm,
929	!bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
930
931	for_each_power_domain(domain, mask) {
932	intel_wakeref_t __maybe_unused wf = INTEL_WAKEREF_DEF;
933
934	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
935	wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
936	#endif
937	intel_display_power_put(display, domain, wakeref: wf);
938	clear_bit(nr: domain, addr: power_domain_set->mask.bits);
939	}
940	}
941
942	static int
943	sanitize_disable_power_well_option(int disable_power_well)
944	{
945	if (disable_power_well >= 0)
946	return !!disable_power_well;
947
948	return 1;
949	}
950
951	static u32 get_allowed_dc_mask(struct intel_display *display, int enable_dc)
952	{
953	u32 mask;
954	int requested_dc;
955	int max_dc;
956
957	if (!HAS_DISPLAY(display))
958	return 0;
959
960	if (DISPLAY_VER(display) >= 20)
961	max_dc = 2;
962	else if (display->platform.dg2)
963	max_dc = 1;
964	else if (display->platform.dg1)
965	max_dc = 3;
966	else if (DISPLAY_VER(display) >= 12)
967	max_dc = 4;
968	else if (display->platform.geminilake || display->platform.broxton)
969	max_dc = 1;
970	else if (DISPLAY_VER(display) >= 9)
971	max_dc = 2;
972	else
973	max_dc = 0;
974
975	/*
976	* DC9 has a separate HW flow from the rest of the DC states,
977	* not depending on the DMC firmware. It's needed by system
978	* suspend/resume, so allow it unconditionally.
979	*/
980	mask = display->platform.geminilake || display->platform.broxton ||
981	DISPLAY_VER(display) >= 11 ? DC_STATE_EN_DC9 : 0;
982
983	if (!display->params.disable_power_well)
984	max_dc = 0;
985
986	if (enable_dc >= 0 && enable_dc <= max_dc) {
987	requested_dc = enable_dc;
988	} else if (enable_dc == -1) {
989	requested_dc = max_dc;
990	} else if (enable_dc > max_dc && enable_dc <= 4) {
991	drm_dbg_kms(display->drm,
992	"Adjusting requested max DC state (%d->%d)\n",
993	enable_dc, max_dc);
994	requested_dc = max_dc;
995	} else {
996	drm_err(display->drm,
997	"Unexpected value for enable_dc (%d)\n", enable_dc);
998	requested_dc = max_dc;
999	}
1000
1001	switch (requested_dc) {
1002	case 4:
1003	mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
1004	break;
1005	case 3:
1006	mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
1007	break;
1008	case 2:
1009	mask |= DC_STATE_EN_UPTO_DC6;
1010	break;
1011	case 1:
1012	mask |= DC_STATE_EN_UPTO_DC5;
1013	break;
1014	}
1015
1016	drm_dbg_kms(display->drm, "Allowed DC state mask %02x\n", mask);
1017
1018	return mask;
1019	}
1020
1021	/**
1022	* intel_power_domains_init - initializes the power domain structures
1023	* @display: display device instance
1024	*
1025	* Initializes the power domain structures for @display depending upon the
1026	* supported platform.
1027	*/
1028	int intel_power_domains_init(struct intel_display *display)
1029	{
1030	struct i915_power_domains *power_domains = &display->power.domains;
1031
1032	display->params.disable_power_well =
1033	sanitize_disable_power_well_option(disable_power_well: display->params.disable_power_well);
1034	power_domains->allowed_dc_mask =
1035	get_allowed_dc_mask(display, enable_dc: display->params.enable_dc);
1036
1037	power_domains->target_dc_state =
1038	sanitize_target_dc_state(display, DC_STATE_EN_UPTO_DC6);
1039
1040	mutex_init(&power_domains->lock);
1041
1042	INIT_DELAYED_WORK(&power_domains->async_put_work,
1043	intel_display_power_put_async_work);
1044
1045	return intel_display_power_map_init(power_domains);
1046	}
1047
1048	/**
1049	* intel_power_domains_cleanup - clean up power domains resources
1050	* @display: display device instance
1051	*
1052	* Release any resources acquired by intel_power_domains_init()
1053	*/
1054	void intel_power_domains_cleanup(struct intel_display *display)
1055	{
1056	intel_display_power_map_cleanup(power_domains: &display->power.domains);
1057	}
1058
1059	static void intel_power_domains_sync_hw(struct intel_display *display)
1060	{
1061	struct i915_power_domains *power_domains = &display->power.domains;
1062	struct i915_power_well *power_well;
1063
1064	mutex_lock(&power_domains->lock);
1065	for_each_power_well(display, power_well)
1066	intel_power_well_sync_hw(display, power_well);
1067	mutex_unlock(lock: &power_domains->lock);
1068	}
1069
1070	static void gen9_dbuf_slice_set(struct intel_display *display,
1071	enum dbuf_slice slice, bool enable)
1072	{
1073	i915_reg_t reg = DBUF_CTL_S(slice);
1074	bool state;
1075
1076	intel_de_rmw(display, reg, DBUF_POWER_REQUEST,
1077	set: enable ? DBUF_POWER_REQUEST : 0);
1078	intel_de_posting_read(display, reg);
1079	udelay(usec: 10);
1080
1081	state = intel_de_read(display, reg) & DBUF_POWER_STATE;
1082	drm_WARN(display->drm, enable != state,
1083	"DBuf slice %d power %s timeout!\n",
1084	slice, str_enable_disable(enable));
1085	}
1086
1087	void gen9_dbuf_slices_update(struct intel_display *display,
1088	u8 req_slices)
1089	{
1090	struct i915_power_domains *power_domains = &display->power.domains;
1091	u8 slice_mask = DISPLAY_INFO(display)->dbuf.slice_mask;
1092	enum dbuf_slice slice;
1093
1094	drm_WARN(display->drm, req_slices & ~slice_mask,
1095	"Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
1096	req_slices, slice_mask);
1097
1098	drm_dbg_kms(display->drm, "Updating dbuf slices to 0x%x\n",
1099	req_slices);
1100
1101	/*
1102	* Might be running this in parallel to gen9_dc_off_power_well_enable
1103	* being called from intel_dp_detect for instance,
1104	* which causes assertion triggered by race condition,
1105	* as gen9_assert_dbuf_enabled might preempt this when registers
1106	* were already updated, while dev_priv was not.
1107	*/
1108	mutex_lock(&power_domains->lock);
1109
1110	for_each_dbuf_slice(display, slice)
1111	gen9_dbuf_slice_set(display, slice, enable: req_slices & BIT(slice));
1112
1113	display->dbuf.enabled_slices = req_slices;
1114
1115	mutex_unlock(lock: &power_domains->lock);
1116	}
1117
1118	static void gen9_dbuf_enable(struct intel_display *display)
1119	{
1120	u8 slices_mask;
1121
1122	display->dbuf.enabled_slices = intel_enabled_dbuf_slices_mask(display);
1123
1124	slices_mask = BIT(DBUF_S1) | display->dbuf.enabled_slices;
1125
1126	if (DISPLAY_VER(display) >= 14)
1127	intel_pmdemand_program_dbuf(display, dbuf_slices: slices_mask);
1128
1129	/*
1130	* Just power up at least 1 slice, we will
1131	* figure out later which slices we have and what we need.
1132	*/
1133	gen9_dbuf_slices_update(display, req_slices: slices_mask);
1134	}
1135
1136	static void gen9_dbuf_disable(struct intel_display *display)
1137	{
1138	gen9_dbuf_slices_update(display, req_slices: 0);
1139
1140	if (DISPLAY_VER(display) >= 14)
1141	intel_pmdemand_program_dbuf(display, dbuf_slices: 0);
1142	}
1143
1144	static void gen12_dbuf_slices_config(struct intel_display *display)
1145	{
1146	enum dbuf_slice slice;
1147
1148	for_each_dbuf_slice(display, slice)
1149	intel_de_rmw(display, DBUF_CTL_S(slice),
1150	DBUF_TRACKER_STATE_SERVICE_MASK,
1151	DBUF_TRACKER_STATE_SERVICE(8));
1152	}
1153
1154	static void icl_mbus_init(struct intel_display *display)
1155	{
1156	unsigned long abox_regs = DISPLAY_INFO(display)->abox_mask;
1157	u32 mask, val, i;
1158
1159	if (display->platform.alderlake_p || DISPLAY_VER(display) >= 14)
1160	return;
1161
1162	mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
1163	MBUS_ABOX_BT_CREDIT_POOL2_MASK |
1164	MBUS_ABOX_B_CREDIT_MASK |
1165	MBUS_ABOX_BW_CREDIT_MASK;
1166	val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
1167	MBUS_ABOX_BT_CREDIT_POOL2(16) |
1168	MBUS_ABOX_B_CREDIT(1) |
1169	MBUS_ABOX_BW_CREDIT(1);
1170
1171	/*
1172	* gen12 platforms that use abox1 and abox2 for pixel data reads still
1173	* expect us to program the abox_ctl0 register as well, even though
1174	* we don't have to program other instance-0 registers like BW_BUDDY.
1175	*/
1176	if (DISPLAY_VER(display) == 12)
1177	abox_regs |= BIT(0);
1178
1179	for_each_set_bit(i, &abox_regs, BITS_PER_TYPE(abox_regs))
1180	intel_de_rmw(display, MBUS_ABOX_CTL(i), clear: mask, set: val);
1181	}
1182
1183	static void hsw_assert_cdclk(struct intel_display *display)
1184	{
1185	u32 val = intel_de_read(display, LCPLL_CTL);
1186
1187	/*
1188	* The LCPLL register should be turned on by the BIOS. For now
1189	* let's just check its state and print errors in case
1190	* something is wrong. Don't even try to turn it on.
1191	*/
1192
1193	if (val & LCPLL_CD_SOURCE_FCLK)
1194	drm_err(display->drm, "CDCLK source is not LCPLL\n");
1195
1196	if (val & LCPLL_PLL_DISABLE)
1197	drm_err(display->drm, "LCPLL is disabled\n");
1198
1199	if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
1200	drm_err(display->drm, "LCPLL not using non-SSC reference\n");
1201	}
1202
1203	static void assert_can_disable_lcpll(struct intel_display *display)
1204	{
1205	struct drm_i915_private *dev_priv = to_i915(dev: display->drm);
1206	struct intel_crtc *crtc;
1207
1208	for_each_intel_crtc(display->drm, crtc)
1209	INTEL_DISPLAY_STATE_WARN(display, crtc->active,
1210	"CRTC for pipe %c enabled\n",
1211	pipe_name(crtc->pipe));
1212
1213	INTEL_DISPLAY_STATE_WARN(display, intel_de_read(display, HSW_PWR_WELL_CTL2),
1214	"Display power well on\n");
1215	INTEL_DISPLAY_STATE_WARN(display,
1216	intel_de_read(display, SPLL_CTL) & SPLL_PLL_ENABLE,
1217	"SPLL enabled\n");
1218	INTEL_DISPLAY_STATE_WARN(display,
1219	intel_de_read(display, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
1220	"WRPLL1 enabled\n");
1221	INTEL_DISPLAY_STATE_WARN(display,
1222	intel_de_read(display, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
1223	"WRPLL2 enabled\n");
1224	INTEL_DISPLAY_STATE_WARN(display,
1225	intel_de_read(display, PP_STATUS(display, 0)) & PP_ON,
1226	"Panel power on\n");
1227	INTEL_DISPLAY_STATE_WARN(display,
1228	intel_de_read(display, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
1229	"CPU PWM1 enabled\n");
1230	if (display->platform.haswell)
1231	INTEL_DISPLAY_STATE_WARN(display,
1232	intel_de_read(display, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
1233	"CPU PWM2 enabled\n");
1234	INTEL_DISPLAY_STATE_WARN(display,
1235	intel_de_read(display, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
1236	"PCH PWM1 enabled\n");
1237	INTEL_DISPLAY_STATE_WARN(display,
1238	(intel_de_read(display, UTIL_PIN_CTL) & (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) == (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
1239	"Utility pin enabled in PWM mode\n");
1240	INTEL_DISPLAY_STATE_WARN(display,
1241	intel_de_read(display, PCH_GTC_CTL) & PCH_GTC_ENABLE,
1242	"PCH GTC enabled\n");
1243
1244	/*
1245	* In theory we can still leave IRQs enabled, as long as only the HPD
1246	* interrupts remain enabled. We used to check for that, but since it's
1247	* gen-specific and since we only disable LCPLL after we fully disable
1248	* the interrupts, the check below should be enough.
1249	*/
1250	INTEL_DISPLAY_STATE_WARN(display, intel_irqs_enabled(dev_priv),
1251	"IRQs enabled\n");
1252	}
1253
1254	static u32 hsw_read_dcomp(struct intel_display *display)
1255	{
1256	if (display->platform.haswell)
1257	return intel_de_read(display, D_COMP_HSW);
1258	else
1259	return intel_de_read(display, D_COMP_BDW);
1260	}
1261
1262	static void hsw_write_dcomp(struct intel_display *display, u32 val)
1263	{
1264	if (display->platform.haswell) {
1265	if (intel_pcode_write(display->drm, GEN6_PCODE_WRITE_D_COMP, val))
1266	drm_dbg_kms(display->drm, "Failed to write to D_COMP\n");
1267	} else {
1268	intel_de_write(display, D_COMP_BDW, val);
1269	intel_de_posting_read(display, D_COMP_BDW);
1270	}
1271	}
1272
1273	/*
1274	* This function implements pieces of two sequences from BSpec:
1275	* - Sequence for display software to disable LCPLL
1276	* - Sequence for display software to allow package C8+
1277	* The steps implemented here are just the steps that actually touch the LCPLL
1278	* register. Callers should take care of disabling all the display engine
1279	* functions, doing the mode unset, fixing interrupts, etc.
1280	*/
1281	static void hsw_disable_lcpll(struct intel_display *display,
1282	bool switch_to_fclk, bool allow_power_down)
1283	{
1284	u32 val;
1285	int ret;
1286
1287	assert_can_disable_lcpll(display);
1288
1289	val = intel_de_read(display, LCPLL_CTL);
1290
1291	if (switch_to_fclk) {
1292	val |= LCPLL_CD_SOURCE_FCLK;
1293	intel_de_write(display, LCPLL_CTL, val);
1294
1295	ret = intel_de_wait_for_set_us(display, LCPLL_CTL,
1296	LCPLL_CD_SOURCE_FCLK_DONE, timeout_us: 1);
1297	if (ret)
1298	drm_err(display->drm, "Switching to FCLK failed\n");
1299
1300	val = intel_de_read(display, LCPLL_CTL);
1301	}
1302
1303	val |= LCPLL_PLL_DISABLE;
1304	intel_de_write(display, LCPLL_CTL, val);
1305	intel_de_posting_read(display, LCPLL_CTL);
1306
1307	if (intel_de_wait_for_clear_ms(display, LCPLL_CTL, LCPLL_PLL_LOCK, timeout_ms: 1))
1308	drm_err(display->drm, "LCPLL still locked\n");
1309
1310	val = hsw_read_dcomp(display);
1311	val |= D_COMP_COMP_DISABLE;
1312	hsw_write_dcomp(display, val);
1313	ndelay(100);
1314
1315	ret = poll_timeout_us(val = hsw_read_dcomp(display),
1316	(val & D_COMP_RCOMP_IN_PROGRESS) == 0,
1317	100, 1000, false);
1318	if (ret)
1319	drm_err(display->drm, "D_COMP RCOMP still in progress\n");
1320
1321	if (allow_power_down) {
1322	intel_de_rmw(display, LCPLL_CTL, clear: 0, LCPLL_POWER_DOWN_ALLOW);
1323	intel_de_posting_read(display, LCPLL_CTL);
1324	}
1325	}
1326
1327	/*
1328	* Fully restores LCPLL, disallowing power down and switching back to LCPLL
1329	* source.
1330	*/
1331	static void hsw_restore_lcpll(struct intel_display *display)
1332	{
1333	struct drm_i915_private __maybe_unused *dev_priv = to_i915(dev: display->drm);
1334	u32 val;
1335	int ret;
1336
1337	val = intel_de_read(display, LCPLL_CTL);
1338
1339	if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
1340	LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
1341	return;
1342
1343	/*
1344	* Make sure we're not on PC8 state before disabling PC8, otherwise
1345	* we'll hang the machine. To prevent PC8 state, just enable force_wake.
1346	*/
1347	intel_uncore_forcewake_get(uncore: &dev_priv->uncore, domains: FORCEWAKE_ALL);
1348
1349	if (val & LCPLL_POWER_DOWN_ALLOW) {
1350	val &= ~LCPLL_POWER_DOWN_ALLOW;
1351	intel_de_write(display, LCPLL_CTL, val);
1352	intel_de_posting_read(display, LCPLL_CTL);
1353	}
1354
1355	val = hsw_read_dcomp(display);
1356	val |= D_COMP_COMP_FORCE;
1357	val &= ~D_COMP_COMP_DISABLE;
1358	hsw_write_dcomp(display, val);
1359
1360	val = intel_de_read(display, LCPLL_CTL);
1361	val &= ~LCPLL_PLL_DISABLE;
1362	intel_de_write(display, LCPLL_CTL, val);
1363
1364	if (intel_de_wait_for_set_ms(display, LCPLL_CTL, LCPLL_PLL_LOCK, timeout_ms: 5))
1365	drm_err(display->drm, "LCPLL not locked yet\n");
1366
1367	if (val & LCPLL_CD_SOURCE_FCLK) {
1368	intel_de_rmw(display, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, set: 0);
1369
1370	ret = intel_de_wait_for_clear_us(display, LCPLL_CTL,
1371	LCPLL_CD_SOURCE_FCLK_DONE, timeout_us: 1);
1372	if (ret)
1373	drm_err(display->drm,
1374	"Switching back to LCPLL failed\n");
1375	}
1376
1377	intel_uncore_forcewake_put(uncore: &dev_priv->uncore, domains: FORCEWAKE_ALL);
1378
1379	intel_update_cdclk(display);
1380	intel_cdclk_dump_config(display, cdclk_config: &display->cdclk.hw, context: "Current CDCLK");
1381	}
1382
1383	/*
1384	* Package states C8 and deeper are really deep PC states that can only be
1385	* reached when all the devices on the system allow it, so even if the graphics
1386	* device allows PC8+, it doesn't mean the system will actually get to these
1387	* states. Our driver only allows PC8+ when going into runtime PM.
1388	*
1389	* The requirements for PC8+ are that all the outputs are disabled, the power
1390	* well is disabled and most interrupts are disabled, and these are also
1391	* requirements for runtime PM. When these conditions are met, we manually do
1392	* the other conditions: disable the interrupts, clocks and switch LCPLL refclk
1393	* to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
1394	* hang the machine.
1395	*
1396	* When we really reach PC8 or deeper states (not just when we allow it) we lose
1397	* the state of some registers, so when we come back from PC8+ we need to
1398	* restore this state. We don't get into PC8+ if we're not in RC6, so we don't
1399	* need to take care of the registers kept by RC6. Notice that this happens even
1400	* if we don't put the device in PCI D3 state (which is what currently happens
1401	* because of the runtime PM support).
1402	*
1403	* For more, read "Display Sequences for Package C8" on the hardware
1404	* documentation.
1405	*/
1406	static void hsw_enable_pc8(struct intel_display *display)
1407	{
1408	drm_dbg_kms(display->drm, "Enabling package C8+\n");
1409
1410	if (HAS_PCH_LPT_LP(display))
1411	intel_de_rmw(display, SOUTH_DSPCLK_GATE_D,
1412	PCH_LP_PARTITION_LEVEL_DISABLE, set: 0);
1413
1414	lpt_disable_clkout_dp(display);
1415	hsw_disable_lcpll(display, switch_to_fclk: true, allow_power_down: true);
1416	}
1417
1418	static void hsw_disable_pc8(struct intel_display *display)
1419	{
1420	struct drm_i915_private __maybe_unused *dev_priv = to_i915(dev: display->drm);
1421
1422	drm_dbg_kms(display->drm, "Disabling package C8+\n");
1423
1424	hsw_restore_lcpll(display);
1425	intel_init_pch_refclk(display);
1426
1427	/* Many display registers don't survive PC8+ */
1428	#ifdef I915 /* FIXME */
1429	intel_clock_gating_init(i915: dev_priv);
1430	#endif
1431	}
1432
1433	static void intel_pch_reset_handshake(struct intel_display *display,
1434	bool enable)
1435	{
1436	i915_reg_t reg;
1437	u32 reset_bits;
1438
1439	if (DISPLAY_VER(display) >= 35)
1440	return;
1441
1442	if (display->platform.ivybridge) {
1443	reg = GEN7_MSG_CTL;
1444	reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
1445	} else {
1446	reg = HSW_NDE_RSTWRN_OPT;
1447	reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
1448	}
1449
1450	if (DISPLAY_VER(display) >= 14)
1451	reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
1452
1453	intel_de_rmw(display, reg, clear: reset_bits, set: enable ? reset_bits : 0);
1454	}
1455
1456	static void skl_display_core_init(struct intel_display *display,
1457	bool resume)
1458	{
1459	struct i915_power_domains *power_domains = &display->power.domains;
1460	struct i915_power_well *well;
1461
1462	gen9_set_dc_state(display, DC_STATE_DISABLE);
1463
1464	/* enable PCH reset handshake */
1465	intel_pch_reset_handshake(display, enable: !HAS_PCH_NOP(display));
1466
1467	if (!HAS_DISPLAY(display))
1468	return;
1469
1470	/* enable PG1 and Misc I/O */
1471	mutex_lock(&power_domains->lock);
1472
1473	well = lookup_power_well(display, id: SKL_DISP_PW_1);
1474	intel_power_well_enable(display, power_well: well);
1475
1476	well = lookup_power_well(display, id: SKL_DISP_PW_MISC_IO);
1477	intel_power_well_enable(display, power_well: well);
1478
1479	mutex_unlock(lock: &power_domains->lock);
1480
1481	intel_cdclk_init_hw(display);
1482
1483	gen9_dbuf_enable(display);
1484
1485	if (resume)
1486	intel_dmc_load_program(display);
1487	}
1488
1489	static void skl_display_core_uninit(struct intel_display *display)
1490	{
1491	struct i915_power_domains *power_domains = &display->power.domains;
1492	struct i915_power_well *well;
1493
1494	if (!HAS_DISPLAY(display))
1495	return;
1496
1497	gen9_disable_dc_states(display);
1498	/* TODO: disable DMC program */
1499
1500	gen9_dbuf_disable(display);
1501
1502	intel_cdclk_uninit_hw(display);
1503
1504	/* The spec doesn't call for removing the reset handshake flag */
1505	/* disable PG1 and Misc I/O */
1506
1507	mutex_lock(&power_domains->lock);
1508
1509	/*
1510	* BSpec says to keep the MISC IO power well enabled here, only
1511	* remove our request for power well 1.
1512	* Note that even though the driver's request is removed power well 1
1513	* may stay enabled after this due to DMC's own request on it.
1514	*/
1515	well = lookup_power_well(display, id: SKL_DISP_PW_1);
1516	intel_power_well_disable(display, power_well: well);
1517
1518	mutex_unlock(lock: &power_domains->lock);
1519
1520	usleep_range(min: 10, max: 30); /* 10 us delay per Bspec */
1521	}
1522
1523	static void bxt_display_core_init(struct intel_display *display, bool resume)
1524	{
1525	struct i915_power_domains *power_domains = &display->power.domains;
1526	struct i915_power_well *well;
1527
1528	gen9_set_dc_state(display, DC_STATE_DISABLE);
1529
1530	/*
1531	* NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
1532	* or else the reset will hang because there is no PCH to respond.
1533	* Move the handshake programming to initialization sequence.
1534	* Previously was left up to BIOS.
1535	*/
1536	intel_pch_reset_handshake(display, enable: false);
1537
1538	if (!HAS_DISPLAY(display))
1539	return;
1540
1541	/* Enable PG1 */
1542	mutex_lock(&power_domains->lock);
1543
1544	well = lookup_power_well(display, id: SKL_DISP_PW_1);
1545	intel_power_well_enable(display, power_well: well);
1546
1547	mutex_unlock(lock: &power_domains->lock);
1548
1549	intel_cdclk_init_hw(display);
1550
1551	gen9_dbuf_enable(display);
1552
1553	if (resume)
1554	intel_dmc_load_program(display);
1555	}
1556
1557	static void bxt_display_core_uninit(struct intel_display *display)
1558	{
1559	struct i915_power_domains *power_domains = &display->power.domains;
1560	struct i915_power_well *well;
1561
1562	if (!HAS_DISPLAY(display))
1563	return;
1564
1565	gen9_disable_dc_states(display);
1566	/* TODO: disable DMC program */
1567
1568	gen9_dbuf_disable(display);
1569
1570	intel_cdclk_uninit_hw(display);
1571
1572	/* The spec doesn't call for removing the reset handshake flag */
1573
1574	/*
1575	* Disable PW1 (PG1).
1576	* Note that even though the driver's request is removed power well 1
1577	* may stay enabled after this due to DMC's own request on it.
1578	*/
1579	mutex_lock(&power_domains->lock);
1580
1581	well = lookup_power_well(display, id: SKL_DISP_PW_1);
1582	intel_power_well_disable(display, power_well: well);
1583
1584	mutex_unlock(lock: &power_domains->lock);
1585
1586	usleep_range(min: 10, max: 30); /* 10 us delay per Bspec */
1587	}
1588
1589	struct buddy_page_mask {
1590	u32 page_mask;
1591	u8 type;
1592	u8 num_channels;
1593	};
1594
1595	static const struct buddy_page_mask tgl_buddy_page_masks[] = {
1596	{ .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0xF },
1597	{ .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0xF },
1598	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
1599	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
1600	{ .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x1F },
1601	{ .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x1E },
1602	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
1603	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
1604	{}
1605	};
1606
1607	static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
1608	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
1609	{ .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0x1 },
1610	{ .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0x1 },
1611	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
1612	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
1613	{ .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x3 },
1614	{ .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x3 },
1615	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
1616	{}
1617	};
1618
1619	static void tgl_bw_buddy_init(struct intel_display *display)
1620	{
1621	const struct dram_info *dram_info = intel_dram_info(drm: display->drm);
1622	const struct buddy_page_mask *table;
1623	unsigned long abox_mask = DISPLAY_INFO(display)->abox_mask;
1624	int config, i;
1625
1626	/* BW_BUDDY registers are not used on dgpu's beyond DG1 */
1627	if (display->platform.dgfx && !display->platform.dg1)
1628	return;
1629
1630	if (display->platform.alderlake_s ||
1631	(display->platform.rocketlake && IS_DISPLAY_STEP(display, STEP_A0, STEP_B0)))
1632	/* Wa_1409767108 */
1633	table = wa_1409767108_buddy_page_masks;
1634	else
1635	table = tgl_buddy_page_masks;
1636
1637	for (config = 0; table[config].page_mask != 0; config++)
1638	if (table[config].num_channels == dram_info->num_channels &&
1639	table[config].type == dram_info->type)
1640	break;
1641
1642	if (table[config].page_mask == 0) {
1643	drm_dbg_kms(display->drm,
1644	"Unknown memory configuration; disabling address buddy logic.\n");
1645	for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask))
1646	intel_de_write(display, BW_BUDDY_CTL(i),
1647	BW_BUDDY_DISABLE);
1648	} else {
1649	for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask)) {
1650	intel_de_write(display, BW_BUDDY_PAGE_MASK(i),
1651	val: table[config].page_mask);
1652
1653	/* Wa_22010178259:tgl,dg1,rkl,adl-s */
1654	if (DISPLAY_VER(display) == 12)
1655	intel_de_rmw(display, BW_BUDDY_CTL(i),
1656	BW_BUDDY_TLB_REQ_TIMER_MASK,
1657	BW_BUDDY_TLB_REQ_TIMER(0x8));
1658	}
1659	}
1660	}
1661
1662	static void icl_display_core_init(struct intel_display *display,
1663	bool resume)
1664	{
1665	struct i915_power_domains *power_domains = &display->power.domains;
1666	struct i915_power_well *well;
1667
1668	gen9_set_dc_state(display, DC_STATE_DISABLE);
1669
1670	/* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
1671	if (INTEL_PCH_TYPE(display) >= PCH_TGP &&
1672	INTEL_PCH_TYPE(display) < PCH_DG1)
1673	intel_de_rmw(display, SOUTH_DSPCLK_GATE_D, clear: 0,
1674	PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
1675
1676	/* 1. Enable PCH reset handshake. */
1677	intel_pch_reset_handshake(display, enable: !HAS_PCH_NOP(display));
1678
1679	if (!HAS_DISPLAY(display))
1680	return;
1681
1682	/* 2. Initialize all combo phys */
1683	intel_combo_phy_init(display);
1684
1685	/*
1686	* 3. Enable Power Well 1 (PG1).
1687	* The AUX IO power wells will be enabled on demand.
1688	*/
1689	mutex_lock(&power_domains->lock);
1690	well = lookup_power_well(display, id: SKL_DISP_PW_1);
1691	intel_power_well_enable(display, power_well: well);
1692	mutex_unlock(lock: &power_domains->lock);
1693
1694	if (DISPLAY_VER(display) == 14)
1695	intel_de_rmw(display, DC_STATE_EN,
1696	HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, set: 0);
1697
1698	/* 4. Enable CDCLK. */
1699	intel_cdclk_init_hw(display);
1700
1701	if (DISPLAY_VER(display) == 12 || display->platform.dg2)
1702	gen12_dbuf_slices_config(display);
1703
1704	/* 5. Enable DBUF. */
1705	gen9_dbuf_enable(display);
1706
1707	/* 6. Setup MBUS. */
1708	icl_mbus_init(display);
1709
1710	/* 7. Program arbiter BW_BUDDY registers */
1711	if (DISPLAY_VER(display) >= 12)
1712	tgl_bw_buddy_init(display);
1713
1714	/* 8. Ensure PHYs have completed calibration and adaptation */
1715	if (display->platform.dg2)
1716	intel_snps_phy_wait_for_calibration(display);
1717
1718	/* 9. XE2_HPD: Program CHICKEN_MISC_2 before any cursor or planes are enabled */
1719	if (DISPLAY_VERx100(display) == 1401)
1720	intel_de_rmw(display, CHICKEN_MISC_2, BMG_DARB_HALF_BLK_END_BURST, set: 1);
1721
1722	if (resume)
1723	intel_dmc_load_program(display);
1724
1725	/* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p,dg2 */
1726	if (IS_DISPLAY_VERx100(display, 1200, 1300))
1727	intel_de_rmw(display, GEN11_CHICKEN_DCPR_2, clear: 0,
1728	DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
1729	DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
1730
1731	/* Wa_14011503030:xelpd */
1732	if (DISPLAY_VER(display) == 13)
1733	intel_de_write(display, XELPD_DISPLAY_ERR_FATAL_MASK, val: ~0);
1734
1735	/* Wa_15013987218 */
1736	if (DISPLAY_VER(display) == 20) {
1737	intel_de_rmw(display, SOUTH_DSPCLK_GATE_D,
1738	clear: 0, PCH_GMBUSUNIT_CLOCK_GATE_DISABLE);
1739	intel_de_rmw(display, SOUTH_DSPCLK_GATE_D,
1740	PCH_GMBUSUNIT_CLOCK_GATE_DISABLE, set: 0);
1741	}
1742	}
1743
1744	static void icl_display_core_uninit(struct intel_display *display)
1745	{
1746	struct i915_power_domains *power_domains = &display->power.domains;
1747	struct i915_power_well *well;
1748
1749	if (!HAS_DISPLAY(display))
1750	return;
1751
1752	gen9_disable_dc_states(display);
1753	intel_dmc_disable_program(display);
1754
1755	/* 1. Disable all display engine functions -> already done */
1756
1757	/* 2. Disable DBUF */
1758	gen9_dbuf_disable(display);
1759
1760	/* 3. Disable CD clock */
1761	intel_cdclk_uninit_hw(display);
1762
1763	if (DISPLAY_VER(display) == 14)
1764	intel_de_rmw(display, DC_STATE_EN, clear: 0,
1765	HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
1766
1767	/*
1768	* 4. Disable Power Well 1 (PG1).
1769	* The AUX IO power wells are toggled on demand, so they are already
1770	* disabled at this point.
1771	*/
1772	mutex_lock(&power_domains->lock);
1773	well = lookup_power_well(display, id: SKL_DISP_PW_1);
1774	intel_power_well_disable(display, power_well: well);
1775	mutex_unlock(lock: &power_domains->lock);
1776
1777	/* 5. */
1778	intel_combo_phy_uninit(display);
1779	}
1780
1781	static void chv_phy_control_init(struct intel_display *display)
1782	{
1783	struct i915_power_well *cmn_bc =
1784	lookup_power_well(display, id: VLV_DISP_PW_DPIO_CMN_BC);
1785	struct i915_power_well *cmn_d =
1786	lookup_power_well(display, id: CHV_DISP_PW_DPIO_CMN_D);
1787
1788	/*
1789	* DISPLAY_PHY_CONTROL can get corrupted if read. As a
1790	* workaround never ever read DISPLAY_PHY_CONTROL, and
1791	* instead maintain a shadow copy ourselves. Use the actual
1792	* power well state and lane status to reconstruct the
1793	* expected initial value.
1794	*/
1795	display->power.chv_phy_control =
1796	PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1797	PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1798	PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
1799	PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
1800	PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
1801
1802	/*
1803	* If all lanes are disabled we leave the override disabled
1804	* with all power down bits cleared to match the state we
1805	* would use after disabling the port. Otherwise enable the
1806	* override and set the lane powerdown bits accding to the
1807	* current lane status.
1808	*/
1809	if (intel_power_well_is_enabled(display, power_well: cmn_bc)) {
1810	u32 status = intel_de_read(display, DPLL(display, PIPE_A));
1811	unsigned int mask;
1812
1813	mask = status & DPLL_PORTB_READY_MASK;
1814	if (mask == 0xf)
1815	mask = 0x0;
1816	else
1817	display->power.chv_phy_control |=
1818	PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
1819
1820	display->power.chv_phy_control |=
1821	PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
1822
1823	mask = (status & DPLL_PORTC_READY_MASK) >> 4;
1824	if (mask == 0xf)
1825	mask = 0x0;
1826	else
1827	display->power.chv_phy_control |=
1828	PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
1829
1830	display->power.chv_phy_control |=
1831	PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
1832
1833	display->power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1834
1835	display->power.chv_phy_assert[DPIO_PHY0] = false;
1836	} else {
1837	display->power.chv_phy_assert[DPIO_PHY0] = true;
1838	}
1839
1840	if (intel_power_well_is_enabled(display, power_well: cmn_d)) {
1841	u32 status = intel_de_read(display, DPIO_PHY_STATUS);
1842	unsigned int mask;
1843
1844	mask = status & DPLL_PORTD_READY_MASK;
1845
1846	if (mask == 0xf)
1847	mask = 0x0;
1848	else
1849	display->power.chv_phy_control |=
1850	PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
1851
1852	display->power.chv_phy_control |=
1853	PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
1854
1855	display->power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1856
1857	display->power.chv_phy_assert[DPIO_PHY1] = false;
1858	} else {
1859	display->power.chv_phy_assert[DPIO_PHY1] = true;
1860	}
1861
1862	drm_dbg_kms(display->drm, "Initial PHY_CONTROL=0x%08x\n",
1863	display->power.chv_phy_control);
1864
1865	/* Defer application of initial phy_control to enabling the powerwell */
1866	}
1867
1868	static void vlv_cmnlane_wa(struct intel_display *display)
1869	{
1870	struct i915_power_well *cmn =
1871	lookup_power_well(display, id: VLV_DISP_PW_DPIO_CMN_BC);
1872	struct i915_power_well *disp2d =
1873	lookup_power_well(display, id: VLV_DISP_PW_DISP2D);
1874
1875	/* If the display might be already active skip this */
1876	if (intel_power_well_is_enabled(display, power_well: cmn) &&
1877	intel_power_well_is_enabled(display, power_well: disp2d) &&
1878	intel_de_read(display, DPIO_CTL) & DPIO_CMNRST)
1879	return;
1880
1881	drm_dbg_kms(display->drm, "toggling display PHY side reset\n");
1882
1883	/* cmnlane needs DPLL registers */
1884	intel_power_well_enable(display, power_well: disp2d);
1885
1886	/*
1887	* From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1888	* Need to assert and de-assert PHY SB reset by gating the
1889	* common lane power, then un-gating it.
1890	* Simply ungating isn't enough to reset the PHY enough to get
1891	* ports and lanes running.
1892	*/
1893	intel_power_well_disable(display, power_well: cmn);
1894	}
1895
1896	static bool vlv_punit_is_power_gated(struct intel_display *display, u32 reg0)
1897	{
1898	bool ret;
1899
1900	vlv_punit_get(drm: display->drm);
1901	ret = (vlv_punit_read(drm: display->drm, addr: reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
1902	vlv_punit_put(drm: display->drm);
1903
1904	return ret;
1905	}
1906
1907	static void assert_ved_power_gated(struct intel_display *display)
1908	{
1909	drm_WARN(display->drm,
1910	!vlv_punit_is_power_gated(display, PUNIT_REG_VEDSSPM0),
1911	"VED not power gated\n");
1912	}
1913
1914	static void assert_isp_power_gated(struct intel_display *display)
1915	{
1916	static const struct pci_device_id isp_ids[] = {
1917	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
1918	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
1919	{}
1920	};
1921
1922	drm_WARN(display->drm, !pci_dev_present(isp_ids) &&
1923	!vlv_punit_is_power_gated(display, PUNIT_REG_ISPSSPM0),
1924	"ISP not power gated\n");
1925	}
1926
1927	static void intel_power_domains_verify_state(struct intel_display *display);
1928
1929	/**
1930	* intel_power_domains_init_hw - initialize hardware power domain state
1931	* @display: display device instance
1932	* @resume: Called from resume code paths or not
1933	*
1934	* This function initializes the hardware power domain state and enables all
1935	* power wells belonging to the INIT power domain. Power wells in other
1936	* domains (and not in the INIT domain) are referenced or disabled by
1937	* intel_modeset_readout_hw_state(). After that the reference count of each
1938	* power well must match its HW enabled state, see
1939	* intel_power_domains_verify_state().
1940	*
1941	* It will return with power domains disabled (to be enabled later by
1942	* intel_power_domains_enable()) and must be paired with
1943	* intel_power_domains_driver_remove().
1944	*/
1945	void intel_power_domains_init_hw(struct intel_display *display, bool resume)
1946	{
1947	struct i915_power_domains *power_domains = &display->power.domains;
1948
1949	power_domains->initializing = true;
1950
1951	if (DISPLAY_VER(display) >= 11) {
1952	icl_display_core_init(display, resume);
1953	} else if (display->platform.geminilake || display->platform.broxton) {
1954	bxt_display_core_init(display, resume);
1955	} else if (DISPLAY_VER(display) == 9) {
1956	skl_display_core_init(display, resume);
1957	} else if (display->platform.cherryview) {
1958	mutex_lock(&power_domains->lock);
1959	chv_phy_control_init(display);
1960	mutex_unlock(lock: &power_domains->lock);
1961	assert_isp_power_gated(display);
1962	} else if (display->platform.valleyview) {
1963	mutex_lock(&power_domains->lock);
1964	vlv_cmnlane_wa(display);
1965	mutex_unlock(lock: &power_domains->lock);
1966	assert_ved_power_gated(display);
1967	assert_isp_power_gated(display);
1968	} else if (display->platform.broadwell || display->platform.haswell) {
1969	hsw_assert_cdclk(display);
1970	intel_pch_reset_handshake(display, enable: !HAS_PCH_NOP(display));
1971	} else if (display->platform.ivybridge) {
1972	intel_pch_reset_handshake(display, enable: !HAS_PCH_NOP(display));
1973	}
1974
1975	/*
1976	* Keep all power wells enabled for any dependent HW access during
1977	* initialization and to make sure we keep BIOS enabled display HW
1978	* resources powered until display HW readout is complete. We drop
1979	* this reference in intel_power_domains_enable().
1980	*/
1981	drm_WARN_ON(display->drm, power_domains->init_wakeref);
1982	power_domains->init_wakeref =
1983	intel_display_power_get(display, domain: POWER_DOMAIN_INIT);
1984
1985	/* Disable power support if the user asked so. */
1986	if (!display->params.disable_power_well) {
1987	drm_WARN_ON(display->drm, power_domains->disable_wakeref);
1988	display->power.domains.disable_wakeref = intel_display_power_get(display,
1989	domain: POWER_DOMAIN_INIT);
1990	}
1991	intel_power_domains_sync_hw(display);
1992
1993	power_domains->initializing = false;
1994	}
1995
1996	/**
1997	* intel_power_domains_driver_remove - deinitialize hw power domain state
1998	* @display: display device instance
1999	*
2000	* De-initializes the display power domain HW state. It also ensures that the
2001	* device stays powered up so that the driver can be reloaded.
2002	*
2003	* It must be called with power domains already disabled (after a call to
2004	* intel_power_domains_disable()) and must be paired with
2005	* intel_power_domains_init_hw().
2006	*/
2007	void intel_power_domains_driver_remove(struct intel_display *display)
2008	{
2009	intel_wakeref_t wakeref __maybe_unused =
2010	fetch_and_zero(&display->power.domains.init_wakeref);
2011
2012	/* Remove the refcount we took to keep power well support disabled. */
2013	if (!display->params.disable_power_well)
2014	intel_display_power_put(display, domain: POWER_DOMAIN_INIT,
2015	fetch_and_zero(&display->power.domains.disable_wakeref));
2016
2017	intel_display_power_flush_work_sync(display);
2018
2019	intel_power_domains_verify_state(display);
2020
2021	/* Keep the power well enabled, but cancel its rpm wakeref. */
2022	intel_display_rpm_put(display, wakeref);
2023	}
2024
2025	/**
2026	* intel_power_domains_sanitize_state - sanitize power domains state
2027	* @display: display device instance
2028	*
2029	* Sanitize the power domains state during driver loading and system resume.
2030	* The function will disable all display power wells that BIOS has enabled
2031	* without a user for it (any user for a power well has taken a reference
2032	* on it by the time this function is called, after the state of all the
2033	* pipe, encoder, etc. HW resources have been sanitized).
2034	*/
2035	void intel_power_domains_sanitize_state(struct intel_display *display)
2036	{
2037	struct i915_power_domains *power_domains = &display->power.domains;
2038	struct i915_power_well *power_well;
2039
2040	mutex_lock(&power_domains->lock);
2041
2042	for_each_power_well_reverse(display, power_well) {
2043	if (power_well->desc->always_on || power_well->count ||
2044	!intel_power_well_is_enabled(display, power_well))
2045	continue;
2046
2047	drm_dbg_kms(display->drm,
2048	"BIOS left unused %s power well enabled, disabling it\n",
2049	intel_power_well_name(power_well));
2050	intel_power_well_disable(display, power_well);
2051	}
2052
2053	mutex_unlock(lock: &power_domains->lock);
2054	}
2055
2056	/**
2057	* intel_power_domains_enable - enable toggling of display power wells
2058	* @display: display device instance
2059	*
2060	* Enable the ondemand enabling/disabling of the display power wells. Note that
2061	* power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
2062	* only at specific points of the display modeset sequence, thus they are not
2063	* affected by the intel_power_domains_enable()/disable() calls. The purpose
2064	* of these function is to keep the rest of power wells enabled until the end
2065	* of display HW readout (which will acquire the power references reflecting
2066	* the current HW state).
2067	*/
2068	void intel_power_domains_enable(struct intel_display *display)
2069	{
2070	intel_wakeref_t wakeref __maybe_unused =
2071	fetch_and_zero(&display->power.domains.init_wakeref);
2072
2073	intel_display_power_put(display, domain: POWER_DOMAIN_INIT, wakeref);
2074	intel_power_domains_verify_state(display);
2075	}
2076
2077	/**
2078	* intel_power_domains_disable - disable toggling of display power wells
2079	* @display: display device instance
2080	*
2081	* Disable the ondemand enabling/disabling of the display power wells. See
2082	* intel_power_domains_enable() for which power wells this call controls.
2083	*/
2084	void intel_power_domains_disable(struct intel_display *display)
2085	{
2086	struct i915_power_domains *power_domains = &display->power.domains;
2087
2088	drm_WARN_ON(display->drm, power_domains->init_wakeref);
2089	power_domains->init_wakeref =
2090	intel_display_power_get(display, domain: POWER_DOMAIN_INIT);
2091
2092	intel_power_domains_verify_state(display);
2093	}
2094
2095	/**
2096	* intel_power_domains_suspend - suspend power domain state
2097	* @display: display device instance
2098	* @s2idle: specifies whether we go to idle, or deeper sleep
2099	*
2100	* This function prepares the hardware power domain state before entering
2101	* system suspend.
2102	*
2103	* It must be called with power domains already disabled (after a call to
2104	* intel_power_domains_disable()) and paired with intel_power_domains_resume().
2105	*/
2106	void intel_power_domains_suspend(struct intel_display *display, bool s2idle)
2107	{
2108	struct i915_power_domains *power_domains = &display->power.domains;
2109	intel_wakeref_t wakeref __maybe_unused =
2110	fetch_and_zero(&power_domains->init_wakeref);
2111
2112	intel_display_power_put(display, domain: POWER_DOMAIN_INIT, wakeref);
2113
2114	/*
2115	* In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
2116	* support don't manually deinit the power domains. This also means the
2117	* DMC firmware will stay active, it will power down any HW
2118	* resources as required and also enable deeper system power states
2119	* that would be blocked if the firmware was inactive.
2120	*/
2121	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) && s2idle &&
2122	intel_dmc_has_payload(display)) {
2123	intel_display_power_flush_work(display);
2124	intel_power_domains_verify_state(display);
2125	return;
2126	}
2127
2128	/*
2129	* Even if power well support was disabled we still want to disable
2130	* power wells if power domains must be deinitialized for suspend.
2131	*/
2132	if (!display->params.disable_power_well)
2133	intel_display_power_put(display, domain: POWER_DOMAIN_INIT,
2134	fetch_and_zero(&display->power.domains.disable_wakeref));
2135
2136	intel_display_power_flush_work(display);
2137	intel_power_domains_verify_state(display);
2138
2139	if (DISPLAY_VER(display) >= 11)
2140	icl_display_core_uninit(display);
2141	else if (display->platform.geminilake || display->platform.broxton)
2142	bxt_display_core_uninit(display);
2143	else if (DISPLAY_VER(display) == 9)
2144	skl_display_core_uninit(display);
2145
2146	power_domains->display_core_suspended = true;
2147	}
2148
2149	/**
2150	* intel_power_domains_resume - resume power domain state
2151	* @display: display device instance
2152	*
2153	* This function resume the hardware power domain state during system resume.
2154	*
2155	* It will return with power domain support disabled (to be enabled later by
2156	* intel_power_domains_enable()) and must be paired with
2157	* intel_power_domains_suspend().
2158	*/
2159	void intel_power_domains_resume(struct intel_display *display)
2160	{
2161	struct i915_power_domains *power_domains = &display->power.domains;
2162
2163	if (power_domains->display_core_suspended) {
2164	intel_power_domains_init_hw(display, resume: true);
2165	power_domains->display_core_suspended = false;
2166	} else {
2167	drm_WARN_ON(display->drm, power_domains->init_wakeref);
2168	power_domains->init_wakeref =
2169	intel_display_power_get(display, domain: POWER_DOMAIN_INIT);
2170	}
2171	}
2172
2173	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2174
2175	static void intel_power_domains_dump_info(struct intel_display *display)
2176	{
2177	struct i915_power_domains *power_domains = &display->power.domains;
2178	struct i915_power_well *power_well;
2179
2180	for_each_power_well(display, power_well) {
2181	enum intel_display_power_domain domain;
2182
2183	drm_dbg_kms(display->drm, "%-25s %d\n",
2184	intel_power_well_name(power_well), intel_power_well_refcount(power_well));
2185
2186	for_each_power_domain(domain, intel_power_well_domains(power_well))
2187	drm_dbg_kms(display->drm, " %-23s %d\n",
2188	intel_display_power_domain_str(domain),
2189	power_domains->domain_use_count[domain]);
2190	}
2191	}
2192
2193	/**
2194	* intel_power_domains_verify_state - verify the HW/SW state for all power wells
2195	* @display: display device instance
2196	*
2197	* Verify if the reference count of each power well matches its HW enabled
2198	* state and the total refcount of the domains it belongs to. This must be
2199	* called after modeset HW state sanitization, which is responsible for
2200	* acquiring reference counts for any power wells in use and disabling the
2201	* ones left on by BIOS but not required by any active output.
2202	*/
2203	static void intel_power_domains_verify_state(struct intel_display *display)
2204	{
2205	struct i915_power_domains *power_domains = &display->power.domains;
2206	struct i915_power_well *power_well;
2207	bool dump_domain_info;
2208
2209	mutex_lock(&power_domains->lock);
2210
2211	verify_async_put_domains_state(power_domains);
2212
2213	dump_domain_info = false;
2214	for_each_power_well(display, power_well) {
2215	enum intel_display_power_domain domain;
2216	int domains_count;
2217	bool enabled;
2218
2219	enabled = intel_power_well_is_enabled(display, power_well);
2220	if ((intel_power_well_refcount(power_well) ||
2221	intel_power_well_is_always_on(power_well)) !=
2222	enabled)
2223	drm_err(display->drm,
2224	"power well %s state mismatch (refcount %d/enabled %d)",
2225	intel_power_well_name(power_well),
2226	intel_power_well_refcount(power_well), enabled);
2227
2228	domains_count = 0;
2229	for_each_power_domain(domain, intel_power_well_domains(power_well))
2230	domains_count += power_domains->domain_use_count[domain];
2231
2232	if (intel_power_well_refcount(power_well) != domains_count) {
2233	drm_err(display->drm,
2234	"power well %s refcount/domain refcount mismatch "
2235	"(refcount %d/domains refcount %d)\n",
2236	intel_power_well_name(power_well),
2237	intel_power_well_refcount(power_well),
2238	domains_count);
2239	dump_domain_info = true;
2240	}
2241	}
2242
2243	if (dump_domain_info) {
2244	static bool dumped;
2245
2246	if (!dumped) {
2247	intel_power_domains_dump_info(display);
2248	dumped = true;
2249	}
2250	}
2251
2252	mutex_unlock(lock: &power_domains->lock);
2253	}
2254
2255	#else
2256
2257	static void intel_power_domains_verify_state(struct intel_display *display)
2258	{
2259	}
2260
2261	#endif
2262
2263	void intel_display_power_suspend_late(struct intel_display *display, bool s2idle)
2264	{
2265	intel_power_domains_suspend(display, s2idle);
2266
2267	if (DISPLAY_VER(display) >= 11 || display->platform.geminilake ||
2268	display->platform.broxton) {
2269	bxt_enable_dc9(display);
2270	} else if (display->platform.haswell || display->platform.broadwell) {
2271	hsw_enable_pc8(display);
2272	}
2273
2274	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2275	if (INTEL_PCH_TYPE(display) >= PCH_CNP && INTEL_PCH_TYPE(display) < PCH_DG1)
2276	intel_de_rmw(display, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
2277	}
2278
2279	void intel_display_power_resume_early(struct intel_display *display)
2280	{
2281	if (DISPLAY_VER(display) >= 11 || display->platform.geminilake ||
2282	display->platform.broxton) {
2283	gen9_sanitize_dc_state(display);
2284	bxt_disable_dc9(display);
2285	} else if (display->platform.haswell || display->platform.broadwell) {
2286	hsw_disable_pc8(display);
2287	}
2288
2289	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2290	if (INTEL_PCH_TYPE(display) >= PCH_CNP && INTEL_PCH_TYPE(display) < PCH_DG1)
2291	intel_de_rmw(display, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, set: 0);
2292
2293	intel_power_domains_resume(display);
2294	}
2295
2296	void intel_display_power_suspend(struct intel_display *display)
2297	{
2298	if (DISPLAY_VER(display) >= 11) {
2299	icl_display_core_uninit(display);
2300	bxt_enable_dc9(display);
2301	} else if (display->platform.geminilake || display->platform.broxton) {
2302	bxt_display_core_uninit(display);
2303	bxt_enable_dc9(display);
2304	} else if (display->platform.haswell || display->platform.broadwell) {
2305	hsw_enable_pc8(display);
2306	}
2307	}
2308
2309	void intel_display_power_resume(struct intel_display *display)
2310	{
2311	struct i915_power_domains *power_domains = &display->power.domains;
2312
2313	if (DISPLAY_VER(display) >= 11) {
2314	bxt_disable_dc9(display);
2315	icl_display_core_init(display, resume: true);
2316	if (intel_dmc_has_payload(display)) {
2317	if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
2318	skl_enable_dc6(display);
2319	else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
2320	gen9_enable_dc5(display);
2321	}
2322	} else if (display->platform.geminilake || display->platform.broxton) {
2323	bxt_disable_dc9(display);
2324	bxt_display_core_init(display, resume: true);
2325	if (intel_dmc_has_payload(display) &&
2326	(power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2327	gen9_enable_dc5(display);
2328	} else if (display->platform.haswell || display->platform.broadwell) {
2329	hsw_disable_pc8(display);
2330	}
2331	}
2332
2333	void intel_display_power_debug(struct intel_display *display, struct seq_file *m)
2334	{
2335	struct i915_power_domains *power_domains = &display->power.domains;
2336	int i;
2337
2338	mutex_lock(&power_domains->lock);
2339
2340	seq_printf(m, fmt: "Runtime power status: %s\n",
2341	str_enabled_disabled(v: !power_domains->init_wakeref));
2342
2343	seq_printf(m, fmt: "%-25s %s\n", "Power well/domain", "Use count");
2344	for (i = 0; i < power_domains->power_well_count; i++) {
2345	struct i915_power_well *power_well;
2346	enum intel_display_power_domain power_domain;
2347
2348	power_well = &power_domains->power_wells[i];
2349	seq_printf(m, fmt: "%-25s %d\n", intel_power_well_name(power_well),
2350	intel_power_well_refcount(power_well));
2351
2352	for_each_power_domain(power_domain, intel_power_well_domains(power_well))
2353	seq_printf(m, fmt: " %-23s %d\n",
2354	intel_display_power_domain_str(domain: power_domain),
2355	power_domains->domain_use_count[power_domain]);
2356	}
2357
2358	mutex_unlock(lock: &power_domains->lock);
2359	}
2360
2361	struct intel_ddi_port_domains {
2362	enum port port_start;
2363	enum port port_end;
2364	enum aux_ch aux_ch_start;
2365	enum aux_ch aux_ch_end;
2366
2367	enum intel_display_power_domain ddi_lanes;
2368	enum intel_display_power_domain ddi_io;
2369	enum intel_display_power_domain aux_io;
2370	enum intel_display_power_domain aux_legacy_usbc;
2371	enum intel_display_power_domain aux_tbt;
2372	};
2373
2374	static const struct intel_ddi_port_domains
2375	i9xx_port_domains[] = {
2376	{
2377	.port_start = PORT_A,
2378	.port_end = PORT_F,
2379	.aux_ch_start = AUX_CH_A,
2380	.aux_ch_end = AUX_CH_F,
2381
2382	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2383	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2384	.aux_io = POWER_DOMAIN_AUX_IO_A,
2385	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2386	.aux_tbt = POWER_DOMAIN_INVALID,
2387	},
2388	};
2389
2390	static const struct intel_ddi_port_domains
2391	d11_port_domains[] = {
2392	{
2393	.port_start = PORT_A,
2394	.port_end = PORT_B,
2395	.aux_ch_start = AUX_CH_A,
2396	.aux_ch_end = AUX_CH_B,
2397
2398	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2399	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2400	.aux_io = POWER_DOMAIN_AUX_IO_A,
2401	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2402	.aux_tbt = POWER_DOMAIN_INVALID,
2403	}, {
2404	.port_start = PORT_C,
2405	.port_end = PORT_F,
2406	.aux_ch_start = AUX_CH_C,
2407	.aux_ch_end = AUX_CH_F,
2408
2409	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_C,
2410	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_C,
2411	.aux_io = POWER_DOMAIN_AUX_IO_C,
2412	.aux_legacy_usbc = POWER_DOMAIN_AUX_C,
2413	.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2414	},
2415	};
2416
2417	static const struct intel_ddi_port_domains
2418	d12_port_domains[] = {
2419	{
2420	.port_start = PORT_A,
2421	.port_end = PORT_C,
2422	.aux_ch_start = AUX_CH_A,
2423	.aux_ch_end = AUX_CH_C,
2424
2425	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2426	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2427	.aux_io = POWER_DOMAIN_AUX_IO_A,
2428	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2429	.aux_tbt = POWER_DOMAIN_INVALID,
2430	}, {
2431	.port_start = PORT_TC1,
2432	.port_end = PORT_TC6,
2433	.aux_ch_start = AUX_CH_USBC1,
2434	.aux_ch_end = AUX_CH_USBC6,
2435
2436	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2437	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2438	.aux_io = POWER_DOMAIN_INVALID,
2439	.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2440	.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2441	},
2442	};
2443
2444	static const struct intel_ddi_port_domains
2445	d13_port_domains[] = {
2446	{
2447	.port_start = PORT_A,
2448	.port_end = PORT_C,
2449	.aux_ch_start = AUX_CH_A,
2450	.aux_ch_end = AUX_CH_C,
2451
2452	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2453	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2454	.aux_io = POWER_DOMAIN_AUX_IO_A,
2455	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2456	.aux_tbt = POWER_DOMAIN_INVALID,
2457	}, {
2458	.port_start = PORT_TC1,
2459	.port_end = PORT_TC4,
2460	.aux_ch_start = AUX_CH_USBC1,
2461	.aux_ch_end = AUX_CH_USBC4,
2462
2463	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2464	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2465	.aux_io = POWER_DOMAIN_INVALID,
2466	.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2467	.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2468	}, {
2469	.port_start = PORT_D_XELPD,
2470	.port_end = PORT_E_XELPD,
2471	.aux_ch_start = AUX_CH_D_XELPD,
2472	.aux_ch_end = AUX_CH_E_XELPD,
2473
2474	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_D,
2475	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_D,
2476	.aux_io = POWER_DOMAIN_AUX_IO_D,
2477	.aux_legacy_usbc = POWER_DOMAIN_AUX_D,
2478	.aux_tbt = POWER_DOMAIN_INVALID,
2479	},
2480	};
2481
2482	static void
2483	intel_port_domains_for_platform(struct intel_display *display,
2484	const struct intel_ddi_port_domains **domains,
2485	int *domains_size)
2486	{
2487	if (DISPLAY_VER(display) >= 13) {
2488	*domains = d13_port_domains;
2489	*domains_size = ARRAY_SIZE(d13_port_domains);
2490	} else if (DISPLAY_VER(display) >= 12) {
2491	*domains = d12_port_domains;
2492	*domains_size = ARRAY_SIZE(d12_port_domains);
2493	} else if (DISPLAY_VER(display) >= 11) {
2494	*domains = d11_port_domains;
2495	*domains_size = ARRAY_SIZE(d11_port_domains);
2496	} else {
2497	*domains = i9xx_port_domains;
2498	*domains_size = ARRAY_SIZE(i9xx_port_domains);
2499	}
2500	}
2501
2502	static const struct intel_ddi_port_domains *
2503	intel_port_domains_for_port(struct intel_display *display, enum port port)
2504	{
2505	const struct intel_ddi_port_domains *domains;
2506	int domains_size;
2507	int i;
2508
2509	intel_port_domains_for_platform(display, domains: &domains, domains_size: &domains_size);
2510	for (i = 0; i < domains_size; i++)
2511	if (port >= domains[i].port_start && port <= domains[i].port_end)
2512	return &domains[i];
2513
2514	return NULL;
2515	}
2516
2517	enum intel_display_power_domain
2518	intel_display_power_ddi_io_domain(struct intel_display *display, enum port port)
2519	{
2520	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(display, port);
2521
2522	if (drm_WARN_ON(display->drm, !domains || domains->ddi_io == POWER_DOMAIN_INVALID))
2523	return POWER_DOMAIN_PORT_DDI_IO_A;
2524
2525	return domains->ddi_io + (int)(port - domains->port_start);
2526	}
2527
2528	enum intel_display_power_domain
2529	intel_display_power_ddi_lanes_domain(struct intel_display *display, enum port port)
2530	{
2531	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(display, port);
2532
2533	if (drm_WARN_ON(display->drm, !domains || domains->ddi_lanes == POWER_DOMAIN_INVALID))
2534	return POWER_DOMAIN_PORT_DDI_LANES_A;
2535
2536	return domains->ddi_lanes + (int)(port - domains->port_start);
2537	}
2538
2539	static const struct intel_ddi_port_domains *
2540	intel_port_domains_for_aux_ch(struct intel_display *display, enum aux_ch aux_ch)
2541	{
2542	const struct intel_ddi_port_domains *domains;
2543	int domains_size;
2544	int i;
2545
2546	intel_port_domains_for_platform(display, domains: &domains, domains_size: &domains_size);
2547	for (i = 0; i < domains_size; i++)
2548	if (aux_ch >= domains[i].aux_ch_start && aux_ch <= domains[i].aux_ch_end)
2549	return &domains[i];
2550
2551	return NULL;
2552	}
2553
2554	enum intel_display_power_domain
2555	intel_display_power_aux_io_domain(struct intel_display *display, enum aux_ch aux_ch)
2556	{
2557	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(display, aux_ch);
2558
2559	if (drm_WARN_ON(display->drm, !domains || domains->aux_io == POWER_DOMAIN_INVALID))
2560	return POWER_DOMAIN_AUX_IO_A;
2561
2562	return domains->aux_io + (int)(aux_ch - domains->aux_ch_start);
2563	}
2564
2565	enum intel_display_power_domain
2566	intel_display_power_legacy_aux_domain(struct intel_display *display, enum aux_ch aux_ch)
2567	{
2568	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(display, aux_ch);
2569
2570	if (drm_WARN_ON(display->drm, !domains || domains->aux_legacy_usbc == POWER_DOMAIN_INVALID))
2571	return POWER_DOMAIN_AUX_A;
2572
2573	return domains->aux_legacy_usbc + (int)(aux_ch - domains->aux_ch_start);
2574	}
2575
2576	enum intel_display_power_domain
2577	intel_display_power_tbt_aux_domain(struct intel_display *display, enum aux_ch aux_ch)
2578	{
2579	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(display, aux_ch);
2580
2581	if (drm_WARN_ON(display->drm, !domains || domains->aux_tbt == POWER_DOMAIN_INVALID))
2582	return POWER_DOMAIN_AUX_TBT1;
2583
2584	return domains->aux_tbt + (int)(aux_ch - domains->aux_ch_start);
2585	}
2586