1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2019 Intel Corporation
4	*/
5
6	#include <linux/string_helpers.h>
7
8	#include <drm/intel/i915_drm.h>
9
10	#include "display/intel_display_rps.h"
11	#include "display/vlv_clock.h"
12	#include "soc/intel_dram.h"
13
14	#include "i915_drv.h"
15	#include "i915_irq.h"
16	#include "i915_reg.h"
17	#include "i915_wait_util.h"
18	#include "intel_breadcrumbs.h"
19	#include "intel_gt.h"
20	#include "intel_gt_clock_utils.h"
21	#include "intel_gt_irq.h"
22	#include "intel_gt_pm.h"
23	#include "intel_gt_pm_irq.h"
24	#include "intel_gt_print.h"
25	#include "intel_gt_regs.h"
26	#include "intel_mchbar_regs.h"
27	#include "intel_pcode.h"
28	#include "intel_rps.h"
29	#include "vlv_iosf_sb.h"
30	#include "../../../platform/x86/intel_ips.h"
31
32	#define BUSY_MAX_EI 20u /* ms */
33
34	/*
35	* Lock protecting IPS related data structures
36	*/
37	static DEFINE_SPINLOCK(mchdev_lock);
38
39	static struct intel_gt *rps_to_gt(struct intel_rps *rps)
40	{
41	return container_of(rps, struct intel_gt, rps);
42	}
43
44	static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
45	{
46	return rps_to_gt(rps)->i915;
47	}
48
49	static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
50	{
51	return rps_to_gt(rps)->uncore;
52	}
53
54	static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
55	{
56	struct intel_gt *gt = rps_to_gt(rps);
57
58	return &gt_to_guc(gt)->slpc;
59	}
60
61	static bool rps_uses_slpc(struct intel_rps *rps)
62	{
63	struct intel_gt *gt = rps_to_gt(rps);
64
65	return intel_uc_uses_guc_slpc(uc: &gt->uc);
66	}
67
68	static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
69	{
70	return mask & ~rps->pm_intrmsk_mbz;
71	}
72
73	static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
74	{
75	intel_uncore_write_fw(uncore, reg, val);
76	}
77
78	static void rps_timer(struct timer_list *t)
79	{
80	struct intel_rps *rps = timer_container_of(rps, t, timer);
81	struct intel_gt *gt = rps_to_gt(rps);
82	struct intel_engine_cs *engine;
83	ktime_t dt, last, timestamp;
84	enum intel_engine_id id;
85	s64 max_busy[3] = {};
86
87	timestamp = 0;
88	for_each_engine(engine, gt, id) {
89	s64 busy;
90	int i;
91
92	dt = intel_engine_get_busy_time(engine, now: &timestamp);
93	last = engine->stats.rps;
94	engine->stats.rps = dt;
95
96	busy = ktime_to_ns(ktime_sub(dt, last));
97	for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
98	if (busy > max_busy[i])
99	swap(busy, max_busy[i]);
100	}
101	}
102	last = rps->pm_timestamp;
103	rps->pm_timestamp = timestamp;
104
105	if (intel_rps_is_active(rps)) {
106	s64 busy;
107	int i;
108
109	dt = ktime_sub(timestamp, last);
110
111	/*
112	* Our goal is to evaluate each engine independently, so we run
113	* at the lowest clocks required to sustain the heaviest
114	* workload. However, a task may be split into sequential
115	* dependent operations across a set of engines, such that
116	* the independent contributions do not account for high load,
117	* but overall the task is GPU bound. For example, consider
118	* video decode on vcs followed by colour post-processing
119	* on vecs, followed by general post-processing on rcs.
120	* Since multi-engines being active does imply a single
121	* continuous workload across all engines, we hedge our
122	* bets by only contributing a factor of the distributed
123	* load into our busyness calculation.
124	*/
125	busy = max_busy[0];
126	for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
127	if (!max_busy[i])
128	break;
129
130	busy += div_u64(dividend: max_busy[i], divisor: 1 << i);
131	}
132	GT_TRACE(gt,
133	"busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
134	busy, (int)div64_u64(100 * busy, dt),
135	max_busy[0], max_busy[1], max_busy[2],
136	rps->pm_interval);
137
138	if (100 * busy > rps->power.up_threshold * dt &&
139	rps->cur_freq < rps->max_freq_softlimit) {
140	rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
141	rps->pm_interval = 1;
142	queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
143	} else if (100 * busy < rps->power.down_threshold * dt &&
144	rps->cur_freq > rps->min_freq_softlimit) {
145	rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
146	rps->pm_interval = 1;
147	queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
148	} else {
149	rps->last_adj = 0;
150	}
151
152	mod_timer(timer: &rps->timer,
153	expires: jiffies + msecs_to_jiffies(m: rps->pm_interval));
154	rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
155	}
156	}
157
158	static void rps_start_timer(struct intel_rps *rps)
159	{
160	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
161	rps->pm_interval = 1;
162	mod_timer(timer: &rps->timer, expires: jiffies + 1);
163	}
164
165	static void rps_stop_timer(struct intel_rps *rps)
166	{
167	timer_delete_sync(timer: &rps->timer);
168	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
169	cancel_work_sync(work: &rps->work);
170	}
171
172	static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
173	{
174	u32 mask = 0;
175
176	/* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
177	if (val > rps->min_freq_softlimit)
178	mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
179	GEN6_PM_RP_DOWN_THRESHOLD |
180	GEN6_PM_RP_DOWN_TIMEOUT);
181
182	if (val < rps->max_freq_softlimit)
183	mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
184
185	mask &= rps->pm_events;
186
187	return rps_pm_sanitize_mask(rps, mask: ~mask);
188	}
189
190	static void rps_reset_ei(struct intel_rps *rps)
191	{
192	memset(&rps->ei, 0, sizeof(rps->ei));
193	}
194
195	static void rps_enable_interrupts(struct intel_rps *rps)
196	{
197	struct intel_gt *gt = rps_to_gt(rps);
198
199	GEM_BUG_ON(rps_uses_slpc(rps));
200
201	GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
202	rps->pm_events, rps_pm_mask(rps, rps->last_freq));
203
204	rps_reset_ei(rps);
205
206	spin_lock_irq(lock: gt->irq_lock);
207	gen6_gt_pm_enable_irq(gt, enable_mask: rps->pm_events);
208	spin_unlock_irq(lock: gt->irq_lock);
209
210	intel_uncore_write(uncore: gt->uncore,
211	GEN6_PMINTRMSK, val: rps_pm_mask(rps, val: rps->last_freq));
212	}
213
214	static void gen6_rps_reset_interrupts(struct intel_rps *rps)
215	{
216	gen6_gt_pm_reset_iir(gt: rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
217	}
218
219	static void gen11_rps_reset_interrupts(struct intel_rps *rps)
220	{
221	while (gen11_gt_reset_one_iir(gt: rps_to_gt(rps), bank: 0, GEN11_GTPM))
222	;
223	}
224
225	static void rps_reset_interrupts(struct intel_rps *rps)
226	{
227	struct intel_gt *gt = rps_to_gt(rps);
228
229	spin_lock_irq(lock: gt->irq_lock);
230	if (GRAPHICS_VER(gt->i915) >= 11)
231	gen11_rps_reset_interrupts(rps);
232	else
233	gen6_rps_reset_interrupts(rps);
234
235	rps->pm_iir = 0;
236	spin_unlock_irq(lock: gt->irq_lock);
237	}
238
239	static void rps_disable_interrupts(struct intel_rps *rps)
240	{
241	struct intel_gt *gt = rps_to_gt(rps);
242
243	intel_uncore_write(uncore: gt->uncore,
244	GEN6_PMINTRMSK, val: rps_pm_sanitize_mask(rps, mask: ~0u));
245
246	spin_lock_irq(lock: gt->irq_lock);
247	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
248	spin_unlock_irq(lock: gt->irq_lock);
249
250	intel_synchronize_irq(i915: gt->i915);
251
252	/*
253	* Now that we will not be generating any more work, flush any
254	* outstanding tasks. As we are called on the RPS idle path,
255	* we will reset the GPU to minimum frequencies, so the current
256	* state of the worker can be discarded.
257	*/
258	cancel_work_sync(work: &rps->work);
259
260	rps_reset_interrupts(rps);
261	GT_TRACE(gt, "interrupts:off\n");
262	}
263
264	static const struct cparams {
265	u16 i;
266	u16 t;
267	u16 m;
268	u16 c;
269	} cparams[] = {
270	{ 1, 1333, 301, 28664 },
271	{ 1, 1067, 294, 24460 },
272	{ 1, 800, 294, 25192 },
273	{ 0, 1333, 276, 27605 },
274	{ 0, 1067, 276, 27605 },
275	{ 0, 800, 231, 23784 },
276	};
277
278	static void gen5_rps_init(struct intel_rps *rps)
279	{
280	struct drm_i915_private *i915 = rps_to_i915(rps);
281	struct intel_uncore *uncore = rps_to_uncore(rps);
282	unsigned int fsb_freq, mem_freq;
283	u8 fmax, fmin, fstart;
284	u32 rgvmodectl;
285	int c_m, i;
286
287	fsb_freq = intel_fsb_freq(i915);
288	mem_freq = intel_mem_freq(i915);
289
290	if (fsb_freq <= 3200000)
291	c_m = 0;
292	else if (fsb_freq <= 4800000)
293	c_m = 1;
294	else
295	c_m = 2;
296
297	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
298	if (cparams[i].i == c_m &&
299	cparams[i].t == DIV_ROUND_CLOSEST(mem_freq, 1000)) {
300	rps->ips.m = cparams[i].m;
301	rps->ips.c = cparams[i].c;
302	break;
303	}
304	}
305
306	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
307
308	/* Set up min, max, and cur for interrupt handling */
309	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
310	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
311	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
312	MEMMODE_FSTART_SHIFT;
313	drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
314	fmax, fmin, fstart);
315
316	rps->min_freq = fmax;
317	rps->efficient_freq = fstart;
318	rps->max_freq = fmin;
319	}
320
321	static unsigned long
322	__ips_chipset_val(struct intel_ips *ips)
323	{
324	struct intel_uncore *uncore =
325	rps_to_uncore(container_of(ips, struct intel_rps, ips));
326	unsigned long now = jiffies_to_msecs(j: jiffies), dt;
327	unsigned long result;
328	u64 total, delta;
329
330	lockdep_assert_held(&mchdev_lock);
331
332	/*
333	* Prevent division-by-zero if we are asking too fast.
334	* Also, we don't get interesting results if we are polling
335	* faster than once in 10ms, so just return the saved value
336	* in such cases.
337	*/
338	dt = now - ips->last_time1;
339	if (dt <= 10)
340	return ips->chipset_power;
341
342	/* FIXME: handle per-counter overflow */
343	total = intel_uncore_read(uncore, DMIEC);
344	total += intel_uncore_read(uncore, DDREC);
345	total += intel_uncore_read(uncore, CSIEC);
346
347	delta = total - ips->last_count1;
348
349	result = div_u64(dividend: div_u64(dividend: ips->m * delta, divisor: dt) + ips->c, divisor: 10);
350
351	ips->last_count1 = total;
352	ips->last_time1 = now;
353
354	ips->chipset_power = result;
355
356	return result;
357	}
358
359	static unsigned long ips_mch_val(struct intel_uncore *uncore)
360	{
361	unsigned int m, x, b;
362	u32 tsfs;
363
364	tsfs = intel_uncore_read(uncore, TSFS);
365	x = intel_uncore_read8(uncore, TR1);
366
367	b = tsfs & TSFS_INTR_MASK;
368	m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
369
370	return m * x / 127 - b;
371	}
372
373	static int _pxvid_to_vd(u8 pxvid)
374	{
375	if (pxvid == 0)
376	return 0;
377
378	if (pxvid >= 8 && pxvid < 31)
379	pxvid = 31;
380
381	return (pxvid + 2) * 125;
382	}
383
384	static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
385	{
386	const int vd = _pxvid_to_vd(pxvid);
387
388	if (INTEL_INFO(i915)->is_mobile)
389	return max(vd - 1125, 0);
390
391	return vd;
392	}
393
394	static void __gen5_ips_update(struct intel_ips *ips)
395	{
396	struct intel_uncore *uncore =
397	rps_to_uncore(container_of(ips, struct intel_rps, ips));
398	u64 now, delta, dt;
399	u32 count;
400
401	lockdep_assert_held(&mchdev_lock);
402
403	now = ktime_get_raw_ns();
404	dt = now - ips->last_time2;
405	do_div(dt, NSEC_PER_MSEC);
406
407	/* Don't divide by 0 */
408	if (dt <= 10)
409	return;
410
411	count = intel_uncore_read(uncore, GFXEC);
412	delta = count - ips->last_count2;
413
414	ips->last_count2 = count;
415	ips->last_time2 = now;
416
417	/* More magic constants... */
418	ips->gfx_power = div_u64(dividend: delta * 1181, divisor: dt * 10);
419	}
420
421	static void gen5_rps_update(struct intel_rps *rps)
422	{
423	spin_lock_irq(lock: &mchdev_lock);
424	__gen5_ips_update(ips: &rps->ips);
425	spin_unlock_irq(lock: &mchdev_lock);
426	}
427
428	static unsigned int gen5_invert_freq(struct intel_rps *rps,
429	unsigned int val)
430	{
431	/* Invert the frequency bin into an ips delay */
432	val = rps->max_freq - val;
433	val = rps->min_freq + val;
434
435	return val;
436	}
437
438	static int __gen5_rps_set(struct intel_rps *rps, u8 val)
439	{
440	struct intel_uncore *uncore = rps_to_uncore(rps);
441	u16 rgvswctl;
442
443	lockdep_assert_held(&mchdev_lock);
444
445	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
446	if (rgvswctl & MEMCTL_CMD_STS) {
447	drm_dbg(&rps_to_i915(rps)->drm,
448	"gpu busy, RCS change rejected\n");
449	return -EBUSY; /* still busy with another command */
450	}
451
452	/* Invert the frequency bin into an ips delay */
453	val = gen5_invert_freq(rps, val);
454
455	rgvswctl =
456	(MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
457	(val << MEMCTL_FREQ_SHIFT) |
458	MEMCTL_SFCAVM;
459	intel_uncore_write16(uncore, MEMSWCTL, val: rgvswctl);
460	intel_uncore_posting_read16(uncore, MEMSWCTL);
461
462	rgvswctl |= MEMCTL_CMD_STS;
463	intel_uncore_write16(uncore, MEMSWCTL, val: rgvswctl);
464
465	return 0;
466	}
467
468	static int gen5_rps_set(struct intel_rps *rps, u8 val)
469	{
470	int err;
471
472	spin_lock_irq(lock: &mchdev_lock);
473	err = __gen5_rps_set(rps, val);
474	spin_unlock_irq(lock: &mchdev_lock);
475
476	return err;
477	}
478
479	static unsigned long intel_pxfreq(u32 vidfreq)
480	{
481	int div = (vidfreq & 0x3f0000) >> 16;
482	int post = (vidfreq & 0x3000) >> 12;
483	int pre = (vidfreq & 0x7);
484
485	if (!pre)
486	return 0;
487
488	return div * 133333 / (pre << post);
489	}
490
491	static unsigned int init_emon(struct intel_uncore *uncore)
492	{
493	u8 pxw[16];
494	int i;
495
496	/* Disable to program */
497	intel_uncore_write(uncore, ECR, val: 0);
498	intel_uncore_posting_read(uncore, ECR);
499
500	/* Program energy weights for various events */
501	intel_uncore_write(uncore, SDEW, val: 0x15040d00);
502	intel_uncore_write(uncore, CSIEW0, val: 0x007f0000);
503	intel_uncore_write(uncore, CSIEW1, val: 0x1e220004);
504	intel_uncore_write(uncore, CSIEW2, val: 0x04000004);
505
506	for (i = 0; i < 5; i++)
507	intel_uncore_write(uncore, PEW(i), val: 0);
508	for (i = 0; i < 3; i++)
509	intel_uncore_write(uncore, DEW(i), val: 0);
510
511	/* Program P-state weights to account for frequency power adjustment */
512	for (i = 0; i < 16; i++) {
513	u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
514	unsigned int freq = intel_pxfreq(vidfreq: pxvidfreq);
515	unsigned int vid =
516	(pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
517	unsigned int val;
518
519	val = vid * vid * freq / 1000 * 255;
520	val /= 127 * 127 * 900;
521
522	pxw[i] = val;
523	}
524	/* Render standby states get 0 weight */
525	pxw[14] = 0;
526	pxw[15] = 0;
527
528	for (i = 0; i < 4; i++) {
529	intel_uncore_write(uncore, PXW(i),
530	val: pxw[i * 4 + 0] << 24 |
531	pxw[i * 4 + 1] << 16 |
532	pxw[i * 4 + 2] << 8 |
533	pxw[i * 4 + 3] << 0);
534	}
535
536	/* Adjust magic regs to magic values (more experimental results) */
537	intel_uncore_write(uncore, OGW0, val: 0);
538	intel_uncore_write(uncore, OGW1, val: 0);
539	intel_uncore_write(uncore, EG0, val: 0x00007f00);
540	intel_uncore_write(uncore, EG1, val: 0x0000000e);
541	intel_uncore_write(uncore, EG2, val: 0x000e0000);
542	intel_uncore_write(uncore, EG3, val: 0x68000300);
543	intel_uncore_write(uncore, EG4, val: 0x42000000);
544	intel_uncore_write(uncore, EG5, val: 0x00140031);
545	intel_uncore_write(uncore, EG6, val: 0);
546	intel_uncore_write(uncore, EG7, val: 0);
547
548	for (i = 0; i < 8; i++)
549	intel_uncore_write(uncore, PXWL(i), val: 0);
550
551	/* Enable PMON + select events */
552	intel_uncore_write(uncore, ECR, val: 0x80000019);
553
554	return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
555	}
556
557	static bool gen5_rps_enable(struct intel_rps *rps)
558	{
559	struct drm_i915_private *i915 = rps_to_i915(rps);
560	struct intel_display *display = i915->display;
561	struct intel_uncore *uncore = rps_to_uncore(rps);
562	u8 fstart, vstart;
563	u32 rgvmodectl;
564
565	spin_lock_irq(lock: &mchdev_lock);
566
567	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
568
569	/* Enable temp reporting */
570	intel_uncore_write16(uncore, PMMISC,
571	val: intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
572	intel_uncore_write16(uncore, TSC1,
573	val: intel_uncore_read16(uncore, TSC1) | TSE);
574
575	/* 100ms RC evaluation intervals */
576	intel_uncore_write(uncore, RCUPEI, val: 100000);
577	intel_uncore_write(uncore, RCDNEI, val: 100000);
578
579	/* Set max/min thresholds to 90ms and 80ms respectively */
580	intel_uncore_write(uncore, RCBMAXAVG, val: 90000);
581	intel_uncore_write(uncore, RCBMINAVG, val: 80000);
582
583	intel_uncore_write(uncore, MEMIHYST, val: 1);
584
585	/* Set up min, max, and cur for interrupt handling */
586	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
587	MEMMODE_FSTART_SHIFT;
588
589	vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
590	PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
591
592	intel_uncore_write(uncore,
593	MEMINTREN,
594	MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
595
596	intel_uncore_write(uncore, VIDSTART, val: vstart);
597	intel_uncore_posting_read(uncore, VIDSTART);
598
599	rgvmodectl |= MEMMODE_SWMODE_EN;
600	intel_uncore_write(uncore, MEMMODECTL, val: rgvmodectl);
601
602	if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
603	MEMCTL_CMD_STS) == 0, 10))
604	drm_err(&uncore->i915->drm,
605	"stuck trying to change perf mode\n");
606	mdelay(1);
607
608	__gen5_rps_set(rps, val: rps->cur_freq);
609
610	rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
611	rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
612	rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
613	rps->ips.last_time1 = jiffies_to_msecs(j: jiffies);
614
615	rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
616	rps->ips.last_time2 = ktime_get_raw_ns();
617
618	ilk_display_rps_enable(display);
619
620	spin_unlock_irq(lock: &mchdev_lock);
621
622	rps->ips.corr = init_emon(uncore);
623
624	return true;
625	}
626
627	static void gen5_rps_disable(struct intel_rps *rps)
628	{
629	struct drm_i915_private *i915 = rps_to_i915(rps);
630	struct intel_display *display = i915->display;
631	struct intel_uncore *uncore = rps_to_uncore(rps);
632	u16 rgvswctl;
633
634	spin_lock_irq(lock: &mchdev_lock);
635
636	ilk_display_rps_disable(display);
637
638	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
639
640	/* Ack interrupts, disable EFC interrupt */
641	intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, set: 0);
642	intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
643
644	/* Go back to the starting frequency */
645	__gen5_rps_set(rps, val: rps->idle_freq);
646	mdelay(1);
647	rgvswctl |= MEMCTL_CMD_STS;
648	intel_uncore_write(uncore, MEMSWCTL, val: rgvswctl);
649	mdelay(1);
650
651	spin_unlock_irq(lock: &mchdev_lock);
652	}
653
654	static u32 rps_limits(struct intel_rps *rps, u8 val)
655	{
656	u32 limits;
657
658	/*
659	* Only set the down limit when we've reached the lowest level to avoid
660	* getting more interrupts, otherwise leave this clear. This prevents a
661	* race in the hw when coming out of rc6: There's a tiny window where
662	* the hw runs at the minimal clock before selecting the desired
663	* frequency, if the down threshold expires in that window we will not
664	* receive a down interrupt.
665	*/
666	if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
667	limits = rps->max_freq_softlimit << 23;
668	if (val <= rps->min_freq_softlimit)
669	limits |= rps->min_freq_softlimit << 14;
670	} else {
671	limits = rps->max_freq_softlimit << 24;
672	if (val <= rps->min_freq_softlimit)
673	limits |= rps->min_freq_softlimit << 16;
674	}
675
676	return limits;
677	}
678
679	static void rps_set_power(struct intel_rps *rps, int new_power)
680	{
681	struct intel_gt *gt = rps_to_gt(rps);
682	struct intel_uncore *uncore = gt->uncore;
683	u32 ei_up = 0, ei_down = 0;
684
685	lockdep_assert_held(&rps->power.mutex);
686
687	if (new_power == rps->power.mode)
688	return;
689
690	/* Note the units here are not exactly 1us, but 1280ns. */
691	switch (new_power) {
692	case LOW_POWER:
693	ei_up = 16000;
694	ei_down = 32000;
695	break;
696
697	case BETWEEN:
698	ei_up = 13000;
699	ei_down = 32000;
700	break;
701
702	case HIGH_POWER:
703	ei_up = 10000;
704	ei_down = 32000;
705	break;
706	}
707
708	/* When byt can survive without system hang with dynamic
709	* sw freq adjustments, this restriction can be lifted.
710	*/
711	if (IS_VALLEYVIEW(gt->i915))
712	goto skip_hw_write;
713
714	GT_TRACE(gt,
715	"changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
716	new_power,
717	rps->power.up_threshold, ei_up,
718	rps->power.down_threshold, ei_down);
719
720	set(uncore, GEN6_RP_UP_EI,
721	val: intel_gt_ns_to_pm_interval(gt, ns: ei_up * 1000));
722	set(uncore, GEN6_RP_UP_THRESHOLD,
723	val: intel_gt_ns_to_pm_interval(gt,
724	ns: ei_up * rps->power.up_threshold * 10));
725
726	set(uncore, GEN6_RP_DOWN_EI,
727	val: intel_gt_ns_to_pm_interval(gt, ns: ei_down * 1000));
728	set(uncore, GEN6_RP_DOWN_THRESHOLD,
729	val: intel_gt_ns_to_pm_interval(gt,
730	ns: ei_down *
731	rps->power.down_threshold * 10));
732
733	set(uncore, GEN6_RP_CONTROL,
734	val: (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
735	GEN6_RP_MEDIA_HW_NORMAL_MODE |
736	GEN6_RP_MEDIA_IS_GFX |
737	GEN6_RP_ENABLE |
738	GEN6_RP_UP_BUSY_AVG |
739	GEN6_RP_DOWN_IDLE_AVG);
740
741	skip_hw_write:
742	rps->power.mode = new_power;
743	}
744
745	static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
746	{
747	int new_power;
748
749	new_power = rps->power.mode;
750	switch (rps->power.mode) {
751	case LOW_POWER:
752	if (val > rps->efficient_freq + 1 &&
753	val > rps->cur_freq)
754	new_power = BETWEEN;
755	break;
756
757	case BETWEEN:
758	if (val <= rps->efficient_freq &&
759	val < rps->cur_freq)
760	new_power = LOW_POWER;
761	else if (val >= rps->rp0_freq &&
762	val > rps->cur_freq)
763	new_power = HIGH_POWER;
764	break;
765
766	case HIGH_POWER:
767	if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
768	val < rps->cur_freq)
769	new_power = BETWEEN;
770	break;
771	}
772	/* Max/min bins are special */
773	if (val <= rps->min_freq_softlimit)
774	new_power = LOW_POWER;
775	if (val >= rps->max_freq_softlimit)
776	new_power = HIGH_POWER;
777
778	mutex_lock(&rps->power.mutex);
779	if (rps->power.interactive)
780	new_power = HIGH_POWER;
781	rps_set_power(rps, new_power);
782	mutex_unlock(lock: &rps->power.mutex);
783	}
784
785	void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
786	{
787	GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
788	str_yes_no(interactive));
789
790	mutex_lock(&rps->power.mutex);
791	if (interactive) {
792	if (!rps->power.interactive++ && intel_rps_is_active(rps))
793	rps_set_power(rps, new_power: HIGH_POWER);
794	} else {
795	GEM_BUG_ON(!rps->power.interactive);
796	rps->power.interactive--;
797	}
798	mutex_unlock(lock: &rps->power.mutex);
799	}
800
801	static int gen6_rps_set(struct intel_rps *rps, u8 val)
802	{
803	struct intel_uncore *uncore = rps_to_uncore(rps);
804	struct drm_i915_private *i915 = rps_to_i915(rps);
805	u32 swreq;
806
807	GEM_BUG_ON(rps_uses_slpc(rps));
808
809	if (GRAPHICS_VER(i915) >= 9)
810	swreq = GEN9_FREQUENCY(val);
811	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
812	swreq = HSW_FREQUENCY(val);
813	else
814	swreq = (GEN6_FREQUENCY(val) |
815	GEN6_OFFSET(0) |
816	GEN6_AGGRESSIVE_TURBO);
817	set(uncore, GEN6_RPNSWREQ, val: swreq);
818
819	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
820	val, intel_gpu_freq(rps, val), swreq);
821
822	return 0;
823	}
824
825	static int vlv_rps_set(struct intel_rps *rps, u8 val)
826	{
827	struct drm_i915_private *i915 = rps_to_i915(rps);
828	int err;
829
830	vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
831	err = vlv_iosf_sb_write(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_REQ, val);
832	vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
833
834	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
835	val, intel_gpu_freq(rps, val));
836
837	return err;
838	}
839
840	static int rps_set(struct intel_rps *rps, u8 val, bool update)
841	{
842	struct drm_i915_private *i915 = rps_to_i915(rps);
843	int err;
844
845	if (val == rps->last_freq)
846	return 0;
847
848	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
849	err = vlv_rps_set(rps, val);
850	else if (GRAPHICS_VER(i915) >= 6)
851	err = gen6_rps_set(rps, val);
852	else
853	err = gen5_rps_set(rps, val);
854	if (err)
855	return err;
856
857	if (update && GRAPHICS_VER(i915) >= 6)
858	gen6_rps_set_thresholds(rps, val);
859	rps->last_freq = val;
860
861	return 0;
862	}
863
864	void intel_rps_unpark(struct intel_rps *rps)
865	{
866	if (!intel_rps_is_enabled(rps))
867	return;
868
869	GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
870
871	/*
872	* Use the user's desired frequency as a guide, but for better
873	* performance, jump directly to RPe as our starting frequency.
874	*/
875	mutex_lock(&rps->lock);
876
877	intel_rps_set_active(rps);
878	intel_rps_set(rps,
879	clamp(rps->cur_freq,
880	rps->min_freq_softlimit,
881	rps->max_freq_softlimit));
882
883	mutex_unlock(lock: &rps->lock);
884
885	rps->pm_iir = 0;
886	if (intel_rps_has_interrupts(rps))
887	rps_enable_interrupts(rps);
888	if (intel_rps_uses_timer(rps))
889	rps_start_timer(rps);
890
891	if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
892	gen5_rps_update(rps);
893	}
894
895	void intel_rps_park(struct intel_rps *rps)
896	{
897	int adj;
898
899	if (!intel_rps_is_enabled(rps))
900	return;
901
902	if (!intel_rps_clear_active(rps))
903	return;
904
905	if (intel_rps_uses_timer(rps))
906	rps_stop_timer(rps);
907	if (intel_rps_has_interrupts(rps))
908	rps_disable_interrupts(rps);
909
910	if (rps->last_freq <= rps->idle_freq)
911	return;
912
913	/*
914	* The punit delays the write of the frequency and voltage until it
915	* determines the GPU is awake. During normal usage we don't want to
916	* waste power changing the frequency if the GPU is sleeping (rc6).
917	* However, the GPU and driver is now idle and we do not want to delay
918	* switching to minimum voltage (reducing power whilst idle) as we do
919	* not expect to be woken in the near future and so must flush the
920	* change by waking the device.
921	*
922	* We choose to take the media powerwell (either would do to trick the
923	* punit into committing the voltage change) as that takes a lot less
924	* power than the render powerwell.
925	*/
926	intel_uncore_forcewake_get(uncore: rps_to_uncore(rps), domains: FORCEWAKE_MEDIA);
927	rps_set(rps, val: rps->idle_freq, update: false);
928	intel_uncore_forcewake_put(uncore: rps_to_uncore(rps), domains: FORCEWAKE_MEDIA);
929
930	/*
931	* Since we will try and restart from the previously requested
932	* frequency on unparking, treat this idle point as a downclock
933	* interrupt and reduce the frequency for resume. If we park/unpark
934	* more frequently than the rps worker can run, we will not respond
935	* to any EI and never see a change in frequency.
936	*
937	* (Note we accommodate Cherryview's limitation of only using an
938	* even bin by applying it to all.)
939	*/
940	adj = rps->last_adj;
941	if (adj < 0)
942	adj *= 2;
943	else /* CHV needs even encode values */
944	adj = -2;
945	rps->last_adj = adj;
946	rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
947	if (rps->cur_freq < rps->efficient_freq) {
948	rps->cur_freq = rps->efficient_freq;
949	rps->last_adj = 0;
950	}
951
952	GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
953	}
954
955	u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
956	{
957	struct intel_guc_slpc *slpc;
958
959	if (rps_uses_slpc(rps)) {
960	slpc = rps_to_slpc(rps);
961
962	return slpc->boost_freq;
963	} else {
964	return intel_gpu_freq(rps, val: rps->boost_freq);
965	}
966	}
967
968	static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
969	{
970	bool boost = false;
971
972	/* Validate against (static) hardware limits */
973	val = intel_freq_opcode(rps, val);
974	if (val < rps->min_freq || val > rps->max_freq)
975	return -EINVAL;
976
977	mutex_lock(&rps->lock);
978	if (val != rps->boost_freq) {
979	rps->boost_freq = val;
980	boost = atomic_read(v: &rps->num_waiters);
981	}
982	mutex_unlock(lock: &rps->lock);
983	if (boost)
984	queue_work(wq: rps_to_gt(rps)->i915->unordered_wq, work: &rps->work);
985
986	return 0;
987	}
988
989	int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
990	{
991	struct intel_guc_slpc *slpc;
992
993	if (rps_uses_slpc(rps)) {
994	slpc = rps_to_slpc(rps);
995
996	return intel_guc_slpc_set_boost_freq(slpc, val: freq);
997	} else {
998	return rps_set_boost_freq(rps, val: freq);
999	}
1000	}
1001
1002	void intel_rps_dec_waiters(struct intel_rps *rps)
1003	{
1004	struct intel_guc_slpc *slpc;
1005
1006	if (rps_uses_slpc(rps)) {
1007	slpc = rps_to_slpc(rps);
1008
1009	/* Don't decrement num_waiters for req where increment was skipped */
1010	if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING)
1011	return;
1012
1013	intel_guc_slpc_dec_waiters(slpc);
1014	} else {
1015	atomic_dec(v: &rps->num_waiters);
1016	}
1017	}
1018
1019	void intel_rps_boost(struct i915_request *rq)
1020	{
1021	struct intel_guc_slpc *slpc;
1022
1023	if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1024	return;
1025
1026	/* Waitboost is not needed for contexts marked with a Freq hint */
1027	if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags))
1028	return;
1029
1030	/* Serializes with i915_request_retire() */
1031	if (!test_and_set_bit(nr: I915_FENCE_FLAG_BOOST, addr: &rq->fence.flags)) {
1032	struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1033
1034	if (rps_uses_slpc(rps)) {
1035	slpc = rps_to_slpc(rps);
1036
1037	/* Waitboost should not be done with power saving profile */
1038	if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING)
1039	return;
1040
1041	/* Return if old value is non zero */
1042	if (!atomic_fetch_inc(v: &slpc->num_waiters)) {
1043	/*
1044	* Skip queuing boost work if frequency is already boosted,
1045	* but still increment num_waiters.
1046	*/
1047	if (slpc->min_freq_softlimit >= slpc->boost_freq)
1048	return;
1049
1050	GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1051	rq->fence.context, rq->fence.seqno);
1052	queue_work(wq: rps_to_gt(rps)->i915->unordered_wq,
1053	work: &slpc->boost_work);
1054	}
1055
1056	return;
1057	}
1058
1059	if (atomic_fetch_inc(v: &rps->num_waiters))
1060	return;
1061
1062	if (!intel_rps_is_active(rps))
1063	return;
1064
1065	GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1066	rq->fence.context, rq->fence.seqno);
1067
1068	if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1069	queue_work(wq: rps_to_gt(rps)->i915->unordered_wq, work: &rps->work);
1070
1071	WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1072	}
1073	}
1074
1075	int intel_rps_set(struct intel_rps *rps, u8 val)
1076	{
1077	int err;
1078
1079	lockdep_assert_held(&rps->lock);
1080	GEM_BUG_ON(val > rps->max_freq);
1081	GEM_BUG_ON(val < rps->min_freq);
1082
1083	if (intel_rps_is_active(rps)) {
1084	err = rps_set(rps, val, update: true);
1085	if (err)
1086	return err;
1087
1088	/*
1089	* Make sure we continue to get interrupts
1090	* until we hit the minimum or maximum frequencies.
1091	*/
1092	if (intel_rps_has_interrupts(rps)) {
1093	struct intel_uncore *uncore = rps_to_uncore(rps);
1094
1095	set(uncore,
1096	GEN6_RP_INTERRUPT_LIMITS, val: rps_limits(rps, val));
1097
1098	set(uncore, GEN6_PMINTRMSK, val: rps_pm_mask(rps, val));
1099	}
1100	}
1101
1102	rps->cur_freq = val;
1103	return 0;
1104	}
1105
1106	static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1107	{
1108	struct drm_i915_private *i915 = rps_to_i915(rps);
1109	struct intel_uncore *uncore = rps_to_uncore(rps);
1110
1111	if (IS_GEN9_LP(i915))
1112	return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1113	else
1114	return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1115	}
1116
1117	static void
1118	mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1119	{
1120	struct intel_uncore *uncore = rps_to_uncore(rps);
1121	u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1122	intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1123	intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1124	u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1125	intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1126	intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1127
1128	/* MTL values are in units of 16.67 MHz */
1129	caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1130	caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1131	caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1132	}
1133
1134	static void
1135	__gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1136	{
1137	struct drm_i915_private *i915 = rps_to_i915(rps);
1138	u32 rp_state_cap;
1139
1140	rp_state_cap = intel_rps_read_state_cap(rps);
1141
1142	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
1143	if (IS_GEN9_LP(i915)) {
1144	caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1145	caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1146	caps->min_freq = (rp_state_cap >> 0) & 0xff;
1147	} else {
1148	caps->rp0_freq = (rp_state_cap >> 0) & 0xff;
1149	if (GRAPHICS_VER(i915) >= 10)
1150	caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1151	intel_uncore_read(to_gt(i915)->uncore,
1152	GEN10_FREQ_INFO_REC));
1153	else
1154	caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1155	caps->min_freq = (rp_state_cap >> 16) & 0xff;
1156	}
1157
1158	if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1159	/*
1160	* In this case rp_state_cap register reports frequencies in
1161	* units of 50 MHz. Convert these to the actual "hw unit", i.e.
1162	* units of 16.67 MHz
1163	*/
1164	caps->rp0_freq *= GEN9_FREQ_SCALER;
1165	caps->rp1_freq *= GEN9_FREQ_SCALER;
1166	caps->min_freq *= GEN9_FREQ_SCALER;
1167	}
1168	}
1169
1170	/**
1171	* gen6_rps_get_freq_caps - Get freq caps exposed by HW
1172	* @rps: the intel_rps structure
1173	* @caps: returned freq caps
1174	*
1175	* Returned "caps" frequencies should be converted to MHz using
1176	* intel_gpu_freq()
1177	*/
1178	void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1179	{
1180	struct drm_i915_private *i915 = rps_to_i915(rps);
1181
1182	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1183	return mtl_get_freq_caps(rps, caps);
1184	else
1185	return __gen6_rps_get_freq_caps(rps, caps);
1186	}
1187
1188	static void gen6_rps_init(struct intel_rps *rps)
1189	{
1190	struct drm_i915_private *i915 = rps_to_i915(rps);
1191	struct intel_rps_freq_caps caps;
1192
1193	gen6_rps_get_freq_caps(rps, caps: &caps);
1194	rps->rp0_freq = caps.rp0_freq;
1195	rps->rp1_freq = caps.rp1_freq;
1196	rps->min_freq = caps.min_freq;
1197
1198	/* hw_max = RP0 until we check for overclocking */
1199	rps->max_freq = rps->rp0_freq;
1200
1201	rps->efficient_freq = rps->rp1_freq;
1202	if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1203	IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1204	u32 ddcc_status = 0;
1205	u32 mult = 1;
1206
1207	if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1208	mult = GEN9_FREQ_SCALER;
1209	if (snb_pcode_read(uncore: rps_to_gt(rps)->uncore,
1210	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1211	val: &ddcc_status, NULL) == 0)
1212	rps->efficient_freq =
1213	clamp_t(u32,
1214	((ddcc_status >> 8) & 0xff) * mult,
1215	rps->min_freq,
1216	rps->max_freq);
1217	}
1218	}
1219
1220	static bool rps_reset(struct intel_rps *rps)
1221	{
1222	struct drm_i915_private *i915 = rps_to_i915(rps);
1223
1224	/* force a reset */
1225	rps->power.mode = -1;
1226	rps->last_freq = -1;
1227
1228	if (rps_set(rps, val: rps->min_freq, update: true)) {
1229	drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1230	return false;
1231	}
1232
1233	rps->cur_freq = rps->min_freq;
1234	return true;
1235	}
1236
1237	/* See the Gen9_GT_PM_Programming_Guide doc for the below */
1238	static bool gen9_rps_enable(struct intel_rps *rps)
1239	{
1240	struct intel_gt *gt = rps_to_gt(rps);
1241	struct intel_uncore *uncore = gt->uncore;
1242
1243	/* Program defaults and thresholds for RPS */
1244	if (GRAPHICS_VER(gt->i915) == 9)
1245	intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1246	GEN9_FREQUENCY(rps->rp1_freq));
1247
1248	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1249
1250	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1251
1252	return rps_reset(rps);
1253	}
1254
1255	static bool gen8_rps_enable(struct intel_rps *rps)
1256	{
1257	struct intel_uncore *uncore = rps_to_uncore(rps);
1258
1259	intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1260	HSW_FREQUENCY(rps->rp1_freq));
1261
1262	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1263
1264	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1265
1266	return rps_reset(rps);
1267	}
1268
1269	static bool gen6_rps_enable(struct intel_rps *rps)
1270	{
1271	struct intel_uncore *uncore = rps_to_uncore(rps);
1272
1273	/* Power down if completely idle for over 50ms */
1274	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1275	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1276
1277	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1278	GEN6_PM_RP_DOWN_THRESHOLD |
1279	GEN6_PM_RP_DOWN_TIMEOUT);
1280
1281	return rps_reset(rps);
1282	}
1283
1284	static int chv_rps_max_freq(struct intel_rps *rps)
1285	{
1286	struct drm_i915_private *i915 = rps_to_i915(rps);
1287	struct intel_gt *gt = rps_to_gt(rps);
1288	u32 val;
1289
1290	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE);
1291
1292	switch (gt->info.sseu.eu_total) {
1293	case 8:
1294	/* (2 * 4) config */
1295	val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1296	break;
1297	case 12:
1298	/* (2 * 6) config */
1299	val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1300	break;
1301	case 16:
1302	/* (2 * 8) config */
1303	default:
1304	/* Setting (2 * 8) Min RP0 for any other combination */
1305	val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1306	break;
1307	}
1308
1309	return val & FB_GFX_FREQ_FUSE_MASK;
1310	}
1311
1312	static int chv_rps_rpe_freq(struct intel_rps *rps)
1313	{
1314	struct drm_i915_private *i915 = rps_to_i915(rps);
1315	u32 val;
1316
1317	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_GPU_DUTYCYCLE_REG);
1318	val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1319
1320	return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1321	}
1322
1323	static int chv_rps_guar_freq(struct intel_rps *rps)
1324	{
1325	struct drm_i915_private *i915 = rps_to_i915(rps);
1326	u32 val;
1327
1328	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE);
1329
1330	return val & FB_GFX_FREQ_FUSE_MASK;
1331	}
1332
1333	static u32 chv_rps_min_freq(struct intel_rps *rps)
1334	{
1335	struct drm_i915_private *i915 = rps_to_i915(rps);
1336	u32 val;
1337
1338	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, FB_GFX_FMIN_AT_VMIN_FUSE);
1339	val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1340
1341	return val & FB_GFX_FREQ_FUSE_MASK;
1342	}
1343
1344	static bool chv_rps_enable(struct intel_rps *rps)
1345	{
1346	struct intel_uncore *uncore = rps_to_uncore(rps);
1347	struct drm_i915_private *i915 = rps_to_i915(rps);
1348	u32 val;
1349
1350	/* 1: Program defaults and thresholds for RPS*/
1351	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1352	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1353	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1354	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1355	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1356
1357	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1358
1359	/* 2: Enable RPS */
1360	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1361	GEN6_RP_MEDIA_HW_NORMAL_MODE |
1362	GEN6_RP_MEDIA_IS_GFX |
1363	GEN6_RP_ENABLE |
1364	GEN6_RP_UP_BUSY_AVG |
1365	GEN6_RP_DOWN_IDLE_AVG);
1366
1367	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1368	GEN6_PM_RP_DOWN_THRESHOLD |
1369	GEN6_PM_RP_DOWN_TIMEOUT);
1370
1371	/* Setting Fixed Bias */
1372	vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1373
1374	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1375	vlv_iosf_sb_write(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val);
1376
1377	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
1378
1379	vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1380
1381	/* RPS code assumes GPLL is used */
1382	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1383	"GPLL not enabled\n");
1384
1385	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1386	str_yes_no(val & GPLLENABLE));
1387	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1388
1389	return rps_reset(rps);
1390	}
1391
1392	static int vlv_rps_guar_freq(struct intel_rps *rps)
1393	{
1394	struct drm_i915_private *i915 = rps_to_i915(rps);
1395	u32 val, rp1;
1396
1397	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE);
1398
1399	rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1400	rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1401
1402	return rp1;
1403	}
1404
1405	static int vlv_rps_max_freq(struct intel_rps *rps)
1406	{
1407	struct drm_i915_private *i915 = rps_to_i915(rps);
1408	u32 val, rp0;
1409
1410	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE);
1411
1412	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1413	/* Clamp to max */
1414	rp0 = min_t(u32, rp0, 0xea);
1415
1416	return rp0;
1417	}
1418
1419	static int vlv_rps_rpe_freq(struct intel_rps *rps)
1420	{
1421	struct drm_i915_private *i915 = rps_to_i915(rps);
1422	u32 val, rpe;
1423
1424	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1425	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1426	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1427	rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1428
1429	return rpe;
1430	}
1431
1432	static int vlv_rps_min_freq(struct intel_rps *rps)
1433	{
1434	struct drm_i915_private *i915 = rps_to_i915(rps);
1435	u32 val;
1436
1437	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_LFM) & 0xff;
1438	/*
1439	* According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1440	* for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1441	* a BYT-M B0 the above register contains 0xbf. Moreover when setting
1442	* a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1443	* to make sure it matches what Punit accepts.
1444	*/
1445	return max_t(u32, val, 0xc0);
1446	}
1447
1448	static bool vlv_rps_enable(struct intel_rps *rps)
1449	{
1450	struct intel_uncore *uncore = rps_to_uncore(rps);
1451	struct drm_i915_private *i915 = rps_to_i915(rps);
1452	u32 val;
1453
1454	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1455	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1456	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1457	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1458	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1459
1460	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1461
1462	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1463	GEN6_RP_MEDIA_TURBO |
1464	GEN6_RP_MEDIA_HW_NORMAL_MODE |
1465	GEN6_RP_MEDIA_IS_GFX |
1466	GEN6_RP_ENABLE |
1467	GEN6_RP_UP_BUSY_AVG |
1468	GEN6_RP_DOWN_IDLE_CONT);
1469
1470	/* WaGsvRC0ResidencyMethod:vlv */
1471	rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1472
1473	vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1474
1475	/* Setting Fixed Bias */
1476	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1477	vlv_iosf_sb_write(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val);
1478
1479	val = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
1480
1481	vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
1482
1483	/* RPS code assumes GPLL is used */
1484	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1485	"GPLL not enabled\n");
1486
1487	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1488	str_yes_no(val & GPLLENABLE));
1489	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1490
1491	return rps_reset(rps);
1492	}
1493
1494	static unsigned long __ips_gfx_val(struct intel_ips *ips)
1495	{
1496	struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1497	struct intel_uncore *uncore = rps_to_uncore(rps);
1498	unsigned int t, state1, state2;
1499	u32 pxvid, ext_v;
1500	u64 corr, corr2;
1501
1502	lockdep_assert_held(&mchdev_lock);
1503
1504	pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1505	pxvid = (pxvid >> 24) & 0x7f;
1506	ext_v = pvid_to_extvid(i915: rps_to_i915(rps), pxvid);
1507
1508	state1 = ext_v;
1509
1510	/* Revel in the empirically derived constants */
1511
1512	/* Correction factor in 1/100000 units */
1513	t = ips_mch_val(uncore);
1514	if (t > 80)
1515	corr = t * 2349 + 135940;
1516	else if (t >= 50)
1517	corr = t * 964 + 29317;
1518	else /* < 50 */
1519	corr = t * 301 + 1004;
1520
1521	corr = div_u64(dividend: corr * 150142 * state1, divisor: 10000) - 78642;
1522	corr2 = div_u64(dividend: corr, divisor: 100000) * ips->corr;
1523
1524	state2 = div_u64(dividend: corr2 * state1, divisor: 10000);
1525	state2 /= 100; /* convert to mW */
1526
1527	__gen5_ips_update(ips);
1528
1529	return ips->gfx_power + state2;
1530	}
1531
1532	static bool has_busy_stats(struct intel_rps *rps)
1533	{
1534	struct intel_engine_cs *engine;
1535	enum intel_engine_id id;
1536
1537	for_each_engine(engine, rps_to_gt(rps), id) {
1538	if (!intel_engine_supports_stats(engine))
1539	return false;
1540	}
1541
1542	return true;
1543	}
1544
1545	void intel_rps_enable(struct intel_rps *rps)
1546	{
1547	struct drm_i915_private *i915 = rps_to_i915(rps);
1548	struct intel_uncore *uncore = rps_to_uncore(rps);
1549	bool enabled = false;
1550
1551	if (!HAS_RPS(i915))
1552	return;
1553
1554	if (rps_uses_slpc(rps))
1555	return;
1556
1557	intel_gt_check_clock_frequency(gt: rps_to_gt(rps));
1558
1559	intel_uncore_forcewake_get(uncore, domains: FORCEWAKE_ALL);
1560	if (rps->max_freq <= rps->min_freq)
1561	/* leave disabled, no room for dynamic reclocking */;
1562	else if (IS_CHERRYVIEW(i915))
1563	enabled = chv_rps_enable(rps);
1564	else if (IS_VALLEYVIEW(i915))
1565	enabled = vlv_rps_enable(rps);
1566	else if (GRAPHICS_VER(i915) >= 9)
1567	enabled = gen9_rps_enable(rps);
1568	else if (GRAPHICS_VER(i915) >= 8)
1569	enabled = gen8_rps_enable(rps);
1570	else if (GRAPHICS_VER(i915) >= 6)
1571	enabled = gen6_rps_enable(rps);
1572	else if (IS_IRONLAKE_M(i915))
1573	enabled = gen5_rps_enable(rps);
1574	else
1575	MISSING_CASE(GRAPHICS_VER(i915));
1576	intel_uncore_forcewake_put(uncore, domains: FORCEWAKE_ALL);
1577	if (!enabled)
1578	return;
1579
1580	GT_TRACE(rps_to_gt(rps),
1581	"min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",
1582	rps->min_freq, rps->max_freq,
1583	intel_gpu_freq(rps, rps->min_freq),
1584	intel_gpu_freq(rps, rps->max_freq),
1585	rps->power.up_threshold,
1586	rps->power.down_threshold);
1587
1588	GEM_BUG_ON(rps->max_freq < rps->min_freq);
1589	GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1590
1591	GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1592	GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1593
1594	if (has_busy_stats(rps))
1595	intel_rps_set_timer(rps);
1596	else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1597	intel_rps_set_interrupts(rps);
1598	else
1599	/* Ironlake currently uses intel_ips.ko */ {}
1600
1601	intel_rps_set_enabled(rps);
1602	}
1603
1604	static void gen6_rps_disable(struct intel_rps *rps)
1605	{
1606	set(uncore: rps_to_uncore(rps), GEN6_RP_CONTROL, val: 0);
1607	}
1608
1609	void intel_rps_disable(struct intel_rps *rps)
1610	{
1611	struct drm_i915_private *i915 = rps_to_i915(rps);
1612
1613	if (!intel_rps_is_enabled(rps))
1614	return;
1615
1616	intel_rps_clear_enabled(rps);
1617	intel_rps_clear_interrupts(rps);
1618	intel_rps_clear_timer(rps);
1619
1620	if (GRAPHICS_VER(i915) >= 6)
1621	gen6_rps_disable(rps);
1622	else if (IS_IRONLAKE_M(i915))
1623	gen5_rps_disable(rps);
1624	}
1625
1626	static int byt_gpu_freq(struct intel_rps *rps, int val)
1627	{
1628	/*
1629	* N = val - 0xb7
1630	* Slow = Fast = GPLL ref * N
1631	*/
1632	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1633	}
1634
1635	static int byt_freq_opcode(struct intel_rps *rps, int val)
1636	{
1637	return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1638	}
1639
1640	static int chv_gpu_freq(struct intel_rps *rps, int val)
1641	{
1642	/*
1643	* N = val / 2
1644	* CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1645	*/
1646	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1647	}
1648
1649	static int chv_freq_opcode(struct intel_rps *rps, int val)
1650	{
1651	/* CHV needs even values */
1652	return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1653	}
1654
1655	int intel_gpu_freq(struct intel_rps *rps, int val)
1656	{
1657	struct drm_i915_private *i915 = rps_to_i915(rps);
1658
1659	if (GRAPHICS_VER(i915) >= 9)
1660	return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1661	GEN9_FREQ_SCALER);
1662	else if (IS_CHERRYVIEW(i915))
1663	return chv_gpu_freq(rps, val);
1664	else if (IS_VALLEYVIEW(i915))
1665	return byt_gpu_freq(rps, val);
1666	else if (GRAPHICS_VER(i915) >= 6)
1667	return val * GT_FREQUENCY_MULTIPLIER;
1668	else
1669	return val;
1670	}
1671
1672	int intel_freq_opcode(struct intel_rps *rps, int val)
1673	{
1674	struct drm_i915_private *i915 = rps_to_i915(rps);
1675
1676	if (GRAPHICS_VER(i915) >= 9)
1677	return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1678	GT_FREQUENCY_MULTIPLIER);
1679	else if (IS_CHERRYVIEW(i915))
1680	return chv_freq_opcode(rps, val);
1681	else if (IS_VALLEYVIEW(i915))
1682	return byt_freq_opcode(rps, val);
1683	else if (GRAPHICS_VER(i915) >= 6)
1684	return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1685	else
1686	return val;
1687	}
1688
1689	static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1690	{
1691	struct drm_i915_private *i915 = rps_to_i915(rps);
1692
1693	rps->gpll_ref_freq = vlv_clock_get_gpll(drm: &i915->drm);
1694
1695	drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1696	rps->gpll_ref_freq);
1697	}
1698
1699	static void vlv_rps_init(struct intel_rps *rps)
1700	{
1701	struct drm_i915_private *i915 = rps_to_i915(rps);
1702
1703	vlv_init_gpll_ref_freq(rps);
1704
1705	vlv_iosf_sb_get(drm: &i915->drm,
1706	BIT(VLV_IOSF_SB_PUNIT) |
1707	BIT(VLV_IOSF_SB_NC) |
1708	BIT(VLV_IOSF_SB_CCK));
1709
1710	rps->max_freq = vlv_rps_max_freq(rps);
1711	rps->rp0_freq = rps->max_freq;
1712	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1713	intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1714
1715	rps->efficient_freq = vlv_rps_rpe_freq(rps);
1716	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1717	intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1718
1719	rps->rp1_freq = vlv_rps_guar_freq(rps);
1720	drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1721	intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1722
1723	rps->min_freq = vlv_rps_min_freq(rps);
1724	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1725	intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1726
1727	vlv_iosf_sb_put(drm: &i915->drm,
1728	BIT(VLV_IOSF_SB_PUNIT) |
1729	BIT(VLV_IOSF_SB_NC) |
1730	BIT(VLV_IOSF_SB_CCK));
1731	}
1732
1733	static void chv_rps_init(struct intel_rps *rps)
1734	{
1735	struct drm_i915_private *i915 = rps_to_i915(rps);
1736
1737	vlv_init_gpll_ref_freq(rps);
1738
1739	vlv_iosf_sb_get(drm: &i915->drm,
1740	BIT(VLV_IOSF_SB_PUNIT) |
1741	BIT(VLV_IOSF_SB_NC) |
1742	BIT(VLV_IOSF_SB_CCK));
1743
1744	rps->max_freq = chv_rps_max_freq(rps);
1745	rps->rp0_freq = rps->max_freq;
1746	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1747	intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1748
1749	rps->efficient_freq = chv_rps_rpe_freq(rps);
1750	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1751	intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1752
1753	rps->rp1_freq = chv_rps_guar_freq(rps);
1754	drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1755	intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1756
1757	rps->min_freq = chv_rps_min_freq(rps);
1758	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1759	intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1760
1761	vlv_iosf_sb_put(drm: &i915->drm,
1762	BIT(VLV_IOSF_SB_PUNIT) |
1763	BIT(VLV_IOSF_SB_NC) |
1764	BIT(VLV_IOSF_SB_CCK));
1765
1766	drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1767	rps->rp1_freq | rps->min_freq) & 1,
1768	"Odd GPU freq values\n");
1769	}
1770
1771	static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1772	{
1773	ei->ktime = ktime_get_raw();
1774	ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1775	ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1776	}
1777
1778	static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1779	{
1780	struct drm_i915_private *i915 = rps_to_i915(rps);
1781	struct intel_uncore *uncore = rps_to_uncore(rps);
1782	const struct intel_rps_ei *prev = &rps->ei;
1783	struct intel_rps_ei now;
1784	u32 events = 0;
1785
1786	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1787	return 0;
1788
1789	vlv_c0_read(uncore, ei: &now);
1790
1791	if (prev->ktime) {
1792	u64 time, c0;
1793	u32 render, media;
1794
1795	time = ktime_us_delta(later: now.ktime, earlier: prev->ktime);
1796
1797	time *= vlv_clock_get_czclk(drm: &i915->drm);
1798
1799	/* Workload can be split between render + media,
1800	* e.g. SwapBuffers being blitted in X after being rendered in
1801	* mesa. To account for this we need to combine both engines
1802	* into our activity counter.
1803	*/
1804	render = now.render_c0 - prev->render_c0;
1805	media = now.media_c0 - prev->media_c0;
1806	c0 = max(render, media);
1807	c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1808
1809	if (c0 > time * rps->power.up_threshold)
1810	events = GEN6_PM_RP_UP_THRESHOLD;
1811	else if (c0 < time * rps->power.down_threshold)
1812	events = GEN6_PM_RP_DOWN_THRESHOLD;
1813	}
1814
1815	rps->ei = now;
1816	return events;
1817	}
1818
1819	static void rps_work(struct work_struct *work)
1820	{
1821	struct intel_rps *rps = container_of(work, typeof(*rps), work);
1822	struct intel_gt *gt = rps_to_gt(rps);
1823	struct drm_i915_private *i915 = rps_to_i915(rps);
1824	bool client_boost = false;
1825	int new_freq, adj, min, max;
1826	u32 pm_iir = 0;
1827
1828	spin_lock_irq(lock: gt->irq_lock);
1829	pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1830	client_boost = atomic_read(v: &rps->num_waiters);
1831	spin_unlock_irq(lock: gt->irq_lock);
1832
1833	/* Make sure we didn't queue anything we're not going to process. */
1834	if (!pm_iir && !client_boost)
1835	goto out;
1836
1837	mutex_lock(&rps->lock);
1838	if (!intel_rps_is_active(rps)) {
1839	mutex_unlock(lock: &rps->lock);
1840	return;
1841	}
1842
1843	pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1844
1845	adj = rps->last_adj;
1846	new_freq = rps->cur_freq;
1847	min = rps->min_freq_softlimit;
1848	max = rps->max_freq_softlimit;
1849	if (client_boost)
1850	max = rps->max_freq;
1851
1852	GT_TRACE(gt,
1853	"pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1854	pm_iir, str_yes_no(client_boost),
1855	adj, new_freq, min, max);
1856
1857	if (client_boost && new_freq < rps->boost_freq) {
1858	new_freq = rps->boost_freq;
1859	adj = 0;
1860	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1861	if (adj > 0)
1862	adj *= 2;
1863	else /* CHV needs even encode values */
1864	adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1865
1866	if (new_freq >= rps->max_freq_softlimit)
1867	adj = 0;
1868	} else if (client_boost) {
1869	adj = 0;
1870	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1871	if (rps->cur_freq > rps->efficient_freq)
1872	new_freq = rps->efficient_freq;
1873	else if (rps->cur_freq > rps->min_freq_softlimit)
1874	new_freq = rps->min_freq_softlimit;
1875	adj = 0;
1876	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1877	if (adj < 0)
1878	adj *= 2;
1879	else /* CHV needs even encode values */
1880	adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1881
1882	if (new_freq <= rps->min_freq_softlimit)
1883	adj = 0;
1884	} else { /* unknown event */
1885	adj = 0;
1886	}
1887
1888	/*
1889	* sysfs frequency limits may have snuck in while
1890	* servicing the interrupt
1891	*/
1892	new_freq += adj;
1893	new_freq = clamp_t(int, new_freq, min, max);
1894
1895	if (intel_rps_set(rps, val: new_freq)) {
1896	drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1897	adj = 0;
1898	}
1899	rps->last_adj = adj;
1900
1901	mutex_unlock(lock: &rps->lock);
1902
1903	out:
1904	spin_lock_irq(lock: gt->irq_lock);
1905	gen6_gt_pm_unmask_irq(gt, mask: rps->pm_events);
1906	spin_unlock_irq(lock: gt->irq_lock);
1907	}
1908
1909	void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1910	{
1911	struct intel_gt *gt = rps_to_gt(rps);
1912	const u32 events = rps->pm_events & pm_iir;
1913
1914	lockdep_assert_held(gt->irq_lock);
1915
1916	if (unlikely(!events))
1917	return;
1918
1919	GT_TRACE(gt, "irq events:%x\n", events);
1920
1921	gen6_gt_pm_mask_irq(gt, mask: events);
1922
1923	rps->pm_iir |= events;
1924	queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
1925	}
1926
1927	void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1928	{
1929	struct intel_gt *gt = rps_to_gt(rps);
1930	u32 events;
1931
1932	events = pm_iir & rps->pm_events;
1933	if (events) {
1934	spin_lock(lock: gt->irq_lock);
1935
1936	GT_TRACE(gt, "irq events:%x\n", events);
1937
1938	gen6_gt_pm_mask_irq(gt, mask: events);
1939	rps->pm_iir |= events;
1940
1941	queue_work(wq: gt->i915->unordered_wq, work: &rps->work);
1942	spin_unlock(lock: gt->irq_lock);
1943	}
1944
1945	if (GRAPHICS_VER(gt->i915) >= 8)
1946	return;
1947
1948	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1949	intel_engine_cs_irq(engine: gt->engine[VECS0], iir: pm_iir >> 10);
1950
1951	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1952	drm_dbg(&rps_to_i915(rps)->drm,
1953	"Command parser error, pm_iir 0x%08x\n", pm_iir);
1954	}
1955
1956	void gen5_rps_irq_handler(struct intel_rps *rps)
1957	{
1958	struct intel_uncore *uncore = rps_to_uncore(rps);
1959	u32 busy_up, busy_down, max_avg, min_avg;
1960	u8 new_freq;
1961
1962	spin_lock(lock: &mchdev_lock);
1963
1964	intel_uncore_write16(uncore,
1965	MEMINTRSTS,
1966	val: intel_uncore_read(uncore, MEMINTRSTS));
1967
1968	intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1969	busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1970	busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1971	max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1972	min_avg = intel_uncore_read(uncore, RCBMINAVG);
1973
1974	/* Handle RCS change request from hw */
1975	new_freq = rps->cur_freq;
1976	if (busy_up > max_avg)
1977	new_freq++;
1978	else if (busy_down < min_avg)
1979	new_freq--;
1980	new_freq = clamp(new_freq,
1981	rps->min_freq_softlimit,
1982	rps->max_freq_softlimit);
1983
1984	if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, val: new_freq))
1985	rps->cur_freq = new_freq;
1986
1987	spin_unlock(lock: &mchdev_lock);
1988	}
1989
1990	void intel_rps_init_early(struct intel_rps *rps)
1991	{
1992	mutex_init(&rps->lock);
1993	mutex_init(&rps->power.mutex);
1994
1995	INIT_WORK(&rps->work, rps_work);
1996	timer_setup(&rps->timer, rps_timer, 0);
1997
1998	atomic_set(v: &rps->num_waiters, i: 0);
1999	}
2000
2001	void intel_rps_init(struct intel_rps *rps)
2002	{
2003	struct drm_i915_private *i915 = rps_to_i915(rps);
2004
2005	if (rps_uses_slpc(rps))
2006	return;
2007
2008	if (IS_CHERRYVIEW(i915))
2009	chv_rps_init(rps);
2010	else if (IS_VALLEYVIEW(i915))
2011	vlv_rps_init(rps);
2012	else if (GRAPHICS_VER(i915) >= 6)
2013	gen6_rps_init(rps);
2014	else if (IS_IRONLAKE_M(i915))
2015	gen5_rps_init(rps);
2016
2017	/* Derive initial user preferences/limits from the hardware limits */
2018	rps->max_freq_softlimit = rps->max_freq;
2019	rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
2020	rps->min_freq_softlimit = rps->min_freq;
2021	rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
2022
2023	/* After setting max-softlimit, find the overclock max freq */
2024	if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2025	u32 params = 0;
2026
2027	snb_pcode_read(uncore: rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, val: &params, NULL);
2028	if (params & BIT(31)) { /* OC supported */
2029	drm_dbg(&i915->drm,
2030	"Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2031	(rps->max_freq & 0xff) * 50,
2032	(params & 0xff) * 50);
2033	rps->max_freq = params & 0xff;
2034	}
2035	}
2036
2037	/* Set default thresholds in % */
2038	rps->power.up_threshold = 95;
2039	rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;
2040	rps->power.down_threshold = 85;
2041	rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;
2042
2043	/* Finally allow us to boost to max by default */
2044	rps->boost_freq = rps->max_freq;
2045	rps->idle_freq = rps->min_freq;
2046
2047	/* Start in the middle, from here we will autotune based on workload */
2048	rps->cur_freq = rps->efficient_freq;
2049
2050	rps->pm_intrmsk_mbz = 0;
2051
2052	/*
2053	* SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2054	* if GEN6_PM_UP_EI_EXPIRED is masked.
2055	*
2056	* TODO: verify if this can be reproduced on VLV,CHV.
2057	*/
2058	if (GRAPHICS_VER(i915) <= 7)
2059	rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2060
2061	if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2062	rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2063
2064	/* GuC needs ARAT expired interrupt unmasked */
2065	if (intel_uc_uses_guc_submission(uc: &rps_to_gt(rps)->uc))
2066	rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2067	}
2068
2069	void intel_rps_sanitize(struct intel_rps *rps)
2070	{
2071	if (rps_uses_slpc(rps))
2072	return;
2073
2074	if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2075	rps_disable_interrupts(rps);
2076	}
2077
2078	u32 intel_rps_read_rpstat(struct intel_rps *rps)
2079	{
2080	struct drm_i915_private *i915 = rps_to_i915(rps);
2081	i915_reg_t rpstat;
2082
2083	rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2084
2085	return intel_uncore_read(uncore: rps_to_gt(rps)->uncore, reg: rpstat);
2086	}
2087
2088	static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2089	{
2090	struct drm_i915_private *i915 = rps_to_i915(rps);
2091	u32 cagf;
2092
2093	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2094	cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2095	else if (GRAPHICS_VER(i915) >= 12)
2096	cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2097	else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2098	cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2099	else if (GRAPHICS_VER(i915) >= 9)
2100	cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2101	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2102	cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2103	else if (GRAPHICS_VER(i915) >= 6)
2104	cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2105	else
2106	cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2107
2108	return cagf;
2109	}
2110
2111	static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2112	{
2113	struct drm_i915_private *i915 = rps_to_i915(rps);
2114	struct intel_uncore *uncore = rps_to_uncore(rps);
2115	i915_reg_t r = INVALID_MMIO_REG;
2116	u32 freq;
2117
2118	/*
2119	* For Gen12+ reading freq from HW does not need a forcewake and
2120	* registers will return 0 freq when GT is in RC6
2121	*/
2122	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2123	r = MTL_MIRROR_TARGET_WP1;
2124	} else if (GRAPHICS_VER(i915) >= 12) {
2125	r = GEN12_RPSTAT1;
2126	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2127	vlv_iosf_sb_get(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
2128	freq = vlv_iosf_sb_read(drm: &i915->drm, unit: VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
2129	vlv_iosf_sb_put(drm: &i915->drm, BIT(VLV_IOSF_SB_PUNIT));
2130	} else if (GRAPHICS_VER(i915) >= 6) {
2131	r = GEN6_RPSTAT1;
2132	} else {
2133	r = MEMSTAT_ILK;
2134	}
2135
2136	if (i915_mmio_reg_valid(r))
2137	freq = take_fw ? intel_uncore_read(uncore, reg: r) : intel_uncore_read_fw(uncore, r);
2138
2139	return intel_rps_get_cagf(rps, rpstat: freq);
2140	}
2141
2142	static u32 read_cagf(struct intel_rps *rps)
2143	{
2144	return __read_cagf(rps, take_fw: true);
2145	}
2146
2147	u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2148	{
2149	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2150	intel_wakeref_t wakeref;
2151	u32 freq = 0;
2152
2153	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2154	freq = intel_gpu_freq(rps, val: read_cagf(rps));
2155
2156	return freq;
2157	}
2158
2159	u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2160	{
2161	return intel_gpu_freq(rps, val: __read_cagf(rps, take_fw: false));
2162	}
2163
2164	static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2165	{
2166	struct intel_uncore *uncore = rps_to_uncore(rps);
2167	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2168	intel_wakeref_t wakeref;
2169	u32 freq = 0;
2170
2171	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2172	freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2173
2174	return freq;
2175	}
2176
2177	static u32 intel_rps_get_req(u32 pureq)
2178	{
2179	u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2180
2181	return req;
2182	}
2183
2184	u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2185	{
2186	u32 freq = intel_rps_get_req(pureq: intel_rps_read_punit_req(rps));
2187
2188	return intel_gpu_freq(rps, val: freq);
2189	}
2190
2191	u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2192	{
2193	if (rps_uses_slpc(rps))
2194	return intel_rps_read_punit_req_frequency(rps);
2195	else
2196	return intel_gpu_freq(rps, val: rps->cur_freq);
2197	}
2198
2199	u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2200	{
2201	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2202
2203	if (rps_uses_slpc(rps))
2204	return slpc->max_freq_softlimit;
2205	else
2206	return intel_gpu_freq(rps, val: rps->max_freq_softlimit);
2207	}
2208
2209	/**
2210	* intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2211	* @rps: the intel_rps structure
2212	*
2213	* Returns the max frequency in a raw format. In newer platforms raw is in
2214	* units of 50 MHz.
2215	*/
2216	u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2217	{
2218	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2219	u32 freq;
2220
2221	if (rps_uses_slpc(rps)) {
2222	return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2223	GT_FREQUENCY_MULTIPLIER);
2224	} else {
2225	freq = rps->max_freq;
2226	if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2227	/* Convert GT frequency to 50 MHz units */
2228	freq /= GEN9_FREQ_SCALER;
2229	}
2230	return freq;
2231	}
2232	}
2233
2234	u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2235	{
2236	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2237
2238	if (rps_uses_slpc(rps))
2239	return slpc->rp0_freq;
2240	else
2241	return intel_gpu_freq(rps, val: rps->rp0_freq);
2242	}
2243
2244	u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2245	{
2246	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2247
2248	if (rps_uses_slpc(rps))
2249	return slpc->rp1_freq;
2250	else
2251	return intel_gpu_freq(rps, val: rps->rp1_freq);
2252	}
2253
2254	u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2255	{
2256	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2257
2258	if (rps_uses_slpc(rps))
2259	return slpc->min_freq;
2260	else
2261	return intel_gpu_freq(rps, val: rps->min_freq);
2262	}
2263
2264	static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2265	{
2266	struct intel_gt *gt = rps_to_gt(rps);
2267	struct drm_i915_private *i915 = gt->i915;
2268	struct intel_uncore *uncore = gt->uncore;
2269	struct intel_rps_freq_caps caps;
2270	u32 rp_state_limits;
2271	u32 gt_perf_status;
2272	u32 rpmodectl, rpinclimit, rpdeclimit;
2273	u32 rpstat, cagf, reqf;
2274	u32 rpcurupei, rpcurup, rpprevup;
2275	u32 rpcurdownei, rpcurdown, rpprevdown;
2276	u32 rpupei, rpupt, rpdownei, rpdownt;
2277	u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2278
2279	rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2280	gen6_rps_get_freq_caps(rps, caps: &caps);
2281	if (IS_GEN9_LP(i915))
2282	gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2283	else
2284	gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2285
2286	/* RPSTAT1 is in the GT power well */
2287	intel_uncore_forcewake_get(uncore, domains: FORCEWAKE_ALL);
2288
2289	reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2290	if (GRAPHICS_VER(i915) >= 9) {
2291	reqf >>= 23;
2292	} else {
2293	reqf &= ~GEN6_TURBO_DISABLE;
2294	if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2295	reqf >>= 24;
2296	else
2297	reqf >>= 25;
2298	}
2299	reqf = intel_gpu_freq(rps, val: reqf);
2300
2301	rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2302	rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2303	rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2304
2305	rpstat = intel_rps_read_rpstat(rps);
2306	rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2307	rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2308	rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2309	rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2310	rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2311	rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2312
2313	rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2314	rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2315
2316	rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2317	rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2318
2319	cagf = intel_rps_read_actual_frequency(rps);
2320
2321	intel_uncore_forcewake_put(uncore, domains: FORCEWAKE_ALL);
2322
2323	if (GRAPHICS_VER(i915) >= 11) {
2324	pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2325	pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2326	/*
2327	* The equivalent to the PM ISR & IIR cannot be read
2328	* without affecting the current state of the system
2329	*/
2330	pm_isr = 0;
2331	pm_iir = 0;
2332	} else if (GRAPHICS_VER(i915) >= 8) {
2333	pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2334	pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2335	pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2336	pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2337	} else {
2338	pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2339	pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2340	pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2341	pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2342	}
2343	pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2344
2345	drm_printf(p, f: "Video Turbo Mode: %s\n",
2346	str_yes_no(v: rpmodectl & GEN6_RP_MEDIA_TURBO));
2347	drm_printf(p, f: "HW control enabled: %s\n",
2348	str_yes_no(v: rpmodectl & GEN6_RP_ENABLE));
2349	drm_printf(p, f: "SW control enabled: %s\n",
2350	str_yes_no(v: (rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2351
2352	drm_printf(p, f: "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2353	pm_ier, pm_imr, pm_mask);
2354	if (GRAPHICS_VER(i915) <= 10)
2355	drm_printf(p, f: "PM ISR=0x%08x IIR=0x%08x\n",
2356	pm_isr, pm_iir);
2357	drm_printf(p, f: "pm_intrmsk_mbz: 0x%08x\n",
2358	rps->pm_intrmsk_mbz);
2359	drm_printf(p, f: "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2360	drm_printf(p, f: "Render p-state ratio: %d\n",
2361	(gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2362	drm_printf(p, f: "Render p-state VID: %d\n",
2363	gt_perf_status & 0xff);
2364	drm_printf(p, f: "Render p-state limit: %d\n",
2365	rp_state_limits & 0xff);
2366	drm_printf(p, f: "RPSTAT1: 0x%08x\n", rpstat);
2367	drm_printf(p, f: "RPMODECTL: 0x%08x\n", rpmodectl);
2368	drm_printf(p, f: "RPINCLIMIT: 0x%08x\n", rpinclimit);
2369	drm_printf(p, f: "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2370	drm_printf(p, f: "RPNSWREQ: %dMHz\n", reqf);
2371	drm_printf(p, f: "CAGF: %dMHz\n", cagf);
2372	drm_printf(p, f: "RP CUR UP EI: %d (%lldns)\n",
2373	rpcurupei,
2374	intel_gt_pm_interval_to_ns(gt, count: rpcurupei));
2375	drm_printf(p, f: "RP CUR UP: %d (%lldns)\n",
2376	rpcurup, intel_gt_pm_interval_to_ns(gt, count: rpcurup));
2377	drm_printf(p, f: "RP PREV UP: %d (%lldns)\n",
2378	rpprevup, intel_gt_pm_interval_to_ns(gt, count: rpprevup));
2379	drm_printf(p, f: "Up threshold: %d%%\n",
2380	rps->power.up_threshold);
2381	drm_printf(p, f: "RP UP EI: %d (%lldns)\n",
2382	rpupei, intel_gt_pm_interval_to_ns(gt, count: rpupei));
2383	drm_printf(p, f: "RP UP THRESHOLD: %d (%lldns)\n",
2384	rpupt, intel_gt_pm_interval_to_ns(gt, count: rpupt));
2385
2386	drm_printf(p, f: "RP CUR DOWN EI: %d (%lldns)\n",
2387	rpcurdownei,
2388	intel_gt_pm_interval_to_ns(gt, count: rpcurdownei));
2389	drm_printf(p, f: "RP CUR DOWN: %d (%lldns)\n",
2390	rpcurdown,
2391	intel_gt_pm_interval_to_ns(gt, count: rpcurdown));
2392	drm_printf(p, f: "RP PREV DOWN: %d (%lldns)\n",
2393	rpprevdown,
2394	intel_gt_pm_interval_to_ns(gt, count: rpprevdown));
2395	drm_printf(p, f: "Down threshold: %d%%\n",
2396	rps->power.down_threshold);
2397	drm_printf(p, f: "RP DOWN EI: %d (%lldns)\n",
2398	rpdownei, intel_gt_pm_interval_to_ns(gt, count: rpdownei));
2399	drm_printf(p, f: "RP DOWN THRESHOLD: %d (%lldns)\n",
2400	rpdownt, intel_gt_pm_interval_to_ns(gt, count: rpdownt));
2401
2402	drm_printf(p, f: "Lowest (RPN) frequency: %dMHz\n",
2403	intel_gpu_freq(rps, val: caps.min_freq));
2404	drm_printf(p, f: "Nominal (RP1) frequency: %dMHz\n",
2405	intel_gpu_freq(rps, val: caps.rp1_freq));
2406	drm_printf(p, f: "Max non-overclocked (RP0) frequency: %dMHz\n",
2407	intel_gpu_freq(rps, val: caps.rp0_freq));
2408	drm_printf(p, f: "Max overclocked frequency: %dMHz\n",
2409	intel_gpu_freq(rps, val: rps->max_freq));
2410
2411	drm_printf(p, f: "Current freq: %d MHz\n",
2412	intel_gpu_freq(rps, val: rps->cur_freq));
2413	drm_printf(p, f: "Actual freq: %d MHz\n", cagf);
2414	drm_printf(p, f: "Idle freq: %d MHz\n",
2415	intel_gpu_freq(rps, val: rps->idle_freq));
2416	drm_printf(p, f: "Min freq: %d MHz\n",
2417	intel_gpu_freq(rps, val: rps->min_freq));
2418	drm_printf(p, f: "Boost freq: %d MHz\n",
2419	intel_gpu_freq(rps, val: rps->boost_freq));
2420	drm_printf(p, f: "Max freq: %d MHz\n",
2421	intel_gpu_freq(rps, val: rps->max_freq));
2422	drm_printf(p,
2423	f: "efficient (RPe) frequency: %d MHz\n",
2424	intel_gpu_freq(rps, val: rps->efficient_freq));
2425	}
2426
2427	static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2428	{
2429	struct intel_gt *gt = rps_to_gt(rps);
2430	struct intel_uncore *uncore = gt->uncore;
2431	struct intel_rps_freq_caps caps;
2432	u32 pm_mask;
2433
2434	gen6_rps_get_freq_caps(rps, caps: &caps);
2435	pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2436
2437	drm_printf(p, f: "PM MASK=0x%08x\n", pm_mask);
2438	drm_printf(p, f: "pm_intrmsk_mbz: 0x%08x\n",
2439	rps->pm_intrmsk_mbz);
2440	drm_printf(p, f: "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2441	drm_printf(p, f: "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2442	drm_printf(p, f: "Lowest (RPN) frequency: %dMHz\n",
2443	intel_gpu_freq(rps, val: caps.min_freq));
2444	drm_printf(p, f: "Nominal (RP1) frequency: %dMHz\n",
2445	intel_gpu_freq(rps, val: caps.rp1_freq));
2446	drm_printf(p, f: "Max non-overclocked (RP0) frequency: %dMHz\n",
2447	intel_gpu_freq(rps, val: caps.rp0_freq));
2448	drm_printf(p, f: "Current freq: %d MHz\n",
2449	intel_rps_get_requested_frequency(rps));
2450	drm_printf(p, f: "Actual freq: %d MHz\n",
2451	intel_rps_read_actual_frequency(rps));
2452	drm_printf(p, f: "Min freq: %d MHz\n",
2453	intel_rps_get_min_frequency(rps));
2454	drm_printf(p, f: "Boost freq: %d MHz\n",
2455	intel_rps_get_boost_frequency(rps));
2456	drm_printf(p, f: "Max freq: %d MHz\n",
2457	intel_rps_get_max_frequency(rps));
2458	drm_printf(p,
2459	f: "efficient (RPe) frequency: %d MHz\n",
2460	intel_gpu_freq(rps, val: caps.rp1_freq));
2461	}
2462
2463	void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2464	{
2465	if (rps_uses_slpc(rps))
2466	return slpc_frequency_dump(rps, p);
2467	else
2468	return rps_frequency_dump(rps, p);
2469	}
2470
2471	static int set_max_freq(struct intel_rps *rps, u32 val)
2472	{
2473	struct drm_i915_private *i915 = rps_to_i915(rps);
2474	int ret = 0;
2475
2476	mutex_lock(&rps->lock);
2477
2478	val = intel_freq_opcode(rps, val);
2479	if (val < rps->min_freq ||
2480	val > rps->max_freq ||
2481	val < rps->min_freq_softlimit) {
2482	ret = -EINVAL;
2483	goto unlock;
2484	}
2485
2486	if (val > rps->rp0_freq)
2487	drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2488	intel_gpu_freq(rps, val));
2489
2490	rps->max_freq_softlimit = val;
2491
2492	val = clamp_t(int, rps->cur_freq,
2493	rps->min_freq_softlimit,
2494	rps->max_freq_softlimit);
2495
2496	/*
2497	* We still need *_set_rps to process the new max_delay and
2498	* update the interrupt limits and PMINTRMSK even though
2499	* frequency request may be unchanged.
2500	*/
2501	intel_rps_set(rps, val);
2502
2503	unlock:
2504	mutex_unlock(lock: &rps->lock);
2505
2506	return ret;
2507	}
2508
2509	int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2510	{
2511	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2512
2513	if (rps_uses_slpc(rps))
2514	return intel_guc_slpc_set_max_freq(slpc, val);
2515	else
2516	return set_max_freq(rps, val);
2517	}
2518
2519	u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2520	{
2521	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2522
2523	if (rps_uses_slpc(rps))
2524	return slpc->min_freq_softlimit;
2525	else
2526	return intel_gpu_freq(rps, val: rps->min_freq_softlimit);
2527	}
2528
2529	/**
2530	* intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2531	* @rps: the intel_rps structure
2532	*
2533	* Returns the min frequency in a raw format. In newer platforms raw is in
2534	* units of 50 MHz.
2535	*/
2536	u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2537	{
2538	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2539	u32 freq;
2540
2541	if (rps_uses_slpc(rps)) {
2542	return DIV_ROUND_CLOSEST(slpc->min_freq,
2543	GT_FREQUENCY_MULTIPLIER);
2544	} else {
2545	freq = rps->min_freq;
2546	if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2547	/* Convert GT frequency to 50 MHz units */
2548	freq /= GEN9_FREQ_SCALER;
2549	}
2550	return freq;
2551	}
2552	}
2553
2554	static int set_min_freq(struct intel_rps *rps, u32 val)
2555	{
2556	int ret = 0;
2557
2558	mutex_lock(&rps->lock);
2559
2560	val = intel_freq_opcode(rps, val);
2561	if (val < rps->min_freq ||
2562	val > rps->max_freq ||
2563	val > rps->max_freq_softlimit) {
2564	ret = -EINVAL;
2565	goto unlock;
2566	}
2567
2568	rps->min_freq_softlimit = val;
2569
2570	val = clamp_t(int, rps->cur_freq,
2571	rps->min_freq_softlimit,
2572	rps->max_freq_softlimit);
2573
2574	/*
2575	* We still need *_set_rps to process the new min_delay and
2576	* update the interrupt limits and PMINTRMSK even though
2577	* frequency request may be unchanged.
2578	*/
2579	intel_rps_set(rps, val);
2580
2581	unlock:
2582	mutex_unlock(lock: &rps->lock);
2583
2584	return ret;
2585	}
2586
2587	int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2588	{
2589	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2590
2591	if (rps_uses_slpc(rps))
2592	return intel_guc_slpc_set_min_freq(slpc, val);
2593	else
2594	return set_min_freq(rps, val);
2595	}
2596
2597	u8 intel_rps_get_up_threshold(struct intel_rps *rps)
2598	{
2599	return rps->power.up_threshold;
2600	}
2601
2602	static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
2603	{
2604	int ret;
2605
2606	if (val > 100)
2607	return -EINVAL;
2608
2609	ret = mutex_lock_interruptible(&rps->lock);
2610	if (ret)
2611	return ret;
2612
2613	if (*threshold == val)
2614	goto out_unlock;
2615
2616	*threshold = val;
2617
2618	/* Force reset. */
2619	rps->last_freq = -1;
2620	mutex_lock(&rps->power.mutex);
2621	rps->power.mode = -1;
2622	mutex_unlock(lock: &rps->power.mutex);
2623
2624	intel_rps_set(rps, clamp(rps->cur_freq,
2625	rps->min_freq_softlimit,
2626	rps->max_freq_softlimit));
2627
2628	out_unlock:
2629	mutex_unlock(lock: &rps->lock);
2630
2631	return ret;
2632	}
2633
2634	int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
2635	{
2636	return rps_set_threshold(rps, threshold: &rps->power.up_threshold, val: threshold);
2637	}
2638
2639	u8 intel_rps_get_down_threshold(struct intel_rps *rps)
2640	{
2641	return rps->power.down_threshold;
2642	}
2643
2644	int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
2645	{
2646	return rps_set_threshold(rps, threshold: &rps->power.down_threshold, val: threshold);
2647	}
2648
2649	static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2650	{
2651	struct intel_uncore *uncore = rps_to_uncore(rps);
2652	u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2653
2654	/* Allow punit to process software requests */
2655	intel_uncore_write(uncore, GEN6_RP_CONTROL, val: state);
2656	}
2657
2658	void intel_rps_raise_unslice(struct intel_rps *rps)
2659	{
2660	struct intel_uncore *uncore = rps_to_uncore(rps);
2661
2662	mutex_lock(&rps->lock);
2663
2664	if (rps_uses_slpc(rps)) {
2665	/* RP limits have not been initialized yet for SLPC path */
2666	struct intel_rps_freq_caps caps;
2667
2668	gen6_rps_get_freq_caps(rps, caps: &caps);
2669
2670	intel_rps_set_manual(rps, enable: true);
2671	intel_uncore_write(uncore, GEN6_RPNSWREQ,
2672	val: ((caps.rp0_freq <<
2673	GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2674	GEN9_IGNORE_SLICE_RATIO));
2675	intel_rps_set_manual(rps, enable: false);
2676	} else {
2677	intel_rps_set(rps, val: rps->rp0_freq);
2678	}
2679
2680	mutex_unlock(lock: &rps->lock);
2681	}
2682
2683	void intel_rps_lower_unslice(struct intel_rps *rps)
2684	{
2685	struct intel_uncore *uncore = rps_to_uncore(rps);
2686
2687	mutex_lock(&rps->lock);
2688
2689	if (rps_uses_slpc(rps)) {
2690	/* RP limits have not been initialized yet for SLPC path */
2691	struct intel_rps_freq_caps caps;
2692
2693	gen6_rps_get_freq_caps(rps, caps: &caps);
2694
2695	intel_rps_set_manual(rps, enable: true);
2696	intel_uncore_write(uncore, GEN6_RPNSWREQ,
2697	val: ((caps.min_freq <<
2698	GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2699	GEN9_IGNORE_SLICE_RATIO));
2700	intel_rps_set_manual(rps, enable: false);
2701	} else {
2702	intel_rps_set(rps, val: rps->min_freq);
2703	}
2704
2705	mutex_unlock(lock: &rps->lock);
2706	}
2707
2708	static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2709	{
2710	struct intel_gt *gt = rps_to_gt(rps);
2711	intel_wakeref_t wakeref;
2712	u32 val;
2713
2714	with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2715	val = intel_uncore_read(uncore: gt->uncore, reg: reg32);
2716
2717	return val;
2718	}
2719
2720	bool rps_read_mask_mmio(struct intel_rps *rps,
2721	i915_reg_t reg32, u32 mask)
2722	{
2723	return rps_read_mmio(rps, reg32) & mask;
2724	}
2725
2726	/* External interface for intel_ips.ko */
2727
2728	static struct drm_i915_private __rcu *ips_mchdev;
2729
2730	/*
2731	* Tells the intel_ips driver that the i915 driver is now loaded, if
2732	* IPS got loaded first.
2733	*
2734	* This awkward dance is so that neither module has to depend on the
2735	* other in order for IPS to do the appropriate communication of
2736	* GPU turbo limits to i915.
2737	*/
2738	static void
2739	ips_ping_for_i915_load(void)
2740	{
2741	void (*link)(void);
2742
2743	link = symbol_get(ips_link_to_i915_driver);
2744	if (link) {
2745	link();
2746	symbol_put(ips_link_to_i915_driver);
2747	}
2748	}
2749
2750	void intel_rps_driver_register(struct intel_rps *rps)
2751	{
2752	struct intel_gt *gt = rps_to_gt(rps);
2753
2754	/*
2755	* We only register the i915 ips part with intel-ips once everything is
2756	* set up, to avoid intel-ips sneaking in and reading bogus values.
2757	*/
2758	if (GRAPHICS_VER(gt->i915) == 5) {
2759	GEM_BUG_ON(ips_mchdev);
2760	rcu_assign_pointer(ips_mchdev, gt->i915);
2761	ips_ping_for_i915_load();
2762	}
2763	}
2764
2765	void intel_rps_driver_unregister(struct intel_rps *rps)
2766	{
2767	if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2768	rcu_assign_pointer(ips_mchdev, NULL);
2769	}
2770
2771	static struct drm_i915_private *mchdev_get(void)
2772	{
2773	struct drm_i915_private *i915;
2774
2775	rcu_read_lock();
2776	i915 = rcu_dereference(ips_mchdev);
2777	if (i915 && !kref_get_unless_zero(kref: &i915->drm.ref))
2778	i915 = NULL;
2779	rcu_read_unlock();
2780
2781	return i915;
2782	}
2783
2784	/**
2785	* i915_read_mch_val - return value for IPS use
2786	*
2787	* Calculate and return a value for the IPS driver to use when deciding whether
2788	* we have thermal and power headroom to increase CPU or GPU power budget.
2789	*/
2790	unsigned long i915_read_mch_val(void)
2791	{
2792	struct drm_i915_private *i915;
2793	unsigned long chipset_val = 0;
2794	unsigned long graphics_val = 0;
2795	intel_wakeref_t wakeref;
2796
2797	i915 = mchdev_get();
2798	if (!i915)
2799	return 0;
2800
2801	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2802	struct intel_ips *ips = &to_gt(i915)->rps.ips;
2803
2804	spin_lock_irq(lock: &mchdev_lock);
2805	chipset_val = __ips_chipset_val(ips);
2806	graphics_val = __ips_gfx_val(ips);
2807	spin_unlock_irq(lock: &mchdev_lock);
2808	}
2809
2810	drm_dev_put(dev: &i915->drm);
2811	return chipset_val + graphics_val;
2812	}
2813	EXPORT_SYMBOL_GPL(i915_read_mch_val);
2814
2815	/**
2816	* i915_gpu_raise - raise GPU frequency limit
2817	*
2818	* Raise the limit; IPS indicates we have thermal headroom.
2819	*/
2820	bool i915_gpu_raise(void)
2821	{
2822	struct drm_i915_private *i915;
2823	struct intel_rps *rps;
2824
2825	i915 = mchdev_get();
2826	if (!i915)
2827	return false;
2828
2829	rps = &to_gt(i915)->rps;
2830
2831	spin_lock_irq(lock: &mchdev_lock);
2832	if (rps->max_freq_softlimit < rps->max_freq)
2833	rps->max_freq_softlimit++;
2834	spin_unlock_irq(lock: &mchdev_lock);
2835
2836	drm_dev_put(dev: &i915->drm);
2837	return true;
2838	}
2839	EXPORT_SYMBOL_GPL(i915_gpu_raise);
2840
2841	/**
2842	* i915_gpu_lower - lower GPU frequency limit
2843	*
2844	* IPS indicates we're close to a thermal limit, so throttle back the GPU
2845	* frequency maximum.
2846	*/
2847	bool i915_gpu_lower(void)
2848	{
2849	struct drm_i915_private *i915;
2850	struct intel_rps *rps;
2851
2852	i915 = mchdev_get();
2853	if (!i915)
2854	return false;
2855
2856	rps = &to_gt(i915)->rps;
2857
2858	spin_lock_irq(lock: &mchdev_lock);
2859	if (rps->max_freq_softlimit > rps->min_freq)
2860	rps->max_freq_softlimit--;
2861	spin_unlock_irq(lock: &mchdev_lock);
2862
2863	drm_dev_put(dev: &i915->drm);
2864	return true;
2865	}
2866	EXPORT_SYMBOL_GPL(i915_gpu_lower);
2867
2868	/**
2869	* i915_gpu_busy - indicate GPU business to IPS
2870	*
2871	* Tell the IPS driver whether or not the GPU is busy.
2872	*/
2873	bool i915_gpu_busy(void)
2874	{
2875	struct drm_i915_private *i915;
2876	bool ret;
2877
2878	i915 = mchdev_get();
2879	if (!i915)
2880	return false;
2881
2882	ret = to_gt(i915)->awake;
2883
2884	drm_dev_put(dev: &i915->drm);
2885	return ret;
2886	}
2887	EXPORT_SYMBOL_GPL(i915_gpu_busy);
2888
2889	/**
2890	* i915_gpu_turbo_disable - disable graphics turbo
2891	*
2892	* Disable graphics turbo by resetting the max frequency and setting the
2893	* current frequency to the default.
2894	*/
2895	bool i915_gpu_turbo_disable(void)
2896	{
2897	struct drm_i915_private *i915;
2898	struct intel_rps *rps;
2899	bool ret;
2900
2901	i915 = mchdev_get();
2902	if (!i915)
2903	return false;
2904
2905	rps = &to_gt(i915)->rps;
2906
2907	spin_lock_irq(lock: &mchdev_lock);
2908	rps->max_freq_softlimit = rps->min_freq;
2909	ret = !__gen5_rps_set(rps: &to_gt(i915)->rps, val: rps->min_freq);
2910	spin_unlock_irq(lock: &mchdev_lock);
2911
2912	drm_dev_put(dev: &i915->drm);
2913	return ret;
2914	}
2915	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2916
2917	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2918	#include "selftest_rps.c"
2919	#include "selftest_slpc.c"
2920	#endif
2921