1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5
6#include <linux/string_helpers.h>
7#include <linux/kernel.h>
8
9#include <drm/drm_print.h>
10
11#include "gt/intel_gt_regs.h"
12
13#include "i915_drv.h"
14#include "i915_reg.h"
15#include "i915_trace.h"
16#include "i915_utils.h"
17#include "i915_wait_util.h"
18#include "intel_clock_gating.h"
19#include "intel_uncore_trace.h"
20#include "vlv_suspend.h"
21
22struct vlv_s0ix_state {
23 /* GAM */
24 u32 wr_watermark;
25 u32 gfx_prio_ctrl;
26 u32 arb_mode;
27 u32 gfx_pend_tlb0;
28 u32 gfx_pend_tlb1;
29 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
30 u32 media_max_req_count;
31 u32 gfx_max_req_count;
32 u32 render_hwsp;
33 u32 ecochk;
34 u32 bsd_hwsp;
35 u32 blt_hwsp;
36 u32 tlb_rd_addr;
37
38 /* MBC */
39 u32 g3dctl;
40 u32 gsckgctl;
41 u32 mbctl;
42
43 /* GCP */
44 u32 ucgctl1;
45 u32 ucgctl3;
46 u32 rcgctl1;
47 u32 rcgctl2;
48 u32 rstctl;
49 u32 misccpctl;
50
51 /* GPM */
52 u32 gfxpause;
53 u32 rpdeuhwtc;
54 u32 rpdeuc;
55 u32 ecobus;
56 u32 pwrdwnupctl;
57 u32 rp_down_timeout;
58 u32 rp_deucsw;
59 u32 rcubmabdtmr;
60 u32 rcedata;
61 u32 spare2gh;
62
63 /* Display 1 CZ domain */
64 u32 gt_imr;
65 u32 gt_ier;
66 u32 pm_imr;
67 u32 pm_ier;
68 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
69
70 /* GT SA CZ domain */
71 u32 tilectl;
72 u32 gt_fifoctl;
73 u32 gtlc_wake_ctrl;
74 u32 gtlc_survive;
75 u32 pmwgicz;
76
77 /* Display 2 CZ domain */
78 u32 gu_ctl0;
79 u32 gu_ctl1;
80 u32 pcbr;
81 u32 clock_gate_dis2;
82};
83
84/*
85 * Save all Gunit registers that may be lost after a D3 and a subsequent
86 * S0i[R123] transition. The list of registers needing a save/restore is
87 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
88 * registers in the following way:
89 * - Driver: saved/restored by the driver
90 * - Punit : saved/restored by the Punit firmware
91 * - No, w/o marking: no need to save/restore, since the register is R/O or
92 * used internally by the HW in a way that doesn't depend
93 * keeping the content across a suspend/resume.
94 * - Debug : used for debugging
95 *
96 * We save/restore all registers marked with 'Driver', with the following
97 * exceptions:
98 * - Registers out of use, including also registers marked with 'Debug'.
99 * These have no effect on the driver's operation, so we don't save/restore
100 * them to reduce the overhead.
101 * - Registers that are fully setup by an initialization function called from
102 * the resume path. For example many clock gating and RPS/RC6 registers.
103 * - Registers that provide the right functionality with their reset defaults.
104 *
105 * TODO: Except for registers that based on the above 3 criteria can be safely
106 * ignored, we save/restore all others, practically treating the HW context as
107 * a black-box for the driver. Further investigation is needed to reduce the
108 * saved/restored registers even further, by following the same 3 criteria.
109 */
110static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915)
111{
112 struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
113 struct intel_uncore *uncore = &i915->uncore;
114 int i;
115
116 if (!s)
117 return;
118
119 /* GAM 0x4000-0x4770 */
120 s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK);
121 s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL);
122 s->arb_mode = intel_uncore_read(uncore, ARB_MODE);
123 s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0);
124 s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1);
125
126 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
127 s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i));
128
129 s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT);
130 s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT);
131
132 s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7);
133 s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
134 s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7);
135 s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7);
136
137 s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR);
138
139 /* MBC 0x9024-0x91D0, 0x8500 */
140 s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL);
141 s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL);
142 s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL);
143
144 /* GCP 0x9400-0x9424, 0x8100-0x810C */
145 s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1);
146 s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3);
147 s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1);
148 s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2);
149 s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL);
150 s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL);
151
152 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
153 s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE);
154 s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC);
155 s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC);
156 s->ecobus = intel_uncore_read(uncore, ECOBUS);
157 s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL);
158 s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT);
159 s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW);
160 s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR);
161 s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA);
162 s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H);
163
164 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
165 s->gt_imr = intel_uncore_read(uncore, GTIMR);
166 s->gt_ier = intel_uncore_read(uncore, GTIER);
167 s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
168 s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
169
170 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
171 s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i));
172
173 /* GT SA CZ domain, 0x100000-0x138124 */
174 s->tilectl = intel_uncore_read(uncore, TILECTL);
175 s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL);
176 s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
177 s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
178 s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ);
179
180 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
181 s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0);
182 s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1);
183 s->pcbr = intel_uncore_read(uncore, VLV_PCBR);
184 s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2);
185
186 /*
187 * Not saving any of:
188 * DFT, 0x9800-0x9EC0
189 * SARB, 0xB000-0xB1FC
190 * GAC, 0x5208-0x524C, 0x14000-0x14C000
191 * PCI CFG
192 */
193}
194
195static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915)
196{
197 struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
198 struct intel_uncore *uncore = &i915->uncore;
199 int i;
200
201 if (!s)
202 return;
203
204 /* GAM 0x4000-0x4770 */
205 intel_uncore_write(uncore, GEN7_WR_WATERMARK, val: s->wr_watermark);
206 intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, val: s->gfx_prio_ctrl);
207 intel_uncore_write(uncore, ARB_MODE, val: s->arb_mode | (0xffff << 16));
208 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, val: s->gfx_pend_tlb0);
209 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, val: s->gfx_pend_tlb1);
210
211 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
212 intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), val: s->lra_limits[i]);
213
214 intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, val: s->media_max_req_count);
215 intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, val: s->gfx_max_req_count);
216
217 intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, val: s->render_hwsp);
218 intel_uncore_write(uncore, GAM_ECOCHK, val: s->ecochk);
219 intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, val: s->bsd_hwsp);
220 intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, val: s->blt_hwsp);
221
222 intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, val: s->tlb_rd_addr);
223
224 /* MBC 0x9024-0x91D0, 0x8500 */
225 intel_uncore_write(uncore, VLV_G3DCTL, val: s->g3dctl);
226 intel_uncore_write(uncore, VLV_GSCKGCTL, val: s->gsckgctl);
227 intel_uncore_write(uncore, GEN6_MBCTL, val: s->mbctl);
228
229 /* GCP 0x9400-0x9424, 0x8100-0x810C */
230 intel_uncore_write(uncore, GEN6_UCGCTL1, val: s->ucgctl1);
231 intel_uncore_write(uncore, GEN6_UCGCTL3, val: s->ucgctl3);
232 intel_uncore_write(uncore, GEN6_RCGCTL1, val: s->rcgctl1);
233 intel_uncore_write(uncore, GEN6_RCGCTL2, val: s->rcgctl2);
234 intel_uncore_write(uncore, GEN6_RSTCTL, val: s->rstctl);
235 intel_uncore_write(uncore, GEN7_MISCCPCTL, val: s->misccpctl);
236
237 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
238 intel_uncore_write(uncore, GEN6_GFXPAUSE, val: s->gfxpause);
239 intel_uncore_write(uncore, GEN6_RPDEUHWTC, val: s->rpdeuhwtc);
240 intel_uncore_write(uncore, GEN6_RPDEUC, val: s->rpdeuc);
241 intel_uncore_write(uncore, ECOBUS, val: s->ecobus);
242 intel_uncore_write(uncore, VLV_PWRDWNUPCTL, val: s->pwrdwnupctl);
243 intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, val: s->rp_down_timeout);
244 intel_uncore_write(uncore, GEN6_RPDEUCSW, val: s->rp_deucsw);
245 intel_uncore_write(uncore, GEN6_RCUBMABDTMR, val: s->rcubmabdtmr);
246 intel_uncore_write(uncore, VLV_RCEDATA, val: s->rcedata);
247 intel_uncore_write(uncore, VLV_SPAREG2H, val: s->spare2gh);
248
249 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
250 intel_uncore_write(uncore, GTIMR, val: s->gt_imr);
251 intel_uncore_write(uncore, GTIER, val: s->gt_ier);
252 intel_uncore_write(uncore, GEN6_PMIMR, val: s->pm_imr);
253 intel_uncore_write(uncore, GEN6_PMIER, val: s->pm_ier);
254
255 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
256 intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), val: s->gt_scratch[i]);
257
258 /* GT SA CZ domain, 0x100000-0x138124 */
259 intel_uncore_write(uncore, TILECTL, val: s->tilectl);
260 intel_uncore_write(uncore, GTFIFOCTL, val: s->gt_fifoctl);
261 /*
262 * Preserve the GT allow wake and GFX force clock bit, they are not
263 * be restored, as they are used to control the s0ix suspend/resume
264 * sequence by the caller.
265 */
266 intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, clear: ~VLV_GTLC_ALLOWWAKEREQ,
267 set: s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ);
268
269 intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, clear: ~VLV_GFX_CLK_FORCE_ON_BIT,
270 set: s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT);
271
272 intel_uncore_write(uncore, VLV_PMWGICZ, val: s->pmwgicz);
273
274 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
275 intel_uncore_write(uncore, VLV_GU_CTL0, val: s->gu_ctl0);
276 intel_uncore_write(uncore, VLV_GU_CTL1, val: s->gu_ctl1);
277 intel_uncore_write(uncore, VLV_PCBR, val: s->pcbr);
278 intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, val: s->clock_gate_dis2);
279}
280
281static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
282 u32 mask, u32 val)
283{
284 i915_reg_t reg = VLV_GTLC_PW_STATUS;
285 u32 reg_value;
286 int ret;
287
288 /* The HW does not like us polling for PW_STATUS frequently, so
289 * use the sleeping loop rather than risk the busy spin within
290 * intel_wait_for_register().
291 *
292 * Transitioning between RC6 states should be at most 2ms (see
293 * valleyview_enable_rps) so use a 3ms timeout.
294 */
295 ret = wait_for(((reg_value =
296 intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
297 == val, 3);
298
299 /* just trace the final value */
300 trace_i915_reg_rw(write: false, reg, val: reg_value, len: sizeof(reg_value), trace: true);
301
302 return ret;
303}
304
305static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on)
306{
307 struct intel_uncore *uncore = &i915->uncore;
308 int err;
309
310 intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, VLV_GFX_CLK_FORCE_ON_BIT,
311 set: force_on ? VLV_GFX_CLK_FORCE_ON_BIT : 0);
312
313 if (!force_on)
314 return 0;
315
316 err = intel_wait_for_register(uncore,
317 VLV_GTLC_SURVIVABILITY_REG,
318 VLV_GFX_CLK_STATUS_BIT,
319 VLV_GFX_CLK_STATUS_BIT,
320 timeout_ms: 20);
321 if (err)
322 drm_err(&i915->drm,
323 "timeout waiting for GFX clock force-on (%08x)\n",
324 intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG));
325
326 return err;
327}
328
329static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow)
330{
331 struct intel_uncore *uncore = &i915->uncore;
332 u32 mask;
333 u32 val;
334 int err;
335
336 intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ,
337 set: allow ? VLV_GTLC_ALLOWWAKEREQ : 0);
338 intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL);
339
340 mask = VLV_GTLC_ALLOWWAKEACK;
341 val = allow ? mask : 0;
342
343 err = vlv_wait_for_pw_status(i915, mask, val);
344 if (err)
345 drm_err(&i915->drm, "timeout disabling GT waking\n");
346
347 return err;
348}
349
350static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
351 bool wait_for_on)
352{
353 u32 mask;
354 u32 val;
355
356 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
357 val = wait_for_on ? mask : 0;
358
359 /*
360 * RC6 transitioning can be delayed up to 2 msec (see
361 * valleyview_enable_rps), use 3 msec for safety.
362 *
363 * This can fail to turn off the rc6 if the GPU is stuck after a failed
364 * reset and we are trying to force the machine to sleep.
365 */
366 if (vlv_wait_for_pw_status(i915: dev_priv, mask, val))
367 drm_dbg(&dev_priv->drm,
368 "timeout waiting for GT wells to go %s\n",
369 str_on_off(wait_for_on));
370}
371
372static void vlv_check_no_gt_access(struct drm_i915_private *i915)
373{
374 struct intel_uncore *uncore = &i915->uncore;
375
376 if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
377 return;
378
379 drm_dbg(&i915->drm, "GT register access while GT waking disabled\n");
380 intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
381}
382
383int vlv_suspend_complete(struct drm_i915_private *dev_priv)
384{
385 u32 mask;
386 int err;
387
388 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
389 return 0;
390
391 /*
392 * Bspec defines the following GT well on flags as debug only, so
393 * don't treat them as hard failures.
394 */
395 vlv_wait_for_gt_wells(dev_priv, wait_for_on: false);
396
397 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
398 drm_WARN_ON(&dev_priv->drm,
399 (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask);
400
401 vlv_check_no_gt_access(i915: dev_priv);
402
403 err = vlv_force_gfx_clock(i915: dev_priv, force_on: true);
404 if (err)
405 goto err1;
406
407 err = vlv_allow_gt_wake(i915: dev_priv, allow: false);
408 if (err)
409 goto err2;
410
411 vlv_save_gunit_s0ix_state(i915: dev_priv);
412
413 err = vlv_force_gfx_clock(i915: dev_priv, force_on: false);
414 if (err)
415 goto err2;
416
417 return 0;
418
419err2:
420 /* For safety always re-enable waking and disable gfx clock forcing */
421 vlv_allow_gt_wake(i915: dev_priv, allow: true);
422err1:
423 vlv_force_gfx_clock(i915: dev_priv, force_on: false);
424
425 return err;
426}
427
428int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
429{
430 int err;
431 int ret;
432
433 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
434 return 0;
435
436 /*
437 * If any of the steps fail just try to continue, that's the best we
438 * can do at this point. Return the first error code (which will also
439 * leave RPM permanently disabled).
440 */
441 ret = vlv_force_gfx_clock(i915: dev_priv, force_on: true);
442
443 vlv_restore_gunit_s0ix_state(i915: dev_priv);
444
445 err = vlv_allow_gt_wake(i915: dev_priv, allow: true);
446 if (!ret)
447 ret = err;
448
449 err = vlv_force_gfx_clock(i915: dev_priv, force_on: false);
450 if (!ret)
451 ret = err;
452
453 vlv_check_no_gt_access(i915: dev_priv);
454
455 if (rpm_resume)
456 intel_clock_gating_init(i915: dev_priv);
457
458 return ret;
459}
460
461int vlv_suspend_init(struct drm_i915_private *i915)
462{
463 if (!IS_VALLEYVIEW(i915))
464 return 0;
465
466 /* we write all the values in the struct, so no need to zero it out */
467 i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
468 GFP_KERNEL);
469 if (!i915->vlv_s0ix_state)
470 return -ENOMEM;
471
472 return 0;
473}
474
475void vlv_suspend_cleanup(struct drm_i915_private *i915)
476{
477 if (!i915->vlv_s0ix_state)
478 return;
479
480 kfree(objp: i915->vlv_s0ix_state);
481 i915->vlv_s0ix_state = NULL;
482}
483

source code of linux/drivers/gpu/drm/i915/vlv_suspend.c