1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include "gem/i915_gem_pm.h"
7#include "gem/i915_gem_ttm_pm.h"
8#include "gt/intel_gt.h"
9#include "gt/intel_gt_pm.h"
10#include "gt/intel_gt_requests.h"
11
12#include "i915_driver.h"
13#include "i915_drv.h"
14
15#if IS_ENABLED(CONFIG_X86)
16#include <asm/smp.h>
17#else
18#define wbinvd_on_all_cpus() \
19 pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
20#endif
21
22void i915_gem_suspend(struct drm_i915_private *i915)
23{
24 struct intel_gt *gt;
25 unsigned int i;
26
27 GEM_TRACE("%s\n", dev_name(i915->drm.dev));
28
29 intel_wakeref_auto(wf: &i915->runtime_pm.userfault_wakeref, timeout: 0);
30 /*
31 * On rare occasions, we've observed the fence completion triggers
32 * free_engines asynchronously via rcu_call. Ensure those are done.
33 * This path is only called on suspend, so it's an acceptable cost.
34 */
35 rcu_barrier();
36
37 flush_workqueue(i915->wq);
38
39 /*
40 * We have to flush all the executing contexts to main memory so
41 * that they can saved in the hibernation image. To ensure the last
42 * context image is coherent, we have to switch away from it. That
43 * leaves the i915->kernel_context still active when
44 * we actually suspend, and its image in memory may not match the GPU
45 * state. Fortunately, the kernel_context is disposable and we do
46 * not rely on its state.
47 */
48 for_each_gt(gt, i915, i)
49 intel_gt_suspend_prepare(gt);
50
51 i915_gem_drain_freed_objects(i915);
52}
53
54static int lmem_restore(struct drm_i915_private *i915, u32 flags)
55{
56 struct intel_memory_region *mr;
57 int ret = 0, id;
58
59 for_each_memory_region(mr, i915, id) {
60 if (mr->type == INTEL_MEMORY_LOCAL) {
61 ret = i915_ttm_restore_region(mr, flags);
62 if (ret)
63 break;
64 }
65 }
66
67 return ret;
68}
69
70static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
71{
72 struct intel_memory_region *mr;
73 int ret = 0, id;
74
75 for_each_memory_region(mr, i915, id) {
76 if (mr->type == INTEL_MEMORY_LOCAL) {
77 ret = i915_ttm_backup_region(mr, flags);
78 if (ret)
79 break;
80 }
81 }
82
83 return ret;
84}
85
86static void lmem_recover(struct drm_i915_private *i915)
87{
88 struct intel_memory_region *mr;
89 int id;
90
91 for_each_memory_region(mr, i915, id)
92 if (mr->type == INTEL_MEMORY_LOCAL)
93 i915_ttm_recover_region(mr);
94}
95
96int i915_gem_backup_suspend(struct drm_i915_private *i915)
97{
98 int ret;
99
100 /* Opportunistically try to evict unpinned objects */
101 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
102 if (ret)
103 goto out_recover;
104
105 i915_gem_suspend(i915);
106
107 /*
108 * More objects may have become unpinned as requests were
109 * retired. Now try to evict again. The gt may be wedged here
110 * in which case we automatically fall back to memcpy.
111 * We allow also backing up pinned objects that have not been
112 * marked for early recover, and that may contain, for example,
113 * page-tables for the migrate context.
114 */
115 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
116 I915_TTM_BACKUP_PINNED);
117 if (ret)
118 goto out_recover;
119
120 /*
121 * Remaining objects are backed up using memcpy once we've stopped
122 * using the migrate context.
123 */
124 ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
125 if (ret)
126 goto out_recover;
127
128 return 0;
129
130out_recover:
131 lmem_recover(i915);
132
133 return ret;
134}
135
136void i915_gem_suspend_late(struct drm_i915_private *i915)
137{
138 struct drm_i915_gem_object *obj;
139 struct list_head *phases[] = {
140 &i915->mm.shrink_list,
141 &i915->mm.purge_list,
142 NULL
143 }, **phase;
144 struct intel_gt *gt;
145 unsigned long flags;
146 unsigned int i;
147 bool flush = false;
148
149 /*
150 * Neither the BIOS, ourselves or any other kernel
151 * expects the system to be in execlists mode on startup,
152 * so we need to reset the GPU back to legacy mode. And the only
153 * known way to disable logical contexts is through a GPU reset.
154 *
155 * So in order to leave the system in a known default configuration,
156 * always reset the GPU upon unload and suspend. Afterwards we then
157 * clean up the GEM state tracking, flushing off the requests and
158 * leaving the system in a known idle state.
159 *
160 * Note that is of the upmost importance that the GPU is idle and
161 * all stray writes are flushed *before* we dismantle the backing
162 * storage for the pinned objects.
163 *
164 * However, since we are uncertain that resetting the GPU on older
165 * machines is a good idea, we don't - just in case it leaves the
166 * machine in an unusable condition.
167 */
168
169 /* Like i915_gem_suspend, flush tasks staged from fence triggers */
170 rcu_barrier();
171
172 for_each_gt(gt, i915, i)
173 intel_gt_suspend_late(gt);
174
175 spin_lock_irqsave(&i915->mm.obj_lock, flags);
176 for (phase = phases; *phase; phase++) {
177 list_for_each_entry(obj, *phase, mm.link) {
178 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
179 flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
180 __start_cpu_write(obj); /* presume auto-hibernate */
181 }
182 }
183 spin_unlock_irqrestore(lock: &i915->mm.obj_lock, flags);
184 if (flush)
185 wbinvd_on_all_cpus();
186}
187
188int i915_gem_freeze(struct drm_i915_private *i915)
189{
190 /* Discard all purgeable objects, let userspace recover those as
191 * required after resuming.
192 */
193 i915_gem_shrink_all(i915);
194
195 return 0;
196}
197
198int i915_gem_freeze_late(struct drm_i915_private *i915)
199{
200 struct drm_i915_gem_object *obj;
201 intel_wakeref_t wakeref;
202
203 /*
204 * Called just before we write the hibernation image.
205 *
206 * We need to update the domain tracking to reflect that the CPU
207 * will be accessing all the pages to create and restore from the
208 * hibernation, and so upon restoration those pages will be in the
209 * CPU domain.
210 *
211 * To make sure the hibernation image contains the latest state,
212 * we update that state just before writing out the image.
213 *
214 * To try and reduce the hibernation image, we manually shrink
215 * the objects as well, see i915_gem_freeze()
216 */
217
218 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
219 i915_gem_shrink(NULL, i915, target: -1UL, NULL, flags: ~0);
220 i915_gem_drain_freed_objects(i915);
221
222 wbinvd_on_all_cpus();
223 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
224 __start_cpu_write(obj);
225
226 return 0;
227}
228
229void i915_gem_resume(struct drm_i915_private *i915)
230{
231 struct intel_gt *gt;
232 int ret, i, j;
233
234 GEM_TRACE("%s\n", dev_name(i915->drm.dev));
235
236 ret = lmem_restore(i915, flags: 0);
237 GEM_WARN_ON(ret);
238
239 /*
240 * As we didn't flush the kernel context before suspend, we cannot
241 * guarantee that the context image is complete. So let's just reset
242 * it and start again.
243 */
244 for_each_gt(gt, i915, i)
245 if (intel_gt_resume(gt))
246 goto err_wedged;
247
248 ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
249 GEM_WARN_ON(ret);
250
251 return;
252
253err_wedged:
254 for_each_gt(gt, i915, j) {
255 if (!intel_gt_is_wedged(gt)) {
256 dev_err(i915->drm.dev,
257 "Failed to re-initialize GPU[%u], declaring it wedged!\n",
258 j);
259 intel_gt_set_wedged(gt);
260 }
261
262 if (j == i)
263 break;
264 }
265}
266

source code of linux/drivers/gpu/drm/i915/gem/i915_gem_pm.c