1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include "msm_gpu.h"
8#include "msm_gpu_trace.h"
9
10#include <linux/devfreq.h>
11#include <linux/devfreq_cooling.h>
12#include <linux/math64.h>
13#include <linux/units.h>
14
15/*
16 * Power Management:
17 */
18
19static int msm_devfreq_target(struct device *dev, unsigned long *freq,
20 u32 flags)
21{
22 struct msm_gpu *gpu = dev_to_gpu(dev);
23 struct msm_gpu_devfreq *df = &gpu->devfreq;
24 struct dev_pm_opp *opp;
25
26 /*
27 * Note that devfreq_recommended_opp() can modify the freq
28 * to something that actually is in the opp table:
29 */
30 opp = devfreq_recommended_opp(dev, freq, flags);
31 if (IS_ERR(ptr: opp))
32 return PTR_ERR(ptr: opp);
33
34 trace_msm_gpu_freq_change(freq: dev_pm_opp_get_freq(opp));
35
36 /*
37 * If the GPU is idle, devfreq is not aware, so just stash
38 * the new target freq (to use when we return to active)
39 */
40 if (df->idle_freq) {
41 df->idle_freq = *freq;
42 dev_pm_opp_put(opp);
43 return 0;
44 }
45
46 if (gpu->funcs->gpu_set_freq) {
47 mutex_lock(&df->lock);
48 gpu->funcs->gpu_set_freq(gpu, opp, df->suspended);
49 mutex_unlock(lock: &df->lock);
50 } else {
51 dev_pm_opp_set_rate(dev, target_freq: *freq);
52 }
53
54 dev_pm_opp_put(opp);
55
56 return 0;
57}
58
59static unsigned long get_freq(struct msm_gpu *gpu)
60{
61 struct msm_gpu_devfreq *df = &gpu->devfreq;
62
63 /*
64 * If the GPU is idle, use the shadow/saved freq to avoid
65 * confusing devfreq (which is unaware that we are switching
66 * to lowest freq until the device is active again)
67 */
68 if (df->idle_freq)
69 return df->idle_freq;
70
71 if (gpu->funcs->gpu_get_freq)
72 return gpu->funcs->gpu_get_freq(gpu);
73
74 return clk_get_rate(clk: gpu->core_clk);
75}
76
77static int msm_devfreq_get_dev_status(struct device *dev,
78 struct devfreq_dev_status *status)
79{
80 struct msm_gpu *gpu = dev_to_gpu(dev);
81 struct msm_gpu_devfreq *df = &gpu->devfreq;
82 u64 busy_cycles, busy_time;
83 unsigned long sample_rate;
84 ktime_t time;
85
86 mutex_lock(&df->lock);
87
88 status->current_frequency = get_freq(gpu);
89 time = ktime_get();
90 status->total_time = ktime_us_delta(later: time, earlier: df->time);
91 df->time = time;
92
93 if (df->suspended) {
94 mutex_unlock(lock: &df->lock);
95 status->busy_time = 0;
96 return 0;
97 }
98
99 busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate);
100 busy_time = busy_cycles - df->busy_cycles;
101 df->busy_cycles = busy_cycles;
102
103 mutex_unlock(lock: &df->lock);
104
105 busy_time *= USEC_PER_SEC;
106 busy_time = div64_ul(busy_time, sample_rate);
107 if (WARN_ON(busy_time > ~0LU))
108 busy_time = ~0LU;
109
110 status->busy_time = busy_time;
111
112 return 0;
113}
114
115static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
116{
117 *freq = get_freq(gpu: dev_to_gpu(dev));
118
119 return 0;
120}
121
122static struct devfreq_dev_profile msm_devfreq_profile = {
123 .timer = DEVFREQ_TIMER_DELAYED,
124 .polling_ms = 50,
125 .target = msm_devfreq_target,
126 .get_dev_status = msm_devfreq_get_dev_status,
127 .get_cur_freq = msm_devfreq_get_cur_freq,
128};
129
130static void msm_devfreq_boost_work(struct kthread_work *work);
131static void msm_devfreq_idle_work(struct kthread_work *work);
132
133static bool has_devfreq(struct msm_gpu *gpu)
134{
135 struct msm_gpu_devfreq *df = &gpu->devfreq;
136 return !!df->devfreq;
137}
138
139void msm_devfreq_init(struct msm_gpu *gpu)
140{
141 struct msm_gpu_devfreq *df = &gpu->devfreq;
142 struct msm_drm_private *priv = gpu->dev->dev_private;
143 int ret;
144
145 /* We need target support to do devfreq */
146 if (!gpu->funcs->gpu_busy)
147 return;
148
149 /*
150 * Setup default values for simple_ondemand governor tuning. We
151 * want to throttle up at 50% load for the double-buffer case,
152 * where due to stalling waiting for vblank we could get stuck
153 * at (for ex) 30fps at 50% utilization.
154 */
155 priv->gpu_devfreq_config.upthreshold = 50;
156 priv->gpu_devfreq_config.downdifferential = 10;
157
158 mutex_init(&df->lock);
159 df->suspended = true;
160
161 ret = dev_pm_qos_add_request(dev: &gpu->pdev->dev, req: &df->boost_freq,
162 type: DEV_PM_QOS_MIN_FREQUENCY, value: 0);
163 if (ret < 0) {
164 DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize QoS\n");
165 return;
166 }
167
168 msm_devfreq_profile.initial_freq = gpu->fast_rate;
169
170 /*
171 * Don't set the freq_table or max_state and let devfreq build the table
172 * from OPP
173 * After a deferred probe, these may have be left to non-zero values,
174 * so set them back to zero before creating the devfreq device
175 */
176 msm_devfreq_profile.freq_table = NULL;
177 msm_devfreq_profile.max_state = 0;
178
179 df->devfreq = devm_devfreq_add_device(dev: &gpu->pdev->dev,
180 profile: &msm_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND,
181 data: &priv->gpu_devfreq_config);
182
183 if (IS_ERR(ptr: df->devfreq)) {
184 DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n");
185 dev_pm_qos_remove_request(req: &df->boost_freq);
186 df->devfreq = NULL;
187 return;
188 }
189
190 devfreq_suspend_device(devfreq: df->devfreq);
191
192 gpu->cooling = of_devfreq_cooling_register(np: gpu->pdev->dev.of_node, df: df->devfreq);
193 if (IS_ERR(ptr: gpu->cooling)) {
194 DRM_DEV_ERROR(&gpu->pdev->dev,
195 "Couldn't register GPU cooling device\n");
196 gpu->cooling = NULL;
197 }
198
199 msm_hrtimer_work_init(work: &df->boost_work, worker: gpu->worker, fn: msm_devfreq_boost_work,
200 CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
201 msm_hrtimer_work_init(work: &df->idle_work, worker: gpu->worker, fn: msm_devfreq_idle_work,
202 CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
203}
204
205static void cancel_idle_work(struct msm_gpu_devfreq *df)
206{
207 hrtimer_cancel(timer: &df->idle_work.timer);
208 kthread_cancel_work_sync(work: &df->idle_work.work);
209}
210
211static void cancel_boost_work(struct msm_gpu_devfreq *df)
212{
213 hrtimer_cancel(timer: &df->boost_work.timer);
214 kthread_cancel_work_sync(work: &df->boost_work.work);
215}
216
217void msm_devfreq_cleanup(struct msm_gpu *gpu)
218{
219 struct msm_gpu_devfreq *df = &gpu->devfreq;
220
221 if (!has_devfreq(gpu))
222 return;
223
224 devfreq_cooling_unregister(dfc: gpu->cooling);
225 dev_pm_qos_remove_request(req: &df->boost_freq);
226}
227
228void msm_devfreq_resume(struct msm_gpu *gpu)
229{
230 struct msm_gpu_devfreq *df = &gpu->devfreq;
231 unsigned long sample_rate;
232
233 if (!has_devfreq(gpu))
234 return;
235
236 mutex_lock(&df->lock);
237 df->busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate);
238 df->time = ktime_get();
239 df->suspended = false;
240 mutex_unlock(lock: &df->lock);
241
242 devfreq_resume_device(devfreq: df->devfreq);
243}
244
245void msm_devfreq_suspend(struct msm_gpu *gpu)
246{
247 struct msm_gpu_devfreq *df = &gpu->devfreq;
248
249 if (!has_devfreq(gpu))
250 return;
251
252 mutex_lock(&df->lock);
253 df->suspended = true;
254 mutex_unlock(lock: &df->lock);
255
256 devfreq_suspend_device(devfreq: df->devfreq);
257
258 cancel_idle_work(df);
259 cancel_boost_work(df);
260}
261
262static void msm_devfreq_boost_work(struct kthread_work *work)
263{
264 struct msm_gpu_devfreq *df = container_of(work,
265 struct msm_gpu_devfreq, boost_work.work);
266
267 dev_pm_qos_update_request(req: &df->boost_freq, new_value: 0);
268}
269
270void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor)
271{
272 struct msm_gpu_devfreq *df = &gpu->devfreq;
273 uint64_t freq;
274
275 if (!has_devfreq(gpu))
276 return;
277
278 freq = get_freq(gpu);
279 freq *= factor;
280
281 /*
282 * A nice little trap is that PM QoS operates in terms of KHz,
283 * while devfreq operates in terms of Hz:
284 */
285 do_div(freq, HZ_PER_KHZ);
286
287 dev_pm_qos_update_request(req: &df->boost_freq, new_value: freq);
288
289 msm_hrtimer_queue_work(work: &df->boost_work,
290 wakeup_time: ms_to_ktime(ms: msm_devfreq_profile.polling_ms),
291 mode: HRTIMER_MODE_REL);
292}
293
294void msm_devfreq_active(struct msm_gpu *gpu)
295{
296 struct msm_gpu_devfreq *df = &gpu->devfreq;
297 unsigned int idle_time;
298 unsigned long target_freq;
299
300 if (!has_devfreq(gpu))
301 return;
302
303 /*
304 * Cancel any pending transition to idle frequency:
305 */
306 cancel_idle_work(df);
307
308 /*
309 * Hold devfreq lock to synchronize with get_dev_status()/
310 * target() callbacks
311 */
312 mutex_lock(&df->devfreq->lock);
313
314 target_freq = df->idle_freq;
315
316 idle_time = ktime_to_ms(ktime_sub(ktime_get(), df->idle_time));
317
318 df->idle_freq = 0;
319
320 /*
321 * We could have become active again before the idle work had a
322 * chance to run, in which case the df->idle_freq would have
323 * still been zero. In this case, no need to change freq.
324 */
325 if (target_freq)
326 msm_devfreq_target(dev: &gpu->pdev->dev, freq: &target_freq, flags: 0);
327
328 mutex_unlock(lock: &df->devfreq->lock);
329
330 /*
331 * If we've been idle for a significant fraction of a polling
332 * interval, then we won't meet the threshold of busyness for
333 * the governor to ramp up the freq.. so give some boost
334 */
335 if (idle_time > msm_devfreq_profile.polling_ms) {
336 msm_devfreq_boost(gpu, factor: 2);
337 }
338}
339
340
341static void msm_devfreq_idle_work(struct kthread_work *work)
342{
343 struct msm_gpu_devfreq *df = container_of(work,
344 struct msm_gpu_devfreq, idle_work.work);
345 struct msm_gpu *gpu = container_of(df, struct msm_gpu, devfreq);
346 struct msm_drm_private *priv = gpu->dev->dev_private;
347 unsigned long idle_freq, target_freq = 0;
348
349 /*
350 * Hold devfreq lock to synchronize with get_dev_status()/
351 * target() callbacks
352 */
353 mutex_lock(&df->devfreq->lock);
354
355 idle_freq = get_freq(gpu);
356
357 if (priv->gpu_clamp_to_idle)
358 msm_devfreq_target(dev: &gpu->pdev->dev, freq: &target_freq, flags: 0);
359
360 df->idle_time = ktime_get();
361 df->idle_freq = idle_freq;
362
363 mutex_unlock(lock: &df->devfreq->lock);
364}
365
366void msm_devfreq_idle(struct msm_gpu *gpu)
367{
368 struct msm_gpu_devfreq *df = &gpu->devfreq;
369
370 if (!has_devfreq(gpu))
371 return;
372
373 msm_hrtimer_queue_work(work: &df->idle_work, wakeup_time: ms_to_ktime(ms: 1),
374 mode: HRTIMER_MODE_REL);
375}
376

source code of linux/drivers/gpu/drm/msm/msm_gpu_devfreq.c