1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * x86 APERF/MPERF KHz calculation for
4 * /sys/.../cpufreq/scaling_cur_freq
5 *
6 * Copyright (C) 2017 Intel Corp.
7 * Author: Len Brown <len.brown@intel.com>
8 */
9#include <linux/cpufreq.h>
10#include <linux/delay.h>
11#include <linux/ktime.h>
12#include <linux/math64.h>
13#include <linux/percpu.h>
14#include <linux/rcupdate.h>
15#include <linux/sched/isolation.h>
16#include <linux/sched/topology.h>
17#include <linux/smp.h>
18#include <linux/syscore_ops.h>
19
20#include <asm/cpu.h>
21#include <asm/cpu_device_id.h>
22#include <asm/intel-family.h>
23#include <asm/msr.h>
24
25#include "cpu.h"
26
27struct aperfmperf {
28 seqcount_t seq;
29 unsigned long last_update;
30 u64 acnt;
31 u64 mcnt;
32 u64 aperf;
33 u64 mperf;
34};
35
36static DEFINE_PER_CPU_SHARED_ALIGNED(struct aperfmperf, cpu_samples) = {
37 .seq = SEQCNT_ZERO(cpu_samples.seq)
38};
39
40static void init_counter_refs(void *data)
41{
42 u64 aperf, mperf;
43
44 rdmsrq(MSR_IA32_APERF, aperf);
45 rdmsrq(MSR_IA32_MPERF, mperf);
46
47 this_cpu_write(cpu_samples.aperf, aperf);
48 this_cpu_write(cpu_samples.mperf, mperf);
49}
50
51#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
52/*
53 * APERF/MPERF frequency ratio computation.
54 *
55 * The scheduler wants to do frequency invariant accounting and needs a <1
56 * ratio to account for the 'current' frequency, corresponding to
57 * freq_curr / freq_max.
58 *
59 * Since the frequency freq_curr on x86 is controlled by micro-controller and
60 * our P-state setting is little more than a request/hint, we need to observe
61 * the effective frequency 'BusyMHz', i.e. the average frequency over a time
62 * interval after discarding idle time. This is given by:
63 *
64 * BusyMHz = delta_APERF / delta_MPERF * freq_base
65 *
66 * where freq_base is the max non-turbo P-state.
67 *
68 * The freq_max term has to be set to a somewhat arbitrary value, because we
69 * can't know which turbo states will be available at a given point in time:
70 * it all depends on the thermal headroom of the entire package. We set it to
71 * the turbo level with 4 cores active.
72 *
73 * Benchmarks show that's a good compromise between the 1C turbo ratio
74 * (freq_curr/freq_max would rarely reach 1) and something close to freq_base,
75 * which would ignore the entire turbo range (a conspicuous part, making
76 * freq_curr/freq_max always maxed out).
77 *
78 * An exception to the heuristic above is the Atom uarch, where we choose the
79 * highest turbo level for freq_max since Atom's are generally oriented towards
80 * power efficiency.
81 *
82 * Setting freq_max to anything less than the 1C turbo ratio makes the ratio
83 * freq_curr / freq_max to eventually grow >1, in which case we clip it to 1.
84 */
85
86DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key);
87
88static u64 arch_turbo_freq_ratio = SCHED_CAPACITY_SCALE;
89static u64 arch_max_freq_ratio = SCHED_CAPACITY_SCALE;
90
91void arch_set_max_freq_ratio(bool turbo_disabled)
92{
93 arch_max_freq_ratio = turbo_disabled ? SCHED_CAPACITY_SCALE :
94 arch_turbo_freq_ratio;
95}
96EXPORT_SYMBOL_GPL(arch_set_max_freq_ratio);
97
98static bool __init turbo_disabled(void)
99{
100 u64 misc_en;
101 int err;
102
103 err = rdmsrq_safe(MSR_IA32_MISC_ENABLE, p: &misc_en);
104 if (err)
105 return false;
106
107 return (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
108}
109
110static bool __init slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
111{
112 int err;
113
114 err = rdmsrq_safe(MSR_ATOM_CORE_RATIOS, p: base_freq);
115 if (err)
116 return false;
117
118 err = rdmsrq_safe(MSR_ATOM_CORE_TURBO_RATIOS, p: turbo_freq);
119 if (err)
120 return false;
121
122 *base_freq = (*base_freq >> 16) & 0x3F; /* max P state */
123 *turbo_freq = *turbo_freq & 0x3F; /* 1C turbo */
124
125 return true;
126}
127
128#define X86_MATCH(vfm) \
129 X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, NULL)
130
131static const struct x86_cpu_id has_knl_turbo_ratio_limits[] __initconst = {
132 X86_MATCH(INTEL_XEON_PHI_KNL),
133 X86_MATCH(INTEL_XEON_PHI_KNM),
134 {}
135};
136
137static const struct x86_cpu_id has_skx_turbo_ratio_limits[] __initconst = {
138 X86_MATCH(INTEL_SKYLAKE_X),
139 {}
140};
141
142static const struct x86_cpu_id has_glm_turbo_ratio_limits[] __initconst = {
143 X86_MATCH(INTEL_ATOM_GOLDMONT),
144 X86_MATCH(INTEL_ATOM_GOLDMONT_D),
145 X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS),
146 {}
147};
148
149static bool __init knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq,
150 int num_delta_fratio)
151{
152 int fratio, delta_fratio, found;
153 int err, i;
154 u64 msr;
155
156 err = rdmsrq_safe(MSR_PLATFORM_INFO, p: base_freq);
157 if (err)
158 return false;
159
160 *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */
161
162 err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT, p: &msr);
163 if (err)
164 return false;
165
166 fratio = (msr >> 8) & 0xFF;
167 i = 16;
168 found = 0;
169 do {
170 if (found >= num_delta_fratio) {
171 *turbo_freq = fratio;
172 return true;
173 }
174
175 delta_fratio = (msr >> (i + 5)) & 0x7;
176
177 if (delta_fratio) {
178 found += 1;
179 fratio -= delta_fratio;
180 }
181
182 i += 8;
183 } while (i < 64);
184
185 return true;
186}
187
188static bool __init skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int size)
189{
190 u64 ratios, counts;
191 u32 group_size;
192 int err, i;
193
194 err = rdmsrq_safe(MSR_PLATFORM_INFO, p: base_freq);
195 if (err)
196 return false;
197
198 *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */
199
200 err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT, p: &ratios);
201 if (err)
202 return false;
203
204 err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT1, p: &counts);
205 if (err)
206 return false;
207
208 for (i = 0; i < 64; i += 8) {
209 group_size = (counts >> i) & 0xFF;
210 if (group_size >= size) {
211 *turbo_freq = (ratios >> i) & 0xFF;
212 return true;
213 }
214 }
215
216 return false;
217}
218
219static bool __init core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
220{
221 u64 msr;
222 int err;
223
224 err = rdmsrq_safe(MSR_PLATFORM_INFO, p: base_freq);
225 if (err)
226 return false;
227
228 err = rdmsrq_safe(MSR_TURBO_RATIO_LIMIT, p: &msr);
229 if (err)
230 return false;
231
232 *base_freq = (*base_freq >> 8) & 0xFF; /* max P state */
233 *turbo_freq = (msr >> 24) & 0xFF; /* 4C turbo */
234
235 /* The CPU may have less than 4 cores */
236 if (!*turbo_freq)
237 *turbo_freq = msr & 0xFF; /* 1C turbo */
238
239 return true;
240}
241
242static bool __init intel_set_max_freq_ratio(void)
243{
244 u64 base_freq, turbo_freq;
245 u64 turbo_ratio;
246
247 if (slv_set_max_freq_ratio(base_freq: &base_freq, turbo_freq: &turbo_freq))
248 goto out;
249
250 if (x86_match_cpu(match: has_glm_turbo_ratio_limits) &&
251 skx_set_max_freq_ratio(base_freq: &base_freq, turbo_freq: &turbo_freq, size: 1))
252 goto out;
253
254 if (x86_match_cpu(match: has_knl_turbo_ratio_limits) &&
255 knl_set_max_freq_ratio(base_freq: &base_freq, turbo_freq: &turbo_freq, num_delta_fratio: 1))
256 goto out;
257
258 if (x86_match_cpu(match: has_skx_turbo_ratio_limits) &&
259 skx_set_max_freq_ratio(base_freq: &base_freq, turbo_freq: &turbo_freq, size: 4))
260 goto out;
261
262 if (core_set_max_freq_ratio(base_freq: &base_freq, turbo_freq: &turbo_freq))
263 goto out;
264
265 return false;
266
267out:
268 /*
269 * Some hypervisors advertise X86_FEATURE_APERFMPERF
270 * but then fill all MSR's with zeroes.
271 * Some CPUs have turbo boost but don't declare any turbo ratio
272 * in MSR_TURBO_RATIO_LIMIT.
273 */
274 if (!base_freq || !turbo_freq) {
275 pr_debug("Couldn't determine cpu base or turbo frequency, necessary for scale-invariant accounting.\n");
276 return false;
277 }
278
279 turbo_ratio = div_u64(dividend: turbo_freq * SCHED_CAPACITY_SCALE, divisor: base_freq);
280 if (!turbo_ratio) {
281 pr_debug("Non-zero turbo and base frequencies led to a 0 ratio.\n");
282 return false;
283 }
284
285 arch_turbo_freq_ratio = turbo_ratio;
286 arch_set_max_freq_ratio(turbo_disabled());
287
288 return true;
289}
290
291#ifdef CONFIG_PM_SLEEP
292static const struct syscore_ops freq_invariance_syscore_ops = {
293 .resume = init_counter_refs,
294};
295
296static struct syscore freq_invariance_syscore = {
297 .ops = &freq_invariance_syscore_ops,
298};
299
300static void register_freq_invariance_syscore(void)
301{
302 register_syscore(syscore: &freq_invariance_syscore);
303}
304#else
305static inline void register_freq_invariance_syscore(void) {}
306#endif
307
308static void freq_invariance_enable(void)
309{
310 if (static_branch_unlikely(&arch_scale_freq_key)) {
311 WARN_ON_ONCE(1);
312 return;
313 }
314 static_branch_enable_cpuslocked(&arch_scale_freq_key);
315 register_freq_invariance_syscore();
316 pr_info("Estimated ratio of average max frequency by base frequency (times 1024): %llu\n", arch_max_freq_ratio);
317}
318
319void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled)
320{
321 arch_turbo_freq_ratio = ratio;
322 arch_set_max_freq_ratio(turbo_disabled);
323 freq_invariance_enable();
324}
325
326static void __init bp_init_freq_invariance(void)
327{
328 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
329 return;
330
331 if (intel_set_max_freq_ratio()) {
332 guard(cpus_read_lock)();
333 freq_invariance_enable();
334 }
335}
336
337static void disable_freq_invariance_workfn(struct work_struct *work)
338{
339 int cpu;
340
341 static_branch_disable(&arch_scale_freq_key);
342
343 /*
344 * Set arch_freq_scale to a default value on all cpus
345 * This negates the effect of scaling
346 */
347 for_each_possible_cpu(cpu)
348 per_cpu(arch_freq_scale, cpu) = SCHED_CAPACITY_SCALE;
349}
350
351static DECLARE_WORK(disable_freq_invariance_work,
352 disable_freq_invariance_workfn);
353
354DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE;
355EXPORT_PER_CPU_SYMBOL_GPL(arch_freq_scale);
356
357static DEFINE_STATIC_KEY_FALSE(arch_hybrid_cap_scale_key);
358
359struct arch_hybrid_cpu_scale {
360 unsigned long capacity;
361 unsigned long freq_ratio;
362};
363
364static struct arch_hybrid_cpu_scale __percpu *arch_cpu_scale;
365
366/**
367 * arch_enable_hybrid_capacity_scale() - Enable hybrid CPU capacity scaling
368 *
369 * Allocate memory for per-CPU data used by hybrid CPU capacity scaling,
370 * initialize it and set the static key controlling its code paths.
371 *
372 * Must be called before arch_set_cpu_capacity().
373 */
374bool arch_enable_hybrid_capacity_scale(void)
375{
376 int cpu;
377
378 if (static_branch_unlikely(&arch_hybrid_cap_scale_key)) {
379 WARN_ONCE(1, "Hybrid CPU capacity scaling already enabled");
380 return true;
381 }
382
383 arch_cpu_scale = alloc_percpu(struct arch_hybrid_cpu_scale);
384 if (!arch_cpu_scale)
385 return false;
386
387 for_each_possible_cpu(cpu) {
388 per_cpu_ptr(arch_cpu_scale, cpu)->capacity = SCHED_CAPACITY_SCALE;
389 per_cpu_ptr(arch_cpu_scale, cpu)->freq_ratio = arch_max_freq_ratio;
390 }
391
392 static_branch_enable(&arch_hybrid_cap_scale_key);
393
394 pr_info("Hybrid CPU capacity scaling enabled\n");
395
396 return true;
397}
398
399/**
400 * arch_set_cpu_capacity() - Set scale-invariance parameters for a CPU
401 * @cpu: Target CPU.
402 * @cap: Capacity of @cpu at its maximum frequency, relative to @max_cap.
403 * @max_cap: System-wide maximum CPU capacity.
404 * @cap_freq: Frequency of @cpu corresponding to @cap.
405 * @base_freq: Frequency of @cpu at which MPERF counts.
406 *
407 * The units in which @cap and @max_cap are expressed do not matter, so long
408 * as they are consistent, because the former is effectively divided by the
409 * latter. Analogously for @cap_freq and @base_freq.
410 *
411 * After calling this function for all CPUs, call arch_rebuild_sched_domains()
412 * to let the scheduler know that capacity-aware scheduling can be used going
413 * forward.
414 */
415void arch_set_cpu_capacity(int cpu, unsigned long cap, unsigned long max_cap,
416 unsigned long cap_freq, unsigned long base_freq)
417{
418 if (static_branch_likely(&arch_hybrid_cap_scale_key)) {
419 WRITE_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->capacity,
420 div_u64(cap << SCHED_CAPACITY_SHIFT, max_cap));
421 WRITE_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->freq_ratio,
422 div_u64(cap_freq << SCHED_CAPACITY_SHIFT, base_freq));
423 } else {
424 WARN_ONCE(1, "Hybrid CPU capacity scaling not enabled");
425 }
426}
427
428unsigned long arch_scale_cpu_capacity(int cpu)
429{
430 if (static_branch_unlikely(&arch_hybrid_cap_scale_key))
431 return READ_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->capacity);
432
433 return SCHED_CAPACITY_SCALE;
434}
435EXPORT_SYMBOL_GPL(arch_scale_cpu_capacity);
436
437static void scale_freq_tick(u64 acnt, u64 mcnt)
438{
439 u64 freq_scale, freq_ratio;
440
441 if (!arch_scale_freq_invariant())
442 return;
443
444 if (check_shl_overflow(acnt, 2*SCHED_CAPACITY_SHIFT, &acnt))
445 goto error;
446
447 if (static_branch_unlikely(&arch_hybrid_cap_scale_key))
448 freq_ratio = READ_ONCE(this_cpu_ptr(arch_cpu_scale)->freq_ratio);
449 else
450 freq_ratio = arch_max_freq_ratio;
451
452 if (check_mul_overflow(mcnt, freq_ratio, &mcnt) || !mcnt)
453 goto error;
454
455 freq_scale = div64_u64(dividend: acnt, divisor: mcnt);
456 if (!freq_scale)
457 goto error;
458
459 if (freq_scale > SCHED_CAPACITY_SCALE)
460 freq_scale = SCHED_CAPACITY_SCALE;
461
462 this_cpu_write(arch_freq_scale, freq_scale);
463 return;
464
465error:
466 pr_warn("Scheduler frequency invariance went wobbly, disabling!\n");
467 schedule_work(work: &disable_freq_invariance_work);
468}
469#else
470static inline void bp_init_freq_invariance(void) { }
471static inline void scale_freq_tick(u64 acnt, u64 mcnt) { }
472#endif /* CONFIG_X86_64 && CONFIG_SMP */
473
474void arch_scale_freq_tick(void)
475{
476 struct aperfmperf *s = this_cpu_ptr(&cpu_samples);
477 u64 acnt, mcnt, aperf, mperf;
478
479 if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
480 return;
481
482 rdmsrq(MSR_IA32_APERF, aperf);
483 rdmsrq(MSR_IA32_MPERF, mperf);
484 acnt = aperf - s->aperf;
485 mcnt = mperf - s->mperf;
486
487 s->aperf = aperf;
488 s->mperf = mperf;
489
490 raw_write_seqcount_begin(&s->seq);
491 s->last_update = jiffies;
492 s->acnt = acnt;
493 s->mcnt = mcnt;
494 raw_write_seqcount_end(&s->seq);
495
496 scale_freq_tick(acnt, mcnt);
497}
498
499/*
500 * Discard samples older than the define maximum sample age of 20ms. There
501 * is no point in sending IPIs in such a case. If the scheduler tick was
502 * not running then the CPU is either idle or isolated.
503 */
504#define MAX_SAMPLE_AGE ((unsigned long)HZ / 50)
505
506int arch_freq_get_on_cpu(int cpu)
507{
508 struct aperfmperf *s = per_cpu_ptr(&cpu_samples, cpu);
509 unsigned int seq, freq;
510 unsigned long last;
511 u64 acnt, mcnt;
512
513 if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
514 goto fallback;
515
516 do {
517 seq = raw_read_seqcount_begin(&s->seq);
518 last = s->last_update;
519 acnt = s->acnt;
520 mcnt = s->mcnt;
521 } while (read_seqcount_retry(&s->seq, seq));
522
523 /*
524 * Bail on invalid count and when the last update was too long ago,
525 * which covers idle and NOHZ full CPUs.
526 */
527 if (!mcnt || (jiffies - last) > MAX_SAMPLE_AGE)
528 goto fallback;
529
530 return div64_u64(dividend: (cpu_khz * acnt), divisor: mcnt);
531
532fallback:
533 freq = cpufreq_quick_get(cpu);
534 return freq ? freq : cpu_khz;
535}
536
537static int __init bp_init_aperfmperf(void)
538{
539 if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
540 return 0;
541
542 init_counter_refs(NULL);
543 bp_init_freq_invariance();
544 return 0;
545}
546early_initcall(bp_init_aperfmperf);
547
548void ap_init_aperfmperf(void)
549{
550 if (cpu_feature_enabled(X86_FEATURE_APERFMPERF))
551 init_counter_refs(NULL);
552}
553

source code of linux/arch/x86/kernel/cpu/aperfmperf.c