1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Common code for Intel Running Average Power Limit (RAPL) support.
4 * Copyright (c) 2019, Intel Corporation.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/bitmap.h>
9#include <linux/cleanup.h>
10#include <linux/cpu.h>
11#include <linux/delay.h>
12#include <linux/device.h>
13#include <linux/intel_rapl.h>
14#include <linux/kernel.h>
15#include <linux/list.h>
16#include <linux/log2.h>
17#include <linux/module.h>
18#include <linux/nospec.h>
19#include <linux/perf_event.h>
20#include <linux/platform_device.h>
21#include <linux/powercap.h>
22#include <linux/processor.h>
23#include <linux/slab.h>
24#include <linux/suspend.h>
25#include <linux/sysfs.h>
26#include <linux/types.h>
27
28#include <asm/cpu_device_id.h>
29#include <asm/intel-family.h>
30#include <asm/iosf_mbi.h>
31#include <asm/msr.h>
32
33/* bitmasks for RAPL MSRs, used by primitive access functions */
34#define ENERGY_STATUS_MASK 0xffffffff
35
36#define POWER_LIMIT1_MASK 0x7FFF
37#define POWER_LIMIT1_ENABLE BIT(15)
38#define POWER_LIMIT1_CLAMP BIT(16)
39
40#define POWER_LIMIT2_MASK (0x7FFFULL<<32)
41#define POWER_LIMIT2_ENABLE BIT_ULL(47)
42#define POWER_LIMIT2_CLAMP BIT_ULL(48)
43#define POWER_HIGH_LOCK BIT_ULL(63)
44#define POWER_LOW_LOCK BIT(31)
45
46#define POWER_LIMIT4_MASK 0x1FFF
47
48#define TIME_WINDOW1_MASK (0x7FULL<<17)
49#define TIME_WINDOW2_MASK (0x7FULL<<49)
50
51#define POWER_UNIT_OFFSET 0
52#define POWER_UNIT_MASK 0x0F
53
54#define ENERGY_UNIT_OFFSET 0x08
55#define ENERGY_UNIT_MASK 0x1F00
56
57#define TIME_UNIT_OFFSET 0x10
58#define TIME_UNIT_MASK 0xF0000
59
60#define POWER_INFO_MAX_MASK (0x7fffULL<<32)
61#define POWER_INFO_MIN_MASK (0x7fffULL<<16)
62#define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)
63#define POWER_INFO_THERMAL_SPEC_MASK 0x7fff
64
65#define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
66#define PP_POLICY_MASK 0x1F
67
68/*
69 * SPR has different layout for Psys Domain PowerLimit registers.
70 * There are 17 bits of PL1 and PL2 instead of 15 bits.
71 * The Enable bits and TimeWindow bits are also shifted as a result.
72 */
73#define PSYS_POWER_LIMIT1_MASK 0x1FFFF
74#define PSYS_POWER_LIMIT1_ENABLE BIT(17)
75
76#define PSYS_POWER_LIMIT2_MASK (0x1FFFFULL<<32)
77#define PSYS_POWER_LIMIT2_ENABLE BIT_ULL(49)
78
79#define PSYS_TIME_WINDOW1_MASK (0x7FULL<<19)
80#define PSYS_TIME_WINDOW2_MASK (0x7FULL<<51)
81
82/* bitmasks for RAPL TPMI, used by primitive access functions */
83#define TPMI_POWER_LIMIT_MASK 0x3FFFF
84#define TPMI_POWER_LIMIT_ENABLE BIT_ULL(62)
85#define TPMI_TIME_WINDOW_MASK (0x7FULL<<18)
86#define TPMI_INFO_SPEC_MASK 0x3FFFF
87#define TPMI_INFO_MIN_MASK (0x3FFFFULL << 18)
88#define TPMI_INFO_MAX_MASK (0x3FFFFULL << 36)
89#define TPMI_INFO_MAX_TIME_WIN_MASK (0x7FULL << 54)
90
91/* Non HW constants */
92#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
93#define RAPL_PRIMITIVE_DUMMY BIT(2)
94
95#define TIME_WINDOW_MAX_MSEC 40000
96#define TIME_WINDOW_MIN_MSEC 250
97#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
98enum unit_type {
99 ARBITRARY_UNIT, /* no translation */
100 POWER_UNIT,
101 ENERGY_UNIT,
102 TIME_UNIT,
103};
104
105/* per domain data, some are optional */
106#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
107
108#define DOMAIN_STATE_INACTIVE BIT(0)
109#define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)
110
111static const char *pl_names[NR_POWER_LIMITS] = {
112 [POWER_LIMIT1] = "long_term",
113 [POWER_LIMIT2] = "short_term",
114 [POWER_LIMIT4] = "peak_power",
115};
116
117enum pl_prims {
118 PL_ENABLE,
119 PL_CLAMP,
120 PL_LIMIT,
121 PL_TIME_WINDOW,
122 PL_MAX_POWER,
123 PL_LOCK,
124};
125
126static bool is_pl_valid(struct rapl_domain *rd, int pl)
127{
128 if (pl < POWER_LIMIT1 || pl > POWER_LIMIT4)
129 return false;
130 return rd->rpl[pl].name ? true : false;
131}
132
133static int get_pl_lock_prim(struct rapl_domain *rd, int pl)
134{
135 if (rd->rp->priv->type == RAPL_IF_TPMI) {
136 if (pl == POWER_LIMIT1)
137 return PL1_LOCK;
138 if (pl == POWER_LIMIT2)
139 return PL2_LOCK;
140 if (pl == POWER_LIMIT4)
141 return PL4_LOCK;
142 }
143
144 /* MSR/MMIO Interface doesn't have Lock bit for PL4 */
145 if (pl == POWER_LIMIT4)
146 return -EINVAL;
147
148 /*
149 * Power Limit register that supports two power limits has a different
150 * bit position for the Lock bit.
151 */
152 if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))
153 return FW_HIGH_LOCK;
154 return FW_LOCK;
155}
156
157static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim)
158{
159 switch (pl) {
160 case POWER_LIMIT1:
161 if (prim == PL_ENABLE)
162 return PL1_ENABLE;
163 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)
164 return PL1_CLAMP;
165 if (prim == PL_LIMIT)
166 return POWER_LIMIT1;
167 if (prim == PL_TIME_WINDOW)
168 return TIME_WINDOW1;
169 if (prim == PL_MAX_POWER)
170 return THERMAL_SPEC_POWER;
171 if (prim == PL_LOCK)
172 return get_pl_lock_prim(rd, pl);
173 return -EINVAL;
174 case POWER_LIMIT2:
175 if (prim == PL_ENABLE)
176 return PL2_ENABLE;
177 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)
178 return PL2_CLAMP;
179 if (prim == PL_LIMIT)
180 return POWER_LIMIT2;
181 if (prim == PL_TIME_WINDOW)
182 return TIME_WINDOW2;
183 if (prim == PL_MAX_POWER)
184 return MAX_POWER;
185 if (prim == PL_LOCK)
186 return get_pl_lock_prim(rd, pl);
187 return -EINVAL;
188 case POWER_LIMIT4:
189 if (prim == PL_LIMIT)
190 return POWER_LIMIT4;
191 if (prim == PL_ENABLE)
192 return PL4_ENABLE;
193 /* PL4 would be around two times PL2, use same prim as PL2. */
194 if (prim == PL_MAX_POWER)
195 return MAX_POWER;
196 if (prim == PL_LOCK)
197 return get_pl_lock_prim(rd, pl);
198 return -EINVAL;
199 default:
200 return -EINVAL;
201 }
202}
203
204#define power_zone_to_rapl_domain(_zone) \
205 container_of(_zone, struct rapl_domain, power_zone)
206
207struct rapl_defaults {
208 u8 floor_freq_reg_addr;
209 int (*check_unit)(struct rapl_domain *rd);
210 void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
211 u64 (*compute_time_window)(struct rapl_domain *rd, u64 val,
212 bool to_raw);
213 unsigned int dram_domain_energy_unit;
214 unsigned int psys_domain_energy_unit;
215 bool spr_psys_bits;
216};
217static struct rapl_defaults *defaults_msr;
218static const struct rapl_defaults defaults_tpmi;
219
220static struct rapl_defaults *get_defaults(struct rapl_package *rp)
221{
222 return rp->priv->defaults;
223}
224
225/* Sideband MBI registers */
226#define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2)
227#define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf)
228
229#define PACKAGE_PLN_INT_SAVED BIT(0)
230#define MAX_PRIM_NAME (32)
231
232/* per domain data. used to describe individual knobs such that access function
233 * can be consolidated into one instead of many inline functions.
234 */
235struct rapl_primitive_info {
236 const char *name;
237 u64 mask;
238 int shift;
239 enum rapl_domain_reg_id id;
240 enum unit_type unit;
241 u32 flag;
242};
243
244#define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \
245 .name = #p, \
246 .mask = m, \
247 .shift = s, \
248 .id = i, \
249 .unit = u, \
250 .flag = f \
251 }
252
253static void rapl_init_domains(struct rapl_package *rp);
254static int rapl_read_data_raw(struct rapl_domain *rd,
255 enum rapl_primitives prim,
256 bool xlate, u64 *data,
257 bool atomic);
258static int rapl_write_data_raw(struct rapl_domain *rd,
259 enum rapl_primitives prim,
260 unsigned long long value);
261static int rapl_read_pl_data(struct rapl_domain *rd, int pl,
262 enum pl_prims pl_prim,
263 bool xlate, u64 *data);
264static int rapl_write_pl_data(struct rapl_domain *rd, int pl,
265 enum pl_prims pl_prim,
266 unsigned long long value);
267static u64 rapl_unit_xlate(struct rapl_domain *rd,
268 enum unit_type type, u64 value, int to_raw);
269static void package_power_limit_irq_save(struct rapl_package *rp);
270
271static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
272
273static const char *const rapl_domain_names[] = {
274 "package",
275 "core",
276 "uncore",
277 "dram",
278 "psys",
279};
280
281static int get_energy_counter(struct powercap_zone *power_zone,
282 u64 *energy_raw)
283{
284 struct rapl_domain *rd;
285 u64 energy_now;
286
287 /* prevent CPU hotplug, make sure the RAPL domain does not go
288 * away while reading the counter.
289 */
290 cpus_read_lock();
291 rd = power_zone_to_rapl_domain(power_zone);
292
293 if (!rapl_read_data_raw(rd, prim: ENERGY_COUNTER, xlate: true, data: &energy_now, atomic: false)) {
294 *energy_raw = energy_now;
295 cpus_read_unlock();
296
297 return 0;
298 }
299 cpus_read_unlock();
300
301 return -EIO;
302}
303
304static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)
305{
306 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);
307
308 *energy = rapl_unit_xlate(rd, type: ENERGY_UNIT, ENERGY_STATUS_MASK, to_raw: 0);
309 return 0;
310}
311
312static int release_zone(struct powercap_zone *power_zone)
313{
314 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
315 struct rapl_package *rp = rd->rp;
316
317 /* package zone is the last zone of a package, we can free
318 * memory here since all children has been unregistered.
319 */
320 if (rd->id == RAPL_DOMAIN_PACKAGE) {
321 kfree(objp: rd);
322 rp->domains = NULL;
323 }
324
325 return 0;
326
327}
328
329static int find_nr_power_limit(struct rapl_domain *rd)
330{
331 int i, nr_pl = 0;
332
333 for (i = 0; i < NR_POWER_LIMITS; i++) {
334 if (is_pl_valid(rd, pl: i))
335 nr_pl++;
336 }
337
338 return nr_pl;
339}
340
341static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
342{
343 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
344 struct rapl_defaults *defaults = get_defaults(rp: rd->rp);
345 u64 val;
346 int ret;
347
348 cpus_read_lock();
349 ret = rapl_write_pl_data(rd, pl: POWER_LIMIT1, pl_prim: PL_ENABLE, value: mode);
350 if (ret)
351 goto end;
352
353 ret = rapl_read_pl_data(rd, pl: POWER_LIMIT1, pl_prim: PL_ENABLE, xlate: false, data: &val);
354 if (ret)
355 goto end;
356
357 if (mode != val) {
358 pr_debug("%s cannot be %s\n", power_zone->name,
359 str_enabled_disabled(mode));
360 goto end;
361 }
362
363 if (defaults->set_floor_freq)
364 defaults->set_floor_freq(rd, mode);
365
366end:
367 cpus_read_unlock();
368
369 return ret;
370}
371
372static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)
373{
374 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
375 u64 val;
376 int ret;
377
378 if (rd->rpl[POWER_LIMIT1].locked) {
379 *mode = false;
380 return 0;
381 }
382 cpus_read_lock();
383 ret = rapl_read_pl_data(rd, pl: POWER_LIMIT1, pl_prim: PL_ENABLE, xlate: true, data: &val);
384 if (!ret)
385 *mode = val;
386 cpus_read_unlock();
387
388 return ret;
389}
390
391/* per RAPL domain ops, in the order of rapl_domain_type */
392static const struct powercap_zone_ops zone_ops[] = {
393 /* RAPL_DOMAIN_PACKAGE */
394 {
395 .get_energy_uj = get_energy_counter,
396 .get_max_energy_range_uj = get_max_energy_counter,
397 .release = release_zone,
398 .set_enable = set_domain_enable,
399 .get_enable = get_domain_enable,
400 },
401 /* RAPL_DOMAIN_PP0 */
402 {
403 .get_energy_uj = get_energy_counter,
404 .get_max_energy_range_uj = get_max_energy_counter,
405 .release = release_zone,
406 .set_enable = set_domain_enable,
407 .get_enable = get_domain_enable,
408 },
409 /* RAPL_DOMAIN_PP1 */
410 {
411 .get_energy_uj = get_energy_counter,
412 .get_max_energy_range_uj = get_max_energy_counter,
413 .release = release_zone,
414 .set_enable = set_domain_enable,
415 .get_enable = get_domain_enable,
416 },
417 /* RAPL_DOMAIN_DRAM */
418 {
419 .get_energy_uj = get_energy_counter,
420 .get_max_energy_range_uj = get_max_energy_counter,
421 .release = release_zone,
422 .set_enable = set_domain_enable,
423 .get_enable = get_domain_enable,
424 },
425 /* RAPL_DOMAIN_PLATFORM */
426 {
427 .get_energy_uj = get_energy_counter,
428 .get_max_energy_range_uj = get_max_energy_counter,
429 .release = release_zone,
430 .set_enable = set_domain_enable,
431 .get_enable = get_domain_enable,
432 },
433};
434
435/*
436 * Constraint index used by powercap can be different than power limit (PL)
437 * index in that some PLs maybe missing due to non-existent MSRs. So we
438 * need to convert here by finding the valid PLs only (name populated).
439 */
440static int contraint_to_pl(struct rapl_domain *rd, int cid)
441{
442 int i, j;
443
444 for (i = POWER_LIMIT1, j = 0; i < NR_POWER_LIMITS; i++) {
445 if (is_pl_valid(rd, pl: i) && j++ == cid) {
446 pr_debug("%s: index %d\n", __func__, i);
447 return i;
448 }
449 }
450 pr_err("Cannot find matching power limit for constraint %d\n", cid);
451
452 return -EINVAL;
453}
454
455static int set_power_limit(struct powercap_zone *power_zone, int cid,
456 u64 power_limit)
457{
458 struct rapl_domain *rd;
459 struct rapl_package *rp;
460 int ret = 0;
461 int id;
462
463 cpus_read_lock();
464 rd = power_zone_to_rapl_domain(power_zone);
465 id = contraint_to_pl(rd, cid);
466 rp = rd->rp;
467
468 ret = rapl_write_pl_data(rd, pl: id, pl_prim: PL_LIMIT, value: power_limit);
469 if (!ret)
470 package_power_limit_irq_save(rp);
471 cpus_read_unlock();
472 return ret;
473}
474
475static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
476 u64 *data)
477{
478 struct rapl_domain *rd;
479 u64 val;
480 int ret = 0;
481 int id;
482
483 cpus_read_lock();
484 rd = power_zone_to_rapl_domain(power_zone);
485 id = contraint_to_pl(rd, cid);
486
487 ret = rapl_read_pl_data(rd, pl: id, pl_prim: PL_LIMIT, xlate: true, data: &val);
488 if (!ret)
489 *data = val;
490
491 cpus_read_unlock();
492
493 return ret;
494}
495
496static int set_time_window(struct powercap_zone *power_zone, int cid,
497 u64 window)
498{
499 struct rapl_domain *rd;
500 int ret = 0;
501 int id;
502
503 cpus_read_lock();
504 rd = power_zone_to_rapl_domain(power_zone);
505 id = contraint_to_pl(rd, cid);
506
507 ret = rapl_write_pl_data(rd, pl: id, pl_prim: PL_TIME_WINDOW, value: window);
508
509 cpus_read_unlock();
510 return ret;
511}
512
513static int get_time_window(struct powercap_zone *power_zone, int cid,
514 u64 *data)
515{
516 struct rapl_domain *rd;
517 u64 val;
518 int ret = 0;
519 int id;
520
521 cpus_read_lock();
522 rd = power_zone_to_rapl_domain(power_zone);
523 id = contraint_to_pl(rd, cid);
524
525 ret = rapl_read_pl_data(rd, pl: id, pl_prim: PL_TIME_WINDOW, xlate: true, data: &val);
526 if (!ret)
527 *data = val;
528
529 cpus_read_unlock();
530
531 return ret;
532}
533
534static const char *get_constraint_name(struct powercap_zone *power_zone,
535 int cid)
536{
537 struct rapl_domain *rd;
538 int id;
539
540 rd = power_zone_to_rapl_domain(power_zone);
541 id = contraint_to_pl(rd, cid);
542 if (id >= 0)
543 return rd->rpl[id].name;
544
545 return NULL;
546}
547
548static int get_max_power(struct powercap_zone *power_zone, int cid, u64 *data)
549{
550 struct rapl_domain *rd;
551 u64 val;
552 int ret = 0;
553 int id;
554
555 cpus_read_lock();
556 rd = power_zone_to_rapl_domain(power_zone);
557 id = contraint_to_pl(rd, cid);
558
559 ret = rapl_read_pl_data(rd, pl: id, pl_prim: PL_MAX_POWER, xlate: true, data: &val);
560 if (!ret)
561 *data = val;
562
563 /* As a generalization rule, PL4 would be around two times PL2. */
564 if (id == POWER_LIMIT4)
565 *data = *data * 2;
566
567 cpus_read_unlock();
568
569 return ret;
570}
571
572static const struct powercap_zone_constraint_ops constraint_ops = {
573 .set_power_limit_uw = set_power_limit,
574 .get_power_limit_uw = get_current_power_limit,
575 .set_time_window_us = set_time_window,
576 .get_time_window_us = get_time_window,
577 .get_max_power_uw = get_max_power,
578 .get_name = get_constraint_name,
579};
580
581/* Return the id used for read_raw/write_raw callback */
582static int get_rid(struct rapl_package *rp)
583{
584 return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id;
585}
586
587/* called after domain detection and package level data are set */
588static void rapl_init_domains(struct rapl_package *rp)
589{
590 enum rapl_domain_type i;
591 enum rapl_domain_reg_id j;
592 struct rapl_domain *rd = rp->domains;
593
594 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
595 unsigned int mask = rp->domain_map & (1 << i);
596 int t;
597
598 if (!mask)
599 continue;
600
601 rd->rp = rp;
602
603 if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {
604 snprintf(buf: rd->name, RAPL_DOMAIN_NAME_LENGTH, fmt: "psys-%d",
605 rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) :
606 rp->id);
607 } else {
608 snprintf(buf: rd->name, RAPL_DOMAIN_NAME_LENGTH, fmt: "%s",
609 rapl_domain_names[i]);
610 }
611
612 rd->id = i;
613
614 /* PL1 is supported by default */
615 rp->priv->limits[i] |= BIT(POWER_LIMIT1);
616
617 for (t = POWER_LIMIT1; t < NR_POWER_LIMITS; t++) {
618 if (rp->priv->limits[i] & BIT(t))
619 rd->rpl[t].name = pl_names[t];
620 }
621
622 for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++)
623 rd->regs[j] = rp->priv->regs[i][j];
624
625 rd++;
626 }
627}
628
629static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
630 u64 value, int to_raw)
631{
632 u64 units = 1;
633 struct rapl_defaults *defaults = get_defaults(rp: rd->rp);
634 u64 scale = 1;
635
636 switch (type) {
637 case POWER_UNIT:
638 units = rd->power_unit;
639 break;
640 case ENERGY_UNIT:
641 scale = ENERGY_UNIT_SCALE;
642 units = rd->energy_unit;
643 break;
644 case TIME_UNIT:
645 return defaults->compute_time_window(rd, value, to_raw);
646 case ARBITRARY_UNIT:
647 default:
648 return value;
649 }
650
651 if (to_raw)
652 return div64_u64(dividend: value, divisor: units) * scale;
653
654 value *= units;
655
656 return div64_u64(dividend: value, divisor: scale);
657}
658
659/* RAPL primitives for MSR and MMIO I/F */
660static struct rapl_primitive_info rpi_msr[NR_RAPL_PRIMITIVES] = {
661 /* name, mask, shift, msr index, unit divisor */
662 [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
663 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
664 [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
665 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
666 [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0,
667 RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
668 [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
669 RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
670 [FW_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31,
671 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
672 [FW_HIGH_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_HIGH_LOCK, 63,
673 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
674 [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
675 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
676 [PL1_CLAMP] = PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
677 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
678 [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
679 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
680 [PL2_CLAMP] = PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
681 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
682 [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
683 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
684 [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
685 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
686 [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
687 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
688 [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
689 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
690 [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
691 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
692 [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
693 RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
694 [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
695 RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
696 [PRIORITY_LEVEL] = PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
697 RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
698 [PSYS_POWER_LIMIT1] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0,
699 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
700 [PSYS_POWER_LIMIT2] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK, 32,
701 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
702 [PSYS_PL1_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE, 17,
703 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
704 [PSYS_PL2_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE, 49,
705 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
706 [PSYS_TIME_WINDOW1] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK, 19,
707 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
708 [PSYS_TIME_WINDOW2] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK, 51,
709 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
710 /* non-hardware */
711 [AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
712 RAPL_PRIMITIVE_DERIVED),
713};
714
715/* RAPL primitives for TPMI I/F */
716static struct rapl_primitive_info rpi_tpmi[NR_RAPL_PRIMITIVES] = {
717 /* name, mask, shift, msr index, unit divisor */
718 [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, TPMI_POWER_LIMIT_MASK, 0,
719 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
720 [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, TPMI_POWER_LIMIT_MASK, 0,
721 RAPL_DOMAIN_REG_PL2, POWER_UNIT, 0),
722 [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, TPMI_POWER_LIMIT_MASK, 0,
723 RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
724 [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
725 RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
726 [PL1_LOCK] = PRIMITIVE_INFO_INIT(PL1_LOCK, POWER_HIGH_LOCK, 63,
727 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
728 [PL2_LOCK] = PRIMITIVE_INFO_INIT(PL2_LOCK, POWER_HIGH_LOCK, 63,
729 RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
730 [PL4_LOCK] = PRIMITIVE_INFO_INIT(PL4_LOCK, POWER_HIGH_LOCK, 63,
731 RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
732 [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
733 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
734 [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
735 RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),
736 [PL4_ENABLE] = PRIMITIVE_INFO_INIT(PL4_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,
737 RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
738 [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TPMI_TIME_WINDOW_MASK, 18,
739 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
740 [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TPMI_TIME_WINDOW_MASK, 18,
741 RAPL_DOMAIN_REG_PL2, TIME_UNIT, 0),
742 [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, TPMI_INFO_SPEC_MASK, 0,
743 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
744 [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, TPMI_INFO_MAX_MASK, 36,
745 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
746 [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, TPMI_INFO_MIN_MASK, 18,
747 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
748 [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, TPMI_INFO_MAX_TIME_WIN_MASK, 54,
749 RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
750 [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
751 RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
752 /* non-hardware */
753 [AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0,
754 POWER_UNIT, RAPL_PRIMITIVE_DERIVED),
755};
756
757static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)
758{
759 struct rapl_primitive_info *rpi = rp->priv->rpi;
760
761 if (prim < 0 || prim >= NR_RAPL_PRIMITIVES || !rpi)
762 return NULL;
763
764 return &rpi[prim];
765}
766
767static int rapl_config(struct rapl_package *rp)
768{
769 switch (rp->priv->type) {
770 /* MMIO I/F shares the same register layout as MSR registers */
771 case RAPL_IF_MMIO:
772 case RAPL_IF_MSR:
773 rp->priv->defaults = (void *)defaults_msr;
774 rp->priv->rpi = (void *)rpi_msr;
775 break;
776 case RAPL_IF_TPMI:
777 rp->priv->defaults = (void *)&defaults_tpmi;
778 rp->priv->rpi = (void *)rpi_tpmi;
779 break;
780 default:
781 return -EINVAL;
782 }
783
784 /* defaults_msr can be NULL on unsupported platforms */
785 if (!rp->priv->defaults || !rp->priv->rpi)
786 return -ENODEV;
787
788 return 0;
789}
790
791static enum rapl_primitives
792prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim)
793{
794 struct rapl_defaults *defaults = get_defaults(rp: rd->rp);
795
796 if (!defaults->spr_psys_bits)
797 return prim;
798
799 if (rd->id != RAPL_DOMAIN_PLATFORM)
800 return prim;
801
802 switch (prim) {
803 case POWER_LIMIT1:
804 return PSYS_POWER_LIMIT1;
805 case POWER_LIMIT2:
806 return PSYS_POWER_LIMIT2;
807 case PL1_ENABLE:
808 return PSYS_PL1_ENABLE;
809 case PL2_ENABLE:
810 return PSYS_PL2_ENABLE;
811 case TIME_WINDOW1:
812 return PSYS_TIME_WINDOW1;
813 case TIME_WINDOW2:
814 return PSYS_TIME_WINDOW2;
815 default:
816 return prim;
817 }
818}
819
820/* Read primitive data based on its related struct rapl_primitive_info.
821 * if xlate flag is set, return translated data based on data units, i.e.
822 * time, energy, and power.
823 * RAPL MSRs are non-architectual and are laid out not consistently across
824 * domains. Here we use primitive info to allow writing consolidated access
825 * functions.
826 * For a given primitive, it is processed by MSR mask and shift. Unit conversion
827 * is pre-assigned based on RAPL unit MSRs read at init time.
828 * 63-------------------------- 31--------------------------- 0
829 * | xxxxx (mask) |
830 * | |<- shift ----------------|
831 * 63-------------------------- 31--------------------------- 0
832 */
833static int rapl_read_data_raw(struct rapl_domain *rd,
834 enum rapl_primitives prim, bool xlate, u64 *data,
835 bool atomic)
836{
837 u64 value;
838 enum rapl_primitives prim_fixed = prim_fixups(rd, prim);
839 struct rapl_primitive_info *rpi = get_rpi(rp: rd->rp, prim: prim_fixed);
840 struct reg_action ra;
841
842 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY)
843 return -EINVAL;
844
845 ra.reg = rd->regs[rpi->id];
846 if (!ra.reg.val)
847 return -EINVAL;
848
849 /* non-hardware data are collected by the polling thread */
850 if (rpi->flag & RAPL_PRIMITIVE_DERIVED) {
851 *data = rd->rdd.primitives[prim];
852 return 0;
853 }
854
855 ra.mask = rpi->mask;
856
857 if (rd->rp->priv->read_raw(get_rid(rp: rd->rp), &ra, atomic)) {
858 pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name);
859 return -EIO;
860 }
861
862 value = ra.value >> rpi->shift;
863
864 if (xlate)
865 *data = rapl_unit_xlate(rd, type: rpi->unit, value, to_raw: 0);
866 else
867 *data = value;
868
869 return 0;
870}
871
872/* Similar use of primitive info in the read counterpart */
873static int rapl_write_data_raw(struct rapl_domain *rd,
874 enum rapl_primitives prim,
875 unsigned long long value)
876{
877 enum rapl_primitives prim_fixed = prim_fixups(rd, prim);
878 struct rapl_primitive_info *rpi = get_rpi(rp: rd->rp, prim: prim_fixed);
879 u64 bits;
880 struct reg_action ra;
881 int ret;
882
883 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY)
884 return -EINVAL;
885
886 bits = rapl_unit_xlate(rd, type: rpi->unit, value, to_raw: 1);
887 bits <<= rpi->shift;
888 bits &= rpi->mask;
889
890 memset(&ra, 0, sizeof(ra));
891
892 ra.reg = rd->regs[rpi->id];
893 ra.mask = rpi->mask;
894 ra.value = bits;
895
896 ret = rd->rp->priv->write_raw(get_rid(rp: rd->rp), &ra);
897
898 return ret;
899}
900
901static int rapl_read_pl_data(struct rapl_domain *rd, int pl,
902 enum pl_prims pl_prim, bool xlate, u64 *data)
903{
904 enum rapl_primitives prim = get_pl_prim(rd, pl, prim: pl_prim);
905
906 if (!is_pl_valid(rd, pl))
907 return -EINVAL;
908
909 return rapl_read_data_raw(rd, prim, xlate, data, atomic: false);
910}
911
912static int rapl_write_pl_data(struct rapl_domain *rd, int pl,
913 enum pl_prims pl_prim,
914 unsigned long long value)
915{
916 enum rapl_primitives prim = get_pl_prim(rd, pl, prim: pl_prim);
917
918 if (!is_pl_valid(rd, pl))
919 return -EINVAL;
920
921 if (rd->rpl[pl].locked) {
922 pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]);
923 return -EACCES;
924 }
925
926 return rapl_write_data_raw(rd, prim, value);
927}
928/*
929 * Raw RAPL data stored in MSRs are in certain scales. We need to
930 * convert them into standard units based on the units reported in
931 * the RAPL unit MSRs. This is specific to CPUs as the method to
932 * calculate units differ on different CPUs.
933 * We convert the units to below format based on CPUs.
934 * i.e.
935 * energy unit: picoJoules : Represented in picoJoules by default
936 * power unit : microWatts : Represented in milliWatts by default
937 * time unit : microseconds: Represented in seconds by default
938 */
939static int rapl_check_unit_core(struct rapl_domain *rd)
940{
941 struct reg_action ra;
942 u32 value;
943
944 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
945 ra.mask = ~0;
946 if (rd->rp->priv->read_raw(get_rid(rp: rd->rp), &ra, false)) {
947 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
948 ra.reg.val, rd->rp->name, rd->name);
949 return -ENODEV;
950 }
951
952 value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
953 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
954
955 value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
956 rd->power_unit = 1000000 / (1 << value);
957
958 value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
959 rd->time_unit = 1000000 / (1 << value);
960
961 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
962 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
963
964 return 0;
965}
966
967static int rapl_check_unit_atom(struct rapl_domain *rd)
968{
969 struct reg_action ra;
970 u32 value;
971
972 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
973 ra.mask = ~0;
974 if (rd->rp->priv->read_raw(get_rid(rp: rd->rp), &ra, false)) {
975 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
976 ra.reg.val, rd->rp->name, rd->name);
977 return -ENODEV;
978 }
979
980 value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
981 rd->energy_unit = ENERGY_UNIT_SCALE * 1 << value;
982
983 value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
984 rd->power_unit = (1 << value) * 1000;
985
986 value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
987 rd->time_unit = 1000000 / (1 << value);
988
989 pr_debug("Atom %s:%s energy=%dpJ, time=%dus, power=%duW\n",
990 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
991
992 return 0;
993}
994
995static void power_limit_irq_save_cpu(void *info)
996{
997 u32 l, h = 0;
998 struct rapl_package *rp = (struct rapl_package *)info;
999
1000 /* save the state of PLN irq mask bit before disabling it */
1001 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
1002 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
1003 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;
1004 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
1005 }
1006 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
1007 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, low: l, high: h);
1008}
1009
1010/* REVISIT:
1011 * When package power limit is set artificially low by RAPL, LVT
1012 * thermal interrupt for package power limit should be ignored
1013 * since we are not really exceeding the real limit. The intention
1014 * is to avoid excessive interrupts while we are trying to save power.
1015 * A useful feature might be routing the package_power_limit interrupt
1016 * to userspace via eventfd. once we have a usecase, this is simple
1017 * to do by adding an atomic notifier.
1018 */
1019
1020static void package_power_limit_irq_save(struct rapl_package *rp)
1021{
1022 if (rp->lead_cpu < 0)
1023 return;
1024
1025 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
1026 return;
1027
1028 smp_call_function_single(cpuid: rp->lead_cpu, func: power_limit_irq_save_cpu, info: rp, wait: 1);
1029}
1030
1031/*
1032 * Restore per package power limit interrupt enable state. Called from cpu
1033 * hotplug code on package removal.
1034 */
1035static void package_power_limit_irq_restore(struct rapl_package *rp)
1036{
1037 u32 l, h;
1038
1039 if (rp->lead_cpu < 0)
1040 return;
1041
1042 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
1043 return;
1044
1045 /* irq enable state not saved, nothing to restore */
1046 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
1047 return;
1048
1049 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
1050
1051 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
1052 l |= PACKAGE_THERM_INT_PLN_ENABLE;
1053 else
1054 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
1055
1056 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, low: l, high: h);
1057}
1058
1059static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
1060{
1061 int i;
1062
1063 /* always enable clamp such that p-state can go below OS requested
1064 * range. power capping priority over guranteed frequency.
1065 */
1066 rapl_write_pl_data(rd, pl: POWER_LIMIT1, pl_prim: PL_CLAMP, value: mode);
1067
1068 for (i = POWER_LIMIT2; i < NR_POWER_LIMITS; i++) {
1069 rapl_write_pl_data(rd, pl: i, pl_prim: PL_ENABLE, value: mode);
1070 rapl_write_pl_data(rd, pl: i, pl_prim: PL_CLAMP, value: mode);
1071 }
1072}
1073
1074static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
1075{
1076 static u32 power_ctrl_orig_val;
1077 struct rapl_defaults *defaults = get_defaults(rp: rd->rp);
1078 u32 mdata;
1079
1080 if (!defaults->floor_freq_reg_addr) {
1081 pr_err("Invalid floor frequency config register\n");
1082 return;
1083 }
1084
1085 if (!power_ctrl_orig_val)
1086 iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,
1087 offset: defaults->floor_freq_reg_addr,
1088 mdr: &power_ctrl_orig_val);
1089 mdata = power_ctrl_orig_val;
1090 if (enable) {
1091 mdata &= ~(0x7f << 8);
1092 mdata |= 1 << 8;
1093 }
1094 iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,
1095 offset: defaults->floor_freq_reg_addr, mdr: mdata);
1096}
1097
1098static u64 rapl_compute_time_window_core(struct rapl_domain *rd, u64 value,
1099 bool to_raw)
1100{
1101 u64 f, y; /* fraction and exp. used for time unit */
1102
1103 /*
1104 * Special processing based on 2^Y*(1+F/4), refer
1105 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
1106 */
1107 if (!to_raw) {
1108 f = (value & 0x60) >> 5;
1109 y = value & 0x1f;
1110 value = (1 << y) * (4 + f) * rd->time_unit / 4;
1111 } else {
1112 if (value < rd->time_unit)
1113 return 0;
1114
1115 do_div(value, rd->time_unit);
1116 y = ilog2(value);
1117
1118 /*
1119 * The target hardware field is 7 bits wide, so return all ones
1120 * if the exponent is too large.
1121 */
1122 if (y > 0x1f)
1123 return 0x7f;
1124
1125 f = div64_u64(dividend: 4 * (value - (1ULL << y)), divisor: 1ULL << y);
1126 value = (y & 0x1f) | ((f & 0x3) << 5);
1127 }
1128 return value;
1129}
1130
1131static u64 rapl_compute_time_window_atom(struct rapl_domain *rd, u64 value,
1132 bool to_raw)
1133{
1134 /*
1135 * Atom time unit encoding is straight forward val * time_unit,
1136 * where time_unit is default to 1 sec. Never 0.
1137 */
1138 if (!to_raw)
1139 return (value) ? value * rd->time_unit : rd->time_unit;
1140
1141 value = div64_u64(dividend: value, divisor: rd->time_unit);
1142
1143 return value;
1144}
1145
1146/* TPMI Unit register has different layout */
1147#define TPMI_POWER_UNIT_OFFSET POWER_UNIT_OFFSET
1148#define TPMI_POWER_UNIT_MASK POWER_UNIT_MASK
1149#define TPMI_ENERGY_UNIT_OFFSET 0x06
1150#define TPMI_ENERGY_UNIT_MASK 0x7C0
1151#define TPMI_TIME_UNIT_OFFSET 0x0C
1152#define TPMI_TIME_UNIT_MASK 0xF000
1153
1154static int rapl_check_unit_tpmi(struct rapl_domain *rd)
1155{
1156 struct reg_action ra;
1157 u32 value;
1158
1159 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
1160 ra.mask = ~0;
1161 if (rd->rp->priv->read_raw(get_rid(rp: rd->rp), &ra, false)) {
1162 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
1163 ra.reg.val, rd->rp->name, rd->name);
1164 return -ENODEV;
1165 }
1166
1167 value = (ra.value & TPMI_ENERGY_UNIT_MASK) >> TPMI_ENERGY_UNIT_OFFSET;
1168 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
1169
1170 value = (ra.value & TPMI_POWER_UNIT_MASK) >> TPMI_POWER_UNIT_OFFSET;
1171 rd->power_unit = 1000000 / (1 << value);
1172
1173 value = (ra.value & TPMI_TIME_UNIT_MASK) >> TPMI_TIME_UNIT_OFFSET;
1174 rd->time_unit = 1000000 / (1 << value);
1175
1176 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
1177 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
1178
1179 return 0;
1180}
1181
1182static const struct rapl_defaults defaults_tpmi = {
1183 .check_unit = rapl_check_unit_tpmi,
1184 /* Reuse existing logic, ignore the PL_CLAMP failures and enable all Power Limits */
1185 .set_floor_freq = set_floor_freq_default,
1186 .compute_time_window = rapl_compute_time_window_core,
1187};
1188
1189static const struct rapl_defaults rapl_defaults_core = {
1190 .floor_freq_reg_addr = 0,
1191 .check_unit = rapl_check_unit_core,
1192 .set_floor_freq = set_floor_freq_default,
1193 .compute_time_window = rapl_compute_time_window_core,
1194};
1195
1196static const struct rapl_defaults rapl_defaults_hsw_server = {
1197 .check_unit = rapl_check_unit_core,
1198 .set_floor_freq = set_floor_freq_default,
1199 .compute_time_window = rapl_compute_time_window_core,
1200 .dram_domain_energy_unit = 15300,
1201};
1202
1203static const struct rapl_defaults rapl_defaults_spr_server = {
1204 .check_unit = rapl_check_unit_core,
1205 .set_floor_freq = set_floor_freq_default,
1206 .compute_time_window = rapl_compute_time_window_core,
1207 .psys_domain_energy_unit = 1000000000,
1208 .spr_psys_bits = true,
1209};
1210
1211static const struct rapl_defaults rapl_defaults_byt = {
1212 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,
1213 .check_unit = rapl_check_unit_atom,
1214 .set_floor_freq = set_floor_freq_atom,
1215 .compute_time_window = rapl_compute_time_window_atom,
1216};
1217
1218static const struct rapl_defaults rapl_defaults_tng = {
1219 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,
1220 .check_unit = rapl_check_unit_atom,
1221 .set_floor_freq = set_floor_freq_atom,
1222 .compute_time_window = rapl_compute_time_window_atom,
1223};
1224
1225static const struct rapl_defaults rapl_defaults_ann = {
1226 .floor_freq_reg_addr = 0,
1227 .check_unit = rapl_check_unit_atom,
1228 .set_floor_freq = NULL,
1229 .compute_time_window = rapl_compute_time_window_atom,
1230};
1231
1232static const struct rapl_defaults rapl_defaults_cht = {
1233 .floor_freq_reg_addr = 0,
1234 .check_unit = rapl_check_unit_atom,
1235 .set_floor_freq = NULL,
1236 .compute_time_window = rapl_compute_time_window_atom,
1237};
1238
1239static const struct rapl_defaults rapl_defaults_amd = {
1240 .check_unit = rapl_check_unit_core,
1241};
1242
1243static const struct x86_cpu_id rapl_ids[] __initconst = {
1244 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &rapl_defaults_core),
1245 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &rapl_defaults_core),
1246
1247 X86_MATCH_VFM(INTEL_IVYBRIDGE, &rapl_defaults_core),
1248 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &rapl_defaults_core),
1249
1250 X86_MATCH_VFM(INTEL_HASWELL, &rapl_defaults_core),
1251 X86_MATCH_VFM(INTEL_HASWELL_L, &rapl_defaults_core),
1252 X86_MATCH_VFM(INTEL_HASWELL_G, &rapl_defaults_core),
1253 X86_MATCH_VFM(INTEL_HASWELL_X, &rapl_defaults_hsw_server),
1254
1255 X86_MATCH_VFM(INTEL_BROADWELL, &rapl_defaults_core),
1256 X86_MATCH_VFM(INTEL_BROADWELL_G, &rapl_defaults_core),
1257 X86_MATCH_VFM(INTEL_BROADWELL_D, &rapl_defaults_core),
1258 X86_MATCH_VFM(INTEL_BROADWELL_X, &rapl_defaults_hsw_server),
1259
1260 X86_MATCH_VFM(INTEL_SKYLAKE, &rapl_defaults_core),
1261 X86_MATCH_VFM(INTEL_SKYLAKE_L, &rapl_defaults_core),
1262 X86_MATCH_VFM(INTEL_SKYLAKE_X, &rapl_defaults_hsw_server),
1263 X86_MATCH_VFM(INTEL_KABYLAKE_L, &rapl_defaults_core),
1264 X86_MATCH_VFM(INTEL_KABYLAKE, &rapl_defaults_core),
1265 X86_MATCH_VFM(INTEL_CANNONLAKE_L, &rapl_defaults_core),
1266 X86_MATCH_VFM(INTEL_ICELAKE_L, &rapl_defaults_core),
1267 X86_MATCH_VFM(INTEL_ICELAKE, &rapl_defaults_core),
1268 X86_MATCH_VFM(INTEL_ICELAKE_NNPI, &rapl_defaults_core),
1269 X86_MATCH_VFM(INTEL_ICELAKE_X, &rapl_defaults_hsw_server),
1270 X86_MATCH_VFM(INTEL_ICELAKE_D, &rapl_defaults_hsw_server),
1271 X86_MATCH_VFM(INTEL_COMETLAKE_L, &rapl_defaults_core),
1272 X86_MATCH_VFM(INTEL_COMETLAKE, &rapl_defaults_core),
1273 X86_MATCH_VFM(INTEL_TIGERLAKE_L, &rapl_defaults_core),
1274 X86_MATCH_VFM(INTEL_TIGERLAKE, &rapl_defaults_core),
1275 X86_MATCH_VFM(INTEL_ROCKETLAKE, &rapl_defaults_core),
1276 X86_MATCH_VFM(INTEL_ALDERLAKE, &rapl_defaults_core),
1277 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &rapl_defaults_core),
1278 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &rapl_defaults_core),
1279 X86_MATCH_VFM(INTEL_RAPTORLAKE, &rapl_defaults_core),
1280 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, &rapl_defaults_core),
1281 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, &rapl_defaults_core),
1282 X86_MATCH_VFM(INTEL_BARTLETTLAKE, &rapl_defaults_core),
1283 X86_MATCH_VFM(INTEL_METEORLAKE, &rapl_defaults_core),
1284 X86_MATCH_VFM(INTEL_METEORLAKE_L, &rapl_defaults_core),
1285 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &rapl_defaults_spr_server),
1286 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &rapl_defaults_spr_server),
1287 X86_MATCH_VFM(INTEL_LUNARLAKE_M, &rapl_defaults_core),
1288 X86_MATCH_VFM(INTEL_PANTHERLAKE_L, &rapl_defaults_core),
1289 X86_MATCH_VFM(INTEL_WILDCATLAKE_L, &rapl_defaults_core),
1290 X86_MATCH_VFM(INTEL_NOVALAKE, &rapl_defaults_core),
1291 X86_MATCH_VFM(INTEL_NOVALAKE_L, &rapl_defaults_core),
1292 X86_MATCH_VFM(INTEL_ARROWLAKE_H, &rapl_defaults_core),
1293 X86_MATCH_VFM(INTEL_ARROWLAKE, &rapl_defaults_core),
1294 X86_MATCH_VFM(INTEL_ARROWLAKE_U, &rapl_defaults_core),
1295 X86_MATCH_VFM(INTEL_LAKEFIELD, &rapl_defaults_core),
1296
1297 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &rapl_defaults_byt),
1298 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &rapl_defaults_cht),
1299 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &rapl_defaults_tng),
1300 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID2,&rapl_defaults_ann),
1301 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &rapl_defaults_core),
1302 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &rapl_defaults_core),
1303 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &rapl_defaults_core),
1304 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &rapl_defaults_core),
1305 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &rapl_defaults_core),
1306 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &rapl_defaults_core),
1307
1308 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &rapl_defaults_hsw_server),
1309 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &rapl_defaults_hsw_server),
1310
1311 X86_MATCH_VENDOR_FAM(AMD, 0x17, &rapl_defaults_amd),
1312 X86_MATCH_VENDOR_FAM(AMD, 0x19, &rapl_defaults_amd),
1313 X86_MATCH_VENDOR_FAM(AMD, 0x1A, &rapl_defaults_amd),
1314 X86_MATCH_VENDOR_FAM(HYGON, 0x18, &rapl_defaults_amd),
1315 {}
1316};
1317MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
1318
1319/* Read once for all raw primitive data for domains */
1320static void rapl_update_domain_data(struct rapl_package *rp)
1321{
1322 int dmn, prim;
1323 u64 val;
1324
1325 for (dmn = 0; dmn < rp->nr_domains; dmn++) {
1326 pr_debug("update %s domain %s data\n", rp->name,
1327 rp->domains[dmn].name);
1328 /* exclude non-raw primitives */
1329 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
1330 struct rapl_primitive_info *rpi = get_rpi(rp, prim);
1331
1332 if (!rapl_read_data_raw(rd: &rp->domains[dmn], prim,
1333 xlate: rpi->unit, data: &val, atomic: false))
1334 rp->domains[dmn].rdd.primitives[prim] = val;
1335 }
1336 }
1337
1338}
1339
1340static int rapl_package_register_powercap(struct rapl_package *rp)
1341{
1342 struct rapl_domain *rd;
1343 struct powercap_zone *power_zone = NULL;
1344 int nr_pl, ret;
1345
1346 /* Update the domain data of the new package */
1347 rapl_update_domain_data(rp);
1348
1349 /* first we register package domain as the parent zone */
1350 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1351 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1352 nr_pl = find_nr_power_limit(rd);
1353 pr_debug("register package domain %s\n", rp->name);
1354 power_zone = powercap_register_zone(power_zone: &rd->power_zone,
1355 control_type: rp->priv->control_type, name: rp->name,
1356 NULL, ops: &zone_ops[rd->id], nr_constraints: nr_pl,
1357 const_ops: &constraint_ops);
1358 if (IS_ERR(ptr: power_zone)) {
1359 pr_debug("failed to register power zone %s\n",
1360 rp->name);
1361 return PTR_ERR(ptr: power_zone);
1362 }
1363 /* track parent zone in per package/socket data */
1364 rp->power_zone = power_zone;
1365 /* done, only one package domain per socket */
1366 break;
1367 }
1368 }
1369 if (!power_zone) {
1370 pr_err("no package domain found, unknown topology!\n");
1371 return -ENODEV;
1372 }
1373 /* now register domains as children of the socket/package */
1374 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1375 struct powercap_zone *parent = rp->power_zone;
1376
1377 if (rd->id == RAPL_DOMAIN_PACKAGE)
1378 continue;
1379 if (rd->id == RAPL_DOMAIN_PLATFORM)
1380 parent = NULL;
1381 /* number of power limits per domain varies */
1382 nr_pl = find_nr_power_limit(rd);
1383 power_zone = powercap_register_zone(power_zone: &rd->power_zone,
1384 control_type: rp->priv->control_type,
1385 name: rd->name, parent,
1386 ops: &zone_ops[rd->id], nr_constraints: nr_pl,
1387 const_ops: &constraint_ops);
1388
1389 if (IS_ERR(ptr: power_zone)) {
1390 pr_debug("failed to register power_zone, %s:%s\n",
1391 rp->name, rd->name);
1392 ret = PTR_ERR(ptr: power_zone);
1393 goto err_cleanup;
1394 }
1395 }
1396 return 0;
1397
1398err_cleanup:
1399 /*
1400 * Clean up previously initialized domains within the package if we
1401 * failed after the first domain setup.
1402 */
1403 while (--rd >= rp->domains) {
1404 pr_debug("unregister %s domain %s\n", rp->name, rd->name);
1405 powercap_unregister_zone(control_type: rp->priv->control_type,
1406 power_zone: &rd->power_zone);
1407 }
1408
1409 return ret;
1410}
1411
1412static int rapl_check_domain(int domain, struct rapl_package *rp)
1413{
1414 struct reg_action ra;
1415
1416 switch (domain) {
1417 case RAPL_DOMAIN_PACKAGE:
1418 case RAPL_DOMAIN_PP0:
1419 case RAPL_DOMAIN_PP1:
1420 case RAPL_DOMAIN_DRAM:
1421 case RAPL_DOMAIN_PLATFORM:
1422 ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];
1423 break;
1424 default:
1425 pr_err("invalid domain id %d\n", domain);
1426 return -EINVAL;
1427 }
1428 /* make sure domain counters are available and contains non-zero
1429 * values, otherwise skip it.
1430 */
1431
1432 ra.mask = ENERGY_STATUS_MASK;
1433 if (rp->priv->read_raw(get_rid(rp), &ra, false) || !ra.value)
1434 return -ENODEV;
1435
1436 return 0;
1437}
1438
1439/*
1440 * Get per domain energy/power/time unit.
1441 * RAPL Interfaces without per domain unit register will use the package
1442 * scope unit register to set per domain units.
1443 */
1444static int rapl_get_domain_unit(struct rapl_domain *rd)
1445{
1446 struct rapl_defaults *defaults = get_defaults(rp: rd->rp);
1447 int ret;
1448
1449 if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) {
1450 if (!rd->rp->priv->reg_unit.val) {
1451 pr_err("No valid Unit register found\n");
1452 return -ENODEV;
1453 }
1454 rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit;
1455 }
1456
1457 if (!defaults->check_unit) {
1458 pr_err("missing .check_unit() callback\n");
1459 return -ENODEV;
1460 }
1461
1462 ret = defaults->check_unit(rd);
1463 if (ret)
1464 return ret;
1465
1466 if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit)
1467 rd->energy_unit = defaults->dram_domain_energy_unit;
1468 if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit)
1469 rd->energy_unit = defaults->psys_domain_energy_unit;
1470 return 0;
1471}
1472
1473/*
1474 * Check if power limits are available. Two cases when they are not available:
1475 * 1. Locked by BIOS, in this case we still provide read-only access so that
1476 * users can see what limit is set by the BIOS.
1477 * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not
1478 * exist at all. In this case, we do not show the constraints in powercap.
1479 *
1480 * Called after domains are detected and initialized.
1481 */
1482static void rapl_detect_powerlimit(struct rapl_domain *rd)
1483{
1484 u64 val64;
1485 int i;
1486
1487 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
1488 if (!rapl_read_pl_data(rd, pl: i, pl_prim: PL_LOCK, xlate: false, data: &val64)) {
1489 if (val64) {
1490 rd->rpl[i].locked = true;
1491 pr_info("%s:%s:%s locked by BIOS\n",
1492 rd->rp->name, rd->name, pl_names[i]);
1493 }
1494 }
1495
1496 if (rapl_read_pl_data(rd, pl: i, pl_prim: PL_LIMIT, xlate: false, data: &val64))
1497 rd->rpl[i].name = NULL;
1498 }
1499}
1500
1501/* Detect active and valid domains for the given CPU, caller must
1502 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
1503 */
1504static int rapl_detect_domains(struct rapl_package *rp)
1505{
1506 struct rapl_domain *rd;
1507 int i;
1508
1509 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
1510 /* use physical package id to read counters */
1511 if (!rapl_check_domain(domain: i, rp)) {
1512 rp->domain_map |= 1 << i;
1513 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
1514 }
1515 }
1516 rp->nr_domains = bitmap_weight(src: &rp->domain_map, nbits: RAPL_DOMAIN_MAX);
1517 if (!rp->nr_domains) {
1518 pr_debug("no valid rapl domains found in %s\n", rp->name);
1519 return -ENODEV;
1520 }
1521 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);
1522
1523 rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain),
1524 GFP_KERNEL);
1525 if (!rp->domains)
1526 return -ENOMEM;
1527
1528 rapl_init_domains(rp);
1529
1530 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1531 rapl_get_domain_unit(rd);
1532 rapl_detect_powerlimit(rd);
1533 }
1534
1535 return 0;
1536}
1537
1538#ifdef CONFIG_PERF_EVENTS
1539
1540/*
1541 * Support for RAPL PMU
1542 *
1543 * Register a PMU if any of the registered RAPL Packages have the requirement
1544 * of exposing its energy counters via Perf PMU.
1545 *
1546 * PMU Name:
1547 * power
1548 *
1549 * Events:
1550 * Name Event id RAPL Domain
1551 * energy_cores 0x01 RAPL_DOMAIN_PP0
1552 * energy_pkg 0x02 RAPL_DOMAIN_PACKAGE
1553 * energy_ram 0x03 RAPL_DOMAIN_DRAM
1554 * energy_gpu 0x04 RAPL_DOMAIN_PP1
1555 * energy_psys 0x05 RAPL_DOMAIN_PLATFORM
1556 *
1557 * Unit:
1558 * Joules
1559 *
1560 * Scale:
1561 * 2.3283064365386962890625e-10
1562 * The same RAPL domain in different RAPL Packages may have different
1563 * energy units. Use 2.3283064365386962890625e-10 (2^-32) Joules as
1564 * the fixed unit for all energy counters, and covert each hardware
1565 * counter increase to N times of PMU event counter increases.
1566 *
1567 * This is fully compatible with the current MSR RAPL PMU. This means that
1568 * userspace programs like turbostat can use the same code to handle RAPL Perf
1569 * PMU, no matter what RAPL Interface driver (MSR/TPMI, etc) is running
1570 * underlying on the platform.
1571 *
1572 * Note that RAPL Packages can be probed/removed dynamically, and the events
1573 * supported by each TPMI RAPL device can be different. Thus the RAPL PMU
1574 * support is done on demand, which means
1575 * 1. PMU is registered only if it is needed by a RAPL Package. PMU events for
1576 * unsupported counters are not exposed.
1577 * 2. PMU is unregistered and registered when a new RAPL Package is probed and
1578 * supports new counters that are not supported by current PMU.
1579 * 3. PMU is unregistered when all registered RAPL Packages don't need PMU.
1580 */
1581
1582struct rapl_pmu {
1583 struct pmu pmu; /* Perf PMU structure */
1584 u64 timer_ms; /* Maximum expiration time to avoid counter overflow */
1585 unsigned long domain_map; /* Events supported by current registered PMU */
1586 bool registered; /* Whether the PMU has been registered or not */
1587};
1588
1589static struct rapl_pmu rapl_pmu;
1590
1591/* PMU helpers */
1592
1593static int get_pmu_cpu(struct rapl_package *rp)
1594{
1595 int cpu;
1596
1597 if (!rp->has_pmu)
1598 return nr_cpu_ids;
1599
1600 /* Only TPMI & MSR RAPL are supported for now */
1601 if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR)
1602 return nr_cpu_ids;
1603
1604 /* TPMI/MSR RAPL uses any CPU in the package for PMU */
1605 for_each_online_cpu(cpu)
1606 if (topology_physical_package_id(cpu) == rp->id)
1607 return cpu;
1608
1609 return nr_cpu_ids;
1610}
1611
1612static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu)
1613{
1614 if (!rp->has_pmu)
1615 return false;
1616
1617 /* Only TPMI & MSR RAPL are supported for now */
1618 if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR)
1619 return false;
1620
1621 /* TPMI/MSR RAPL uses any CPU in the package for PMU */
1622 return topology_physical_package_id(cpu) == rp->id;
1623}
1624
1625static struct rapl_package_pmu_data *event_to_pmu_data(struct perf_event *event)
1626{
1627 struct rapl_package *rp = event->pmu_private;
1628
1629 return &rp->pmu_data;
1630}
1631
1632/* PMU event callbacks */
1633
1634static u64 event_read_counter(struct perf_event *event)
1635{
1636 struct rapl_package *rp = event->pmu_private;
1637 u64 val;
1638 int ret;
1639
1640 /* Return 0 for unsupported events */
1641 if (event->hw.idx < 0)
1642 return 0;
1643
1644 ret = rapl_read_data_raw(rd: &rp->domains[event->hw.idx], prim: ENERGY_COUNTER, xlate: false, data: &val, atomic: true);
1645
1646 /* Return 0 for failed read */
1647 if (ret)
1648 return 0;
1649
1650 return val;
1651}
1652
1653static void __rapl_pmu_event_start(struct perf_event *event)
1654{
1655 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1656
1657 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1658 return;
1659
1660 event->hw.state = 0;
1661
1662 list_add_tail(new: &event->active_entry, head: &data->active_list);
1663
1664 local64_set(&event->hw.prev_count, event_read_counter(event));
1665 if (++data->n_active == 1)
1666 hrtimer_start(timer: &data->hrtimer, tim: data->timer_interval,
1667 mode: HRTIMER_MODE_REL_PINNED);
1668}
1669
1670static void rapl_pmu_event_start(struct perf_event *event, int mode)
1671{
1672 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1673 unsigned long flags;
1674
1675 raw_spin_lock_irqsave(&data->lock, flags);
1676 __rapl_pmu_event_start(event);
1677 raw_spin_unlock_irqrestore(&data->lock, flags);
1678}
1679
1680static u64 rapl_event_update(struct perf_event *event)
1681{
1682 struct hw_perf_event *hwc = &event->hw;
1683 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1684 u64 prev_raw_count, new_raw_count;
1685 s64 delta, sdelta;
1686
1687 /*
1688 * Follow the generic code to drain hwc->prev_count.
1689 * The loop is not expected to run for multiple times.
1690 */
1691 prev_raw_count = local64_read(&hwc->prev_count);
1692 do {
1693 new_raw_count = event_read_counter(event);
1694 } while (!local64_try_cmpxchg(l: &hwc->prev_count,
1695 old: &prev_raw_count, new: new_raw_count));
1696
1697
1698 /*
1699 * Now we have the new raw value and have updated the prev
1700 * timestamp already. We can now calculate the elapsed delta
1701 * (event-)time and add that to the generic event.
1702 */
1703 delta = new_raw_count - prev_raw_count;
1704
1705 /*
1706 * Scale delta to smallest unit (2^-32)
1707 * users must then scale back: count * 1/(1e9*2^32) to get Joules
1708 * or use ldexp(count, -32).
1709 * Watts = Joules/Time delta
1710 */
1711 sdelta = delta * data->scale[event->hw.flags];
1712
1713 local64_add(sdelta, &event->count);
1714
1715 return new_raw_count;
1716}
1717
1718static void rapl_pmu_event_stop(struct perf_event *event, int mode)
1719{
1720 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1721 struct hw_perf_event *hwc = &event->hw;
1722 unsigned long flags;
1723
1724 raw_spin_lock_irqsave(&data->lock, flags);
1725
1726 /* Mark event as deactivated and stopped */
1727 if (!(hwc->state & PERF_HES_STOPPED)) {
1728 WARN_ON_ONCE(data->n_active <= 0);
1729 if (--data->n_active == 0)
1730 hrtimer_cancel(timer: &data->hrtimer);
1731
1732 list_del(entry: &event->active_entry);
1733
1734 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1735 hwc->state |= PERF_HES_STOPPED;
1736 }
1737
1738 /* Check if update of sw counter is necessary */
1739 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1740 /*
1741 * Drain the remaining delta count out of a event
1742 * that we are disabling:
1743 */
1744 rapl_event_update(event);
1745 hwc->state |= PERF_HES_UPTODATE;
1746 }
1747
1748 raw_spin_unlock_irqrestore(&data->lock, flags);
1749}
1750
1751static int rapl_pmu_event_add(struct perf_event *event, int mode)
1752{
1753 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1754 struct hw_perf_event *hwc = &event->hw;
1755 unsigned long flags;
1756
1757 raw_spin_lock_irqsave(&data->lock, flags);
1758
1759 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1760
1761 if (mode & PERF_EF_START)
1762 __rapl_pmu_event_start(event);
1763
1764 raw_spin_unlock_irqrestore(&data->lock, flags);
1765
1766 return 0;
1767}
1768
1769static void rapl_pmu_event_del(struct perf_event *event, int flags)
1770{
1771 rapl_pmu_event_stop(event, PERF_EF_UPDATE);
1772}
1773
1774/* RAPL PMU event ids, same as shown in sysfs */
1775enum perf_rapl_events {
1776 PERF_RAPL_PP0 = 1, /* all cores */
1777 PERF_RAPL_PKG, /* entire package */
1778 PERF_RAPL_RAM, /* DRAM */
1779 PERF_RAPL_PP1, /* gpu */
1780 PERF_RAPL_PSYS, /* psys */
1781 PERF_RAPL_MAX
1782};
1783#define RAPL_EVENT_MASK GENMASK(7, 0)
1784
1785static const int event_to_domain[PERF_RAPL_MAX] = {
1786 [PERF_RAPL_PP0] = RAPL_DOMAIN_PP0,
1787 [PERF_RAPL_PKG] = RAPL_DOMAIN_PACKAGE,
1788 [PERF_RAPL_RAM] = RAPL_DOMAIN_DRAM,
1789 [PERF_RAPL_PP1] = RAPL_DOMAIN_PP1,
1790 [PERF_RAPL_PSYS] = RAPL_DOMAIN_PLATFORM,
1791};
1792
1793static int rapl_pmu_event_init(struct perf_event *event)
1794{
1795 struct rapl_package *pos, *rp = NULL;
1796 u64 cfg = event->attr.config & RAPL_EVENT_MASK;
1797 int domain, idx;
1798
1799 /* Only look at RAPL events */
1800 if (event->attr.type != event->pmu->type)
1801 return -ENOENT;
1802
1803 /* Check for supported events only */
1804 if (!cfg || cfg >= PERF_RAPL_MAX)
1805 return -EINVAL;
1806
1807 if (event->cpu < 0)
1808 return -EINVAL;
1809
1810 /* Find out which Package the event belongs to */
1811 list_for_each_entry(pos, &rapl_packages, plist) {
1812 if (is_rp_pmu_cpu(rp: pos, cpu: event->cpu)) {
1813 rp = pos;
1814 break;
1815 }
1816 }
1817 if (!rp)
1818 return -ENODEV;
1819
1820 /* Find out which RAPL Domain the event belongs to */
1821 domain = event_to_domain[cfg];
1822
1823 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
1824 event->pmu_private = rp; /* Which package */
1825 event->hw.flags = domain; /* Which domain */
1826
1827 event->hw.idx = -1;
1828 /* Find out the index in rp->domains[] to get domain pointer */
1829 for (idx = 0; idx < rp->nr_domains; idx++) {
1830 if (rp->domains[idx].id == domain) {
1831 event->hw.idx = idx;
1832 break;
1833 }
1834 }
1835
1836 return 0;
1837}
1838
1839static void rapl_pmu_event_read(struct perf_event *event)
1840{
1841 rapl_event_update(event);
1842}
1843
1844static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
1845{
1846 struct rapl_package_pmu_data *data =
1847 container_of(hrtimer, struct rapl_package_pmu_data, hrtimer);
1848 struct perf_event *event;
1849 unsigned long flags;
1850
1851 if (!data->n_active)
1852 return HRTIMER_NORESTART;
1853
1854 raw_spin_lock_irqsave(&data->lock, flags);
1855
1856 list_for_each_entry(event, &data->active_list, active_entry)
1857 rapl_event_update(event);
1858
1859 raw_spin_unlock_irqrestore(&data->lock, flags);
1860
1861 hrtimer_forward_now(timer: hrtimer, interval: data->timer_interval);
1862
1863 return HRTIMER_RESTART;
1864}
1865
1866/* PMU sysfs attributes */
1867
1868/*
1869 * There are no default events, but we need to create "events" group (with
1870 * empty attrs) before updating it with detected events.
1871 */
1872static struct attribute *attrs_empty[] = {
1873 NULL,
1874};
1875
1876static struct attribute_group pmu_events_group = {
1877 .name = "events",
1878 .attrs = attrs_empty,
1879};
1880
1881static ssize_t cpumask_show(struct device *dev,
1882 struct device_attribute *attr, char *buf)
1883{
1884 struct rapl_package *rp;
1885 cpumask_var_t cpu_mask;
1886 int cpu;
1887 int ret;
1888
1889 if (!alloc_cpumask_var(mask: &cpu_mask, GFP_KERNEL))
1890 return -ENOMEM;
1891
1892 cpus_read_lock();
1893
1894 cpumask_clear(dstp: cpu_mask);
1895
1896 /* Choose a cpu for each RAPL Package */
1897 list_for_each_entry(rp, &rapl_packages, plist) {
1898 cpu = get_pmu_cpu(rp);
1899 if (cpu < nr_cpu_ids)
1900 cpumask_set_cpu(cpu, dstp: cpu_mask);
1901 }
1902 cpus_read_unlock();
1903
1904 ret = cpumap_print_to_pagebuf(list: true, buf, mask: cpu_mask);
1905
1906 free_cpumask_var(mask: cpu_mask);
1907
1908 return ret;
1909}
1910
1911static DEVICE_ATTR_RO(cpumask);
1912
1913static struct attribute *pmu_cpumask_attrs[] = {
1914 &dev_attr_cpumask.attr,
1915 NULL
1916};
1917
1918static struct attribute_group pmu_cpumask_group = {
1919 .attrs = pmu_cpumask_attrs,
1920};
1921
1922PMU_FORMAT_ATTR(event, "config:0-7");
1923static struct attribute *pmu_format_attr[] = {
1924 &format_attr_event.attr,
1925 NULL
1926};
1927
1928static struct attribute_group pmu_format_group = {
1929 .name = "format",
1930 .attrs = pmu_format_attr,
1931};
1932
1933static const struct attribute_group *pmu_attr_groups[] = {
1934 &pmu_events_group,
1935 &pmu_cpumask_group,
1936 &pmu_format_group,
1937 NULL
1938};
1939
1940#define RAPL_EVENT_ATTR_STR(_name, v, str) \
1941static struct perf_pmu_events_attr event_attr_##v = { \
1942 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
1943 .event_str = str, \
1944}
1945
1946RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
1947RAPL_EVENT_ATTR_STR(energy-pkg, rapl_pkg, "event=0x02");
1948RAPL_EVENT_ATTR_STR(energy-ram, rapl_ram, "event=0x03");
1949RAPL_EVENT_ATTR_STR(energy-gpu, rapl_gpu, "event=0x04");
1950RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05");
1951
1952RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_unit_cores, "Joules");
1953RAPL_EVENT_ATTR_STR(energy-pkg.unit, rapl_unit_pkg, "Joules");
1954RAPL_EVENT_ATTR_STR(energy-ram.unit, rapl_unit_ram, "Joules");
1955RAPL_EVENT_ATTR_STR(energy-gpu.unit, rapl_unit_gpu, "Joules");
1956RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_unit_psys, "Joules");
1957
1958RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_scale_cores, "2.3283064365386962890625e-10");
1959RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_scale_pkg, "2.3283064365386962890625e-10");
1960RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_scale_ram, "2.3283064365386962890625e-10");
1961RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_scale_gpu, "2.3283064365386962890625e-10");
1962RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_scale_psys, "2.3283064365386962890625e-10");
1963
1964#define RAPL_EVENT_GROUP(_name, domain) \
1965static struct attribute *pmu_attr_##_name[] = { \
1966 &event_attr_rapl_##_name.attr.attr, \
1967 &event_attr_rapl_unit_##_name.attr.attr, \
1968 &event_attr_rapl_scale_##_name.attr.attr, \
1969 NULL \
1970}; \
1971static umode_t is_visible_##_name(struct kobject *kobj, struct attribute *attr, int event) \
1972{ \
1973 return rapl_pmu.domain_map & BIT(domain) ? attr->mode : 0; \
1974} \
1975static struct attribute_group pmu_group_##_name = { \
1976 .name = "events", \
1977 .attrs = pmu_attr_##_name, \
1978 .is_visible = is_visible_##_name, \
1979}
1980
1981RAPL_EVENT_GROUP(cores, RAPL_DOMAIN_PP0);
1982RAPL_EVENT_GROUP(pkg, RAPL_DOMAIN_PACKAGE);
1983RAPL_EVENT_GROUP(ram, RAPL_DOMAIN_DRAM);
1984RAPL_EVENT_GROUP(gpu, RAPL_DOMAIN_PP1);
1985RAPL_EVENT_GROUP(psys, RAPL_DOMAIN_PLATFORM);
1986
1987static const struct attribute_group *pmu_attr_update[] = {
1988 &pmu_group_cores,
1989 &pmu_group_pkg,
1990 &pmu_group_ram,
1991 &pmu_group_gpu,
1992 &pmu_group_psys,
1993 NULL
1994};
1995
1996static int rapl_pmu_update(struct rapl_package *rp)
1997{
1998 int ret = 0;
1999
2000 /* Return if PMU already covers all events supported by current RAPL Package */
2001 if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map)))
2002 goto end;
2003
2004 /* Unregister previous registered PMU */
2005 if (rapl_pmu.registered)
2006 perf_pmu_unregister(pmu: &rapl_pmu.pmu);
2007
2008 rapl_pmu.registered = false;
2009 rapl_pmu.domain_map |= rp->domain_map;
2010
2011 memset(&rapl_pmu.pmu, 0, sizeof(struct pmu));
2012 rapl_pmu.pmu.attr_groups = pmu_attr_groups;
2013 rapl_pmu.pmu.attr_update = pmu_attr_update;
2014 rapl_pmu.pmu.task_ctx_nr = perf_invalid_context;
2015 rapl_pmu.pmu.event_init = rapl_pmu_event_init;
2016 rapl_pmu.pmu.add = rapl_pmu_event_add;
2017 rapl_pmu.pmu.del = rapl_pmu_event_del;
2018 rapl_pmu.pmu.start = rapl_pmu_event_start;
2019 rapl_pmu.pmu.stop = rapl_pmu_event_stop;
2020 rapl_pmu.pmu.read = rapl_pmu_event_read;
2021 rapl_pmu.pmu.module = THIS_MODULE;
2022 rapl_pmu.pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT;
2023 ret = perf_pmu_register(pmu: &rapl_pmu.pmu, name: "power", type: -1);
2024 if (ret) {
2025 pr_info("Failed to register PMU\n");
2026 return ret;
2027 }
2028
2029 rapl_pmu.registered = true;
2030end:
2031 rp->has_pmu = true;
2032 return ret;
2033}
2034
2035int rapl_package_add_pmu_locked(struct rapl_package *rp)
2036{
2037 struct rapl_package_pmu_data *data = &rp->pmu_data;
2038 int idx;
2039
2040 if (rp->has_pmu)
2041 return -EEXIST;
2042
2043 for (idx = 0; idx < rp->nr_domains; idx++) {
2044 struct rapl_domain *rd = &rp->domains[idx];
2045 int domain = rd->id;
2046 u64 val;
2047
2048 if (!test_bit(domain, &rp->domain_map))
2049 continue;
2050
2051 /*
2052 * The RAPL PMU granularity is 2^-32 Joules
2053 * data->scale[]: times of 2^-32 Joules for each ENERGY COUNTER increase
2054 */
2055 val = rd->energy_unit * (1ULL << 32);
2056 do_div(val, ENERGY_UNIT_SCALE * 1000000);
2057 data->scale[domain] = val;
2058
2059 if (!rapl_pmu.timer_ms) {
2060 struct rapl_primitive_info *rpi = get_rpi(rp, prim: ENERGY_COUNTER);
2061
2062 /*
2063 * Calculate the timer rate:
2064 * Use reference of 200W for scaling the timeout to avoid counter
2065 * overflows.
2066 *
2067 * max_count = rpi->mask >> rpi->shift + 1
2068 * max_energy_pj = max_count * rd->energy_unit
2069 * max_time_sec = (max_energy_pj / 1000000000) / 200w
2070 *
2071 * rapl_pmu.timer_ms = max_time_sec * 1000 / 2
2072 */
2073 val = (rpi->mask >> rpi->shift) + 1;
2074 val *= rd->energy_unit;
2075 do_div(val, 1000000 * 200 * 2);
2076 rapl_pmu.timer_ms = val;
2077
2078 pr_debug("%llu ms overflow timer\n", rapl_pmu.timer_ms);
2079 }
2080
2081 pr_debug("Domain %s: hw unit %lld * 2^-32 Joules\n", rd->name, data->scale[domain]);
2082 }
2083
2084 /* Initialize per package PMU data */
2085 raw_spin_lock_init(&data->lock);
2086 INIT_LIST_HEAD(list: &data->active_list);
2087 data->timer_interval = ms_to_ktime(ms: rapl_pmu.timer_ms);
2088 hrtimer_setup(timer: &data->hrtimer, function: rapl_hrtimer_handle, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
2089
2090 return rapl_pmu_update(rp);
2091}
2092EXPORT_SYMBOL_GPL(rapl_package_add_pmu_locked);
2093
2094int rapl_package_add_pmu(struct rapl_package *rp)
2095{
2096 guard(cpus_read_lock)();
2097
2098 return rapl_package_add_pmu_locked(rp);
2099}
2100EXPORT_SYMBOL_GPL(rapl_package_add_pmu);
2101
2102void rapl_package_remove_pmu_locked(struct rapl_package *rp)
2103{
2104 struct rapl_package *pos;
2105
2106 if (!rp->has_pmu)
2107 return;
2108
2109 list_for_each_entry(pos, &rapl_packages, plist) {
2110 /* PMU is still needed */
2111 if (pos->has_pmu && pos != rp)
2112 return;
2113 }
2114
2115 perf_pmu_unregister(pmu: &rapl_pmu.pmu);
2116 memset(&rapl_pmu, 0, sizeof(struct rapl_pmu));
2117}
2118EXPORT_SYMBOL_GPL(rapl_package_remove_pmu_locked);
2119
2120void rapl_package_remove_pmu(struct rapl_package *rp)
2121{
2122 guard(cpus_read_lock)();
2123
2124 rapl_package_remove_pmu_locked(rp);
2125}
2126EXPORT_SYMBOL_GPL(rapl_package_remove_pmu);
2127#endif
2128
2129/* called from CPU hotplug notifier, hotplug lock held */
2130void rapl_remove_package_cpuslocked(struct rapl_package *rp)
2131{
2132 struct rapl_domain *rd, *rd_package = NULL;
2133
2134 package_power_limit_irq_restore(rp);
2135
2136 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
2137 int i;
2138
2139 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
2140 rapl_write_pl_data(rd, pl: i, pl_prim: PL_ENABLE, value: 0);
2141 rapl_write_pl_data(rd, pl: i, pl_prim: PL_CLAMP, value: 0);
2142 }
2143
2144 if (rd->id == RAPL_DOMAIN_PACKAGE) {
2145 rd_package = rd;
2146 continue;
2147 }
2148 pr_debug("remove package, undo power limit on %s: %s\n",
2149 rp->name, rd->name);
2150 powercap_unregister_zone(control_type: rp->priv->control_type,
2151 power_zone: &rd->power_zone);
2152 }
2153 /* do parent zone last */
2154 powercap_unregister_zone(control_type: rp->priv->control_type,
2155 power_zone: &rd_package->power_zone);
2156 list_del(entry: &rp->plist);
2157 kfree(objp: rp);
2158}
2159EXPORT_SYMBOL_GPL(rapl_remove_package_cpuslocked);
2160
2161void rapl_remove_package(struct rapl_package *rp)
2162{
2163 guard(cpus_read_lock)();
2164 rapl_remove_package_cpuslocked(rp);
2165}
2166EXPORT_SYMBOL_GPL(rapl_remove_package);
2167
2168/*
2169 * RAPL Package energy counter scope:
2170 * 1. AMD/HYGON platforms use per-PKG package energy counter
2171 * 2. For Intel platforms
2172 * 2.1 CLX-AP platform has per-DIE package energy counter
2173 * 2.2 Other platforms that uses MSR RAPL are single die systems so the
2174 * package energy counter can be considered as per-PKG/per-DIE,
2175 * here it is considered as per-DIE.
2176 * 2.3 New platforms that use TPMI RAPL doesn't care about the
2177 * scope because they are not MSR/CPU based.
2178 */
2179#define rapl_msrs_are_pkg_scope() \
2180 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || \
2181 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
2182
2183/* caller to ensure CPU hotplug lock is held */
2184struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv,
2185 bool id_is_cpu)
2186{
2187 struct rapl_package *rp;
2188 int uid;
2189
2190 if (id_is_cpu) {
2191 uid = rapl_msrs_are_pkg_scope() ?
2192 topology_physical_package_id(id) : topology_logical_die_id(id);
2193 if (uid < 0) {
2194 pr_err("topology_logical_(package/die)_id() returned a negative value");
2195 return NULL;
2196 }
2197 }
2198 else
2199 uid = id;
2200
2201 list_for_each_entry(rp, &rapl_packages, plist) {
2202 if (rp->id == uid
2203 && rp->priv->control_type == priv->control_type)
2204 return rp;
2205 }
2206
2207 return NULL;
2208}
2209EXPORT_SYMBOL_GPL(rapl_find_package_domain_cpuslocked);
2210
2211struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu)
2212{
2213 guard(cpus_read_lock)();
2214 return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu);
2215}
2216EXPORT_SYMBOL_GPL(rapl_find_package_domain);
2217
2218/* called from CPU hotplug notifier, hotplug lock held */
2219struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu)
2220{
2221 struct rapl_package *rp;
2222 int ret;
2223
2224 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
2225 if (!rp)
2226 return ERR_PTR(error: -ENOMEM);
2227
2228 if (id_is_cpu) {
2229 rp->id = rapl_msrs_are_pkg_scope() ?
2230 topology_physical_package_id(id) : topology_logical_die_id(id);
2231 if ((int)(rp->id) < 0) {
2232 pr_err("topology_logical_(package/die)_id() returned a negative value");
2233 return ERR_PTR(error: -EINVAL);
2234 }
2235 rp->lead_cpu = id;
2236 if (!rapl_msrs_are_pkg_scope() && topology_max_dies_per_package() > 1)
2237 snprintf(buf: rp->name, PACKAGE_DOMAIN_NAME_LENGTH, fmt: "package-%d-die-%d",
2238 topology_physical_package_id(id), topology_die_id(id));
2239 else
2240 snprintf(buf: rp->name, PACKAGE_DOMAIN_NAME_LENGTH, fmt: "package-%d",
2241 topology_physical_package_id(id));
2242 } else {
2243 rp->id = id;
2244 rp->lead_cpu = -1;
2245 snprintf(buf: rp->name, PACKAGE_DOMAIN_NAME_LENGTH, fmt: "package-%d", id);
2246 }
2247
2248 rp->priv = priv;
2249 ret = rapl_config(rp);
2250 if (ret)
2251 goto err_free_package;
2252
2253 /* check if the package contains valid domains */
2254 if (rapl_detect_domains(rp)) {
2255 ret = -ENODEV;
2256 goto err_free_package;
2257 }
2258 ret = rapl_package_register_powercap(rp);
2259 if (!ret) {
2260 INIT_LIST_HEAD(list: &rp->plist);
2261 list_add(new: &rp->plist, head: &rapl_packages);
2262 return rp;
2263 }
2264
2265err_free_package:
2266 kfree(objp: rp->domains);
2267 kfree(objp: rp);
2268 return ERR_PTR(error: ret);
2269}
2270EXPORT_SYMBOL_GPL(rapl_add_package_cpuslocked);
2271
2272struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu)
2273{
2274 guard(cpus_read_lock)();
2275 return rapl_add_package_cpuslocked(id, priv, id_is_cpu);
2276}
2277EXPORT_SYMBOL_GPL(rapl_add_package);
2278
2279static void power_limit_state_save(void)
2280{
2281 struct rapl_package *rp;
2282 struct rapl_domain *rd;
2283 int ret, i;
2284
2285 cpus_read_lock();
2286 list_for_each_entry(rp, &rapl_packages, plist) {
2287 if (!rp->power_zone)
2288 continue;
2289 rd = power_zone_to_rapl_domain(rp->power_zone);
2290 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
2291 ret = rapl_read_pl_data(rd, pl: i, pl_prim: PL_LIMIT, xlate: true,
2292 data: &rd->rpl[i].last_power_limit);
2293 if (ret)
2294 rd->rpl[i].last_power_limit = 0;
2295 }
2296 }
2297 cpus_read_unlock();
2298}
2299
2300static void power_limit_state_restore(void)
2301{
2302 struct rapl_package *rp;
2303 struct rapl_domain *rd;
2304 int i;
2305
2306 cpus_read_lock();
2307 list_for_each_entry(rp, &rapl_packages, plist) {
2308 if (!rp->power_zone)
2309 continue;
2310 rd = power_zone_to_rapl_domain(rp->power_zone);
2311 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++)
2312 if (rd->rpl[i].last_power_limit)
2313 rapl_write_pl_data(rd, pl: i, pl_prim: PL_LIMIT,
2314 value: rd->rpl[i].last_power_limit);
2315 }
2316 cpus_read_unlock();
2317}
2318
2319static int rapl_pm_callback(struct notifier_block *nb,
2320 unsigned long mode, void *_unused)
2321{
2322 switch (mode) {
2323 case PM_SUSPEND_PREPARE:
2324 power_limit_state_save();
2325 break;
2326 case PM_POST_SUSPEND:
2327 power_limit_state_restore();
2328 break;
2329 }
2330 return NOTIFY_OK;
2331}
2332
2333static struct notifier_block rapl_pm_notifier = {
2334 .notifier_call = rapl_pm_callback,
2335};
2336
2337static struct platform_device *rapl_msr_platdev;
2338
2339static int __init rapl_init(void)
2340{
2341 const struct x86_cpu_id *id;
2342 int ret;
2343
2344 id = x86_match_cpu(match: rapl_ids);
2345 if (id) {
2346 defaults_msr = (struct rapl_defaults *)id->driver_data;
2347
2348 rapl_msr_platdev = platform_device_alloc(name: "intel_rapl_msr", id: 0);
2349 if (!rapl_msr_platdev)
2350 return -ENOMEM;
2351
2352 ret = platform_device_add(pdev: rapl_msr_platdev);
2353 if (ret) {
2354 platform_device_put(pdev: rapl_msr_platdev);
2355 return ret;
2356 }
2357 }
2358
2359 ret = register_pm_notifier(nb: &rapl_pm_notifier);
2360 if (ret && rapl_msr_platdev) {
2361 platform_device_del(pdev: rapl_msr_platdev);
2362 platform_device_put(pdev: rapl_msr_platdev);
2363 }
2364
2365 return ret;
2366}
2367
2368static void __exit rapl_exit(void)
2369{
2370 platform_device_unregister(rapl_msr_platdev);
2371 unregister_pm_notifier(nb: &rapl_pm_notifier);
2372}
2373
2374fs_initcall(rapl_init);
2375module_exit(rapl_exit);
2376
2377MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code");
2378MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
2379MODULE_LICENSE("GPL v2");
2380

source code of linux/drivers/powercap/intel_rapl_common.c