1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2017-2018 Intel Corporation
5 */
6
7#include <linux/pm_runtime.h>
8
9#include <drm/drm_print.h>
10
11#include "gt/intel_engine.h"
12#include "gt/intel_engine_pm.h"
13#include "gt/intel_engine_regs.h"
14#include "gt/intel_engine_user.h"
15#include "gt/intel_gt.h"
16#include "gt/intel_gt_pm.h"
17#include "gt/intel_gt_regs.h"
18#include "gt/intel_rc6.h"
19#include "gt/intel_rps.h"
20
21#include "i915_drv.h"
22#include "i915_pmu.h"
23
24/* Frequency for the sampling timer for events which need it. */
25#define FREQUENCY 200
26#define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
27
28#define ENGINE_SAMPLE_MASK \
29 (BIT(I915_SAMPLE_BUSY) | \
30 BIT(I915_SAMPLE_WAIT) | \
31 BIT(I915_SAMPLE_SEMA))
32
33static struct i915_pmu *event_to_pmu(struct perf_event *event)
34{
35 return container_of(event->pmu, struct i915_pmu, base);
36}
37
38static struct drm_i915_private *pmu_to_i915(struct i915_pmu *pmu)
39{
40 return container_of(pmu, struct drm_i915_private, pmu);
41}
42
43static u8 engine_config_sample(u64 config)
44{
45 return config & I915_PMU_SAMPLE_MASK;
46}
47
48static u8 engine_event_sample(struct perf_event *event)
49{
50 return engine_config_sample(config: event->attr.config);
51}
52
53static u8 engine_event_class(struct perf_event *event)
54{
55 return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
56}
57
58static u8 engine_event_instance(struct perf_event *event)
59{
60 return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
61}
62
63static bool is_engine_config(const u64 config)
64{
65 return config < __I915_PMU_OTHER(0);
66}
67
68static unsigned int config_gt_id(const u64 config)
69{
70 return config >> __I915_PMU_GT_SHIFT;
71}
72
73static u64 config_counter(const u64 config)
74{
75 return config & ~(~0ULL << __I915_PMU_GT_SHIFT);
76}
77
78static unsigned int other_bit(const u64 config)
79{
80 unsigned int val;
81
82 switch (config_counter(config)) {
83 case I915_PMU_ACTUAL_FREQUENCY:
84 val = __I915_PMU_ACTUAL_FREQUENCY_ENABLED;
85 break;
86 case I915_PMU_REQUESTED_FREQUENCY:
87 val = __I915_PMU_REQUESTED_FREQUENCY_ENABLED;
88 break;
89 case I915_PMU_RC6_RESIDENCY:
90 val = __I915_PMU_RC6_RESIDENCY_ENABLED;
91 break;
92 default:
93 /*
94 * Events that do not require sampling, or tracking state
95 * transitions between enabled and disabled can be ignored.
96 */
97 return -1;
98 }
99
100 return I915_ENGINE_SAMPLE_COUNT +
101 config_gt_id(config) * __I915_PMU_TRACKED_EVENT_COUNT +
102 val;
103}
104
105static unsigned int config_bit(const u64 config)
106{
107 if (is_engine_config(config))
108 return engine_config_sample(config);
109 else
110 return other_bit(config);
111}
112
113static __always_inline u32 config_mask(const u64 config)
114{
115 unsigned int bit = config_bit(config);
116
117 if (__builtin_constant_p(bit))
118 BUILD_BUG_ON(bit >
119 BITS_PER_TYPE(typeof_member(struct i915_pmu,
120 enable)) - 1);
121 else
122 WARN_ON_ONCE(bit >
123 BITS_PER_TYPE(typeof_member(struct i915_pmu,
124 enable)) - 1);
125
126 return BIT(bit);
127}
128
129static bool is_engine_event(struct perf_event *event)
130{
131 return is_engine_config(config: event->attr.config);
132}
133
134static unsigned int event_bit(struct perf_event *event)
135{
136 return config_bit(config: event->attr.config);
137}
138
139static u32 frequency_enabled_mask(void)
140{
141 unsigned int i;
142 u32 mask = 0;
143
144 for (i = 0; i < I915_PMU_MAX_GT; i++)
145 mask |= config_mask(__I915_PMU_ACTUAL_FREQUENCY(i)) |
146 config_mask(__I915_PMU_REQUESTED_FREQUENCY(i));
147
148 return mask;
149}
150
151static bool pmu_needs_timer(struct i915_pmu *pmu)
152{
153 struct drm_i915_private *i915 = pmu_to_i915(pmu);
154 u32 enable;
155
156 /*
157 * Only some counters need the sampling timer.
158 *
159 * We start with a bitmask of all currently enabled events.
160 */
161 enable = pmu->enable;
162
163 /*
164 * Mask out all the ones which do not need the timer, or in
165 * other words keep all the ones that could need the timer.
166 */
167 enable &= frequency_enabled_mask() | ENGINE_SAMPLE_MASK;
168
169 /*
170 * Also there is software busyness tracking available we do not
171 * need the timer for I915_SAMPLE_BUSY counter.
172 */
173 if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS)
174 enable &= ~BIT(I915_SAMPLE_BUSY);
175
176 /*
177 * If some bits remain it means we need the sampling timer running.
178 */
179 return enable;
180}
181
182static u64 __get_rc6(struct intel_gt *gt)
183{
184 struct drm_i915_private *i915 = gt->i915;
185 u64 val;
186
187 val = intel_rc6_residency_ns(rc6: &gt->rc6, id: INTEL_RC6_RES_RC6);
188
189 if (HAS_RC6p(i915))
190 val += intel_rc6_residency_ns(rc6: &gt->rc6, id: INTEL_RC6_RES_RC6p);
191
192 if (HAS_RC6pp(i915))
193 val += intel_rc6_residency_ns(rc6: &gt->rc6, id: INTEL_RC6_RES_RC6pp);
194
195 return val;
196}
197
198static inline s64 ktime_since_raw(const ktime_t kt)
199{
200 return ktime_to_ns(ktime_sub(ktime_get_raw(), kt));
201}
202
203static u64 read_sample(struct i915_pmu *pmu, unsigned int gt_id, int sample)
204{
205 return pmu->sample[gt_id][sample].cur;
206}
207
208static void
209store_sample(struct i915_pmu *pmu, unsigned int gt_id, int sample, u64 val)
210{
211 pmu->sample[gt_id][sample].cur = val;
212}
213
214static void
215add_sample_mult(struct i915_pmu *pmu, unsigned int gt_id, int sample, u32 val, u32 mul)
216{
217 pmu->sample[gt_id][sample].cur += mul_u32_u32(a: val, b: mul);
218}
219
220static u64 get_rc6(struct intel_gt *gt)
221{
222 struct drm_i915_private *i915 = gt->i915;
223 const unsigned int gt_id = gt->info.id;
224 struct i915_pmu *pmu = &i915->pmu;
225 intel_wakeref_t wakeref;
226 unsigned long flags;
227 u64 val;
228
229 wakeref = intel_gt_pm_get_if_awake(gt);
230 if (wakeref) {
231 val = __get_rc6(gt);
232 intel_gt_pm_put_async(gt, handle: wakeref);
233 }
234
235 spin_lock_irqsave(&pmu->lock, flags);
236
237 if (wakeref) {
238 store_sample(pmu, gt_id, sample: __I915_SAMPLE_RC6, val);
239 } else {
240 /*
241 * We think we are runtime suspended.
242 *
243 * Report the delta from when the device was suspended to now,
244 * on top of the last known real value, as the approximated RC6
245 * counter value.
246 */
247 val = ktime_since_raw(kt: pmu->sleep_last[gt_id]);
248 val += read_sample(pmu, gt_id, sample: __I915_SAMPLE_RC6);
249 }
250
251 if (val < read_sample(pmu, gt_id, sample: __I915_SAMPLE_RC6_LAST_REPORTED))
252 val = read_sample(pmu, gt_id, sample: __I915_SAMPLE_RC6_LAST_REPORTED);
253 else
254 store_sample(pmu, gt_id, sample: __I915_SAMPLE_RC6_LAST_REPORTED, val);
255
256 spin_unlock_irqrestore(lock: &pmu->lock, flags);
257
258 return val;
259}
260
261static void init_rc6(struct i915_pmu *pmu)
262{
263 struct drm_i915_private *i915 = pmu_to_i915(pmu);
264 struct intel_gt *gt;
265 unsigned int i;
266
267 for_each_gt(gt, i915, i) {
268 intel_wakeref_t wakeref;
269
270 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
271 u64 val = __get_rc6(gt);
272
273 store_sample(pmu, gt_id: i, sample: __I915_SAMPLE_RC6, val);
274 store_sample(pmu, gt_id: i, sample: __I915_SAMPLE_RC6_LAST_REPORTED,
275 val);
276 pmu->sleep_last[i] = ktime_get_raw();
277 }
278 }
279}
280
281static void park_rc6(struct intel_gt *gt)
282{
283 struct i915_pmu *pmu = &gt->i915->pmu;
284
285 store_sample(pmu, gt_id: gt->info.id, sample: __I915_SAMPLE_RC6, val: __get_rc6(gt));
286 pmu->sleep_last[gt->info.id] = ktime_get_raw();
287}
288
289static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu)
290{
291 if (!pmu->timer_enabled && pmu_needs_timer(pmu)) {
292 pmu->timer_enabled = true;
293 pmu->timer_last = ktime_get();
294 hrtimer_start_range_ns(timer: &pmu->timer,
295 tim: ns_to_ktime(PERIOD), range_ns: 0,
296 mode: HRTIMER_MODE_REL_PINNED);
297 }
298}
299
300void i915_pmu_gt_parked(struct intel_gt *gt)
301{
302 struct i915_pmu *pmu = &gt->i915->pmu;
303
304 if (!pmu->registered)
305 return;
306
307 spin_lock_irq(lock: &pmu->lock);
308
309 park_rc6(gt);
310
311 /*
312 * Signal sampling timer to stop if only engine events are enabled and
313 * GPU went idle.
314 */
315 pmu->unparked &= ~BIT(gt->info.id);
316 if (pmu->unparked == 0)
317 pmu->timer_enabled = false;
318
319 spin_unlock_irq(lock: &pmu->lock);
320}
321
322void i915_pmu_gt_unparked(struct intel_gt *gt)
323{
324 struct i915_pmu *pmu = &gt->i915->pmu;
325
326 if (!pmu->registered)
327 return;
328
329 spin_lock_irq(lock: &pmu->lock);
330
331 /*
332 * Re-enable sampling timer when GPU goes active.
333 */
334 if (pmu->unparked == 0)
335 __i915_pmu_maybe_start_timer(pmu);
336
337 pmu->unparked |= BIT(gt->info.id);
338
339 spin_unlock_irq(lock: &pmu->lock);
340}
341
342static void
343add_sample(struct i915_pmu_sample *sample, u32 val)
344{
345 sample->cur += val;
346}
347
348static bool exclusive_mmio_access(const struct drm_i915_private *i915)
349{
350 /*
351 * We have to avoid concurrent mmio cache line access on gen7 or
352 * risk a machine hang. For a fun history lesson dig out the old
353 * userspace intel_gpu_top and run it on Ivybridge or Haswell!
354 */
355 return GRAPHICS_VER(i915) == 7;
356}
357
358static void gen3_engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
359{
360 struct intel_engine_pmu *pmu = &engine->pmu;
361 bool busy;
362 u32 val;
363
364 val = ENGINE_READ_FW(engine, RING_CTL);
365 if (val == 0) /* powerwell off => engine idle */
366 return;
367
368 if (val & RING_WAIT)
369 add_sample(sample: &pmu->sample[I915_SAMPLE_WAIT], val: period_ns);
370 if (val & RING_WAIT_SEMAPHORE)
371 add_sample(sample: &pmu->sample[I915_SAMPLE_SEMA], val: period_ns);
372
373 /* No need to sample when busy stats are supported. */
374 if (intel_engine_supports_stats(engine))
375 return;
376
377 /*
378 * While waiting on a semaphore or event, MI_MODE reports the
379 * ring as idle. However, previously using the seqno, and with
380 * execlists sampling, we account for the ring waiting as the
381 * engine being busy. Therefore, we record the sample as being
382 * busy if either waiting or !idle.
383 */
384 busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
385 if (!busy) {
386 val = ENGINE_READ_FW(engine, RING_MI_MODE);
387 busy = !(val & MODE_IDLE);
388 }
389 if (busy)
390 add_sample(sample: &pmu->sample[I915_SAMPLE_BUSY], val: period_ns);
391}
392
393static void gen2_engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
394{
395 struct intel_engine_pmu *pmu = &engine->pmu;
396 u32 tail, head, acthd;
397
398 tail = ENGINE_READ_FW(engine, RING_TAIL);
399 head = ENGINE_READ_FW(engine, RING_HEAD);
400 acthd = ENGINE_READ_FW(engine, ACTHD);
401
402 if (head & HEAD_WAIT_I8XX)
403 add_sample(sample: &pmu->sample[I915_SAMPLE_WAIT], val: period_ns);
404
405 if (head & HEAD_WAIT_I8XX || head != acthd ||
406 (head & HEAD_ADDR) != (tail & TAIL_ADDR))
407 add_sample(sample: &pmu->sample[I915_SAMPLE_BUSY], val: period_ns);
408}
409
410static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
411{
412 if (GRAPHICS_VER(engine->i915) >= 3)
413 gen3_engine_sample(engine, period_ns);
414 else
415 gen2_engine_sample(engine, period_ns);
416}
417
418static void
419engines_sample(struct intel_gt *gt, unsigned int period_ns)
420{
421 struct drm_i915_private *i915 = gt->i915;
422 struct intel_engine_cs *engine;
423 enum intel_engine_id id;
424 unsigned long flags;
425
426 if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
427 return;
428
429 if (!intel_gt_pm_is_awake(gt))
430 return;
431
432 for_each_engine(engine, gt, id) {
433 if (!engine->pmu.enable)
434 continue;
435
436 if (!intel_engine_pm_get_if_awake(engine))
437 continue;
438
439 if (exclusive_mmio_access(i915)) {
440 spin_lock_irqsave(&engine->uncore->lock, flags);
441 engine_sample(engine, period_ns);
442 spin_unlock_irqrestore(lock: &engine->uncore->lock, flags);
443 } else {
444 engine_sample(engine, period_ns);
445 }
446
447 intel_engine_pm_put_async(engine);
448 }
449}
450
451static bool
452frequency_sampling_enabled(struct i915_pmu *pmu, unsigned int gt)
453{
454 return pmu->enable &
455 (config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt)) |
456 config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt)));
457}
458
459static void
460frequency_sample(struct intel_gt *gt, unsigned int period_ns)
461{
462 struct drm_i915_private *i915 = gt->i915;
463 const unsigned int gt_id = gt->info.id;
464 struct i915_pmu *pmu = &i915->pmu;
465 struct intel_rps *rps = &gt->rps;
466 intel_wakeref_t wakeref;
467
468 if (!frequency_sampling_enabled(pmu, gt: gt_id))
469 return;
470
471 /* Report 0/0 (actual/requested) frequency while parked. */
472 wakeref = intel_gt_pm_get_if_awake(gt);
473 if (!wakeref)
474 return;
475
476 if (pmu->enable & config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt_id))) {
477 u32 val;
478
479 /*
480 * We take a quick peek here without using forcewake
481 * so that we don't perturb the system under observation
482 * (forcewake => !rc6 => increased power use). We expect
483 * that if the read fails because it is outside of the
484 * mmio power well, then it will return 0 -- in which
485 * case we assume the system is running at the intended
486 * frequency. Fortunately, the read should rarely fail!
487 */
488 val = intel_rps_read_actual_frequency_fw(rps);
489 if (!val)
490 val = intel_gpu_freq(rps, val: rps->cur_freq);
491
492 add_sample_mult(pmu, gt_id, sample: __I915_SAMPLE_FREQ_ACT,
493 val, mul: period_ns / 1000);
494 }
495
496 if (pmu->enable & config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt_id))) {
497 add_sample_mult(pmu, gt_id, sample: __I915_SAMPLE_FREQ_REQ,
498 val: intel_rps_get_requested_frequency(rps),
499 mul: period_ns / 1000);
500 }
501
502 intel_gt_pm_put_async(gt, handle: wakeref);
503}
504
505static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
506{
507 struct i915_pmu *pmu = container_of(hrtimer, struct i915_pmu, timer);
508 struct drm_i915_private *i915 = pmu_to_i915(pmu);
509 unsigned int period_ns;
510 struct intel_gt *gt;
511 unsigned int i;
512 ktime_t now;
513
514 if (!READ_ONCE(pmu->timer_enabled))
515 return HRTIMER_NORESTART;
516
517 now = ktime_get();
518 period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last));
519 pmu->timer_last = now;
520
521 /*
522 * Strictly speaking the passed in period may not be 100% accurate for
523 * all internal calculation, since some amount of time can be spent on
524 * grabbing the forcewake. However the potential error from timer call-
525 * back delay greatly dominates this so we keep it simple.
526 */
527
528 for_each_gt(gt, i915, i) {
529 if (!(pmu->unparked & BIT(i)))
530 continue;
531
532 engines_sample(gt, period_ns);
533 frequency_sample(gt, period_ns);
534 }
535
536 hrtimer_forward(timer: hrtimer, now, interval: ns_to_ktime(PERIOD));
537
538 return HRTIMER_RESTART;
539}
540
541static void i915_pmu_event_destroy(struct perf_event *event)
542{
543 struct i915_pmu *pmu = event_to_pmu(event);
544 struct drm_i915_private *i915 = pmu_to_i915(pmu);
545
546 drm_WARN_ON(&i915->drm, event->parent);
547
548 drm_dev_put(dev: &i915->drm);
549}
550
551static int
552engine_event_status(struct intel_engine_cs *engine,
553 enum drm_i915_pmu_engine_sample sample)
554{
555 switch (sample) {
556 case I915_SAMPLE_BUSY:
557 case I915_SAMPLE_WAIT:
558 break;
559 case I915_SAMPLE_SEMA:
560 if (GRAPHICS_VER(engine->i915) < 6)
561 return -ENODEV;
562 break;
563 default:
564 return -ENOENT;
565 }
566
567 return 0;
568}
569
570static int
571config_status(struct drm_i915_private *i915, u64 config)
572{
573 struct intel_gt *gt = to_gt(i915);
574
575 unsigned int gt_id = config_gt_id(config);
576 unsigned int max_gt_id = HAS_EXTRA_GT_LIST(i915) ? 1 : 0;
577
578 if (gt_id > max_gt_id)
579 return -ENOENT;
580
581 switch (config_counter(config)) {
582 case I915_PMU_ACTUAL_FREQUENCY:
583 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
584 /* Requires a mutex for sampling! */
585 return -ENODEV;
586 fallthrough;
587 case I915_PMU_REQUESTED_FREQUENCY:
588 if (GRAPHICS_VER(i915) < 6)
589 return -ENODEV;
590 break;
591 case I915_PMU_INTERRUPTS:
592 if (gt_id)
593 return -ENOENT;
594 break;
595 case I915_PMU_RC6_RESIDENCY:
596 if (!gt->rc6.supported)
597 return -ENODEV;
598 break;
599 case I915_PMU_SOFTWARE_GT_AWAKE_TIME:
600 break;
601 default:
602 return -ENOENT;
603 }
604
605 return 0;
606}
607
608static int engine_event_init(struct perf_event *event)
609{
610 struct i915_pmu *pmu = event_to_pmu(event);
611 struct drm_i915_private *i915 = pmu_to_i915(pmu);
612 struct intel_engine_cs *engine;
613
614 engine = intel_engine_lookup_user(i915, class: engine_event_class(event),
615 instance: engine_event_instance(event));
616 if (!engine)
617 return -ENODEV;
618
619 return engine_event_status(engine, sample: engine_event_sample(event));
620}
621
622static int i915_pmu_event_init(struct perf_event *event)
623{
624 struct i915_pmu *pmu = event_to_pmu(event);
625 struct drm_i915_private *i915 = pmu_to_i915(pmu);
626 int ret;
627
628 if (!pmu->registered)
629 return -ENODEV;
630
631 if (event->attr.type != event->pmu->type)
632 return -ENOENT;
633
634 /* unsupported modes and filters */
635 if (event->attr.sample_period) /* no sampling */
636 return -EINVAL;
637
638 if (has_branch_stack(event))
639 return -EOPNOTSUPP;
640
641 if (event->cpu < 0)
642 return -EINVAL;
643
644 if (is_engine_event(event))
645 ret = engine_event_init(event);
646 else
647 ret = config_status(i915, config: event->attr.config);
648 if (ret)
649 return ret;
650
651 if (!event->parent) {
652 drm_dev_get(dev: &i915->drm);
653 event->destroy = i915_pmu_event_destroy;
654 }
655
656 return 0;
657}
658
659static u64 __i915_pmu_event_read(struct perf_event *event)
660{
661 struct i915_pmu *pmu = event_to_pmu(event);
662 struct drm_i915_private *i915 = pmu_to_i915(pmu);
663 u64 val = 0;
664
665 if (is_engine_event(event)) {
666 u8 sample = engine_event_sample(event);
667 struct intel_engine_cs *engine;
668
669 engine = intel_engine_lookup_user(i915,
670 class: engine_event_class(event),
671 instance: engine_event_instance(event));
672
673 if (drm_WARN_ON_ONCE(&i915->drm, !engine)) {
674 /* Do nothing */
675 } else if (sample == I915_SAMPLE_BUSY &&
676 intel_engine_supports_stats(engine)) {
677 ktime_t unused;
678
679 val = ktime_to_ns(kt: intel_engine_get_busy_time(engine,
680 now: &unused));
681 } else {
682 val = engine->pmu.sample[sample].cur;
683 }
684 } else {
685 const unsigned int gt_id = config_gt_id(config: event->attr.config);
686 const u64 config = config_counter(config: event->attr.config);
687
688 switch (config) {
689 case I915_PMU_ACTUAL_FREQUENCY:
690 val =
691 div_u64(dividend: read_sample(pmu, gt_id,
692 sample: __I915_SAMPLE_FREQ_ACT),
693 USEC_PER_SEC /* to MHz */);
694 break;
695 case I915_PMU_REQUESTED_FREQUENCY:
696 val =
697 div_u64(dividend: read_sample(pmu, gt_id,
698 sample: __I915_SAMPLE_FREQ_REQ),
699 USEC_PER_SEC /* to MHz */);
700 break;
701 case I915_PMU_INTERRUPTS:
702 val = READ_ONCE(pmu->irq_count);
703 break;
704 case I915_PMU_RC6_RESIDENCY:
705 val = get_rc6(gt: i915->gt[gt_id]);
706 break;
707 case I915_PMU_SOFTWARE_GT_AWAKE_TIME:
708 val = ktime_to_ns(kt: intel_gt_get_awake_time(gt: to_gt(i915)));
709 break;
710 }
711 }
712
713 return val;
714}
715
716static void i915_pmu_event_read(struct perf_event *event)
717{
718 struct i915_pmu *pmu = event_to_pmu(event);
719 struct hw_perf_event *hwc = &event->hw;
720 u64 prev, new;
721
722 if (!pmu->registered) {
723 event->hw.state = PERF_HES_STOPPED;
724 return;
725 }
726
727 prev = local64_read(&hwc->prev_count);
728 do {
729 new = __i915_pmu_event_read(event);
730 } while (!local64_try_cmpxchg(l: &hwc->prev_count, old: &prev, new));
731
732 local64_add(new - prev, &event->count);
733}
734
735static void i915_pmu_enable(struct perf_event *event)
736{
737 struct i915_pmu *pmu = event_to_pmu(event);
738 struct drm_i915_private *i915 = pmu_to_i915(pmu);
739 const unsigned int bit = event_bit(event);
740 unsigned long flags;
741
742 if (bit == -1)
743 goto update;
744
745 spin_lock_irqsave(&pmu->lock, flags);
746
747 /*
748 * Update the bitmask of enabled events and increment
749 * the event reference counter.
750 */
751 BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS);
752 GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
753 GEM_BUG_ON(pmu->enable_count[bit] == ~0);
754
755 pmu->enable |= BIT(bit);
756 pmu->enable_count[bit]++;
757
758 /*
759 * Start the sampling timer if needed and not already enabled.
760 */
761 __i915_pmu_maybe_start_timer(pmu);
762
763 /*
764 * For per-engine events the bitmask and reference counting
765 * is stored per engine.
766 */
767 if (is_engine_event(event)) {
768 u8 sample = engine_event_sample(event);
769 struct intel_engine_cs *engine;
770
771 engine = intel_engine_lookup_user(i915,
772 class: engine_event_class(event),
773 instance: engine_event_instance(event));
774
775 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=
776 I915_ENGINE_SAMPLE_COUNT);
777 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=
778 I915_ENGINE_SAMPLE_COUNT);
779 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
780 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
781 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
782
783 engine->pmu.enable |= BIT(sample);
784 engine->pmu.enable_count[sample]++;
785 }
786
787 spin_unlock_irqrestore(lock: &pmu->lock, flags);
788
789update:
790 /*
791 * Store the current counter value so we can report the correct delta
792 * for all listeners. Even when the event was already enabled and has
793 * an existing non-zero value.
794 */
795 local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
796}
797
798static void i915_pmu_disable(struct perf_event *event)
799{
800 struct i915_pmu *pmu = event_to_pmu(event);
801 struct drm_i915_private *i915 = pmu_to_i915(pmu);
802 const unsigned int bit = event_bit(event);
803 unsigned long flags;
804
805 if (bit == -1)
806 return;
807
808 spin_lock_irqsave(&pmu->lock, flags);
809
810 if (is_engine_event(event)) {
811 u8 sample = engine_event_sample(event);
812 struct intel_engine_cs *engine;
813
814 engine = intel_engine_lookup_user(i915,
815 class: engine_event_class(event),
816 instance: engine_event_instance(event));
817
818 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
819 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
820 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
821
822 /*
823 * Decrement the reference count and clear the enabled
824 * bitmask when the last listener on an event goes away.
825 */
826 if (--engine->pmu.enable_count[sample] == 0)
827 engine->pmu.enable &= ~BIT(sample);
828 }
829
830 GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
831 GEM_BUG_ON(pmu->enable_count[bit] == 0);
832 /*
833 * Decrement the reference count and clear the enabled
834 * bitmask when the last listener on an event goes away.
835 */
836 if (--pmu->enable_count[bit] == 0) {
837 pmu->enable &= ~BIT(bit);
838 pmu->timer_enabled &= pmu_needs_timer(pmu);
839 }
840
841 spin_unlock_irqrestore(lock: &pmu->lock, flags);
842}
843
844static void i915_pmu_event_start(struct perf_event *event, int flags)
845{
846 struct i915_pmu *pmu = event_to_pmu(event);
847
848 if (!pmu->registered)
849 return;
850
851 i915_pmu_enable(event);
852 event->hw.state = 0;
853}
854
855static void i915_pmu_event_stop(struct perf_event *event, int flags)
856{
857 struct i915_pmu *pmu = event_to_pmu(event);
858
859 if (!pmu->registered)
860 goto out;
861
862 if (flags & PERF_EF_UPDATE)
863 i915_pmu_event_read(event);
864
865 i915_pmu_disable(event);
866
867out:
868 event->hw.state = PERF_HES_STOPPED;
869}
870
871static int i915_pmu_event_add(struct perf_event *event, int flags)
872{
873 struct i915_pmu *pmu = event_to_pmu(event);
874
875 if (!pmu->registered)
876 return -ENODEV;
877
878 if (flags & PERF_EF_START)
879 i915_pmu_event_start(event, flags);
880
881 return 0;
882}
883
884static void i915_pmu_event_del(struct perf_event *event, int flags)
885{
886 i915_pmu_event_stop(event, PERF_EF_UPDATE);
887}
888
889struct i915_str_attribute {
890 struct device_attribute attr;
891 const char *str;
892};
893
894static ssize_t i915_pmu_format_show(struct device *dev,
895 struct device_attribute *attr, char *buf)
896{
897 struct i915_str_attribute *eattr;
898
899 eattr = container_of(attr, struct i915_str_attribute, attr);
900 return sysfs_emit(buf, fmt: "%s\n", eattr->str);
901}
902
903#define I915_PMU_FORMAT_ATTR(_name, _config) \
904 (&((struct i915_str_attribute[]) { \
905 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
906 .str = _config, } \
907 })[0].attr.attr)
908
909static struct attribute *i915_pmu_format_attrs[] = {
910 I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
911 NULL,
912};
913
914static const struct attribute_group i915_pmu_format_attr_group = {
915 .name = "format",
916 .attrs = i915_pmu_format_attrs,
917};
918
919struct i915_ext_attribute {
920 struct device_attribute attr;
921 unsigned long val;
922};
923
924static ssize_t i915_pmu_event_show(struct device *dev,
925 struct device_attribute *attr, char *buf)
926{
927 struct i915_ext_attribute *eattr;
928
929 eattr = container_of(attr, struct i915_ext_attribute, attr);
930 return sysfs_emit(buf, fmt: "config=0x%lx\n", eattr->val);
931}
932
933#define __event(__counter, __name, __unit) \
934{ \
935 .counter = (__counter), \
936 .name = (__name), \
937 .unit = (__unit), \
938 .global = false, \
939}
940
941#define __global_event(__counter, __name, __unit) \
942{ \
943 .counter = (__counter), \
944 .name = (__name), \
945 .unit = (__unit), \
946 .global = true, \
947}
948
949#define __engine_event(__sample, __name) \
950{ \
951 .sample = (__sample), \
952 .name = (__name), \
953}
954
955static struct i915_ext_attribute *
956add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
957{
958 sysfs_attr_init(&attr->attr.attr);
959 attr->attr.attr.name = name;
960 attr->attr.attr.mode = 0444;
961 attr->attr.show = i915_pmu_event_show;
962 attr->val = config;
963
964 return ++attr;
965}
966
967static struct perf_pmu_events_attr *
968add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
969 const char *str)
970{
971 sysfs_attr_init(&attr->attr.attr);
972 attr->attr.attr.name = name;
973 attr->attr.attr.mode = 0444;
974 attr->attr.show = perf_event_sysfs_show;
975 attr->event_str = str;
976
977 return ++attr;
978}
979
980static struct attribute **
981create_event_attributes(struct i915_pmu *pmu)
982{
983 struct drm_i915_private *i915 = pmu_to_i915(pmu);
984 static const struct {
985 unsigned int counter;
986 const char *name;
987 const char *unit;
988 bool global;
989 } events[] = {
990 __event(0, "actual-frequency", "M"),
991 __event(1, "requested-frequency", "M"),
992 __global_event(2, "interrupts", NULL),
993 __event(3, "rc6-residency", "ns"),
994 __event(4, "software-gt-awake-time", "ns"),
995 };
996 static const struct {
997 enum drm_i915_pmu_engine_sample sample;
998 char *name;
999 } engine_events[] = {
1000 __engine_event(I915_SAMPLE_BUSY, "busy"),
1001 __engine_event(I915_SAMPLE_SEMA, "sema"),
1002 __engine_event(I915_SAMPLE_WAIT, "wait"),
1003 };
1004 unsigned int count = 0;
1005 struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
1006 struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
1007 struct attribute **attr = NULL, **attr_iter;
1008 struct intel_engine_cs *engine;
1009 struct intel_gt *gt;
1010 unsigned int i, j;
1011
1012 /* Count how many counters we will be exposing. */
1013 for_each_gt(gt, i915, j) {
1014 for (i = 0; i < ARRAY_SIZE(events); i++) {
1015 u64 config = ___I915_PMU_OTHER(j, events[i].counter);
1016
1017 if (!config_status(i915, config))
1018 count++;
1019 }
1020 }
1021
1022 for_each_uabi_engine(engine, i915) {
1023 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
1024 if (!engine_event_status(engine,
1025 sample: engine_events[i].sample))
1026 count++;
1027 }
1028 }
1029
1030 /* Allocate attribute objects and table. */
1031 i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
1032 if (!i915_attr)
1033 goto err_alloc;
1034
1035 pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
1036 if (!pmu_attr)
1037 goto err_alloc;
1038
1039 /* Max one pointer of each attribute type plus a termination entry. */
1040 attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
1041 if (!attr)
1042 goto err_alloc;
1043
1044 i915_iter = i915_attr;
1045 pmu_iter = pmu_attr;
1046 attr_iter = attr;
1047
1048 /* Initialize supported non-engine counters. */
1049 for_each_gt(gt, i915, j) {
1050 for (i = 0; i < ARRAY_SIZE(events); i++) {
1051 u64 config = ___I915_PMU_OTHER(j, events[i].counter);
1052 char *str;
1053
1054 if (config_status(i915, config))
1055 continue;
1056
1057 if (events[i].global || !HAS_EXTRA_GT_LIST(i915))
1058 str = kstrdup(s: events[i].name, GFP_KERNEL);
1059 else
1060 str = kasprintf(GFP_KERNEL, fmt: "%s-gt%u",
1061 events[i].name, j);
1062 if (!str)
1063 goto err;
1064
1065 *attr_iter++ = &i915_iter->attr.attr;
1066 i915_iter = add_i915_attr(attr: i915_iter, name: str, config);
1067
1068 if (events[i].unit) {
1069 if (events[i].global || !HAS_EXTRA_GT_LIST(i915))
1070 str = kasprintf(GFP_KERNEL, fmt: "%s.unit",
1071 events[i].name);
1072 else
1073 str = kasprintf(GFP_KERNEL, fmt: "%s-gt%u.unit",
1074 events[i].name, j);
1075 if (!str)
1076 goto err;
1077
1078 *attr_iter++ = &pmu_iter->attr.attr;
1079 pmu_iter = add_pmu_attr(attr: pmu_iter, name: str,
1080 str: events[i].unit);
1081 }
1082 }
1083 }
1084
1085 /* Initialize supported engine counters. */
1086 for_each_uabi_engine(engine, i915) {
1087 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
1088 char *str;
1089
1090 if (engine_event_status(engine,
1091 sample: engine_events[i].sample))
1092 continue;
1093
1094 str = kasprintf(GFP_KERNEL, fmt: "%s-%s",
1095 engine->name, engine_events[i].name);
1096 if (!str)
1097 goto err;
1098
1099 *attr_iter++ = &i915_iter->attr.attr;
1100 i915_iter =
1101 add_i915_attr(attr: i915_iter, name: str,
1102 __I915_PMU_ENGINE(engine->uabi_class,
1103 engine->uabi_instance,
1104 engine_events[i].sample));
1105
1106 str = kasprintf(GFP_KERNEL, fmt: "%s-%s.unit",
1107 engine->name, engine_events[i].name);
1108 if (!str)
1109 goto err;
1110
1111 *attr_iter++ = &pmu_iter->attr.attr;
1112 pmu_iter = add_pmu_attr(attr: pmu_iter, name: str, str: "ns");
1113 }
1114 }
1115
1116 pmu->i915_attr = i915_attr;
1117 pmu->pmu_attr = pmu_attr;
1118
1119 return attr;
1120
1121err:;
1122 for (attr_iter = attr; *attr_iter; attr_iter++)
1123 kfree(objp: (*attr_iter)->name);
1124
1125err_alloc:
1126 kfree(objp: attr);
1127 kfree(objp: i915_attr);
1128 kfree(objp: pmu_attr);
1129
1130 return NULL;
1131}
1132
1133static void free_event_attributes(struct i915_pmu *pmu)
1134{
1135 struct attribute **attr_iter = pmu->events_attr_group.attrs;
1136
1137 for (; *attr_iter; attr_iter++)
1138 kfree(objp: (*attr_iter)->name);
1139
1140 kfree(objp: pmu->events_attr_group.attrs);
1141 kfree(objp: pmu->i915_attr);
1142 kfree(objp: pmu->pmu_attr);
1143
1144 pmu->events_attr_group.attrs = NULL;
1145 pmu->i915_attr = NULL;
1146 pmu->pmu_attr = NULL;
1147}
1148
1149void i915_pmu_register(struct drm_i915_private *i915)
1150{
1151 struct i915_pmu *pmu = &i915->pmu;
1152 const struct attribute_group *attr_groups[] = {
1153 &i915_pmu_format_attr_group,
1154 &pmu->events_attr_group,
1155 NULL
1156 };
1157 int ret = -ENOMEM;
1158
1159 spin_lock_init(&pmu->lock);
1160 hrtimer_setup(timer: &pmu->timer, function: i915_sample, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
1161 init_rc6(pmu);
1162
1163 if (IS_DGFX(i915)) {
1164 pmu->name = kasprintf(GFP_KERNEL,
1165 fmt: "i915_%s",
1166 dev_name(dev: i915->drm.dev));
1167 if (pmu->name) {
1168 /* tools/perf reserves colons as special. */
1169 strreplace(str: (char *)pmu->name, old: ':', new: '_');
1170 }
1171 } else {
1172 pmu->name = "i915";
1173 }
1174 if (!pmu->name)
1175 goto err;
1176
1177 pmu->events_attr_group.name = "events";
1178 pmu->events_attr_group.attrs = create_event_attributes(pmu);
1179 if (!pmu->events_attr_group.attrs)
1180 goto err_name;
1181
1182 pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups),
1183 GFP_KERNEL);
1184 if (!pmu->base.attr_groups)
1185 goto err_attr;
1186
1187 pmu->base.module = THIS_MODULE;
1188 pmu->base.task_ctx_nr = perf_invalid_context;
1189 pmu->base.scope = PERF_PMU_SCOPE_SYS_WIDE;
1190 pmu->base.event_init = i915_pmu_event_init;
1191 pmu->base.add = i915_pmu_event_add;
1192 pmu->base.del = i915_pmu_event_del;
1193 pmu->base.start = i915_pmu_event_start;
1194 pmu->base.stop = i915_pmu_event_stop;
1195 pmu->base.read = i915_pmu_event_read;
1196
1197 ret = perf_pmu_register(pmu: &pmu->base, name: pmu->name, type: -1);
1198 if (ret)
1199 goto err_groups;
1200
1201 pmu->registered = true;
1202
1203 return;
1204
1205err_groups:
1206 kfree(objp: pmu->base.attr_groups);
1207err_attr:
1208 free_event_attributes(pmu);
1209err_name:
1210 if (IS_DGFX(i915))
1211 kfree(objp: pmu->name);
1212err:
1213 drm_notice(&i915->drm, "Failed to register PMU!\n");
1214}
1215
1216void i915_pmu_unregister(struct drm_i915_private *i915)
1217{
1218 struct i915_pmu *pmu = &i915->pmu;
1219
1220 if (!pmu->registered)
1221 return;
1222
1223 /* Disconnect the PMU callbacks */
1224 pmu->registered = false;
1225
1226 hrtimer_cancel(timer: &pmu->timer);
1227
1228 perf_pmu_unregister(pmu: &pmu->base);
1229 kfree(objp: pmu->base.attr_groups);
1230 if (IS_DGFX(i915))
1231 kfree(objp: pmu->name);
1232 free_event_attributes(pmu);
1233}
1234

source code of linux/drivers/gpu/drm/i915/i915_pmu.c