1	/*
2	* Copyright © 2015-2016 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*
23	* Authors:
24	* Robert Bragg <robert@sixbynine.org>
25	*/
26
27
28	/**
29	* DOC: i915 Perf Overview
30	*
31	* Gen graphics supports a large number of performance counters that can help
32	* driver and application developers understand and optimize their use of the
33	* GPU.
34	*
35	* This i915 perf interface enables userspace to configure and open a file
36	* descriptor representing a stream of GPU metrics which can then be read() as
37	* a stream of sample records.
38	*
39	* The interface is particularly suited to exposing buffered metrics that are
40	* captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
41	*
42	* Streams representing a single context are accessible to applications with a
43	* corresponding drm file descriptor, such that OpenGL can use the interface
44	* without special privileges. Access to system-wide metrics requires root
45	* privileges by default, unless changed via the dev.i915.perf_event_paranoid
46	* sysctl option.
47	*
48	*/
49
50	/**
51	* DOC: i915 Perf History and Comparison with Core Perf
52	*
53	* The interface was initially inspired by the core Perf infrastructure but
54	* some notable differences are:
55	*
56	* i915 perf file descriptors represent a "stream" instead of an "event"; where
57	* a perf event primarily corresponds to a single 64bit value, while a stream
58	* might sample sets of tightly-coupled counters, depending on the
59	* configuration. For example the Gen OA unit isn't designed to support
60	* orthogonal configurations of individual counters; it's configured for a set
61	* of related counters. Samples for an i915 perf stream capturing OA metrics
62	* will include a set of counter values packed in a compact HW specific format.
63	* The OA unit supports a number of different packing formats which can be
64	* selected by the user opening the stream. Perf has support for grouping
65	* events, but each event in the group is configured, validated and
66	* authenticated individually with separate system calls.
67	*
68	* i915 perf stream configurations are provided as an array of u64 (key,value)
69	* pairs, instead of a fixed struct with multiple miscellaneous config members,
70	* interleaved with event-type specific members.
71	*
72	* i915 perf doesn't support exposing metrics via an mmap'd circular buffer.
73	* The supported metrics are being written to memory by the GPU unsynchronized
74	* with the CPU, using HW specific packing formats for counter sets. Sometimes
75	* the constraints on HW configuration require reports to be filtered before it
76	* would be acceptable to expose them to unprivileged applications - to hide
77	* the metrics of other processes/contexts. For these use cases a read() based
78	* interface is a good fit, and provides an opportunity to filter data as it
79	* gets copied from the GPU mapped buffers to userspace buffers.
80	*
81	*
82	* Issues hit with first prototype based on Core Perf
83	* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
84	*
85	* The first prototype of this driver was based on the core perf
86	* infrastructure, and while we did make that mostly work, with some changes to
87	* perf, we found we were breaking or working around too many assumptions baked
88	* into perf's currently cpu centric design.
89	*
90	* In the end we didn't see a clear benefit to making perf's implementation and
91	* interface more complex by changing design assumptions while we knew we still
92	* wouldn't be able to use any existing perf based userspace tools.
93	*
94	* Also considering the Gen specific nature of the Observability hardware and
95	* how userspace will sometimes need to combine i915 perf OA metrics with
96	* side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
97	* expecting the interface to be used by a platform specific userspace such as
98	* OpenGL or tools. This is to say; we aren't inherently missing out on having
99	* a standard vendor/architecture agnostic interface by not using perf.
100	*
101	*
102	* For posterity, in case we might re-visit trying to adapt core perf to be
103	* better suited to exposing i915 metrics these were the main pain points we
104	* hit:
105	*
106	* - The perf based OA PMU driver broke some significant design assumptions:
107	*
108	* Existing perf pmus are used for profiling work on a cpu and we were
109	* introducing the idea of _IS_DEVICE pmus with different security
110	* implications, the need to fake cpu-related data (such as user/kernel
111	* registers) to fit with perf's current design, and adding _DEVICE records
112	* as a way to forward device-specific status records.
113	*
114	* The OA unit writes reports of counters into a circular buffer, without
115	* involvement from the CPU, making our PMU driver the first of a kind.
116	*
117	* Given the way we were periodically forward data from the GPU-mapped, OA
118	* buffer to perf's buffer, those bursts of sample writes looked to perf like
119	* we were sampling too fast and so we had to subvert its throttling checks.
120	*
121	* Perf supports groups of counters and allows those to be read via
122	* transactions internally but transactions currently seem designed to be
123	* explicitly initiated from the cpu (say in response to a userspace read())
124	* and while we could pull a report out of the OA buffer we can't
125	* trigger a report from the cpu on demand.
126	*
127	* Related to being report based; the OA counters are configured in HW as a
128	* set while perf generally expects counter configurations to be orthogonal.
129	* Although counters can be associated with a group leader as they are
130	* opened, there's no clear precedent for being able to provide group-wide
131	* configuration attributes (for example we want to let userspace choose the
132	* OA unit report format used to capture all counters in a set, or specify a
133	* GPU context to filter metrics on). We avoided using perf's grouping
134	* feature and forwarded OA reports to userspace via perf's 'raw' sample
135	* field. This suited our userspace well considering how coupled the counters
136	* are when dealing with normalizing. It would be inconvenient to split
137	* counters up into separate events, only to require userspace to recombine
138	* them. For Mesa it's also convenient to be forwarded raw, periodic reports
139	* for combining with the side-band raw reports it captures using
140	* MI_REPORT_PERF_COUNT commands.
141	*
142	* - As a side note on perf's grouping feature; there was also some concern
143	* that using PERF_FORMAT_GROUP as a way to pack together counter values
144	* would quite drastically inflate our sample sizes, which would likely
145	* lower the effective sampling resolutions we could use when the available
146	* memory bandwidth is limited.
147	*
148	* With the OA unit's report formats, counters are packed together as 32
149	* or 40bit values, with the largest report size being 256 bytes.
150	*
151	* PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a
152	* documented ordering to the values, implying PERF_FORMAT_ID must also be
153	* used to add a 64bit ID before each value; giving 16 bytes per counter.
154	*
155	* Related to counter orthogonality; we can't time share the OA unit, while
156	* event scheduling is a central design idea within perf for allowing
157	* userspace to open + enable more events than can be configured in HW at any
158	* one time. The OA unit is not designed to allow re-configuration while in
159	* use. We can't reconfigure the OA unit without losing internal OA unit
160	* state which we can't access explicitly to save and restore. Reconfiguring
161	* the OA unit is also relatively slow, involving ~100 register writes. From
162	* userspace Mesa also depends on a stable OA configuration when emitting
163	* MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be
164	* disabled while there are outstanding MI_RPC commands lest we hang the
165	* command streamer.
166	*
167	* The contents of sample records aren't extensible by device drivers (i.e.
168	* the sample_type bits). As an example; Sourab Gupta had been looking to
169	* attach GPU timestamps to our OA samples. We were shoehorning OA reports
170	* into sample records by using the 'raw' field, but it's tricky to pack more
171	* than one thing into this field because events/core.c currently only lets a
172	* pmu give a single raw data pointer plus len which will be copied into the
173	* ring buffer. To include more than the OA report we'd have to copy the
174	* report into an intermediate larger buffer. I'd been considering allowing a
175	* vector of data+len values to be specified for copying the raw data, but
176	* it felt like a kludge to being using the raw field for this purpose.
177	*
178	* - It felt like our perf based PMU was making some technical compromises
179	* just for the sake of using perf:
180	*
181	* perf_event_open() requires events to either relate to a pid or a specific
182	* cpu core, while our device pmu related to neither. Events opened with a
183	* pid will be automatically enabled/disabled according to the scheduling of
184	* that process - so not appropriate for us. When an event is related to a
185	* cpu id, perf ensures pmu methods will be invoked via an inter process
186	* interrupt on that core. To avoid invasive changes our userspace opened OA
187	* perf events for a specific cpu. This was workable but it meant the
188	* majority of the OA driver ran in atomic context, including all OA report
189	* forwarding, which wasn't really necessary in our case and seems to make
190	* our locking requirements somewhat complex as we handled the interaction
191	* with the rest of the i915 driver.
192	*/
193
194	#include <linux/anon_inodes.h>
195	#include <linux/nospec.h>
196	#include <linux/sizes.h>
197	#include <linux/uuid.h>
198
199	#include "gem/i915_gem_context.h"
200	#include "gem/i915_gem_internal.h"
201	#include "gt/intel_engine_pm.h"
202	#include "gt/intel_engine_regs.h"
203	#include "gt/intel_engine_user.h"
204	#include "gt/intel_execlists_submission.h"
205	#include "gt/intel_gpu_commands.h"
206	#include "gt/intel_gt.h"
207	#include "gt/intel_gt_clock_utils.h"
208	#include "gt/intel_gt_mcr.h"
209	#include "gt/intel_gt_print.h"
210	#include "gt/intel_gt_regs.h"
211	#include "gt/intel_lrc.h"
212	#include "gt/intel_lrc_reg.h"
213	#include "gt/intel_rc6.h"
214	#include "gt/intel_ring.h"
215	#include "gt/uc/intel_guc_slpc.h"
216
217	#include "i915_drv.h"
218	#include "i915_file_private.h"
219	#include "i915_perf.h"
220	#include "i915_perf_oa_regs.h"
221	#include "i915_reg.h"
222	#include "i915_mmio_range.h"
223
224	/* HW requires this to be a power of two, between 128k and 16M, though driver
225	* is currently generally designed assuming the largest 16M size is used such
226	* that the overflow cases are unlikely in normal operation.
227	*/
228	#define OA_BUFFER_SIZE SZ_16M
229
230	#define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
231
232	/**
233	* DOC: OA Tail Pointer Race
234	*
235	* There's a HW race condition between OA unit tail pointer register updates and
236	* writes to memory whereby the tail pointer can sometimes get ahead of what's
237	* been written out to the OA buffer so far (in terms of what's visible to the
238	* CPU).
239	*
240	* Although this can be observed explicitly while copying reports to userspace
241	* by checking for a zeroed report-id field in tail reports, we want to account
242	* for this earlier, as part of the oa_buffer_check_unlocked to avoid lots of
243	* redundant read() attempts.
244	*
245	* We workaround this issue in oa_buffer_check_unlocked() by reading the reports
246	* in the OA buffer, starting from the tail reported by the HW until we find a
247	* report with its first 2 dwords not 0 meaning its previous report is
248	* completely in memory and ready to be read. Those dwords are also set to 0
249	* once read and the whole buffer is cleared upon OA buffer initialization. The
250	* first dword is the reason for this report while the second is the timestamp,
251	* making the chances of having those 2 fields at 0 fairly unlikely. A more
252	* detailed explanation is available in oa_buffer_check_unlocked().
253	*
254	* Most of the implementation details for this workaround are in
255	* oa_buffer_check_unlocked() and _append_oa_reports()
256	*
257	* Note for posterity: previously the driver used to define an effective tail
258	* pointer that lagged the real pointer by a 'tail margin' measured in bytes
259	* derived from %OA_TAIL_MARGIN_NSEC and the configured sampling frequency.
260	* This was flawed considering that the OA unit may also automatically generate
261	* non-periodic reports (such as on context switch) or the OA unit may be
262	* enabled without any periodic sampling.
263	*/
264	#define OA_TAIL_MARGIN_NSEC 100000ULL
265	#define INVALID_TAIL_PTR 0xffffffff
266
267	/* The default frequency for checking whether the OA unit has written new
268	* reports to the circular OA buffer...
269	*/
270	#define DEFAULT_POLL_FREQUENCY_HZ 200
271	#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
272
273	/* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */
274	static u32 i915_perf_stream_paranoid = true;
275
276	/* The maximum exponent the hardware accepts is 63 (essentially it selects one
277	* of the 64bit timestamp bits to trigger reports from) but there's currently
278	* no known use case for sampling as infrequently as once per 47 thousand years.
279	*
280	* Since the timestamps included in OA reports are only 32bits it seems
281	* reasonable to limit the OA exponent where it's still possible to account for
282	* overflow in OA report timestamps.
283	*/
284	#define OA_EXPONENT_MAX 31
285
286	#define INVALID_CTX_ID 0xffffffff
287
288	/* On Gen8+ automatically triggered OA reports include a 'reason' field... */
289	#define OAREPORT_REASON_MASK 0x3f
290	#define OAREPORT_REASON_MASK_EXTENDED 0x7f
291	#define OAREPORT_REASON_SHIFT 19
292	#define OAREPORT_REASON_TIMER (1<<0)
293	#define OAREPORT_REASON_CTX_SWITCH (1<<3)
294	#define OAREPORT_REASON_CLK_RATIO (1<<5)
295
296	#define HAS_MI_SET_PREDICATE(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
297
298	/* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate
299	*
300	* The highest sampling frequency we can theoretically program the OA unit
301	* with is always half the timestamp frequency: E.g. 6.25Mhz for Haswell.
302	*
303	* Initialized just before we register the sysctl parameter.
304	*/
305	static int oa_sample_rate_hard_limit;
306
307	/* Theoretically we can program the OA unit to sample every 160ns but don't
308	* allow that by default unless root...
309	*
310	* The default threshold of 100000Hz is based on perf's similar
311	* kernel.perf_event_max_sample_rate sysctl parameter.
312	*/
313	static u32 i915_oa_max_sample_rate = 100000;
314
315	/* XXX: beware if future OA HW adds new report formats that the current
316	* code assumes all reports have a power-of-two size and ~(size - 1) can
317	* be used as a mask to align the OA tail pointer.
318	*/
319	static const struct i915_oa_format oa_formats[I915_OA_FORMAT_MAX] = {
320	[I915_OA_FORMAT_A13] = { 0, 64 },
321	[I915_OA_FORMAT_A29] = { 1, 128 },
322	[I915_OA_FORMAT_A13_B8_C8] = { 2, 128 },
323	/* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */
324	[I915_OA_FORMAT_B4_C8] = { 4, 64 },
325	[I915_OA_FORMAT_A45_B8_C8] = { 5, 256 },
326	[I915_OA_FORMAT_B4_C8_A16] = { 6, 128 },
327	[I915_OA_FORMAT_C4_B8] = { 7, 64 },
328	[I915_OA_FORMAT_A12] = { 0, 64 },
329	[I915_OA_FORMAT_A12_B8_C8] = { 2, 128 },
330	[I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 },
331	[I915_OAR_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 },
332	[I915_OA_FORMAT_A24u40_A14u32_B8_C8] = { 5, 256 },
333	[I915_OAM_FORMAT_MPEC8u64_B8_C8] = { 1, 192, TYPE_OAM, HDR_64_BIT },
334	[I915_OAM_FORMAT_MPEC8u32_B8_C8] = { 2, 128, TYPE_OAM, HDR_64_BIT },
335	};
336
337	static const u32 mtl_oa_base[] = {
338	[PERF_GROUP_OAM_SAMEDIA_0] = 0x393000,
339	};
340
341	#define SAMPLE_OA_REPORT (1<<0)
342
343	/**
344	* struct perf_open_properties - for validated properties given to open a stream
345	* @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags
346	* @single_context: Whether a single or all gpu contexts should be monitored
347	* @hold_preemption: Whether the preemption is disabled for the filtered
348	* context
349	* @ctx_handle: A gem ctx handle for use with @single_context
350	* @metrics_set: An ID for an OA unit metric set advertised via sysfs
351	* @oa_format: An OA unit HW report format
352	* @oa_periodic: Whether to enable periodic OA unit sampling
353	* @oa_period_exponent: The OA unit sampling period is derived from this
354	* @engine: The engine (typically rcs0) being monitored by the OA unit
355	* @has_sseu: Whether @sseu was specified by userspace
356	* @sseu: internal SSEU configuration computed either from the userspace
357	* specified configuration in the opening parameters or a default value
358	* (see get_default_sseu_config())
359	* @poll_oa_period: The period in nanoseconds at which the CPU will check for OA
360	* data availability
361	*
362	* As read_properties_unlocked() enumerates and validates the properties given
363	* to open a stream of metrics the configuration is built up in the structure
364	* which starts out zero initialized.
365	*/
366	struct perf_open_properties {
367	u32 sample_flags;
368
369	u64 single_context:1;
370	u64 hold_preemption:1;
371	u64 ctx_handle;
372
373	/* OA sampling state */
374	int metrics_set;
375	int oa_format;
376	bool oa_periodic;
377	int oa_period_exponent;
378
379	struct intel_engine_cs *engine;
380
381	bool has_sseu;
382	struct intel_sseu sseu;
383
384	u64 poll_oa_period;
385	};
386
387	struct i915_oa_config_bo {
388	struct llist_node node;
389
390	struct i915_oa_config *oa_config;
391	struct i915_vma *vma;
392	};
393
394	static struct ctl_table_header *sysctl_header;
395
396	static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer);
397
398	void i915_oa_config_release(struct kref *ref)
399	{
400	struct i915_oa_config *oa_config =
401	container_of(ref, typeof(*oa_config), ref);
402
403	kfree(objp: oa_config->flex_regs);
404	kfree(objp: oa_config->b_counter_regs);
405	kfree(objp: oa_config->mux_regs);
406
407	kfree_rcu(oa_config, rcu);
408	}
409
410	struct i915_oa_config *
411	i915_perf_get_oa_config(struct i915_perf *perf, int metrics_set)
412	{
413	struct i915_oa_config *oa_config;
414
415	rcu_read_lock();
416	oa_config = idr_find(&perf->metrics_idr, id: metrics_set);
417	if (oa_config)
418	oa_config = i915_oa_config_get(oa_config);
419	rcu_read_unlock();
420
421	return oa_config;
422	}
423
424	static void free_oa_config_bo(struct i915_oa_config_bo *oa_bo)
425	{
426	i915_oa_config_put(oa_config: oa_bo->oa_config);
427	i915_vma_put(vma: oa_bo->vma);
428	kfree(objp: oa_bo);
429	}
430
431	static inline const
432	struct i915_perf_regs *__oa_regs(struct i915_perf_stream *stream)
433	{
434	return &stream->engine->oa_group->regs;
435	}
436
437	static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
438	{
439	struct intel_uncore *uncore = stream->uncore;
440
441	return intel_uncore_read(uncore, reg: __oa_regs(stream)->oa_tail_ptr) &
442	GEN12_OAG_OATAILPTR_MASK;
443	}
444
445	static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream)
446	{
447	struct intel_uncore *uncore = stream->uncore;
448
449	return intel_uncore_read(uncore, GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK;
450	}
451
452	static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
453	{
454	struct intel_uncore *uncore = stream->uncore;
455	u32 oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
456
457	return oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
458	}
459
460	#define oa_report_header_64bit(__s) \
461	((__s)->oa_buffer.format->header == HDR_64_BIT)
462
463	static u64 oa_report_id(struct i915_perf_stream *stream, void *report)
464	{
465	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
466	}
467
468	static u64 oa_report_reason(struct i915_perf_stream *stream, void *report)
469	{
470	return (oa_report_id(stream, report) >> OAREPORT_REASON_SHIFT) &
471	(GRAPHICS_VER(stream->perf->i915) == 12 ?
472	OAREPORT_REASON_MASK_EXTENDED :
473	OAREPORT_REASON_MASK);
474	}
475
476	static void oa_report_id_clear(struct i915_perf_stream *stream, u32 *report)
477	{
478	if (oa_report_header_64bit(stream))
479	*(u64 *)report = 0;
480	else
481	*report = 0;
482	}
483
484	static bool oa_report_ctx_invalid(struct i915_perf_stream *stream, void *report)
485	{
486	return !(oa_report_id(stream, report) &
487	stream->perf->gen8_valid_ctx_bit);
488	}
489
490	static u64 oa_timestamp(struct i915_perf_stream *stream, void *report)
491	{
492	return oa_report_header_64bit(stream) ?
493	*((u64 *)report + 1) :
494	*((u32 *)report + 1);
495	}
496
497	static void oa_timestamp_clear(struct i915_perf_stream *stream, u32 *report)
498	{
499	if (oa_report_header_64bit(stream))
500	*(u64 *)&report[2] = 0;
501	else
502	report[1] = 0;
503	}
504
505	static u32 oa_context_id(struct i915_perf_stream *stream, u32 *report)
506	{
507	u32 ctx_id = oa_report_header_64bit(stream) ? report[4] : report[2];
508
509	return ctx_id & stream->specific_ctx_id_mask;
510	}
511
512	static void oa_context_id_squash(struct i915_perf_stream *stream, u32 *report)
513	{
514	if (oa_report_header_64bit(stream))
515	report[4] = INVALID_CTX_ID;
516	else
517	report[2] = INVALID_CTX_ID;
518	}
519
520	/**
521	* oa_buffer_check_unlocked - check for data and update tail ptr state
522	* @stream: i915 stream instance
523	*
524	* This is either called via fops (for blocking reads in user ctx) or the poll
525	* check hrtimer (atomic ctx) to check the OA buffer tail pointer and check
526	* if there is data available for userspace to read.
527	*
528	* This function is central to providing a workaround for the OA unit tail
529	* pointer having a race with respect to what data is visible to the CPU.
530	* It is responsible for reading tail pointers from the hardware and giving
531	* the pointers time to 'age' before they are made available for reading.
532	* (See description of OA_TAIL_MARGIN_NSEC above for further details.)
533	*
534	* Besides returning true when there is data available to read() this function
535	* also updates the tail in the oa_buffer object.
536	*
537	* Note: It's safe to read OA config state here unlocked, assuming that this is
538	* only called while the stream is enabled, while the global OA configuration
539	* can't be modified.
540	*
541	* Returns: %true if the OA buffer contains data, else %false
542	*/
543	static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
544	{
545	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
546	int report_size = stream->oa_buffer.format->size;
547	u32 tail, hw_tail;
548	unsigned long flags;
549	bool pollin;
550	u32 partial_report_size;
551
552	/*
553	* We have to consider the (unlikely) possibility that read() errors
554	* could result in an OA buffer reset which might reset the head and
555	* tail state.
556	*/
557	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
558
559	hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
560	hw_tail -= gtt_offset;
561
562	/*
563	* The tail pointer increases in 64 byte increments, not in report_size
564	* steps. Also the report size may not be a power of 2. Compute
565	* potentially partially landed report in the OA buffer
566	*/
567	partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail);
568	partial_report_size %= report_size;
569
570	/* Subtract partial amount off the tail */
571	hw_tail = OA_TAKEN(hw_tail, partial_report_size);
572
573	tail = hw_tail;
574
575	/*
576	* Walk the stream backward until we find a report with report
577	* id and timestamp not at 0. Since the circular buffer pointers
578	* progress by increments of 64 bytes and that reports can be up
579	* to 256 bytes long, we can't tell whether a report has fully
580	* landed in memory before the report id and timestamp of the
581	* following report have effectively landed.
582	*
583	* This is assuming that the writes of the OA unit land in
584	* memory in the order they were written to.
585	* If not : (╯°□°）╯︵ ┻━┻
586	*/
587	while (OA_TAKEN(tail, stream->oa_buffer.tail) >= report_size) {
588	void *report = stream->oa_buffer.vaddr + tail;
589
590	if (oa_report_id(stream, report) ||
591	oa_timestamp(stream, report))
592	break;
593
594	tail = (tail - report_size) & (OA_BUFFER_SIZE - 1);
595	}
596
597	if (OA_TAKEN(hw_tail, tail) > report_size &&
598	__ratelimit(&stream->perf->tail_pointer_race))
599	drm_notice(&stream->uncore->i915->drm,
600	"unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n",
601	stream->oa_buffer.head, tail, hw_tail);
602
603	stream->oa_buffer.tail = tail;
604
605	pollin = OA_TAKEN(stream->oa_buffer.tail,
606	stream->oa_buffer.head) >= report_size;
607
608	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
609
610	return pollin;
611	}
612
613	/**
614	* append_oa_status - Appends a status record to a userspace read() buffer.
615	* @stream: An i915-perf stream opened for OA metrics
616	* @buf: destination buffer given by userspace
617	* @count: the number of bytes userspace wants to read
618	* @offset: (inout): the current position for writing into @buf
619	* @type: The kind of status to report to userspace
620	*
621	* Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`)
622	* into the userspace read() buffer.
623	*
624	* The @buf @offset will only be updated on success.
625	*
626	* Returns: 0 on success, negative error code on failure.
627	*/
628	static int append_oa_status(struct i915_perf_stream *stream,
629	char __user *buf,
630	size_t count,
631	size_t *offset,
632	enum drm_i915_perf_record_type type)
633	{
634	struct drm_i915_perf_record_header header = { type, 0, sizeof(header) };
635
636	if ((count - *offset) < header.size)
637	return -ENOSPC;
638
639	if (copy_to_user(to: buf + *offset, from: &header, n: sizeof(header)))
640	return -EFAULT;
641
642	(*offset) += header.size;
643
644	return 0;
645	}
646
647	/**
648	* append_oa_sample - Copies single OA report into userspace read() buffer.
649	* @stream: An i915-perf stream opened for OA metrics
650	* @buf: destination buffer given by userspace
651	* @count: the number of bytes userspace wants to read
652	* @offset: (inout): the current position for writing into @buf
653	* @report: A single OA report to (optionally) include as part of the sample
654	*
655	* The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*`
656	* properties when opening a stream, tracked as `stream->sample_flags`. This
657	* function copies the requested components of a single sample to the given
658	* read() @buf.
659	*
660	* The @buf @offset will only be updated on success.
661	*
662	* Returns: 0 on success, negative error code on failure.
663	*/
664	static int append_oa_sample(struct i915_perf_stream *stream,
665	char __user *buf,
666	size_t count,
667	size_t *offset,
668	const u8 *report)
669	{
670	int report_size = stream->oa_buffer.format->size;
671	struct drm_i915_perf_record_header header;
672	int report_size_partial;
673	u8 *oa_buf_end;
674
675	header.type = DRM_I915_PERF_RECORD_SAMPLE;
676	header.pad = 0;
677	header.size = stream->sample_size;
678
679	if ((count - *offset) < header.size)
680	return -ENOSPC;
681
682	buf += *offset;
683	if (copy_to_user(to: buf, from: &header, n: sizeof(header)))
684	return -EFAULT;
685	buf += sizeof(header);
686
687	oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE;
688	report_size_partial = oa_buf_end - report;
689
690	if (report_size_partial < report_size) {
691	if (copy_to_user(to: buf, from: report, n: report_size_partial))
692	return -EFAULT;
693	buf += report_size_partial;
694
695	if (copy_to_user(to: buf, from: stream->oa_buffer.vaddr,
696	n: report_size - report_size_partial))
697	return -EFAULT;
698	} else if (copy_to_user(to: buf, from: report, n: report_size)) {
699	return -EFAULT;
700	}
701
702	(*offset) += header.size;
703
704	return 0;
705	}
706
707	/**
708	* gen8_append_oa_reports - Copies all buffered OA reports into
709	* userspace read() buffer.
710	* @stream: An i915-perf stream opened for OA metrics
711	* @buf: destination buffer given by userspace
712	* @count: the number of bytes userspace wants to read
713	* @offset: (inout): the current position for writing into @buf
714	*
715	* Notably any error condition resulting in a short read (-%ENOSPC or
716	* -%EFAULT) will be returned even though one or more records may
717	* have been successfully copied. In this case it's up to the caller
718	* to decide if the error should be squashed before returning to
719	* userspace.
720	*
721	* Note: reports are consumed from the head, and appended to the
722	* tail, so the tail chases the head?... If you think that's mad
723	* and back-to-front you're not alone, but this follows the
724	* Gen PRM naming convention.
725	*
726	* Returns: 0 on success, negative error code on failure.
727	*/
728	static int gen8_append_oa_reports(struct i915_perf_stream *stream,
729	char __user *buf,
730	size_t count,
731	size_t *offset)
732	{
733	struct intel_uncore *uncore = stream->uncore;
734	int report_size = stream->oa_buffer.format->size;
735	u8 *oa_buf_base = stream->oa_buffer.vaddr;
736	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
737	u32 mask = (OA_BUFFER_SIZE - 1);
738	size_t start_offset = *offset;
739	unsigned long flags;
740	u32 head, tail;
741	int ret = 0;
742
743	if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
744	return -EIO;
745
746	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
747
748	head = stream->oa_buffer.head;
749	tail = stream->oa_buffer.tail;
750
751	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
752
753	/*
754	* An out of bounds or misaligned head or tail pointer implies a driver
755	* bug since we validate + align the tail pointers we read from the
756	* hardware and we are in full control of the head pointer which should
757	* only be incremented by multiples of the report size.
758	*/
759	if (drm_WARN_ONCE(&uncore->i915->drm,
760	head > OA_BUFFER_SIZE ||
761	tail > OA_BUFFER_SIZE,
762	"Inconsistent OA buffer pointers: head = %u, tail = %u\n",
763	head, tail))
764	return -EIO;
765
766
767	for (/* none */;
768	OA_TAKEN(tail, head);
769	head = (head + report_size) & mask) {
770	u8 *report = oa_buf_base + head;
771	u32 *report32 = (void *)report;
772	u32 ctx_id;
773	u64 reason;
774
775	/*
776	* The reason field includes flags identifying what
777	* triggered this specific report (mostly timer
778	* triggered or e.g. due to a context switch).
779	*/
780	reason = oa_report_reason(stream, report);
781	ctx_id = oa_context_id(stream, report: report32);
782
783	/*
784	* Squash whatever is in the CTX_ID field if it's marked as
785	* invalid to be sure we avoid false-positive, single-context
786	* filtering below...
787	*
788	* Note: that we don't clear the valid_ctx_bit so userspace can
789	* understand that the ID has been squashed by the kernel.
790	*
791	* Update:
792	*
793	* On XEHP platforms the behavior of context id valid bit has
794	* changed compared to prior platforms. To describe this, we
795	* define a few terms:
796	*
797	* context-switch-report: This is a report with the reason type
798	* being context-switch. It is generated when a context switches
799	* out.
800	*
801	* context-valid-bit: A bit that is set in the report ID field
802	* to indicate that a valid context has been loaded.
803	*
804	* gpu-idle: A condition characterized by a
805	* context-switch-report with context-valid-bit set to 0.
806	*
807	* On prior platforms, context-id-valid bit is set to 0 only
808	* when GPU goes idle. In all other reports, it is set to 1.
809	*
810	* On XEHP platforms, context-valid-bit is set to 1 in a context
811	* switch report if a new context switched in. For all other
812	* reports it is set to 0.
813	*
814	* This change in behavior causes an issue with MMIO triggered
815	* reports. MMIO triggered reports have the markers in the
816	* context ID field and the context-valid-bit is 0. The logic
817	* below to squash the context ID would render the report
818	* useless since the user will not be able to find it in the OA
819	* buffer. Since MMIO triggered reports exist only on XEHP,
820	* we should avoid squashing these for XEHP platforms.
821	*/
822
823	if (oa_report_ctx_invalid(stream, report) &&
824	GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 55)) {
825	ctx_id = INVALID_CTX_ID;
826	oa_context_id_squash(stream, report: report32);
827	}
828
829	/*
830	* NB: For Gen 8 the OA unit no longer supports clock gating
831	* off for a specific context and the kernel can't securely
832	* stop the counters from updating as system-wide / global
833	* values.
834	*
835	* Automatic reports now include a context ID so reports can be
836	* filtered on the cpu but it's not worth trying to
837	* automatically subtract/hide counter progress for other
838	* contexts while filtering since we can't stop userspace
839	* issuing MI_REPORT_PERF_COUNT commands which would still
840	* provide a side-band view of the real values.
841	*
842	* To allow userspace (such as Mesa/GL_INTEL_performance_query)
843	* to normalize counters for a single filtered context then it
844	* needs be forwarded bookend context-switch reports so that it
845	* can track switches in between MI_REPORT_PERF_COUNT commands
846	* and can itself subtract/ignore the progress of counters
847	* associated with other contexts. Note that the hardware
848	* automatically triggers reports when switching to a new
849	* context which are tagged with the ID of the newly active
850	* context. To avoid the complexity (and likely fragility) of
851	* reading ahead while parsing reports to try and minimize
852	* forwarding redundant context switch reports (i.e. between
853	* other, unrelated contexts) we simply elect to forward them
854	* all.
855	*
856	* We don't rely solely on the reason field to identify context
857	* switches since it's not-uncommon for periodic samples to
858	* identify a switch before any 'context switch' report.
859	*/
860	if (!stream->ctx ||
861	stream->specific_ctx_id == ctx_id ||
862	stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||
863	reason & OAREPORT_REASON_CTX_SWITCH) {
864
865	/*
866	* While filtering for a single context we avoid
867	* leaking the IDs of other contexts.
868	*/
869	if (stream->ctx &&
870	stream->specific_ctx_id != ctx_id) {
871	oa_context_id_squash(stream, report: report32);
872	}
873
874	ret = append_oa_sample(stream, buf, count, offset,
875	report);
876	if (ret)
877	break;
878
879	stream->oa_buffer.last_ctx_id = ctx_id;
880	}
881
882	if (is_power_of_2(n: report_size)) {
883	/*
884	* Clear out the report id and timestamp as a means
885	* to detect unlanded reports.
886	*/
887	oa_report_id_clear(stream, report: report32);
888	oa_timestamp_clear(stream, report: report32);
889	} else {
890	u8 *oa_buf_end = stream->oa_buffer.vaddr +
891	OA_BUFFER_SIZE;
892	u32 part = oa_buf_end - (u8 *)report32;
893
894	/* Zero out the entire report */
895	if (report_size <= part) {
896	memset(report32, 0, report_size);
897	} else {
898	memset(report32, 0, part);
899	memset(oa_buf_base, 0, report_size - part);
900	}
901	}
902	}
903
904	if (start_offset != *offset) {
905	i915_reg_t oaheadptr;
906
907	oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ?
908	__oa_regs(stream)->oa_head_ptr :
909	GEN8_OAHEADPTR;
910
911	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
912
913	/*
914	* We removed the gtt_offset for the copy loop above, indexing
915	* relative to oa_buf_base so put back here...
916	*/
917	intel_uncore_write(uncore, reg: oaheadptr,
918	val: (head + gtt_offset) & GEN12_OAG_OAHEADPTR_MASK);
919	stream->oa_buffer.head = head;
920
921	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
922	}
923
924	return ret;
925	}
926
927	/**
928	* gen8_oa_read - copy status records then buffered OA reports
929	* @stream: An i915-perf stream opened for OA metrics
930	* @buf: destination buffer given by userspace
931	* @count: the number of bytes userspace wants to read
932	* @offset: (inout): the current position for writing into @buf
933	*
934	* Checks OA unit status registers and if necessary appends corresponding
935	* status records for userspace (such as for a buffer full condition) and then
936	* initiate appending any buffered OA reports.
937	*
938	* Updates @offset according to the number of bytes successfully copied into
939	* the userspace buffer.
940	*
941	* NB: some data may be successfully copied to the userspace buffer
942	* even if an error is returned, and this is reflected in the
943	* updated @offset.
944	*
945	* Returns: zero on success or a negative error code
946	*/
947	static int gen8_oa_read(struct i915_perf_stream *stream,
948	char __user *buf,
949	size_t count,
950	size_t *offset)
951	{
952	struct intel_uncore *uncore = stream->uncore;
953	u32 oastatus;
954	i915_reg_t oastatus_reg;
955	int ret;
956
957	if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
958	return -EIO;
959
960	oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ?
961	__oa_regs(stream)->oa_status :
962	GEN8_OASTATUS;
963
964	oastatus = intel_uncore_read(uncore, reg: oastatus_reg);
965
966	/*
967	* We treat OABUFFER_OVERFLOW as a significant error:
968	*
969	* Although theoretically we could handle this more gracefully
970	* sometimes, some Gens don't correctly suppress certain
971	* automatically triggered reports in this condition and so we
972	* have to assume that old reports are now being trampled
973	* over.
974	*
975	* Considering how we don't currently give userspace control
976	* over the OA buffer size and always configure a large 16MB
977	* buffer, then a buffer overflow does anyway likely indicate
978	* that something has gone quite badly wrong.
979	*/
980	if (oastatus & GEN8_OASTATUS_OABUFFER_OVERFLOW) {
981	ret = append_oa_status(stream, buf, count, offset,
982	type: DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
983	if (ret)
984	return ret;
985
986	drm_dbg(&stream->perf->i915->drm,
987	"OA buffer overflow (exponent = %d): force restart\n",
988	stream->period_exponent);
989
990	stream->perf->ops.oa_disable(stream);
991	stream->perf->ops.oa_enable(stream);
992
993	/*
994	* Note: .oa_enable() is expected to re-init the oabuffer and
995	* reset GEN8_OASTATUS for us
996	*/
997	oastatus = intel_uncore_read(uncore, reg: oastatus_reg);
998	}
999
1000	if (oastatus & GEN8_OASTATUS_REPORT_LOST) {
1001	ret = append_oa_status(stream, buf, count, offset,
1002	type: DRM_I915_PERF_RECORD_OA_REPORT_LOST);
1003	if (ret)
1004	return ret;
1005
1006	intel_uncore_rmw(uncore, reg: oastatus_reg,
1007	GEN8_OASTATUS_COUNTER_OVERFLOW |
1008	GEN8_OASTATUS_REPORT_LOST,
1009	IS_GRAPHICS_VER(uncore->i915, 8, 11) ?
1010	(GEN8_OASTATUS_HEAD_POINTER_WRAP |
1011	GEN8_OASTATUS_TAIL_POINTER_WRAP) : 0);
1012	}
1013
1014	return gen8_append_oa_reports(stream, buf, count, offset);
1015	}
1016
1017	/**
1018	* gen7_append_oa_reports - Copies all buffered OA reports into
1019	* userspace read() buffer.
1020	* @stream: An i915-perf stream opened for OA metrics
1021	* @buf: destination buffer given by userspace
1022	* @count: the number of bytes userspace wants to read
1023	* @offset: (inout): the current position for writing into @buf
1024	*
1025	* Notably any error condition resulting in a short read (-%ENOSPC or
1026	* -%EFAULT) will be returned even though one or more records may
1027	* have been successfully copied. In this case it's up to the caller
1028	* to decide if the error should be squashed before returning to
1029	* userspace.
1030	*
1031	* Note: reports are consumed from the head, and appended to the
1032	* tail, so the tail chases the head?... If you think that's mad
1033	* and back-to-front you're not alone, but this follows the
1034	* Gen PRM naming convention.
1035	*
1036	* Returns: 0 on success, negative error code on failure.
1037	*/
1038	static int gen7_append_oa_reports(struct i915_perf_stream *stream,
1039	char __user *buf,
1040	size_t count,
1041	size_t *offset)
1042	{
1043	struct intel_uncore *uncore = stream->uncore;
1044	int report_size = stream->oa_buffer.format->size;
1045	u8 *oa_buf_base = stream->oa_buffer.vaddr;
1046	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1047	u32 mask = (OA_BUFFER_SIZE - 1);
1048	size_t start_offset = *offset;
1049	unsigned long flags;
1050	u32 head, tail;
1051	int ret = 0;
1052
1053	if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
1054	return -EIO;
1055
1056	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1057
1058	head = stream->oa_buffer.head;
1059	tail = stream->oa_buffer.tail;
1060
1061	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1062
1063	/* An out of bounds or misaligned head or tail pointer implies a driver
1064	* bug since we validate + align the tail pointers we read from the
1065	* hardware and we are in full control of the head pointer which should
1066	* only be incremented by multiples of the report size (notably also
1067	* all a power of two).
1068	*/
1069	if (drm_WARN_ONCE(&uncore->i915->drm,
1070	head > OA_BUFFER_SIZE || head % report_size ||
1071	tail > OA_BUFFER_SIZE || tail % report_size,
1072	"Inconsistent OA buffer pointers: head = %u, tail = %u\n",
1073	head, tail))
1074	return -EIO;
1075
1076
1077	for (/* none */;
1078	OA_TAKEN(tail, head);
1079	head = (head + report_size) & mask) {
1080	u8 *report = oa_buf_base + head;
1081	u32 *report32 = (void *)report;
1082
1083	/* All the report sizes factor neatly into the buffer
1084	* size so we never expect to see a report split
1085	* between the beginning and end of the buffer.
1086	*
1087	* Given the initial alignment check a misalignment
1088	* here would imply a driver bug that would result
1089	* in an overrun.
1090	*/
1091	if (drm_WARN_ON(&uncore->i915->drm,
1092	(OA_BUFFER_SIZE - head) < report_size)) {
1093	drm_err(&uncore->i915->drm,
1094	"Spurious OA head ptr: non-integral report offset\n");
1095	break;
1096	}
1097
1098	/* The report-ID field for periodic samples includes
1099	* some undocumented flags related to what triggered
1100	* the report and is never expected to be zero so we
1101	* can check that the report isn't invalid before
1102	* copying it to userspace...
1103	*/
1104	if (report32[0] == 0) {
1105	if (__ratelimit(&stream->perf->spurious_report_rs))
1106	drm_notice(&uncore->i915->drm,
1107	"Skipping spurious, invalid OA report\n");
1108	continue;
1109	}
1110
1111	ret = append_oa_sample(stream, buf, count, offset, report);
1112	if (ret)
1113	break;
1114
1115	/* Clear out the first 2 dwords as a mean to detect unlanded
1116	* reports.
1117	*/
1118	report32[0] = 0;
1119	report32[1] = 0;
1120	}
1121
1122	if (start_offset != *offset) {
1123	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1124
1125	intel_uncore_write(uncore, GEN7_OASTATUS2,
1126	val: ((head + gtt_offset) & GEN7_OASTATUS2_HEAD_MASK) |
1127	GEN7_OASTATUS2_MEM_SELECT_GGTT);
1128	stream->oa_buffer.head = head;
1129
1130	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1131	}
1132
1133	return ret;
1134	}
1135
1136	/**
1137	* gen7_oa_read - copy status records then buffered OA reports
1138	* @stream: An i915-perf stream opened for OA metrics
1139	* @buf: destination buffer given by userspace
1140	* @count: the number of bytes userspace wants to read
1141	* @offset: (inout): the current position for writing into @buf
1142	*
1143	* Checks Gen 7 specific OA unit status registers and if necessary appends
1144	* corresponding status records for userspace (such as for a buffer full
1145	* condition) and then initiate appending any buffered OA reports.
1146	*
1147	* Updates @offset according to the number of bytes successfully copied into
1148	* the userspace buffer.
1149	*
1150	* Returns: zero on success or a negative error code
1151	*/
1152	static int gen7_oa_read(struct i915_perf_stream *stream,
1153	char __user *buf,
1154	size_t count,
1155	size_t *offset)
1156	{
1157	struct intel_uncore *uncore = stream->uncore;
1158	u32 oastatus1;
1159	int ret;
1160
1161	if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
1162	return -EIO;
1163
1164	oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
1165
1166	/* XXX: On Haswell we don't have a safe way to clear oastatus1
1167	* bits while the OA unit is enabled (while the tail pointer
1168	* may be updated asynchronously) so we ignore status bits
1169	* that have already been reported to userspace.
1170	*/
1171	oastatus1 &= ~stream->perf->gen7_latched_oastatus1;
1172
1173	/* We treat OABUFFER_OVERFLOW as a significant error:
1174	*
1175	* - The status can be interpreted to mean that the buffer is
1176	* currently full (with a higher precedence than OA_TAKEN()
1177	* which will start to report a near-empty buffer after an
1178	* overflow) but it's awkward that we can't clear the status
1179	* on Haswell, so without a reset we won't be able to catch
1180	* the state again.
1181	*
1182	* - Since it also implies the HW has started overwriting old
1183	* reports it may also affect our sanity checks for invalid
1184	* reports when copying to userspace that assume new reports
1185	* are being written to cleared memory.
1186	*
1187	* - In the future we may want to introduce a flight recorder
1188	* mode where the driver will automatically maintain a safe
1189	* guard band between head/tail, avoiding this overflow
1190	* condition, but we avoid the added driver complexity for
1191	* now.
1192	*/
1193	if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) {
1194	ret = append_oa_status(stream, buf, count, offset,
1195	type: DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
1196	if (ret)
1197	return ret;
1198
1199	drm_dbg(&stream->perf->i915->drm,
1200	"OA buffer overflow (exponent = %d): force restart\n",
1201	stream->period_exponent);
1202
1203	stream->perf->ops.oa_disable(stream);
1204	stream->perf->ops.oa_enable(stream);
1205
1206	oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
1207	}
1208
1209	if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) {
1210	ret = append_oa_status(stream, buf, count, offset,
1211	type: DRM_I915_PERF_RECORD_OA_REPORT_LOST);
1212	if (ret)
1213	return ret;
1214	stream->perf->gen7_latched_oastatus1 |=
1215	GEN7_OASTATUS1_REPORT_LOST;
1216	}
1217
1218	return gen7_append_oa_reports(stream, buf, count, offset);
1219	}
1220
1221	/**
1222	* i915_oa_wait_unlocked - handles blocking IO until OA data available
1223	* @stream: An i915-perf stream opened for OA metrics
1224	*
1225	* Called when userspace tries to read() from a blocking stream FD opened
1226	* for OA metrics. It waits until the hrtimer callback finds a non-empty
1227	* OA buffer and wakes us.
1228	*
1229	* Note: it's acceptable to have this return with some false positives
1230	* since any subsequent read handling will return -EAGAIN if there isn't
1231	* really data ready for userspace yet.
1232	*
1233	* Returns: zero on success or a negative error code
1234	*/
1235	static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
1236	{
1237	/* We would wait indefinitely if periodic sampling is not enabled */
1238	if (!stream->periodic)
1239	return -EIO;
1240
1241	return wait_event_interruptible(stream->poll_wq,
1242	oa_buffer_check_unlocked(stream));
1243	}
1244
1245	/**
1246	* i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1247	* @stream: An i915-perf stream opened for OA metrics
1248	* @file: An i915 perf stream file
1249	* @wait: poll() state table
1250	*
1251	* For handling userspace polling on an i915 perf stream opened for OA metrics,
1252	* this starts a poll_wait with the wait queue that our hrtimer callback wakes
1253	* when it sees data ready to read in the circular OA buffer.
1254	*/
1255	static void i915_oa_poll_wait(struct i915_perf_stream *stream,
1256	struct file *file,
1257	poll_table *wait)
1258	{
1259	poll_wait(filp: file, wait_address: &stream->poll_wq, p: wait);
1260	}
1261
1262	/**
1263	* i915_oa_read - just calls through to &i915_oa_ops->read
1264	* @stream: An i915-perf stream opened for OA metrics
1265	* @buf: destination buffer given by userspace
1266	* @count: the number of bytes userspace wants to read
1267	* @offset: (inout): the current position for writing into @buf
1268	*
1269	* Updates @offset according to the number of bytes successfully copied into
1270	* the userspace buffer.
1271	*
1272	* Returns: zero on success or a negative error code
1273	*/
1274	static int i915_oa_read(struct i915_perf_stream *stream,
1275	char __user *buf,
1276	size_t count,
1277	size_t *offset)
1278	{
1279	return stream->perf->ops.read(stream, buf, count, offset);
1280	}
1281
1282	static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
1283	{
1284	struct i915_gem_engines_iter it;
1285	struct i915_gem_context *ctx = stream->ctx;
1286	struct intel_context *ce;
1287	struct i915_gem_ww_ctx ww;
1288	int err = -ENODEV;
1289
1290	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1291	if (ce->engine != stream->engine) /* first match! */
1292	continue;
1293
1294	err = 0;
1295	break;
1296	}
1297	i915_gem_context_unlock_engines(ctx);
1298
1299	if (err)
1300	return ERR_PTR(error: err);
1301
1302	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
1303	retry:
1304	/*
1305	* As the ID is the gtt offset of the context's vma we
1306	* pin the vma to ensure the ID remains fixed.
1307	*/
1308	err = intel_context_pin_ww(ce, ww: &ww);
1309	if (err == -EDEADLK) {
1310	err = i915_gem_ww_ctx_backoff(ctx: &ww);
1311	if (!err)
1312	goto retry;
1313	}
1314	i915_gem_ww_ctx_fini(ctx: &ww);
1315
1316	if (err)
1317	return ERR_PTR(error: err);
1318
1319	stream->pinned_ctx = ce;
1320	return stream->pinned_ctx;
1321	}
1322
1323	static int
1324	__store_reg_to_mem(struct i915_request *rq, i915_reg_t reg, u32 ggtt_offset)
1325	{
1326	u32 *cs, cmd;
1327
1328	cmd = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1329	if (GRAPHICS_VER(rq->i915) >= 8)
1330	cmd++;
1331
1332	cs = intel_ring_begin(rq, num_dwords: 4);
1333	if (IS_ERR(ptr: cs))
1334	return PTR_ERR(ptr: cs);
1335
1336	*cs++ = cmd;
1337	*cs++ = i915_mmio_reg_offset(reg);
1338	*cs++ = ggtt_offset;
1339	*cs++ = 0;
1340
1341	intel_ring_advance(rq, cs);
1342
1343	return 0;
1344	}
1345
1346	static int
1347	__read_reg(struct intel_context *ce, i915_reg_t reg, u32 ggtt_offset)
1348	{
1349	struct i915_request *rq;
1350	int err;
1351
1352	rq = i915_request_create(ce);
1353	if (IS_ERR(ptr: rq))
1354	return PTR_ERR(ptr: rq);
1355
1356	i915_request_get(rq);
1357
1358	err = __store_reg_to_mem(rq, reg, ggtt_offset);
1359
1360	i915_request_add(rq);
1361	if (!err && i915_request_wait(rq, flags: 0, HZ / 2) < 0)
1362	err = -ETIME;
1363
1364	i915_request_put(rq);
1365
1366	return err;
1367	}
1368
1369	static int
1370	gen12_guc_sw_ctx_id(struct intel_context *ce, u32 *ctx_id)
1371	{
1372	struct i915_vma *scratch;
1373	u32 *val;
1374	int err;
1375
1376	scratch = __vm_create_scratch_for_read_pinned(vm: &ce->engine->gt->ggtt->vm, size: 4);
1377	if (IS_ERR(ptr: scratch))
1378	return PTR_ERR(ptr: scratch);
1379
1380	err = i915_vma_sync(vma: scratch);
1381	if (err)
1382	goto err_scratch;
1383
1384	err = __read_reg(ce, RING_EXECLIST_STATUS_HI(ce->engine->mmio_base),
1385	ggtt_offset: i915_ggtt_offset(vma: scratch));
1386	if (err)
1387	goto err_scratch;
1388
1389	val = i915_gem_object_pin_map_unlocked(obj: scratch->obj, type: I915_MAP_WB);
1390	if (IS_ERR(ptr: val)) {
1391	err = PTR_ERR(ptr: val);
1392	goto err_scratch;
1393	}
1394
1395	*ctx_id = *val;
1396	i915_gem_object_unpin_map(obj: scratch->obj);
1397
1398	err_scratch:
1399	i915_vma_unpin_and_release(p_vma: &scratch, flags: 0);
1400	return err;
1401	}
1402
1403	/*
1404	* For execlist mode of submission, pick an unused context id
1405	* 0 - (NUM_CONTEXT_TAG -1) are used by other contexts
1406	* XXX_MAX_CONTEXT_HW_ID is used by idle context
1407	*
1408	* For GuC mode of submission read context id from the upper dword of the
1409	* EXECLIST_STATUS register. Note that we read this value only once and expect
1410	* that the value stays fixed for the entire OA use case. There are cases where
1411	* GuC KMD implementation may deregister a context to reuse it's context id, but
1412	* we prevent that from happening to the OA context by pinning it.
1413	*/
1414	static int gen12_get_render_context_id(struct i915_perf_stream *stream)
1415	{
1416	u32 ctx_id, mask;
1417	int ret;
1418
1419	if (intel_engine_uses_guc(engine: stream->engine)) {
1420	ret = gen12_guc_sw_ctx_id(ce: stream->pinned_ctx, ctx_id: &ctx_id);
1421	if (ret)
1422	return ret;
1423
1424	mask = ((1U << GEN12_GUC_SW_CTX_ID_WIDTH) - 1) <<
1425	(GEN12_GUC_SW_CTX_ID_SHIFT - 32);
1426	} else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 55)) {
1427	ctx_id = (XEHP_MAX_CONTEXT_HW_ID - 1) <<
1428	(XEHP_SW_CTX_ID_SHIFT - 32);
1429
1430	mask = ((1U << XEHP_SW_CTX_ID_WIDTH) - 1) <<
1431	(XEHP_SW_CTX_ID_SHIFT - 32);
1432	} else {
1433	ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) <<
1434	(GEN11_SW_CTX_ID_SHIFT - 32);
1435
1436	mask = ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) <<
1437	(GEN11_SW_CTX_ID_SHIFT - 32);
1438	}
1439	stream->specific_ctx_id = ctx_id & mask;
1440	stream->specific_ctx_id_mask = mask;
1441
1442	return 0;
1443	}
1444
1445	static bool oa_find_reg_in_lri(u32 *state, u32 reg, u32 *offset, u32 end)
1446	{
1447	u32 idx = *offset;
1448	u32 len = min(MI_LRI_LEN(state[idx]) + idx, end);
1449	bool found = false;
1450
1451	idx++;
1452	for (; idx < len; idx += 2) {
1453	if (state[idx] == reg) {
1454	found = true;
1455	break;
1456	}
1457	}
1458
1459	*offset = idx;
1460	return found;
1461	}
1462
1463	static u32 oa_context_image_offset(struct intel_context *ce, u32 reg)
1464	{
1465	u32 offset, len = (ce->engine->context_size - PAGE_SIZE) / 4;
1466	u32 *state = ce->lrc_reg_state;
1467
1468	if (drm_WARN_ON(&ce->engine->i915->drm, !state))
1469	return U32_MAX;
1470
1471	for (offset = 0; offset < len; ) {
1472	if (IS_MI_LRI_CMD(state[offset])) {
1473	/*
1474	* We expect reg-value pairs in MI_LRI command, so
1475	* MI_LRI_LEN() should be even, if not, issue a warning.
1476	*/
1477	drm_WARN_ON(&ce->engine->i915->drm,
1478	MI_LRI_LEN(state[offset]) & 0x1);
1479
1480	if (oa_find_reg_in_lri(state, reg, offset: &offset, end: len))
1481	break;
1482	} else {
1483	offset++;
1484	}
1485	}
1486
1487	return offset < len ? offset : U32_MAX;
1488	}
1489
1490	static int set_oa_ctx_ctrl_offset(struct intel_context *ce)
1491	{
1492	i915_reg_t reg = GEN12_OACTXCONTROL(ce->engine->mmio_base);
1493	struct i915_perf *perf = &ce->engine->i915->perf;
1494	u32 offset = perf->ctx_oactxctrl_offset;
1495
1496	/* Do this only once. Failure is stored as offset of U32_MAX */
1497	if (offset)
1498	goto exit;
1499
1500	offset = oa_context_image_offset(ce, i915_mmio_reg_offset(reg));
1501	perf->ctx_oactxctrl_offset = offset;
1502
1503	drm_dbg(&ce->engine->i915->drm,
1504	"%s oa ctx control at 0x%08x dword offset\n",
1505	ce->engine->name, offset);
1506
1507	exit:
1508	return offset && offset != U32_MAX ? 0 : -ENODEV;
1509	}
1510
1511	static bool engine_supports_mi_query(struct intel_engine_cs *engine)
1512	{
1513	return engine->class == RENDER_CLASS;
1514	}
1515
1516	/**
1517	* oa_get_render_ctx_id - determine and hold ctx hw id
1518	* @stream: An i915-perf stream opened for OA metrics
1519	*
1520	* Determine the render context hw id, and ensure it remains fixed for the
1521	* lifetime of the stream. This ensures that we don't have to worry about
1522	* updating the context ID in OACONTROL on the fly.
1523	*
1524	* Returns: zero on success or a negative error code
1525	*/
1526	static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
1527	{
1528	struct intel_context *ce;
1529	int ret = 0;
1530
1531	ce = oa_pin_context(stream);
1532	if (IS_ERR(ptr: ce))
1533	return PTR_ERR(ptr: ce);
1534
1535	if (engine_supports_mi_query(engine: stream->engine) &&
1536	HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) {
1537	/*
1538	* We are enabling perf query here. If we don't find the context
1539	* offset here, just return an error.
1540	*/
1541	ret = set_oa_ctx_ctrl_offset(ce);
1542	if (ret) {
1543	intel_context_unpin(ce);
1544	drm_err(&stream->perf->i915->drm,
1545	"Enabling perf query failed for %s\n",
1546	stream->engine->name);
1547	return ret;
1548	}
1549	}
1550
1551	switch (GRAPHICS_VER(ce->engine->i915)) {
1552	case 7: {
1553	/*
1554	* On Haswell we don't do any post processing of the reports
1555	* and don't need to use the mask.
1556	*/
1557	stream->specific_ctx_id = i915_ggtt_offset(vma: ce->state);
1558	stream->specific_ctx_id_mask = 0;
1559	break;
1560	}
1561
1562	case 8:
1563	case 9:
1564	if (intel_engine_uses_guc(engine: ce->engine)) {
1565	/*
1566	* When using GuC, the context descriptor we write in
1567	* i915 is read by GuC and rewritten before it's
1568	* actually written into the hardware. The LRCA is
1569	* what is put into the context id field of the
1570	* context descriptor by GuC. Because it's aligned to
1571	* a page, the lower 12bits are always at 0 and
1572	* dropped by GuC. They won't be part of the context
1573	* ID in the OA reports, so squash those lower bits.
1574	*/
1575	stream->specific_ctx_id = ce->lrc.lrca >> 12;
1576
1577	/*
1578	* GuC uses the top bit to signal proxy submission, so
1579	* ignore that bit.
1580	*/
1581	stream->specific_ctx_id_mask =
1582	(1U << (GEN8_CTX_ID_WIDTH - 1)) - 1;
1583	} else {
1584	stream->specific_ctx_id_mask =
1585	(1U << GEN8_CTX_ID_WIDTH) - 1;
1586	stream->specific_ctx_id = stream->specific_ctx_id_mask;
1587	}
1588	break;
1589
1590	case 11:
1591	case 12:
1592	ret = gen12_get_render_context_id(stream);
1593	break;
1594
1595	default:
1596	MISSING_CASE(GRAPHICS_VER(ce->engine->i915));
1597	}
1598
1599	ce->tag = stream->specific_ctx_id;
1600
1601	drm_dbg(&stream->perf->i915->drm,
1602	"filtering on ctx_id=0x%x ctx_id_mask=0x%x\n",
1603	stream->specific_ctx_id,
1604	stream->specific_ctx_id_mask);
1605
1606	return ret;
1607	}
1608
1609	/**
1610	* oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1611	* @stream: An i915-perf stream opened for OA metrics
1612	*
1613	* In case anything needed doing to ensure the context HW ID would remain valid
1614	* for the lifetime of the stream, then that can be undone here.
1615	*/
1616	static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
1617	{
1618	struct intel_context *ce;
1619
1620	ce = fetch_and_zero(&stream->pinned_ctx);
1621	if (ce) {
1622	ce->tag = 0; /* recomputed on next submission after parking */
1623	intel_context_unpin(ce);
1624	}
1625
1626	stream->specific_ctx_id = INVALID_CTX_ID;
1627	stream->specific_ctx_id_mask = 0;
1628	}
1629
1630	static void
1631	free_oa_buffer(struct i915_perf_stream *stream)
1632	{
1633	i915_vma_unpin_and_release(p_vma: &stream->oa_buffer.vma,
1634	I915_VMA_RELEASE_MAP);
1635
1636	stream->oa_buffer.vaddr = NULL;
1637	}
1638
1639	static void
1640	free_oa_configs(struct i915_perf_stream *stream)
1641	{
1642	struct i915_oa_config_bo *oa_bo, *tmp;
1643
1644	i915_oa_config_put(oa_config: stream->oa_config);
1645	llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
1646	free_oa_config_bo(oa_bo);
1647	}
1648
1649	static void
1650	free_noa_wait(struct i915_perf_stream *stream)
1651	{
1652	i915_vma_unpin_and_release(p_vma: &stream->noa_wait, flags: 0);
1653	}
1654
1655	static bool engine_supports_oa(const struct intel_engine_cs *engine)
1656	{
1657	return engine->oa_group;
1658	}
1659
1660	static bool engine_supports_oa_format(struct intel_engine_cs *engine, int type)
1661	{
1662	return engine->oa_group && engine->oa_group->type == type;
1663	}
1664
1665	static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1666	{
1667	struct i915_perf *perf = stream->perf;
1668	struct intel_gt *gt = stream->engine->gt;
1669	struct i915_perf_group *g = stream->engine->oa_group;
1670	int m;
1671
1672	if (WARN_ON(stream != g->exclusive_stream))
1673	return;
1674
1675	/*
1676	* Unset exclusive_stream first, it will be checked while disabling
1677	* the metric set on gen8+.
1678	*
1679	* See i915_oa_init_reg_state() and lrc_configure_all_contexts()
1680	*/
1681	WRITE_ONCE(g->exclusive_stream, NULL);
1682	perf->ops.disable_metric_set(stream);
1683
1684	free_oa_buffer(stream);
1685
1686	intel_uncore_forcewake_put(uncore: stream->uncore, domains: FORCEWAKE_ALL);
1687	intel_engine_pm_put(engine: stream->engine);
1688
1689	if (stream->ctx)
1690	oa_put_render_ctx_id(stream);
1691
1692	free_oa_configs(stream);
1693	free_noa_wait(stream);
1694
1695	m = ratelimit_state_get_miss(rs: &perf->spurious_report_rs);
1696	if (m)
1697	gt_notice(gt, "%d spurious OA report notices suppressed due to ratelimiting\n", m);
1698	}
1699
1700	static void gen7_init_oa_buffer(struct i915_perf_stream *stream)
1701	{
1702	struct intel_uncore *uncore = stream->uncore;
1703	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1704	unsigned long flags;
1705
1706	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1707
1708	/* Pre-DevBDW: OABUFFER must be set with counters off,
1709	* before OASTATUS1, but after OASTATUS2
1710	*/
1711	intel_uncore_write(uncore, GEN7_OASTATUS2, /* head */
1712	val: gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT);
1713	stream->oa_buffer.head = 0;
1714
1715	intel_uncore_write(uncore, GEN7_OABUFFER, val: gtt_offset);
1716
1717	intel_uncore_write(uncore, GEN7_OASTATUS1, /* tail */
1718	val: gtt_offset | OABUFFER_SIZE_16M);
1719
1720	/* Mark that we need updated tail pointers to read from... */
1721	stream->oa_buffer.tail = 0;
1722
1723	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1724
1725	/* On Haswell we have to track which OASTATUS1 flags we've
1726	* already seen since they can't be cleared while periodic
1727	* sampling is enabled.
1728	*/
1729	stream->perf->gen7_latched_oastatus1 = 0;
1730
1731	/* NB: although the OA buffer will initially be allocated
1732	* zeroed via shmfs (and so this memset is redundant when
1733	* first allocating), we may re-init the OA buffer, either
1734	* when re-enabling a stream or in error/reset paths.
1735	*
1736	* The reason we clear the buffer for each re-init is for the
1737	* sanity check in gen7_append_oa_reports() that looks at the
1738	* report-id field to make sure it's non-zero which relies on
1739	* the assumption that new reports are being written to zeroed
1740	* memory...
1741	*/
1742	memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1743	}
1744
1745	static void gen8_init_oa_buffer(struct i915_perf_stream *stream)
1746	{
1747	struct intel_uncore *uncore = stream->uncore;
1748	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1749	unsigned long flags;
1750
1751	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1752
1753	intel_uncore_write(uncore, GEN8_OASTATUS, val: 0);
1754	intel_uncore_write(uncore, GEN8_OAHEADPTR, val: gtt_offset);
1755	stream->oa_buffer.head = 0;
1756
1757	intel_uncore_write(uncore, GEN8_OABUFFER_UDW, val: 0);
1758
1759	/*
1760	* PRM says:
1761	*
1762	* "This MMIO must be set before the OATAILPTR
1763	* register and after the OAHEADPTR register. This is
1764	* to enable proper functionality of the overflow
1765	* bit."
1766	*/
1767	intel_uncore_write(uncore, GEN8_OABUFFER, val: gtt_offset |
1768	OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
1769	intel_uncore_write(uncore, GEN8_OATAILPTR, val: gtt_offset & GEN8_OATAILPTR_MASK);
1770
1771	/* Mark that we need updated tail pointers to read from... */
1772	stream->oa_buffer.tail = 0;
1773
1774	/*
1775	* Reset state used to recognise context switches, affecting which
1776	* reports we will forward to userspace while filtering for a single
1777	* context.
1778	*/
1779	stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1780
1781	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1782
1783	/*
1784	* NB: although the OA buffer will initially be allocated
1785	* zeroed via shmfs (and so this memset is redundant when
1786	* first allocating), we may re-init the OA buffer, either
1787	* when re-enabling a stream or in error/reset paths.
1788	*
1789	* The reason we clear the buffer for each re-init is for the
1790	* sanity check in gen8_append_oa_reports() that looks at the
1791	* reason field to make sure it's non-zero which relies on
1792	* the assumption that new reports are being written to zeroed
1793	* memory...
1794	*/
1795	memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1796	}
1797
1798	static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
1799	{
1800	struct intel_uncore *uncore = stream->uncore;
1801	u32 gtt_offset = i915_ggtt_offset(vma: stream->oa_buffer.vma);
1802	unsigned long flags;
1803
1804	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1805
1806	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_status, val: 0);
1807	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_head_ptr,
1808	val: gtt_offset & GEN12_OAG_OAHEADPTR_MASK);
1809	stream->oa_buffer.head = 0;
1810
1811	/*
1812	* PRM says:
1813	*
1814	* "This MMIO must be set before the OATAILPTR
1815	* register and after the OAHEADPTR register. This is
1816	* to enable proper functionality of the overflow
1817	* bit."
1818	*/
1819	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_buffer, val: gtt_offset |
1820	OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
1821	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_tail_ptr,
1822	val: gtt_offset & GEN12_OAG_OATAILPTR_MASK);
1823
1824	/* Mark that we need updated tail pointers to read from... */
1825	stream->oa_buffer.tail = 0;
1826
1827	/*
1828	* Reset state used to recognise context switches, affecting which
1829	* reports we will forward to userspace while filtering for a single
1830	* context.
1831	*/
1832	stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1833
1834	spin_unlock_irqrestore(lock: &stream->oa_buffer.ptr_lock, flags);
1835
1836	/*
1837	* NB: although the OA buffer will initially be allocated
1838	* zeroed via shmfs (and so this memset is redundant when
1839	* first allocating), we may re-init the OA buffer, either
1840	* when re-enabling a stream or in error/reset paths.
1841	*
1842	* The reason we clear the buffer for each re-init is for the
1843	* sanity check in gen8_append_oa_reports() that looks at the
1844	* reason field to make sure it's non-zero which relies on
1845	* the assumption that new reports are being written to zeroed
1846	* memory...
1847	*/
1848	memset(stream->oa_buffer.vaddr, 0,
1849	stream->oa_buffer.vma->size);
1850	}
1851
1852	static int alloc_oa_buffer(struct i915_perf_stream *stream)
1853	{
1854	struct drm_i915_private *i915 = stream->perf->i915;
1855	struct intel_gt *gt = stream->engine->gt;
1856	struct drm_i915_gem_object *bo;
1857	struct i915_vma *vma;
1858	int ret;
1859
1860	if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))
1861	return -ENODEV;
1862
1863	BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE);
1864	BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
1865
1866	bo = i915_gem_object_create_shmem(i915: stream->perf->i915, OA_BUFFER_SIZE);
1867	if (IS_ERR(ptr: bo)) {
1868	drm_err(&i915->drm, "Failed to allocate OA buffer\n");
1869	return PTR_ERR(ptr: bo);
1870	}
1871
1872	i915_gem_object_set_cache_coherency(obj: bo, cache_level: I915_CACHE_LLC);
1873
1874	/* PreHSW required 512K alignment, HSW requires 16M */
1875	vma = i915_vma_instance(obj: bo, vm: &gt->ggtt->vm, NULL);
1876	if (IS_ERR(ptr: vma)) {
1877	ret = PTR_ERR(ptr: vma);
1878	goto err_unref;
1879	}
1880
1881	/*
1882	* PreHSW required 512K alignment.
1883	* HSW and onwards, align to requested size of OA buffer.
1884	*/
1885	ret = i915_vma_pin(vma, size: 0, SZ_16M, PIN_GLOBAL | PIN_HIGH);
1886	if (ret) {
1887	gt_err(gt, "Failed to pin OA buffer %d\n", ret);
1888	goto err_unref;
1889	}
1890
1891	stream->oa_buffer.vma = vma;
1892
1893	stream->oa_buffer.vaddr =
1894	i915_gem_object_pin_map_unlocked(obj: bo, type: I915_MAP_WB);
1895	if (IS_ERR(ptr: stream->oa_buffer.vaddr)) {
1896	ret = PTR_ERR(ptr: stream->oa_buffer.vaddr);
1897	goto err_unpin;
1898	}
1899
1900	return 0;
1901
1902	err_unpin:
1903	__i915_vma_unpin(vma);
1904
1905	err_unref:
1906	i915_gem_object_put(obj: bo);
1907
1908	stream->oa_buffer.vaddr = NULL;
1909	stream->oa_buffer.vma = NULL;
1910
1911	return ret;
1912	}
1913
1914	static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs,
1915	bool save, i915_reg_t reg, u32 offset,
1916	u32 dword_count)
1917	{
1918	u32 cmd;
1919	u32 d;
1920
1921	cmd = save ? MI_STORE_REGISTER_MEM : MI_LOAD_REGISTER_MEM;
1922	cmd |= MI_SRM_LRM_GLOBAL_GTT;
1923	if (GRAPHICS_VER(stream->perf->i915) >= 8)
1924	cmd++;
1925
1926	for (d = 0; d < dword_count; d++) {
1927	*cs++ = cmd;
1928	*cs++ = i915_mmio_reg_offset(reg) + 4 * d;
1929	*cs++ = i915_ggtt_offset(vma: stream->noa_wait) + offset + 4 * d;
1930	*cs++ = 0;
1931	}
1932
1933	return cs;
1934	}
1935
1936	static int alloc_noa_wait(struct i915_perf_stream *stream)
1937	{
1938	struct drm_i915_private *i915 = stream->perf->i915;
1939	struct intel_gt *gt = stream->engine->gt;
1940	struct drm_i915_gem_object *bo;
1941	struct i915_vma *vma;
1942	const u64 delay_ticks = 0xffffffffffffffff -
1943	intel_gt_ns_to_clock_interval(gt: to_gt(i915: stream->perf->i915),
1944	ns: atomic64_read(v: &stream->perf->noa_programming_delay));
1945	const u32 base = stream->engine->mmio_base;
1946	#define CS_GPR(x) GEN8_RING_CS_GPR(base, x)
1947	u32 *batch, *ts0, *cs, *jump;
1948	struct i915_gem_ww_ctx ww;
1949	int ret, i;
1950	enum {
1951	START_TS,
1952	NOW_TS,
1953	DELTA_TS,
1954	JUMP_PREDICATE,
1955	DELTA_TARGET,
1956	N_CS_GPR
1957	};
1958	i915_reg_t mi_predicate_result = HAS_MI_SET_PREDICATE(i915) ?
1959	MI_PREDICATE_RESULT_2_ENGINE(base) :
1960	MI_PREDICATE_RESULT_1(RENDER_RING_BASE);
1961
1962	/*
1963	* gt->scratch was being used to save/restore the GPR registers, but on
1964	* MTL the scratch uses stolen lmem. An MI_SRM to this memory region
1965	* causes an engine hang. Instead allocate an additional page here to
1966	* save/restore GPR registers
1967	*/
1968	bo = i915_gem_object_create_internal(i915, size: 8192);
1969	if (IS_ERR(ptr: bo)) {
1970	drm_err(&i915->drm,
1971	"Failed to allocate NOA wait batchbuffer\n");
1972	return PTR_ERR(ptr: bo);
1973	}
1974
1975	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
1976	retry:
1977	ret = i915_gem_object_lock(obj: bo, ww: &ww);
1978	if (ret)
1979	goto out_ww;
1980
1981	/*
1982	* We pin in GGTT because we jump into this buffer now because
1983	* multiple OA config BOs will have a jump to this address and it
1984	* needs to be fixed during the lifetime of the i915/perf stream.
1985	*/
1986	vma = i915_vma_instance(obj: bo, vm: &gt->ggtt->vm, NULL);
1987	if (IS_ERR(ptr: vma)) {
1988	ret = PTR_ERR(ptr: vma);
1989	goto out_ww;
1990	}
1991
1992	ret = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment: 0, PIN_GLOBAL | PIN_HIGH);
1993	if (ret)
1994	goto out_ww;
1995
1996	batch = cs = i915_gem_object_pin_map(obj: bo, type: I915_MAP_WB);
1997	if (IS_ERR(ptr: batch)) {
1998	ret = PTR_ERR(ptr: batch);
1999	goto err_unpin;
2000	}
2001
2002	stream->noa_wait = vma;
2003
2004	#define GPR_SAVE_OFFSET 4096
2005	#define PREDICATE_SAVE_OFFSET 4160
2006
2007	/* Save registers. */
2008	for (i = 0; i < N_CS_GPR; i++)
2009	cs = save_restore_register(
2010	stream, cs, save: true /* save */, CS_GPR(i),
2011	GPR_SAVE_OFFSET + 8 * i, dword_count: 2);
2012	cs = save_restore_register(
2013	stream, cs, save: true /* save */, reg: mi_predicate_result,
2014	PREDICATE_SAVE_OFFSET, dword_count: 1);
2015
2016	/* First timestamp snapshot location. */
2017	ts0 = cs;
2018
2019	/*
2020	* Initial snapshot of the timestamp register to implement the wait.
2021	* We work with 32b values, so clear out the top 32b bits of the
2022	* register because the ALU works 64bits.
2023	*/
2024	*cs++ = MI_LOAD_REGISTER_IMM(1);
2025	*cs++ = i915_mmio_reg_offset(CS_GPR(START_TS)) + 4;
2026	*cs++ = 0;
2027	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2028	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(base));
2029	*cs++ = i915_mmio_reg_offset(CS_GPR(START_TS));
2030
2031	/*
2032	* This is the location we're going to jump back into until the
2033	* required amount of time has passed.
2034	*/
2035	jump = cs;
2036
2037	/*
2038	* Take another snapshot of the timestamp register. Take care to clear
2039	* up the top 32bits of CS_GPR(1) as we're using it for other
2040	* operations below.
2041	*/
2042	*cs++ = MI_LOAD_REGISTER_IMM(1);
2043	*cs++ = i915_mmio_reg_offset(CS_GPR(NOW_TS)) + 4;
2044	*cs++ = 0;
2045	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2046	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(base));
2047	*cs++ = i915_mmio_reg_offset(CS_GPR(NOW_TS));
2048
2049	/*
2050	* Do a diff between the 2 timestamps and store the result back into
2051	* CS_GPR(1).
2052	*/
2053	*cs++ = MI_MATH(5);
2054	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(NOW_TS));
2055	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(START_TS));
2056	*cs++ = MI_MATH_SUB;
2057	*cs++ = MI_MATH_STORE(MI_MATH_REG(DELTA_TS), MI_MATH_REG_ACCU);
2058	*cs++ = MI_MATH_STORE(MI_MATH_REG(JUMP_PREDICATE), MI_MATH_REG_CF);
2059
2060	/*
2061	* Transfer the carry flag (set to 1 if ts1 < ts0, meaning the
2062	* timestamp have rolled over the 32bits) into the predicate register
2063	* to be used for the predicated jump.
2064	*/
2065	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2066	*cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE));
2067	*cs++ = i915_mmio_reg_offset(mi_predicate_result);
2068
2069	if (HAS_MI_SET_PREDICATE(i915))
2070	*cs++ = MI_SET_PREDICATE | 1;
2071
2072	/* Restart from the beginning if we had timestamps roll over. */
2073	*cs++ = (GRAPHICS_VER(i915) < 8 ?
2074	MI_BATCH_BUFFER_START :
2075	MI_BATCH_BUFFER_START_GEN8) |
2076	MI_BATCH_PREDICATE;
2077	*cs++ = i915_ggtt_offset(vma) + (ts0 - batch) * 4;
2078	*cs++ = 0;
2079
2080	if (HAS_MI_SET_PREDICATE(i915))
2081	*cs++ = MI_SET_PREDICATE;
2082
2083	/*
2084	* Now add the diff between to previous timestamps and add it to :
2085	* (((1 * << 64) - 1) - delay_ns)
2086	*
2087	* When the Carry Flag contains 1 this means the elapsed time is
2088	* longer than the expected delay, and we can exit the wait loop.
2089	*/
2090	*cs++ = MI_LOAD_REGISTER_IMM(2);
2091	*cs++ = i915_mmio_reg_offset(CS_GPR(DELTA_TARGET));
2092	*cs++ = lower_32_bits(delay_ticks);
2093	*cs++ = i915_mmio_reg_offset(CS_GPR(DELTA_TARGET)) + 4;
2094	*cs++ = upper_32_bits(delay_ticks);
2095
2096	*cs++ = MI_MATH(4);
2097	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(DELTA_TS));
2098	*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(DELTA_TARGET));
2099	*cs++ = MI_MATH_ADD;
2100	*cs++ = MI_MATH_STOREINV(MI_MATH_REG(JUMP_PREDICATE), MI_MATH_REG_CF);
2101
2102	*cs++ = MI_ARB_CHECK;
2103
2104	/*
2105	* Transfer the result into the predicate register to be used for the
2106	* predicated jump.
2107	*/
2108	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
2109	*cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE));
2110	*cs++ = i915_mmio_reg_offset(mi_predicate_result);
2111
2112	if (HAS_MI_SET_PREDICATE(i915))
2113	*cs++ = MI_SET_PREDICATE | 1;
2114
2115	/* Predicate the jump. */
2116	*cs++ = (GRAPHICS_VER(i915) < 8 ?
2117	MI_BATCH_BUFFER_START :
2118	MI_BATCH_BUFFER_START_GEN8) |
2119	MI_BATCH_PREDICATE;
2120	*cs++ = i915_ggtt_offset(vma) + (jump - batch) * 4;
2121	*cs++ = 0;
2122
2123	if (HAS_MI_SET_PREDICATE(i915))
2124	*cs++ = MI_SET_PREDICATE;
2125
2126	/* Restore registers. */
2127	for (i = 0; i < N_CS_GPR; i++)
2128	cs = save_restore_register(
2129	stream, cs, save: false /* restore */, CS_GPR(i),
2130	GPR_SAVE_OFFSET + 8 * i, dword_count: 2);
2131	cs = save_restore_register(
2132	stream, cs, save: false /* restore */, reg: mi_predicate_result,
2133	PREDICATE_SAVE_OFFSET, dword_count: 1);
2134
2135	/* And return to the ring. */
2136	*cs++ = MI_BATCH_BUFFER_END;
2137
2138	GEM_BUG_ON(cs - batch > PAGE_SIZE / sizeof(*batch));
2139
2140	i915_gem_object_flush_map(obj: bo);
2141	__i915_gem_object_release_map(obj: bo);
2142
2143	goto out_ww;
2144
2145	err_unpin:
2146	i915_vma_unpin_and_release(p_vma: &vma, flags: 0);
2147	out_ww:
2148	if (ret == -EDEADLK) {
2149	ret = i915_gem_ww_ctx_backoff(ctx: &ww);
2150	if (!ret)
2151	goto retry;
2152	}
2153	i915_gem_ww_ctx_fini(ctx: &ww);
2154	if (ret)
2155	i915_gem_object_put(obj: bo);
2156	return ret;
2157	}
2158
2159	static u32 *write_cs_mi_lri(u32 *cs,
2160	const struct i915_oa_reg *reg_data,
2161	u32 n_regs)
2162	{
2163	u32 i;
2164
2165	for (i = 0; i < n_regs; i++) {
2166	if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) {
2167	u32 n_lri = min_t(u32,
2168	n_regs - i,
2169	MI_LOAD_REGISTER_IMM_MAX_REGS);
2170
2171	*cs++ = MI_LOAD_REGISTER_IMM(n_lri);
2172	}
2173	*cs++ = i915_mmio_reg_offset(reg_data[i].addr);
2174	*cs++ = reg_data[i].value;
2175	}
2176
2177	return cs;
2178	}
2179
2180	static int num_lri_dwords(int num_regs)
2181	{
2182	int count = 0;
2183
2184	if (num_regs > 0) {
2185	count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS);
2186	count += num_regs * 2;
2187	}
2188
2189	return count;
2190	}
2191
2192	static struct i915_oa_config_bo *
2193	alloc_oa_config_buffer(struct i915_perf_stream *stream,
2194	struct i915_oa_config *oa_config)
2195	{
2196	struct drm_i915_gem_object *obj;
2197	struct i915_oa_config_bo *oa_bo;
2198	struct i915_gem_ww_ctx ww;
2199	size_t config_length = 0;
2200	u32 *cs;
2201	int err;
2202
2203	oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL);
2204	if (!oa_bo)
2205	return ERR_PTR(error: -ENOMEM);
2206
2207	config_length += num_lri_dwords(num_regs: oa_config->mux_regs_len);
2208	config_length += num_lri_dwords(num_regs: oa_config->b_counter_regs_len);
2209	config_length += num_lri_dwords(num_regs: oa_config->flex_regs_len);
2210	config_length += 3; /* MI_BATCH_BUFFER_START */
2211	config_length = ALIGN(sizeof(u32) * config_length, I915_GTT_PAGE_SIZE);
2212
2213	obj = i915_gem_object_create_shmem(i915: stream->perf->i915, size: config_length);
2214	if (IS_ERR(ptr: obj)) {
2215	err = PTR_ERR(ptr: obj);
2216	goto err_free;
2217	}
2218
2219	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
2220	retry:
2221	err = i915_gem_object_lock(obj, ww: &ww);
2222	if (err)
2223	goto out_ww;
2224
2225	cs = i915_gem_object_pin_map(obj, type: I915_MAP_WB);
2226	if (IS_ERR(ptr: cs)) {
2227	err = PTR_ERR(ptr: cs);
2228	goto out_ww;
2229	}
2230
2231	cs = write_cs_mi_lri(cs,
2232	reg_data: oa_config->mux_regs,
2233	n_regs: oa_config->mux_regs_len);
2234	cs = write_cs_mi_lri(cs,
2235	reg_data: oa_config->b_counter_regs,
2236	n_regs: oa_config->b_counter_regs_len);
2237	cs = write_cs_mi_lri(cs,
2238	reg_data: oa_config->flex_regs,
2239	n_regs: oa_config->flex_regs_len);
2240
2241	/* Jump into the active wait. */
2242	*cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ?
2243	MI_BATCH_BUFFER_START :
2244	MI_BATCH_BUFFER_START_GEN8);
2245	*cs++ = i915_ggtt_offset(vma: stream->noa_wait);
2246	*cs++ = 0;
2247
2248	i915_gem_object_flush_map(obj);
2249	__i915_gem_object_release_map(obj);
2250
2251	oa_bo->vma = i915_vma_instance(obj,
2252	vm: &stream->engine->gt->ggtt->vm,
2253	NULL);
2254	if (IS_ERR(ptr: oa_bo->vma)) {
2255	err = PTR_ERR(ptr: oa_bo->vma);
2256	goto out_ww;
2257	}
2258
2259	oa_bo->oa_config = i915_oa_config_get(oa_config);
2260	llist_add(new: &oa_bo->node, head: &stream->oa_config_bos);
2261
2262	out_ww:
2263	if (err == -EDEADLK) {
2264	err = i915_gem_ww_ctx_backoff(ctx: &ww);
2265	if (!err)
2266	goto retry;
2267	}
2268	i915_gem_ww_ctx_fini(ctx: &ww);
2269
2270	if (err)
2271	i915_gem_object_put(obj);
2272	err_free:
2273	if (err) {
2274	kfree(objp: oa_bo);
2275	return ERR_PTR(error: err);
2276	}
2277	return oa_bo;
2278	}
2279
2280	static struct i915_vma *
2281	get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config)
2282	{
2283	struct i915_oa_config_bo *oa_bo;
2284
2285	/*
2286	* Look for the buffer in the already allocated BOs attached
2287	* to the stream.
2288	*/
2289	llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
2290	if (oa_bo->oa_config == oa_config &&
2291	memcmp(p: oa_bo->oa_config->uuid,
2292	q: oa_config->uuid,
2293	size: sizeof(oa_config->uuid)) == 0)
2294	goto out;
2295	}
2296
2297	oa_bo = alloc_oa_config_buffer(stream, oa_config);
2298	if (IS_ERR(ptr: oa_bo))
2299	return ERR_CAST(ptr: oa_bo);
2300
2301	out:
2302	return i915_vma_get(vma: oa_bo->vma);
2303	}
2304
2305	static int
2306	emit_oa_config(struct i915_perf_stream *stream,
2307	struct i915_oa_config *oa_config,
2308	struct intel_context *ce,
2309	struct i915_active *active)
2310	{
2311	struct i915_request *rq;
2312	struct i915_vma *vma;
2313	struct i915_gem_ww_ctx ww;
2314	int err;
2315
2316	vma = get_oa_vma(stream, oa_config);
2317	if (IS_ERR(ptr: vma))
2318	return PTR_ERR(ptr: vma);
2319
2320	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
2321	retry:
2322	err = i915_gem_object_lock(obj: vma->obj, ww: &ww);
2323	if (err)
2324	goto err;
2325
2326	err = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment: 0, PIN_GLOBAL | PIN_HIGH);
2327	if (err)
2328	goto err;
2329
2330	intel_engine_pm_get(engine: ce->engine);
2331	rq = i915_request_create(ce);
2332	intel_engine_pm_put(engine: ce->engine);
2333	if (IS_ERR(ptr: rq)) {
2334	err = PTR_ERR(ptr: rq);
2335	goto err_vma_unpin;
2336	}
2337
2338	if (!IS_ERR_OR_NULL(ptr: active)) {
2339	/* After all individual context modifications */
2340	err = i915_request_await_active(rq, ref: active,
2341	I915_ACTIVE_AWAIT_ACTIVE);
2342	if (err)
2343	goto err_add_request;
2344
2345	err = i915_active_add_request(ref: active, rq);
2346	if (err)
2347	goto err_add_request;
2348	}
2349
2350	err = i915_vma_move_to_active(vma, rq, flags: 0);
2351	if (err)
2352	goto err_add_request;
2353
2354	err = rq->engine->emit_bb_start(rq,
2355	i915_vma_offset(vma), 0,
2356	I915_DISPATCH_SECURE);
2357	if (err)
2358	goto err_add_request;
2359
2360	err_add_request:
2361	i915_request_add(rq);
2362	err_vma_unpin:
2363	i915_vma_unpin(vma);
2364	err:
2365	if (err == -EDEADLK) {
2366	err = i915_gem_ww_ctx_backoff(ctx: &ww);
2367	if (!err)
2368	goto retry;
2369	}
2370
2371	i915_gem_ww_ctx_fini(ctx: &ww);
2372	i915_vma_put(vma);
2373	return err;
2374	}
2375
2376	static struct intel_context *oa_context(struct i915_perf_stream *stream)
2377	{
2378	return stream->pinned_ctx ?: stream->engine->kernel_context;
2379	}
2380
2381	static int
2382	hsw_enable_metric_set(struct i915_perf_stream *stream,
2383	struct i915_active *active)
2384	{
2385	struct intel_uncore *uncore = stream->uncore;
2386
2387	/*
2388	* PRM:
2389	*
2390	* OA unit is using “crclk” for its functionality. When trunk
2391	* level clock gating takes place, OA clock would be gated,
2392	* unable to count the events from non-render clock domain.
2393	* Render clock gating must be disabled when OA is enabled to
2394	* count the events from non-render domain. Unit level clock
2395	* gating for RCS should also be disabled.
2396	*/
2397	intel_uncore_rmw(uncore, GEN7_MISCCPCTL,
2398	GEN7_DOP_CLOCK_GATE_ENABLE, set: 0);
2399	intel_uncore_rmw(uncore, GEN6_UCGCTL1,
2400	clear: 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
2401
2402	return emit_oa_config(stream,
2403	oa_config: stream->oa_config, ce: oa_context(stream),
2404	active);
2405	}
2406
2407	static void hsw_disable_metric_set(struct i915_perf_stream *stream)
2408	{
2409	struct intel_uncore *uncore = stream->uncore;
2410
2411	intel_uncore_rmw(uncore, GEN6_UCGCTL1,
2412	GEN6_CSUNIT_CLOCK_GATE_DISABLE, set: 0);
2413	intel_uncore_rmw(uncore, GEN7_MISCCPCTL,
2414	clear: 0, GEN7_DOP_CLOCK_GATE_ENABLE);
2415
2416	intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, set: 0);
2417	}
2418
2419	static u32 oa_config_flex_reg(const struct i915_oa_config *oa_config,
2420	i915_reg_t reg)
2421	{
2422	u32 mmio = i915_mmio_reg_offset(reg);
2423	int i;
2424
2425	/*
2426	* This arbitrary default will select the 'EU FPU0 Pipeline
2427	* Active' event. In the future it's anticipated that there
2428	* will be an explicit 'No Event' we can select, but not yet...
2429	*/
2430	if (!oa_config)
2431	return 0;
2432
2433	for (i = 0; i < oa_config->flex_regs_len; i++) {
2434	if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio)
2435	return oa_config->flex_regs[i].value;
2436	}
2437
2438	return 0;
2439	}
2440	/*
2441	* NB: It must always remain pointer safe to run this even if the OA unit
2442	* has been disabled.
2443	*
2444	* It's fine to put out-of-date values into these per-context registers
2445	* in the case that the OA unit has been disabled.
2446	*/
2447	static void
2448	gen8_update_reg_state_unlocked(const struct intel_context *ce,
2449	const struct i915_perf_stream *stream)
2450	{
2451	u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2452	u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2453	/* The MMIO offsets for Flex EU registers aren't contiguous */
2454	static const i915_reg_t flex_regs[] = {
2455	EU_PERF_CNTL0,
2456	EU_PERF_CNTL1,
2457	EU_PERF_CNTL2,
2458	EU_PERF_CNTL3,
2459	EU_PERF_CNTL4,
2460	EU_PERF_CNTL5,
2461	EU_PERF_CNTL6,
2462	};
2463	u32 *reg_state = ce->lrc_reg_state;
2464	int i;
2465
2466	reg_state[ctx_oactxctrl + 1] =
2467	(stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2468	(stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2469	GEN8_OA_COUNTER_RESUME;
2470
2471	for (i = 0; i < ARRAY_SIZE(flex_regs); i++)
2472	reg_state[ctx_flexeu0 + i * 2 + 1] =
2473	oa_config_flex_reg(oa_config: stream->oa_config, reg: flex_regs[i]);
2474	}
2475
2476	struct flex {
2477	i915_reg_t reg;
2478	u32 offset;
2479	u32 value;
2480	};
2481
2482	static int
2483	gen8_store_flex(struct i915_request *rq,
2484	struct intel_context *ce,
2485	const struct flex *flex, unsigned int count)
2486	{
2487	u32 offset;
2488	u32 *cs;
2489
2490	cs = intel_ring_begin(rq, num_dwords: 4 * count);
2491	if (IS_ERR(ptr: cs))
2492	return PTR_ERR(ptr: cs);
2493
2494	offset = i915_ggtt_offset(vma: ce->state) + LRC_STATE_OFFSET;
2495	do {
2496	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
2497	*cs++ = offset + flex->offset * sizeof(u32);
2498	*cs++ = 0;
2499	*cs++ = flex->value;
2500	} while (flex++, --count);
2501
2502	intel_ring_advance(rq, cs);
2503
2504	return 0;
2505	}
2506
2507	static int
2508	gen8_load_flex(struct i915_request *rq,
2509	struct intel_context *ce,
2510	const struct flex *flex, unsigned int count)
2511	{
2512	u32 *cs;
2513
2514	GEM_BUG_ON(!count || count > 63);
2515
2516	cs = intel_ring_begin(rq, num_dwords: 2 * count + 2);
2517	if (IS_ERR(ptr: cs))
2518	return PTR_ERR(ptr: cs);
2519
2520	*cs++ = MI_LOAD_REGISTER_IMM(count);
2521	do {
2522	*cs++ = i915_mmio_reg_offset(flex->reg);
2523	*cs++ = flex->value;
2524	} while (flex++, --count);
2525	*cs++ = MI_NOOP;
2526
2527	intel_ring_advance(rq, cs);
2528
2529	return 0;
2530	}
2531
2532	static int gen8_modify_context(struct intel_context *ce,
2533	const struct flex *flex, unsigned int count)
2534	{
2535	struct i915_request *rq;
2536	int err;
2537
2538	rq = intel_engine_create_kernel_request(engine: ce->engine);
2539	if (IS_ERR(ptr: rq))
2540	return PTR_ERR(ptr: rq);
2541
2542	/* Serialise with the remote context */
2543	err = intel_context_prepare_remote_request(ce, rq);
2544	if (err == 0)
2545	err = gen8_store_flex(rq, ce, flex, count);
2546
2547	i915_request_add(rq);
2548	return err;
2549	}
2550
2551	static int
2552	gen8_modify_self(struct intel_context *ce,
2553	const struct flex *flex, unsigned int count,
2554	struct i915_active *active)
2555	{
2556	struct i915_request *rq;
2557	int err;
2558
2559	intel_engine_pm_get(engine: ce->engine);
2560	rq = i915_request_create(ce);
2561	intel_engine_pm_put(engine: ce->engine);
2562	if (IS_ERR(ptr: rq))
2563	return PTR_ERR(ptr: rq);
2564
2565	if (!IS_ERR_OR_NULL(ptr: active)) {
2566	err = i915_active_add_request(ref: active, rq);
2567	if (err)
2568	goto err_add_request;
2569	}
2570
2571	err = gen8_load_flex(rq, ce, flex, count);
2572	if (err)
2573	goto err_add_request;
2574
2575	err_add_request:
2576	i915_request_add(rq);
2577	return err;
2578	}
2579
2580	static int gen8_configure_context(struct i915_perf_stream *stream,
2581	struct i915_gem_context *ctx,
2582	struct flex *flex, unsigned int count)
2583	{
2584	struct i915_gem_engines_iter it;
2585	struct intel_context *ce;
2586	int err = 0;
2587
2588	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
2589	GEM_BUG_ON(ce == ce->engine->kernel_context);
2590
2591	if (ce->engine->class != RENDER_CLASS)
2592	continue;
2593
2594	/* Otherwise OA settings will be set upon first use */
2595	if (!intel_context_pin_if_active(ce))
2596	continue;
2597
2598	flex->value = intel_sseu_make_rpcs(gt: ce->engine->gt, req_sseu: &ce->sseu);
2599	err = gen8_modify_context(ce, flex, count);
2600
2601	intel_context_unpin(ce);
2602	if (err)
2603	break;
2604	}
2605	i915_gem_context_unlock_engines(ctx);
2606
2607	return err;
2608	}
2609
2610	static int gen12_configure_oar_context(struct i915_perf_stream *stream,
2611	struct i915_active *active)
2612	{
2613	int err;
2614	struct intel_context *ce = stream->pinned_ctx;
2615	u32 format = stream->oa_buffer.format->format;
2616	u32 offset = stream->perf->ctx_oactxctrl_offset;
2617	struct flex regs_context[] = {
2618	{
2619	GEN8_OACTXCONTROL,
2620	offset + 1,
2621	active ? GEN8_OA_COUNTER_RESUME : 0,
2622	},
2623	};
2624	/* Offsets in regs_lri are not used since this configuration is only
2625	* applied using LRI. Initialize the correct offsets for posterity.
2626	*/
2627	#define GEN12_OAR_OACONTROL_OFFSET 0x5B0
2628	struct flex regs_lri[] = {
2629	{
2630	GEN12_OAR_OACONTROL,
2631	GEN12_OAR_OACONTROL_OFFSET + 1,
2632	(format << GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT) |
2633	(active ? GEN12_OAR_OACONTROL_COUNTER_ENABLE : 0)
2634	},
2635	{
2636	RING_CONTEXT_CONTROL(ce->engine->mmio_base),
2637	CTX_CONTEXT_CONTROL,
2638	_MASKED_FIELD(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE,
2639	active ?
2640	GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE :
2641	0)
2642	},
2643	};
2644
2645	/* Modify the context image of pinned context with regs_context */
2646	err = intel_context_lock_pinned(ce);
2647	if (err)
2648	return err;
2649
2650	err = gen8_modify_context(ce, flex: regs_context,
2651	ARRAY_SIZE(regs_context));
2652	intel_context_unlock_pinned(ce);
2653	if (err)
2654	return err;
2655
2656	/* Apply regs_lri using LRI with pinned context */
2657	return gen8_modify_self(ce, flex: regs_lri, ARRAY_SIZE(regs_lri), active);
2658	}
2659
2660	/*
2661	* Manages updating the per-context aspects of the OA stream
2662	* configuration across all contexts.
2663	*
2664	* The awkward consideration here is that OACTXCONTROL controls the
2665	* exponent for periodic sampling which is primarily used for system
2666	* wide profiling where we'd like a consistent sampling period even in
2667	* the face of context switches.
2668	*
2669	* Our approach of updating the register state context (as opposed to
2670	* say using a workaround batch buffer) ensures that the hardware
2671	* won't automatically reload an out-of-date timer exponent even
2672	* transiently before a WA BB could be parsed.
2673	*
2674	* This function needs to:
2675	* - Ensure the currently running context's per-context OA state is
2676	* updated
2677	* - Ensure that all existing contexts will have the correct per-context
2678	* OA state if they are scheduled for use.
2679	* - Ensure any new contexts will be initialized with the correct
2680	* per-context OA state.
2681	*
2682	* Note: it's only the RCS/Render context that has any OA state.
2683	* Note: the first flex register passed must always be R_PWR_CLK_STATE
2684	*/
2685	static int
2686	oa_configure_all_contexts(struct i915_perf_stream *stream,
2687	struct flex *regs,
2688	size_t num_regs,
2689	struct i915_active *active)
2690	{
2691	struct drm_i915_private *i915 = stream->perf->i915;
2692	struct intel_engine_cs *engine;
2693	struct intel_gt *gt = stream->engine->gt;
2694	struct i915_gem_context *ctx, *cn;
2695	int err;
2696
2697	lockdep_assert_held(&gt->perf.lock);
2698
2699	/*
2700	* The OA register config is setup through the context image. This image
2701	* might be written to by the GPU on context switch (in particular on
2702	* lite-restore). This means we can't safely update a context's image,
2703	* if this context is scheduled/submitted to run on the GPU.
2704	*
2705	* We could emit the OA register config through the batch buffer but
2706	* this might leave small interval of time where the OA unit is
2707	* configured at an invalid sampling period.
2708	*
2709	* Note that since we emit all requests from a single ring, there
2710	* is still an implicit global barrier here that may cause a high
2711	* priority context to wait for an otherwise independent low priority
2712	* context. Contexts idle at the time of reconfiguration are not
2713	* trapped behind the barrier.
2714	*/
2715	spin_lock(lock: &i915->gem.contexts.lock);
2716	list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) {
2717	if (!kref_get_unless_zero(kref: &ctx->ref))
2718	continue;
2719
2720	spin_unlock(lock: &i915->gem.contexts.lock);
2721
2722	err = gen8_configure_context(stream, ctx, flex: regs, count: num_regs);
2723	if (err) {
2724	i915_gem_context_put(ctx);
2725	return err;
2726	}
2727
2728	spin_lock(lock: &i915->gem.contexts.lock);
2729	list_safe_reset_next(ctx, cn, link);
2730	i915_gem_context_put(ctx);
2731	}
2732	spin_unlock(lock: &i915->gem.contexts.lock);
2733
2734	/*
2735	* After updating all other contexts, we need to modify ourselves.
2736	* If we don't modify the kernel_context, we do not get events while
2737	* idle.
2738	*/
2739	for_each_uabi_engine(engine, i915) {
2740	struct intel_context *ce = engine->kernel_context;
2741
2742	if (engine->class != RENDER_CLASS)
2743	continue;
2744
2745	regs[0].value = intel_sseu_make_rpcs(gt: engine->gt, req_sseu: &ce->sseu);
2746
2747	err = gen8_modify_self(ce, flex: regs, count: num_regs, active);
2748	if (err)
2749	return err;
2750	}
2751
2752	return 0;
2753	}
2754
2755	static int
2756	lrc_configure_all_contexts(struct i915_perf_stream *stream,
2757	const struct i915_oa_config *oa_config,
2758	struct i915_active *active)
2759	{
2760	u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2761	/* The MMIO offsets for Flex EU registers aren't contiguous */
2762	const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2763	#define ctx_flexeuN(N) (ctx_flexeu0 + 2 * (N) + 1)
2764	struct flex regs[] = {
2765	{
2766	GEN8_R_PWR_CLK_STATE(RENDER_RING_BASE),
2767	CTX_R_PWR_CLK_STATE,
2768	},
2769	{
2770	GEN8_OACTXCONTROL,
2771	ctx_oactxctrl + 1,
2772	},
2773	{ EU_PERF_CNTL0, ctx_flexeuN(0) },
2774	{ EU_PERF_CNTL1, ctx_flexeuN(1) },
2775	{ EU_PERF_CNTL2, ctx_flexeuN(2) },
2776	{ EU_PERF_CNTL3, ctx_flexeuN(3) },
2777	{ EU_PERF_CNTL4, ctx_flexeuN(4) },
2778	{ EU_PERF_CNTL5, ctx_flexeuN(5) },
2779	{ EU_PERF_CNTL6, ctx_flexeuN(6) },
2780	};
2781	#undef ctx_flexeuN
2782	int i;
2783
2784	regs[1].value =
2785	(stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2786	(stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2787	GEN8_OA_COUNTER_RESUME;
2788
2789	for (i = 2; i < ARRAY_SIZE(regs); i++)
2790	regs[i].value = oa_config_flex_reg(oa_config, reg: regs[i].reg);
2791
2792	return oa_configure_all_contexts(stream,
2793	regs, ARRAY_SIZE(regs),
2794	active);
2795	}
2796
2797	static int
2798	gen8_enable_metric_set(struct i915_perf_stream *stream,
2799	struct i915_active *active)
2800	{
2801	struct intel_uncore *uncore = stream->uncore;
2802	struct i915_oa_config *oa_config = stream->oa_config;
2803	int ret;
2804
2805	/*
2806	* We disable slice/unslice clock ratio change reports on SKL since
2807	* they are too noisy. The HW generates a lot of redundant reports
2808	* where the ratio hasn't really changed causing a lot of redundant
2809	* work to processes and increasing the chances we'll hit buffer
2810	* overruns.
2811	*
2812	* Although we don't currently use the 'disable overrun' OABUFFER
2813	* feature it's worth noting that clock ratio reports have to be
2814	* disabled before considering to use that feature since the HW doesn't
2815	* correctly block these reports.
2816	*
2817	* Currently none of the high-level metrics we have depend on knowing
2818	* this ratio to normalize.
2819	*
2820	* Note: This register is not power context saved and restored, but
2821	* that's OK considering that we disable RC6 while the OA unit is
2822	* enabled.
2823	*
2824	* The _INCLUDE_CLK_RATIO bit allows the slice/unslice frequency to
2825	* be read back from automatically triggered reports, as part of the
2826	* RPT_ID field.
2827	*/
2828	if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) {
2829	intel_uncore_write(uncore, GEN8_OA_DEBUG,
2830	_MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
2831	GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
2832	}
2833
2834	/*
2835	* Update all contexts prior writing the mux configurations as we need
2836	* to make sure all slices/subslices are ON before writing to NOA
2837	* registers.
2838	*/
2839	ret = lrc_configure_all_contexts(stream, oa_config, active);
2840	if (ret)
2841	return ret;
2842
2843	return emit_oa_config(stream,
2844	oa_config: stream->oa_config, ce: oa_context(stream),
2845	active);
2846	}
2847
2848	static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
2849	{
2850	return _MASKED_FIELD(GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
2851	(stream->sample_flags & SAMPLE_OA_REPORT) ?
2852	0 : GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
2853	}
2854
2855	static int
2856	gen12_enable_metric_set(struct i915_perf_stream *stream,
2857	struct i915_active *active)
2858	{
2859	struct drm_i915_private *i915 = stream->perf->i915;
2860	struct intel_uncore *uncore = stream->uncore;
2861	bool periodic = stream->periodic;
2862	u32 period_exponent = stream->period_exponent;
2863	u32 sqcnt1;
2864	int ret;
2865
2866	/*
2867	* Wa_1508761755
2868	* EU NOA signals behave incorrectly if EU clock gating is enabled.
2869	* Disable thread stall DOP gating and EU DOP gating.
2870	*/
2871	if (IS_DG2(i915)) {
2872	intel_gt_mcr_multicast_write(gt: uncore->gt, GEN8_ROW_CHICKEN,
2873	_MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
2874	intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
2875	_MASKED_BIT_ENABLE(GEN12_DISABLE_DOP_GATING));
2876	}
2877
2878	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_debug,
2879	/* Disable clk ratio reports, like previous Gens. */
2880	_MASKED_BIT_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
2881	GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO) |
2882	/*
2883	* If the user didn't require OA reports, instruct
2884	* the hardware not to emit ctx switch reports.
2885	*/
2886	oag_report_ctx_switches(stream));
2887
2888	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_ctx_ctrl, val: periodic ?
2889	(GEN12_OAG_OAGLBCTXCTRL_COUNTER_RESUME |
2890	GEN12_OAG_OAGLBCTXCTRL_TIMER_ENABLE |
2891	(period_exponent << GEN12_OAG_OAGLBCTXCTRL_TIMER_PERIOD_SHIFT))
2892	: 0);
2893
2894	/*
2895	* Initialize Super Queue Internal Cnt Register
2896	* Set PMON Enable in order to collect valid metrics.
2897	* Enable bytes per clock reporting in OA.
2898	*/
2899	sqcnt1 = GEN12_SQCNT1_PMON_ENABLE |
2900	(HAS_OA_BPC_REPORTING(i915) ? GEN12_SQCNT1_OABPC : 0);
2901
2902	intel_uncore_rmw(uncore, GEN12_SQCNT1, clear: 0, set: sqcnt1);
2903
2904	/*
2905	* For Gen12, performance counters are context
2906	* saved/restored. Only enable it for the context that
2907	* requested this.
2908	*/
2909	if (stream->ctx) {
2910	ret = gen12_configure_oar_context(stream, active);
2911	if (ret)
2912	return ret;
2913	}
2914
2915	return emit_oa_config(stream,
2916	oa_config: stream->oa_config, ce: oa_context(stream),
2917	active);
2918	}
2919
2920	static void gen8_disable_metric_set(struct i915_perf_stream *stream)
2921	{
2922	struct intel_uncore *uncore = stream->uncore;
2923
2924	/* Reset all contexts' slices/subslices configurations. */
2925	lrc_configure_all_contexts(stream, NULL, NULL);
2926
2927	intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, set: 0);
2928	}
2929
2930	static void gen11_disable_metric_set(struct i915_perf_stream *stream)
2931	{
2932	struct intel_uncore *uncore = stream->uncore;
2933
2934	/* Reset all contexts' slices/subslices configurations. */
2935	lrc_configure_all_contexts(stream, NULL, NULL);
2936
2937	/* Make sure we disable noa to save power. */
2938	intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, set: 0);
2939	}
2940
2941	static void gen12_disable_metric_set(struct i915_perf_stream *stream)
2942	{
2943	struct intel_uncore *uncore = stream->uncore;
2944	struct drm_i915_private *i915 = stream->perf->i915;
2945	u32 sqcnt1;
2946
2947	/*
2948	* Wa_1508761755: Enable thread stall DOP gating and EU DOP gating.
2949	*/
2950	if (IS_DG2(i915)) {
2951	intel_gt_mcr_multicast_write(gt: uncore->gt, GEN8_ROW_CHICKEN,
2952	_MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
2953	intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
2954	_MASKED_BIT_DISABLE(GEN12_DISABLE_DOP_GATING));
2955	}
2956
2957	/* disable the context save/restore or OAR counters */
2958	if (stream->ctx)
2959	gen12_configure_oar_context(stream, NULL);
2960
2961	/* Make sure we disable noa to save power. */
2962	intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, set: 0);
2963
2964	sqcnt1 = GEN12_SQCNT1_PMON_ENABLE |
2965	(HAS_OA_BPC_REPORTING(i915) ? GEN12_SQCNT1_OABPC : 0);
2966
2967	/* Reset PMON Enable to save power. */
2968	intel_uncore_rmw(uncore, GEN12_SQCNT1, clear: sqcnt1, set: 0);
2969	}
2970
2971	static void gen7_oa_enable(struct i915_perf_stream *stream)
2972	{
2973	struct intel_uncore *uncore = stream->uncore;
2974	struct i915_gem_context *ctx = stream->ctx;
2975	u32 ctx_id = stream->specific_ctx_id;
2976	bool periodic = stream->periodic;
2977	u32 period_exponent = stream->period_exponent;
2978	u32 report_format = stream->oa_buffer.format->format;
2979
2980	/*
2981	* Reset buf pointers so we don't forward reports from before now.
2982	*
2983	* Think carefully if considering trying to avoid this, since it
2984	* also ensures status flags and the buffer itself are cleared
2985	* in error paths, and we have checks for invalid reports based
2986	* on the assumption that certain fields are written to zeroed
2987	* memory which this helps maintains.
2988	*/
2989	gen7_init_oa_buffer(stream);
2990
2991	intel_uncore_write(uncore, GEN7_OACONTROL,
2992	val: (ctx_id & GEN7_OACONTROL_CTX_MASK) |
2993	(period_exponent <<
2994	GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
2995	(periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
2996	(report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
2997	(ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
2998	GEN7_OACONTROL_ENABLE);
2999	}
3000
3001	static void gen8_oa_enable(struct i915_perf_stream *stream)
3002	{
3003	struct intel_uncore *uncore = stream->uncore;
3004	u32 report_format = stream->oa_buffer.format->format;
3005
3006	/*
3007	* Reset buf pointers so we don't forward reports from before now.
3008	*
3009	* Think carefully if considering trying to avoid this, since it
3010	* also ensures status flags and the buffer itself are cleared
3011	* in error paths, and we have checks for invalid reports based
3012	* on the assumption that certain fields are written to zeroed
3013	* memory which this helps maintains.
3014	*/
3015	gen8_init_oa_buffer(stream);
3016
3017	/*
3018	* Note: we don't rely on the hardware to perform single context
3019	* filtering and instead filter on the cpu based on the context-id
3020	* field of reports
3021	*/
3022	intel_uncore_write(uncore, GEN8_OACONTROL,
3023	val: (report_format << GEN8_OA_REPORT_FORMAT_SHIFT) |
3024	GEN8_OA_COUNTER_ENABLE);
3025	}
3026
3027	static void gen12_oa_enable(struct i915_perf_stream *stream)
3028	{
3029	const struct i915_perf_regs *regs;
3030	u32 val;
3031
3032	/*
3033	* If we don't want OA reports from the OA buffer, then we don't even
3034	* need to program the OAG unit.
3035	*/
3036	if (!(stream->sample_flags & SAMPLE_OA_REPORT))
3037	return;
3038
3039	gen12_init_oa_buffer(stream);
3040
3041	regs = __oa_regs(stream);
3042	val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) |
3043	GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE;
3044
3045	intel_uncore_write(uncore: stream->uncore, reg: regs->oa_ctrl, val);
3046	}
3047
3048	/**
3049	* i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
3050	* @stream: An i915 perf stream opened for OA metrics
3051	*
3052	* [Re]enables hardware periodic sampling according to the period configured
3053	* when opening the stream. This also starts a hrtimer that will periodically
3054	* check for data in the circular OA buffer for notifying userspace (e.g.
3055	* during a read() or poll()).
3056	*/
3057	static void i915_oa_stream_enable(struct i915_perf_stream *stream)
3058	{
3059	stream->pollin = false;
3060
3061	stream->perf->ops.oa_enable(stream);
3062
3063	if (stream->sample_flags & SAMPLE_OA_REPORT)
3064	hrtimer_start(timer: &stream->poll_check_timer,
3065	tim: ns_to_ktime(ns: stream->poll_oa_period),
3066	mode: HRTIMER_MODE_REL_PINNED);
3067	}
3068
3069	static void gen7_oa_disable(struct i915_perf_stream *stream)
3070	{
3071	struct intel_uncore *uncore = stream->uncore;
3072
3073	intel_uncore_write(uncore, GEN7_OACONTROL, val: 0);
3074	if (intel_wait_for_register(uncore,
3075	GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, value: 0,
3076	timeout_ms: 50))
3077	drm_err(&stream->perf->i915->drm,
3078	"wait for OA to be disabled timed out\n");
3079	}
3080
3081	static void gen8_oa_disable(struct i915_perf_stream *stream)
3082	{
3083	struct intel_uncore *uncore = stream->uncore;
3084
3085	intel_uncore_write(uncore, GEN8_OACONTROL, val: 0);
3086	if (intel_wait_for_register(uncore,
3087	GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, value: 0,
3088	timeout_ms: 50))
3089	drm_err(&stream->perf->i915->drm,
3090	"wait for OA to be disabled timed out\n");
3091	}
3092
3093	static void gen12_oa_disable(struct i915_perf_stream *stream)
3094	{
3095	struct intel_uncore *uncore = stream->uncore;
3096
3097	intel_uncore_write(uncore, reg: __oa_regs(stream)->oa_ctrl, val: 0);
3098	if (intel_wait_for_register(uncore,
3099	reg: __oa_regs(stream)->oa_ctrl,
3100	GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE, value: 0,
3101	timeout_ms: 50))
3102	drm_err(&stream->perf->i915->drm,
3103	"wait for OA to be disabled timed out\n");
3104
3105	intel_uncore_write(uncore, GEN12_OA_TLB_INV_CR, val: 1);
3106	if (intel_wait_for_register(uncore,
3107	GEN12_OA_TLB_INV_CR,
3108	mask: 1, value: 0,
3109	timeout_ms: 50))
3110	drm_err(&stream->perf->i915->drm,
3111	"wait for OA tlb invalidate timed out\n");
3112	}
3113
3114	/**
3115	* i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
3116	* @stream: An i915 perf stream opened for OA metrics
3117	*
3118	* Stops the OA unit from periodically writing counter reports into the
3119	* circular OA buffer. This also stops the hrtimer that periodically checks for
3120	* data in the circular OA buffer, for notifying userspace.
3121	*/
3122	static void i915_oa_stream_disable(struct i915_perf_stream *stream)
3123	{
3124	stream->perf->ops.oa_disable(stream);
3125
3126	if (stream->sample_flags & SAMPLE_OA_REPORT)
3127	hrtimer_cancel(timer: &stream->poll_check_timer);
3128	}
3129
3130	static const struct i915_perf_stream_ops i915_oa_stream_ops = {
3131	.destroy = i915_oa_stream_destroy,
3132	.enable = i915_oa_stream_enable,
3133	.disable = i915_oa_stream_disable,
3134	.wait_unlocked = i915_oa_wait_unlocked,
3135	.poll_wait = i915_oa_poll_wait,
3136	.read = i915_oa_read,
3137	};
3138
3139	static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream)
3140	{
3141	struct i915_active *active;
3142	int err;
3143
3144	active = i915_active_create();
3145	if (!active)
3146	return -ENOMEM;
3147
3148	err = stream->perf->ops.enable_metric_set(stream, active);
3149	if (err == 0)
3150	__i915_active_wait(ref: active, TASK_UNINTERRUPTIBLE);
3151
3152	i915_active_put(ref: active);
3153	return err;
3154	}
3155
3156	static void
3157	get_default_sseu_config(struct intel_sseu *out_sseu,
3158	struct intel_engine_cs *engine)
3159	{
3160	const struct sseu_dev_info *devinfo_sseu = &engine->gt->info.sseu;
3161
3162	*out_sseu = intel_sseu_from_device_info(sseu: devinfo_sseu);
3163
3164	if (GRAPHICS_VER(engine->i915) == 11) {
3165	/*
3166	* We only need subslice count so it doesn't matter which ones
3167	* we select - just turn off low bits in the amount of half of
3168	* all available subslices per slice.
3169	*/
3170	out_sseu->subslice_mask =
3171	~(~0 << (hweight8(out_sseu->subslice_mask) / 2));
3172	out_sseu->slice_mask = 0x1;
3173	}
3174	}
3175
3176	static int
3177	get_sseu_config(struct intel_sseu *out_sseu,
3178	struct intel_engine_cs *engine,
3179	const struct drm_i915_gem_context_param_sseu *drm_sseu)
3180	{
3181	if (drm_sseu->engine.engine_class != engine->uabi_class ||
3182	drm_sseu->engine.engine_instance != engine->uabi_instance)
3183	return -EINVAL;
3184
3185	return i915_gem_user_to_context_sseu(gt: engine->gt, user: drm_sseu, context: out_sseu);
3186	}
3187
3188	/*
3189	* OA timestamp frequency = CS timestamp frequency in most platforms. On some
3190	* platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such
3191	* cases, return the adjusted CS timestamp frequency to the user.
3192	*/
3193	u32 i915_perf_oa_timestamp_frequency(struct drm_i915_private *i915)
3194	{
3195	struct intel_gt *gt = to_gt(i915);
3196
3197	/* Wa_18013179988 */
3198	if (IS_DG2(i915) || IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74))) {
3199	intel_wakeref_t wakeref;
3200	u32 reg, shift;
3201
3202	with_intel_runtime_pm(to_gt(i915)->uncore->rpm, wakeref)
3203	reg = intel_uncore_read(uncore: to_gt(i915)->uncore, RPM_CONFIG0);
3204
3205	shift = REG_FIELD_GET(GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK,
3206	reg);
3207
3208	return to_gt(i915)->clock_frequency << (3 - shift);
3209	}
3210
3211	return to_gt(i915)->clock_frequency;
3212	}
3213
3214	/**
3215	* i915_oa_stream_init - validate combined props for OA stream and init
3216	* @stream: An i915 perf stream
3217	* @param: The open parameters passed to `DRM_I915_PERF_OPEN`
3218	* @props: The property state that configures stream (individually validated)
3219	*
3220	* While read_properties_unlocked() validates properties in isolation it
3221	* doesn't ensure that the combination necessarily makes sense.
3222	*
3223	* At this point it has been determined that userspace wants a stream of
3224	* OA metrics, but still we need to further validate the combined
3225	* properties are OK.
3226	*
3227	* If the configuration makes sense then we can allocate memory for
3228	* a circular OA buffer and apply the requested metric set configuration.
3229	*
3230	* Returns: zero on success or a negative error code.
3231	*/
3232	static int i915_oa_stream_init(struct i915_perf_stream *stream,
3233	struct drm_i915_perf_open_param *param,
3234	struct perf_open_properties *props)
3235	{
3236	struct drm_i915_private *i915 = stream->perf->i915;
3237	struct i915_perf *perf = stream->perf;
3238	struct i915_perf_group *g;
3239	int ret;
3240
3241	if (!props->engine) {
3242	drm_dbg(&stream->perf->i915->drm,
3243	"OA engine not specified\n");
3244	return -EINVAL;
3245	}
3246	g = props->engine->oa_group;
3247
3248	/*
3249	* If the sysfs metrics/ directory wasn't registered for some
3250	* reason then don't let userspace try their luck with config
3251	* IDs
3252	*/
3253	if (!perf->metrics_kobj) {
3254	drm_dbg(&stream->perf->i915->drm,
3255	"OA metrics weren't advertised via sysfs\n");
3256	return -EINVAL;
3257	}
3258
3259	if (!(props->sample_flags & SAMPLE_OA_REPORT) &&
3260	(GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) {
3261	drm_dbg(&stream->perf->i915->drm,
3262	"Only OA report sampling supported\n");
3263	return -EINVAL;
3264	}
3265
3266	if (!perf->ops.enable_metric_set) {
3267	drm_dbg(&stream->perf->i915->drm,
3268	"OA unit not supported\n");
3269	return -ENODEV;
3270	}
3271
3272	/*
3273	* To avoid the complexity of having to accurately filter
3274	* counter reports and marshal to the appropriate client
3275	* we currently only allow exclusive access
3276	*/
3277	if (g->exclusive_stream) {
3278	drm_dbg(&stream->perf->i915->drm,
3279	"OA unit already in use\n");
3280	return -EBUSY;
3281	}
3282
3283	if (!props->oa_format) {
3284	drm_dbg(&stream->perf->i915->drm,
3285	"OA report format not specified\n");
3286	return -EINVAL;
3287	}
3288
3289	stream->engine = props->engine;
3290	stream->uncore = stream->engine->gt->uncore;
3291
3292	stream->sample_size = sizeof(struct drm_i915_perf_record_header);
3293
3294	stream->oa_buffer.format = &perf->oa_formats[props->oa_format];
3295	if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0))
3296	return -EINVAL;
3297
3298	stream->sample_flags = props->sample_flags;
3299	stream->sample_size += stream->oa_buffer.format->size;
3300
3301	stream->hold_preemption = props->hold_preemption;
3302
3303	stream->periodic = props->oa_periodic;
3304	if (stream->periodic)
3305	stream->period_exponent = props->oa_period_exponent;
3306
3307	if (stream->ctx) {
3308	ret = oa_get_render_ctx_id(stream);
3309	if (ret) {
3310	drm_dbg(&stream->perf->i915->drm,
3311	"Invalid context id to filter with\n");
3312	return ret;
3313	}
3314	}
3315
3316	ret = alloc_noa_wait(stream);
3317	if (ret) {
3318	drm_dbg(&stream->perf->i915->drm,
3319	"Unable to allocate NOA wait batch buffer\n");
3320	goto err_noa_wait_alloc;
3321	}
3322
3323	stream->oa_config = i915_perf_get_oa_config(perf, metrics_set: props->metrics_set);
3324	if (!stream->oa_config) {
3325	drm_dbg(&stream->perf->i915->drm,
3326	"Invalid OA config id=%i\n", props->metrics_set);
3327	ret = -EINVAL;
3328	goto err_config;
3329	}
3330
3331	/* PRM - observability performance counters:
3332	*
3333	* OACONTROL, performance counter enable, note:
3334	*
3335	* "When this bit is set, in order to have coherent counts,
3336	* RC6 power state and trunk clock gating must be disabled.
3337	* This can be achieved by programming MMIO registers as
3338	* 0xA094=0 and 0xA090[31]=1"
3339	*
3340	* In our case we are expecting that taking pm + FORCEWAKE
3341	* references will effectively disable RC6.
3342	*/
3343	intel_engine_pm_get(engine: stream->engine);
3344	intel_uncore_forcewake_get(uncore: stream->uncore, domains: FORCEWAKE_ALL);
3345
3346	ret = alloc_oa_buffer(stream);
3347	if (ret)
3348	goto err_oa_buf_alloc;
3349
3350	stream->ops = &i915_oa_stream_ops;
3351
3352	stream->engine->gt->perf.sseu = props->sseu;
3353	WRITE_ONCE(g->exclusive_stream, stream);
3354
3355	ret = i915_perf_stream_enable_sync(stream);
3356	if (ret) {
3357	drm_dbg(&stream->perf->i915->drm,
3358	"Unable to enable metric set\n");
3359	goto err_enable;
3360	}
3361
3362	drm_dbg(&stream->perf->i915->drm,
3363	"opening stream oa config uuid=%s\n",
3364	stream->oa_config->uuid);
3365
3366	hrtimer_setup(timer: &stream->poll_check_timer, function: oa_poll_check_timer_cb, CLOCK_MONOTONIC,
3367	mode: HRTIMER_MODE_REL);
3368	init_waitqueue_head(&stream->poll_wq);
3369	spin_lock_init(&stream->oa_buffer.ptr_lock);
3370	mutex_init(&stream->lock);
3371
3372	return 0;
3373
3374	err_enable:
3375	WRITE_ONCE(g->exclusive_stream, NULL);
3376	perf->ops.disable_metric_set(stream);
3377
3378	free_oa_buffer(stream);
3379
3380	err_oa_buf_alloc:
3381	intel_uncore_forcewake_put(uncore: stream->uncore, domains: FORCEWAKE_ALL);
3382	intel_engine_pm_put(engine: stream->engine);
3383
3384	free_oa_configs(stream);
3385
3386	err_config:
3387	free_noa_wait(stream);
3388
3389	err_noa_wait_alloc:
3390	if (stream->ctx)
3391	oa_put_render_ctx_id(stream);
3392
3393	return ret;
3394	}
3395
3396	void i915_oa_init_reg_state(const struct intel_context *ce,
3397	const struct intel_engine_cs *engine)
3398	{
3399	struct i915_perf_stream *stream;
3400
3401	if (engine->class != RENDER_CLASS)
3402	return;
3403
3404	/* perf.exclusive_stream serialised by lrc_configure_all_contexts() */
3405	stream = READ_ONCE(engine->oa_group->exclusive_stream);
3406	if (stream && GRAPHICS_VER(stream->perf->i915) < 12)
3407	gen8_update_reg_state_unlocked(ce, stream);
3408	}
3409
3410	/**
3411	* i915_perf_read - handles read() FOP for i915 perf stream FDs
3412	* @file: An i915 perf stream file
3413	* @buf: destination buffer given by userspace
3414	* @count: the number of bytes userspace wants to read
3415	* @ppos: (inout) file seek position (unused)
3416	*
3417	* The entry point for handling a read() on a stream file descriptor from
3418	* userspace. Most of the work is left to the i915_perf_read_locked() and
3419	* &i915_perf_stream_ops->read but to save having stream implementations (of
3420	* which we might have multiple later) we handle blocking read here.
3421	*
3422	* We can also consistently treat trying to read from a disabled stream
3423	* as an IO error so implementations can assume the stream is enabled
3424	* while reading.
3425	*
3426	* Returns: The number of bytes copied or a negative error code on failure.
3427	*/
3428	static ssize_t i915_perf_read(struct file *file,
3429	char __user *buf,
3430	size_t count,
3431	loff_t *ppos)
3432	{
3433	struct i915_perf_stream *stream = file->private_data;
3434	size_t offset = 0;
3435	int ret;
3436
3437	/* To ensure it's handled consistently we simply treat all reads of a
3438	* disabled stream as an error. In particular it might otherwise lead
3439	* to a deadlock for blocking file descriptors...
3440	*/
3441	if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT))
3442	return -EIO;
3443
3444	if (!(file->f_flags & O_NONBLOCK)) {
3445	/* There's the small chance of false positives from
3446	* stream->ops->wait_unlocked.
3447	*
3448	* E.g. with single context filtering since we only wait until
3449	* oabuffer has >= 1 report we don't immediately know whether
3450	* any reports really belong to the current context
3451	*/
3452	do {
3453	ret = stream->ops->wait_unlocked(stream);
3454	if (ret)
3455	return ret;
3456
3457	mutex_lock(&stream->lock);
3458	ret = stream->ops->read(stream, buf, count, &offset);
3459	mutex_unlock(lock: &stream->lock);
3460	} while (!offset && !ret);
3461	} else {
3462	mutex_lock(&stream->lock);
3463	ret = stream->ops->read(stream, buf, count, &offset);
3464	mutex_unlock(lock: &stream->lock);
3465	}
3466
3467	/* We allow the poll checking to sometimes report false positive EPOLLIN
3468	* events where we might actually report EAGAIN on read() if there's
3469	* not really any data available. In this situation though we don't
3470	* want to enter a busy loop between poll() reporting a EPOLLIN event
3471	* and read() returning -EAGAIN. Clearing the oa.pollin state here
3472	* effectively ensures we back off until the next hrtimer callback
3473	* before reporting another EPOLLIN event.
3474	* The exception to this is if ops->read() returned -ENOSPC which means
3475	* that more OA data is available than could fit in the user provided
3476	* buffer. In this case we want the next poll() call to not block.
3477	*/
3478	if (ret != -ENOSPC)
3479	stream->pollin = false;
3480
3481	/* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, ... */
3482	return offset ?: (ret ?: -EAGAIN);
3483	}
3484
3485	static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer)
3486	{
3487	struct i915_perf_stream *stream =
3488	container_of(hrtimer, typeof(*stream), poll_check_timer);
3489
3490	if (oa_buffer_check_unlocked(stream)) {
3491	stream->pollin = true;
3492	wake_up(&stream->poll_wq);
3493	}
3494
3495	hrtimer_forward_now(timer: hrtimer,
3496	interval: ns_to_ktime(ns: stream->poll_oa_period));
3497
3498	return HRTIMER_RESTART;
3499	}
3500
3501	/**
3502	* i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3503	* @stream: An i915 perf stream
3504	* @file: An i915 perf stream file
3505	* @wait: poll() state table
3506	*
3507	* For handling userspace polling on an i915 perf stream, this calls through to
3508	* &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
3509	* will be woken for new stream data.
3510	*
3511	* Returns: any poll events that are ready without sleeping
3512	*/
3513	static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream,
3514	struct file *file,
3515	poll_table *wait)
3516	{
3517	__poll_t events = 0;
3518
3519	stream->ops->poll_wait(stream, file, wait);
3520
3521	/* Note: we don't explicitly check whether there's something to read
3522	* here since this path may be very hot depending on what else
3523	* userspace is polling, or on the timeout in use. We rely solely on
3524	* the hrtimer/oa_poll_check_timer_cb to notify us when there are
3525	* samples to read.
3526	*/
3527	if (stream->pollin)
3528	events |= EPOLLIN;
3529
3530	return events;
3531	}
3532
3533	/**
3534	* i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3535	* @file: An i915 perf stream file
3536	* @wait: poll() state table
3537	*
3538	* For handling userspace polling on an i915 perf stream, this ensures
3539	* poll_wait() gets called with a wait queue that will be woken for new stream
3540	* data.
3541	*
3542	* Note: Implementation deferred to i915_perf_poll_locked()
3543	*
3544	* Returns: any poll events that are ready without sleeping
3545	*/
3546	static __poll_t i915_perf_poll(struct file *file, poll_table *wait)
3547	{
3548	struct i915_perf_stream *stream = file->private_data;
3549	__poll_t ret;
3550
3551	mutex_lock(&stream->lock);
3552	ret = i915_perf_poll_locked(stream, file, wait);
3553	mutex_unlock(lock: &stream->lock);
3554
3555	return ret;
3556	}
3557
3558	/**
3559	* i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
3560	* @stream: A disabled i915 perf stream
3561	*
3562	* [Re]enables the associated capture of data for this stream.
3563	*
3564	* If a stream was previously enabled then there's currently no intention
3565	* to provide userspace any guarantee about the preservation of previously
3566	* buffered data.
3567	*/
3568	static void i915_perf_enable_locked(struct i915_perf_stream *stream)
3569	{
3570	if (stream->enabled)
3571	return;
3572
3573	/* Allow stream->ops->enable() to refer to this */
3574	stream->enabled = true;
3575
3576	if (stream->ops->enable)
3577	stream->ops->enable(stream);
3578
3579	if (stream->hold_preemption)
3580	intel_context_set_nopreempt(ce: stream->pinned_ctx);
3581	}
3582
3583	/**
3584	* i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
3585	* @stream: An enabled i915 perf stream
3586	*
3587	* Disables the associated capture of data for this stream.
3588	*
3589	* The intention is that disabling an re-enabling a stream will ideally be
3590	* cheaper than destroying and re-opening a stream with the same configuration,
3591	* though there are no formal guarantees about what state or buffered data
3592	* must be retained between disabling and re-enabling a stream.
3593	*
3594	* Note: while a stream is disabled it's considered an error for userspace
3595	* to attempt to read from the stream (-EIO).
3596	*/
3597	static void i915_perf_disable_locked(struct i915_perf_stream *stream)
3598	{
3599	if (!stream->enabled)
3600	return;
3601
3602	/* Allow stream->ops->disable() to refer to this */
3603	stream->enabled = false;
3604
3605	if (stream->hold_preemption)
3606	intel_context_clear_nopreempt(ce: stream->pinned_ctx);
3607
3608	if (stream->ops->disable)
3609	stream->ops->disable(stream);
3610	}
3611
3612	static long i915_perf_config_locked(struct i915_perf_stream *stream,
3613	unsigned long metrics_set)
3614	{
3615	struct i915_oa_config *config;
3616	long ret = stream->oa_config->id;
3617
3618	config = i915_perf_get_oa_config(perf: stream->perf, metrics_set);
3619	if (!config)
3620	return -EINVAL;
3621
3622	if (config != stream->oa_config) {
3623	int err;
3624
3625	/*
3626	* If OA is bound to a specific context, emit the
3627	* reconfiguration inline from that context. The update
3628	* will then be ordered with respect to submission on that
3629	* context.
3630	*
3631	* When set globally, we use a low priority kernel context,
3632	* so it will effectively take effect when idle.
3633	*/
3634	err = emit_oa_config(stream, oa_config: config, ce: oa_context(stream), NULL);
3635	if (!err)
3636	config = xchg(&stream->oa_config, config);
3637	else
3638	ret = err;
3639	}
3640
3641	i915_oa_config_put(oa_config: config);
3642
3643	return ret;
3644	}
3645
3646	/**
3647	* i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3648	* @stream: An i915 perf stream
3649	* @cmd: the ioctl request
3650	* @arg: the ioctl data
3651	*
3652	* Returns: zero on success or a negative error code. Returns -EINVAL for
3653	* an unknown ioctl request.
3654	*/
3655	static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
3656	unsigned int cmd,
3657	unsigned long arg)
3658	{
3659	switch (cmd) {
3660	case I915_PERF_IOCTL_ENABLE:
3661	i915_perf_enable_locked(stream);
3662	return 0;
3663	case I915_PERF_IOCTL_DISABLE:
3664	i915_perf_disable_locked(stream);
3665	return 0;
3666	case I915_PERF_IOCTL_CONFIG:
3667	return i915_perf_config_locked(stream, metrics_set: arg);
3668	}
3669
3670	return -EINVAL;
3671	}
3672
3673	/**
3674	* i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3675	* @file: An i915 perf stream file
3676	* @cmd: the ioctl request
3677	* @arg: the ioctl data
3678	*
3679	* Implementation deferred to i915_perf_ioctl_locked().
3680	*
3681	* Returns: zero on success or a negative error code. Returns -EINVAL for
3682	* an unknown ioctl request.
3683	*/
3684	static long i915_perf_ioctl(struct file *file,
3685	unsigned int cmd,
3686	unsigned long arg)
3687	{
3688	struct i915_perf_stream *stream = file->private_data;
3689	long ret;
3690
3691	mutex_lock(&stream->lock);
3692	ret = i915_perf_ioctl_locked(stream, cmd, arg);
3693	mutex_unlock(lock: &stream->lock);
3694
3695	return ret;
3696	}
3697
3698	/**
3699	* i915_perf_destroy_locked - destroy an i915 perf stream
3700	* @stream: An i915 perf stream
3701	*
3702	* Frees all resources associated with the given i915 perf @stream, disabling
3703	* any associated data capture in the process.
3704	*
3705	* Note: The &gt->perf.lock mutex has been taken to serialize
3706	* with any non-file-operation driver hooks.
3707	*/
3708	static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
3709	{
3710	if (stream->enabled)
3711	i915_perf_disable_locked(stream);
3712
3713	if (stream->ops->destroy)
3714	stream->ops->destroy(stream);
3715
3716	if (stream->ctx)
3717	i915_gem_context_put(ctx: stream->ctx);
3718
3719	kfree(objp: stream);
3720	}
3721
3722	/**
3723	* i915_perf_release - handles userspace close() of a stream file
3724	* @inode: anonymous inode associated with file
3725	* @file: An i915 perf stream file
3726	*
3727	* Cleans up any resources associated with an open i915 perf stream file.
3728	*
3729	* NB: close() can't really fail from the userspace point of view.
3730	*
3731	* Returns: zero on success or a negative error code.
3732	*/
3733	static int i915_perf_release(struct inode *inode, struct file *file)
3734	{
3735	struct i915_perf_stream *stream = file->private_data;
3736	struct i915_perf *perf = stream->perf;
3737	struct intel_gt *gt = stream->engine->gt;
3738
3739	/*
3740	* Within this call, we know that the fd is being closed and we have no
3741	* other user of stream->lock. Use the perf lock to destroy the stream
3742	* here.
3743	*/
3744	mutex_lock(&gt->perf.lock);
3745	i915_perf_destroy_locked(stream);
3746	mutex_unlock(lock: &gt->perf.lock);
3747
3748	/* Release the reference the perf stream kept on the driver. */
3749	drm_dev_put(dev: &perf->i915->drm);
3750
3751	return 0;
3752	}
3753
3754
3755	static const struct file_operations fops = {
3756	.owner = THIS_MODULE,
3757	.release = i915_perf_release,
3758	.poll = i915_perf_poll,
3759	.read = i915_perf_read,
3760	.unlocked_ioctl = i915_perf_ioctl,
3761	/* Our ioctl have no arguments, so it's safe to use the same function
3762	* to handle 32bits compatibility.
3763	*/
3764	.compat_ioctl = i915_perf_ioctl,
3765	};
3766
3767
3768	/**
3769	* i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3770	* @perf: i915 perf instance
3771	* @param: The open parameters passed to 'DRM_I915_PERF_OPEN`
3772	* @props: individually validated u64 property value pairs
3773	* @file: drm file
3774	*
3775	* See i915_perf_ioctl_open() for interface details.
3776	*
3777	* Implements further stream config validation and stream initialization on
3778	* behalf of i915_perf_open_ioctl() with the &gt->perf.lock mutex
3779	* taken to serialize with any non-file-operation driver hooks.
3780	*
3781	* Note: at this point the @props have only been validated in isolation and
3782	* it's still necessary to validate that the combination of properties makes
3783	* sense.
3784	*
3785	* In the case where userspace is interested in OA unit metrics then further
3786	* config validation and stream initialization details will be handled by
3787	* i915_oa_stream_init(). The code here should only validate config state that
3788	* will be relevant to all stream types / backends.
3789	*
3790	* Returns: zero on success or a negative error code.
3791	*/
3792	static int
3793	i915_perf_open_ioctl_locked(struct i915_perf *perf,
3794	struct drm_i915_perf_open_param *param,
3795	struct perf_open_properties *props,
3796	struct drm_file *file)
3797	{
3798	struct i915_gem_context *specific_ctx = NULL;
3799	struct i915_perf_stream *stream = NULL;
3800	unsigned long f_flags = 0;
3801	bool privileged_op = true;
3802	int stream_fd;
3803	int ret;
3804
3805	if (props->single_context) {
3806	u32 ctx_handle = props->ctx_handle;
3807	struct drm_i915_file_private *file_priv = file->driver_priv;
3808
3809	specific_ctx = i915_gem_context_lookup(file_priv, id: ctx_handle);
3810	if (IS_ERR(ptr: specific_ctx)) {
3811	drm_dbg(&perf->i915->drm,
3812	"Failed to look up context with ID %u for opening perf stream\n",
3813	ctx_handle);
3814	ret = PTR_ERR(ptr: specific_ctx);
3815	goto err;
3816	}
3817	}
3818
3819	/*
3820	* On Haswell the OA unit supports clock gating off for a specific
3821	* context and in this mode there's no visibility of metrics for the
3822	* rest of the system, which we consider acceptable for a
3823	* non-privileged client.
3824	*
3825	* For Gen8->11 the OA unit no longer supports clock gating off for a
3826	* specific context and the kernel can't securely stop the counters
3827	* from updating as system-wide / global values. Even though we can
3828	* filter reports based on the included context ID we can't block
3829	* clients from seeing the raw / global counter values via
3830	* MI_REPORT_PERF_COUNT commands and so consider it a privileged op to
3831	* enable the OA unit by default.
3832	*
3833	* For Gen12+ we gain a new OAR unit that only monitors the RCS on a
3834	* per context basis. So we can relax requirements there if the user
3835	* doesn't request global stream access (i.e. query based sampling
3836	* using MI_RECORD_PERF_COUNT.
3837	*/
3838	if (IS_HASWELL(perf->i915) && specific_ctx)
3839	privileged_op = false;
3840	else if (GRAPHICS_VER(perf->i915) == 12 && specific_ctx &&
3841	(props->sample_flags & SAMPLE_OA_REPORT) == 0)
3842	privileged_op = false;
3843
3844	if (props->hold_preemption) {
3845	if (!props->single_context) {
3846	drm_dbg(&perf->i915->drm,
3847	"preemption disable with no context\n");
3848	ret = -EINVAL;
3849	goto err;
3850	}
3851	privileged_op = true;
3852	}
3853
3854	/*
3855	* Asking for SSEU configuration is a privileged operation.
3856	*/
3857	if (props->has_sseu)
3858	privileged_op = true;
3859	else
3860	get_default_sseu_config(out_sseu: &props->sseu, engine: props->engine);
3861
3862	/* Similar to perf's kernel.perf_paranoid_cpu sysctl option
3863	* we check a dev.i915.perf_stream_paranoid sysctl option
3864	* to determine if it's ok to access system wide OA counters
3865	* without CAP_PERFMON or CAP_SYS_ADMIN privileges.
3866	*/
3867	if (privileged_op &&
3868	i915_perf_stream_paranoid && !perfmon_capable()) {
3869	drm_dbg(&perf->i915->drm,
3870	"Insufficient privileges to open i915 perf stream\n");
3871	ret = -EACCES;
3872	goto err_ctx;
3873	}
3874
3875	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
3876	if (!stream) {
3877	ret = -ENOMEM;
3878	goto err_ctx;
3879	}
3880
3881	stream->perf = perf;
3882	stream->ctx = specific_ctx;
3883	stream->poll_oa_period = props->poll_oa_period;
3884
3885	ret = i915_oa_stream_init(stream, param, props);
3886	if (ret)
3887	goto err_alloc;
3888
3889	/* we avoid simply assigning stream->sample_flags = props->sample_flags
3890	* to have _stream_init check the combination of sample flags more
3891	* thoroughly, but still this is the expected result at this point.
3892	*/
3893	if (WARN_ON(stream->sample_flags != props->sample_flags)) {
3894	ret = -ENODEV;
3895	goto err_flags;
3896	}
3897
3898	if (param->flags & I915_PERF_FLAG_FD_CLOEXEC)
3899	f_flags |= O_CLOEXEC;
3900	if (param->flags & I915_PERF_FLAG_FD_NONBLOCK)
3901	f_flags |= O_NONBLOCK;
3902
3903	stream_fd = anon_inode_getfd(name: "[i915_perf]", fops: &fops, priv: stream, flags: f_flags);
3904	if (stream_fd < 0) {
3905	ret = stream_fd;
3906	goto err_flags;
3907	}
3908
3909	if (!(param->flags & I915_PERF_FLAG_DISABLED))
3910	i915_perf_enable_locked(stream);
3911
3912	/* Take a reference on the driver that will be kept with stream_fd
3913	* until its release.
3914	*/
3915	drm_dev_get(dev: &perf->i915->drm);
3916
3917	return stream_fd;
3918
3919	err_flags:
3920	if (stream->ops->destroy)
3921	stream->ops->destroy(stream);
3922	err_alloc:
3923	kfree(objp: stream);
3924	err_ctx:
3925	if (specific_ctx)
3926	i915_gem_context_put(ctx: specific_ctx);
3927	err:
3928	return ret;
3929	}
3930
3931	static u64 oa_exponent_to_ns(struct i915_perf *perf, int exponent)
3932	{
3933	u64 nom = (2ULL << exponent) * NSEC_PER_SEC;
3934	u32 den = i915_perf_oa_timestamp_frequency(i915: perf->i915);
3935
3936	return div_u64(dividend: nom + den - 1, divisor: den);
3937	}
3938
3939	static __always_inline bool
3940	oa_format_valid(struct i915_perf *perf, enum drm_i915_oa_format format)
3941	{
3942	return test_bit(format, perf->format_mask);
3943	}
3944
3945	static __always_inline void
3946	oa_format_add(struct i915_perf *perf, enum drm_i915_oa_format format)
3947	{
3948	__set_bit(format, perf->format_mask);
3949	}
3950
3951	/**
3952	* read_properties_unlocked - validate + copy userspace stream open properties
3953	* @perf: i915 perf instance
3954	* @uprops: The array of u64 key value pairs given by userspace
3955	* @n_props: The number of key value pairs expected in @uprops
3956	* @props: The stream configuration built up while validating properties
3957	*
3958	* Note this function only validates properties in isolation it doesn't
3959	* validate that the combination of properties makes sense or that all
3960	* properties necessary for a particular kind of stream have been set.
3961	*
3962	* Note that there currently aren't any ordering requirements for properties so
3963	* we shouldn't validate or assume anything about ordering here. This doesn't
3964	* rule out defining new properties with ordering requirements in the future.
3965	*/
3966	static int read_properties_unlocked(struct i915_perf *perf,
3967	u64 __user *uprops,
3968	u32 n_props,
3969	struct perf_open_properties *props)
3970	{
3971	struct drm_i915_gem_context_param_sseu user_sseu;
3972	const struct i915_oa_format *f;
3973	u64 __user *uprop = uprops;
3974	bool config_instance = false;
3975	bool config_class = false;
3976	bool config_sseu = false;
3977	u8 class, instance;
3978	u32 i;
3979	int ret;
3980
3981	memset(props, 0, sizeof(struct perf_open_properties));
3982	props->poll_oa_period = DEFAULT_POLL_PERIOD_NS;
3983
3984	/* Considering that ID = 0 is reserved and assuming that we don't
3985	* (currently) expect any configurations to ever specify duplicate
3986	* values for a particular property ID then the last _PROP_MAX value is
3987	* one greater than the maximum number of properties we expect to get
3988	* from userspace.
3989	*/
3990	if (!n_props || n_props >= DRM_I915_PERF_PROP_MAX) {
3991	drm_dbg(&perf->i915->drm,
3992	"Invalid number of i915 perf properties given\n");
3993	return -EINVAL;
3994	}
3995
3996	/* Defaults when class:instance is not passed */
3997	class = I915_ENGINE_CLASS_RENDER;
3998	instance = 0;
3999
4000	for (i = 0; i < n_props; i++) {
4001	u64 oa_period, oa_freq_hz;
4002	u64 id, value;
4003
4004	ret = get_user(id, uprop);
4005	if (ret)
4006	return ret;
4007
4008	ret = get_user(value, uprop + 1);
4009	if (ret)
4010	return ret;
4011
4012	if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) {
4013	drm_dbg(&perf->i915->drm,
4014	"Unknown i915 perf property ID\n");
4015	return -EINVAL;
4016	}
4017
4018	switch ((enum drm_i915_perf_property_id)id) {
4019	case DRM_I915_PERF_PROP_CTX_HANDLE:
4020	props->single_context = 1;
4021	props->ctx_handle = value;
4022	break;
4023	case DRM_I915_PERF_PROP_SAMPLE_OA:
4024	if (value)
4025	props->sample_flags |= SAMPLE_OA_REPORT;
4026	break;
4027	case DRM_I915_PERF_PROP_OA_METRICS_SET:
4028	if (value == 0) {
4029	drm_dbg(&perf->i915->drm,
4030	"Unknown OA metric set ID\n");
4031	return -EINVAL;
4032	}
4033	props->metrics_set = value;
4034	break;
4035	case DRM_I915_PERF_PROP_OA_FORMAT:
4036	if (value == 0 || value >= I915_OA_FORMAT_MAX) {
4037	drm_dbg(&perf->i915->drm,
4038	"Out-of-range OA report format %llu\n",
4039	value);
4040	return -EINVAL;
4041	}
4042	if (!oa_format_valid(perf, format: value)) {
4043	drm_dbg(&perf->i915->drm,
4044	"Unsupported OA report format %llu\n",
4045	value);
4046	return -EINVAL;
4047	}
4048	props->oa_format = value;
4049	break;
4050	case DRM_I915_PERF_PROP_OA_EXPONENT:
4051	if (value > OA_EXPONENT_MAX) {
4052	drm_dbg(&perf->i915->drm,
4053	"OA timer exponent too high (> %u)\n",
4054	OA_EXPONENT_MAX);
4055	return -EINVAL;
4056	}
4057
4058	/* Theoretically we can program the OA unit to sample
4059	* e.g. every 160ns for HSW, 167ns for BDW/SKL or 104ns
4060	* for BXT. We don't allow such high sampling
4061	* frequencies by default unless root.
4062	*/
4063
4064	BUILD_BUG_ON(sizeof(oa_period) != 8);
4065	oa_period = oa_exponent_to_ns(perf, exponent: value);
4066
4067	/* This check is primarily to ensure that oa_period <=
4068	* UINT32_MAX (before passing to do_div which only
4069	* accepts a u32 denominator), but we can also skip
4070	* checking anything < 1Hz which implicitly can't be
4071	* limited via an integer oa_max_sample_rate.
4072	*/
4073	if (oa_period <= NSEC_PER_SEC) {
4074	u64 tmp = NSEC_PER_SEC;
4075	do_div(tmp, oa_period);
4076	oa_freq_hz = tmp;
4077	} else
4078	oa_freq_hz = 0;
4079
4080	if (oa_freq_hz > i915_oa_max_sample_rate && !perfmon_capable()) {
4081	drm_dbg(&perf->i915->drm,
4082	"OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without CAP_PERFMON or CAP_SYS_ADMIN privileges\n",
4083	i915_oa_max_sample_rate);
4084	return -EACCES;
4085	}
4086
4087	props->oa_periodic = true;
4088	props->oa_period_exponent = value;
4089	break;
4090	case DRM_I915_PERF_PROP_HOLD_PREEMPTION:
4091	props->hold_preemption = !!value;
4092	break;
4093	case DRM_I915_PERF_PROP_GLOBAL_SSEU: {
4094	if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 55)) {
4095	drm_dbg(&perf->i915->drm,
4096	"SSEU config not supported on gfx %x\n",
4097	GRAPHICS_VER_FULL(perf->i915));
4098	return -ENODEV;
4099	}
4100
4101	if (copy_from_user(to: &user_sseu,
4102	u64_to_user_ptr(value),
4103	n: sizeof(user_sseu))) {
4104	drm_dbg(&perf->i915->drm,
4105	"Unable to copy global sseu parameter\n");
4106	return -EFAULT;
4107	}
4108	config_sseu = true;
4109	break;
4110	}
4111	case DRM_I915_PERF_PROP_POLL_OA_PERIOD:
4112	if (value < 100000 /* 100us */) {
4113	drm_dbg(&perf->i915->drm,
4114	"OA availability timer too small (%lluns < 100us)\n",
4115	value);
4116	return -EINVAL;
4117	}
4118	props->poll_oa_period = value;
4119	break;
4120	case DRM_I915_PERF_PROP_OA_ENGINE_CLASS:
4121	class = (u8)value;
4122	config_class = true;
4123	break;
4124	case DRM_I915_PERF_PROP_OA_ENGINE_INSTANCE:
4125	instance = (u8)value;
4126	config_instance = true;
4127	break;
4128	default:
4129	MISSING_CASE(id);
4130	return -EINVAL;
4131	}
4132
4133	uprop += 2;
4134	}
4135
4136	if ((config_class && !config_instance) ||
4137	(config_instance && !config_class)) {
4138	drm_dbg(&perf->i915->drm,
4139	"OA engine-class and engine-instance parameters must be passed together\n");
4140	return -EINVAL;
4141	}
4142
4143	props->engine = intel_engine_lookup_user(i915: perf->i915, class, instance);
4144	if (!props->engine) {
4145	drm_dbg(&perf->i915->drm,
4146	"OA engine class and instance invalid %d:%d\n",
4147	class, instance);
4148	return -EINVAL;
4149	}
4150
4151	if (!engine_supports_oa(engine: props->engine)) {
4152	drm_dbg(&perf->i915->drm,
4153	"Engine not supported by OA %d:%d\n",
4154	class, instance);
4155	return -EINVAL;
4156	}
4157
4158	/*
4159	* Wa_14017512683: mtl[a0..c0): Use of OAM must be preceded with Media
4160	* C6 disable in BIOS. Fail if Media C6 is enabled on steppings where OAM
4161	* does not work as expected.
4162	*/
4163	if (IS_MEDIA_GT_IP_STEP(props->engine->gt, IP_VER(13, 0), STEP_A0, STEP_C0) &&
4164	props->engine->oa_group->type == TYPE_OAM &&
4165	intel_check_bios_c6_setup(rc6: &props->engine->gt->rc6)) {
4166	drm_dbg(&perf->i915->drm,
4167	"OAM requires media C6 to be disabled in BIOS\n");
4168	return -EINVAL;
4169	}
4170
4171	i = array_index_nospec(props->oa_format, I915_OA_FORMAT_MAX);
4172	f = &perf->oa_formats[i];
4173	if (!engine_supports_oa_format(engine: props->engine, type: f->type)) {
4174	drm_dbg(&perf->i915->drm,
4175	"Invalid OA format %d for class %d\n",
4176	f->type, props->engine->class);
4177	return -EINVAL;
4178	}
4179
4180	if (config_sseu) {
4181	ret = get_sseu_config(out_sseu: &props->sseu, engine: props->engine, drm_sseu: &user_sseu);
4182	if (ret) {
4183	drm_dbg(&perf->i915->drm,
4184	"Invalid SSEU configuration\n");
4185	return ret;
4186	}
4187	props->has_sseu = true;
4188	}
4189
4190	return 0;
4191	}
4192
4193	/**
4194	* i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
4195	* @dev: drm device
4196	* @data: ioctl data copied from userspace (unvalidated)
4197	* @file: drm file
4198	*
4199	* Validates the stream open parameters given by userspace including flags
4200	* and an array of u64 key, value pair properties.
4201	*
4202	* Very little is assumed up front about the nature of the stream being
4203	* opened (for instance we don't assume it's for periodic OA unit metrics). An
4204	* i915-perf stream is expected to be a suitable interface for other forms of
4205	* buffered data written by the GPU besides periodic OA metrics.
4206	*
4207	* Note we copy the properties from userspace outside of the i915 perf
4208	* mutex to avoid an awkward lockdep with mmap_lock.
4209	*
4210	* Most of the implementation details are handled by
4211	* i915_perf_open_ioctl_locked() after taking the &gt->perf.lock
4212	* mutex for serializing with any non-file-operation driver hooks.
4213	*
4214	* Return: A newly opened i915 Perf stream file descriptor or negative
4215	* error code on failure.
4216	*/
4217	int i915_perf_open_ioctl(struct drm_device *dev, void *data,
4218	struct drm_file *file)
4219	{
4220	struct i915_perf *perf = &to_i915(dev)->perf;
4221	struct drm_i915_perf_open_param *param = data;
4222	struct intel_gt *gt;
4223	struct perf_open_properties props;
4224	u32 known_open_flags;
4225	int ret;
4226
4227	if (!perf->i915)
4228	return -ENOTSUPP;
4229
4230	known_open_flags = I915_PERF_FLAG_FD_CLOEXEC |
4231	I915_PERF_FLAG_FD_NONBLOCK |
4232	I915_PERF_FLAG_DISABLED;
4233	if (param->flags & ~known_open_flags) {
4234	drm_dbg(&perf->i915->drm,
4235	"Unknown drm_i915_perf_open_param flag\n");
4236	return -EINVAL;
4237	}
4238
4239	ret = read_properties_unlocked(perf,
4240	u64_to_user_ptr(param->properties_ptr),
4241	n_props: param->num_properties,
4242	props: &props);
4243	if (ret)
4244	return ret;
4245
4246	gt = props.engine->gt;
4247
4248	mutex_lock(&gt->perf.lock);
4249	ret = i915_perf_open_ioctl_locked(perf, param, props: &props, file);
4250	mutex_unlock(lock: &gt->perf.lock);
4251
4252	return ret;
4253	}
4254
4255	/**
4256	* i915_perf_register - exposes i915-perf to userspace
4257	* @i915: i915 device instance
4258	*
4259	* In particular OA metric sets are advertised under a sysfs metrics/
4260	* directory allowing userspace to enumerate valid IDs that can be
4261	* used to open an i915-perf stream.
4262	*/
4263	void i915_perf_register(struct drm_i915_private *i915)
4264	{
4265	struct i915_perf *perf = &i915->perf;
4266	struct intel_gt *gt = to_gt(i915);
4267
4268	if (!perf->i915)
4269	return;
4270
4271	/* To be sure we're synchronized with an attempted
4272	* i915_perf_open_ioctl(); considering that we register after
4273	* being exposed to userspace.
4274	*/
4275	mutex_lock(&gt->perf.lock);
4276
4277	perf->metrics_kobj =
4278	kobject_create_and_add(name: "metrics",
4279	parent: &i915->drm.primary->kdev->kobj);
4280
4281	mutex_unlock(lock: &gt->perf.lock);
4282	}
4283
4284	/**
4285	* i915_perf_unregister - hide i915-perf from userspace
4286	* @i915: i915 device instance
4287	*
4288	* i915-perf state cleanup is split up into an 'unregister' and
4289	* 'deinit' phase where the interface is first hidden from
4290	* userspace by i915_perf_unregister() before cleaning up
4291	* remaining state in i915_perf_fini().
4292	*/
4293	void i915_perf_unregister(struct drm_i915_private *i915)
4294	{
4295	struct i915_perf *perf = &i915->perf;
4296
4297	if (!perf->metrics_kobj)
4298	return;
4299
4300	kobject_put(kobj: perf->metrics_kobj);
4301	perf->metrics_kobj = NULL;
4302	}
4303
4304	static bool gen8_is_valid_flex_addr(struct i915_perf *perf, u32 addr)
4305	{
4306	static const i915_reg_t flex_eu_regs[] = {
4307	EU_PERF_CNTL0,
4308	EU_PERF_CNTL1,
4309	EU_PERF_CNTL2,
4310	EU_PERF_CNTL3,
4311	EU_PERF_CNTL4,
4312	EU_PERF_CNTL5,
4313	EU_PERF_CNTL6,
4314	};
4315	int i;
4316
4317	for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
4318	if (i915_mmio_reg_offset(flex_eu_regs[i]) == addr)
4319	return true;
4320	}
4321	return false;
4322	}
4323
4324	#define REG_EQUAL(addr, mmio) \
4325	((addr) == i915_mmio_reg_offset(mmio))
4326
4327	static const struct i915_mmio_range gen7_oa_b_counters[] = {
4328	{ .start = 0x2710, .end = 0x272c }, /* OASTARTTRIG[1-8] */
4329	{ .start = 0x2740, .end = 0x275c }, /* OAREPORTTRIG[1-8] */
4330	{ .start = 0x2770, .end = 0x27ac }, /* OACEC[0-7][0-1] */
4331	{}
4332	};
4333
4334	static const struct i915_mmio_range gen12_oa_b_counters[] = {
4335	{ .start = 0x2b2c, .end = 0x2b2c }, /* GEN12_OAG_OA_PESS */
4336	{ .start = 0xd900, .end = 0xd91c }, /* GEN12_OAG_OASTARTTRIG[1-8] */
4337	{ .start = 0xd920, .end = 0xd93c }, /* GEN12_OAG_OAREPORTTRIG1[1-8] */
4338	{ .start = 0xd940, .end = 0xd97c }, /* GEN12_OAG_CEC[0-7][0-1] */
4339	{ .start = 0xdc00, .end = 0xdc3c }, /* GEN12_OAG_SCEC[0-7][0-1] */
4340	{ .start = 0xdc40, .end = 0xdc40 }, /* GEN12_OAG_SPCTR_CNF */
4341	{ .start = 0xdc44, .end = 0xdc44 }, /* GEN12_OAA_DBG_REG */
4342	{}
4343	};
4344
4345	static const struct i915_mmio_range mtl_oam_b_counters[] = {
4346	{ .start = 0x393000, .end = 0x39301c }, /* GEN12_OAM_STARTTRIG1[1-8] */
4347	{ .start = 0x393020, .end = 0x39303c }, /* GEN12_OAM_REPORTTRIG1[1-8] */
4348	{ .start = 0x393040, .end = 0x39307c }, /* GEN12_OAM_CEC[0-7][0-1] */
4349	{ .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */
4350	{}
4351	};
4352
4353	static const struct i915_mmio_range xehp_oa_b_counters[] = {
4354	{ .start = 0xdc48, .end = 0xdc48 }, /* OAA_ENABLE_REG */
4355	{ .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */
4356	{}
4357	};
4358
4359	static const struct i915_mmio_range gen7_oa_mux_regs[] = {
4360	{ .start = 0x91b8, .end = 0x91cc }, /* OA_PERFCNT[1-2], OA_PERFMATRIX */
4361	{ .start = 0x9800, .end = 0x9888 }, /* MICRO_BP0_0 - NOA_WRITE */
4362	{ .start = 0xe180, .end = 0xe180 }, /* HALF_SLICE_CHICKEN2 */
4363	{}
4364	};
4365
4366	static const struct i915_mmio_range hsw_oa_mux_regs[] = {
4367	{ .start = 0x09e80, .end = 0x09ea4 }, /* HSW_MBVID2_NOA[0-9] */
4368	{ .start = 0x09ec0, .end = 0x09ec0 }, /* HSW_MBVID2_MISR0 */
4369	{ .start = 0x25100, .end = 0x2ff90 },
4370	{}
4371	};
4372
4373	static const struct i915_mmio_range chv_oa_mux_regs[] = {
4374	{ .start = 0x182300, .end = 0x1823a4 },
4375	{}
4376	};
4377
4378	static const struct i915_mmio_range gen8_oa_mux_regs[] = {
4379	{ .start = 0x0d00, .end = 0x0d2c }, /* RPM_CONFIG[0-1], NOA_CONFIG[0-8] */
4380	{ .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */
4381	{}
4382	};
4383
4384	static const struct i915_mmio_range gen11_oa_mux_regs[] = {
4385	{ .start = 0x91c8, .end = 0x91dc }, /* OA_PERFCNT[3-4] */
4386	{}
4387	};
4388
4389	static const struct i915_mmio_range gen12_oa_mux_regs[] = {
4390	{ .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
4391	{ .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
4392	{ .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */
4393	{ .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */
4394	{ .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */
4395	{}
4396	};
4397
4398	/*
4399	* Ref: 14010536224:
4400	* 0x20cc is repurposed on MTL, so use a separate array for MTL.
4401	*/
4402	static const struct i915_mmio_range mtl_oa_mux_regs[] = {
4403	{ .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
4404	{ .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
4405	{ .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */
4406	{ .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */
4407	{ .start = 0x38d100, .end = 0x38d114}, /* VISACTL */
4408	{}
4409	};
4410
4411	static bool gen7_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
4412	{
4413	return i915_mmio_range_table_contains(addr, table: gen7_oa_b_counters);
4414	}
4415
4416	static bool gen8_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4417	{
4418	return i915_mmio_range_table_contains(addr, table: gen7_oa_mux_regs) ||
4419	i915_mmio_range_table_contains(addr, table: gen8_oa_mux_regs);
4420	}
4421
4422	static bool gen11_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4423	{
4424	return i915_mmio_range_table_contains(addr, table: gen7_oa_mux_regs) ||
4425	i915_mmio_range_table_contains(addr, table: gen8_oa_mux_regs) ||
4426	i915_mmio_range_table_contains(addr, table: gen11_oa_mux_regs);
4427	}
4428
4429	static bool hsw_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4430	{
4431	return i915_mmio_range_table_contains(addr, table: gen7_oa_mux_regs) ||
4432	i915_mmio_range_table_contains(addr, table: hsw_oa_mux_regs);
4433	}
4434
4435	static bool chv_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4436	{
4437	return i915_mmio_range_table_contains(addr, table: gen7_oa_mux_regs) ||
4438	i915_mmio_range_table_contains(addr, table: chv_oa_mux_regs);
4439	}
4440
4441	static bool gen12_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
4442	{
4443	return i915_mmio_range_table_contains(addr, table: gen12_oa_b_counters);
4444	}
4445
4446	static bool mtl_is_valid_oam_b_counter_addr(struct i915_perf *perf, u32 addr)
4447	{
4448	if (HAS_OAM(perf->i915) &&
4449	GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70))
4450	return i915_mmio_range_table_contains(addr, table: mtl_oam_b_counters);
4451
4452	return false;
4453	}
4454
4455	static bool xehp_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
4456	{
4457	return i915_mmio_range_table_contains(addr, table: xehp_oa_b_counters) ||
4458	i915_mmio_range_table_contains(addr, table: gen12_oa_b_counters) ||
4459	mtl_is_valid_oam_b_counter_addr(perf, addr);
4460	}
4461
4462	static bool gen12_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
4463	{
4464	if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 70))
4465	return i915_mmio_range_table_contains(addr, table: mtl_oa_mux_regs);
4466	else
4467	return i915_mmio_range_table_contains(addr, table: gen12_oa_mux_regs);
4468	}
4469
4470	static u32 mask_reg_value(u32 reg, u32 val)
4471	{
4472	/*
4473	* HALF_SLICE_CHICKEN2 is programmed with a the
4474	* WaDisableSTUnitPowerOptimization workaround. Make sure the value
4475	* programmed by userspace doesn't change this.
4476	*/
4477	if (REG_EQUAL(reg, HALF_SLICE_CHICKEN2))
4478	val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE);
4479
4480	/*
4481	* WAIT_FOR_RC6_EXIT has only one bit fulfilling the function
4482	* indicated by its name and a bunch of selection fields used by OA
4483	* configs.
4484	*/
4485	if (REG_EQUAL(reg, WAIT_FOR_RC6_EXIT))
4486	val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
4487
4488	return val;
4489	}
4490
4491	static struct i915_oa_reg *alloc_oa_regs(struct i915_perf *perf,
4492	bool (*is_valid)(struct i915_perf *perf, u32 addr),
4493	u32 __user *regs,
4494	u32 n_regs)
4495	{
4496	struct i915_oa_reg *oa_regs;
4497	int err;
4498	u32 i;
4499
4500	if (!n_regs)
4501	return NULL;
4502
4503	/* No is_valid function means we're not allowing any register to be programmed. */
4504	GEM_BUG_ON(!is_valid);
4505	if (!is_valid)
4506	return ERR_PTR(error: -EINVAL);
4507
4508	oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL);
4509	if (!oa_regs)
4510	return ERR_PTR(error: -ENOMEM);
4511
4512	for (i = 0; i < n_regs; i++) {
4513	u32 addr, value;
4514
4515	err = get_user(addr, regs);
4516	if (err)
4517	goto addr_err;
4518
4519	if (!is_valid(perf, addr)) {
4520	drm_dbg(&perf->i915->drm,
4521	"Invalid oa_reg address: %X\n", addr);
4522	err = -EINVAL;
4523	goto addr_err;
4524	}
4525
4526	err = get_user(value, regs + 1);
4527	if (err)
4528	goto addr_err;
4529
4530	oa_regs[i].addr = _MMIO(addr);
4531	oa_regs[i].value = mask_reg_value(reg: addr, val: value);
4532
4533	regs += 2;
4534	}
4535
4536	return oa_regs;
4537
4538	addr_err:
4539	kfree(objp: oa_regs);
4540	return ERR_PTR(error: err);
4541	}
4542
4543	static ssize_t show_dynamic_id(struct kobject *kobj,
4544	struct kobj_attribute *attr,
4545	char *buf)
4546	{
4547	struct i915_oa_config *oa_config =
4548	container_of(attr, typeof(*oa_config), sysfs_metric_id);
4549
4550	return sprintf(buf, fmt: "%d\n", oa_config->id);
4551	}
4552
4553	static int create_dynamic_oa_sysfs_entry(struct i915_perf *perf,
4554	struct i915_oa_config *oa_config)
4555	{
4556	sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
4557	oa_config->sysfs_metric_id.attr.name = "id";
4558	oa_config->sysfs_metric_id.attr.mode = S_IRUGO;
4559	oa_config->sysfs_metric_id.show = show_dynamic_id;
4560	oa_config->sysfs_metric_id.store = NULL;
4561
4562	oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
4563	oa_config->attrs[1] = NULL;
4564
4565	oa_config->sysfs_metric.name = oa_config->uuid;
4566	oa_config->sysfs_metric.attrs = oa_config->attrs;
4567
4568	return sysfs_create_group(kobj: perf->metrics_kobj,
4569	grp: &oa_config->sysfs_metric);
4570	}
4571
4572	/**
4573	* i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
4574	* @dev: drm device
4575	* @data: ioctl data (pointer to struct drm_i915_perf_oa_config) copied from
4576	* userspace (unvalidated)
4577	* @file: drm file
4578	*
4579	* Validates the submitted OA register to be saved into a new OA config that
4580	* can then be used for programming the OA unit and its NOA network.
4581	*
4582	* Returns: A new allocated config number to be used with the perf open ioctl
4583	* or a negative error code on failure.
4584	*/
4585	int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
4586	struct drm_file *file)
4587	{
4588	struct i915_perf *perf = &to_i915(dev)->perf;
4589	struct drm_i915_perf_oa_config *args = data;
4590	struct i915_oa_config *oa_config, *tmp;
4591	struct i915_oa_reg *regs;
4592	int err, id;
4593
4594	if (!perf->i915)
4595	return -ENOTSUPP;
4596
4597	if (!perf->metrics_kobj) {
4598	drm_dbg(&perf->i915->drm,
4599	"OA metrics weren't advertised via sysfs\n");
4600	return -EINVAL;
4601	}
4602
4603	if (i915_perf_stream_paranoid && !perfmon_capable()) {
4604	drm_dbg(&perf->i915->drm,
4605	"Insufficient privileges to add i915 OA config\n");
4606	return -EACCES;
4607	}
4608
4609	if ((!args->mux_regs_ptr || !args->n_mux_regs) &&
4610	(!args->boolean_regs_ptr || !args->n_boolean_regs) &&
4611	(!args->flex_regs_ptr || !args->n_flex_regs)) {
4612	drm_dbg(&perf->i915->drm,
4613	"No OA registers given\n");
4614	return -EINVAL;
4615	}
4616
4617	oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL);
4618	if (!oa_config) {
4619	drm_dbg(&perf->i915->drm,
4620	"Failed to allocate memory for the OA config\n");
4621	return -ENOMEM;
4622	}
4623
4624	oa_config->perf = perf;
4625	kref_init(kref: &oa_config->ref);
4626
4627	if (!uuid_is_valid(uuid: args->uuid)) {
4628	drm_dbg(&perf->i915->drm,
4629	"Invalid uuid format for OA config\n");
4630	err = -EINVAL;
4631	goto reg_err;
4632	}
4633
4634	/* Last character in oa_config->uuid will be 0 because oa_config is
4635	* kzalloc.
4636	*/
4637	memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid));
4638
4639	oa_config->mux_regs_len = args->n_mux_regs;
4640	regs = alloc_oa_regs(perf,
4641	is_valid: perf->ops.is_valid_mux_reg,
4642	u64_to_user_ptr(args->mux_regs_ptr),
4643	n_regs: args->n_mux_regs);
4644
4645	if (IS_ERR(ptr: regs)) {
4646	drm_dbg(&perf->i915->drm,
4647	"Failed to create OA config for mux_regs\n");
4648	err = PTR_ERR(ptr: regs);
4649	goto reg_err;
4650	}
4651	oa_config->mux_regs = regs;
4652
4653	oa_config->b_counter_regs_len = args->n_boolean_regs;
4654	regs = alloc_oa_regs(perf,
4655	is_valid: perf->ops.is_valid_b_counter_reg,
4656	u64_to_user_ptr(args->boolean_regs_ptr),
4657	n_regs: args->n_boolean_regs);
4658
4659	if (IS_ERR(ptr: regs)) {
4660	drm_dbg(&perf->i915->drm,
4661	"Failed to create OA config for b_counter_regs\n");
4662	err = PTR_ERR(ptr: regs);
4663	goto reg_err;
4664	}
4665	oa_config->b_counter_regs = regs;
4666
4667	if (GRAPHICS_VER(perf->i915) < 8) {
4668	if (args->n_flex_regs != 0) {
4669	err = -EINVAL;
4670	goto reg_err;
4671	}
4672	} else {
4673	oa_config->flex_regs_len = args->n_flex_regs;
4674	regs = alloc_oa_regs(perf,
4675	is_valid: perf->ops.is_valid_flex_reg,
4676	u64_to_user_ptr(args->flex_regs_ptr),
4677	n_regs: args->n_flex_regs);
4678
4679	if (IS_ERR(ptr: regs)) {
4680	drm_dbg(&perf->i915->drm,
4681	"Failed to create OA config for flex_regs\n");
4682	err = PTR_ERR(ptr: regs);
4683	goto reg_err;
4684	}
4685	oa_config->flex_regs = regs;
4686	}
4687
4688	err = mutex_lock_interruptible(&perf->metrics_lock);
4689	if (err)
4690	goto reg_err;
4691
4692	/* We shouldn't have too many configs, so this iteration shouldn't be
4693	* too costly.
4694	*/
4695	idr_for_each_entry(&perf->metrics_idr, tmp, id) {
4696	if (!strcmp(tmp->uuid, oa_config->uuid)) {
4697	drm_dbg(&perf->i915->drm,
4698	"OA config already exists with this uuid\n");
4699	err = -EADDRINUSE;
4700	goto sysfs_err;
4701	}
4702	}
4703
4704	err = create_dynamic_oa_sysfs_entry(perf, oa_config);
4705	if (err) {
4706	drm_dbg(&perf->i915->drm,
4707	"Failed to create sysfs entry for OA config\n");
4708	goto sysfs_err;
4709	}
4710
4711	/* Config id 0 is invalid, id 1 for kernel stored test config. */
4712	oa_config->id = idr_alloc(&perf->metrics_idr,
4713	ptr: oa_config, start: 2,
4714	end: 0, GFP_KERNEL);
4715	if (oa_config->id < 0) {
4716	drm_dbg(&perf->i915->drm,
4717	"Failed to create sysfs entry for OA config\n");
4718	err = oa_config->id;
4719	goto sysfs_err;
4720	}
4721	id = oa_config->id;
4722
4723	drm_dbg(&perf->i915->drm,
4724	"Added config %s id=%i\n", oa_config->uuid, oa_config->id);
4725	mutex_unlock(lock: &perf->metrics_lock);
4726
4727	return id;
4728
4729	sysfs_err:
4730	mutex_unlock(lock: &perf->metrics_lock);
4731	reg_err:
4732	i915_oa_config_put(oa_config);
4733	drm_dbg(&perf->i915->drm,
4734	"Failed to add new OA config\n");
4735	return err;
4736	}
4737
4738	/**
4739	* i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
4740	* @dev: drm device
4741	* @data: ioctl data (pointer to u64 integer) copied from userspace
4742	* @file: drm file
4743	*
4744	* Configs can be removed while being used, the will stop appearing in sysfs
4745	* and their content will be freed when the stream using the config is closed.
4746	*
4747	* Returns: 0 on success or a negative error code on failure.
4748	*/
4749	int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
4750	struct drm_file *file)
4751	{
4752	struct i915_perf *perf = &to_i915(dev)->perf;
4753	u64 *arg = data;
4754	struct i915_oa_config *oa_config;
4755	int ret;
4756
4757	if (!perf->i915)
4758	return -ENOTSUPP;
4759
4760	if (i915_perf_stream_paranoid && !perfmon_capable()) {
4761	drm_dbg(&perf->i915->drm,
4762	"Insufficient privileges to remove i915 OA config\n");
4763	return -EACCES;
4764	}
4765
4766	ret = mutex_lock_interruptible(&perf->metrics_lock);
4767	if (ret)
4768	return ret;
4769
4770	oa_config = idr_find(&perf->metrics_idr, id: *arg);
4771	if (!oa_config) {
4772	drm_dbg(&perf->i915->drm,
4773	"Failed to remove unknown OA config\n");
4774	ret = -ENOENT;
4775	goto err_unlock;
4776	}
4777
4778	GEM_BUG_ON(*arg != oa_config->id);
4779
4780	sysfs_remove_group(kobj: perf->metrics_kobj, grp: &oa_config->sysfs_metric);
4781
4782	idr_remove(&perf->metrics_idr, id: *arg);
4783
4784	mutex_unlock(lock: &perf->metrics_lock);
4785
4786	drm_dbg(&perf->i915->drm,
4787	"Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
4788
4789	i915_oa_config_put(oa_config);
4790
4791	return 0;
4792
4793	err_unlock:
4794	mutex_unlock(lock: &perf->metrics_lock);
4795	return ret;
4796	}
4797
4798	static const struct ctl_table oa_table[] = {
4799	{
4800	.procname = "perf_stream_paranoid",
4801	.data = &i915_perf_stream_paranoid,
4802	.maxlen = sizeof(i915_perf_stream_paranoid),
4803	.mode = 0644,
4804	.proc_handler = proc_dointvec_minmax,
4805	.extra1 = SYSCTL_ZERO,
4806	.extra2 = SYSCTL_ONE,
4807	},
4808	{
4809	.procname = "oa_max_sample_rate",
4810	.data = &i915_oa_max_sample_rate,
4811	.maxlen = sizeof(i915_oa_max_sample_rate),
4812	.mode = 0644,
4813	.proc_handler = proc_dointvec_minmax,
4814	.extra1 = SYSCTL_ZERO,
4815	.extra2 = &oa_sample_rate_hard_limit,
4816	},
4817	};
4818
4819	static u32 num_perf_groups_per_gt(struct intel_gt *gt)
4820	{
4821	return 1;
4822	}
4823
4824	static u32 __oam_engine_group(struct intel_engine_cs *engine)
4825	{
4826	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) {
4827	/*
4828	* There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices
4829	* within the gt use the same OAM. All MTL SKUs list 1 SA MEDIA.
4830	*/
4831	drm_WARN_ON(&engine->i915->drm,
4832	engine->gt->type != GT_MEDIA);
4833
4834	return PERF_GROUP_OAM_SAMEDIA_0;
4835	}
4836
4837	return PERF_GROUP_INVALID;
4838	}
4839
4840	static u32 __oa_engine_group(struct intel_engine_cs *engine)
4841	{
4842	switch (engine->class) {
4843	case RENDER_CLASS:
4844	return PERF_GROUP_OAG;
4845
4846	case VIDEO_DECODE_CLASS:
4847	case VIDEO_ENHANCEMENT_CLASS:
4848	return __oam_engine_group(engine);
4849
4850	default:
4851	return PERF_GROUP_INVALID;
4852	}
4853	}
4854
4855	static struct i915_perf_regs __oam_regs(u32 base)
4856	{
4857	return (struct i915_perf_regs) {
4858	base,
4859	GEN12_OAM_HEAD_POINTER(base),
4860	GEN12_OAM_TAIL_POINTER(base),
4861	GEN12_OAM_BUFFER(base),
4862	GEN12_OAM_CONTEXT_CONTROL(base),
4863	GEN12_OAM_CONTROL(base),
4864	GEN12_OAM_DEBUG(base),
4865	GEN12_OAM_STATUS(base),
4866	GEN12_OAM_CONTROL_COUNTER_FORMAT_SHIFT,
4867	};
4868	}
4869
4870	static struct i915_perf_regs __oag_regs(void)
4871	{
4872	return (struct i915_perf_regs) {
4873	0,
4874	GEN12_OAG_OAHEADPTR,
4875	GEN12_OAG_OATAILPTR,
4876	GEN12_OAG_OABUFFER,
4877	GEN12_OAG_OAGLBCTXCTRL,
4878	GEN12_OAG_OACONTROL,
4879	GEN12_OAG_OA_DEBUG,
4880	GEN12_OAG_OASTATUS,
4881	GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT,
4882	};
4883	}
4884
4885	static void oa_init_groups(struct intel_gt *gt)
4886	{
4887	int i, num_groups = gt->perf.num_perf_groups;
4888
4889	for (i = 0; i < num_groups; i++) {
4890	struct i915_perf_group *g = &gt->perf.group[i];
4891
4892	/* Fused off engines can result in a group with num_engines == 0 */
4893	if (g->num_engines == 0)
4894	continue;
4895
4896	if (i == PERF_GROUP_OAG && gt->type != GT_MEDIA) {
4897	g->regs = __oag_regs();
4898	g->type = TYPE_OAG;
4899	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
4900	g->regs = __oam_regs(base: mtl_oa_base[i]);
4901	g->type = TYPE_OAM;
4902	}
4903	}
4904	}
4905
4906	static int oa_init_gt(struct intel_gt *gt)
4907	{
4908	u32 num_groups = num_perf_groups_per_gt(gt);
4909	struct intel_engine_cs *engine;
4910	struct i915_perf_group *g;
4911	intel_engine_mask_t tmp;
4912
4913	g = kcalloc(num_groups, sizeof(*g), GFP_KERNEL);
4914	if (!g)
4915	return -ENOMEM;
4916
4917	for_each_engine_masked(engine, gt, ALL_ENGINES, tmp) {
4918	u32 index = __oa_engine_group(engine);
4919
4920	engine->oa_group = NULL;
4921	if (index < num_groups) {
4922	g[index].num_engines++;
4923	engine->oa_group = &g[index];
4924	}
4925	}
4926
4927	gt->perf.num_perf_groups = num_groups;
4928	gt->perf.group = g;
4929
4930	oa_init_groups(gt);
4931
4932	return 0;
4933	}
4934
4935	static int oa_init_engine_groups(struct i915_perf *perf)
4936	{
4937	struct intel_gt *gt;
4938	int i, ret;
4939
4940	for_each_gt(gt, perf->i915, i) {
4941	ret = oa_init_gt(gt);
4942	if (ret)
4943	return ret;
4944	}
4945
4946	return 0;
4947	}
4948
4949	static void oa_init_supported_formats(struct i915_perf *perf)
4950	{
4951	struct drm_i915_private *i915 = perf->i915;
4952	enum intel_platform platform = INTEL_INFO(i915)->platform;
4953
4954	switch (platform) {
4955	case INTEL_HASWELL:
4956	oa_format_add(perf, format: I915_OA_FORMAT_A13);
4957	oa_format_add(perf, format: I915_OA_FORMAT_A13);
4958	oa_format_add(perf, format: I915_OA_FORMAT_A29);
4959	oa_format_add(perf, format: I915_OA_FORMAT_A13_B8_C8);
4960	oa_format_add(perf, format: I915_OA_FORMAT_B4_C8);
4961	oa_format_add(perf, format: I915_OA_FORMAT_A45_B8_C8);
4962	oa_format_add(perf, format: I915_OA_FORMAT_B4_C8_A16);
4963	oa_format_add(perf, format: I915_OA_FORMAT_C4_B8);
4964	break;
4965
4966	case INTEL_BROADWELL:
4967	case INTEL_CHERRYVIEW:
4968	case INTEL_SKYLAKE:
4969	case INTEL_BROXTON:
4970	case INTEL_KABYLAKE:
4971	case INTEL_GEMINILAKE:
4972	case INTEL_COFFEELAKE:
4973	case INTEL_COMETLAKE:
4974	case INTEL_ICELAKE:
4975	case INTEL_ELKHARTLAKE:
4976	case INTEL_JASPERLAKE:
4977	case INTEL_TIGERLAKE:
4978	case INTEL_ROCKETLAKE:
4979	case INTEL_DG1:
4980	case INTEL_ALDERLAKE_S:
4981	case INTEL_ALDERLAKE_P:
4982	oa_format_add(perf, format: I915_OA_FORMAT_A12);
4983	oa_format_add(perf, format: I915_OA_FORMAT_A12_B8_C8);
4984	oa_format_add(perf, format: I915_OA_FORMAT_A32u40_A4u32_B8_C8);
4985	oa_format_add(perf, format: I915_OA_FORMAT_C4_B8);
4986	break;
4987
4988	case INTEL_DG2:
4989	oa_format_add(perf, format: I915_OAR_FORMAT_A32u40_A4u32_B8_C8);
4990	oa_format_add(perf, format: I915_OA_FORMAT_A24u40_A14u32_B8_C8);
4991	break;
4992
4993	case INTEL_METEORLAKE:
4994	oa_format_add(perf, format: I915_OAR_FORMAT_A32u40_A4u32_B8_C8);
4995	oa_format_add(perf, format: I915_OA_FORMAT_A24u40_A14u32_B8_C8);
4996	oa_format_add(perf, format: I915_OAM_FORMAT_MPEC8u64_B8_C8);
4997	oa_format_add(perf, format: I915_OAM_FORMAT_MPEC8u32_B8_C8);
4998	break;
4999
5000	default:
5001	MISSING_CASE(platform);
5002	}
5003	}
5004
5005	static void i915_perf_init_info(struct drm_i915_private *i915)
5006	{
5007	struct i915_perf *perf = &i915->perf;
5008
5009	switch (GRAPHICS_VER(i915)) {
5010	case 8:
5011	perf->ctx_oactxctrl_offset = 0x120;
5012	perf->ctx_flexeu0_offset = 0x2ce;
5013	perf->gen8_valid_ctx_bit = BIT(25);
5014	break;
5015	case 9:
5016	perf->ctx_oactxctrl_offset = 0x128;
5017	perf->ctx_flexeu0_offset = 0x3de;
5018	perf->gen8_valid_ctx_bit = BIT(16);
5019	break;
5020	case 11:
5021	perf->ctx_oactxctrl_offset = 0x124;
5022	perf->ctx_flexeu0_offset = 0x78e;
5023	perf->gen8_valid_ctx_bit = BIT(16);
5024	break;
5025	case 12:
5026	perf->gen8_valid_ctx_bit = BIT(16);
5027	/*
5028	* Calculate offset at runtime in oa_pin_context for gen12 and
5029	* cache the value in perf->ctx_oactxctrl_offset.
5030	*/
5031	break;
5032	default:
5033	MISSING_CASE(GRAPHICS_VER(i915));
5034	}
5035	}
5036
5037	/**
5038	* i915_perf_init - initialize i915-perf state on module bind
5039	* @i915: i915 device instance
5040	*
5041	* Initializes i915-perf state without exposing anything to userspace.
5042	*
5043	* Note: i915-perf initialization is split into an 'init' and 'register'
5044	* phase with the i915_perf_register() exposing state to userspace.
5045	*/
5046	int i915_perf_init(struct drm_i915_private *i915)
5047	{
5048	struct i915_perf *perf = &i915->perf;
5049
5050	perf->oa_formats = oa_formats;
5051	if (IS_HASWELL(i915)) {
5052	perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr;
5053	perf->ops.is_valid_mux_reg = hsw_is_valid_mux_addr;
5054	perf->ops.is_valid_flex_reg = NULL;
5055	perf->ops.enable_metric_set = hsw_enable_metric_set;
5056	perf->ops.disable_metric_set = hsw_disable_metric_set;
5057	perf->ops.oa_enable = gen7_oa_enable;
5058	perf->ops.oa_disable = gen7_oa_disable;
5059	perf->ops.read = gen7_oa_read;
5060	perf->ops.oa_hw_tail_read = gen7_oa_hw_tail_read;
5061	} else if (HAS_LOGICAL_RING_CONTEXTS(i915)) {
5062	/* Note: that although we could theoretically also support the
5063	* legacy ringbuffer mode on BDW (and earlier iterations of
5064	* this driver, before upstreaming did this) it didn't seem
5065	* worth the complexity to maintain now that BDW+ enable
5066	* execlist mode by default.
5067	*/
5068	perf->ops.read = gen8_oa_read;
5069	i915_perf_init_info(i915);
5070
5071	if (IS_GRAPHICS_VER(i915, 8, 9)) {
5072	perf->ops.is_valid_b_counter_reg =
5073	gen7_is_valid_b_counter_addr;
5074	perf->ops.is_valid_mux_reg =
5075	gen8_is_valid_mux_addr;
5076	perf->ops.is_valid_flex_reg =
5077	gen8_is_valid_flex_addr;
5078
5079	if (IS_CHERRYVIEW(i915)) {
5080	perf->ops.is_valid_mux_reg =
5081	chv_is_valid_mux_addr;
5082	}
5083
5084	perf->ops.oa_enable = gen8_oa_enable;
5085	perf->ops.oa_disable = gen8_oa_disable;
5086	perf->ops.enable_metric_set = gen8_enable_metric_set;
5087	perf->ops.disable_metric_set = gen8_disable_metric_set;
5088	perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
5089	} else if (GRAPHICS_VER(i915) == 11) {
5090	perf->ops.is_valid_b_counter_reg =
5091	gen7_is_valid_b_counter_addr;
5092	perf->ops.is_valid_mux_reg =
5093	gen11_is_valid_mux_addr;
5094	perf->ops.is_valid_flex_reg =
5095	gen8_is_valid_flex_addr;
5096
5097	perf->ops.oa_enable = gen8_oa_enable;
5098	perf->ops.oa_disable = gen8_oa_disable;
5099	perf->ops.enable_metric_set = gen8_enable_metric_set;
5100	perf->ops.disable_metric_set = gen11_disable_metric_set;
5101	perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
5102	} else if (GRAPHICS_VER(i915) == 12) {
5103	perf->ops.is_valid_b_counter_reg =
5104	HAS_OA_SLICE_CONTRIB_LIMITS(i915) ?
5105	xehp_is_valid_b_counter_addr :
5106	gen12_is_valid_b_counter_addr;
5107	perf->ops.is_valid_mux_reg =
5108	gen12_is_valid_mux_addr;
5109	perf->ops.is_valid_flex_reg =
5110	gen8_is_valid_flex_addr;
5111
5112	perf->ops.oa_enable = gen12_oa_enable;
5113	perf->ops.oa_disable = gen12_oa_disable;
5114	perf->ops.enable_metric_set = gen12_enable_metric_set;
5115	perf->ops.disable_metric_set = gen12_disable_metric_set;
5116	perf->ops.oa_hw_tail_read = gen12_oa_hw_tail_read;
5117	}
5118	}
5119
5120	if (perf->ops.enable_metric_set) {
5121	struct intel_gt *gt;
5122	int i, ret;
5123
5124	for_each_gt(gt, i915, i)
5125	mutex_init(&gt->perf.lock);
5126
5127	/* Choose a representative limit */
5128	oa_sample_rate_hard_limit = to_gt(i915)->clock_frequency / 2;
5129
5130	mutex_init(&perf->metrics_lock);
5131	idr_init_base(idr: &perf->metrics_idr, base: 1);
5132
5133	/* We set up some ratelimit state to potentially throttle any
5134	* _NOTES about spurious, invalid OA reports which we don't
5135	* forward to userspace.
5136	*
5137	* We print a _NOTE about any throttling when closing the
5138	* stream instead of waiting until driver _fini which no one
5139	* would ever see.
5140	*
5141	* Using the same limiting factors as printk_ratelimit()
5142	*/
5143	ratelimit_state_init(rs: &perf->spurious_report_rs, interval: 5 * HZ, burst: 10);
5144	/* Since we use a DRM_NOTE for spurious reports it would be
5145	* inconsistent to let __ratelimit() automatically print a
5146	* warning for throttling.
5147	*/
5148	ratelimit_set_flags(rs: &perf->spurious_report_rs,
5149	RATELIMIT_MSG_ON_RELEASE);
5150
5151	ratelimit_state_init(rs: &perf->tail_pointer_race,
5152	interval: 5 * HZ, burst: 10);
5153	ratelimit_set_flags(rs: &perf->tail_pointer_race,
5154	RATELIMIT_MSG_ON_RELEASE);
5155
5156	atomic64_set(v: &perf->noa_programming_delay,
5157	i: 500 * 1000 /* 500us */);
5158
5159	perf->i915 = i915;
5160
5161	ret = oa_init_engine_groups(perf);
5162	if (ret) {
5163	drm_err(&i915->drm,
5164	"OA initialization failed %d\n", ret);
5165	return ret;
5166	}
5167
5168	oa_init_supported_formats(perf);
5169	}
5170
5171	return 0;
5172	}
5173
5174	static int destroy_config(int id, void *p, void *data)
5175	{
5176	i915_oa_config_put(oa_config: p);
5177	return 0;
5178	}
5179
5180	int i915_perf_sysctl_register(void)
5181	{
5182	sysctl_header = register_sysctl("dev/i915", oa_table);
5183	return 0;
5184	}
5185
5186	void i915_perf_sysctl_unregister(void)
5187	{
5188	unregister_sysctl_table(table: sysctl_header);
5189	}
5190
5191	/**
5192	* i915_perf_fini - Counter part to i915_perf_init()
5193	* @i915: i915 device instance
5194	*/
5195	void i915_perf_fini(struct drm_i915_private *i915)
5196	{
5197	struct i915_perf *perf = &i915->perf;
5198	struct intel_gt *gt;
5199	int i;
5200
5201	if (!perf->i915)
5202	return;
5203
5204	for_each_gt(gt, perf->i915, i)
5205	kfree(objp: gt->perf.group);
5206
5207	idr_for_each(&perf->metrics_idr, fn: destroy_config, data: perf);
5208	idr_destroy(&perf->metrics_idr);
5209
5210	memset(&perf->ops, 0, sizeof(perf->ops));
5211	perf->i915 = NULL;
5212	}
5213
5214	/**
5215	* i915_perf_ioctl_version - Version of the i915-perf subsystem
5216	* @i915: The i915 device
5217	*
5218	* This version number is used by userspace to detect available features.
5219	*/
5220	int i915_perf_ioctl_version(struct drm_i915_private *i915)
5221	{
5222	/*
5223	* 1: Initial version
5224	* I915_PERF_IOCTL_ENABLE
5225	* I915_PERF_IOCTL_DISABLE
5226	*
5227	* 2: Added runtime modification of OA config.
5228	* I915_PERF_IOCTL_CONFIG
5229	*
5230	* 3: Add DRM_I915_PERF_PROP_HOLD_PREEMPTION parameter to hold
5231	* preemption on a particular context so that performance data is
5232	* accessible from a delta of MI_RPC reports without looking at the
5233	* OA buffer.
5234	*
5235	* 4: Add DRM_I915_PERF_PROP_ALLOWED_SSEU to limit what contexts can
5236	* be run for the duration of the performance recording based on
5237	* their SSEU configuration.
5238	*
5239	* 5: Add DRM_I915_PERF_PROP_POLL_OA_PERIOD parameter that controls the
5240	* interval for the hrtimer used to check for OA data.
5241	*
5242	* 6: Add DRM_I915_PERF_PROP_OA_ENGINE_CLASS and
5243	* DRM_I915_PERF_PROP_OA_ENGINE_INSTANCE
5244	*
5245	* 7: Add support for video decode and enhancement classes.
5246	*/
5247
5248	/*
5249	* Wa_14017512683: mtl[a0..c0): Use of OAM must be preceded with Media
5250	* C6 disable in BIOS. If Media C6 is enabled in BIOS, return version 6
5251	* to indicate that OA media is not supported.
5252	*/
5253	if (IS_MEDIA_GT_IP_STEP(i915->media_gt, IP_VER(13, 0), STEP_A0, STEP_C0) &&
5254	intel_check_bios_c6_setup(rc6: &i915->media_gt->rc6))
5255	return 6;
5256
5257	return 7;
5258	}
5259
5260	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
5261	#include "selftests/i915_perf.c"
5262	#endif
5263