1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2025 Intel Corporation
4 */
5
6#include <linux/anon_inodes.h>
7#include <linux/fs.h>
8#include <linux/poll.h>
9#include <linux/types.h>
10
11#include <drm/drm_drv.h>
12#include <generated/xe_wa_oob.h>
13#include <uapi/drm/xe_drm.h>
14
15#include "xe_bo.h"
16#include "xe_device.h"
17#include "xe_eu_stall.h"
18#include "xe_force_wake.h"
19#include "xe_gt_mcr.h"
20#include "xe_gt_printk.h"
21#include "xe_gt_topology.h"
22#include "xe_macros.h"
23#include "xe_observation.h"
24#include "xe_pm.h"
25#include "xe_trace.h"
26#include "xe_wa.h"
27
28#include "regs/xe_eu_stall_regs.h"
29#include "regs/xe_gt_regs.h"
30
31#define POLL_PERIOD_MS 5
32
33static size_t per_xecore_buf_size = SZ_512K;
34
35struct per_xecore_buf {
36 /* Buffer vaddr */
37 u8 *vaddr;
38 /* Write pointer */
39 u32 write;
40 /* Read pointer */
41 u32 read;
42};
43
44struct xe_eu_stall_data_stream {
45 bool pollin;
46 bool enabled;
47 int wait_num_reports;
48 int sampling_rate_mult;
49 wait_queue_head_t poll_wq;
50 size_t data_record_size;
51 size_t per_xecore_buf_size;
52 unsigned int fw_ref;
53
54 struct xe_gt *gt;
55 struct xe_bo *bo;
56 /* Lock to protect data buffer pointers */
57 struct mutex xecore_buf_lock;
58 struct per_xecore_buf *xecore_buf;
59 struct {
60 bool reported_to_user;
61 xe_dss_mask_t mask;
62 } data_drop;
63 struct delayed_work buf_poll_work;
64};
65
66struct xe_eu_stall_gt {
67 /* Lock to protect stream */
68 struct mutex stream_lock;
69 /* EU stall data stream */
70 struct xe_eu_stall_data_stream *stream;
71 /* Workqueue to schedule buffer pointers polling work */
72 struct workqueue_struct *buf_ptr_poll_wq;
73};
74
75/**
76 * struct eu_stall_open_properties - EU stall sampling properties received
77 * from user space at open.
78 * @sampling_rate_mult: EU stall sampling rate multiplier.
79 * HW will sample every (sampling_rate_mult x 251) cycles.
80 * @wait_num_reports: Minimum number of EU stall data reports to unblock poll().
81 * @gt: GT on which EU stall data will be captured.
82 */
83struct eu_stall_open_properties {
84 int sampling_rate_mult;
85 int wait_num_reports;
86 struct xe_gt *gt;
87};
88
89/*
90 * EU stall data format for PVC
91 */
92struct xe_eu_stall_data_pvc {
93 __u64 ip_addr:29; /* Bits 0 to 28 */
94 __u64 active_count:8; /* Bits 29 to 36 */
95 __u64 other_count:8; /* Bits 37 to 44 */
96 __u64 control_count:8; /* Bits 45 to 52 */
97 __u64 pipestall_count:8; /* Bits 53 to 60 */
98 __u64 send_count:8; /* Bits 61 to 68 */
99 __u64 dist_acc_count:8; /* Bits 69 to 76 */
100 __u64 sbid_count:8; /* Bits 77 to 84 */
101 __u64 sync_count:8; /* Bits 85 to 92 */
102 __u64 inst_fetch_count:8; /* Bits 93 to 100 */
103 __u64 unused_bits:27;
104 __u64 unused[6];
105} __packed;
106
107/*
108 * EU stall data format for Xe2 arch GPUs (LNL, BMG).
109 */
110struct xe_eu_stall_data_xe2 {
111 __u64 ip_addr:29; /* Bits 0 to 28 */
112 __u64 tdr_count:8; /* Bits 29 to 36 */
113 __u64 other_count:8; /* Bits 37 to 44 */
114 __u64 control_count:8; /* Bits 45 to 52 */
115 __u64 pipestall_count:8; /* Bits 53 to 60 */
116 __u64 send_count:8; /* Bits 61 to 68 */
117 __u64 dist_acc_count:8; /* Bits 69 to 76 */
118 __u64 sbid_count:8; /* Bits 77 to 84 */
119 __u64 sync_count:8; /* Bits 85 to 92 */
120 __u64 inst_fetch_count:8; /* Bits 93 to 100 */
121 __u64 active_count:8; /* Bits 101 to 108 */
122 __u64 ex_id:3; /* Bits 109 to 111 */
123 __u64 end_flag:1; /* Bit 112 */
124 __u64 unused_bits:15;
125 __u64 unused[6];
126} __packed;
127
128/*
129 * EU stall data format for Xe3p arch GPUs.
130 */
131struct xe_eu_stall_data_xe3p {
132 __u64 ip_addr:61; /* Bits 0 to 60 */
133 __u64 tdr_count:8; /* Bits 61 to 68 */
134 __u64 other_count:8; /* Bits 69 to 76 */
135 __u64 control_count:8; /* Bits 77 to 84 */
136 __u64 pipestall_count:8; /* Bits 85 to 92 */
137 __u64 send_count:8; /* Bits 93 to 100 */
138 __u64 dist_acc_count:8; /* Bits 101 to 108 */
139 __u64 sbid_count:8; /* Bits 109 to 116 */
140 __u64 sync_count:8; /* Bits 117 to 124 */
141 __u64 inst_fetch_count:8; /* Bits 125 to 132 */
142 __u64 active_count:8; /* Bits 133 to 140 */
143 __u64 ex_id:3; /* Bits 141 to 143 */
144 __u64 end_flag:1; /* Bit 144 */
145 __u64 unused_bits:47;
146 __u64 unused[5];
147} __packed;
148
149const u64 eu_stall_sampling_rates[] = {251, 251 * 2, 251 * 3, 251 * 4, 251 * 5, 251 * 6, 251 * 7};
150
151/**
152 * xe_eu_stall_get_sampling_rates - get EU stall sampling rates information.
153 *
154 * @num_rates: Pointer to a u32 to return the number of sampling rates.
155 * @rates: double u64 pointer to point to an array of sampling rates.
156 *
157 * Stores the number of sampling rates and pointer to the array of
158 * sampling rates in the input pointers.
159 *
160 * Returns: Size of the EU stall sampling rates array.
161 */
162size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates)
163{
164 *num_rates = ARRAY_SIZE(eu_stall_sampling_rates);
165 *rates = eu_stall_sampling_rates;
166
167 return sizeof(eu_stall_sampling_rates);
168}
169
170/**
171 * xe_eu_stall_get_per_xecore_buf_size - get per XeCore buffer size.
172 *
173 * Returns: The per XeCore buffer size used to allocate the per GT
174 * EU stall data buffer.
175 */
176size_t xe_eu_stall_get_per_xecore_buf_size(void)
177{
178 return per_xecore_buf_size;
179}
180
181/**
182 * xe_eu_stall_data_record_size - get EU stall data record size.
183 *
184 * @xe: Pointer to a Xe device.
185 *
186 * Returns: EU stall data record size.
187 */
188size_t xe_eu_stall_data_record_size(struct xe_device *xe)
189{
190 size_t record_size = 0;
191
192 if (GRAPHICS_VER(xe) >= 35)
193 record_size = sizeof(struct xe_eu_stall_data_xe3p);
194 else if (GRAPHICS_VER(xe) >= 20)
195 record_size = sizeof(struct xe_eu_stall_data_xe2);
196 else if (xe->info.platform == XE_PVC)
197 record_size = sizeof(struct xe_eu_stall_data_pvc);
198
199
200 xe_assert(xe, is_power_of_2(record_size));
201
202 return record_size;
203}
204
205/**
206 * num_data_rows - Return the number of EU stall data rows of 64B each
207 * for a given data size.
208 *
209 * @data_size: EU stall data size
210 */
211static u32 num_data_rows(u32 data_size)
212{
213 return data_size >> 6;
214}
215
216static void xe_eu_stall_fini(void *arg)
217{
218 struct xe_gt *gt = arg;
219
220 destroy_workqueue(wq: gt->eu_stall->buf_ptr_poll_wq);
221 mutex_destroy(lock: &gt->eu_stall->stream_lock);
222 kfree(objp: gt->eu_stall);
223}
224
225/**
226 * xe_eu_stall_init() - Allocate and initialize GT level EU stall data
227 * structure xe_eu_stall_gt within struct xe_gt.
228 *
229 * @gt: GT being initialized.
230 *
231 * Returns: zero on success or a negative error code.
232 */
233int xe_eu_stall_init(struct xe_gt *gt)
234{
235 struct xe_device *xe = gt_to_xe(gt);
236 int ret;
237
238 if (!xe_eu_stall_supported_on_platform(xe))
239 return 0;
240
241 gt->eu_stall = kzalloc(sizeof(*gt->eu_stall), GFP_KERNEL);
242 if (!gt->eu_stall) {
243 ret = -ENOMEM;
244 goto exit;
245 }
246
247 mutex_init(&gt->eu_stall->stream_lock);
248
249 gt->eu_stall->buf_ptr_poll_wq = alloc_ordered_workqueue("xe_eu_stall", 0);
250 if (!gt->eu_stall->buf_ptr_poll_wq) {
251 ret = -ENOMEM;
252 goto exit_free;
253 }
254
255 return devm_add_action_or_reset(xe->drm.dev, xe_eu_stall_fini, gt);
256exit_free:
257 mutex_destroy(lock: &gt->eu_stall->stream_lock);
258 kfree(objp: gt->eu_stall);
259exit:
260 return ret;
261}
262
263static int set_prop_eu_stall_sampling_rate(struct xe_device *xe, u64 value,
264 struct eu_stall_open_properties *props)
265{
266 value = div_u64(dividend: value, divisor: 251);
267 if (value == 0 || value > 7) {
268 drm_dbg(&xe->drm, "Invalid EU stall sampling rate %llu\n", value);
269 return -EINVAL;
270 }
271 props->sampling_rate_mult = value;
272 return 0;
273}
274
275static int set_prop_eu_stall_wait_num_reports(struct xe_device *xe, u64 value,
276 struct eu_stall_open_properties *props)
277{
278 props->wait_num_reports = value;
279
280 return 0;
281}
282
283static int set_prop_eu_stall_gt_id(struct xe_device *xe, u64 value,
284 struct eu_stall_open_properties *props)
285{
286 struct xe_gt *gt = xe_device_get_gt(xe, gt_id: value);
287
288 if (!gt) {
289 drm_dbg(&xe->drm, "Invalid GT ID %llu for EU stall sampling\n", value);
290 return -EINVAL;
291 }
292 props->gt = gt;
293 return 0;
294}
295
296typedef int (*set_eu_stall_property_fn)(struct xe_device *xe, u64 value,
297 struct eu_stall_open_properties *props);
298
299static const set_eu_stall_property_fn xe_set_eu_stall_property_funcs[] = {
300 [DRM_XE_EU_STALL_PROP_SAMPLE_RATE] = set_prop_eu_stall_sampling_rate,
301 [DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS] = set_prop_eu_stall_wait_num_reports,
302 [DRM_XE_EU_STALL_PROP_GT_ID] = set_prop_eu_stall_gt_id,
303};
304
305static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension,
306 struct eu_stall_open_properties *props)
307{
308 u64 __user *address = u64_to_user_ptr(extension);
309 struct drm_xe_ext_set_property ext;
310 int err;
311 u32 idx;
312
313 err = copy_from_user(to: &ext, from: address, n: sizeof(ext));
314 if (XE_IOCTL_DBG(xe, err))
315 return -EFAULT;
316
317 if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) ||
318 XE_IOCTL_DBG(xe, !ext.property) || XE_IOCTL_DBG(xe, ext.pad))
319 return -EINVAL;
320
321 idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs));
322 return xe_set_eu_stall_property_funcs[idx](xe, ext.value, props);
323}
324
325typedef int (*xe_eu_stall_user_extension_fn)(struct xe_device *xe, u64 extension,
326 struct eu_stall_open_properties *props);
327static const xe_eu_stall_user_extension_fn xe_eu_stall_user_extension_funcs[] = {
328 [DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY] = xe_eu_stall_user_ext_set_property,
329};
330
331#define MAX_USER_EXTENSIONS 5
332static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension,
333 int ext_number, struct eu_stall_open_properties *props)
334{
335 u64 __user *address = u64_to_user_ptr(extension);
336 struct drm_xe_user_extension ext;
337 int err;
338 u32 idx;
339
340 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
341 return -E2BIG;
342
343 err = copy_from_user(to: &ext, from: address, n: sizeof(ext));
344 if (XE_IOCTL_DBG(xe, err))
345 return -EFAULT;
346
347 if (XE_IOCTL_DBG(xe, ext.pad) ||
348 XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(xe_eu_stall_user_extension_funcs)))
349 return -EINVAL;
350
351 idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_eu_stall_user_extension_funcs));
352 err = xe_eu_stall_user_extension_funcs[idx](xe, extension, props);
353 if (XE_IOCTL_DBG(xe, err))
354 return err;
355
356 if (ext.next_extension)
357 return xe_eu_stall_user_extensions(xe, extension: ext.next_extension, ext_number: ++ext_number, props);
358
359 return 0;
360}
361
362/**
363 * buf_data_size - Calculate the number of bytes in a circular buffer
364 * given the read and write pointers and the size of
365 * the buffer.
366 *
367 * @buf_size: Size of the circular buffer
368 * @read_ptr: Read pointer with an additional overflow bit
369 * @write_ptr: Write pointer with an additional overflow bit
370 *
371 * Since the read and write pointers have an additional overflow bit,
372 * this function calculates the offsets from the pointers and use the
373 * offsets to calculate the data size in the buffer.
374 *
375 * Returns: number of bytes of data in the buffer
376 */
377static u32 buf_data_size(size_t buf_size, u32 read_ptr, u32 write_ptr)
378{
379 u32 read_offset, write_offset, size = 0;
380
381 if (read_ptr == write_ptr)
382 goto exit;
383
384 read_offset = read_ptr & (buf_size - 1);
385 write_offset = write_ptr & (buf_size - 1);
386
387 if (write_offset > read_offset)
388 size = write_offset - read_offset;
389 else
390 size = buf_size - read_offset + write_offset;
391exit:
392 return size;
393}
394
395/**
396 * eu_stall_data_buf_poll - Poll for EU stall data in the buffer.
397 *
398 * @stream: xe EU stall data stream instance
399 *
400 * Returns: true if the EU stall buffer contains minimum stall data as
401 * specified by the event report count, else false.
402 */
403static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream)
404{
405 u32 read_ptr, write_ptr_reg, write_ptr, total_data = 0;
406 u32 buf_size = stream->per_xecore_buf_size;
407 struct per_xecore_buf *xecore_buf;
408 struct xe_gt *gt = stream->gt;
409 bool min_data_present = false;
410 u16 group, instance;
411 unsigned int xecore;
412
413 mutex_lock(&stream->xecore_buf_lock);
414 for_each_dss_steering(xecore, gt, group, instance) {
415 xecore_buf = &stream->xecore_buf[xecore];
416 read_ptr = xecore_buf->read;
417 write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT,
418 group, instance);
419 write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg);
420 write_ptr <<= 6;
421 write_ptr &= ((buf_size << 1) - 1);
422 if (!min_data_present) {
423 total_data += buf_data_size(buf_size, read_ptr, write_ptr);
424 if (num_data_rows(data_size: total_data) >= stream->wait_num_reports)
425 min_data_present = true;
426 }
427 if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP)
428 set_bit(nr: xecore, addr: stream->data_drop.mask);
429 xecore_buf->write = write_ptr;
430 }
431 mutex_unlock(lock: &stream->xecore_buf_lock);
432
433 return min_data_present;
434}
435
436static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance)
437{
438 struct xe_device *xe = gt_to_xe(gt);
439 u32 write_ptr_reg;
440
441 /* On PVC, the overflow bit has to be cleared by writing 1 to it.
442 * On Xe2 and later GPUs, the bit has to be cleared by writing 0 to it.
443 */
444 if (GRAPHICS_VER(xe) >= 20)
445 write_ptr_reg = _MASKED_BIT_DISABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
446 else
447 write_ptr_reg = _MASKED_BIT_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
448
449 xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT, value: write_ptr_reg, group, instance);
450}
451
452static int xe_eu_stall_data_buf_read(struct xe_eu_stall_data_stream *stream,
453 char __user *buf, size_t count,
454 size_t *total_data_size, struct xe_gt *gt,
455 u16 group, u16 instance, unsigned int xecore)
456{
457 size_t read_data_size, copy_size, buf_size;
458 u32 read_ptr_reg, read_ptr, write_ptr;
459 u8 *xecore_start_vaddr, *read_vaddr;
460 struct per_xecore_buf *xecore_buf;
461 u32 read_offset, write_offset;
462
463 /* Hardware increments the read and write pointers such that they can
464 * overflow into one additional bit. For example, a 256KB size buffer
465 * offset pointer needs 18 bits. But HW uses 19 bits for the read and
466 * write pointers. This technique avoids wasting a slot in the buffer.
467 * Read and write offsets are calculated from the pointers in order to
468 * check if the write pointer has wrapped around the array.
469 */
470 xecore_buf = &stream->xecore_buf[xecore];
471 xecore_start_vaddr = xecore_buf->vaddr;
472 read_ptr = xecore_buf->read;
473 write_ptr = xecore_buf->write;
474 buf_size = stream->per_xecore_buf_size;
475
476 read_data_size = buf_data_size(buf_size, read_ptr, write_ptr);
477 /* Read only the data that the user space buffer can accommodate */
478 read_data_size = min_t(size_t, count - *total_data_size, read_data_size);
479 if (read_data_size == 0)
480 goto exit_drop;
481
482 read_offset = read_ptr & (buf_size - 1);
483 write_offset = write_ptr & (buf_size - 1);
484 read_vaddr = xecore_start_vaddr + read_offset;
485
486 if (write_offset > read_offset) {
487 if (copy_to_user(to: buf + *total_data_size, from: read_vaddr, n: read_data_size))
488 return -EFAULT;
489 } else {
490 if (read_data_size >= buf_size - read_offset)
491 copy_size = buf_size - read_offset;
492 else
493 copy_size = read_data_size;
494 if (copy_to_user(to: buf + *total_data_size, from: read_vaddr, n: copy_size))
495 return -EFAULT;
496 if (copy_to_user(to: buf + *total_data_size + copy_size,
497 from: xecore_start_vaddr, n: read_data_size - copy_size))
498 return -EFAULT;
499 }
500
501 *total_data_size += read_data_size;
502 read_ptr += read_data_size;
503
504 /* Read pointer can overflow into one additional bit */
505 read_ptr &= (buf_size << 1) - 1;
506 read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, (read_ptr >> 6));
507 read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg);
508 xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, value: read_ptr_reg, group, instance);
509 xecore_buf->read = read_ptr;
510 trace_xe_eu_stall_data_read(slice: group, subslice: instance, read_ptr, write_ptr,
511 read_size: read_data_size, total_size: *total_data_size);
512exit_drop:
513 /* Clear drop bit (if set) after any data was read or if the buffer was empty.
514 * Drop bit can be set even if the buffer is empty as the buffer may have been emptied
515 * in the previous read() and the data drop bit was set during the previous read().
516 */
517 if (test_bit(xecore, stream->data_drop.mask)) {
518 clear_dropped_eviction_line_bit(gt, group, instance);
519 clear_bit(nr: xecore, addr: stream->data_drop.mask);
520 }
521 return 0;
522}
523
524/**
525 * xe_eu_stall_stream_read_locked - copy EU stall counters data from the
526 * per xecore buffers to the userspace buffer
527 * @stream: A stream opened for EU stall count metrics
528 * @file: An xe EU stall data stream file
529 * @buf: destination buffer given by userspace
530 * @count: the number of bytes userspace wants to read
531 *
532 * Returns: Number of bytes copied or a negative error code
533 * If we've successfully copied any data then reporting that takes
534 * precedence over any internal error status, so the data isn't lost.
535 */
536static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *stream,
537 struct file *file, char __user *buf,
538 size_t count)
539{
540 struct xe_gt *gt = stream->gt;
541 size_t total_size = 0;
542 u16 group, instance;
543 unsigned int xecore;
544 int ret = 0;
545
546 mutex_lock(&stream->xecore_buf_lock);
547 if (bitmap_weight(src: stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) {
548 if (!stream->data_drop.reported_to_user) {
549 stream->data_drop.reported_to_user = true;
550 xe_gt_dbg(gt, "EU stall data dropped in XeCores: %*pb\n",
551 XE_MAX_DSS_FUSE_BITS, stream->data_drop.mask);
552 mutex_unlock(lock: &stream->xecore_buf_lock);
553 return -EIO;
554 }
555 stream->data_drop.reported_to_user = false;
556 }
557
558 for_each_dss_steering(xecore, gt, group, instance) {
559 ret = xe_eu_stall_data_buf_read(stream, buf, count, total_data_size: &total_size,
560 gt, group, instance, xecore);
561 if (ret || count == total_size)
562 break;
563 }
564 mutex_unlock(lock: &stream->xecore_buf_lock);
565 return total_size ?: (ret ?: -EAGAIN);
566}
567
568/*
569 * Userspace must enable the EU stall stream with DRM_XE_OBSERVATION_IOCTL_ENABLE
570 * before calling read().
571 *
572 * Returns: The number of bytes copied or a negative error code on failure.
573 * -EIO if HW drops any EU stall data when the buffer is full.
574 */
575static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf,
576 size_t count, loff_t *ppos)
577{
578 struct xe_eu_stall_data_stream *stream = file->private_data;
579 struct xe_gt *gt = stream->gt;
580 ssize_t ret, aligned_count;
581
582 aligned_count = ALIGN_DOWN(count, stream->data_record_size);
583 if (aligned_count == 0)
584 return -EINVAL;
585
586 if (!stream->enabled) {
587 xe_gt_dbg(gt, "EU stall data stream not enabled to read\n");
588 return -EINVAL;
589 }
590
591 if (!(file->f_flags & O_NONBLOCK)) {
592 do {
593 ret = wait_event_interruptible(stream->poll_wq, stream->pollin);
594 if (ret)
595 return -EINTR;
596
597 mutex_lock(&gt->eu_stall->stream_lock);
598 ret = xe_eu_stall_stream_read_locked(stream, file, buf, count: aligned_count);
599 mutex_unlock(lock: &gt->eu_stall->stream_lock);
600 } while (ret == -EAGAIN);
601 } else {
602 mutex_lock(&gt->eu_stall->stream_lock);
603 ret = xe_eu_stall_stream_read_locked(stream, file, buf, count: aligned_count);
604 mutex_unlock(lock: &gt->eu_stall->stream_lock);
605 }
606
607 /*
608 * This may not work correctly if the user buffer is very small.
609 * We don't want to block the next read() when there is data in the buffer
610 * now, but couldn't be accommodated in the small user buffer.
611 */
612 stream->pollin = false;
613
614 return ret;
615}
616
617static void xe_eu_stall_stream_free(struct xe_eu_stall_data_stream *stream)
618{
619 struct xe_gt *gt = stream->gt;
620
621 mutex_destroy(lock: &stream->xecore_buf_lock);
622 gt->eu_stall->stream = NULL;
623 kfree(objp: stream);
624}
625
626static void xe_eu_stall_data_buf_destroy(struct xe_eu_stall_data_stream *stream)
627{
628 xe_bo_unpin_map_no_vm(bo: stream->bo);
629 kfree(objp: stream->xecore_buf);
630}
631
632static int xe_eu_stall_data_buf_alloc(struct xe_eu_stall_data_stream *stream,
633 u16 last_xecore)
634{
635 struct xe_tile *tile = stream->gt->tile;
636 struct xe_bo *bo;
637 u32 size;
638
639 stream->xecore_buf = kcalloc(last_xecore, sizeof(*stream->xecore_buf), GFP_KERNEL);
640 if (!stream->xecore_buf)
641 return -ENOMEM;
642
643 size = stream->per_xecore_buf_size * last_xecore;
644
645 bo = xe_bo_create_pin_map_at_novm(xe: tile->xe, tile, size, offset: ~0ull, type: ttm_bo_type_kernel,
646 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, SZ_64, intr: false);
647 if (IS_ERR(ptr: bo)) {
648 kfree(objp: stream->xecore_buf);
649 return PTR_ERR(ptr: bo);
650 }
651
652 XE_WARN_ON(!IS_ALIGNED(xe_bo_ggtt_addr(bo), SZ_64));
653 stream->bo = bo;
654
655 return 0;
656}
657
658static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream)
659{
660 u32 write_ptr_reg, write_ptr, read_ptr_reg, reg_value;
661 struct per_xecore_buf *xecore_buf;
662 struct xe_gt *gt = stream->gt;
663 u16 group, instance;
664 int xecore;
665
666 /* Take runtime pm ref and forcewake to disable RC6 */
667 xe_pm_runtime_get(gt_to_xe(gt));
668 stream->fw_ref = xe_force_wake_get(fw: gt_to_fw(gt), domains: XE_FW_RENDER);
669 if (!xe_force_wake_ref_has_domain(fw_ref: stream->fw_ref, domain: XE_FW_RENDER)) {
670 xe_gt_err(gt, "Failed to get RENDER forcewake\n");
671 xe_pm_runtime_put(gt_to_xe(gt));
672 return -ETIMEDOUT;
673 }
674
675 if (XE_GT_WA(gt, 22016596838))
676 xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2,
677 _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
678
679 for_each_dss_steering(xecore, gt, group, instance) {
680 write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT, group, instance);
681 /* Clear any drop bits set and not cleared in the previous session. */
682 if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP)
683 clear_dropped_eviction_line_bit(gt, group, instance);
684 write_ptr = REG_FIELD_GET(XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK, write_ptr_reg);
685 read_ptr_reg = REG_FIELD_PREP(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, write_ptr);
686 read_ptr_reg = _MASKED_FIELD(XEHPC_EUSTALL_REPORT1_READ_PTR_MASK, read_ptr_reg);
687 /* Initialize the read pointer to the write pointer */
688 xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, value: read_ptr_reg, group, instance);
689 write_ptr <<= 6;
690 write_ptr &= (stream->per_xecore_buf_size << 1) - 1;
691 xecore_buf = &stream->xecore_buf[xecore];
692 xecore_buf->write = write_ptr;
693 xecore_buf->read = write_ptr;
694 }
695 stream->data_drop.reported_to_user = false;
696 bitmap_zero(dst: stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS);
697
698 reg_value = _MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE,
699 REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) |
700 REG_FIELD_PREP(EUSTALL_SAMPLE_RATE,
701 stream->sampling_rate_mult));
702 xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_CTRL, value: reg_value);
703 /* GGTT addresses can never be > 32 bits */
704 xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE_UPPER, value: 0);
705 reg_value = xe_bo_ggtt_addr(bo: stream->bo);
706 reg_value |= REG_FIELD_PREP(XEHPC_EUSTALL_BASE_XECORE_BUF_SZ,
707 stream->per_xecore_buf_size / SZ_256K);
708 reg_value |= XEHPC_EUSTALL_BASE_ENABLE_SAMPLING;
709 xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, value: reg_value);
710
711 return 0;
712}
713
714static void eu_stall_data_buf_poll_work_fn(struct work_struct *work)
715{
716 struct xe_eu_stall_data_stream *stream =
717 container_of(work, typeof(*stream), buf_poll_work.work);
718 struct xe_gt *gt = stream->gt;
719
720 if (eu_stall_data_buf_poll(stream)) {
721 stream->pollin = true;
722 wake_up(&stream->poll_wq);
723 }
724 queue_delayed_work(wq: gt->eu_stall->buf_ptr_poll_wq,
725 dwork: &stream->buf_poll_work,
726 delay: msecs_to_jiffies(POLL_PERIOD_MS));
727}
728
729static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream,
730 struct eu_stall_open_properties *props)
731{
732 unsigned int max_wait_num_reports, xecore, last_xecore, num_xecores;
733 struct per_xecore_buf *xecore_buf;
734 struct xe_gt *gt = stream->gt;
735 xe_dss_mask_t all_xecores;
736 u16 group, instance;
737 u32 vaddr_offset;
738 int ret;
739
740 bitmap_or(dst: all_xecores, src1: gt->fuse_topo.g_dss_mask, src2: gt->fuse_topo.c_dss_mask,
741 XE_MAX_DSS_FUSE_BITS);
742 num_xecores = bitmap_weight(src: all_xecores, XE_MAX_DSS_FUSE_BITS);
743 last_xecore = xe_gt_topology_mask_last_dss(mask: all_xecores) + 1;
744
745 max_wait_num_reports = num_data_rows(data_size: per_xecore_buf_size * num_xecores);
746 if (props->wait_num_reports == 0 || props->wait_num_reports > max_wait_num_reports) {
747 xe_gt_dbg(gt, "Invalid EU stall event report count %u\n",
748 props->wait_num_reports);
749 xe_gt_dbg(gt, "Minimum event report count is 1, maximum is %u\n",
750 max_wait_num_reports);
751 return -EINVAL;
752 }
753
754 init_waitqueue_head(&stream->poll_wq);
755 mutex_init(&stream->xecore_buf_lock);
756 INIT_DELAYED_WORK(&stream->buf_poll_work, eu_stall_data_buf_poll_work_fn);
757 stream->per_xecore_buf_size = per_xecore_buf_size;
758 stream->sampling_rate_mult = props->sampling_rate_mult;
759 stream->wait_num_reports = props->wait_num_reports;
760 stream->data_record_size = xe_eu_stall_data_record_size(gt_to_xe(gt));
761
762 ret = xe_eu_stall_data_buf_alloc(stream, last_xecore);
763 if (ret)
764 return ret;
765
766 for_each_dss_steering(xecore, gt, group, instance) {
767 xecore_buf = &stream->xecore_buf[xecore];
768 vaddr_offset = xecore * stream->per_xecore_buf_size;
769 xecore_buf->vaddr = stream->bo->vmap.vaddr + vaddr_offset;
770 }
771 return 0;
772}
773
774static __poll_t xe_eu_stall_stream_poll_locked(struct xe_eu_stall_data_stream *stream,
775 struct file *file, poll_table *wait)
776{
777 __poll_t events = 0;
778
779 poll_wait(filp: file, wait_address: &stream->poll_wq, p: wait);
780
781 if (stream->pollin)
782 events |= EPOLLIN;
783
784 return events;
785}
786
787static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait)
788{
789 struct xe_eu_stall_data_stream *stream = file->private_data;
790 struct xe_gt *gt = stream->gt;
791 __poll_t ret;
792
793 mutex_lock(&gt->eu_stall->stream_lock);
794 ret = xe_eu_stall_stream_poll_locked(stream, file, wait);
795 mutex_unlock(lock: &gt->eu_stall->stream_lock);
796
797 return ret;
798}
799
800static int xe_eu_stall_enable_locked(struct xe_eu_stall_data_stream *stream)
801{
802 struct xe_gt *gt = stream->gt;
803 int ret = 0;
804
805 if (stream->enabled)
806 return ret;
807
808 stream->enabled = true;
809
810 ret = xe_eu_stall_stream_enable(stream);
811
812 queue_delayed_work(wq: gt->eu_stall->buf_ptr_poll_wq,
813 dwork: &stream->buf_poll_work,
814 delay: msecs_to_jiffies(POLL_PERIOD_MS));
815 return ret;
816}
817
818static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream)
819{
820 struct xe_gt *gt = stream->gt;
821
822 if (!stream->enabled)
823 return 0;
824
825 stream->enabled = false;
826
827 xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, value: 0);
828
829 cancel_delayed_work_sync(dwork: &stream->buf_poll_work);
830
831 if (XE_GT_WA(gt, 22016596838))
832 xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2,
833 _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
834
835 xe_force_wake_put(fw: gt_to_fw(gt), fw_ref: stream->fw_ref);
836 xe_pm_runtime_put(gt_to_xe(gt));
837
838 return 0;
839}
840
841static long xe_eu_stall_stream_ioctl_locked(struct xe_eu_stall_data_stream *stream,
842 unsigned int cmd, unsigned long arg)
843{
844 switch (cmd) {
845 case DRM_XE_OBSERVATION_IOCTL_ENABLE:
846 return xe_eu_stall_enable_locked(stream);
847 case DRM_XE_OBSERVATION_IOCTL_DISABLE:
848 return xe_eu_stall_disable_locked(stream);
849 }
850
851 return -EINVAL;
852}
853
854static long xe_eu_stall_stream_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
855{
856 struct xe_eu_stall_data_stream *stream = file->private_data;
857 struct xe_gt *gt = stream->gt;
858 long ret;
859
860 mutex_lock(&gt->eu_stall->stream_lock);
861 ret = xe_eu_stall_stream_ioctl_locked(stream, cmd, arg);
862 mutex_unlock(lock: &gt->eu_stall->stream_lock);
863
864 return ret;
865}
866
867static int xe_eu_stall_stream_close(struct inode *inode, struct file *file)
868{
869 struct xe_eu_stall_data_stream *stream = file->private_data;
870 struct xe_gt *gt = stream->gt;
871
872 drm_dev_put(dev: &gt->tile->xe->drm);
873
874 mutex_lock(&gt->eu_stall->stream_lock);
875 xe_eu_stall_disable_locked(stream);
876 xe_eu_stall_data_buf_destroy(stream);
877 xe_eu_stall_stream_free(stream);
878 mutex_unlock(lock: &gt->eu_stall->stream_lock);
879
880 return 0;
881}
882
883static const struct file_operations fops_eu_stall = {
884 .owner = THIS_MODULE,
885 .llseek = noop_llseek,
886 .release = xe_eu_stall_stream_close,
887 .poll = xe_eu_stall_stream_poll,
888 .read = xe_eu_stall_stream_read,
889 .unlocked_ioctl = xe_eu_stall_stream_ioctl,
890 .compat_ioctl = xe_eu_stall_stream_ioctl,
891};
892
893static int xe_eu_stall_stream_open_locked(struct drm_device *dev,
894 struct eu_stall_open_properties *props,
895 struct drm_file *file)
896{
897 struct xe_eu_stall_data_stream *stream;
898 struct xe_gt *gt = props->gt;
899 unsigned long f_flags = 0;
900 int ret, stream_fd;
901
902 /* Only one session can be active at any time */
903 if (gt->eu_stall->stream) {
904 xe_gt_dbg(gt, "EU stall sampling session already active\n");
905 return -EBUSY;
906 }
907
908 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
909 if (!stream)
910 return -ENOMEM;
911
912 gt->eu_stall->stream = stream;
913 stream->gt = gt;
914
915 ret = xe_eu_stall_stream_init(stream, props);
916 if (ret) {
917 xe_gt_dbg(gt, "EU stall stream init failed : %d\n", ret);
918 goto err_free;
919 }
920
921 stream_fd = anon_inode_getfd(name: "[xe_eu_stall]", fops: &fops_eu_stall, priv: stream, flags: f_flags);
922 if (stream_fd < 0) {
923 ret = stream_fd;
924 xe_gt_dbg(gt, "EU stall inode get fd failed : %d\n", ret);
925 goto err_destroy;
926 }
927
928 /* Take a reference on the driver that will be kept with stream_fd
929 * until its release.
930 */
931 drm_dev_get(dev: &gt->tile->xe->drm);
932
933 return stream_fd;
934
935err_destroy:
936 xe_eu_stall_data_buf_destroy(stream);
937err_free:
938 xe_eu_stall_stream_free(stream);
939 return ret;
940}
941
942/**
943 * xe_eu_stall_stream_open - Open a xe EU stall data stream fd
944 *
945 * @dev: DRM device pointer
946 * @data: pointer to first struct @drm_xe_ext_set_property in
947 * the chain of input properties from the user space.
948 * @file: DRM file pointer
949 *
950 * This function opens a EU stall data stream with input properties from
951 * the user space.
952 *
953 * Returns: EU stall data stream fd on success or a negative error code.
954 */
955int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file)
956{
957 struct xe_device *xe = to_xe_device(dev);
958 struct eu_stall_open_properties props = {};
959 int ret;
960
961 if (!xe_eu_stall_supported_on_platform(xe)) {
962 drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n");
963 return -ENODEV;
964 }
965
966 if (xe_observation_paranoid && !perfmon_capable()) {
967 drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n");
968 return -EACCES;
969 }
970
971 /* Initialize and set default values */
972 props.wait_num_reports = 1;
973 props.sampling_rate_mult = 4;
974
975 ret = xe_eu_stall_user_extensions(xe, extension: data, ext_number: 0, props: &props);
976 if (ret)
977 return ret;
978
979 if (!props.gt) {
980 drm_dbg(&xe->drm, "GT ID not provided for EU stall sampling\n");
981 return -EINVAL;
982 }
983
984 mutex_lock(&props.gt->eu_stall->stream_lock);
985 ret = xe_eu_stall_stream_open_locked(dev, props: &props, file);
986 mutex_unlock(lock: &props.gt->eu_stall->stream_lock);
987
988 return ret;
989}
990

source code of linux/drivers/gpu/drm/xe/xe_eu_stall.c