1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_device.h"
7
8#include <linux/aperture.h>
9#include <linux/delay.h>
10#include <linux/fault-inject.h>
11#include <linux/iopoll.h>
12#include <linux/units.h>
13
14#include <drm/drm_atomic_helper.h>
15#include <drm/drm_client.h>
16#include <drm/drm_gem_ttm_helper.h>
17#include <drm/drm_ioctl.h>
18#include <drm/drm_managed.h>
19#include <drm/drm_print.h>
20#include <uapi/drm/xe_drm.h>
21
22#include "display/xe_display.h"
23#include "instructions/xe_gpu_commands.h"
24#include "regs/xe_gt_regs.h"
25#include "regs/xe_regs.h"
26#include "xe_bo.h"
27#include "xe_bo_evict.h"
28#include "xe_debugfs.h"
29#include "xe_devcoredump.h"
30#include "xe_device_sysfs.h"
31#include "xe_dma_buf.h"
32#include "xe_drm_client.h"
33#include "xe_drv.h"
34#include "xe_exec.h"
35#include "xe_exec_queue.h"
36#include "xe_force_wake.h"
37#include "xe_ggtt.h"
38#include "xe_gsc_proxy.h"
39#include "xe_gt.h"
40#include "xe_gt_mcr.h"
41#include "xe_gt_printk.h"
42#include "xe_gt_sriov_vf.h"
43#include "xe_guc.h"
44#include "xe_guc_pc.h"
45#include "xe_hw_engine_group.h"
46#include "xe_hwmon.h"
47#include "xe_i2c.h"
48#include "xe_irq.h"
49#include "xe_late_bind_fw.h"
50#include "xe_mmio.h"
51#include "xe_module.h"
52#include "xe_nvm.h"
53#include "xe_oa.h"
54#include "xe_observation.h"
55#include "xe_pagefault.h"
56#include "xe_pat.h"
57#include "xe_pcode.h"
58#include "xe_pm.h"
59#include "xe_pmu.h"
60#include "xe_psmi.h"
61#include "xe_pxp.h"
62#include "xe_query.h"
63#include "xe_shrinker.h"
64#include "xe_survivability_mode.h"
65#include "xe_sriov.h"
66#include "xe_tile.h"
67#include "xe_ttm_stolen_mgr.h"
68#include "xe_ttm_sys_mgr.h"
69#include "xe_vm.h"
70#include "xe_vm_madvise.h"
71#include "xe_vram.h"
72#include "xe_vram_types.h"
73#include "xe_vsec.h"
74#include "xe_wait_user_fence.h"
75#include "xe_wa.h"
76
77#include <generated/xe_device_wa_oob.h>
78#include <generated/xe_wa_oob.h>
79
80static int xe_file_open(struct drm_device *dev, struct drm_file *file)
81{
82 struct xe_device *xe = to_xe_device(dev);
83 struct xe_drm_client *client;
84 struct xe_file *xef;
85 int ret = -ENOMEM;
86 struct task_struct *task = NULL;
87
88 xef = kzalloc(sizeof(*xef), GFP_KERNEL);
89 if (!xef)
90 return ret;
91
92 client = xe_drm_client_alloc();
93 if (!client) {
94 kfree(objp: xef);
95 return ret;
96 }
97
98 xef->drm = file;
99 xef->client = client;
100 xef->xe = xe;
101
102 mutex_init(&xef->vm.lock);
103 xa_init_flags(xa: &xef->vm.xa, XA_FLAGS_ALLOC1);
104
105 mutex_init(&xef->exec_queue.lock);
106 xa_init_flags(xa: &xef->exec_queue.xa, XA_FLAGS_ALLOC1);
107
108 file->driver_priv = xef;
109 kref_init(kref: &xef->refcount);
110
111 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
112 if (task) {
113 xef->process_name = kstrdup(s: task->comm, GFP_KERNEL);
114 xef->pid = task->pid;
115 put_task_struct(t: task);
116 }
117
118 return 0;
119}
120
121static void xe_file_destroy(struct kref *ref)
122{
123 struct xe_file *xef = container_of(ref, struct xe_file, refcount);
124
125 xa_destroy(&xef->exec_queue.xa);
126 mutex_destroy(lock: &xef->exec_queue.lock);
127 xa_destroy(&xef->vm.xa);
128 mutex_destroy(lock: &xef->vm.lock);
129
130 xe_drm_client_put(client: xef->client);
131 kfree(objp: xef->process_name);
132 kfree(objp: xef);
133}
134
135/**
136 * xe_file_get() - Take a reference to the xe file object
137 * @xef: Pointer to the xe file
138 *
139 * Anyone with a pointer to xef must take a reference to the xe file
140 * object using this call.
141 *
142 * Return: xe file pointer
143 */
144struct xe_file *xe_file_get(struct xe_file *xef)
145{
146 kref_get(kref: &xef->refcount);
147 return xef;
148}
149
150/**
151 * xe_file_put() - Drop a reference to the xe file object
152 * @xef: Pointer to the xe file
153 *
154 * Used to drop reference to the xef object
155 */
156void xe_file_put(struct xe_file *xef)
157{
158 kref_put(kref: &xef->refcount, release: xe_file_destroy);
159}
160
161static void xe_file_close(struct drm_device *dev, struct drm_file *file)
162{
163 struct xe_device *xe = to_xe_device(dev);
164 struct xe_file *xef = file->driver_priv;
165 struct xe_vm *vm;
166 struct xe_exec_queue *q;
167 unsigned long idx;
168
169 xe_pm_runtime_get(xe);
170
171 /*
172 * No need for exec_queue.lock here as there is no contention for it
173 * when FD is closing as IOCTLs presumably can't be modifying the
174 * xarray. Taking exec_queue.lock here causes undue dependency on
175 * vm->lock taken during xe_exec_queue_kill().
176 */
177 xa_for_each(&xef->exec_queue.xa, idx, q) {
178 if (q->vm && q->hwe->hw_engine_group)
179 xe_hw_engine_group_del_exec_queue(group: q->hwe->hw_engine_group, q);
180 xe_exec_queue_kill(q);
181 xe_exec_queue_put(q);
182 }
183 xa_for_each(&xef->vm.xa, idx, vm)
184 xe_vm_close_and_put(vm);
185
186 xe_file_put(xef);
187
188 xe_pm_runtime_put(xe);
189}
190
191static const struct drm_ioctl_desc xe_ioctls[] = {
192 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
193 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
194 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
195 DRM_RENDER_ALLOW),
196 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
197 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
198 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
199 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
200 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
201 DRM_RENDER_ALLOW),
202 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
203 DRM_RENDER_ALLOW),
204 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
205 DRM_RENDER_ALLOW),
206 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
207 DRM_RENDER_ALLOW),
208 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
209 DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
210 DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
211 DRM_RENDER_ALLOW),
212};
213
214static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
215{
216 struct drm_file *file_priv = file->private_data;
217 struct xe_device *xe = to_xe_device(dev: file_priv->minor->dev);
218 long ret;
219
220 if (xe_device_wedged(xe))
221 return -ECANCELED;
222
223 ret = xe_pm_runtime_get_ioctl(xe);
224 if (ret >= 0)
225 ret = drm_ioctl(filp: file, cmd, arg);
226 xe_pm_runtime_put(xe);
227
228 return ret;
229}
230
231#ifdef CONFIG_COMPAT
232static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
233{
234 struct drm_file *file_priv = file->private_data;
235 struct xe_device *xe = to_xe_device(dev: file_priv->minor->dev);
236 long ret;
237
238 if (xe_device_wedged(xe))
239 return -ECANCELED;
240
241 ret = xe_pm_runtime_get_ioctl(xe);
242 if (ret >= 0)
243 ret = drm_compat_ioctl(filp: file, cmd, arg);
244 xe_pm_runtime_put(xe);
245
246 return ret;
247}
248#else
249/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
250#define xe_drm_compat_ioctl NULL
251#endif
252
253static void barrier_open(struct vm_area_struct *vma)
254{
255 drm_dev_get(dev: vma->vm_private_data);
256}
257
258static void barrier_close(struct vm_area_struct *vma)
259{
260 drm_dev_put(dev: vma->vm_private_data);
261}
262
263static void barrier_release_dummy_page(struct drm_device *dev, void *res)
264{
265 struct page *dummy_page = (struct page *)res;
266
267 __free_page(dummy_page);
268}
269
270static vm_fault_t barrier_fault(struct vm_fault *vmf)
271{
272 struct drm_device *dev = vmf->vma->vm_private_data;
273 struct vm_area_struct *vma = vmf->vma;
274 vm_fault_t ret = VM_FAULT_NOPAGE;
275 pgprot_t prot;
276 int idx;
277
278 prot = vm_get_page_prot(vm_flags: vma->vm_flags);
279
280 if (drm_dev_enter(dev, idx: &idx)) {
281 unsigned long pfn;
282
283#define LAST_DB_PAGE_OFFSET 0x7ff001
284 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
285 LAST_DB_PAGE_OFFSET);
286 ret = vmf_insert_pfn_prot(vma, addr: vma->vm_start, pfn,
287 pgprot_noncached(prot));
288 drm_dev_exit(idx);
289 } else {
290 struct page *page;
291
292 /* Allocate new dummy page to map all the VA range in this VMA to it*/
293 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
294 if (!page)
295 return VM_FAULT_OOM;
296
297 /* Set the page to be freed using drmm release action */
298 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
299 return VM_FAULT_OOM;
300
301 ret = vmf_insert_pfn_prot(vma, addr: vma->vm_start, page_to_pfn(page),
302 pgprot: prot);
303 }
304
305 return ret;
306}
307
308static const struct vm_operations_struct vm_ops_barrier = {
309 .open = barrier_open,
310 .close = barrier_close,
311 .fault = barrier_fault,
312};
313
314static int xe_pci_barrier_mmap(struct file *filp,
315 struct vm_area_struct *vma)
316{
317 struct drm_file *priv = filp->private_data;
318 struct drm_device *dev = priv->minor->dev;
319 struct xe_device *xe = to_xe_device(dev);
320
321 if (!IS_DGFX(xe))
322 return -EINVAL;
323
324 if (vma->vm_end - vma->vm_start > SZ_4K)
325 return -EINVAL;
326
327 if (is_cow_mapping(flags: vma->vm_flags))
328 return -EINVAL;
329
330 if (vma->vm_flags & (VM_READ | VM_EXEC))
331 return -EINVAL;
332
333 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
334 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
335 vma->vm_ops = &vm_ops_barrier;
336 vma->vm_private_data = dev;
337 drm_dev_get(dev: vma->vm_private_data);
338
339 return 0;
340}
341
342static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
343{
344 struct drm_file *priv = filp->private_data;
345 struct drm_device *dev = priv->minor->dev;
346
347 if (drm_dev_is_unplugged(dev))
348 return -ENODEV;
349
350 switch (vma->vm_pgoff) {
351 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
352 return xe_pci_barrier_mmap(filp, vma);
353 }
354
355 return drm_gem_mmap(filp, vma);
356}
357
358static const struct file_operations xe_driver_fops = {
359 .owner = THIS_MODULE,
360 .open = drm_open,
361 .release = drm_release_noglobal,
362 .unlocked_ioctl = xe_drm_ioctl,
363 .mmap = xe_mmap,
364 .poll = drm_poll,
365 .read = drm_read,
366 .compat_ioctl = xe_drm_compat_ioctl,
367 .llseek = noop_llseek,
368#ifdef CONFIG_PROC_FS
369 .show_fdinfo = drm_show_fdinfo,
370#endif
371 .fop_flags = FOP_UNSIGNED_OFFSET,
372};
373
374static struct drm_driver driver = {
375 /* Don't use MTRRs here; the Xserver or userspace app should
376 * deal with them for Intel hardware.
377 */
378 .driver_features =
379 DRIVER_GEM |
380 DRIVER_RENDER | DRIVER_SYNCOBJ |
381 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
382 .open = xe_file_open,
383 .postclose = xe_file_close,
384
385 .gem_prime_import = xe_gem_prime_import,
386
387 .dumb_create = xe_bo_dumb_create,
388 .dumb_map_offset = drm_gem_ttm_dumb_map_offset,
389#ifdef CONFIG_PROC_FS
390 .show_fdinfo = xe_drm_client_fdinfo,
391#endif
392 .ioctls = xe_ioctls,
393 .num_ioctls = ARRAY_SIZE(xe_ioctls),
394 .fops = &xe_driver_fops,
395 .name = DRIVER_NAME,
396 .desc = DRIVER_DESC,
397 .major = DRIVER_MAJOR,
398 .minor = DRIVER_MINOR,
399 .patchlevel = DRIVER_PATCHLEVEL,
400};
401
402static void xe_device_destroy(struct drm_device *dev, void *dummy)
403{
404 struct xe_device *xe = to_xe_device(dev);
405
406 xe_bo_dev_fini(bo_device: &xe->bo_device);
407
408 if (xe->preempt_fence_wq)
409 destroy_workqueue(wq: xe->preempt_fence_wq);
410
411 if (xe->ordered_wq)
412 destroy_workqueue(wq: xe->ordered_wq);
413
414 if (xe->unordered_wq)
415 destroy_workqueue(wq: xe->unordered_wq);
416
417 if (xe->destroy_wq)
418 destroy_workqueue(wq: xe->destroy_wq);
419
420 ttm_device_fini(bdev: &xe->ttm);
421}
422
423struct xe_device *xe_device_create(struct pci_dev *pdev,
424 const struct pci_device_id *ent)
425{
426 struct xe_device *xe;
427 int err;
428
429 xe_display_driver_set_hooks(driver: &driver);
430
431 err = aperture_remove_conflicting_pci_devices(pdev, name: driver.name);
432 if (err)
433 return ERR_PTR(error: err);
434
435 xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
436 if (IS_ERR(ptr: xe))
437 return xe;
438
439 err = ttm_device_init(bdev: &xe->ttm, funcs: &xe_ttm_funcs, dev: xe->drm.dev,
440 mapping: xe->drm.anon_inode->i_mapping,
441 vma_manager: xe->drm.vma_offset_manager, alloc_flags: 0);
442 if (WARN_ON(err))
443 goto err;
444
445 xe_bo_dev_init(bo_device: &xe->bo_device);
446 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
447 if (err)
448 goto err;
449
450 err = xe_shrinker_create(xe);
451 if (err)
452 goto err;
453
454 xe->info.devid = pdev->device;
455 xe->info.revid = pdev->revision;
456 xe->info.force_execlist = xe_modparam.force_execlist;
457 xe->atomic_svm_timeslice_ms = 5;
458
459 err = xe_irq_init(xe);
460 if (err)
461 goto err;
462
463 xe_validation_device_init(val: &xe->val);
464
465 init_waitqueue_head(&xe->ufence_wq);
466
467 init_rwsem(&xe->usm.lock);
468
469 xa_init_flags(xa: &xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
470
471 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
472 /* Trigger a large asid and an early asid wrap. */
473 u32 asid;
474
475 BUILD_BUG_ON(XE_MAX_ASID < 2);
476 err = xa_alloc_cyclic(xa: &xe->usm.asid_to_vm, id: &asid, NULL,
477 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
478 next: &xe->usm.next_asid, GFP_KERNEL);
479 drm_WARN_ON(&xe->drm, err);
480 if (err >= 0)
481 xa_erase(&xe->usm.asid_to_vm, index: asid);
482 }
483
484 err = xe_bo_pinned_init(xe);
485 if (err)
486 goto err;
487
488 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
489 WQ_MEM_RECLAIM);
490 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
491 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
492 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);
493 if (!xe->ordered_wq || !xe->unordered_wq ||
494 !xe->preempt_fence_wq || !xe->destroy_wq) {
495 /*
496 * Cleanup done in xe_device_destroy via
497 * drmm_add_action_or_reset register above
498 */
499 drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
500 err = -ENOMEM;
501 goto err;
502 }
503
504 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
505 if (err)
506 goto err;
507
508 return xe;
509
510err:
511 return ERR_PTR(error: err);
512}
513ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
514
515static bool xe_driver_flr_disabled(struct xe_device *xe)
516{
517 if (IS_SRIOV_VF(xe))
518 return true;
519
520 if (xe_mmio_read32(mmio: xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
521 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
522 return true;
523 }
524
525 return false;
526}
527
528/*
529 * The driver-initiated FLR is the highest level of reset that we can trigger
530 * from within the driver. It is different from the PCI FLR in that it doesn't
531 * fully reset the SGUnit and doesn't modify the PCI config space and therefore
532 * it doesn't require a re-enumeration of the PCI BARs. However, the
533 * driver-initiated FLR does still cause a reset of both GT and display and a
534 * memory wipe of local and stolen memory, so recovery would require a full HW
535 * re-init and saving/restoring (or re-populating) the wiped memory. Since we
536 * perform the FLR as the very last action before releasing access to the HW
537 * during the driver release flow, we don't attempt recovery at all, because
538 * if/when a new instance of Xe is bound to the device it will do a full
539 * re-init anyway.
540 */
541static void __xe_driver_flr(struct xe_device *xe)
542{
543 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
544 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
545 int ret;
546
547 drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
548
549 /*
550 * Make sure any pending FLR requests have cleared by waiting for the
551 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
552 * to make sure it's not still set from a prior attempt (it's a write to
553 * clear bit).
554 * Note that we should never be in a situation where a previous attempt
555 * is still pending (unless the HW is totally dead), but better to be
556 * safe in case something unexpected happens
557 */
558 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, val: 0, timeout_us: flr_timeout, NULL, atomic: false);
559 if (ret) {
560 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
561 return;
562 }
563 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
564
565 /* Trigger the actual Driver-FLR */
566 xe_mmio_rmw32(mmio, GU_CNTL, clr: 0, DRIVERFLR);
567
568 /* Wait for hardware teardown to complete */
569 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, val: 0, timeout_us: flr_timeout, NULL, atomic: false);
570 if (ret) {
571 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
572 return;
573 }
574
575 /* Wait for hardware/firmware re-init to complete */
576 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
577 timeout_us: flr_timeout, NULL, atomic: false);
578 if (ret) {
579 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
580 return;
581 }
582
583 /* Clear sticky completion status */
584 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
585}
586
587static void xe_driver_flr(struct xe_device *xe)
588{
589 if (xe_driver_flr_disabled(xe))
590 return;
591
592 __xe_driver_flr(xe);
593}
594
595static void xe_driver_flr_fini(void *arg)
596{
597 struct xe_device *xe = arg;
598
599 if (xe->needs_flr_on_fini)
600 xe_driver_flr(xe);
601}
602
603static void xe_device_sanitize(void *arg)
604{
605 struct xe_device *xe = arg;
606 struct xe_gt *gt;
607 u8 id;
608
609 for_each_gt(gt, xe, id)
610 xe_gt_sanitize(gt);
611}
612
613static int xe_set_dma_info(struct xe_device *xe)
614{
615 unsigned int mask_size = xe->info.dma_mask_size;
616 int err;
617
618 dma_set_max_seg_size(dev: xe->drm.dev, size: xe_sg_segment_size(dev: xe->drm.dev));
619
620 err = dma_set_mask(dev: xe->drm.dev, DMA_BIT_MASK(mask_size));
621 if (err)
622 goto mask_err;
623
624 err = dma_set_coherent_mask(dev: xe->drm.dev, DMA_BIT_MASK(mask_size));
625 if (err)
626 goto mask_err;
627
628 return 0;
629
630mask_err:
631 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
632 return err;
633}
634
635static int lmem_initializing(struct xe_device *xe)
636{
637 if (xe_mmio_read32(mmio: xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT)
638 return 0;
639
640 if (signal_pending(current))
641 return -EINTR;
642
643 return 1;
644}
645
646static int wait_for_lmem_ready(struct xe_device *xe)
647{
648 const unsigned long TIMEOUT_SEC = 60;
649 unsigned long prev_jiffies;
650 int initializing;
651
652 if (!IS_DGFX(xe))
653 return 0;
654
655 if (IS_SRIOV_VF(xe))
656 return 0;
657
658 if (!lmem_initializing(xe))
659 return 0;
660
661 drm_dbg(&xe->drm, "Waiting for lmem initialization\n");
662 prev_jiffies = jiffies;
663
664 /*
665 * The boot firmware initializes local memory and
666 * assesses its health. If memory training fails,
667 * the punit will have been instructed to keep the GT powered
668 * down.we won't be able to communicate with it
669 *
670 * If the status check is done before punit updates the register,
671 * it can lead to the system being unusable.
672 * use a timeout and defer the probe to prevent this.
673 */
674 poll_timeout_us(initializing = lmem_initializing(xe),
675 initializing <= 0,
676 20 * USEC_PER_MSEC, TIMEOUT_SEC * USEC_PER_SEC, true);
677 if (initializing < 0)
678 return initializing;
679
680 if (initializing) {
681 drm_dbg(&xe->drm, "lmem not initialized by firmware\n");
682 return -EPROBE_DEFER;
683 }
684
685 drm_dbg(&xe->drm, "lmem ready after %ums",
686 jiffies_to_msecs(jiffies - prev_jiffies));
687
688 return 0;
689}
690ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */
691
692static void vf_update_device_info(struct xe_device *xe)
693{
694 xe_assert(xe, IS_SRIOV_VF(xe));
695 /* disable features that are not available/applicable to VFs */
696 xe->info.probe_display = 0;
697 xe->info.has_heci_cscfi = 0;
698 xe->info.has_heci_gscfi = 0;
699 xe->info.has_late_bind = 0;
700 xe->info.skip_guc_pc = 1;
701 xe->info.skip_pcode = 1;
702}
703
704static int xe_device_vram_alloc(struct xe_device *xe)
705{
706 struct xe_vram_region *vram;
707
708 if (!IS_DGFX(xe))
709 return 0;
710
711 vram = drmm_kzalloc(dev: &xe->drm, size: sizeof(*vram), GFP_KERNEL);
712 if (!vram)
713 return -ENOMEM;
714
715 xe->mem.vram = vram;
716 return 0;
717}
718
719/**
720 * xe_device_probe_early: Device early probe
721 * @xe: xe device instance
722 *
723 * Initialize MMIO resources that don't require any
724 * knowledge about tile count. Also initialize pcode and
725 * check vram initialization on root tile.
726 *
727 * Return: 0 on success, error code on failure
728 */
729int xe_device_probe_early(struct xe_device *xe)
730{
731 int err;
732
733 xe_wa_device_init(xe);
734 xe_wa_process_device_oob(xe);
735
736 err = xe_mmio_probe_early(xe);
737 if (err)
738 return err;
739
740 xe_sriov_probe_early(xe);
741
742 if (IS_SRIOV_VF(xe))
743 vf_update_device_info(xe);
744
745 err = xe_pcode_probe_early(xe);
746 if (err || xe_survivability_mode_is_requested(xe)) {
747 int save_err = err;
748
749 /*
750 * Try to leave device in survivability mode if device is
751 * possible, but still return the previous error for error
752 * propagation
753 */
754 err = xe_survivability_mode_boot_enable(xe);
755 if (err)
756 return err;
757
758 return save_err;
759 }
760
761 err = wait_for_lmem_ready(xe);
762 if (err)
763 return err;
764
765 xe->wedged.mode = xe_modparam.wedged_mode;
766
767 err = xe_device_vram_alloc(xe);
768 if (err)
769 return err;
770
771 return 0;
772}
773ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
774
775static int probe_has_flat_ccs(struct xe_device *xe)
776{
777 struct xe_gt *gt;
778 unsigned int fw_ref;
779 u32 reg;
780
781 /* Always enabled/disabled, no runtime check to do */
782 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
783 return 0;
784
785 gt = xe_root_mmio_gt(xe);
786 if (!gt)
787 return 0;
788
789 fw_ref = xe_force_wake_get(fw: gt_to_fw(gt), domains: XE_FW_GT);
790 if (!fw_ref)
791 return -ETIMEDOUT;
792
793 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
794 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
795
796 if (!xe->info.has_flat_ccs)
797 drm_dbg(&xe->drm,
798 "Flat CCS has been disabled in bios, May lead to performance impact");
799
800 xe_force_wake_put(fw: gt_to_fw(gt), fw_ref);
801
802 return 0;
803}
804
805int xe_device_probe(struct xe_device *xe)
806{
807 struct xe_tile *tile;
808 struct xe_gt *gt;
809 int err;
810 u8 id;
811
812 xe_pat_init_early(xe);
813
814 err = xe_sriov_init(xe);
815 if (err)
816 return err;
817
818 xe->info.mem_region_mask = 1;
819
820 err = xe_set_dma_info(xe);
821 if (err)
822 return err;
823
824 err = xe_mmio_probe_tiles(xe);
825 if (err)
826 return err;
827
828 for_each_gt(gt, xe, id) {
829 err = xe_gt_init_early(gt);
830 if (err)
831 return err;
832 }
833
834 for_each_tile(tile, xe, id) {
835 err = xe_ggtt_init_early(ggtt: tile->mem.ggtt);
836 if (err)
837 return err;
838 }
839
840 /*
841 * From here on, if a step fails, make sure a Driver-FLR is triggereed
842 */
843 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
844 if (err)
845 return err;
846
847 err = probe_has_flat_ccs(xe);
848 if (err)
849 return err;
850
851 err = xe_vram_probe(xe);
852 if (err)
853 return err;
854
855 for_each_tile(tile, xe, id) {
856 err = xe_tile_init_noalloc(tile);
857 if (err)
858 return err;
859 }
860
861 /*
862 * Allow allocations only now to ensure xe_display_init_early()
863 * is the first to allocate, always.
864 */
865 err = xe_ttm_sys_mgr_init(xe);
866 if (err)
867 return err;
868
869 /* Allocate and map stolen after potential VRAM resize */
870 err = xe_ttm_stolen_mgr_init(xe);
871 if (err)
872 return err;
873
874 /*
875 * Now that GT is initialized (TTM in particular),
876 * we can try to init display, and inherit the initial fb.
877 * This is the reason the first allocation needs to be done
878 * inside display.
879 */
880 err = xe_display_init_early(xe);
881 if (err)
882 return err;
883
884 for_each_tile(tile, xe, id) {
885 err = xe_tile_init(tile);
886 if (err)
887 return err;
888 }
889
890 err = xe_irq_install(xe);
891 if (err)
892 return err;
893
894 for_each_gt(gt, xe, id) {
895 err = xe_gt_init(gt);
896 if (err)
897 return err;
898 }
899
900 err = xe_pagefault_init(xe);
901 if (err)
902 return err;
903
904 if (xe->tiles->media_gt &&
905 XE_GT_WA(xe->tiles->media_gt, 15015404425_disable))
906 XE_DEVICE_WA_DISABLE(xe, 15015404425);
907
908 err = xe_devcoredump_init(xe);
909 if (err)
910 return err;
911
912 xe_nvm_init(xe);
913
914 err = xe_heci_gsc_init(xe);
915 if (err)
916 return err;
917
918 err = xe_late_bind_init(late_bind: &xe->late_bind);
919 if (err)
920 return err;
921
922 err = xe_oa_init(xe);
923 if (err)
924 return err;
925
926 err = xe_display_init(xe);
927 if (err)
928 return err;
929
930 err = xe_pxp_init(xe);
931 if (err)
932 return err;
933
934 err = xe_psmi_init(xe);
935 if (err)
936 return err;
937
938 err = drm_dev_register(dev: &xe->drm, flags: 0);
939 if (err)
940 return err;
941
942 xe_display_register(xe);
943
944 err = xe_oa_register(xe);
945 if (err)
946 goto err_unregister_display;
947
948 err = xe_pmu_register(pmu: &xe->pmu);
949 if (err)
950 goto err_unregister_display;
951
952 err = xe_device_sysfs_init(xe);
953 if (err)
954 goto err_unregister_display;
955
956 xe_debugfs_register(xe);
957
958 err = xe_hwmon_register(xe);
959 if (err)
960 goto err_unregister_display;
961
962 err = xe_i2c_probe(xe);
963 if (err)
964 goto err_unregister_display;
965
966 for_each_gt(gt, xe, id)
967 xe_gt_sanitize_freq(gt);
968
969 xe_vsec_init(xe);
970
971 err = xe_sriov_init_late(xe);
972 if (err)
973 goto err_unregister_display;
974
975 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
976
977err_unregister_display:
978 xe_display_unregister(xe);
979
980 return err;
981}
982
983void xe_device_remove(struct xe_device *xe)
984{
985 xe_display_unregister(xe);
986
987 drm_dev_unplug(dev: &xe->drm);
988
989 xe_bo_pci_dev_remove_all(xe);
990}
991
992void xe_device_shutdown(struct xe_device *xe)
993{
994 struct xe_gt *gt;
995 u8 id;
996
997 drm_dbg(&xe->drm, "Shutting down device\n");
998
999 xe_display_pm_shutdown(xe);
1000
1001 xe_irq_suspend(xe);
1002
1003 for_each_gt(gt, xe, id)
1004 xe_gt_shutdown(gt);
1005
1006 xe_display_pm_shutdown_late(xe);
1007
1008 if (!xe_driver_flr_disabled(xe)) {
1009 /* BOOM! */
1010 __xe_driver_flr(xe);
1011 }
1012}
1013
1014/**
1015 * xe_device_wmb() - Device specific write memory barrier
1016 * @xe: the &xe_device
1017 *
1018 * While wmb() is sufficient for a barrier if we use system memory, on discrete
1019 * platforms with device memory we additionally need to issue a register write.
1020 * Since it doesn't matter which register we write to, use the read-only VF_CAP
1021 * register that is also marked as accessible by the VFs.
1022 */
1023void xe_device_wmb(struct xe_device *xe)
1024{
1025 wmb();
1026 if (IS_DGFX(xe))
1027 xe_mmio_write32(mmio: xe_root_tile_mmio(xe), VF_CAP_REG, val: 0);
1028}
1029
1030/*
1031 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
1032 */
1033static void tdf_request_sync(struct xe_device *xe)
1034{
1035 unsigned int fw_ref;
1036 struct xe_gt *gt;
1037 u8 id;
1038
1039 for_each_gt(gt, xe, id) {
1040 if (xe_gt_is_media_type(gt))
1041 continue;
1042
1043 fw_ref = xe_force_wake_get(fw: gt_to_fw(gt), domains: XE_FW_GT);
1044 if (!fw_ref)
1045 return;
1046
1047 xe_mmio_write32(mmio: &gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
1048
1049 /*
1050 * FIXME: We can likely do better here with our choice of
1051 * timeout. Currently we just assume the worst case, i.e. 150us,
1052 * which is believed to be sufficient to cover the worst case
1053 * scenario on current platforms if all cache entries are
1054 * transient and need to be flushed..
1055 */
1056 if (xe_mmio_wait32(mmio: &gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, val: 0,
1057 timeout_us: 300, NULL, atomic: false))
1058 xe_gt_err_once(gt, "TD flush timeout\n");
1059
1060 xe_force_wake_put(fw: gt_to_fw(gt), fw_ref);
1061 }
1062}
1063
1064void xe_device_l2_flush(struct xe_device *xe)
1065{
1066 struct xe_gt *gt;
1067 unsigned int fw_ref;
1068
1069 gt = xe_root_mmio_gt(xe);
1070 if (!gt)
1071 return;
1072
1073 if (!XE_GT_WA(gt, 16023588340))
1074 return;
1075
1076 fw_ref = xe_force_wake_get(fw: gt_to_fw(gt), domains: XE_FW_GT);
1077 if (!fw_ref)
1078 return;
1079
1080 spin_lock(lock: &gt->global_invl_lock);
1081
1082 xe_mmio_write32(mmio: &gt->mmio, XE2_GLOBAL_INVAL, val: 0x1);
1083 if (xe_mmio_wait32(mmio: &gt->mmio, XE2_GLOBAL_INVAL, mask: 0x1, val: 0x0, timeout_us: 1000, NULL, atomic: true))
1084 xe_gt_err_once(gt, "Global invalidation timeout\n");
1085
1086 spin_unlock(lock: &gt->global_invl_lock);
1087
1088 xe_force_wake_put(fw: gt_to_fw(gt), fw_ref);
1089}
1090
1091/**
1092 * xe_device_td_flush() - Flush transient L3 cache entries
1093 * @xe: The device
1094 *
1095 * Display engine has direct access to memory and is never coherent with L3/L4
1096 * caches (or CPU caches), however KMD is responsible for specifically flushing
1097 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
1098 * can happen from such a surface without seeing corruption.
1099 *
1100 * Display surfaces can be tagged as transient by mapping it using one of the
1101 * various L3:XD PAT index modes on Xe2.
1102 *
1103 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
1104 * at the end of each submission via PIPE_CONTROL for compute/render, since SA
1105 * Media is not coherent with L3 and we want to support render-vs-media
1106 * usescases. For other engines like copy/blt the HW internally forces uncached
1107 * behaviour, hence why we can skip the TDF on such platforms.
1108 */
1109void xe_device_td_flush(struct xe_device *xe)
1110{
1111 struct xe_gt *root_gt;
1112
1113 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1114 return;
1115
1116 root_gt = xe_root_mmio_gt(xe);
1117 if (!root_gt)
1118 return;
1119
1120 if (XE_GT_WA(root_gt, 16023588340)) {
1121 /* A transient flush is not sufficient: flush the L2 */
1122 xe_device_l2_flush(xe);
1123 } else {
1124 xe_guc_pc_apply_flush_freq_limit(pc: &root_gt->uc.guc.pc);
1125 tdf_request_sync(xe);
1126 xe_guc_pc_remove_flush_freq_limit(pc: &root_gt->uc.guc.pc);
1127 }
1128}
1129
1130u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1131{
1132 return xe_device_has_flat_ccs(xe) ?
1133 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1134}
1135
1136/**
1137 * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1138 * @xe: xe device instance
1139 *
1140 * To be used before any kind of memory access. It will splat a debug warning
1141 * if the device is currently sleeping. But it doesn't guarantee in any way
1142 * that the device is going to remain awake. Xe PM runtime get and put
1143 * functions might be added to the outer bound of the memory access, while
1144 * this check is intended for inner usage to splat some warning if the worst
1145 * case has just happened.
1146 */
1147void xe_device_assert_mem_access(struct xe_device *xe)
1148{
1149 xe_assert(xe, !xe_pm_runtime_suspended(xe));
1150}
1151
1152void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1153{
1154 struct xe_gt *gt;
1155 u8 id;
1156
1157 drm_printf(p, f: "PCI ID: 0x%04x\n", xe->info.devid);
1158 drm_printf(p, f: "PCI revision: 0x%02x\n", xe->info.revid);
1159
1160 for_each_gt(gt, xe, id) {
1161 drm_printf(p, f: "GT id: %u\n", id);
1162 drm_printf(p, f: "\tTile: %u\n", gt->tile->id);
1163 drm_printf(p, f: "\tType: %s\n",
1164 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1165 drm_printf(p, f: "\tIP ver: %u.%u.%u\n",
1166 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1167 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1168 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1169 drm_printf(p, f: "\tCS reference clock: %u\n", gt->info.reference_clock);
1170 }
1171}
1172
1173u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1174{
1175 return sign_extend64(value: address, index: xe->info.va_bits - 1);
1176}
1177
1178u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1179{
1180 return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1181}
1182
1183static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
1184{
1185 struct xe_device *xe = arg;
1186
1187 xe_pm_runtime_put(xe);
1188}
1189
1190/**
1191 * DOC: Xe Device Wedging
1192 *
1193 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst.
1194 * When device is in wedged state, every IOCTL will be blocked and GT cannot be
1195 * used. Certain critical errors like gt reset failure, firmware failures can cause
1196 * the device to be wedged. The default recovery method for a wedged state
1197 * is rebind/bus-reset.
1198 *
1199 * Another recovery method is vendor-specific. Below are the cases that send
1200 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent.
1201 *
1202 * Case: Firmware Flash
1203 * --------------------
1204 *
1205 * Identification Hint
1206 * +++++++++++++++++++
1207 *
1208 * ``WEDGED=vendor-specific`` drm device wedged uevent with
1209 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify
1210 * admin/userspace consumer about the need for a firmware flash.
1211 *
1212 * Recovery Procedure
1213 * ++++++++++++++++++
1214 *
1215 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow
1216 * the below steps
1217 *
1218 * - Check Runtime Survivability mode sysfs.
1219 * If enabled, firmware flash is required to recover the device.
1220 *
1221 * /sys/bus/pci/devices/<device>/survivability_mode
1222 *
1223 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash
1224 * firmware and restore device to normal operation.
1225 */
1226
1227/**
1228 * xe_device_set_wedged_method - Set wedged recovery method
1229 * @xe: xe device instance
1230 * @method: recovery method to set
1231 *
1232 * Set wedged recovery method to be sent in drm wedged uevent.
1233 */
1234void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
1235{
1236 xe->wedged.method = method;
1237}
1238
1239/**
1240 * xe_device_declare_wedged - Declare device wedged
1241 * @xe: xe device instance
1242 *
1243 * This is a final state that can only be cleared with the recovery method
1244 * specified in the drm wedged uevent. The method can be set using
1245 * xe_device_set_wedged_method before declaring the device as wedged. If no method
1246 * is set, reprobe (unbind/re-bind) will be sent by default.
1247 *
1248 * In this state every IOCTL will be blocked so the GT cannot be used.
1249 * In general it will be called upon any critical error such as gt reset
1250 * failure or guc loading failure. Userspace will be notified of this state
1251 * through device wedged uevent.
1252 * If xe.wedged module parameter is set to 2, this function will be called
1253 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1254 * snapshot capture. In this mode, GT reset won't be attempted so the state of
1255 * the issue is preserved for further debugging.
1256 */
1257void xe_device_declare_wedged(struct xe_device *xe)
1258{
1259 struct xe_gt *gt;
1260 u8 id;
1261
1262 if (xe->wedged.mode == 0) {
1263 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1264 return;
1265 }
1266
1267 xe_pm_runtime_get_noresume(xe);
1268
1269 if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) {
1270 drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n");
1271 return;
1272 }
1273
1274 if (!atomic_xchg(v: &xe->wedged.flag, new: 1)) {
1275 xe->needs_flr_on_fini = true;
1276 drm_err(&xe->drm,
1277 "CRITICAL: Xe has declared device %s as wedged.\n"
1278 "IOCTLs and executions are blocked. Only a rebind may clear the failure\n"
1279 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
1280 dev_name(xe->drm.dev));
1281 }
1282
1283 for_each_gt(gt, xe, id)
1284 xe_gt_declare_wedged(gt);
1285
1286 if (xe_device_wedged(xe)) {
1287 /* If no wedge recovery method is set, use default */
1288 if (!xe->wedged.method)
1289 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND |
1290 DRM_WEDGE_RECOVERY_BUS_RESET);
1291
1292 /* Notify userspace of wedged device */
1293 drm_dev_wedged_event(dev: &xe->drm, method: xe->wedged.method, NULL);
1294 }
1295}
1296

source code of linux/drivers/gpu/drm/xe/xe_device.c