1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include "xe_guc.h"
7
8	#include <linux/iopoll.h>
9	#include <drm/drm_managed.h>
10
11	#include <generated/xe_wa_oob.h>
12
13	#include "abi/guc_actions_abi.h"
14	#include "abi/guc_errors_abi.h"
15	#include "regs/xe_gt_regs.h"
16	#include "regs/xe_gtt_defs.h"
17	#include "regs/xe_guc_regs.h"
18	#include "regs/xe_irq_regs.h"
19	#include "xe_bo.h"
20	#include "xe_configfs.h"
21	#include "xe_device.h"
22	#include "xe_force_wake.h"
23	#include "xe_gt.h"
24	#include "xe_gt_printk.h"
25	#include "xe_gt_sriov_vf.h"
26	#include "xe_gt_throttle.h"
27	#include "xe_gt_sriov_pf_migration.h"
28	#include "xe_guc_ads.h"
29	#include "xe_guc_buf.h"
30	#include "xe_guc_capture.h"
31	#include "xe_guc_ct.h"
32	#include "xe_guc_db_mgr.h"
33	#include "xe_guc_engine_activity.h"
34	#include "xe_guc_hwconfig.h"
35	#include "xe_guc_klv_helpers.h"
36	#include "xe_guc_log.h"
37	#include "xe_guc_pc.h"
38	#include "xe_guc_relay.h"
39	#include "xe_guc_submit.h"
40	#include "xe_memirq.h"
41	#include "xe_mmio.h"
42	#include "xe_platform_types.h"
43	#include "xe_sriov.h"
44	#include "xe_sriov_pf_migration.h"
45	#include "xe_uc.h"
46	#include "xe_uc_fw.h"
47	#include "xe_wa.h"
48	#include "xe_wopcm.h"
49
50	static u32 guc_bo_ggtt_addr(struct xe_guc *guc,
51	struct xe_bo *bo)
52	{
53	struct xe_device *xe = guc_to_xe(guc);
54	u32 addr;
55
56	/*
57	* For most BOs, the address on the allocating tile is fine. However for
58	* some, e.g. G2G CTB, the address on a specific tile is required as it
59	* might be different for each tile. So, just always ask for the address
60	* on the target GuC.
61	*/
62	addr = __xe_bo_ggtt_addr(bo, gt_to_tile(guc_to_gt(guc))->id);
63
64	/* GuC addresses above GUC_GGTT_TOP don't map through the GTT */
65	xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc)));
66	xe_assert(xe, addr < GUC_GGTT_TOP);
67	xe_assert(xe, xe_bo_size(bo) <= GUC_GGTT_TOP - addr);
68
69	return addr;
70	}
71
72	static u32 guc_ctl_debug_flags(struct xe_guc *guc)
73	{
74	u32 level = xe_guc_log_get_level(log: &guc->log);
75	u32 flags = 0;
76
77	if (!GUC_LOG_LEVEL_IS_VERBOSE(level))
78	flags |= GUC_LOG_DISABLED;
79	else
80	flags |= FIELD_PREP(GUC_LOG_VERBOSITY, GUC_LOG_LEVEL_TO_VERBOSITY(level));
81
82	return flags;
83	}
84
85	static u32 guc_ctl_feature_flags(struct xe_guc *guc)
86	{
87	struct xe_device *xe = guc_to_xe(guc);
88	u32 flags = GUC_CTL_ENABLE_LITE_RESTORE;
89
90	if (!xe->info.skip_guc_pc)
91	flags |= GUC_CTL_ENABLE_SLPC;
92
93	if (xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev)))
94	flags |= GUC_CTL_ENABLE_PSMI_LOGGING;
95
96	if (xe_guc_using_main_gamctrl_queues(guc))
97	flags |= GUC_CTL_MAIN_GAMCTRL_QUEUES;
98
99	return flags;
100	}
101
102	static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
103	{
104	u32 offset = guc_bo_ggtt_addr(guc, bo: guc->log.bo) >> PAGE_SHIFT;
105	u32 flags;
106
107	#if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
108	#define LOG_UNIT SZ_1M
109	#define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
110	#else
111	#define LOG_UNIT SZ_4K
112	#define LOG_FLAG 0
113	#endif
114
115	#if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
116	#define CAPTURE_UNIT SZ_1M
117	#define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
118	#else
119	#define CAPTURE_UNIT SZ_4K
120	#define CAPTURE_FLAG 0
121	#endif
122
123	BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
124	BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
125	BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
126	BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
127	BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
128	BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
129
130	flags = GUC_LOG_VALID |
131	GUC_LOG_NOTIFY_ON_HALF_FULL |
132	CAPTURE_FLAG |
133	LOG_FLAG |
134	FIELD_PREP(GUC_LOG_CRASH, CRASH_BUFFER_SIZE / LOG_UNIT - 1) |
135	FIELD_PREP(GUC_LOG_DEBUG, DEBUG_BUFFER_SIZE / LOG_UNIT - 1) |
136	FIELD_PREP(GUC_LOG_CAPTURE, CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) |
137	FIELD_PREP(GUC_LOG_BUF_ADDR, offset);
138
139	#undef LOG_UNIT
140	#undef LOG_FLAG
141	#undef CAPTURE_UNIT
142	#undef CAPTURE_FLAG
143
144	return flags;
145	}
146
147	static u32 guc_ctl_ads_flags(struct xe_guc *guc)
148	{
149	u32 ads = guc_bo_ggtt_addr(guc, bo: guc->ads.bo) >> PAGE_SHIFT;
150	u32 flags = FIELD_PREP(GUC_ADS_ADDR, ads);
151
152	return flags;
153	}
154
155	static bool needs_wa_dual_queue(struct xe_gt *gt)
156	{
157	/*
158	* The DUAL_QUEUE_WA tells the GuC to not allow concurrent submissions
159	* on RCS and CCSes with different address spaces, which on DG2 is
160	* required as a WA for an HW bug.
161	*/
162	if (XE_GT_WA(gt, 22011391025))
163	return true;
164
165	/*
166	* On newer platforms, the HW has been updated to not allow parallel
167	* execution of different address spaces, so the RCS/CCS will stall the
168	* context switch if one of the other RCS/CCSes is busy with a different
169	* address space. While functionally correct, having a submission
170	* stalled on the HW limits the GuC ability to shuffle things around and
171	* can cause complications if the non-stalled submission runs for a long
172	* time, because the GuC doesn't know that the stalled submission isn't
173	* actually running and might declare it as hung. Therefore, we enable
174	* the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines
175	* to move management back to the GuC.
176	*/
177	if (CCS_MASK(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
178	return true;
179
180	return false;
181	}
182
183	static u32 guc_ctl_wa_flags(struct xe_guc *guc)
184	{
185	struct xe_device *xe = guc_to_xe(guc);
186	struct xe_gt *gt = guc_to_gt(guc);
187	u32 flags = 0;
188
189	if (XE_GT_WA(gt, 22012773006))
190	flags |= GUC_WA_POLLCS;
191
192	if (XE_GT_WA(gt, 14014475959))
193	flags |= GUC_WA_HOLD_CCS_SWITCHOUT;
194
195	if (needs_wa_dual_queue(gt))
196	flags |= GUC_WA_DUAL_QUEUE;
197
198	/*
199	* Wa_22011802037: FIXME - there's more to be done than simply setting
200	* this flag: make sure each CS is stopped when preparing for GT reset
201	* and wait for pending MI_FW.
202	*/
203	if (GRAPHICS_VERx100(xe) < 1270)
204	flags |= GUC_WA_PRE_PARSER;
205
206	if (XE_GT_WA(gt, 22012727170) || XE_GT_WA(gt, 22012727685))
207	flags |= GUC_WA_CONTEXT_ISOLATION;
208
209	if (XE_GT_WA(gt, 18020744125) &&
210	!xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
211	flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
212
213	if (XE_GT_WA(gt, 1509372804))
214	flags |= GUC_WA_RENDER_RST_RC6_EXIT;
215
216	if (XE_GT_WA(gt, 14018913170))
217	flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
218
219	if (XE_GT_WA(gt, 16023683509))
220	flags |= GUC_WA_SAVE_RESTORE_MCFG_REG_AT_MC6;
221
222	return flags;
223	}
224
225	static u32 guc_ctl_devid(struct xe_guc *guc)
226	{
227	struct xe_device *xe = guc_to_xe(guc);
228
229	return (((u32)xe->info.devid) << 16) | xe->info.revid;
230	}
231
232	static void guc_print_params(struct xe_guc *guc)
233	{
234	struct xe_gt *gt = guc_to_gt(guc);
235	u32 *params = guc->params;
236	int i;
237
238	BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
239	BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
240
241	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
242	xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]);
243	}
244
245	static void guc_init_params(struct xe_guc *guc)
246	{
247	u32 *params = guc->params;
248
249	params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
250	params[GUC_CTL_FEATURE] = 0;
251	params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
252	params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
253	params[GUC_CTL_WA] = 0;
254	params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
255
256	guc_print_params(guc);
257	}
258
259	static void guc_init_params_post_hwconfig(struct xe_guc *guc)
260	{
261	u32 *params = guc->params;
262
263	params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
264	params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc);
265	params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
266	params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
267	params[GUC_CTL_WA] = guc_ctl_wa_flags(guc);
268	params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
269
270	guc_print_params(guc);
271	}
272
273	/*
274	* Initialize the GuC parameter block before starting the firmware
275	* transfer. These parameters are read by the firmware on startup
276	* and cannot be changed thereafter.
277	*/
278	static void guc_write_params(struct xe_guc *guc)
279	{
280	struct xe_gt *gt = guc_to_gt(guc);
281	int i;
282
283	xe_force_wake_assert_held(fw: gt_to_fw(gt), domain: XE_FW_GT);
284
285	xe_mmio_write32(mmio: &gt->mmio, SOFT_SCRATCH(0), val: 0);
286
287	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
288	xe_mmio_write32(mmio: &gt->mmio, SOFT_SCRATCH(1 + i), val: guc->params[i]);
289	}
290
291	static int guc_action_register_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev,
292	u32 desc_addr, u32 buff_addr, u32 size)
293	{
294	struct xe_gt *gt = guc_to_gt(guc);
295	struct xe_device *xe = gt_to_xe(gt);
296	u32 action[] = {
297	XE_GUC_ACTION_REGISTER_G2G,
298	FIELD_PREP(XE_G2G_REGISTER_SIZE, size / SZ_4K - 1) |
299	FIELD_PREP(XE_G2G_REGISTER_TYPE, type) |
300	FIELD_PREP(XE_G2G_REGISTER_TILE, dst_tile) |
301	FIELD_PREP(XE_G2G_REGISTER_DEVICE, dst_dev),
302	desc_addr,
303	buff_addr,
304	};
305
306	xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
307	xe_assert(xe, !(size % SZ_4K));
308
309	return xe_guc_ct_send_block(ct: &guc->ct, action, ARRAY_SIZE(action));
310	}
311
312	static int guc_action_deregister_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev)
313	{
314	struct xe_gt *gt = guc_to_gt(guc);
315	struct xe_device *xe = gt_to_xe(gt);
316	u32 action[] = {
317	XE_GUC_ACTION_DEREGISTER_G2G,
318	FIELD_PREP(XE_G2G_DEREGISTER_TYPE, type) |
319	FIELD_PREP(XE_G2G_DEREGISTER_TILE, dst_tile) |
320	FIELD_PREP(XE_G2G_DEREGISTER_DEVICE, dst_dev),
321	};
322
323	xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
324
325	return xe_guc_ct_send_block(ct: &guc->ct, action, ARRAY_SIZE(action));
326	}
327
328	#define G2G_DEV(gt) (((gt)->info.type == XE_GT_TYPE_MAIN) ? 0 : 1)
329
330	#define G2G_BUFFER_SIZE (SZ_4K)
331	#define G2G_DESC_SIZE (64)
332	#define G2G_DESC_AREA_SIZE (SZ_4K)
333
334	/*
335	* Generate a unique id for each bi-directional CTB for each pair of
336	* near and far tiles/devices. The id can then be used as an index into
337	* a single allocation that is sub-divided into multiple CTBs.
338	*
339	* For example, with two devices per tile and two tiles, the table should
340	* look like:
341	* Far <tile>.<dev>
342	* 0.0 0.1 1.0 1.1
343	* N 0.0 --/-- 00/01 02/03 04/05
344	* e 0.1 01/00 --/-- 06/07 08/09
345	* a 1.0 03/02 07/06 --/-- 10/11
346	* r 1.1 05/04 09/08 11/10 --/--
347	*
348	* Where each entry is Rx/Tx channel id.
349	*
350	* So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would
351	* be reading from channel #11 and writing to channel #10. Whereas,
352	* GuC #2 talking to GuC #3 would be read on #10 and write to #11.
353	*/
354	static unsigned int g2g_slot(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev,
355	u32 type, u32 max_inst, bool have_dev)
356	{
357	u32 near = near_tile, far = far_tile;
358	u32 idx = 0, x, y, direction;
359	int i;
360
361	if (have_dev) {
362	near = (near << 1) | near_dev;
363	far = (far << 1) | far_dev;
364	}
365
366	/* No need to send to one's self */
367	if (far == near)
368	return -1;
369
370	if (far > near) {
371	/* Top right table half */
372	x = far;
373	y = near;
374
375	/* T/R is 'forwards' direction */
376	direction = type;
377	} else {
378	/* Bottom left table half */
379	x = near;
380	y = far;
381
382	/* B/L is 'backwards' direction */
383	direction = (1 - type);
384	}
385
386	/* Count the rows prior to the target */
387	for (i = y; i > 0; i--)
388	idx += max_inst - i;
389
390	/* Count this row up to the target */
391	idx += (x - 1 - y);
392
393	/* Slots are in Rx/Tx pairs */
394	idx *= 2;
395
396	/* Pick Rx/Tx direction */
397	idx += direction;
398
399	return idx;
400	}
401
402	static int guc_g2g_register(struct xe_guc *near_guc, struct xe_gt *far_gt, u32 type, bool have_dev)
403	{
404	struct xe_gt *near_gt = guc_to_gt(guc: near_guc);
405	struct xe_device *xe = gt_to_xe(near_gt);
406	struct xe_bo *g2g_bo;
407	u32 near_tile = gt_to_tile(near_gt)->id;
408	u32 near_dev = G2G_DEV(near_gt);
409	u32 far_tile = gt_to_tile(far_gt)->id;
410	u32 far_dev = G2G_DEV(far_gt);
411	u32 max = xe->info.gt_count;
412	u32 base, desc, buf;
413	int slot;
414
415	/* G2G is not allowed between different cards */
416	xe_assert(xe, xe == gt_to_xe(far_gt));
417
418	g2g_bo = near_guc->g2g.bo;
419	xe_assert(xe, g2g_bo);
420
421	slot = g2g_slot(near_tile, near_dev, far_tile, far_dev, type, max_inst: max, have_dev);
422	xe_assert(xe, slot >= 0);
423
424	base = guc_bo_ggtt_addr(guc: near_guc, bo: g2g_bo);
425	desc = base + slot * G2G_DESC_SIZE;
426	buf = base + G2G_DESC_AREA_SIZE + slot * G2G_BUFFER_SIZE;
427
428	xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE);
429	xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= xe_bo_size(g2g_bo));
430
431	return guc_action_register_g2g_buffer(guc: near_guc, type, dst_tile: far_tile, dst_dev: far_dev,
432	desc_addr: desc, buff_addr: buf, G2G_BUFFER_SIZE);
433	}
434
435	static void guc_g2g_deregister(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type)
436	{
437	guc_action_deregister_g2g_buffer(guc, type, dst_tile: far_tile, dst_dev: far_dev);
438	}
439
440	static u32 guc_g2g_size(struct xe_guc *guc)
441	{
442	struct xe_gt *gt = guc_to_gt(guc);
443	struct xe_device *xe = gt_to_xe(gt);
444	unsigned int count = xe->info.gt_count;
445	u32 num_channels = (count * (count - 1)) / 2;
446
447	xe_assert(xe, num_channels * XE_G2G_TYPE_LIMIT * G2G_DESC_SIZE <= G2G_DESC_AREA_SIZE);
448
449	return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE;
450	}
451
452	static bool xe_guc_g2g_wanted(struct xe_device *xe)
453	{
454	/* Can't do GuC to GuC communication if there is only one GuC */
455	if (xe->info.gt_count <= 1)
456	return false;
457
458	/* No current user */
459	return false;
460	}
461
462	static int guc_g2g_alloc(struct xe_guc *guc)
463	{
464	struct xe_gt *gt = guc_to_gt(guc);
465	struct xe_device *xe = gt_to_xe(gt);
466	struct xe_tile *tile = gt_to_tile(gt);
467	struct xe_bo *bo;
468	u32 g2g_size;
469
470	if (guc->g2g.bo)
471	return 0;
472
473	if (gt->info.id != 0) {
474	struct xe_gt *root_gt = xe_device_get_gt(xe, gt_id: 0);
475	struct xe_guc *root_guc = &root_gt->uc.guc;
476	struct xe_bo *bo;
477
478	bo = xe_bo_get(bo: root_guc->g2g.bo);
479	if (!bo)
480	return -ENODEV;
481
482	guc->g2g.bo = bo;
483	guc->g2g.owned = false;
484	return 0;
485	}
486
487	g2g_size = guc_g2g_size(guc);
488	bo = xe_managed_bo_create_pin_map(xe, tile, size: g2g_size,
489	XE_BO_FLAG_VRAM_IF_DGFX(tile) |
490	XE_BO_FLAG_GGTT |
491	XE_BO_FLAG_GGTT_ALL |
492	XE_BO_FLAG_GGTT_INVALIDATE |
493	XE_BO_FLAG_PINNED_NORESTORE);
494	if (IS_ERR(ptr: bo))
495	return PTR_ERR(ptr: bo);
496
497	xe_map_memset(xe, dst: &bo->vmap, offset: 0, value: 0, len: g2g_size);
498	guc->g2g.bo = bo;
499	guc->g2g.owned = true;
500
501	return 0;
502	}
503
504	static void guc_g2g_fini(struct xe_guc *guc)
505	{
506	if (!guc->g2g.bo)
507	return;
508
509	/* Unpinning the owned object is handled by generic shutdown */
510	if (!guc->g2g.owned)
511	xe_bo_put(bo: guc->g2g.bo);
512
513	guc->g2g.bo = NULL;
514	}
515
516	static int guc_g2g_start(struct xe_guc *guc)
517	{
518	struct xe_gt *far_gt, *gt = guc_to_gt(guc);
519	struct xe_device *xe = gt_to_xe(gt);
520	unsigned int i, j;
521	int t, err;
522	bool have_dev;
523
524	if (!guc->g2g.bo) {
525	int ret;
526
527	ret = guc_g2g_alloc(guc);
528	if (ret)
529	return ret;
530	}
531
532	/* GuC interface will need extending if more GT device types are ever created. */
533	xe_gt_assert(gt, (gt->info.type == XE_GT_TYPE_MAIN) || (gt->info.type == XE_GT_TYPE_MEDIA));
534
535	/* Channel numbering depends on whether there are multiple GTs per tile */
536	have_dev = xe->info.gt_count > xe->info.tile_count;
537
538	for_each_gt(far_gt, xe, i) {
539	u32 far_tile, far_dev;
540
541	if (far_gt->info.id == gt->info.id)
542	continue;
543
544	far_tile = gt_to_tile(far_gt)->id;
545	far_dev = G2G_DEV(far_gt);
546
547	for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) {
548	err = guc_g2g_register(near_guc: guc, far_gt, type: t, have_dev);
549	if (err) {
550	while (--t >= 0)
551	guc_g2g_deregister(guc, far_tile, far_dev, type: t);
552	goto err_deregister;
553	}
554	}
555	}
556
557	return 0;
558
559	err_deregister:
560	for_each_gt(far_gt, xe, j) {
561	u32 tile, dev;
562
563	if (far_gt->info.id == gt->info.id)
564	continue;
565
566	if (j >= i)
567	break;
568
569	tile = gt_to_tile(far_gt)->id;
570	dev = G2G_DEV(far_gt);
571
572	for (t = 0; t < XE_G2G_TYPE_LIMIT; t++)
573	guc_g2g_deregister(guc, far_tile: tile, far_dev: dev, type: t);
574	}
575
576	return err;
577	}
578
579	static int __guc_opt_in_features_enable(struct xe_guc *guc, u64 addr, u32 num_dwords)
580	{
581	u32 action[] = {
582	XE_GUC_ACTION_OPT_IN_FEATURE_KLV,
583	lower_32_bits(addr),
584	upper_32_bits(addr),
585	num_dwords
586	};
587
588	return xe_guc_ct_send_block(ct: &guc->ct, action, ARRAY_SIZE(action));
589	}
590
591	static bool supports_dynamic_ics(struct xe_guc *guc)
592	{
593	struct xe_device *xe = guc_to_xe(guc);
594	struct xe_gt *gt = guc_to_gt(guc);
595
596	/* Dynamic ICS is available for PVC and Xe2 and newer platforms. */
597	if (xe->info.platform != XE_PVC && GRAPHICS_VER(xe) < 20)
598	return false;
599
600	/*
601	* The feature is currently not compatible with multi-lrc, so the GuC
602	* does not support it at all on the media engines (which are the main
603	* users of mlrc). On the primary GT side, to avoid it being used in
604	* conjunction with mlrc, we only enable it if we are in single CCS
605	* mode.
606	*/
607	if (xe_gt_is_media_type(gt) || gt->ccs_mode > 1)
608	return false;
609
610	/*
611	* Dynamic ICS requires GuC v70.40.1, which maps to compatibility
612	* version v1.18.4.
613	*/
614	return GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 18, 4);
615	}
616
617	#define OPT_IN_MAX_DWORDS 16
618	int xe_guc_opt_in_features_enable(struct xe_guc *guc)
619	{
620	struct xe_device *xe = guc_to_xe(guc);
621	CLASS(xe_guc_buf, buf)(cache: &guc->buf, OPT_IN_MAX_DWORDS);
622	u32 count = 0;
623	u32 *klvs;
624	int ret;
625
626	if (!xe_guc_buf_is_valid(buf))
627	return -ENOBUFS;
628
629	klvs = xe_guc_buf_cpu_ptr(buf);
630
631	/*
632	* The extra CAT error type opt-in was added in GuC v70.17.0, which maps
633	* to compatibility version v1.7.0.
634	* Note that the GuC allows enabling this KLV even on platforms that do
635	* not support the extra type; in such case the returned type variable
636	* will be set to a known invalid value which we can check against.
637	*/
638	if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 7, 0))
639	klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_EXT_CAT_ERR_TYPE);
640
641	if (supports_dynamic_ics(guc))
642	klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH);
643
644	if (count) {
645	xe_assert(xe, count <= OPT_IN_MAX_DWORDS);
646
647	ret = __guc_opt_in_features_enable(guc, addr: xe_guc_buf_flush(buf), num_dwords: count);
648	if (ret < 0) {
649	xe_gt_err(guc_to_gt(guc),
650	"failed to enable GuC opt-in features: %pe\n",
651	ERR_PTR(ret));
652	return ret;
653	}
654	}
655
656	return 0;
657	}
658
659	static void guc_fini_hw(void *arg)
660	{
661	struct xe_guc *guc = arg;
662	struct xe_gt *gt = guc_to_gt(guc);
663	unsigned int fw_ref;
664
665	fw_ref = xe_force_wake_get(fw: gt_to_fw(gt), domains: XE_FORCEWAKE_ALL);
666	xe_uc_sanitize_reset(uc: &guc_to_gt(guc)->uc);
667	xe_force_wake_put(fw: gt_to_fw(gt), fw_ref);
668
669	guc_g2g_fini(guc);
670	}
671
672	/**
673	* xe_guc_comm_init_early - early initialization of GuC communication
674	* @guc: the &xe_guc to initialize
675	*
676	* Must be called prior to first MMIO communication with GuC firmware.
677	*/
678	void xe_guc_comm_init_early(struct xe_guc *guc)
679	{
680	struct xe_gt *gt = guc_to_gt(guc);
681
682	if (xe_gt_is_media_type(gt))
683	guc->notify_reg = MED_GUC_HOST_INTERRUPT;
684	else
685	guc->notify_reg = GUC_HOST_INTERRUPT;
686	}
687
688	static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc)
689	{
690	struct xe_tile *tile = gt_to_tile(guc_to_gt(guc));
691	struct xe_device *xe = guc_to_xe(guc);
692	int ret;
693
694	if (!IS_DGFX(guc_to_xe(guc)))
695	return 0;
696
697	ret = xe_managed_bo_reinit_in_vram(xe, tile, src: &guc->fw.bo);
698	if (ret)
699	return ret;
700
701	ret = xe_managed_bo_reinit_in_vram(xe, tile, src: &guc->log.bo);
702	if (ret)
703	return ret;
704
705	ret = xe_managed_bo_reinit_in_vram(xe, tile, src: &guc->ads.bo);
706	if (ret)
707	return ret;
708
709	return 0;
710	}
711
712	static int vf_guc_init_noalloc(struct xe_guc *guc)
713	{
714	struct xe_gt *gt = guc_to_gt(guc);
715	int err;
716
717	err = xe_gt_sriov_vf_bootstrap(gt);
718	if (err)
719	return err;
720
721	err = xe_gt_sriov_vf_query_config(gt);
722	if (err)
723	return err;
724
725	return 0;
726	}
727
728	int xe_guc_init_noalloc(struct xe_guc *guc)
729	{
730	struct xe_device *xe = guc_to_xe(guc);
731	struct xe_gt *gt = guc_to_gt(guc);
732	int ret;
733
734	xe_guc_comm_init_early(guc);
735
736	ret = xe_guc_ct_init_noalloc(ct: &guc->ct);
737	if (ret)
738	goto out;
739
740	ret = xe_guc_relay_init(relay: &guc->relay);
741	if (ret)
742	goto out;
743
744	if (IS_SRIOV_VF(xe)) {
745	ret = vf_guc_init_noalloc(guc);
746	if (ret)
747	goto out;
748	}
749
750	return 0;
751
752	out:
753	xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
754	return ret;
755	}
756
757	int xe_guc_init(struct xe_guc *guc)
758	{
759	struct xe_device *xe = guc_to_xe(guc);
760	struct xe_gt *gt = guc_to_gt(guc);
761	int ret;
762
763	guc->fw.type = XE_UC_FW_TYPE_GUC;
764	ret = xe_uc_fw_init(uc_fw: &guc->fw);
765	if (ret)
766	return ret;
767
768	if (!xe_uc_fw_is_enabled(uc_fw: &guc->fw))
769	return 0;
770
771	if (IS_SRIOV_VF(xe)) {
772	ret = xe_guc_ct_init(ct: &guc->ct);
773	if (ret)
774	goto out;
775	return 0;
776	}
777
778	ret = xe_guc_log_init(log: &guc->log);
779	if (ret)
780	goto out;
781
782	ret = xe_guc_capture_init(guc);
783	if (ret)
784	goto out;
785
786	ret = xe_guc_ads_init(ads: &guc->ads);
787	if (ret)
788	goto out;
789
790	ret = xe_guc_ct_init(ct: &guc->ct);
791	if (ret)
792	goto out;
793
794	xe_uc_fw_change_status(uc_fw: &guc->fw, status: XE_UC_FIRMWARE_LOADABLE);
795
796	ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc);
797	if (ret)
798	goto out;
799
800	guc_init_params(guc);
801
802	return 0;
803
804	out:
805	xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
806	return ret;
807	}
808
809	static int vf_guc_init_post_hwconfig(struct xe_guc *guc)
810	{
811	int err;
812
813	err = xe_guc_submit_init(guc, num_ids: xe_gt_sriov_vf_guc_ids(gt: guc_to_gt(guc)));
814	if (err)
815	return err;
816
817	err = xe_guc_buf_cache_init(cache: &guc->buf);
818	if (err)
819	return err;
820
821	/* XXX xe_guc_db_mgr_init not needed for now */
822
823	return 0;
824	}
825
826	static u32 guc_additional_cache_size(struct xe_device *xe)
827	{
828	if (IS_SRIOV_PF(xe) && xe_sriov_pf_migration_supported(xe))
829	return XE_GT_SRIOV_PF_MIGRATION_GUC_DATA_MAX_SIZE;
830	else
831	return 0; /* Fallback to default size */
832	}
833
834	/**
835	* xe_guc_init_post_hwconfig - initialize GuC post hwconfig load
836	* @guc: The GuC object
837	*
838	* Return: 0 on success, negative error code on error.
839	*/
840	int xe_guc_init_post_hwconfig(struct xe_guc *guc)
841	{
842	int ret;
843
844	if (IS_SRIOV_VF(guc_to_xe(guc)))
845	return vf_guc_init_post_hwconfig(guc);
846
847	ret = xe_guc_realloc_post_hwconfig(guc);
848	if (ret)
849	return ret;
850
851	ret = xe_guc_ct_init_post_hwconfig(ct: &guc->ct);
852	if (ret)
853	return ret;
854
855	guc_init_params_post_hwconfig(guc);
856
857	ret = xe_guc_submit_init(guc, num_ids: ~0);
858	if (ret)
859	return ret;
860
861	ret = xe_guc_db_mgr_init(dbm: &guc->dbm, count: ~0);
862	if (ret)
863	return ret;
864
865	ret = xe_guc_pc_init(pc: &guc->pc);
866	if (ret)
867	return ret;
868
869	ret = xe_guc_engine_activity_init(guc);
870	if (ret)
871	return ret;
872
873	ret = xe_guc_buf_cache_init_with_size(cache: &guc->buf,
874	size: guc_additional_cache_size(xe: guc_to_xe(guc)));
875	if (ret)
876	return ret;
877
878	return xe_guc_ads_init_post_hwconfig(ads: &guc->ads);
879	}
880
881	int xe_guc_post_load_init(struct xe_guc *guc)
882	{
883	int ret;
884
885	xe_guc_ads_populate_post_load(ads: &guc->ads);
886
887	ret = xe_guc_opt_in_features_enable(guc);
888	if (ret)
889	return ret;
890
891	if (xe_guc_g2g_wanted(xe: guc_to_xe(guc))) {
892	ret = guc_g2g_start(guc);
893	if (ret)
894	return ret;
895	}
896
897	return xe_guc_submit_enable(guc);
898	}
899
900	int xe_guc_reset(struct xe_guc *guc)
901	{
902	struct xe_gt *gt = guc_to_gt(guc);
903	struct xe_mmio *mmio = &gt->mmio;
904	u32 guc_status, gdrst;
905	int ret;
906
907	xe_force_wake_assert_held(fw: gt_to_fw(gt), domain: XE_FW_GT);
908
909	if (IS_SRIOV_VF(gt_to_xe(gt)))
910	return xe_gt_sriov_vf_bootstrap(gt);
911
912	xe_mmio_write32(mmio, GDRST, GRDOM_GUC);
913
914	ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, val: 0, timeout_us: 5000, out_val: &gdrst, atomic: false);
915	if (ret) {
916	xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst);
917	goto err_out;
918	}
919
920	guc_status = xe_mmio_read32(mmio, GUC_STATUS);
921	if (!(guc_status & GS_MIA_IN_RESET)) {
922	xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n",
923	guc_status);
924	ret = -EIO;
925	goto err_out;
926	}
927
928	return 0;
929
930	err_out:
931
932	return ret;
933	}
934
935	static void guc_prepare_xfer(struct xe_guc *guc)
936	{
937	struct xe_gt *gt = guc_to_gt(guc);
938	struct xe_mmio *mmio = &gt->mmio;
939	struct xe_device *xe = guc_to_xe(guc);
940	u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC |
941	GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
942	GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
943	GUC_ENABLE_MIA_CLOCK_GATING;
944
945	if (GRAPHICS_VERx100(xe) < 1250)
946	shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES |
947	GUC_ENABLE_MIA_CACHING;
948
949	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
950	shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index);
951
952	/* Must program this register before loading the ucode with DMA */
953	xe_mmio_write32(mmio, GUC_SHIM_CONTROL, val: shim_flags);
954
955	xe_mmio_write32(mmio, GT_PM_CONFIG, GT_DOORBELL_ENABLE);
956
957	/* Make sure GuC receives ARAT interrupts */
958	xe_mmio_rmw32(mmio, PMINTRMSK, ARAT_EXPIRED_INTRMSK, set: 0);
959	}
960
961	/*
962	* Supporting MMIO & in memory RSA
963	*/
964	static int guc_xfer_rsa(struct xe_guc *guc)
965	{
966	struct xe_gt *gt = guc_to_gt(guc);
967	u32 rsa[UOS_RSA_SCRATCH_COUNT];
968	size_t copied;
969	int i;
970
971	if (guc->fw.rsa_size > 256) {
972	u32 rsa_ggtt_addr = xe_bo_ggtt_addr(bo: guc->fw.bo) +
973	xe_uc_fw_rsa_offset(uc_fw: &guc->fw);
974	xe_mmio_write32(mmio: &gt->mmio, UOS_RSA_SCRATCH(0), val: rsa_ggtt_addr);
975	return 0;
976	}
977
978	copied = xe_uc_fw_copy_rsa(uc_fw: &guc->fw, dst: rsa, max_len: sizeof(rsa));
979	if (copied < sizeof(rsa))
980	return -ENOMEM;
981
982	for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++)
983	xe_mmio_write32(mmio: &gt->mmio, UOS_RSA_SCRATCH(i), val: rsa[i]);
984
985	return 0;
986	}
987
988	/*
989	* Wait for the GuC to start up.
990	*
991	* Measurements indicate this should take no more than 20ms (assuming the GT
992	* clock is at maximum frequency). However, thermal throttling and other issues
993	* can prevent the clock hitting max and thus making the load take significantly
994	* longer. Allow up to 3s as a safety margin in normal builds. For
995	* CONFIG_DRM_XE_DEBUG allow up to 10s to account for slower execution, issues
996	* in PCODE, driver, fan, etc.
997	*
998	* Keep checking the GUC_STATUS every 10ms with a debug message every 100
999	* attempts as a "I'm slow, but alive" message. Regardless, if it takes more
1000	* than 200ms, emit a warning.
1001	*/
1002
1003	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
1004	#define GUC_LOAD_TIMEOUT_SEC 20
1005	#else
1006	#define GUC_LOAD_TIMEOUT_SEC 3
1007	#endif
1008	#define GUC_LOAD_TIME_WARN_MSEC 200
1009
1010	static void print_load_status_err(struct xe_gt *gt, u32 status)
1011	{
1012	struct xe_mmio *mmio = &gt->mmio;
1013	u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status);
1014	u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status);
1015
1016	xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
1017	REG_FIELD_GET(GS_MIA_IN_RESET, status),
1018	bootrom, ukernel,
1019	REG_FIELD_GET(GS_MIA_MASK, status),
1020	REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
1021
1022	switch (bootrom) {
1023	case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1024	xe_gt_err(gt, "invalid key requested, header = 0x%08X\n",
1025	xe_mmio_read32(mmio, GUC_HEADER_INFO));
1026	break;
1027	case XE_BOOTROM_STATUS_RSA_FAILED:
1028	xe_gt_err(gt, "firmware signature verification failed\n");
1029	break;
1030	case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1031	xe_gt_err(gt, "firmware production part check failure\n");
1032	break;
1033	}
1034
1035	switch (ukernel) {
1036	case XE_GUC_LOAD_STATUS_HWCONFIG_START:
1037	xe_gt_err(gt, "still extracting hwconfig table.\n");
1038	break;
1039	case XE_GUC_LOAD_STATUS_EXCEPTION:
1040	xe_gt_err(gt, "firmware exception. EIP: %#x\n",
1041	xe_mmio_read32(mmio, SOFT_SCRATCH(13)));
1042	break;
1043	case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
1044	xe_gt_err(gt, "illegal init/ADS data\n");
1045	break;
1046	case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1047	xe_gt_err(gt, "illegal register in save/restore workaround list\n");
1048	break;
1049	case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR:
1050	xe_gt_err(gt, "illegal workaround KLV data\n");
1051	break;
1052	case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG:
1053	xe_gt_err(gt, "illegal feature flag specified\n");
1054	break;
1055	}
1056	}
1057
1058	/*
1059	* Check GUC_STATUS looking for known terminal states (either completion or
1060	* failure) of either the microkernel status field or the boot ROM status field.
1061	*
1062	* Returns 1 for successful completion, -1 for failure and 0 for any
1063	* intermediate state.
1064	*/
1065	static int guc_load_done(struct xe_gt *gt, u32 *status, u32 *tries)
1066	{
1067	u32 ukernel, bootrom;
1068
1069	*status = xe_mmio_read32(mmio: &gt->mmio, GUC_STATUS);
1070	ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, *status);
1071	bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, *status);
1072
1073	switch (ukernel) {
1074	case XE_GUC_LOAD_STATUS_READY:
1075	return 1;
1076	case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH:
1077	case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH:
1078	case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE:
1079	case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR:
1080	case XE_GUC_LOAD_STATUS_BOOTROM_VERSION_MISMATCH:
1081	case XE_GUC_LOAD_STATUS_DPC_ERROR:
1082	case XE_GUC_LOAD_STATUS_EXCEPTION:
1083	case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
1084	case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID:
1085	case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1086	case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR:
1087	case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG:
1088	return -1;
1089	}
1090
1091	switch (bootrom) {
1092	case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1093	case XE_BOOTROM_STATUS_RSA_FAILED:
1094	case XE_BOOTROM_STATUS_PAVPC_FAILED:
1095	case XE_BOOTROM_STATUS_WOPCM_FAILED:
1096	case XE_BOOTROM_STATUS_LOADLOC_FAILED:
1097	case XE_BOOTROM_STATUS_JUMP_FAILED:
1098	case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED:
1099	case XE_BOOTROM_STATUS_MPUMAP_INCORRECT:
1100	case XE_BOOTROM_STATUS_EXCEPTION:
1101	case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1102	return -1;
1103	}
1104
1105	if (++*tries >= 100) {
1106	struct xe_guc_pc *guc_pc = &gt->uc.guc.pc;
1107
1108	*tries = 0;
1109	xe_gt_dbg(gt, "GuC load still in progress, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n",
1110	xe_guc_pc_get_act_freq(guc_pc),
1111	xe_guc_pc_get_cur_freq_fw(guc_pc),
1112	*status, ukernel, bootrom);
1113	}
1114
1115	return 0;
1116	}
1117
1118	static int guc_wait_ucode(struct xe_guc *guc)
1119	{
1120	struct xe_gt *gt = guc_to_gt(guc);
1121	struct xe_guc_pc *guc_pc = &gt->uc.guc.pc;
1122	u32 before_freq, act_freq, cur_freq;
1123	u32 status = 0, tries = 0;
1124	ktime_t before;
1125	u64 delta_ms;
1126	int ret;
1127
1128	before_freq = xe_guc_pc_get_act_freq(pc: guc_pc);
1129	before = ktime_get();
1130
1131	ret = poll_timeout_us(ret = guc_load_done(gt, &status, &tries), ret,
1132	10 * USEC_PER_MSEC,
1133	GUC_LOAD_TIMEOUT_SEC * USEC_PER_SEC, false);
1134
1135	delta_ms = ktime_to_ms(ktime_sub(ktime_get(), before));
1136	act_freq = xe_guc_pc_get_act_freq(pc: guc_pc);
1137	cur_freq = xe_guc_pc_get_cur_freq_fw(pc: guc_pc);
1138
1139	if (ret) {
1140	xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz)\n",
1141	status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
1142	xe_guc_pc_get_cur_freq_fw(guc_pc));
1143	print_load_status_err(gt, status);
1144
1145	return -EPROTO;
1146	}
1147
1148	if (delta_ms > GUC_LOAD_TIME_WARN_MSEC) {
1149	xe_gt_warn(gt, "GuC load: excessive init time: %lldms! [status = 0x%08X]\n",
1150	delta_ms, status);
1151	xe_gt_warn(gt, "GuC load: excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n",
1152	act_freq, cur_freq, before_freq,
1153	xe_gt_throttle_get_limit_reasons(gt));
1154	} else {
1155	xe_gt_dbg(gt, "GuC load: init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X\n",
1156	delta_ms, act_freq, cur_freq, before_freq, status);
1157	}
1158
1159	return 0;
1160	}
1161	ALLOW_ERROR_INJECTION(guc_wait_ucode, ERRNO);
1162
1163	static int __xe_guc_upload(struct xe_guc *guc)
1164	{
1165	int ret;
1166
1167	/* Raise GT freq to speed up HuC/GuC load */
1168	xe_guc_pc_raise_unslice(pc: &guc->pc);
1169
1170	guc_write_params(guc);
1171	guc_prepare_xfer(guc);
1172
1173	/*
1174	* Note that GuC needs the CSS header plus uKernel code to be copied
1175	* by the DMA engine in one operation, whereas the RSA signature is
1176	* loaded separately, either by copying it to the UOS_RSA_SCRATCH
1177	* register (if key size <= 256) or through a ggtt-pinned vma (if key
1178	* size > 256). The RSA size and therefore the way we provide it to the
1179	* HW is fixed for each platform and hard-coded in the bootrom.
1180	*/
1181	ret = guc_xfer_rsa(guc);
1182	if (ret)
1183	goto out;
1184	/*
1185	* Current uCode expects the code to be loaded at 8k; locations below
1186	* this are used for the stack.
1187	*/
1188	ret = xe_uc_fw_upload(uc_fw: &guc->fw, offset: 0x2000, UOS_MOVE);
1189	if (ret)
1190	goto out;
1191
1192	/* Wait for authentication */
1193	ret = guc_wait_ucode(guc);
1194	if (ret)
1195	goto out;
1196
1197	xe_uc_fw_change_status(uc_fw: &guc->fw, status: XE_UC_FIRMWARE_RUNNING);
1198	return 0;
1199
1200	out:
1201	xe_uc_fw_change_status(uc_fw: &guc->fw, status: XE_UC_FIRMWARE_LOAD_FAIL);
1202	return ret;
1203	}
1204
1205	static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc)
1206	{
1207	struct xe_gt *gt = guc_to_gt(guc);
1208	int ret;
1209
1210	ret = xe_guc_hwconfig_init(guc);
1211	if (ret)
1212	return ret;
1213
1214	ret = xe_guc_enable_communication(guc);
1215	if (ret)
1216	return ret;
1217
1218	ret = xe_gt_sriov_vf_connect(gt);
1219	if (ret)
1220	goto err_out;
1221
1222	ret = xe_gt_sriov_vf_query_runtime(gt);
1223	if (ret)
1224	goto err_out;
1225
1226	return 0;
1227
1228	err_out:
1229	xe_guc_sanitize(guc);
1230	return ret;
1231	}
1232
1233	/**
1234	* xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table
1235	* @guc: The GuC object
1236	*
1237	* This function uploads a minimal GuC that does not support submissions but
1238	* in a state where the hwconfig table can be read. Next, it reads and parses
1239	* the hwconfig table so it can be used for subsequent steps in the driver load.
1240	* Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only).
1241	*
1242	* Return: 0 on success, negative error code on error.
1243	*/
1244	int xe_guc_min_load_for_hwconfig(struct xe_guc *guc)
1245	{
1246	int ret;
1247
1248	if (IS_SRIOV_VF(guc_to_xe(guc)))
1249	return vf_guc_min_load_for_hwconfig(guc);
1250
1251	xe_guc_ads_populate_minimal(ads: &guc->ads);
1252
1253	xe_guc_pc_init_early(pc: &guc->pc);
1254
1255	ret = __xe_guc_upload(guc);
1256	if (ret)
1257	return ret;
1258
1259	ret = xe_guc_hwconfig_init(guc);
1260	if (ret)
1261	return ret;
1262
1263	ret = xe_guc_enable_communication(guc);
1264	if (ret)
1265	return ret;
1266
1267	return 0;
1268	}
1269
1270	int xe_guc_upload(struct xe_guc *guc)
1271	{
1272	struct xe_gt *gt = guc_to_gt(guc);
1273
1274	xe_guc_ads_populate(ads: &guc->ads);
1275
1276	if (xe_guc_using_main_gamctrl_queues(guc))
1277	xe_mmio_write32(mmio: &gt->mmio, MAIN_GAMCTRL_MODE, MAIN_GAMCTRL_QUEUE_SELECT);
1278
1279	return __xe_guc_upload(guc);
1280	}
1281
1282	static void guc_handle_mmio_msg(struct xe_guc *guc)
1283	{
1284	struct xe_gt *gt = guc_to_gt(guc);
1285	u32 msg;
1286
1287	if (IS_SRIOV_VF(guc_to_xe(guc)))
1288	return;
1289
1290	xe_force_wake_assert_held(fw: gt_to_fw(gt), domain: XE_FW_GT);
1291
1292	msg = xe_mmio_read32(mmio: &gt->mmio, SOFT_SCRATCH(15));
1293	msg &= XE_GUC_RECV_MSG_EXCEPTION |
1294	XE_GUC_RECV_MSG_CRASH_DUMP_POSTED;
1295	xe_mmio_write32(mmio: &gt->mmio, SOFT_SCRATCH(15), val: 0);
1296
1297	if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED)
1298	xe_gt_err(gt, "Received early GuC crash dump notification!\n");
1299
1300	if (msg & XE_GUC_RECV_MSG_EXCEPTION)
1301	xe_gt_err(gt, "Received early GuC exception notification!\n");
1302	}
1303
1304	static void guc_enable_irq(struct xe_guc *guc)
1305	{
1306	struct xe_gt *gt = guc_to_gt(guc);
1307	u32 events = xe_gt_is_media_type(gt) ?
1308	REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) :
1309	REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
1310
1311	/* Primary GuC and media GuC share a single enable bit */
1312	xe_mmio_write32(mmio: &gt->mmio, GUC_SG_INTR_ENABLE,
1313	REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST));
1314
1315	/*
1316	* There are separate mask bits for primary and media GuCs, so use
1317	* a RMW operation to avoid clobbering the other GuC's setting.
1318	*/
1319	xe_mmio_rmw32(mmio: &gt->mmio, GUC_SG_INTR_MASK, clr: events, set: 0);
1320	}
1321
1322	int xe_guc_enable_communication(struct xe_guc *guc)
1323	{
1324	struct xe_device *xe = guc_to_xe(guc);
1325	int err;
1326
1327	if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) {
1328	struct xe_gt *gt = guc_to_gt(guc);
1329	struct xe_tile *tile = gt_to_tile(gt);
1330
1331	err = xe_memirq_init_guc(memirq: &tile->memirq, guc);
1332	if (err)
1333	return err;
1334	} else {
1335	guc_enable_irq(guc);
1336	}
1337
1338	err = xe_guc_ct_enable(ct: &guc->ct);
1339	if (err)
1340	return err;
1341
1342	guc_handle_mmio_msg(guc);
1343
1344	return 0;
1345	}
1346
1347	int xe_guc_suspend(struct xe_guc *guc)
1348	{
1349	struct xe_gt *gt = guc_to_gt(guc);
1350	u32 action[] = {
1351	XE_GUC_ACTION_CLIENT_SOFT_RESET,
1352	};
1353	int ret;
1354
1355	ret = xe_guc_mmio_send(guc, request: action, ARRAY_SIZE(action));
1356	if (ret) {
1357	xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
1358	return ret;
1359	}
1360
1361	xe_guc_sanitize(guc);
1362	return 0;
1363	}
1364
1365	void xe_guc_notify(struct xe_guc *guc)
1366	{
1367	struct xe_gt *gt = guc_to_gt(guc);
1368	const u32 default_notify_data = 0;
1369
1370	/*
1371	* Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass
1372	* additional payload data to the GuC but this capability is not
1373	* used by the firmware yet. Use default value in the meantime.
1374	*/
1375	xe_mmio_write32(mmio: &gt->mmio, reg: guc->notify_reg, val: default_notify_data);
1376	}
1377
1378	int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr)
1379	{
1380	u32 action[] = {
1381	XE_GUC_ACTION_AUTHENTICATE_HUC,
1382	rsa_addr
1383	};
1384
1385	return xe_guc_ct_send_block(ct: &guc->ct, action, ARRAY_SIZE(action));
1386	}
1387
1388	int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
1389	u32 len, u32 *response_buf)
1390	{
1391	struct xe_device *xe = guc_to_xe(guc);
1392	struct xe_gt *gt = guc_to_gt(guc);
1393	struct xe_mmio *mmio = &gt->mmio;
1394	u32 header, reply;
1395	struct xe_reg reply_reg = xe_gt_is_media_type(gt) ?
1396	MED_VF_SW_FLAG(0) : VF_SW_FLAG(0);
1397	const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1;
1398	bool lost = false;
1399	int ret;
1400	int i;
1401
1402	BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT);
1403
1404	xe_assert(xe, len);
1405	xe_assert(xe, len <= VF_SW_FLAG_COUNT);
1406	xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT);
1407	xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) ==
1408	GUC_HXG_ORIGIN_HOST);
1409	xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) ==
1410	GUC_HXG_TYPE_REQUEST);
1411
1412	retry:
1413	/* Not in critical data-path, just do if else for GT type */
1414	if (xe_gt_is_media_type(gt)) {
1415	for (i = 0; i < len; ++i)
1416	xe_mmio_write32(mmio, MED_VF_SW_FLAG(i),
1417	val: request[i]);
1418	xe_mmio_read32(mmio, MED_VF_SW_FLAG(LAST_INDEX));
1419	} else {
1420	for (i = 0; i < len; ++i)
1421	xe_mmio_write32(mmio, VF_SW_FLAG(i),
1422	val: request[i]);
1423	xe_mmio_read32(mmio, VF_SW_FLAG(LAST_INDEX));
1424	}
1425
1426	xe_guc_notify(guc);
1427
1428	ret = xe_mmio_wait32(mmio, reg: reply_reg, GUC_HXG_MSG_0_ORIGIN,
1429	FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC),
1430	timeout_us: 50000, out_val: &reply, atomic: false);
1431	if (ret) {
1432	/* scratch registers might be cleared during FLR, try once more */
1433	if (!reply && !lost) {
1434	xe_gt_dbg(gt, "GuC mmio request %#x: lost, trying again\n", request[0]);
1435	lost = true;
1436	goto retry;
1437	}
1438	timeout:
1439	xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
1440	request[0], reply);
1441	return ret;
1442	}
1443
1444	header = xe_mmio_read32(mmio, reg: reply_reg);
1445	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1446	GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
1447	/*
1448	* Once we got a BUSY reply we must wait again for the final
1449	* response but this time we can't use ORIGIN mask anymore.
1450	* To spot a right change in the reply, we take advantage that
1451	* response SUCCESS and FAILURE differ only by the single bit
1452	* and all other bits are set and can be used as a new mask.
1453	*/
1454	u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE;
1455	u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits);
1456
1457	BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS);
1458	BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1);
1459
1460	ret = xe_mmio_wait32(mmio, reg: reply_reg, mask: resp_mask, val: resp_mask,
1461	timeout_us: 2000000, out_val: &header, atomic: false);
1462
1463	if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) !=
1464	GUC_HXG_ORIGIN_GUC))
1465	goto proto;
1466	if (unlikely(ret)) {
1467	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1468	GUC_HXG_TYPE_NO_RESPONSE_BUSY)
1469	goto proto;
1470	goto timeout;
1471	}
1472	}
1473
1474	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1475	GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1476	u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header);
1477
1478	xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
1479	request[0], reason);
1480	goto retry;
1481	}
1482
1483	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1484	GUC_HXG_TYPE_RESPONSE_FAILURE) {
1485	u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
1486	u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
1487
1488	xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
1489	request[0], error, hint);
1490	return -ENXIO;
1491	}
1492
1493	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1494	GUC_HXG_TYPE_RESPONSE_SUCCESS) {
1495	proto:
1496	xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
1497	request[0], header);
1498	return -EPROTO;
1499	}
1500
1501	/* Just copy entire possible message response */
1502	if (response_buf) {
1503	response_buf[0] = header;
1504
1505	for (i = 1; i < VF_SW_FLAG_COUNT; i++) {
1506	reply_reg.addr += sizeof(u32);
1507	response_buf[i] = xe_mmio_read32(mmio, reg: reply_reg);
1508	}
1509	}
1510
1511	/* Use data from the GuC response as our return value */
1512	return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header);
1513	}
1514	ALLOW_ERROR_INJECTION(xe_guc_mmio_send_recv, ERRNO);
1515
1516	int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len)
1517	{
1518	return xe_guc_mmio_send_recv(guc, request, len, NULL);
1519	}
1520
1521	static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val)
1522	{
1523	struct xe_device *xe = guc_to_xe(guc);
1524	u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = {
1525	FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
1526	FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
1527	FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
1528	GUC_ACTION_HOST2GUC_SELF_CFG),
1529	FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) |
1530	FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len),
1531	FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32,
1532	lower_32_bits(val)),
1533	FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64,
1534	upper_32_bits(val)),
1535	};
1536	int ret;
1537
1538	xe_assert(xe, len <= 2);
1539	xe_assert(xe, len != 1 || !upper_32_bits(val));
1540
1541	/* Self config must go over MMIO */
1542	ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
1543
1544	if (unlikely(ret < 0))
1545	return ret;
1546	if (unlikely(ret > 1))
1547	return -EPROTO;
1548	if (unlikely(!ret))
1549	return -ENOKEY;
1550
1551	return 0;
1552	}
1553
1554	int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val)
1555	{
1556	return guc_self_cfg(guc, key, len: 1, val);
1557	}
1558
1559	int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val)
1560	{
1561	return guc_self_cfg(guc, key, len: 2, val);
1562	}
1563
1564	static void xe_guc_sw_0_irq_handler(struct xe_guc *guc)
1565	{
1566	struct xe_gt *gt = guc_to_gt(guc);
1567
1568	if (IS_SRIOV_VF(gt_to_xe(gt)))
1569	xe_gt_sriov_vf_migrated_event_handler(gt);
1570	}
1571
1572	void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir)
1573	{
1574	if (iir & GUC_INTR_GUC2HOST)
1575	xe_guc_ct_irq_handler(ct: &guc->ct);
1576
1577	if (iir & GUC_INTR_SW_INT_0)
1578	xe_guc_sw_0_irq_handler(guc);
1579	}
1580
1581	void xe_guc_sanitize(struct xe_guc *guc)
1582	{
1583	xe_uc_fw_sanitize(uc_fw: &guc->fw);
1584	xe_guc_ct_disable(ct: &guc->ct);
1585	xe_guc_submit_disable(guc);
1586	}
1587
1588	int xe_guc_reset_prepare(struct xe_guc *guc)
1589	{
1590	return xe_guc_submit_reset_prepare(guc);
1591	}
1592
1593	void xe_guc_reset_wait(struct xe_guc *guc)
1594	{
1595	xe_guc_submit_reset_wait(guc);
1596	}
1597
1598	void xe_guc_stop_prepare(struct xe_guc *guc)
1599	{
1600	if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1601	int err;
1602
1603	err = xe_guc_pc_stop(pc: &guc->pc);
1604	xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n",
1605	ERR_PTR(err));
1606	}
1607	}
1608
1609	void xe_guc_stop(struct xe_guc *guc)
1610	{
1611	xe_guc_ct_stop(ct: &guc->ct);
1612
1613	xe_guc_submit_stop(guc);
1614	}
1615
1616	int xe_guc_start(struct xe_guc *guc)
1617	{
1618	return xe_guc_submit_start(guc);
1619	}
1620
1621	int xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
1622	{
1623	struct xe_gt *gt = guc_to_gt(guc);
1624	unsigned int fw_ref;
1625	u32 status;
1626	int i;
1627
1628	xe_uc_fw_print(uc_fw: &guc->fw, p);
1629
1630	if (!IS_SRIOV_VF(gt_to_xe(gt))) {
1631	fw_ref = xe_force_wake_get(fw: gt_to_fw(gt), domains: XE_FW_GT);
1632	if (!fw_ref)
1633	return -EIO;
1634
1635	status = xe_mmio_read32(mmio: &gt->mmio, GUC_STATUS);
1636
1637	drm_printf(p, f: "\nGuC status 0x%08x:\n", status);
1638	drm_printf(p, f: "\tBootrom status = 0x%x\n",
1639	REG_FIELD_GET(GS_BOOTROM_MASK, status));
1640	drm_printf(p, f: "\tuKernel status = 0x%x\n",
1641	REG_FIELD_GET(GS_UKERNEL_MASK, status));
1642	drm_printf(p, f: "\tMIA Core status = 0x%x\n",
1643	REG_FIELD_GET(GS_MIA_MASK, status));
1644	drm_printf(p, f: "\tLog level = %d\n",
1645	xe_guc_log_get_level(log: &guc->log));
1646
1647	drm_puts(p, str: "\nScratch registers:\n");
1648	for (i = 0; i < SOFT_SCRATCH_COUNT; i++) {
1649	drm_printf(p, f: "\t%2d: \t0x%x\n",
1650	i, xe_mmio_read32(mmio: &gt->mmio, SOFT_SCRATCH(i)));
1651	}
1652
1653	xe_force_wake_put(fw: gt_to_fw(gt), fw_ref);
1654	}
1655
1656	drm_puts(p, str: "\n");
1657	xe_guc_ct_print(ct: &guc->ct, p, want_ctb: false);
1658
1659	drm_puts(p, str: "\n");
1660	xe_guc_submit_print(guc, p);
1661
1662	return 0;
1663	}
1664
1665	/**
1666	* xe_guc_declare_wedged() - Declare GuC wedged
1667	* @guc: the GuC object
1668	*
1669	* Wedge the GuC which stops all submission, saves desired debug state, and
1670	* cleans up anything which could timeout.
1671	*/
1672	void xe_guc_declare_wedged(struct xe_guc *guc)
1673	{
1674	xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
1675
1676	xe_guc_reset_prepare(guc);
1677	xe_guc_ct_stop(ct: &guc->ct);
1678	xe_guc_submit_wedge(guc);
1679	}
1680
1681	/**
1682	* xe_guc_using_main_gamctrl_queues() - Detect which reporting queues to use.
1683	* @guc: The GuC object
1684	*
1685	* For Xe3p and beyond, we want to program the hardware to use the
1686	* "Main GAMCTRL queue" rather than the legacy queue before we upload
1687	* the GuC firmware. This will allow the GuC to use a new set of
1688	* registers for pagefault handling and avoid some unnecessary
1689	* complications with MCR register range handling.
1690	*
1691	* Return: true if can use new main gamctrl queues.
1692	*/
1693	bool xe_guc_using_main_gamctrl_queues(struct xe_guc *guc)
1694	{
1695	struct xe_gt *gt = guc_to_gt(guc);
1696
1697	/*
1698	* For Xe3p media gt (35), the GuC and the CS subunits may be still Xe3
1699	* that lacks the Main GAMCTRL support. Reserved bits from the GMD_ID
1700	* inform the IP version of the subunits.
1701	*/
1702	if (xe_gt_is_media_type(gt) && MEDIA_VER(gt_to_xe(gt)) == 35) {
1703	u32 val = xe_mmio_read32(mmio: &gt->mmio, GMD_ID);
1704	u32 subip = REG_FIELD_GET(GMD_ID_SUBIP_FLAG_MASK, val);
1705
1706	if (!subip)
1707	return true;
1708
1709	xe_gt_WARN(gt, subip != 1,
1710	"GMD_ID has unknown value in the SUBIP_FLAG field - 0x%x\n",
1711	subip);
1712
1713	return false;
1714	}
1715
1716	return GT_VER(gt) >= 35;
1717	}
1718
1719	#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1720	#include "tests/xe_guc_g2g_test.c"
1721	#endif
1722