1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_ring_ops.h"
7
8#include <generated/xe_wa_oob.h>
9
10#include "instructions/xe_gpu_commands.h"
11#include "instructions/xe_mi_commands.h"
12#include "regs/xe_engine_regs.h"
13#include "regs/xe_gt_regs.h"
14#include "regs/xe_lrc_layout.h"
15#include "xe_exec_queue_types.h"
16#include "xe_gt.h"
17#include "xe_lrc.h"
18#include "xe_macros.h"
19#include "xe_sched_job.h"
20#include "xe_sriov.h"
21#include "xe_vm_types.h"
22#include "xe_vm.h"
23#include "xe_wa.h"
24
25/*
26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D
27 * pipeline (i.e., CCS engines).
28 */
29#define PIPE_CONTROL_3D_ENGINE_FLAGS (\
30 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
31 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
32 PIPE_CONTROL_TILE_CACHE_FLUSH | \
33 PIPE_CONTROL_DEPTH_STALL | \
34 PIPE_CONTROL_STALL_AT_SCOREBOARD | \
35 PIPE_CONTROL_PSD_SYNC | \
36 PIPE_CONTROL_AMFS_FLUSH | \
37 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
38 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
39
40/* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
41#define PIPE_CONTROL_3D_ARCH_FLAGS ( \
42 PIPE_CONTROL_3D_ENGINE_FLAGS | \
43 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
44 PIPE_CONTROL_FLUSH_ENABLE | \
45 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
46 PIPE_CONTROL_DC_FLUSH_ENABLE)
47
48static u32 preparser_disable(bool state)
49{
50 return MI_ARB_CHECK | BIT(8) | state;
51}
52
53static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
54 u32 *dw, int i)
55{
56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 dw[i++] = reg.addr + gt->mmio.adj_offset;
58 dw[i++] = AUX_INV;
59 dw[i++] = MI_NOOP;
60
61 return i;
62}
63
64static int emit_user_interrupt(u32 *dw, int i)
65{
66 dw[i++] = MI_USER_INTERRUPT;
67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 dw[i++] = MI_ARB_CHECK;
69
70 return i;
71}
72
73static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
74{
75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 dw[i++] = addr;
77 dw[i++] = 0;
78 dw[i++] = value;
79
80 return i;
81}
82
83static int emit_flush_dw(u32 *dw, int i)
84{
85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
86 dw[i++] = 0;
87 dw[i++] = 0;
88 dw[i++] = 0;
89
90 return i;
91}
92
93static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)
94{
95 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
96 flags;
97 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
98 dw[i++] = 0;
99 dw[i++] = value;
100
101 return i;
102}
103
104static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
105{
106 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
107 dw[i++] = lower_32_bits(batch_addr);
108 dw[i++] = upper_32_bits(batch_addr);
109
110 return i;
111}
112
113static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i)
114{
115 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW |
116 MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0;
117
118 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
119 dw[i++] = 0;
120 dw[i++] = val;
121
122 return i;
123}
124
125static int
126emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
127{
128 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
129 dw[i++] = bit_group_1;
130 dw[i++] = offset;
131 dw[i++] = 0;
132 dw[i++] = value;
133 dw[i++] = 0;
134
135 return i;
136}
137
138static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
139 int i)
140{
141 u32 flags0 = 0;
142 u32 flags1 = PIPE_CONTROL_CS_STALL |
143 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
144 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
145 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
146 PIPE_CONTROL_VF_CACHE_INVALIDATE |
147 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
148 PIPE_CONTROL_STATE_CACHE_INVALIDATE |
149 PIPE_CONTROL_QW_WRITE |
150 PIPE_CONTROL_STORE_DATA_INDEX;
151
152 if (invalidate_tlb)
153 flags1 |= PIPE_CONTROL_TLB_INVALIDATE;
154
155 flags1 &= ~mask_flags;
156
157 if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
158 flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE;
159
160 return emit_pipe_control(dw, i, bit_group_0: flags0, bit_group_1: flags1,
161 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, value: 0);
162}
163
164static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
165 u32 *dw, int i)
166{
167 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
168 dw[i++] = lower_32_bits(addr);
169 dw[i++] = upper_32_bits(addr);
170 dw[i++] = lower_32_bits(value);
171 dw[i++] = upper_32_bits(value);
172
173 return i;
174}
175
176static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
177{
178 struct xe_gt *gt = job->q->gt;
179 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
180 u32 flags;
181
182 if (XE_GT_WA(gt, 14016712196))
183 i = emit_pipe_control(dw, i, bit_group_0: 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
184 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, value: 0);
185
186 flags = (PIPE_CONTROL_CS_STALL |
187 PIPE_CONTROL_TILE_CACHE_FLUSH |
188 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
189 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
190 PIPE_CONTROL_DC_FLUSH_ENABLE |
191 PIPE_CONTROL_FLUSH_ENABLE);
192
193 if (XE_GT_WA(gt, 1409600907))
194 flags |= PIPE_CONTROL_DEPTH_STALL;
195
196 if (lacks_render)
197 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
198 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
199 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
200
201 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, bit_group_1: flags, offset: 0, value: 0);
202}
203
204static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
205{
206 if (hwe->class != XE_ENGINE_CLASS_RENDER)
207 return i;
208
209 if (XE_GT_WA(hwe->gt, 16020292621))
210 i = emit_pipe_control(dw, i, bit_group_0: 0, PIPE_CONTROL_LRI_POST_SYNC,
211 RING_NOPID(hwe->mmio_base).addr, value: 0);
212
213 return i;
214}
215
216static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
217 int i)
218{
219 u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
220 PIPE_CONTROL_QW_WRITE;
221
222 if (!stall_only)
223 flags |= PIPE_CONTROL_FLUSH_ENABLE;
224
225 return emit_pipe_control(dw, i, bit_group_0: 0, bit_group_1: flags, offset: addr, value);
226}
227
228static u32 get_ppgtt_flag(struct xe_sched_job *job)
229{
230 if (job->q->vm && !job->ggtt)
231 return BIT(8);
232
233 return 0;
234}
235
236static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
237{
238 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
239 dw[i++] = RING_CTX_TIMESTAMP(0).addr;
240 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
241 dw[i++] = 0;
242
243 return i;
244}
245
246/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
247static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
248 u64 batch_addr, u32 *head, u32 seqno)
249{
250 u32 dw[MAX_JOB_SIZE_DW], i = 0;
251 u32 ppgtt_flag = get_ppgtt_flag(job);
252 struct xe_gt *gt = job->q->gt;
253
254 *head = lrc->ring.tail;
255
256 i = emit_copy_timestamp(lrc, dw, i);
257
258 if (job->ring_ops_flush_tlb) {
259 dw[i++] = preparser_disable(state: true);
260 i = emit_flush_imm_ggtt(addr: xe_lrc_start_seqno_ggtt_addr(lrc),
261 value: seqno, MI_INVALIDATE_TLB, dw, i);
262 dw[i++] = preparser_disable(state: false);
263 } else {
264 i = emit_store_imm_ggtt(addr: xe_lrc_start_seqno_ggtt_addr(lrc),
265 value: seqno, dw, i);
266 }
267
268 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
269
270 if (job->user_fence.used) {
271 i = emit_flush_dw(dw, i);
272 i = emit_store_imm_ppgtt_posted(addr: job->user_fence.addr,
273 value: job->user_fence.value,
274 dw, i);
275 }
276
277 i = emit_flush_imm_ggtt(addr: xe_lrc_seqno_ggtt_addr(lrc), value: seqno, flags: 0, dw, i);
278
279 i = emit_user_interrupt(dw, i);
280
281 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
282
283 xe_lrc_write_ring(lrc, data: dw, size: i * sizeof(*dw));
284}
285
286static bool has_aux_ccs(struct xe_device *xe)
287{
288 /*
289 * PVC is a special case that has no compression of either type
290 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2
291 * onward, so any future platforms with no FlatCCS will not have
292 * AuxCCS either.
293 */
294 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
295 return false;
296
297 return !xe->info.has_flat_ccs;
298}
299
300static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
301 u64 batch_addr, u32 *head, u32 seqno)
302{
303 u32 dw[MAX_JOB_SIZE_DW], i = 0;
304 u32 ppgtt_flag = get_ppgtt_flag(job);
305 struct xe_gt *gt = job->q->gt;
306 struct xe_device *xe = gt_to_xe(gt);
307 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
308
309 *head = lrc->ring.tail;
310
311 i = emit_copy_timestamp(lrc, dw, i);
312
313 dw[i++] = preparser_disable(state: true);
314
315 /* hsdes: 1809175790 */
316 if (has_aux_ccs(xe)) {
317 if (decode)
318 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
319 else
320 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
321 }
322
323 if (job->ring_ops_flush_tlb)
324 i = emit_flush_imm_ggtt(addr: xe_lrc_start_seqno_ggtt_addr(lrc),
325 value: seqno, MI_INVALIDATE_TLB, dw, i);
326
327 dw[i++] = preparser_disable(state: false);
328
329 if (!job->ring_ops_flush_tlb)
330 i = emit_store_imm_ggtt(addr: xe_lrc_start_seqno_ggtt_addr(lrc),
331 value: seqno, dw, i);
332
333 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
334
335 if (job->user_fence.used) {
336 i = emit_flush_dw(dw, i);
337 i = emit_store_imm_ppgtt_posted(addr: job->user_fence.addr,
338 value: job->user_fence.value,
339 dw, i);
340 }
341
342 i = emit_flush_imm_ggtt(addr: xe_lrc_seqno_ggtt_addr(lrc), value: seqno, flags: 0, dw, i);
343
344 i = emit_user_interrupt(dw, i);
345
346 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
347
348 xe_lrc_write_ring(lrc, data: dw, size: i * sizeof(*dw));
349}
350
351static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
352 struct xe_lrc *lrc,
353 u64 batch_addr, u32 *head,
354 u32 seqno)
355{
356 u32 dw[MAX_JOB_SIZE_DW], i = 0;
357 u32 ppgtt_flag = get_ppgtt_flag(job);
358 struct xe_gt *gt = job->q->gt;
359 struct xe_device *xe = gt_to_xe(gt);
360 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
361 u32 mask_flags = 0;
362
363 *head = lrc->ring.tail;
364
365 i = emit_copy_timestamp(lrc, dw, i);
366
367 dw[i++] = preparser_disable(state: true);
368 if (lacks_render)
369 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
370 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
371 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
372
373 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
374 i = emit_pipe_invalidate(mask_flags, invalidate_tlb: job->ring_ops_flush_tlb, dw, i);
375
376 /* hsdes: 1809175790 */
377 if (has_aux_ccs(xe))
378 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
379
380 dw[i++] = preparser_disable(state: false);
381
382 i = emit_store_imm_ggtt(addr: xe_lrc_start_seqno_ggtt_addr(lrc),
383 value: seqno, dw, i);
384
385 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
386
387 i = emit_render_cache_flush(job, dw, i);
388
389 if (job->user_fence.used)
390 i = emit_store_imm_ppgtt_posted(addr: job->user_fence.addr,
391 value: job->user_fence.value,
392 dw, i);
393
394 i = emit_pipe_imm_ggtt(addr: xe_lrc_seqno_ggtt_addr(lrc), value: seqno, stall_only: lacks_render, dw, i);
395
396 i = emit_user_interrupt(dw, i);
397
398 i = emit_pipe_control_to_ring_end(hwe: job->q->hwe, dw, i);
399
400 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
401
402 xe_lrc_write_ring(lrc, data: dw, size: i * sizeof(*dw));
403}
404
405static void emit_migration_job_gen12(struct xe_sched_job *job,
406 struct xe_lrc *lrc, u32 *head,
407 u32 seqno)
408{
409 u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc);
410 u32 dw[MAX_JOB_SIZE_DW], i = 0;
411
412 *head = lrc->ring.tail;
413
414 i = emit_copy_timestamp(lrc, dw, i);
415
416 i = emit_store_imm_ggtt(addr: saddr, value: seqno, dw, i);
417
418 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
419
420 i = emit_bb_start(batch_addr: job->ptrs[0].batch_addr, BIT(8), dw, i);
421
422 dw[i++] = preparser_disable(state: true);
423 i = emit_flush_invalidate(addr: saddr, val: seqno, flush_flags: job->migrate_flush_flags, dw, i);
424 dw[i++] = preparser_disable(state: false);
425
426 i = emit_bb_start(batch_addr: job->ptrs[1].batch_addr, BIT(8), dw, i);
427
428 i = emit_flush_imm_ggtt(addr: xe_lrc_seqno_ggtt_addr(lrc), value: seqno,
429 flags: job->migrate_flush_flags,
430 dw, i);
431
432 i = emit_user_interrupt(dw, i);
433
434 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
435
436 xe_lrc_write_ring(lrc, data: dw, size: i * sizeof(*dw));
437}
438
439static void emit_job_gen12_gsc(struct xe_sched_job *job)
440{
441 struct xe_gt *gt = job->q->gt;
442
443 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
444
445 __emit_job_gen12_simple(job, lrc: job->q->lrc[0],
446 batch_addr: job->ptrs[0].batch_addr,
447 head: &job->ptrs[0].head,
448 seqno: xe_sched_job_lrc_seqno(job));
449}
450
451static void emit_job_gen12_copy(struct xe_sched_job *job)
452{
453 int i;
454
455 if (xe_sched_job_is_migration(q: job->q)) {
456 emit_migration_job_gen12(job, lrc: job->q->lrc[0],
457 head: &job->ptrs[0].head,
458 seqno: xe_sched_job_lrc_seqno(job));
459 return;
460 }
461
462 for (i = 0; i < job->q->width; ++i)
463 __emit_job_gen12_simple(job, lrc: job->q->lrc[i],
464 batch_addr: job->ptrs[i].batch_addr,
465 head: &job->ptrs[i].head,
466 seqno: xe_sched_job_lrc_seqno(job));
467}
468
469static void emit_job_gen12_video(struct xe_sched_job *job)
470{
471 int i;
472
473 /* FIXME: Not doing parallel handshake for now */
474 for (i = 0; i < job->q->width; ++i)
475 __emit_job_gen12_video(job, lrc: job->q->lrc[i],
476 batch_addr: job->ptrs[i].batch_addr,
477 head: &job->ptrs[i].head,
478 seqno: xe_sched_job_lrc_seqno(job));
479}
480
481static void emit_job_gen12_render_compute(struct xe_sched_job *job)
482{
483 int i;
484
485 for (i = 0; i < job->q->width; ++i)
486 __emit_job_gen12_render_compute(job, lrc: job->q->lrc[i],
487 batch_addr: job->ptrs[i].batch_addr,
488 head: &job->ptrs[i].head,
489 seqno: xe_sched_job_lrc_seqno(job));
490}
491
492static const struct xe_ring_ops ring_ops_gen12_gsc = {
493 .emit_job = emit_job_gen12_gsc,
494};
495
496static const struct xe_ring_ops ring_ops_gen12_copy = {
497 .emit_job = emit_job_gen12_copy,
498};
499
500static const struct xe_ring_ops ring_ops_gen12_video = {
501 .emit_job = emit_job_gen12_video,
502};
503
504static const struct xe_ring_ops ring_ops_gen12_render_compute = {
505 .emit_job = emit_job_gen12_render_compute,
506};
507
508const struct xe_ring_ops *
509xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
510{
511 switch (class) {
512 case XE_ENGINE_CLASS_OTHER:
513 return &ring_ops_gen12_gsc;
514 case XE_ENGINE_CLASS_COPY:
515 return &ring_ops_gen12_copy;
516 case XE_ENGINE_CLASS_VIDEO_DECODE:
517 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
518 return &ring_ops_gen12_video;
519 case XE_ENGINE_CLASS_RENDER:
520 case XE_ENGINE_CLASS_COMPUTE:
521 return &ring_ops_gen12_render_compute;
522 default:
523 return NULL;
524 }
525}
526

source code of linux/drivers/gpu/drm/xe/xe_ring_ops.c