1// SPDX-License-Identifier: MIT
2/*
3 * Copyright 2023 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/firmware.h>
26#include <linux/module.h>
27#include <linux/debugfs.h>
28#include <drm/drm_exec.h>
29#include <drm/drm_drv.h>
30
31#include "amdgpu.h"
32#include "amdgpu_umsch_mm.h"
33#include "umsch_mm_v4_0.h"
34
35MODULE_FIRMWARE("amdgpu/umsch_mm_4_0_0.bin");
36
37int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws)
38{
39 struct amdgpu_ring *ring = &umsch->ring;
40
41 if (amdgpu_ring_alloc(ring, ndw: ndws))
42 return -ENOMEM;
43
44 amdgpu_ring_write_multiple(ring, src: pkt, count_dw: ndws);
45 amdgpu_ring_commit(ring);
46
47 return 0;
48}
49
50int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch)
51{
52 struct amdgpu_ring *ring = &umsch->ring;
53 struct amdgpu_device *adev = ring->adev;
54 int r;
55
56 r = amdgpu_fence_wait_polling(ring, wait_seq: ring->fence_drv.sync_seq, timeout: adev->usec_timeout);
57 if (r < 1) {
58 dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n",
59 ring->fence_drv.sync_seq);
60 return -ETIMEDOUT;
61 }
62
63 return 0;
64}
65
66static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring)
67{
68 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
69 struct amdgpu_device *adev = ring->adev;
70
71 if (ring->use_doorbell)
72 WDOORBELL32(ring->doorbell_index, ring->wptr << 2);
73 else
74 WREG32(umsch->rb_wptr, ring->wptr << 2);
75}
76
77static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring)
78{
79 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
80 struct amdgpu_device *adev = ring->adev;
81
82 return RREG32(umsch->rb_rptr);
83}
84
85static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring)
86{
87 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
88 struct amdgpu_device *adev = ring->adev;
89
90 return RREG32(umsch->rb_wptr);
91}
92
93static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = {
94 .type = AMDGPU_RING_TYPE_UMSCH_MM,
95 .align_mask = 0,
96 .nop = 0,
97 .support_64bit_ptrs = false,
98 .get_rptr = umsch_mm_ring_get_rptr,
99 .get_wptr = umsch_mm_ring_get_wptr,
100 .set_wptr = umsch_mm_ring_set_wptr,
101 .insert_nop = amdgpu_ring_insert_nop,
102};
103
104int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch)
105{
106 struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);
107 struct amdgpu_ring *ring = &umsch->ring;
108
109 ring->vm_hub = AMDGPU_MMHUB0(0);
110 ring->use_doorbell = true;
111 ring->no_scheduler = true;
112 ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6;
113
114 snprintf(buf: ring->name, size: sizeof(ring->name), fmt: "umsch");
115
116 return amdgpu_ring_init(adev, ring, max_dw: 1024, NULL, irq_type: 0, hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
117}
118
119int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch)
120{
121 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
122 struct amdgpu_device *adev = umsch->ring.adev;
123 const char *fw_name = NULL;
124 int r;
125
126 switch (amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0)) {
127 case IP_VERSION(4, 0, 5):
128 case IP_VERSION(4, 0, 6):
129 fw_name = "4_0_0";
130 break;
131 default:
132 return -EINVAL;
133 }
134
135 r = amdgpu_ucode_request(adev, fw: &adev->umsch_mm.fw, required: AMDGPU_UCODE_REQUIRED,
136 fmt: "amdgpu/umsch_mm_%s.bin", fw_name);
137 if (r) {
138 release_firmware(fw: adev->umsch_mm.fw);
139 adev->umsch_mm.fw = NULL;
140 return r;
141 }
142
143 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data;
144
145 adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
146 adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
147
148 adev->umsch_mm.irq_start_addr =
149 le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) |
150 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32);
151 adev->umsch_mm.uc_start_addr =
152 le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) |
153 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32);
154 adev->umsch_mm.data_start_addr =
155 le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) |
156 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32);
157
158 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
159 struct amdgpu_firmware_info *info;
160
161 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE];
162 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE;
163 info->fw = adev->umsch_mm.fw;
164 adev->firmware.fw_size +=
165 ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE);
166
167 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA];
168 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA;
169 info->fw = adev->umsch_mm.fw;
170 adev->firmware.fw_size +=
171 ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE);
172 }
173
174 return 0;
175}
176
177int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch)
178{
179 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
180 struct amdgpu_device *adev = umsch->ring.adev;
181 const __le32 *fw_data;
182 uint32_t fw_size;
183 int r;
184
185 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
186 adev->umsch_mm.fw->data;
187
188 fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
189 le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes));
190 fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
191
192 r = amdgpu_bo_create_reserved(adev, size: fw_size,
193 align: 4 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
194 bo_ptr: &adev->umsch_mm.ucode_fw_obj,
195 gpu_addr: &adev->umsch_mm.ucode_fw_gpu_addr,
196 cpu_addr: (void **)&adev->umsch_mm.ucode_fw_ptr);
197 if (r) {
198 dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r);
199 return r;
200 }
201
202 memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size);
203
204 amdgpu_bo_kunmap(bo: adev->umsch_mm.ucode_fw_obj);
205 amdgpu_bo_unreserve(bo: adev->umsch_mm.ucode_fw_obj);
206 return 0;
207}
208
209int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch)
210{
211 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
212 struct amdgpu_device *adev = umsch->ring.adev;
213 const __le32 *fw_data;
214 uint32_t fw_size;
215 int r;
216
217 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
218 adev->umsch_mm.fw->data;
219
220 fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
221 le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes));
222 fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
223
224 r = amdgpu_bo_create_reserved(adev, size: fw_size,
225 align: 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
226 bo_ptr: &adev->umsch_mm.data_fw_obj,
227 gpu_addr: &adev->umsch_mm.data_fw_gpu_addr,
228 cpu_addr: (void **)&adev->umsch_mm.data_fw_ptr);
229 if (r) {
230 dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r);
231 return r;
232 }
233
234 memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size);
235
236 amdgpu_bo_kunmap(bo: adev->umsch_mm.data_fw_obj);
237 amdgpu_bo_unreserve(bo: adev->umsch_mm.data_fw_obj);
238 return 0;
239}
240
241int amdgpu_umsch_mm_psp_execute_cmd_buf(struct amdgpu_umsch_mm *umsch)
242{
243 struct amdgpu_device *adev = umsch->ring.adev;
244 struct amdgpu_firmware_info ucode = {
245 .ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER,
246 .mc_addr = adev->umsch_mm.cmd_buf_gpu_addr,
247 .ucode_size = ((uintptr_t)adev->umsch_mm.cmd_buf_curr_ptr -
248 (uintptr_t)adev->umsch_mm.cmd_buf_ptr),
249 };
250
251 return psp_execute_ip_fw_load(psp: &adev->psp, ucode: &ucode);
252}
253
254static void umsch_mm_agdb_index_init(struct amdgpu_device *adev)
255{
256 uint32_t umsch_mm_agdb_start;
257 int i;
258
259 umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1;
260 umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024);
261 umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1);
262
263 for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++)
264 adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i;
265}
266
267static int umsch_mm_init(struct amdgpu_device *adev)
268{
269 int r;
270
271 adev->umsch_mm.vmid_mask_mm_vpe = 0xf00;
272 adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE);
273 adev->umsch_mm.vpe_hqd_mask = 0xfe;
274
275 r = amdgpu_device_wb_get(adev, wb: &adev->umsch_mm.wb_index);
276 if (r) {
277 dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r);
278 return r;
279 }
280
281 adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr +
282 (adev->umsch_mm.wb_index * 4);
283
284 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
285 AMDGPU_GEM_DOMAIN_GTT,
286 bo_ptr: &adev->umsch_mm.cmd_buf_obj,
287 gpu_addr: &adev->umsch_mm.cmd_buf_gpu_addr,
288 cpu_addr: (void **)&adev->umsch_mm.cmd_buf_ptr);
289 if (r) {
290 dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r);
291 amdgpu_device_wb_free(adev, wb: adev->umsch_mm.wb_index);
292 return r;
293 }
294
295 r = amdgpu_bo_create_kernel(adev, AMDGPU_UMSCHFW_LOG_SIZE, PAGE_SIZE,
296 AMDGPU_GEM_DOMAIN_VRAM |
297 AMDGPU_GEM_DOMAIN_GTT,
298 bo_ptr: &adev->umsch_mm.dbglog_bo,
299 gpu_addr: &adev->umsch_mm.log_gpu_addr,
300 cpu_addr: &adev->umsch_mm.log_cpu_addr);
301 if (r) {
302 dev_err(adev->dev, "(%d) failed to allocate umsch debug bo\n", r);
303 return r;
304 }
305
306 mutex_init(&adev->umsch_mm.mutex_hidden);
307
308 umsch_mm_agdb_index_init(adev);
309
310 return 0;
311}
312
313
314static int umsch_mm_early_init(struct amdgpu_ip_block *ip_block)
315{
316 struct amdgpu_device *adev = ip_block->adev;
317
318 switch (amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0)) {
319 case IP_VERSION(4, 0, 5):
320 case IP_VERSION(4, 0, 6):
321 umsch_mm_v4_0_set_funcs(umsch: &adev->umsch_mm);
322 break;
323 default:
324 return -EINVAL;
325 }
326
327 adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs;
328 umsch_mm_set_regs(&adev->umsch_mm);
329
330 return 0;
331}
332
333static int umsch_mm_late_init(struct amdgpu_ip_block *ip_block)
334{
335 struct amdgpu_device *adev = ip_block->adev;
336
337 if (amdgpu_in_reset(adev) || adev->in_s0ix || adev->in_suspend)
338 return 0;
339
340 return 0;
341}
342
343static int umsch_mm_sw_init(struct amdgpu_ip_block *ip_block)
344{
345 struct amdgpu_device *adev = ip_block->adev;
346 int r;
347
348 r = umsch_mm_init(adev);
349 if (r)
350 return r;
351
352 amdgpu_umsch_fwlog_init(umsch_mm: &adev->umsch_mm);
353 r = umsch_mm_ring_init(&adev->umsch_mm);
354 if (r)
355 return r;
356
357 r = umsch_mm_init_microcode(&adev->umsch_mm);
358 if (r)
359 return r;
360
361 return 0;
362}
363
364static int umsch_mm_sw_fini(struct amdgpu_ip_block *ip_block)
365{
366 struct amdgpu_device *adev = ip_block->adev;
367
368 release_firmware(fw: adev->umsch_mm.fw);
369 adev->umsch_mm.fw = NULL;
370
371 amdgpu_ring_fini(ring: &adev->umsch_mm.ring);
372
373 mutex_destroy(lock: &adev->umsch_mm.mutex_hidden);
374
375 amdgpu_bo_free_kernel(bo: &adev->umsch_mm.cmd_buf_obj,
376 gpu_addr: &adev->umsch_mm.cmd_buf_gpu_addr,
377 cpu_addr: (void **)&adev->umsch_mm.cmd_buf_ptr);
378
379 amdgpu_bo_free_kernel(bo: &adev->umsch_mm.dbglog_bo,
380 gpu_addr: &adev->umsch_mm.log_gpu_addr,
381 cpu_addr: (void **)&adev->umsch_mm.log_cpu_addr);
382
383 amdgpu_device_wb_free(adev, wb: adev->umsch_mm.wb_index);
384
385 return 0;
386}
387
388static int umsch_mm_hw_init(struct amdgpu_ip_block *ip_block)
389{
390 struct amdgpu_device *adev = ip_block->adev;
391 int r;
392
393 r = umsch_mm_load_microcode(&adev->umsch_mm);
394 if (r)
395 return r;
396
397 umsch_mm_ring_start(&adev->umsch_mm);
398
399 r = umsch_mm_set_hw_resources(&adev->umsch_mm);
400 if (r)
401 return r;
402
403 return 0;
404}
405
406static int umsch_mm_hw_fini(struct amdgpu_ip_block *ip_block)
407{
408 struct amdgpu_device *adev = ip_block->adev;
409
410 umsch_mm_ring_stop(&adev->umsch_mm);
411
412 amdgpu_bo_free_kernel(bo: &adev->umsch_mm.data_fw_obj,
413 gpu_addr: &adev->umsch_mm.data_fw_gpu_addr,
414 cpu_addr: (void **)&adev->umsch_mm.data_fw_ptr);
415
416 amdgpu_bo_free_kernel(bo: &adev->umsch_mm.ucode_fw_obj,
417 gpu_addr: &adev->umsch_mm.ucode_fw_gpu_addr,
418 cpu_addr: (void **)&adev->umsch_mm.ucode_fw_ptr);
419 return 0;
420}
421
422static int umsch_mm_suspend(struct amdgpu_ip_block *ip_block)
423{
424 return umsch_mm_hw_fini(ip_block);
425}
426
427static int umsch_mm_resume(struct amdgpu_ip_block *ip_block)
428{
429 return umsch_mm_hw_init(ip_block);
430}
431
432void amdgpu_umsch_fwlog_init(struct amdgpu_umsch_mm *umsch_mm)
433{
434#if defined(CONFIG_DEBUG_FS)
435 void *fw_log_cpu_addr = umsch_mm->log_cpu_addr;
436 volatile struct amdgpu_umsch_fwlog *log_buf = fw_log_cpu_addr;
437
438 log_buf->header_size = sizeof(struct amdgpu_umsch_fwlog);
439 log_buf->buffer_size = AMDGPU_UMSCHFW_LOG_SIZE;
440 log_buf->rptr = log_buf->header_size;
441 log_buf->wptr = log_buf->header_size;
442 log_buf->wrapped = 0;
443#endif
444}
445
446/*
447 * debugfs for mapping umsch firmware log buffer.
448 */
449#if defined(CONFIG_DEBUG_FS)
450static ssize_t amdgpu_debugfs_umsch_fwlog_read(struct file *f, char __user *buf,
451 size_t size, loff_t *pos)
452{
453 struct amdgpu_umsch_mm *umsch_mm;
454 void *log_buf;
455 volatile struct amdgpu_umsch_fwlog *plog;
456 unsigned int read_pos, write_pos, available, i, read_bytes = 0;
457 unsigned int read_num[2] = {0};
458
459 umsch_mm = file_inode(f)->i_private;
460 if (!umsch_mm)
461 return -ENODEV;
462
463 if (!umsch_mm->log_cpu_addr)
464 return -EFAULT;
465
466 log_buf = umsch_mm->log_cpu_addr;
467
468 plog = (volatile struct amdgpu_umsch_fwlog *)log_buf;
469 read_pos = plog->rptr;
470 write_pos = plog->wptr;
471
472 if (read_pos > AMDGPU_UMSCHFW_LOG_SIZE || write_pos > AMDGPU_UMSCHFW_LOG_SIZE)
473 return -EFAULT;
474
475 if (!size || (read_pos == write_pos))
476 return 0;
477
478 if (write_pos > read_pos) {
479 available = write_pos - read_pos;
480 read_num[0] = min_t(size_t, size, available);
481 } else {
482 read_num[0] = AMDGPU_UMSCHFW_LOG_SIZE - read_pos;
483 available = read_num[0] + write_pos - plog->header_size;
484 if (size > available)
485 read_num[1] = write_pos - plog->header_size;
486 else if (size > read_num[0])
487 read_num[1] = size - read_num[0];
488 else
489 read_num[0] = size;
490 }
491
492 for (i = 0; i < 2; i++) {
493 if (read_num[i]) {
494 if (read_pos == AMDGPU_UMSCHFW_LOG_SIZE)
495 read_pos = plog->header_size;
496 if (read_num[i] == copy_to_user(to: (buf + read_bytes),
497 from: (log_buf + read_pos), n: read_num[i]))
498 return -EFAULT;
499
500 read_bytes += read_num[i];
501 read_pos += read_num[i];
502 }
503 }
504
505 plog->rptr = read_pos;
506 *pos += read_bytes;
507 return read_bytes;
508}
509
510static const struct file_operations amdgpu_debugfs_umschfwlog_fops = {
511 .owner = THIS_MODULE,
512 .read = amdgpu_debugfs_umsch_fwlog_read,
513 .llseek = default_llseek
514};
515#endif
516
517void amdgpu_debugfs_umsch_fwlog_init(struct amdgpu_device *adev,
518 struct amdgpu_umsch_mm *umsch_mm)
519{
520#if defined(CONFIG_DEBUG_FS)
521 struct drm_minor *minor = adev_to_drm(adev)->primary;
522 struct dentry *root = minor->debugfs_root;
523 char name[32];
524
525 sprintf(buf: name, fmt: "amdgpu_umsch_fwlog");
526 debugfs_create_file_size(name, S_IFREG | 0444, parent: root, data: umsch_mm,
527 fops: &amdgpu_debugfs_umschfwlog_fops,
528 AMDGPU_UMSCHFW_LOG_SIZE);
529#endif
530}
531
532static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = {
533 .name = "umsch_mm_v4_0",
534 .early_init = umsch_mm_early_init,
535 .late_init = umsch_mm_late_init,
536 .sw_init = umsch_mm_sw_init,
537 .sw_fini = umsch_mm_sw_fini,
538 .hw_init = umsch_mm_hw_init,
539 .hw_fini = umsch_mm_hw_fini,
540 .suspend = umsch_mm_suspend,
541 .resume = umsch_mm_resume,
542};
543
544const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = {
545 .type = AMD_IP_BLOCK_TYPE_UMSCH_MM,
546 .major = 4,
547 .minor = 0,
548 .rev = 0,
549 .funcs = &umsch_mm_v4_0_ip_funcs,
550};
551

source code of linux/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c