1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2021 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#include <linux/vmalloc.h>
9#include <uapi/drm/habanalabs_accel.h>
10#include "habanalabs.h"
11
12/**
13 * hl_format_as_binary - helper function, format an integer as binary
14 * using supplied scratch buffer
15 * @buf: the buffer to use
16 * @buf_len: buffer capacity
17 * @n: number to format
18 *
19 * Returns pointer to buffer
20 */
21char *hl_format_as_binary(char *buf, size_t buf_len, u32 n)
22{
23 int i;
24 u32 bit;
25 bool leading0 = true;
26 char *wrptr = buf;
27
28 if (buf_len > 0 && buf_len < 3) {
29 *wrptr = '\0';
30 return buf;
31 }
32
33 wrptr[0] = '0';
34 wrptr[1] = 'b';
35 wrptr += 2;
36 /* Remove 3 characters from length for '0b' and '\0' termination */
37 buf_len -= 3;
38
39 for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {
40 /* Writing bit calculation in one line would cause a false
41 * positive static code analysis error, so splitting.
42 */
43 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));
44 bit = !!bit;
45 leading0 &= !bit;
46 if (!leading0) {
47 *wrptr = '0' + bit;
48 ++wrptr;
49 }
50 }
51
52 *wrptr = '\0';
53
54 return buf;
55}
56
57/**
58 * resize_to_fit - helper function, resize buffer to fit given amount of data
59 * @buf: destination buffer double pointer
60 * @size: pointer to the size container
61 * @desired_size: size the buffer must contain
62 *
63 * Returns 0 on success or error code on failure.
64 * On success, the size of buffer is at least desired_size. Buffer is allocated
65 * via vmalloc and must be freed with vfree.
66 */
67static int resize_to_fit(char **buf, size_t *size, size_t desired_size)
68{
69 char *resized_buf;
70 size_t new_size;
71
72 if (*size >= desired_size)
73 return 0;
74
75 /* Not enough space to print all, have to resize */
76 new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));
77 resized_buf = vmalloc(new_size);
78 if (!resized_buf)
79 return -ENOMEM;
80 memcpy(resized_buf, *buf, *size);
81 vfree(addr: *buf);
82 *buf = resized_buf;
83 *size = new_size;
84
85 return 1;
86}
87
88/**
89 * hl_snprintf_resize() - print formatted data to buffer, resize as needed
90 * @buf: buffer double pointer, to be written to and resized, must be either
91 * NULL or allocated with vmalloc.
92 * @size: current size of the buffer
93 * @offset: current offset to write to
94 * @format: format of the data
95 *
96 * This function will write formatted data into the buffer. If buffer is not
97 * large enough, it will be resized using vmalloc. Size may be modified if the
98 * buffer was resized, offset will be advanced by the number of bytes written
99 * not including the terminating character
100 *
101 * Returns 0 on success or error code on failure
102 *
103 * Note that the buffer has to be manually released using vfree.
104 */
105int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
106 const char *format, ...)
107{
108 va_list args;
109 size_t length;
110 int rc;
111
112 if (*buf == NULL && (*size != 0 || *offset != 0))
113 return -EINVAL;
114
115 va_start(args, format);
116 length = vsnprintf(buf: *buf + *offset, size: *size - *offset, fmt: format, args);
117 va_end(args);
118
119 rc = resize_to_fit(buf, size, desired_size: *offset + length + 1);
120 if (rc < 0)
121 return rc;
122 else if (rc > 0) {
123 /* Resize was needed, write again */
124 va_start(args, format);
125 length = vsnprintf(buf: *buf + *offset, size: *size - *offset, fmt: format,
126 args);
127 va_end(args);
128 }
129
130 *offset += length;
131
132 return 0;
133}
134
135/**
136 * hl_sync_engine_to_string - convert engine type enum to string literal
137 * @engine_type: engine type (TPC/MME/DMA)
138 *
139 * Return the resolved string literal
140 */
141const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)
142{
143 switch (engine_type) {
144 case ENGINE_DMA:
145 return "DMA";
146 case ENGINE_MME:
147 return "MME";
148 case ENGINE_TPC:
149 return "TPC";
150 }
151 return "Invalid Engine Type";
152}
153
154/**
155 * hl_print_resize_sync_engine - helper function, format engine name and ID
156 * using hl_snprintf_resize
157 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
158 * @size: pointer to the size container
159 * @offset: pointer to the offset container
160 * @engine_type: engine type (TPC/MME/DMA)
161 * @engine_id: engine numerical id
162 *
163 * Returns 0 on success or error code on failure
164 */
165static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset,
166 enum hl_sync_engine_type engine_type,
167 u32 engine_id)
168{
169 return hl_snprintf_resize(buf, size, offset, format: "%s%u",
170 hl_sync_engine_to_string(engine_type), engine_id);
171}
172
173/**
174 * hl_state_dump_get_sync_name - transform sync object id to name if available
175 * @hdev: pointer to the device
176 * @sync_id: sync object id
177 *
178 * Returns a name literal or NULL if not resolved.
179 * Note: returning NULL shall not be considered as a failure, as not all
180 * sync objects are named.
181 */
182const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id)
183{
184 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
185 struct hl_hw_obj_name_entry *entry;
186
187 hash_for_each_possible(sds->so_id_to_str_tb, entry,
188 node, sync_id)
189 if (sync_id == entry->id)
190 return entry->name;
191
192 return NULL;
193}
194
195/**
196 * hl_state_dump_get_monitor_name - transform monitor object dump to monitor
197 * name if available
198 * @hdev: pointer to the device
199 * @mon: monitor state dump
200 *
201 * Returns a name literal or NULL if not resolved.
202 * Note: returning NULL shall not be considered as a failure, as not all
203 * monitors are named.
204 */
205const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
206 struct hl_mon_state_dump *mon)
207{
208 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
209 struct hl_hw_obj_name_entry *entry;
210
211 hash_for_each_possible(sds->monitor_id_to_str_tb,
212 entry, node, mon->id)
213 if (mon->id == entry->id)
214 return entry->name;
215
216 return NULL;
217}
218
219/**
220 * hl_state_dump_free_sync_to_engine_map - free sync object to engine map
221 * @map: sync object to engine map
222 *
223 * Note: generic free implementation, the allocation is implemented per ASIC.
224 */
225void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)
226{
227 struct hl_sync_to_engine_map_entry *entry;
228 struct hlist_node *tmp_node;
229 int i;
230
231 hash_for_each_safe(map->tb, i, tmp_node, entry, node) {
232 hash_del(node: &entry->node);
233 kfree(objp: entry);
234 }
235}
236
237/**
238 * hl_state_dump_get_sync_to_engine - transform sync_id to
239 * hl_sync_to_engine_map_entry if available for current id
240 * @map: sync object to engine map
241 * @sync_id: sync object id
242 *
243 * Returns the translation entry if found or NULL if not.
244 * Note, returned NULL shall not be considered as a failure as the map
245 * does not cover all possible, it is a best effort sync ids.
246 */
247static struct hl_sync_to_engine_map_entry *
248hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)
249{
250 struct hl_sync_to_engine_map_entry *entry;
251
252 hash_for_each_possible(map->tb, entry, node, sync_id)
253 if (entry->sync_id == sync_id)
254 return entry;
255 return NULL;
256}
257
258/**
259 * hl_state_dump_read_sync_objects - read sync objects array
260 * @hdev: pointer to the device
261 * @index: sync manager block index starting with E_N
262 *
263 * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure
264 */
265static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index)
266{
267 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
268 u32 *sync_objects;
269 s64 base_addr; /* Base addr can be negative */
270 int i;
271
272 base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
273 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
274
275 sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));
276 if (!sync_objects)
277 return NULL;
278
279 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)
280 sync_objects[i] = RREG32(base_addr + i * sizeof(u32));
281
282 return sync_objects;
283}
284
285/**
286 * hl_state_dump_free_sync_objects - free sync objects array allocated by
287 * hl_state_dump_read_sync_objects
288 * @sync_objects: sync objects array
289 */
290static void hl_state_dump_free_sync_objects(u32 *sync_objects)
291{
292 vfree(addr: sync_objects);
293}
294
295
296/**
297 * hl_state_dump_print_syncs_single_block - print active sync objects on a
298 * single block
299 * @hdev: pointer to the device
300 * @index: sync manager block index starting with E_N
301 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
302 * @size: pointer to the size container
303 * @offset: pointer to the offset container
304 * @map: sync engines names map
305 *
306 * Returns 0 on success or error code on failure
307 */
308static int
309hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,
310 char **buf, size_t *size, size_t *offset,
311 struct hl_sync_to_engine_map *map)
312{
313 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
314 const char *sync_name;
315 u32 *sync_objects = NULL;
316 int rc = 0, i;
317
318 if (sds->sync_namager_names) {
319 rc = hl_snprintf_resize(
320 buf, size, offset, format: "%s\n",
321 sds->sync_namager_names[index]);
322 if (rc)
323 goto out;
324 }
325
326 sync_objects = hl_state_dump_read_sync_objects(hdev, index);
327 if (!sync_objects) {
328 rc = -ENOMEM;
329 goto out;
330 }
331
332 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {
333 struct hl_sync_to_engine_map_entry *entry;
334 u64 sync_object_addr;
335
336 if (!sync_objects[i])
337 continue;
338
339 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
340 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +
341 i * sizeof(u32);
342
343 rc = hl_snprintf_resize(buf, size, offset, format: "sync id: %u", i);
344 if (rc)
345 goto free_sync_objects;
346 sync_name = hl_state_dump_get_sync_name(hdev, sync_id: i);
347 if (sync_name) {
348 rc = hl_snprintf_resize(buf, size, offset, format: " %s",
349 sync_name);
350 if (rc)
351 goto free_sync_objects;
352 }
353 rc = hl_snprintf_resize(buf, size, offset, format: ", value: %u",
354 sync_objects[i]);
355 if (rc)
356 goto free_sync_objects;
357
358 /* Append engine string */
359 entry = hl_state_dump_get_sync_to_engine(map,
360 sync_id: (u32)sync_object_addr);
361 if (entry) {
362 rc = hl_snprintf_resize(buf, size, offset,
363 format: ", Engine: ");
364 if (rc)
365 goto free_sync_objects;
366 rc = hl_print_resize_sync_engine(buf, size, offset,
367 engine_type: entry->engine_type,
368 engine_id: entry->engine_id);
369 if (rc)
370 goto free_sync_objects;
371 }
372
373 rc = hl_snprintf_resize(buf, size, offset, format: "\n");
374 if (rc)
375 goto free_sync_objects;
376 }
377
378free_sync_objects:
379 hl_state_dump_free_sync_objects(sync_objects);
380out:
381 return rc;
382}
383
384/**
385 * hl_state_dump_print_syncs - print active sync objects
386 * @hdev: pointer to the device
387 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
388 * @size: pointer to the size container
389 * @offset: pointer to the offset container
390 *
391 * Returns 0 on success or error code on failure
392 */
393static int hl_state_dump_print_syncs(struct hl_device *hdev,
394 char **buf, size_t *size,
395 size_t *offset)
396
397{
398 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
399 struct hl_sync_to_engine_map *map;
400 u32 index;
401 int rc = 0;
402
403 map = kzalloc(sizeof(*map), GFP_KERNEL);
404 if (!map)
405 return -ENOMEM;
406
407 rc = sds->funcs.gen_sync_to_engine_map(hdev, map);
408 if (rc)
409 goto free_map_mem;
410
411 rc = hl_snprintf_resize(buf, size, offset, format: "Non zero sync objects:\n");
412 if (rc)
413 goto out;
414
415 if (sds->sync_namager_names) {
416 for (index = 0; sds->sync_namager_names[index]; ++index) {
417 rc = hl_state_dump_print_syncs_single_block(
418 hdev, index, buf, size, offset, map);
419 if (rc)
420 goto out;
421 }
422 } else {
423 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
424 rc = hl_state_dump_print_syncs_single_block(
425 hdev, index, buf, size, offset, map);
426 if (rc)
427 goto out;
428 }
429 }
430
431out:
432 hl_state_dump_free_sync_to_engine_map(map);
433free_map_mem:
434 kfree(objp: map);
435
436 return rc;
437}
438
439/**
440 * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific
441 * block
442 * @hdev: pointer to the device
443 * @index: sync manager block index starting with E_N
444 *
445 * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL
446 * on error
447 */
448static struct hl_mon_state_dump *
449hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)
450{
451 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
452 struct hl_mon_state_dump *monitors;
453 s64 base_addr; /* Base addr can be negative */
454 int i;
455
456 monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *
457 sizeof(struct hl_mon_state_dump));
458 if (!monitors)
459 return NULL;
460
461 base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
462
463 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
464 monitors[i].id = i;
465 monitors[i].wr_addr_low =
466 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +
467 i * sizeof(u32));
468
469 monitors[i].wr_addr_high =
470 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +
471 i * sizeof(u32));
472
473 monitors[i].wr_data =
474 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +
475 i * sizeof(u32));
476
477 monitors[i].arm_data =
478 RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +
479 i * sizeof(u32));
480
481 monitors[i].status =
482 RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +
483 i * sizeof(u32));
484 }
485
486 return monitors;
487}
488
489/**
490 * hl_state_dump_free_monitors - free the monitors structure
491 * @monitors: monitors array created with
492 * hl_state_dump_alloc_read_sm_block_monitors
493 */
494static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)
495{
496 vfree(addr: monitors);
497}
498
499/**
500 * hl_state_dump_print_monitors_single_block - print active monitors on a
501 * single block
502 * @hdev: pointer to the device
503 * @index: sync manager block index starting with E_N
504 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
505 * @size: pointer to the size container
506 * @offset: pointer to the offset container
507 *
508 * Returns 0 on success or error code on failure
509 */
510static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,
511 u32 index,
512 char **buf, size_t *size,
513 size_t *offset)
514{
515 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
516 struct hl_mon_state_dump *monitors = NULL;
517 int rc = 0, i;
518
519 if (sds->sync_namager_names) {
520 rc = hl_snprintf_resize(
521 buf, size, offset, format: "%s\n",
522 sds->sync_namager_names[index]);
523 if (rc)
524 goto out;
525 }
526
527 monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);
528 if (!monitors) {
529 rc = -ENOMEM;
530 goto out;
531 }
532
533 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
534 if (!(sds->funcs.monitor_valid(&monitors[i])))
535 continue;
536
537 /* Monitor is valid, dump it */
538 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,
539 &monitors[i]);
540 if (rc)
541 goto free_monitors;
542
543 hl_snprintf_resize(buf, size, offset, format: "\n");
544 }
545
546free_monitors:
547 hl_state_dump_free_monitors(monitors);
548out:
549 return rc;
550}
551
552/**
553 * hl_state_dump_print_monitors - print active monitors
554 * @hdev: pointer to the device
555 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
556 * @size: pointer to the size container
557 * @offset: pointer to the offset container
558 *
559 * Returns 0 on success or error code on failure
560 */
561static int hl_state_dump_print_monitors(struct hl_device *hdev,
562 char **buf, size_t *size,
563 size_t *offset)
564{
565 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
566 u32 index;
567 int rc = 0;
568
569 rc = hl_snprintf_resize(buf, size, offset,
570 format: "Valid (armed) monitor objects:\n");
571 if (rc)
572 goto out;
573
574 if (sds->sync_namager_names) {
575 for (index = 0; sds->sync_namager_names[index]; ++index) {
576 rc = hl_state_dump_print_monitors_single_block(
577 hdev, index, buf, size, offset);
578 if (rc)
579 goto out;
580 }
581 } else {
582 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
583 rc = hl_state_dump_print_monitors_single_block(
584 hdev, index, buf, size, offset);
585 if (rc)
586 goto out;
587 }
588 }
589
590out:
591 return rc;
592}
593
594/**
595 * hl_state_dump_print_engine_fences - print active fences for a specific
596 * engine
597 * @hdev: pointer to the device
598 * @engine_type: engine type to use
599 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
600 * @size: pointer to the size container
601 * @offset: pointer to the offset container
602 */
603static int
604hl_state_dump_print_engine_fences(struct hl_device *hdev,
605 enum hl_sync_engine_type engine_type,
606 char **buf, size_t *size, size_t *offset)
607{
608 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
609 int rc = 0, i, n_fences;
610 u64 base_addr, next_fence;
611
612 switch (engine_type) {
613 case ENGINE_TPC:
614 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];
615 base_addr = sds->props[SP_TPC0_CMDQ];
616 next_fence = sds->props[SP_NEXT_TPC];
617 break;
618 case ENGINE_MME:
619 n_fences = sds->props[SP_NUM_OF_MME_ENGINES];
620 base_addr = sds->props[SP_MME_CMDQ];
621 next_fence = sds->props[SP_NEXT_MME];
622 break;
623 case ENGINE_DMA:
624 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];
625 base_addr = sds->props[SP_DMA_CMDQ];
626 next_fence = sds->props[SP_DMA_QUEUES_OFFSET];
627 break;
628 default:
629 return -EINVAL;
630 }
631 for (i = 0; i < n_fences; ++i) {
632 rc = sds->funcs.print_fences_single_engine(
633 hdev,
634 base_addr + next_fence * i +
635 sds->props[SP_FENCE0_CNT_OFFSET],
636 base_addr + next_fence * i +
637 sds->props[SP_CP_STS_OFFSET],
638 engine_type, i, buf, size, offset);
639 if (rc)
640 goto out;
641 }
642out:
643 return rc;
644}
645
646/**
647 * hl_state_dump_print_fences - print active fences
648 * @hdev: pointer to the device
649 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
650 * @size: pointer to the size container
651 * @offset: pointer to the offset container
652 */
653static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf,
654 size_t *size, size_t *offset)
655{
656 int rc = 0;
657
658 rc = hl_snprintf_resize(buf, size, offset, format: "Valid (armed) fences:\n");
659 if (rc)
660 goto out;
661
662 rc = hl_state_dump_print_engine_fences(hdev, engine_type: ENGINE_TPC, buf, size, offset);
663 if (rc)
664 goto out;
665
666 rc = hl_state_dump_print_engine_fences(hdev, engine_type: ENGINE_MME, buf, size, offset);
667 if (rc)
668 goto out;
669
670 rc = hl_state_dump_print_engine_fences(hdev, engine_type: ENGINE_DMA, buf, size, offset);
671 if (rc)
672 goto out;
673
674out:
675 return rc;
676}
677
678/**
679 * hl_state_dump() - dump system state
680 * @hdev: pointer to device structure
681 */
682int hl_state_dump(struct hl_device *hdev)
683{
684 char *buf = NULL;
685 size_t offset = 0, size = 0;
686 int rc;
687
688 rc = hl_snprintf_resize(buf: &buf, size: &size, offset: &offset,
689 format: "Timestamp taken on: %llu\n\n",
690 ktime_to_ns(kt: ktime_get()));
691 if (rc)
692 goto err;
693
694 rc = hl_state_dump_print_syncs(hdev, buf: &buf, size: &size, offset: &offset);
695 if (rc)
696 goto err;
697
698 hl_snprintf_resize(buf: &buf, size: &size, offset: &offset, format: "\n");
699
700 rc = hl_state_dump_print_monitors(hdev, buf: &buf, size: &size, offset: &offset);
701 if (rc)
702 goto err;
703
704 hl_snprintf_resize(buf: &buf, size: &size, offset: &offset, format: "\n");
705
706 rc = hl_state_dump_print_fences(hdev, buf: &buf, size: &size, offset: &offset);
707 if (rc)
708 goto err;
709
710 hl_snprintf_resize(buf: &buf, size: &size, offset: &offset, format: "\n");
711
712 hl_debugfs_set_state_dump(hdev, data: buf, length: size);
713
714 return 0;
715err:
716 vfree(addr: buf);
717 return rc;
718}
719

source code of linux/drivers/accel/habanalabs/common/state_dump.c