1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include "pvr_free_list.h"
5#include "pvr_hwrt.h"
6#include "pvr_gem.h"
7#include "pvr_rogue_cr_defs_client.h"
8#include "pvr_rogue_fwif.h"
9
10#include <drm/drm_gem.h>
11#include <linux/bitops.h>
12#include <linux/math.h>
13#include <linux/slab.h>
14#include <linux/xarray.h>
15#include <uapi/drm/pvr_drm.h>
16
17static_assert(ROGUE_FWIF_NUM_RTDATAS == 2);
18static_assert(ROGUE_FWIF_NUM_GEOMDATAS == 1);
19static_assert(ROGUE_FWIF_NUM_RTDATA_FREELISTS == 2);
20
21/*
22 * struct pvr_rt_mtile_info - Render target macrotile information
23 */
24struct pvr_rt_mtile_info {
25 u32 mtile_x[3];
26 u32 mtile_y[3];
27 u32 tile_max_x;
28 u32 tile_max_y;
29 u32 tile_size_x;
30 u32 tile_size_y;
31 u32 num_tiles_x;
32 u32 num_tiles_y;
33};
34
35/* Size of Shadow Render Target Cache entry */
36#define SRTC_ENTRY_SIZE sizeof(u32)
37/* Size of Renders Accumulation Array entry */
38#define RAA_ENTRY_SIZE sizeof(u32)
39
40static int
41hwrt_init_kernel_structure(struct pvr_file *pvr_file,
42 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,
43 struct pvr_hwrt_dataset *hwrt)
44{
45 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
46 int err;
47
48 hwrt->pvr_dev = pvr_dev;
49 hwrt->max_rts = args->layers;
50
51 /* Get pointers to the free lists */
52 for (int i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {
53 hwrt->free_lists[i] = pvr_free_list_lookup(pvr_file, handle: args->free_list_handles[i]);
54 if (!hwrt->free_lists[i]) {
55 err = -EINVAL;
56 goto err_put_free_lists;
57 }
58 }
59
60 if (hwrt->free_lists[ROGUE_FW_LOCAL_FREELIST]->current_pages <
61 pvr_get_free_list_min_pages(pvr_dev)) {
62 err = -EINVAL;
63 goto err_put_free_lists;
64 }
65
66 return 0;
67
68err_put_free_lists:
69 for (int i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {
70 pvr_free_list_put(free_list: hwrt->free_lists[i]);
71 hwrt->free_lists[i] = NULL;
72 }
73
74 return err;
75}
76
77static void
78hwrt_fini_kernel_structure(struct pvr_hwrt_dataset *hwrt)
79{
80 for (int i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {
81 pvr_free_list_put(free_list: hwrt->free_lists[i]);
82 hwrt->free_lists[i] = NULL;
83 }
84}
85
86static void
87hwrt_fini_common_fw_structure(struct pvr_hwrt_dataset *hwrt)
88{
89 pvr_fw_object_destroy(fw_obj: hwrt->common_fw_obj);
90}
91
92static int
93get_cr_isp_mtile_size_val(struct pvr_device *pvr_dev, u32 samples,
94 struct pvr_rt_mtile_info *info, u32 *value_out)
95{
96 u32 x = info->mtile_x[0];
97 u32 y = info->mtile_y[0];
98 u32 samples_per_pixel;
99 int err;
100
101 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel);
102 if (err)
103 return err;
104
105 if (samples_per_pixel == 1) {
106 if (samples >= 4)
107 x <<= 1;
108 if (samples >= 2)
109 y <<= 1;
110 } else if (samples_per_pixel == 2) {
111 if (samples >= 8)
112 x <<= 1;
113 if (samples >= 4)
114 y <<= 1;
115 } else if (samples_per_pixel == 4) {
116 if (samples >= 8)
117 y <<= 1;
118 } else {
119 WARN(true, "Unsupported ISP samples per pixel value");
120 return -EINVAL;
121 }
122
123 *value_out = ((x << ROGUE_CR_ISP_MTILE_SIZE_X_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_X_CLRMSK) |
124 ((y << ROGUE_CR_ISP_MTILE_SIZE_Y_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_Y_CLRMSK);
125
126 return 0;
127}
128
129static int
130get_cr_multisamplectl_val(u32 samples, bool y_flip, u64 *value_out)
131{
132 static const struct {
133 u8 x[8];
134 u8 y[8];
135 } sample_positions[4] = {
136 /* 1 sample */
137 {
138 .x = { 8 },
139 .y = { 8 },
140 },
141 /* 2 samples */
142 {
143 .x = { 12, 4 },
144 .y = { 12, 4 },
145 },
146 /* 4 samples */
147 {
148 .x = { 6, 14, 2, 10 },
149 .y = { 2, 6, 10, 14 },
150 },
151 /* 8 samples */
152 {
153 .x = { 9, 7, 13, 5, 3, 1, 11, 15 },
154 .y = { 5, 11, 9, 3, 13, 7, 15, 1 },
155 },
156 };
157 const int idx = fls(x: samples) - 1;
158 u64 value = 0;
159
160 if (idx < 0 || idx > 3)
161 return -EINVAL;
162
163 for (u32 i = 0; i < 8; i++) {
164 value |= ((u64)sample_positions[idx].x[i]) << (i * 8);
165 if (y_flip)
166 value |= (((u64)(16 - sample_positions[idx].y[i]) & 0xf)) << (i * 8 + 4);
167 else
168 value |= ((u64)sample_positions[idx].y[i]) << (i * 8 + 4);
169 }
170
171 *value_out = value;
172
173 return 0;
174}
175
176static int
177get_cr_te_aa_val(struct pvr_device *pvr_dev, u32 samples, u32 *value_out)
178{
179 u32 samples_per_pixel;
180 u32 value = 0;
181 int err = 0;
182
183 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel);
184 if (err)
185 return err;
186
187 switch (samples_per_pixel) {
188 case 1:
189 if (samples >= 2)
190 value |= ROGUE_CR_TE_AA_Y_EN;
191 if (samples >= 4)
192 value |= ROGUE_CR_TE_AA_X_EN;
193 break;
194 case 2:
195 if (samples >= 2)
196 value |= ROGUE_CR_TE_AA_X2_EN;
197 if (samples >= 4)
198 value |= ROGUE_CR_TE_AA_Y_EN;
199 if (samples >= 8)
200 value |= ROGUE_CR_TE_AA_X_EN;
201 break;
202 case 4:
203 if (samples >= 2)
204 value |= ROGUE_CR_TE_AA_X2_EN;
205 if (samples >= 4)
206 value |= ROGUE_CR_TE_AA_Y2_EN;
207 if (samples >= 8)
208 value |= ROGUE_CR_TE_AA_Y_EN;
209 break;
210 default:
211 WARN(true, "Unsupported ISP samples per pixel value");
212 return -EINVAL;
213 }
214
215 *value_out = value;
216
217 return 0;
218}
219
220static void
221hwrtdata_common_init(void *cpu_ptr, void *priv)
222{
223 struct pvr_hwrt_dataset *hwrt = priv;
224
225 memcpy(cpu_ptr, &hwrt->common, sizeof(hwrt->common));
226}
227
228static int
229hwrt_init_common_fw_structure(struct pvr_file *pvr_file,
230 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,
231 struct pvr_hwrt_dataset *hwrt)
232{
233 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args;
234 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
235 struct pvr_rt_mtile_info info;
236 int err;
237
238 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_x, &info.tile_size_x);
239 if (WARN_ON(err))
240 return err;
241
242 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_y, &info.tile_size_y);
243 if (WARN_ON(err))
244 return err;
245
246 info.num_tiles_x = DIV_ROUND_UP(args->width, info.tile_size_x);
247 info.num_tiles_y = DIV_ROUND_UP(args->height, info.tile_size_y);
248
249 if (PVR_HAS_FEATURE(pvr_dev, simple_parameter_format_version)) {
250 u32 parameter_format;
251
252 err = PVR_FEATURE_VALUE(pvr_dev, simple_parameter_format_version,
253 &parameter_format);
254 if (WARN_ON(err))
255 return err;
256
257 WARN_ON(parameter_format != 2);
258
259 /*
260 * Set up 16 macrotiles with a multiple of 2x2 tiles per macrotile, which is
261 * aligned to a tile group.
262 */
263 info.mtile_x[0] = DIV_ROUND_UP(info.num_tiles_x, 8) * 2;
264 info.mtile_y[0] = DIV_ROUND_UP(info.num_tiles_y, 8) * 2;
265 info.mtile_x[1] = 0;
266 info.mtile_y[1] = 0;
267 info.mtile_x[2] = 0;
268 info.mtile_y[2] = 0;
269 info.tile_max_x = round_up(info.num_tiles_x, 2) - 1;
270 info.tile_max_y = round_up(info.num_tiles_y, 2) - 1;
271 } else {
272 /* Set up 16 macrotiles with a multiple of 4x4 tiles per macrotile. */
273 info.mtile_x[0] = round_up(DIV_ROUND_UP(info.num_tiles_x, 4), 4);
274 info.mtile_y[0] = round_up(DIV_ROUND_UP(info.num_tiles_y, 4), 4);
275 info.mtile_x[1] = info.mtile_x[0] * 2;
276 info.mtile_y[1] = info.mtile_y[0] * 2;
277 info.mtile_x[2] = info.mtile_x[0] * 3;
278 info.mtile_y[2] = info.mtile_y[0] * 3;
279 info.tile_max_x = info.num_tiles_x - 1;
280 info.tile_max_y = info.num_tiles_y - 1;
281 }
282
283 hwrt->common.geom_caches_need_zeroing = false;
284
285 hwrt->common.isp_merge_lower_x = args->isp_merge_lower_x;
286 hwrt->common.isp_merge_lower_y = args->isp_merge_lower_y;
287 hwrt->common.isp_merge_upper_x = args->isp_merge_upper_x;
288 hwrt->common.isp_merge_upper_y = args->isp_merge_upper_y;
289 hwrt->common.isp_merge_scale_x = args->isp_merge_scale_x;
290 hwrt->common.isp_merge_scale_y = args->isp_merge_scale_y;
291
292 err = get_cr_multisamplectl_val(samples: args->samples, y_flip: false,
293 value_out: &hwrt->common.multi_sample_ctl);
294 if (err)
295 return err;
296
297 err = get_cr_multisamplectl_val(samples: args->samples, y_flip: true,
298 value_out: &hwrt->common.flipped_multi_sample_ctl);
299 if (err)
300 return err;
301
302 hwrt->common.mtile_stride = info.mtile_x[0] * info.mtile_y[0];
303
304 err = get_cr_te_aa_val(pvr_dev, samples: args->samples, value_out: &hwrt->common.teaa);
305 if (err)
306 return err;
307
308 hwrt->common.screen_pixel_max =
309 (((args->width - 1) << ROGUE_CR_PPP_SCREEN_PIXXMAX_SHIFT) &
310 ~ROGUE_CR_PPP_SCREEN_PIXXMAX_CLRMSK) |
311 (((args->height - 1) << ROGUE_CR_PPP_SCREEN_PIXYMAX_SHIFT) &
312 ~ROGUE_CR_PPP_SCREEN_PIXYMAX_CLRMSK);
313
314 hwrt->common.te_screen =
315 ((info.tile_max_x << ROGUE_CR_TE_SCREEN_XMAX_SHIFT) &
316 ~ROGUE_CR_TE_SCREEN_XMAX_CLRMSK) |
317 ((info.tile_max_y << ROGUE_CR_TE_SCREEN_YMAX_SHIFT) &
318 ~ROGUE_CR_TE_SCREEN_YMAX_CLRMSK);
319 hwrt->common.te_mtile1 =
320 ((info.mtile_x[0] << ROGUE_CR_TE_MTILE1_X1_SHIFT) & ~ROGUE_CR_TE_MTILE1_X1_CLRMSK) |
321 ((info.mtile_x[1] << ROGUE_CR_TE_MTILE1_X2_SHIFT) & ~ROGUE_CR_TE_MTILE1_X2_CLRMSK) |
322 ((info.mtile_x[2] << ROGUE_CR_TE_MTILE1_X3_SHIFT) & ~ROGUE_CR_TE_MTILE1_X3_CLRMSK);
323 hwrt->common.te_mtile2 =
324 ((info.mtile_y[0] << ROGUE_CR_TE_MTILE2_Y1_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y1_CLRMSK) |
325 ((info.mtile_y[1] << ROGUE_CR_TE_MTILE2_Y2_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y2_CLRMSK) |
326 ((info.mtile_y[2] << ROGUE_CR_TE_MTILE2_Y3_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y3_CLRMSK);
327
328 err = get_cr_isp_mtile_size_val(pvr_dev, samples: args->samples, info: &info,
329 value_out: &hwrt->common.isp_mtile_size);
330 if (err)
331 return err;
332
333 hwrt->common.tpc_stride = geom_data_args->tpc_stride;
334 hwrt->common.tpc_size = geom_data_args->tpc_size;
335
336 hwrt->common.rgn_header_size = args->region_header_size;
337
338 err = pvr_fw_object_create(pvr_dev, size: sizeof(struct rogue_fwif_hwrtdata_common),
339 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, init: hwrtdata_common_init, init_priv: hwrt,
340 pvr_obj_out: &hwrt->common_fw_obj);
341
342 return err;
343}
344
345static void
346hwrt_fw_data_init(void *cpu_ptr, void *priv)
347{
348 struct pvr_hwrt_data *hwrt_data = priv;
349
350 memcpy(cpu_ptr, &hwrt_data->data, sizeof(hwrt_data->data));
351}
352
353static int
354hwrt_data_init_fw_structure(struct pvr_file *pvr_file,
355 struct pvr_hwrt_dataset *hwrt,
356 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,
357 struct drm_pvr_create_hwrt_rt_data_args *rt_data_args,
358 struct pvr_hwrt_data *hwrt_data)
359{
360 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args;
361 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
362 struct rogue_fwif_rta_ctl *rta_ctl;
363 int err;
364
365 pvr_fw_object_get_fw_addr(fw_obj: hwrt->common_fw_obj,
366 fw_addr_out: &hwrt_data->data.hwrt_data_common_fw_addr);
367
368 for (int free_list_i = 0; free_list_i < ARRAY_SIZE(hwrt->free_lists); free_list_i++) {
369 pvr_fw_object_get_fw_addr(fw_obj: hwrt->free_lists[free_list_i]->fw_obj,
370 fw_addr_out: &hwrt_data->data.freelists_fw_addr[free_list_i]);
371 }
372
373 hwrt_data->data.tail_ptrs_dev_addr = geom_data_args->tpc_dev_addr;
374 hwrt_data->data.vheap_table_dev_addr = geom_data_args->vheap_table_dev_addr;
375 hwrt_data->data.rtc_dev_addr = geom_data_args->rtc_dev_addr;
376
377 hwrt_data->data.pm_mlist_dev_addr = rt_data_args->pm_mlist_dev_addr;
378 hwrt_data->data.macrotile_array_dev_addr = rt_data_args->macrotile_array_dev_addr;
379 hwrt_data->data.rgn_header_dev_addr = rt_data_args->region_header_dev_addr;
380
381 rta_ctl = &hwrt_data->data.rta_ctl;
382
383 rta_ctl->render_target_index = 0;
384 rta_ctl->active_render_targets = 0;
385 rta_ctl->valid_render_targets_fw_addr = 0;
386 rta_ctl->rta_num_partial_renders_fw_addr = 0;
387 rta_ctl->max_rts = args->layers;
388
389 if (args->layers > 1) {
390 err = pvr_fw_object_create(pvr_dev, size: args->layers * SRTC_ENTRY_SIZE,
391 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
392 NULL, NULL, pvr_obj_out: &hwrt_data->srtc_obj);
393 if (err)
394 return err;
395 pvr_fw_object_get_fw_addr(fw_obj: hwrt_data->srtc_obj,
396 fw_addr_out: &rta_ctl->valid_render_targets_fw_addr);
397
398 err = pvr_fw_object_create(pvr_dev, size: args->layers * RAA_ENTRY_SIZE,
399 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
400 NULL, NULL, pvr_obj_out: &hwrt_data->raa_obj);
401 if (err)
402 goto err_put_shadow_rt_cache;
403 pvr_fw_object_get_fw_addr(fw_obj: hwrt_data->raa_obj,
404 fw_addr_out: &rta_ctl->rta_num_partial_renders_fw_addr);
405 }
406
407 err = pvr_fw_object_create(pvr_dev, size: sizeof(struct rogue_fwif_hwrtdata),
408 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
409 init: hwrt_fw_data_init, init_priv: hwrt_data, pvr_obj_out: &hwrt_data->fw_obj);
410 if (err)
411 goto err_put_raa_obj;
412
413 pvr_free_list_add_hwrt(free_list: hwrt->free_lists[0], hwrt_data);
414
415 return 0;
416
417err_put_raa_obj:
418 if (args->layers > 1)
419 pvr_fw_object_destroy(fw_obj: hwrt_data->raa_obj);
420
421err_put_shadow_rt_cache:
422 if (args->layers > 1)
423 pvr_fw_object_destroy(fw_obj: hwrt_data->srtc_obj);
424
425 return err;
426}
427
428static void
429hwrt_data_fini_fw_structure(struct pvr_hwrt_dataset *hwrt, int hwrt_nr)
430{
431 struct pvr_hwrt_data *hwrt_data = &hwrt->data[hwrt_nr];
432
433 pvr_free_list_remove_hwrt(free_list: hwrt->free_lists[0], hwrt_data);
434
435 if (hwrt->max_rts > 1) {
436 pvr_fw_object_destroy(fw_obj: hwrt_data->raa_obj);
437 pvr_fw_object_destroy(fw_obj: hwrt_data->srtc_obj);
438 }
439
440 pvr_fw_object_destroy(fw_obj: hwrt_data->fw_obj);
441}
442
443/**
444 * pvr_hwrt_dataset_create() - Create a new HWRT dataset
445 * @pvr_file: Pointer to pvr_file structure.
446 * @args: Creation arguments from userspace.
447 *
448 * Return:
449 * * Pointer to new HWRT, or
450 * * ERR_PTR(-%ENOMEM) on out of memory.
451 */
452struct pvr_hwrt_dataset *
453pvr_hwrt_dataset_create(struct pvr_file *pvr_file,
454 struct drm_pvr_ioctl_create_hwrt_dataset_args *args)
455{
456 struct pvr_hwrt_dataset *hwrt;
457 int err, i = 0;
458
459 /* Create and fill out the kernel structure */
460 hwrt = kzalloc(sizeof(*hwrt), GFP_KERNEL);
461
462 if (!hwrt)
463 return ERR_PTR(error: -ENOMEM);
464
465 err = hwrt_init_kernel_structure(pvr_file, args, hwrt);
466 if (err < 0)
467 goto err_free;
468
469 err = hwrt_init_common_fw_structure(pvr_file, args, hwrt);
470 if (err < 0)
471 goto err_fini_kernel_structure;
472
473 for (; i < ARRAY_SIZE(hwrt->data); i++) {
474 err = hwrt_data_init_fw_structure(pvr_file, hwrt, args,
475 rt_data_args: &args->rt_data_args[i],
476 hwrt_data: &hwrt->data[i]);
477 if (err < 0)
478 goto err_fini_data_structures;
479
480 hwrt->data[i].hwrt_dataset = hwrt;
481 }
482
483 kref_init(kref: &hwrt->ref_count);
484 return hwrt;
485
486err_fini_data_structures:
487 while (--i >= 0)
488 hwrt_data_fini_fw_structure(hwrt, hwrt_nr: i);
489
490err_fini_kernel_structure:
491 hwrt_fini_kernel_structure(hwrt);
492
493err_free:
494 kfree(objp: hwrt);
495
496 return ERR_PTR(error: err);
497}
498
499static void
500pvr_hwrt_dataset_release(struct kref *ref_count)
501{
502 struct pvr_hwrt_dataset *hwrt =
503 container_of(ref_count, struct pvr_hwrt_dataset, ref_count);
504
505 for (int i = ARRAY_SIZE(hwrt->data) - 1; i >= 0; i--) {
506 WARN_ON(pvr_fw_structure_cleanup(hwrt->pvr_dev, ROGUE_FWIF_CLEANUP_HWRTDATA,
507 hwrt->data[i].fw_obj, 0));
508 hwrt_data_fini_fw_structure(hwrt, hwrt_nr: i);
509 }
510
511 hwrt_fini_common_fw_structure(hwrt);
512 hwrt_fini_kernel_structure(hwrt);
513
514 kfree(objp: hwrt);
515}
516
517/**
518 * pvr_destroy_hwrt_datasets_for_file: Destroy any HWRT datasets associated
519 * with the given file.
520 * @pvr_file: Pointer to pvr_file structure.
521 *
522 * Removes all HWRT datasets associated with @pvr_file from the device
523 * hwrt_dataset list and drops initial references. HWRT datasets will then be
524 * destroyed once all outstanding references are dropped.
525 */
526void pvr_destroy_hwrt_datasets_for_file(struct pvr_file *pvr_file)
527{
528 struct pvr_hwrt_dataset *hwrt;
529 unsigned long handle;
530
531 xa_for_each(&pvr_file->hwrt_handles, handle, hwrt) {
532 (void)hwrt;
533 pvr_hwrt_dataset_put(hwrt: xa_erase(&pvr_file->hwrt_handles, index: handle));
534 }
535}
536
537/**
538 * pvr_hwrt_dataset_put() - Release reference on HWRT dataset
539 * @hwrt: Pointer to HWRT dataset to release reference on
540 */
541void
542pvr_hwrt_dataset_put(struct pvr_hwrt_dataset *hwrt)
543{
544 if (hwrt)
545 kref_put(kref: &hwrt->ref_count, release: pvr_hwrt_dataset_release);
546}
547

source code of linux/drivers/gpu/drm/imagination/pvr_hwrt.c