1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include "pvr_vm.h"
5
6#include "pvr_device.h"
7#include "pvr_drv.h"
8#include "pvr_gem.h"
9#include "pvr_mmu.h"
10#include "pvr_rogue_fwif.h"
11#include "pvr_rogue_heap_config.h"
12
13#include <drm/drm_exec.h>
14#include <drm/drm_gem.h>
15#include <drm/drm_gpuvm.h>
16#include <drm/drm_print.h>
17
18#include <linux/bug.h>
19#include <linux/container_of.h>
20#include <linux/err.h>
21#include <linux/errno.h>
22#include <linux/gfp_types.h>
23#include <linux/kref.h>
24#include <linux/mutex.h>
25#include <linux/stddef.h>
26
27/**
28 * DOC: Memory context
29 *
30 * This is the "top level" datatype in the VM code. It's exposed in the public
31 * API as an opaque handle.
32 */
33
34/**
35 * struct pvr_vm_context - Context type used to represent a single VM.
36 */
37struct pvr_vm_context {
38 /**
39 * @pvr_dev: The PowerVR device to which this context is bound.
40 * This binding is immutable for the life of the context.
41 */
42 struct pvr_device *pvr_dev;
43
44 /** @mmu_ctx: The context for binding to physical memory. */
45 struct pvr_mmu_context *mmu_ctx;
46
47 /** @gpuvm_mgr: GPUVM object associated with this context. */
48 struct drm_gpuvm gpuvm_mgr;
49
50 /** @lock: Global lock on this VM. */
51 struct mutex lock;
52
53 /**
54 * @fw_mem_ctx_obj: Firmware object representing firmware memory
55 * context.
56 */
57 struct pvr_fw_object *fw_mem_ctx_obj;
58
59 /** @ref_count: Reference count of object. */
60 struct kref ref_count;
61
62 /**
63 * @dummy_gem: GEM object to enable VM reservation. All private BOs
64 * should use the @dummy_gem.resv and not their own _resv field.
65 */
66 struct drm_gem_object dummy_gem;
67};
68
69static inline
70struct pvr_vm_context *to_pvr_vm_context(struct drm_gpuvm *gpuvm)
71{
72 return container_of(gpuvm, struct pvr_vm_context, gpuvm_mgr);
73}
74
75struct pvr_vm_context *pvr_vm_context_get(struct pvr_vm_context *vm_ctx)
76{
77 if (vm_ctx)
78 kref_get(kref: &vm_ctx->ref_count);
79
80 return vm_ctx;
81}
82
83/**
84 * pvr_vm_get_page_table_root_addr() - Get the DMA address of the root of the
85 * page table structure behind a VM context.
86 * @vm_ctx: Target VM context.
87 */
88dma_addr_t pvr_vm_get_page_table_root_addr(struct pvr_vm_context *vm_ctx)
89{
90 return pvr_mmu_get_root_table_dma_addr(ctx: vm_ctx->mmu_ctx);
91}
92
93/**
94 * pvr_vm_get_dma_resv() - Expose the dma_resv owned by the VM context.
95 * @vm_ctx: Target VM context.
96 *
97 * This is used to allow private BOs to share a dma_resv for faster fence
98 * updates.
99 *
100 * Returns: The dma_resv pointer.
101 */
102struct dma_resv *pvr_vm_get_dma_resv(struct pvr_vm_context *vm_ctx)
103{
104 return vm_ctx->dummy_gem.resv;
105}
106
107/**
108 * DOC: Memory mappings
109 */
110
111/**
112 * struct pvr_vm_gpuva - Wrapper type representing a single VM mapping.
113 */
114struct pvr_vm_gpuva {
115 /** @base: The wrapped drm_gpuva object. */
116 struct drm_gpuva base;
117};
118
119#define to_pvr_vm_gpuva(va) container_of_const(va, struct pvr_vm_gpuva, base)
120
121enum pvr_vm_bind_type {
122 PVR_VM_BIND_TYPE_MAP,
123 PVR_VM_BIND_TYPE_UNMAP,
124};
125
126/**
127 * struct pvr_vm_bind_op - Context of a map/unmap operation.
128 */
129struct pvr_vm_bind_op {
130 /** @type: Map or unmap. */
131 enum pvr_vm_bind_type type;
132
133 /** @pvr_obj: Object associated with mapping (map only). */
134 struct pvr_gem_object *pvr_obj;
135
136 /**
137 * @vm_ctx: VM context where the mapping will be created or destroyed.
138 */
139 struct pvr_vm_context *vm_ctx;
140
141 /** @mmu_op_ctx: MMU op context. */
142 struct pvr_mmu_op_context *mmu_op_ctx;
143
144 /** @gpuvm_bo: Prealloced wrapped BO for attaching to the gpuvm. */
145 struct drm_gpuvm_bo *gpuvm_bo;
146
147 /**
148 * @new_va: Prealloced VA mapping object (init in callback).
149 * Used when creating a mapping.
150 */
151 struct pvr_vm_gpuva *new_va;
152
153 /**
154 * @prev_va: Prealloced VA mapping object (init in callback).
155 * Used when a mapping or unmapping operation overlaps an existing
156 * mapping and splits away the beginning into a new mapping.
157 */
158 struct pvr_vm_gpuva *prev_va;
159
160 /**
161 * @next_va: Prealloced VA mapping object (init in callback).
162 * Used when a mapping or unmapping operation overlaps an existing
163 * mapping and splits away the end into a new mapping.
164 */
165 struct pvr_vm_gpuva *next_va;
166
167 /** @offset: Offset into @pvr_obj to begin mapping from. */
168 u64 offset;
169
170 /** @device_addr: Device-virtual address at the start of the mapping. */
171 u64 device_addr;
172
173 /** @size: Size of the desired mapping. */
174 u64 size;
175};
176
177/**
178 * pvr_vm_bind_op_exec() - Execute a single bind op.
179 * @bind_op: Bind op context.
180 *
181 * Returns:
182 * * 0 on success,
183 * * Any error code returned by drm_gpuva_sm_map(), drm_gpuva_sm_unmap(), or
184 * a callback function.
185 */
186static int pvr_vm_bind_op_exec(struct pvr_vm_bind_op *bind_op)
187{
188 switch (bind_op->type) {
189 case PVR_VM_BIND_TYPE_MAP: {
190 const struct drm_gpuvm_map_req map_req = {
191 .map.va.addr = bind_op->device_addr,
192 .map.va.range = bind_op->size,
193 .map.gem.obj = gem_from_pvr_gem(bind_op->pvr_obj),
194 .map.gem.offset = bind_op->offset,
195 };
196
197 return drm_gpuvm_sm_map(gpuvm: &bind_op->vm_ctx->gpuvm_mgr,
198 priv: bind_op, req: &map_req);
199 }
200
201 case PVR_VM_BIND_TYPE_UNMAP:
202 return drm_gpuvm_sm_unmap(gpuvm: &bind_op->vm_ctx->gpuvm_mgr,
203 priv: bind_op, addr: bind_op->device_addr,
204 range: bind_op->size);
205 }
206
207 /*
208 * This shouldn't happen unless something went wrong
209 * in drm_sched.
210 */
211 WARN_ON(1);
212 return -EINVAL;
213}
214
215static void pvr_vm_bind_op_fini(struct pvr_vm_bind_op *bind_op)
216{
217 drm_gpuvm_bo_put(vm_bo: bind_op->gpuvm_bo);
218
219 kfree(objp: bind_op->new_va);
220 kfree(objp: bind_op->prev_va);
221 kfree(objp: bind_op->next_va);
222
223 if (bind_op->pvr_obj)
224 pvr_gem_object_put(pvr_obj: bind_op->pvr_obj);
225
226 if (bind_op->mmu_op_ctx)
227 pvr_mmu_op_context_destroy(op_ctx: bind_op->mmu_op_ctx);
228}
229
230static int
231pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op,
232 struct pvr_vm_context *vm_ctx,
233 struct pvr_gem_object *pvr_obj, u64 offset,
234 u64 device_addr, u64 size)
235{
236 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
237 const bool is_user = vm_ctx != vm_ctx->pvr_dev->kernel_vm_ctx;
238 const u64 pvr_obj_size = pvr_gem_object_size(pvr_obj);
239 struct sg_table *sgt;
240 u64 offset_plus_size;
241 int err;
242
243 if (check_add_overflow(offset, size, &offset_plus_size))
244 return -EINVAL;
245
246 if (is_user &&
247 !pvr_find_heap_containing(pvr_dev: vm_ctx->pvr_dev, addr: device_addr, size)) {
248 return -EINVAL;
249 }
250
251 if (!pvr_device_addr_and_size_are_valid(vm_ctx, device_addr, size) ||
252 offset & ~PAGE_MASK || size & ~PAGE_MASK ||
253 offset >= pvr_obj_size || offset_plus_size > pvr_obj_size)
254 return -EINVAL;
255
256 bind_op->type = PVR_VM_BIND_TYPE_MAP;
257
258 dma_resv_lock(obj: obj->resv, NULL);
259 bind_op->gpuvm_bo = drm_gpuvm_bo_obtain(gpuvm: &vm_ctx->gpuvm_mgr, obj);
260 dma_resv_unlock(obj: obj->resv);
261 if (IS_ERR(ptr: bind_op->gpuvm_bo))
262 return PTR_ERR(ptr: bind_op->gpuvm_bo);
263
264 bind_op->new_va = kzalloc(sizeof(*bind_op->new_va), GFP_KERNEL);
265 bind_op->prev_va = kzalloc(sizeof(*bind_op->prev_va), GFP_KERNEL);
266 bind_op->next_va = kzalloc(sizeof(*bind_op->next_va), GFP_KERNEL);
267 if (!bind_op->new_va || !bind_op->prev_va || !bind_op->next_va) {
268 err = -ENOMEM;
269 goto err_bind_op_fini;
270 }
271
272 /* Pin pages so they're ready for use. */
273 sgt = pvr_gem_object_get_pages_sgt(pvr_obj);
274 err = PTR_ERR_OR_ZERO(ptr: sgt);
275 if (err)
276 goto err_bind_op_fini;
277
278 bind_op->mmu_op_ctx =
279 pvr_mmu_op_context_create(ctx: vm_ctx->mmu_ctx, sgt, sgt_offset: offset, size);
280 err = PTR_ERR_OR_ZERO(ptr: bind_op->mmu_op_ctx);
281 if (err) {
282 bind_op->mmu_op_ctx = NULL;
283 goto err_bind_op_fini;
284 }
285
286 bind_op->pvr_obj = pvr_obj;
287 bind_op->vm_ctx = vm_ctx;
288 bind_op->device_addr = device_addr;
289 bind_op->size = size;
290 bind_op->offset = offset;
291
292 return 0;
293
294err_bind_op_fini:
295 pvr_vm_bind_op_fini(bind_op);
296
297 return err;
298}
299
300static int
301pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op,
302 struct pvr_vm_context *vm_ctx,
303 struct pvr_gem_object *pvr_obj,
304 u64 device_addr, u64 size)
305{
306 int err;
307
308 if (!pvr_device_addr_and_size_are_valid(vm_ctx, device_addr, size))
309 return -EINVAL;
310
311 bind_op->type = PVR_VM_BIND_TYPE_UNMAP;
312
313 bind_op->prev_va = kzalloc(sizeof(*bind_op->prev_va), GFP_KERNEL);
314 bind_op->next_va = kzalloc(sizeof(*bind_op->next_va), GFP_KERNEL);
315 if (!bind_op->prev_va || !bind_op->next_va) {
316 err = -ENOMEM;
317 goto err_bind_op_fini;
318 }
319
320 bind_op->mmu_op_ctx =
321 pvr_mmu_op_context_create(ctx: vm_ctx->mmu_ctx, NULL, sgt_offset: 0, size: 0);
322 err = PTR_ERR_OR_ZERO(ptr: bind_op->mmu_op_ctx);
323 if (err) {
324 bind_op->mmu_op_ctx = NULL;
325 goto err_bind_op_fini;
326 }
327
328 bind_op->pvr_obj = pvr_obj;
329 bind_op->vm_ctx = vm_ctx;
330 bind_op->device_addr = device_addr;
331 bind_op->size = size;
332
333 return 0;
334
335err_bind_op_fini:
336 pvr_vm_bind_op_fini(bind_op);
337
338 return err;
339}
340
341/**
342 * pvr_vm_gpuva_map() - Insert a mapping into a memory context.
343 * @op: gpuva op containing the remap details.
344 * @op_ctx: Operation context.
345 *
346 * Context: Called by drm_gpuvm_sm_map following a successful mapping while
347 * @op_ctx.vm_ctx mutex is held.
348 *
349 * Return:
350 * * 0 on success, or
351 * * Any error returned by pvr_mmu_map().
352 */
353static int
354pvr_vm_gpuva_map(struct drm_gpuva_op *op, void *op_ctx)
355{
356 struct pvr_gem_object *pvr_gem = gem_to_pvr_gem(op->map.gem.obj);
357 struct pvr_vm_bind_op *ctx = op_ctx;
358 int err;
359
360 if ((op->map.gem.offset | op->map.va.range) & ~PVR_DEVICE_PAGE_MASK)
361 return -EINVAL;
362
363 err = pvr_mmu_map(op_ctx: ctx->mmu_op_ctx, size: op->map.va.range, flags: pvr_gem->flags,
364 device_addr: op->map.va.addr);
365 if (err)
366 return err;
367
368 drm_gpuva_map(gpuvm: &ctx->vm_ctx->gpuvm_mgr, va: &ctx->new_va->base, op: &op->map);
369 drm_gpuva_link(va: &ctx->new_va->base, vm_bo: ctx->gpuvm_bo);
370 ctx->new_va = NULL;
371
372 return 0;
373}
374
375/**
376 * pvr_vm_gpuva_unmap() - Remove a mapping from a memory context.
377 * @op: gpuva op containing the unmap details.
378 * @op_ctx: Operation context.
379 *
380 * Context: Called by drm_gpuvm_sm_unmap following a successful unmapping while
381 * @op_ctx.vm_ctx mutex is held.
382 *
383 * Return:
384 * * 0 on success, or
385 * * Any error returned by pvr_mmu_unmap().
386 */
387static int
388pvr_vm_gpuva_unmap(struct drm_gpuva_op *op, void *op_ctx)
389{
390 struct pvr_vm_bind_op *ctx = op_ctx;
391
392 int err = pvr_mmu_unmap(op_ctx: ctx->mmu_op_ctx, device_addr: op->unmap.va->va.addr,
393 size: op->unmap.va->va.range);
394
395 if (err)
396 return err;
397
398 drm_gpuva_unmap(op: &op->unmap);
399 drm_gpuva_unlink(va: op->unmap.va);
400 kfree(to_pvr_vm_gpuva(op->unmap.va));
401
402 return 0;
403}
404
405/**
406 * pvr_vm_gpuva_remap() - Remap a mapping within a memory context.
407 * @op: gpuva op containing the remap details.
408 * @op_ctx: Operation context.
409 *
410 * Context: Called by either drm_gpuvm_sm_map or drm_gpuvm_sm_unmap when a
411 * mapping or unmapping operation causes a region to be split. The
412 * @op_ctx.vm_ctx mutex is held.
413 *
414 * Return:
415 * * 0 on success, or
416 * * Any error returned by pvr_vm_gpuva_unmap() or pvr_vm_gpuva_unmap().
417 */
418static int
419pvr_vm_gpuva_remap(struct drm_gpuva_op *op, void *op_ctx)
420{
421 struct pvr_vm_bind_op *ctx = op_ctx;
422 u64 va_start = 0, va_range = 0;
423 int err;
424
425 drm_gpuva_op_remap_to_unmap_range(op: &op->remap, start_addr: &va_start, range: &va_range);
426 err = pvr_mmu_unmap(op_ctx: ctx->mmu_op_ctx, device_addr: va_start, size: va_range);
427 if (err)
428 return err;
429
430 /* No actual remap required: the page table tree depth is fixed to 3,
431 * and we use 4k page table entries only for now.
432 */
433 drm_gpuva_remap(prev: &ctx->prev_va->base, next: &ctx->next_va->base, op: &op->remap);
434
435 if (op->remap.prev) {
436 pvr_gem_object_get(gem_to_pvr_gem(ctx->prev_va->base.gem.obj));
437 drm_gpuva_link(va: &ctx->prev_va->base, vm_bo: ctx->gpuvm_bo);
438 ctx->prev_va = NULL;
439 }
440
441 if (op->remap.next) {
442 pvr_gem_object_get(gem_to_pvr_gem(ctx->next_va->base.gem.obj));
443 drm_gpuva_link(va: &ctx->next_va->base, vm_bo: ctx->gpuvm_bo);
444 ctx->next_va = NULL;
445 }
446
447 drm_gpuva_unlink(va: op->remap.unmap->va);
448 kfree(to_pvr_vm_gpuva(op->remap.unmap->va));
449
450 return 0;
451}
452
453/*
454 * Public API
455 *
456 * For an overview of these functions, see *DOC: Public API* in "pvr_vm.h".
457 */
458
459/**
460 * pvr_device_addr_is_valid() - Tests whether a device-virtual address
461 * is valid.
462 * @device_addr: Virtual device address to test.
463 *
464 * Return:
465 * * %true if @device_addr is within the valid range for a device page
466 * table and is aligned to the device page size, or
467 * * %false otherwise.
468 */
469bool
470pvr_device_addr_is_valid(u64 device_addr)
471{
472 return (device_addr & ~PVR_PAGE_TABLE_ADDR_MASK) == 0 &&
473 (device_addr & ~PVR_DEVICE_PAGE_MASK) == 0;
474}
475
476/**
477 * pvr_device_addr_and_size_are_valid() - Tests whether a device-virtual
478 * address and associated size are both valid.
479 * @vm_ctx: Target VM context.
480 * @device_addr: Virtual device address to test.
481 * @size: Size of the range based at @device_addr to test.
482 *
483 * Calling pvr_device_addr_is_valid() twice (once on @size, and again on
484 * @device_addr + @size) to verify a device-virtual address range initially
485 * seems intuitive, but it produces a false-negative when the address range
486 * is right at the end of device-virtual address space.
487 *
488 * This function catches that corner case, as well as checking that
489 * @size is non-zero.
490 *
491 * Return:
492 * * %true if @device_addr is device page aligned; @size is device page
493 * aligned; the range specified by @device_addr and @size is within the
494 * bounds of the device-virtual address space, and @size is non-zero, or
495 * * %false otherwise.
496 */
497bool
498pvr_device_addr_and_size_are_valid(struct pvr_vm_context *vm_ctx,
499 u64 device_addr, u64 size)
500{
501 return pvr_device_addr_is_valid(device_addr) &&
502 drm_gpuvm_range_valid(gpuvm: &vm_ctx->gpuvm_mgr, addr: device_addr, range: size) &&
503 size != 0 && (size & ~PVR_DEVICE_PAGE_MASK) == 0 &&
504 (device_addr + size <= PVR_PAGE_TABLE_ADDR_SPACE_SIZE);
505}
506
507static void pvr_gpuvm_free(struct drm_gpuvm *gpuvm)
508{
509 kfree(objp: to_pvr_vm_context(gpuvm));
510}
511
512static const struct drm_gpuvm_ops pvr_vm_gpuva_ops = {
513 .vm_free = pvr_gpuvm_free,
514 .sm_step_map = pvr_vm_gpuva_map,
515 .sm_step_remap = pvr_vm_gpuva_remap,
516 .sm_step_unmap = pvr_vm_gpuva_unmap,
517};
518
519static void
520fw_mem_context_init(void *cpu_ptr, void *priv)
521{
522 struct rogue_fwif_fwmemcontext *fw_mem_ctx = cpu_ptr;
523 struct pvr_vm_context *vm_ctx = priv;
524
525 fw_mem_ctx->pc_dev_paddr = pvr_vm_get_page_table_root_addr(vm_ctx);
526 fw_mem_ctx->page_cat_base_reg_set = ROGUE_FW_BIF_INVALID_PCSET;
527}
528
529/**
530 * pvr_vm_create_context() - Create a new VM context.
531 * @pvr_dev: Target PowerVR device.
532 * @is_userspace_context: %true if this context is for userspace. This will
533 * create a firmware memory context for the VM context
534 * and disable warnings when tearing down mappings.
535 *
536 * Return:
537 * * A handle to the newly-minted VM context on success,
538 * * -%EINVAL if the feature "virtual address space bits" on @pvr_dev is
539 * missing or has an unsupported value,
540 * * -%ENOMEM if allocation of the structure behind the opaque handle fails,
541 * or
542 * * Any error encountered while setting up internal structures.
543 */
544struct pvr_vm_context *
545pvr_vm_create_context(struct pvr_device *pvr_dev, bool is_userspace_context)
546{
547 struct drm_device *drm_dev = from_pvr_device(pvr_dev);
548
549 struct pvr_vm_context *vm_ctx;
550 u16 device_addr_bits;
551
552 int err;
553
554 err = PVR_FEATURE_VALUE(pvr_dev, virtual_address_space_bits,
555 &device_addr_bits);
556 if (err) {
557 drm_err(drm_dev,
558 "Failed to get device virtual address space bits\n");
559 return ERR_PTR(error: err);
560 }
561
562 if (device_addr_bits != PVR_PAGE_TABLE_ADDR_BITS) {
563 drm_err(drm_dev,
564 "Device has unsupported virtual address space size\n");
565 return ERR_PTR(error: -EINVAL);
566 }
567
568 vm_ctx = kzalloc(sizeof(*vm_ctx), GFP_KERNEL);
569 if (!vm_ctx)
570 return ERR_PTR(error: -ENOMEM);
571
572 vm_ctx->pvr_dev = pvr_dev;
573
574 vm_ctx->mmu_ctx = pvr_mmu_context_create(pvr_dev);
575 err = PTR_ERR_OR_ZERO(ptr: vm_ctx->mmu_ctx);
576 if (err)
577 goto err_free;
578
579 if (is_userspace_context) {
580 err = pvr_fw_object_create(pvr_dev, size: sizeof(struct rogue_fwif_fwmemcontext),
581 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
582 init: fw_mem_context_init, init_priv: vm_ctx, pvr_obj_out: &vm_ctx->fw_mem_ctx_obj);
583
584 if (err)
585 goto err_page_table_destroy;
586 }
587
588 drm_gem_private_object_init(dev: &pvr_dev->base, obj: &vm_ctx->dummy_gem, size: 0);
589 drm_gpuvm_init(gpuvm: &vm_ctx->gpuvm_mgr,
590 name: is_userspace_context ? "PowerVR-user-VM" : "PowerVR-FW-VM",
591 flags: 0, drm: &pvr_dev->base, r_obj: &vm_ctx->dummy_gem,
592 start_offset: 0, range: 1ULL << device_addr_bits, reserve_offset: 0, reserve_range: 0, ops: &pvr_vm_gpuva_ops);
593
594 mutex_init(&vm_ctx->lock);
595 kref_init(kref: &vm_ctx->ref_count);
596
597 return vm_ctx;
598
599err_page_table_destroy:
600 pvr_mmu_context_destroy(ctx: vm_ctx->mmu_ctx);
601
602err_free:
603 kfree(objp: vm_ctx);
604
605 return ERR_PTR(error: err);
606}
607
608/**
609 * pvr_vm_context_release() - Teardown a VM context.
610 * @ref_count: Pointer to reference counter of the VM context.
611 *
612 * This function also ensures that no mappings are left dangling by calling
613 * pvr_vm_unmap_all.
614 */
615static void
616pvr_vm_context_release(struct kref *ref_count)
617{
618 struct pvr_vm_context *vm_ctx =
619 container_of(ref_count, struct pvr_vm_context, ref_count);
620
621 if (vm_ctx->fw_mem_ctx_obj)
622 pvr_fw_object_destroy(fw_obj: vm_ctx->fw_mem_ctx_obj);
623
624 pvr_vm_unmap_all(vm_ctx);
625
626 pvr_mmu_context_destroy(ctx: vm_ctx->mmu_ctx);
627 drm_gem_private_object_fini(obj: &vm_ctx->dummy_gem);
628 mutex_destroy(lock: &vm_ctx->lock);
629
630 drm_gpuvm_put(gpuvm: &vm_ctx->gpuvm_mgr);
631}
632
633/**
634 * pvr_vm_context_lookup() - Look up VM context from handle
635 * @pvr_file: Pointer to pvr_file structure.
636 * @handle: Object handle.
637 *
638 * Takes reference on VM context object. Call pvr_vm_context_put() to release.
639 *
640 * Returns:
641 * * The requested object on success, or
642 * * %NULL on failure (object does not exist in list, or is not a VM context)
643 */
644struct pvr_vm_context *
645pvr_vm_context_lookup(struct pvr_file *pvr_file, u32 handle)
646{
647 struct pvr_vm_context *vm_ctx;
648
649 xa_lock(&pvr_file->vm_ctx_handles);
650 vm_ctx = xa_load(&pvr_file->vm_ctx_handles, index: handle);
651 pvr_vm_context_get(vm_ctx);
652 xa_unlock(&pvr_file->vm_ctx_handles);
653
654 return vm_ctx;
655}
656
657/**
658 * pvr_vm_context_put() - Release a reference on a VM context
659 * @vm_ctx: Target VM context.
660 *
661 * Returns:
662 * * %true if the VM context was destroyed, or
663 * * %false if there are any references still remaining.
664 */
665bool
666pvr_vm_context_put(struct pvr_vm_context *vm_ctx)
667{
668 if (vm_ctx)
669 return kref_put(kref: &vm_ctx->ref_count, release: pvr_vm_context_release);
670
671 return true;
672}
673
674/**
675 * pvr_destroy_vm_contexts_for_file: Destroy any VM contexts associated with the
676 * given file.
677 * @pvr_file: Pointer to pvr_file structure.
678 *
679 * Removes all vm_contexts associated with @pvr_file from the device VM context
680 * list and drops initial references. vm_contexts will then be destroyed once
681 * all outstanding references are dropped.
682 */
683void pvr_destroy_vm_contexts_for_file(struct pvr_file *pvr_file)
684{
685 struct pvr_vm_context *vm_ctx;
686 unsigned long handle;
687
688 xa_for_each(&pvr_file->vm_ctx_handles, handle, vm_ctx) {
689 /* vm_ctx is not used here because that would create a race with xa_erase */
690 pvr_vm_context_put(vm_ctx: xa_erase(&pvr_file->vm_ctx_handles, index: handle));
691 }
692}
693
694static int
695pvr_vm_lock_extra(struct drm_gpuvm_exec *vm_exec)
696{
697 struct pvr_vm_bind_op *bind_op = vm_exec->extra.priv;
698 struct pvr_gem_object *pvr_obj = bind_op->pvr_obj;
699
700 /* Acquire lock on the GEM object being mapped/unmapped. */
701 return drm_exec_lock_obj(exec: &vm_exec->exec, gem_from_pvr_gem(pvr_obj));
702}
703
704/**
705 * pvr_vm_map() - Map a section of physical memory into a section of
706 * device-virtual memory.
707 * @vm_ctx: Target VM context.
708 * @pvr_obj: Target PowerVR memory object.
709 * @pvr_obj_offset: Offset into @pvr_obj to map from.
710 * @device_addr: Virtual device address at the start of the requested mapping.
711 * @size: Size of the requested mapping.
712 *
713 * No handle is returned to represent the mapping. Instead, callers should
714 * remember @device_addr and use that as a handle.
715 *
716 * Return:
717 * * 0 on success,
718 * * -%EINVAL if @device_addr is not a valid page-aligned device-virtual
719 * address; the region specified by @pvr_obj_offset and @size does not fall
720 * entirely within @pvr_obj, or any part of the specified region of @pvr_obj
721 * is not device-virtual page-aligned,
722 * * Any error encountered while performing internal operations required to
723 * destroy the mapping (returned from pvr_vm_gpuva_map or
724 * pvr_vm_gpuva_remap).
725 */
726int
727pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
728 u64 pvr_obj_offset, u64 device_addr, u64 size)
729{
730 struct pvr_vm_bind_op bind_op = {0};
731 struct drm_gpuvm_exec vm_exec = {
732 .vm = &vm_ctx->gpuvm_mgr,
733 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT |
734 DRM_EXEC_IGNORE_DUPLICATES,
735 .extra = {
736 .fn = pvr_vm_lock_extra,
737 .priv = &bind_op,
738 },
739 };
740
741 int err = pvr_vm_bind_op_map_init(bind_op: &bind_op, vm_ctx, pvr_obj,
742 offset: pvr_obj_offset, device_addr,
743 size);
744
745 if (err)
746 return err;
747
748 pvr_gem_object_get(pvr_obj);
749
750 err = drm_gpuvm_exec_lock(vm_exec: &vm_exec);
751 if (err)
752 goto err_cleanup;
753
754 err = pvr_vm_bind_op_exec(bind_op: &bind_op);
755
756 drm_gpuvm_exec_unlock(vm_exec: &vm_exec);
757
758err_cleanup:
759 pvr_vm_bind_op_fini(bind_op: &bind_op);
760
761 return err;
762}
763
764/**
765 * pvr_vm_unmap_obj_locked() - Unmap an already mapped section of device-virtual
766 * memory.
767 * @vm_ctx: Target VM context.
768 * @pvr_obj: Target PowerVR memory object.
769 * @device_addr: Virtual device address at the start of the target mapping.
770 * @size: Size of the target mapping.
771 *
772 * Return:
773 * * 0 on success,
774 * * -%EINVAL if @device_addr is not a valid page-aligned device-virtual
775 * address,
776 * * Any error encountered while performing internal operations required to
777 * destroy the mapping (returned from pvr_vm_gpuva_unmap or
778 * pvr_vm_gpuva_remap).
779 *
780 * The vm_ctx->lock must be held when calling this function.
781 */
782static int
783pvr_vm_unmap_obj_locked(struct pvr_vm_context *vm_ctx,
784 struct pvr_gem_object *pvr_obj,
785 u64 device_addr, u64 size)
786{
787 struct pvr_vm_bind_op bind_op = {0};
788 struct drm_gpuvm_exec vm_exec = {
789 .vm = &vm_ctx->gpuvm_mgr,
790 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT |
791 DRM_EXEC_IGNORE_DUPLICATES,
792 .extra = {
793 .fn = pvr_vm_lock_extra,
794 .priv = &bind_op,
795 },
796 };
797
798 int err = pvr_vm_bind_op_unmap_init(bind_op: &bind_op, vm_ctx, pvr_obj,
799 device_addr, size);
800 if (err)
801 return err;
802
803 pvr_gem_object_get(pvr_obj);
804
805 err = drm_gpuvm_exec_lock(vm_exec: &vm_exec);
806 if (err)
807 goto err_cleanup;
808
809 err = pvr_vm_bind_op_exec(bind_op: &bind_op);
810
811 drm_gpuvm_exec_unlock(vm_exec: &vm_exec);
812
813err_cleanup:
814 pvr_vm_bind_op_fini(bind_op: &bind_op);
815
816 return err;
817}
818
819/**
820 * pvr_vm_unmap_obj() - Unmap an already mapped section of device-virtual
821 * memory.
822 * @vm_ctx: Target VM context.
823 * @pvr_obj: Target PowerVR memory object.
824 * @device_addr: Virtual device address at the start of the target mapping.
825 * @size: Size of the target mapping.
826 *
827 * Return:
828 * * 0 on success,
829 * * Any error encountered by pvr_vm_unmap_obj_locked.
830 */
831int
832pvr_vm_unmap_obj(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
833 u64 device_addr, u64 size)
834{
835 int err;
836
837 mutex_lock(&vm_ctx->lock);
838 err = pvr_vm_unmap_obj_locked(vm_ctx, pvr_obj, device_addr, size);
839 mutex_unlock(lock: &vm_ctx->lock);
840
841 return err;
842}
843
844/**
845 * pvr_vm_unmap() - Unmap an already mapped section of device-virtual memory.
846 * @vm_ctx: Target VM context.
847 * @device_addr: Virtual device address at the start of the target mapping.
848 * @size: Size of the target mapping.
849 *
850 * Return:
851 * * 0 on success,
852 * * Any error encountered by drm_gpuva_find,
853 * * Any error encountered by pvr_vm_unmap_obj_locked.
854 */
855int
856pvr_vm_unmap(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size)
857{
858 struct pvr_gem_object *pvr_obj;
859 struct drm_gpuva *va;
860 int err;
861
862 mutex_lock(&vm_ctx->lock);
863
864 va = drm_gpuva_find(gpuvm: &vm_ctx->gpuvm_mgr, addr: device_addr, range: size);
865 if (va) {
866 pvr_obj = gem_to_pvr_gem(va->gem.obj);
867 err = pvr_vm_unmap_obj_locked(vm_ctx, pvr_obj,
868 device_addr: va->va.addr, size: va->va.range);
869 } else {
870 err = -ENOENT;
871 }
872
873 mutex_unlock(lock: &vm_ctx->lock);
874
875 return err;
876}
877
878/**
879 * pvr_vm_unmap_all() - Unmap all mappings associated with a VM context.
880 * @vm_ctx: Target VM context.
881 *
882 * This function ensures that no mappings are left dangling by unmapping them
883 * all in order of ascending device-virtual address.
884 */
885void
886pvr_vm_unmap_all(struct pvr_vm_context *vm_ctx)
887{
888 mutex_lock(&vm_ctx->lock);
889
890 for (;;) {
891 struct pvr_gem_object *pvr_obj;
892 struct drm_gpuva *va;
893
894 va = drm_gpuva_find_first(gpuvm: &vm_ctx->gpuvm_mgr,
895 addr: vm_ctx->gpuvm_mgr.mm_start,
896 range: vm_ctx->gpuvm_mgr.mm_range);
897 if (!va)
898 break;
899
900 pvr_obj = gem_to_pvr_gem(va->gem.obj);
901
902 WARN_ON(pvr_vm_unmap_obj_locked(vm_ctx, pvr_obj,
903 va->va.addr, va->va.range));
904 }
905
906 mutex_unlock(lock: &vm_ctx->lock);
907}
908
909/* Static data areas are determined by firmware. */
910static const struct drm_pvr_static_data_area static_data_areas[] = {
911 {
912 .area_usage = DRM_PVR_STATIC_DATA_AREA_FENCE,
913 .location_heap_id = DRM_PVR_HEAP_GENERAL,
914 .offset = 0,
915 .size = 128,
916 },
917 {
918 .area_usage = DRM_PVR_STATIC_DATA_AREA_YUV_CSC,
919 .location_heap_id = DRM_PVR_HEAP_GENERAL,
920 .offset = 128,
921 .size = 1024,
922 },
923 {
924 .area_usage = DRM_PVR_STATIC_DATA_AREA_VDM_SYNC,
925 .location_heap_id = DRM_PVR_HEAP_PDS_CODE_DATA,
926 .offset = 0,
927 .size = 128,
928 },
929 {
930 .area_usage = DRM_PVR_STATIC_DATA_AREA_EOT,
931 .location_heap_id = DRM_PVR_HEAP_PDS_CODE_DATA,
932 .offset = 128,
933 .size = 128,
934 },
935 {
936 .area_usage = DRM_PVR_STATIC_DATA_AREA_VDM_SYNC,
937 .location_heap_id = DRM_PVR_HEAP_USC_CODE,
938 .offset = 0,
939 .size = 128,
940 },
941};
942
943#define GET_RESERVED_SIZE(last_offset, last_size) round_up((last_offset) + (last_size), PAGE_SIZE)
944
945/*
946 * The values given to GET_RESERVED_SIZE() are taken from the last entry in the corresponding
947 * static data area for each heap.
948 */
949static const struct drm_pvr_heap pvr_heaps[] = {
950 [DRM_PVR_HEAP_GENERAL] = {
951 .base = ROGUE_GENERAL_HEAP_BASE,
952 .size = ROGUE_GENERAL_HEAP_SIZE,
953 .flags = 0,
954 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
955 },
956 [DRM_PVR_HEAP_PDS_CODE_DATA] = {
957 .base = ROGUE_PDSCODEDATA_HEAP_BASE,
958 .size = ROGUE_PDSCODEDATA_HEAP_SIZE,
959 .flags = 0,
960 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
961 },
962 [DRM_PVR_HEAP_USC_CODE] = {
963 .base = ROGUE_USCCODE_HEAP_BASE,
964 .size = ROGUE_USCCODE_HEAP_SIZE,
965 .flags = 0,
966 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
967 },
968 [DRM_PVR_HEAP_RGNHDR] = {
969 .base = ROGUE_RGNHDR_HEAP_BASE,
970 .size = ROGUE_RGNHDR_HEAP_SIZE,
971 .flags = 0,
972 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
973 },
974 [DRM_PVR_HEAP_VIS_TEST] = {
975 .base = ROGUE_VISTEST_HEAP_BASE,
976 .size = ROGUE_VISTEST_HEAP_SIZE,
977 .flags = 0,
978 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
979 },
980 [DRM_PVR_HEAP_TRANSFER_FRAG] = {
981 .base = ROGUE_TRANSFER_FRAG_HEAP_BASE,
982 .size = ROGUE_TRANSFER_FRAG_HEAP_SIZE,
983 .flags = 0,
984 .page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
985 },
986};
987
988int
989pvr_static_data_areas_get(const struct pvr_device *pvr_dev,
990 struct drm_pvr_ioctl_dev_query_args *args)
991{
992 struct drm_pvr_dev_query_static_data_areas query = {0};
993 int err;
994
995 if (!args->pointer) {
996 args->size = sizeof(struct drm_pvr_dev_query_static_data_areas);
997 return 0;
998 }
999
1000 err = PVR_UOBJ_GET(query, args->size, args->pointer);
1001 if (err < 0)
1002 return err;
1003
1004 if (!query.static_data_areas.array) {
1005 query.static_data_areas.count = ARRAY_SIZE(static_data_areas);
1006 query.static_data_areas.stride = sizeof(struct drm_pvr_static_data_area);
1007 goto copy_out;
1008 }
1009
1010 if (query.static_data_areas.count > ARRAY_SIZE(static_data_areas))
1011 query.static_data_areas.count = ARRAY_SIZE(static_data_areas);
1012
1013 err = PVR_UOBJ_SET_ARRAY(&query.static_data_areas, static_data_areas);
1014 if (err < 0)
1015 return err;
1016
1017copy_out:
1018 err = PVR_UOBJ_SET(args->pointer, args->size, query);
1019 if (err < 0)
1020 return err;
1021
1022 args->size = sizeof(query);
1023 return 0;
1024}
1025
1026int
1027pvr_heap_info_get(const struct pvr_device *pvr_dev,
1028 struct drm_pvr_ioctl_dev_query_args *args)
1029{
1030 struct drm_pvr_dev_query_heap_info query = {0};
1031 u64 dest;
1032 int err;
1033
1034 if (!args->pointer) {
1035 args->size = sizeof(struct drm_pvr_dev_query_heap_info);
1036 return 0;
1037 }
1038
1039 err = PVR_UOBJ_GET(query, args->size, args->pointer);
1040 if (err < 0)
1041 return err;
1042
1043 if (!query.heaps.array) {
1044 query.heaps.count = ARRAY_SIZE(pvr_heaps);
1045 query.heaps.stride = sizeof(struct drm_pvr_heap);
1046 goto copy_out;
1047 }
1048
1049 if (query.heaps.count > ARRAY_SIZE(pvr_heaps))
1050 query.heaps.count = ARRAY_SIZE(pvr_heaps);
1051
1052 /* Region header heap is only present if BRN63142 is present. */
1053 dest = query.heaps.array;
1054 for (size_t i = 0; i < query.heaps.count; i++) {
1055 struct drm_pvr_heap heap = pvr_heaps[i];
1056
1057 if (i == DRM_PVR_HEAP_RGNHDR && !PVR_HAS_QUIRK(pvr_dev, 63142))
1058 heap.size = 0;
1059
1060 err = PVR_UOBJ_SET(dest, query.heaps.stride, heap);
1061 if (err < 0)
1062 return err;
1063
1064 dest += query.heaps.stride;
1065 }
1066
1067copy_out:
1068 err = PVR_UOBJ_SET(args->pointer, args->size, query);
1069 if (err < 0)
1070 return err;
1071
1072 args->size = sizeof(query);
1073 return 0;
1074}
1075
1076/**
1077 * pvr_heap_contains_range() - Determine if a given heap contains the specified
1078 * device-virtual address range.
1079 * @pvr_heap: Target heap.
1080 * @start: Inclusive start of the target range.
1081 * @end: Inclusive end of the target range.
1082 *
1083 * It is an error to call this function with values of @start and @end that do
1084 * not satisfy the condition @start <= @end.
1085 */
1086static __always_inline bool
1087pvr_heap_contains_range(const struct drm_pvr_heap *pvr_heap, u64 start, u64 end)
1088{
1089 return pvr_heap->base <= start && end < pvr_heap->base + pvr_heap->size;
1090}
1091
1092/**
1093 * pvr_find_heap_containing() - Find a heap which contains the specified
1094 * device-virtual address range.
1095 * @pvr_dev: Target PowerVR device.
1096 * @start: Start of the target range.
1097 * @size: Size of the target range.
1098 *
1099 * Return:
1100 * * A pointer to a constant instance of struct drm_pvr_heap representing the
1101 * heap containing the entire range specified by @start and @size on
1102 * success, or
1103 * * %NULL if no such heap exists.
1104 */
1105const struct drm_pvr_heap *
1106pvr_find_heap_containing(struct pvr_device *pvr_dev, u64 start, u64 size)
1107{
1108 u64 end;
1109
1110 if (check_add_overflow(start, size - 1, &end))
1111 return NULL;
1112
1113 /*
1114 * There are no guarantees about the order of address ranges in
1115 * &pvr_heaps, so iterate over the entire array for a heap whose
1116 * range completely encompasses the given range.
1117 */
1118 for (u32 heap_id = 0; heap_id < ARRAY_SIZE(pvr_heaps); heap_id++) {
1119 /* Filter heaps that present only with an associated quirk */
1120 if (heap_id == DRM_PVR_HEAP_RGNHDR &&
1121 !PVR_HAS_QUIRK(pvr_dev, 63142)) {
1122 continue;
1123 }
1124
1125 if (pvr_heap_contains_range(pvr_heap: &pvr_heaps[heap_id], start, end))
1126 return &pvr_heaps[heap_id];
1127 }
1128
1129 return NULL;
1130}
1131
1132/**
1133 * pvr_vm_find_gem_object() - Look up a buffer object from a given
1134 * device-virtual address.
1135 * @vm_ctx: [IN] Target VM context.
1136 * @device_addr: [IN] Virtual device address at the start of the required
1137 * object.
1138 * @mapped_offset_out: [OUT] Pointer to location to write offset of the start
1139 * of the mapped region within the buffer object. May be
1140 * %NULL if this information is not required.
1141 * @mapped_size_out: [OUT] Pointer to location to write size of the mapped
1142 * region. May be %NULL if this information is not required.
1143 *
1144 * If successful, a reference will be taken on the buffer object. The caller
1145 * must drop the reference with pvr_gem_object_put().
1146 *
1147 * Return:
1148 * * The PowerVR buffer object mapped at @device_addr if one exists, or
1149 * * %NULL otherwise.
1150 */
1151struct pvr_gem_object *
1152pvr_vm_find_gem_object(struct pvr_vm_context *vm_ctx, u64 device_addr,
1153 u64 *mapped_offset_out, u64 *mapped_size_out)
1154{
1155 struct pvr_gem_object *pvr_obj;
1156 struct drm_gpuva *va;
1157
1158 mutex_lock(&vm_ctx->lock);
1159
1160 va = drm_gpuva_find_first(gpuvm: &vm_ctx->gpuvm_mgr, addr: device_addr, range: 1);
1161 if (!va)
1162 goto err_unlock;
1163
1164 pvr_obj = gem_to_pvr_gem(va->gem.obj);
1165 pvr_gem_object_get(pvr_obj);
1166
1167 if (mapped_offset_out)
1168 *mapped_offset_out = va->gem.offset;
1169 if (mapped_size_out)
1170 *mapped_size_out = va->va.range;
1171
1172 mutex_unlock(lock: &vm_ctx->lock);
1173
1174 return pvr_obj;
1175
1176err_unlock:
1177 mutex_unlock(lock: &vm_ctx->lock);
1178
1179 return NULL;
1180}
1181
1182/**
1183 * pvr_vm_get_fw_mem_context: Get object representing firmware memory context
1184 * @vm_ctx: Target VM context.
1185 *
1186 * Returns:
1187 * * FW object representing firmware memory context, or
1188 * * %NULL if this VM context does not have a firmware memory context.
1189 */
1190struct pvr_fw_object *
1191pvr_vm_get_fw_mem_context(struct pvr_vm_context *vm_ctx)
1192{
1193 return vm_ctx->fw_mem_ctx_obj;
1194}
1195

source code of linux/drivers/gpu/drm/imagination/pvr_vm.c