1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include "pvr_device.h"
5#include "pvr_gem.h"
6#include "pvr_vm.h"
7
8#include <drm/drm_gem.h>
9#include <drm/drm_prime.h>
10
11#include <linux/compiler.h>
12#include <linux/compiler_attributes.h>
13#include <linux/dma-buf.h>
14#include <linux/dma-direction.h>
15#include <linux/dma-mapping.h>
16#include <linux/err.h>
17#include <linux/gfp.h>
18#include <linux/iosys-map.h>
19#include <linux/log2.h>
20#include <linux/mutex.h>
21#include <linux/pagemap.h>
22#include <linux/property.h>
23#include <linux/refcount.h>
24#include <linux/scatterlist.h>
25
26static void pvr_gem_object_free(struct drm_gem_object *obj)
27{
28 drm_gem_shmem_object_free(obj);
29}
30
31static struct dma_buf *pvr_gem_export(struct drm_gem_object *obj, int flags)
32{
33 struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(obj);
34
35 if (pvr_obj->flags & DRM_PVR_BO_PM_FW_PROTECT)
36 return ERR_PTR(error: -EPERM);
37
38 return drm_gem_prime_export(obj, flags);
39}
40
41static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *vma)
42{
43 struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(gem_obj);
44 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
45
46 if (!(pvr_obj->flags & DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS))
47 return -EINVAL;
48
49 return drm_gem_shmem_mmap(shmem: shmem_obj, vma);
50}
51
52static const struct drm_gem_object_funcs pvr_gem_object_funcs = {
53 .free = pvr_gem_object_free,
54 .print_info = drm_gem_shmem_object_print_info,
55 .export = pvr_gem_export,
56 .pin = drm_gem_shmem_object_pin,
57 .unpin = drm_gem_shmem_object_unpin,
58 .get_sg_table = drm_gem_shmem_object_get_sg_table,
59 .vmap = drm_gem_shmem_object_vmap,
60 .vunmap = drm_gem_shmem_object_vunmap,
61 .mmap = pvr_gem_mmap,
62 .vm_ops = &drm_gem_shmem_vm_ops,
63};
64
65/**
66 * pvr_gem_object_flags_validate() - Verify that a collection of PowerVR GEM
67 * mapping and/or creation flags form a valid combination.
68 * @flags: PowerVR GEM mapping/creation flags to validate.
69 *
70 * This function explicitly allows kernel-only flags. All ioctl entrypoints
71 * should do their own validation as well as relying on this function.
72 *
73 * Return:
74 * * %true if @flags contains valid mapping and/or creation flags, or
75 * * %false otherwise.
76 */
77static bool
78pvr_gem_object_flags_validate(u64 flags)
79{
80 static const u64 invalid_combinations[] = {
81 /*
82 * Memory flagged as PM/FW-protected cannot be mapped to
83 * userspace. To make this explicit, we require that the two
84 * flags allowing each of these respective features are never
85 * specified together.
86 */
87 (DRM_PVR_BO_PM_FW_PROTECT |
88 DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS),
89 };
90
91 /*
92 * Check for bits set in undefined regions. Reserved regions refer to
93 * options that can only be set by the kernel. These are explicitly
94 * allowed in most cases, and must be checked specifically in IOCTL
95 * callback code.
96 */
97 if ((flags & PVR_BO_UNDEFINED_MASK) != 0)
98 return false;
99
100 /*
101 * Check for all combinations of flags marked as invalid in the array
102 * above.
103 */
104 for (int i = 0; i < ARRAY_SIZE(invalid_combinations); ++i) {
105 u64 combo = invalid_combinations[i];
106
107 if ((flags & combo) == combo)
108 return false;
109 }
110
111 return true;
112}
113
114/**
115 * pvr_gem_object_into_handle() - Convert a reference to an object into a
116 * userspace-accessible handle.
117 * @pvr_obj: [IN] Target PowerVR-specific object.
118 * @pvr_file: [IN] File to associate the handle with.
119 * @handle: [OUT] Pointer to store the created handle in. Remains unmodified if
120 * an error is encountered.
121 *
122 * If an error is encountered, ownership of @pvr_obj will not have been
123 * transferred. If this function succeeds, however, further use of @pvr_obj is
124 * considered undefined behaviour unless another reference to it is explicitly
125 * held.
126 *
127 * Return:
128 * * 0 on success, or
129 * * Any error encountered while attempting to allocate a handle on @pvr_file.
130 */
131int
132pvr_gem_object_into_handle(struct pvr_gem_object *pvr_obj,
133 struct pvr_file *pvr_file, u32 *handle)
134{
135 struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj);
136 struct drm_file *file = from_pvr_file(pvr_file);
137
138 u32 new_handle;
139 int err;
140
141 err = drm_gem_handle_create(file_priv: file, obj: gem_obj, handlep: &new_handle);
142 if (err)
143 return err;
144
145 /*
146 * Release our reference to @pvr_obj, effectively transferring
147 * ownership to the handle.
148 */
149 pvr_gem_object_put(pvr_obj);
150
151 /*
152 * Do not store the new handle in @handle until no more errors can
153 * occur.
154 */
155 *handle = new_handle;
156
157 return 0;
158}
159
160/**
161 * pvr_gem_object_from_handle() - Obtain a reference to an object from a
162 * userspace handle.
163 * @pvr_file: PowerVR-specific file to which @handle is associated.
164 * @handle: Userspace handle referencing the target object.
165 *
166 * On return, @handle always maintains its reference to the requested object
167 * (if it had one in the first place). If this function succeeds, the returned
168 * object will hold an additional reference. When the caller is finished with
169 * the returned object, they should call pvr_gem_object_put() on it to release
170 * this reference.
171 *
172 * Return:
173 * * A pointer to the requested PowerVR-specific object on success, or
174 * * %NULL otherwise.
175 */
176struct pvr_gem_object *
177pvr_gem_object_from_handle(struct pvr_file *pvr_file, u32 handle)
178{
179 struct drm_file *file = from_pvr_file(pvr_file);
180 struct drm_gem_object *gem_obj;
181
182 gem_obj = drm_gem_object_lookup(filp: file, handle);
183 if (!gem_obj)
184 return NULL;
185
186 return gem_to_pvr_gem(gem_obj);
187}
188
189/**
190 * pvr_gem_object_vmap() - Map a PowerVR GEM object into CPU virtual address
191 * space.
192 * @pvr_obj: Target PowerVR GEM object.
193 *
194 * Once the caller is finished with the CPU mapping, they must call
195 * pvr_gem_object_vunmap() on @pvr_obj.
196 *
197 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_cpu() is called to make
198 * sure the CPU mapping is consistent.
199 *
200 * Return:
201 * * A pointer to the CPU mapping on success,
202 * * -%ENOMEM if the mapping fails, or
203 * * Any error encountered while attempting to acquire a reference to the
204 * backing pages for @pvr_obj.
205 */
206void *
207pvr_gem_object_vmap(struct pvr_gem_object *pvr_obj)
208{
209 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
210 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
211 struct iosys_map map;
212 int err;
213
214 dma_resv_lock(obj: obj->resv, NULL);
215
216 err = drm_gem_shmem_vmap_locked(shmem: shmem_obj, map: &map);
217 if (err)
218 goto err_unlock;
219
220 if (pvr_obj->flags & PVR_BO_CPU_CACHED) {
221 struct device *dev = shmem_obj->base.dev->dev;
222
223 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped
224 * in GPU space yet.
225 */
226 if (shmem_obj->sgt)
227 dma_sync_sgtable_for_cpu(dev, sgt: shmem_obj->sgt, dir: DMA_BIDIRECTIONAL);
228 }
229
230 dma_resv_unlock(obj: obj->resv);
231
232 return map.vaddr;
233
234err_unlock:
235 dma_resv_unlock(obj: obj->resv);
236
237 return ERR_PTR(error: err);
238}
239
240/**
241 * pvr_gem_object_vunmap() - Unmap a PowerVR memory object from CPU virtual
242 * address space.
243 * @pvr_obj: Target PowerVR GEM object.
244 *
245 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_device() is called to make
246 * sure the GPU mapping is consistent.
247 */
248void
249pvr_gem_object_vunmap(struct pvr_gem_object *pvr_obj)
250{
251 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
252 struct iosys_map map = IOSYS_MAP_INIT_VADDR(shmem_obj->vaddr);
253 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
254
255 if (WARN_ON(!map.vaddr))
256 return;
257
258 dma_resv_lock(obj: obj->resv, NULL);
259
260 if (pvr_obj->flags & PVR_BO_CPU_CACHED) {
261 struct device *dev = shmem_obj->base.dev->dev;
262
263 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped
264 * in GPU space yet.
265 */
266 if (shmem_obj->sgt)
267 dma_sync_sgtable_for_device(dev, sgt: shmem_obj->sgt, dir: DMA_BIDIRECTIONAL);
268 }
269
270 drm_gem_shmem_vunmap_locked(shmem: shmem_obj, map: &map);
271
272 dma_resv_unlock(obj: obj->resv);
273}
274
275/**
276 * pvr_gem_object_zero() - Zeroes the physical memory behind an object.
277 * @pvr_obj: Target PowerVR GEM object.
278 *
279 * Return:
280 * * 0 on success, or
281 * * Any error encountered while attempting to map @pvr_obj to the CPU (see
282 * pvr_gem_object_vmap()).
283 */
284static int
285pvr_gem_object_zero(struct pvr_gem_object *pvr_obj)
286{
287 void *cpu_ptr;
288
289 cpu_ptr = pvr_gem_object_vmap(pvr_obj);
290 if (IS_ERR(ptr: cpu_ptr))
291 return PTR_ERR(ptr: cpu_ptr);
292
293 memset(cpu_ptr, 0, pvr_gem_object_size(pvr_obj));
294
295 /* Make sure the zero-ing is done before vumap-ing the object. */
296 wmb();
297
298 pvr_gem_object_vunmap(pvr_obj);
299
300 return 0;
301}
302
303/**
304 * pvr_gem_create_object() - Allocate and pre-initializes a pvr_gem_object
305 * @drm_dev: DRM device creating this object.
306 * @size: Size of the object to allocate in bytes.
307 *
308 * Return:
309 * * The new pre-initialized GEM object on success,
310 * * -ENOMEM if the allocation failed.
311 */
312struct drm_gem_object *pvr_gem_create_object(struct drm_device *drm_dev, size_t size)
313{
314 struct drm_gem_object *gem_obj;
315 struct pvr_gem_object *pvr_obj;
316
317 pvr_obj = kzalloc(sizeof(*pvr_obj), GFP_KERNEL);
318 if (!pvr_obj)
319 return ERR_PTR(error: -ENOMEM);
320
321 gem_obj = gem_from_pvr_gem(pvr_obj);
322 gem_obj->funcs = &pvr_gem_object_funcs;
323
324 return gem_obj;
325}
326
327/**
328 * pvr_gem_object_create() - Creates a PowerVR-specific buffer object.
329 * @pvr_dev: Target PowerVR device.
330 * @size: Size of the object to allocate in bytes. Must be greater than zero.
331 * Any value which is not an exact multiple of the system page size will be
332 * rounded up to satisfy this condition.
333 * @flags: Options which affect both this operation and future mapping
334 * operations performed on the returned object. Must be a combination of
335 * DRM_PVR_BO_* and/or PVR_BO_* flags.
336 *
337 * The created object may be larger than @size, but can never be smaller. To
338 * get the exact size, call pvr_gem_object_size() on the returned pointer.
339 *
340 * Return:
341 * * The newly-minted PowerVR-specific buffer object on success,
342 * * -%EINVAL if @size is zero or @flags is not valid,
343 * * -%ENOMEM if sufficient physical memory cannot be allocated, or
344 * * Any other error returned by drm_gem_create_mmap_offset().
345 */
346struct pvr_gem_object *
347pvr_gem_object_create(struct pvr_device *pvr_dev, size_t size, u64 flags)
348{
349 struct drm_device *drm_dev = from_pvr_device(pvr_dev);
350 struct drm_gem_shmem_object *shmem_obj;
351 struct pvr_gem_object *pvr_obj;
352 struct sg_table *sgt;
353 int err;
354
355 /* Verify @size and @flags before continuing. */
356 if (size == 0 || !pvr_gem_object_flags_validate(flags))
357 return ERR_PTR(error: -EINVAL);
358
359 if (device_get_dma_attr(dev: drm_dev->dev) == DEV_DMA_COHERENT)
360 flags |= PVR_BO_CPU_CACHED;
361
362 shmem_obj = drm_gem_shmem_create(dev: drm_dev, size);
363 if (IS_ERR(ptr: shmem_obj))
364 return ERR_CAST(ptr: shmem_obj);
365
366 shmem_obj->pages_mark_dirty_on_put = true;
367 shmem_obj->map_wc = !(flags & PVR_BO_CPU_CACHED);
368 pvr_obj = shmem_gem_to_pvr_gem(shmem_obj);
369 pvr_obj->flags = flags;
370
371 sgt = drm_gem_shmem_get_pages_sgt(shmem: shmem_obj);
372 if (IS_ERR(ptr: sgt)) {
373 err = PTR_ERR(ptr: sgt);
374 goto err_shmem_object_free;
375 }
376
377 dma_sync_sgtable_for_device(dev: drm_dev->dev, sgt, dir: DMA_BIDIRECTIONAL);
378
379 /*
380 * Do this last because pvr_gem_object_zero() requires a fully
381 * configured instance of struct pvr_gem_object.
382 */
383 pvr_gem_object_zero(pvr_obj);
384
385 return pvr_obj;
386
387err_shmem_object_free:
388 drm_gem_shmem_free(shmem: shmem_obj);
389
390 return ERR_PTR(error: err);
391}
392
393/**
394 * pvr_gem_get_dma_addr() - Get DMA address for given offset in object
395 * @pvr_obj: Pointer to object to lookup address in.
396 * @offset: Offset within object to lookup address at.
397 * @dma_addr_out: Pointer to location to store DMA address.
398 *
399 * Returns:
400 * * 0 on success, or
401 * * -%EINVAL if object is not currently backed, or if @offset is out of valid
402 * range for this object.
403 */
404int
405pvr_gem_get_dma_addr(struct pvr_gem_object *pvr_obj, u32 offset,
406 dma_addr_t *dma_addr_out)
407{
408 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);
409 u32 accumulated_offset = 0;
410 struct scatterlist *sgl;
411 unsigned int sgt_idx;
412
413 WARN_ON(!shmem_obj->sgt);
414 for_each_sgtable_dma_sg(shmem_obj->sgt, sgl, sgt_idx) {
415 u32 new_offset = accumulated_offset + sg_dma_len(sgl);
416
417 if (offset >= accumulated_offset && offset < new_offset) {
418 *dma_addr_out = sg_dma_address(sgl) +
419 (offset - accumulated_offset);
420 return 0;
421 }
422
423 accumulated_offset = new_offset;
424 }
425
426 return -EINVAL;
427}
428

source code of linux/drivers/gpu/drm/imagination/pvr_gem.c