1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include <uapi/drm/pvr_drm.h>
5
6#include <drm/drm_syncobj.h>
7#include <drm/gpu_scheduler.h>
8#include <linux/xarray.h>
9#include <linux/dma-fence-unwrap.h>
10
11#include "pvr_device.h"
12#include "pvr_queue.h"
13#include "pvr_sync.h"
14
15static int
16pvr_check_sync_op(const struct drm_pvr_sync_op *sync_op)
17{
18 u8 handle_type;
19
20 if (sync_op->flags & ~DRM_PVR_SYNC_OP_FLAGS_MASK)
21 return -EINVAL;
22
23 handle_type = sync_op->flags & DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK;
24 if (handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&
25 handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ)
26 return -EINVAL;
27
28 if (handle_type == DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&
29 sync_op->value != 0)
30 return -EINVAL;
31
32 return 0;
33}
34
35static void
36pvr_sync_signal_free(struct pvr_sync_signal *sig_sync)
37{
38 if (!sig_sync)
39 return;
40
41 drm_syncobj_put(obj: sig_sync->syncobj);
42 dma_fence_chain_free(chain: sig_sync->chain);
43 dma_fence_put(fence: sig_sync->fence);
44 kfree(objp: sig_sync);
45}
46
47void
48pvr_sync_signal_array_cleanup(struct xarray *array)
49{
50 struct pvr_sync_signal *sig_sync;
51 unsigned long i;
52
53 xa_for_each(array, i, sig_sync)
54 pvr_sync_signal_free(sig_sync);
55
56 xa_destroy(array);
57}
58
59static struct pvr_sync_signal *
60pvr_sync_signal_array_add(struct xarray *array, struct drm_file *file, u32 handle, u64 point)
61{
62 struct pvr_sync_signal *sig_sync;
63 struct dma_fence *cur_fence;
64 int err;
65 u32 id;
66
67 sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL);
68 if (!sig_sync)
69 return ERR_PTR(error: -ENOMEM);
70
71 sig_sync->handle = handle;
72 sig_sync->point = point;
73
74 if (point > 0) {
75 sig_sync->chain = dma_fence_chain_alloc();
76 if (!sig_sync->chain) {
77 err = -ENOMEM;
78 goto err_free_sig_sync;
79 }
80 }
81
82 sig_sync->syncobj = drm_syncobj_find(file_private: file, handle);
83 if (!sig_sync->syncobj) {
84 err = -EINVAL;
85 goto err_free_sig_sync;
86 }
87
88 /* Retrieve the current fence attached to that point. It's
89 * perfectly fine to get a NULL fence here, it just means there's
90 * no fence attached to that point yet.
91 */
92 if (!drm_syncobj_find_fence(file_private: file, handle, point, flags: 0, fence: &cur_fence))
93 sig_sync->fence = cur_fence;
94
95 err = xa_alloc(xa: array, id: &id, entry: sig_sync, xa_limit_32b, GFP_KERNEL);
96 if (err)
97 goto err_free_sig_sync;
98
99 return sig_sync;
100
101err_free_sig_sync:
102 pvr_sync_signal_free(sig_sync);
103 return ERR_PTR(error: err);
104}
105
106static struct pvr_sync_signal *
107pvr_sync_signal_array_search(struct xarray *array, u32 handle, u64 point)
108{
109 struct pvr_sync_signal *sig_sync;
110 unsigned long i;
111
112 xa_for_each(array, i, sig_sync) {
113 if (handle == sig_sync->handle && point == sig_sync->point)
114 return sig_sync;
115 }
116
117 return NULL;
118}
119
120static struct pvr_sync_signal *
121pvr_sync_signal_array_get(struct xarray *array, struct drm_file *file, u32 handle, u64 point)
122{
123 struct pvr_sync_signal *sig_sync;
124
125 sig_sync = pvr_sync_signal_array_search(array, handle, point);
126 if (sig_sync)
127 return sig_sync;
128
129 return pvr_sync_signal_array_add(array, file, handle, point);
130}
131
132int
133pvr_sync_signal_array_collect_ops(struct xarray *array,
134 struct drm_file *file,
135 u32 sync_op_count,
136 const struct drm_pvr_sync_op *sync_ops)
137{
138 for (u32 i = 0; i < sync_op_count; i++) {
139 struct pvr_sync_signal *sig_sync;
140 int ret;
141
142 if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))
143 continue;
144
145 ret = pvr_check_sync_op(sync_op: &sync_ops[i]);
146 if (ret)
147 return ret;
148
149 sig_sync = pvr_sync_signal_array_get(array, file,
150 handle: sync_ops[i].handle,
151 point: sync_ops[i].value);
152 if (IS_ERR(ptr: sig_sync))
153 return PTR_ERR(ptr: sig_sync);
154 }
155
156 return 0;
157}
158
159int
160pvr_sync_signal_array_update_fences(struct xarray *array,
161 u32 sync_op_count,
162 const struct drm_pvr_sync_op *sync_ops,
163 struct dma_fence *done_fence)
164{
165 for (u32 i = 0; i < sync_op_count; i++) {
166 struct dma_fence *old_fence;
167 struct pvr_sync_signal *sig_sync;
168
169 if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))
170 continue;
171
172 sig_sync = pvr_sync_signal_array_search(array, handle: sync_ops[i].handle,
173 point: sync_ops[i].value);
174 if (WARN_ON(!sig_sync))
175 return -EINVAL;
176
177 old_fence = sig_sync->fence;
178 sig_sync->fence = dma_fence_get(fence: done_fence);
179 dma_fence_put(fence: old_fence);
180
181 if (WARN_ON(!sig_sync->fence))
182 return -EINVAL;
183 }
184
185 return 0;
186}
187
188void
189pvr_sync_signal_array_push_fences(struct xarray *array)
190{
191 struct pvr_sync_signal *sig_sync;
192 unsigned long i;
193
194 xa_for_each(array, i, sig_sync) {
195 if (sig_sync->chain) {
196 drm_syncobj_add_point(syncobj: sig_sync->syncobj, chain: sig_sync->chain,
197 fence: sig_sync->fence, point: sig_sync->point);
198 sig_sync->chain = NULL;
199 } else {
200 drm_syncobj_replace_fence(syncobj: sig_sync->syncobj, fence: sig_sync->fence);
201 }
202 }
203}
204
205static int
206pvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f)
207{
208 struct dma_fence_unwrap iter;
209 u32 native_fence_count = 0;
210 struct dma_fence *uf;
211 int err = 0;
212
213 dma_fence_unwrap_for_each(uf, &iter, f) {
214 if (pvr_queue_fence_is_ufo_backed(f: uf))
215 native_fence_count++;
216 }
217
218 /* No need to unwrap the fence if it's fully non-native. */
219 if (!native_fence_count)
220 return drm_sched_job_add_dependency(job, fence: f);
221
222 dma_fence_unwrap_for_each(uf, &iter, f) {
223 /* There's no dma_fence_unwrap_stop() helper cleaning up the refs
224 * owned by dma_fence_unwrap(), so let's just iterate over all
225 * entries without doing anything when something failed.
226 */
227 if (err)
228 continue;
229
230 if (pvr_queue_fence_is_ufo_backed(f: uf)) {
231 struct drm_sched_fence *s_fence = to_drm_sched_fence(f: uf);
232
233 /* If this is a native dependency, we wait for the scheduled fence,
234 * and we will let pvr_queue_run_job() issue FW waits.
235 */
236 err = drm_sched_job_add_dependency(job,
237 fence: dma_fence_get(fence: &s_fence->scheduled));
238 } else {
239 err = drm_sched_job_add_dependency(job, fence: dma_fence_get(fence: uf));
240 }
241 }
242
243 dma_fence_put(fence: f);
244 return err;
245}
246
247int
248pvr_sync_add_deps_to_job(struct pvr_file *pvr_file, struct drm_sched_job *job,
249 u32 sync_op_count,
250 const struct drm_pvr_sync_op *sync_ops,
251 struct xarray *signal_array)
252{
253 int err = 0;
254
255 if (!sync_op_count)
256 return 0;
257
258 for (u32 i = 0; i < sync_op_count; i++) {
259 struct pvr_sync_signal *sig_sync;
260 struct dma_fence *fence;
261
262 if (sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL)
263 continue;
264
265 err = pvr_check_sync_op(sync_op: &sync_ops[i]);
266 if (err)
267 return err;
268
269 sig_sync = pvr_sync_signal_array_search(array: signal_array, handle: sync_ops[i].handle,
270 point: sync_ops[i].value);
271 if (sig_sync) {
272 if (WARN_ON(!sig_sync->fence))
273 return -EINVAL;
274
275 fence = dma_fence_get(fence: sig_sync->fence);
276 } else {
277 err = drm_syncobj_find_fence(from_pvr_file(pvr_file), handle: sync_ops[i].handle,
278 point: sync_ops[i].value, flags: 0, fence: &fence);
279 if (err)
280 return err;
281 }
282
283 err = pvr_sync_add_dep_to_job(job, f: fence);
284 if (err)
285 return err;
286 }
287
288 return 0;
289}
290

source code of linux/drivers/gpu/drm/imagination/pvr_sync.c