1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Sync File validation framework
4 *
5 * Copyright (C) 2012 Google, Inc.
6 */
7
8#include <linux/file.h>
9#include <linux/fs.h>
10#include <linux/uaccess.h>
11#include <linux/panic.h>
12#include <linux/slab.h>
13#include <linux/sync_file.h>
14
15#include "sync_debug.h"
16
17#define CREATE_TRACE_POINTS
18#include "sync_trace.h"
19
20/*
21 * SW SYNC validation framework
22 *
23 * A sync object driver that uses a 32bit counter to coordinate
24 * synchronization. Useful when there is no hardware primitive backing
25 * the synchronization.
26 *
27 * To start the framework just open:
28 *
29 * <debugfs>/sync/sw_sync
30 *
31 * That will create a sync timeline, all fences created under this timeline
32 * file descriptor will belong to the this timeline.
33 *
34 * The 'sw_sync' file can be opened many times as to create different
35 * timelines.
36 *
37 * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct
38 * sw_sync_create_fence_data as parameter.
39 *
40 * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used
41 * with the increment as u32. This will update the last signaled value
42 * from the timeline and signal any fence that has a seqno smaller or equal
43 * to it.
44 *
45 * struct sw_sync_create_fence_data
46 * @value: the seqno to initialise the fence with
47 * @name: the name of the new sync point
48 * @fence: return the fd of the new sync_file with the created fence
49 */
50struct sw_sync_create_fence_data {
51 __u32 value;
52 char name[32];
53 __s32 fence; /* fd of new fence */
54};
55
56/**
57 * struct sw_sync_get_deadline - get the deadline hint of a sw_sync fence
58 * @deadline_ns: absolute time of the deadline
59 * @pad: must be zero
60 * @fence_fd: the sw_sync fence fd (in)
61 *
62 * Return the earliest deadline set on the fence. The timebase for the
63 * deadline is CLOCK_MONOTONIC (same as vblank). If there is no deadline
64 * set on the fence, this ioctl will return -ENOENT.
65 */
66struct sw_sync_get_deadline {
67 __u64 deadline_ns;
68 __u32 pad;
69 __s32 fence_fd;
70};
71
72#define SW_SYNC_IOC_MAGIC 'W'
73
74#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\
75 struct sw_sync_create_fence_data)
76
77#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
78#define SW_SYNC_GET_DEADLINE _IOWR(SW_SYNC_IOC_MAGIC, 2, \
79 struct sw_sync_get_deadline)
80
81
82#define SW_SYNC_HAS_DEADLINE_BIT DMA_FENCE_FLAG_USER_BITS
83
84static const struct dma_fence_ops timeline_fence_ops;
85
86static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence)
87{
88 if (fence->ops != &timeline_fence_ops)
89 return NULL;
90 return container_of(fence, struct sync_pt, base);
91}
92
93/**
94 * sync_timeline_create() - creates a sync object
95 * @name: sync_timeline name
96 *
97 * Creates a new sync_timeline. Returns the sync_timeline object or NULL in
98 * case of error.
99 */
100static struct sync_timeline *sync_timeline_create(const char *name)
101{
102 struct sync_timeline *obj;
103
104 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
105 if (!obj)
106 return NULL;
107
108 kref_init(kref: &obj->kref);
109 obj->context = dma_fence_context_alloc(num: 1);
110 strscpy(obj->name, name, sizeof(obj->name));
111
112 obj->pt_tree = RB_ROOT;
113 INIT_LIST_HEAD(list: &obj->pt_list);
114 spin_lock_init(&obj->lock);
115
116 sync_timeline_debug_add(obj);
117
118 return obj;
119}
120
121static void sync_timeline_free(struct kref *kref)
122{
123 struct sync_timeline *obj =
124 container_of(kref, struct sync_timeline, kref);
125
126 sync_timeline_debug_remove(obj);
127
128 kfree(objp: obj);
129}
130
131static void sync_timeline_get(struct sync_timeline *obj)
132{
133 kref_get(kref: &obj->kref);
134}
135
136static void sync_timeline_put(struct sync_timeline *obj)
137{
138 kref_put(kref: &obj->kref, release: sync_timeline_free);
139}
140
141static const char *timeline_fence_get_driver_name(struct dma_fence *fence)
142{
143 return "sw_sync";
144}
145
146static const char *timeline_fence_get_timeline_name(struct dma_fence *fence)
147{
148 struct sync_timeline *parent = dma_fence_parent(fence);
149
150 return parent->name;
151}
152
153static void timeline_fence_release(struct dma_fence *fence)
154{
155 struct sync_pt *pt = dma_fence_to_sync_pt(fence);
156 struct sync_timeline *parent = dma_fence_parent(fence);
157 unsigned long flags;
158
159 spin_lock_irqsave(fence->lock, flags);
160 if (!list_empty(head: &pt->link)) {
161 list_del(entry: &pt->link);
162 rb_erase(&pt->node, &parent->pt_tree);
163 }
164 spin_unlock_irqrestore(lock: fence->lock, flags);
165
166 sync_timeline_put(obj: parent);
167 dma_fence_free(fence);
168}
169
170static bool timeline_fence_signaled(struct dma_fence *fence)
171{
172 struct sync_timeline *parent = dma_fence_parent(fence);
173
174 return !__dma_fence_is_later(fence, f1: fence->seqno, f2: parent->value);
175}
176
177static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
178{
179 struct sync_pt *pt = dma_fence_to_sync_pt(fence);
180 unsigned long flags;
181
182 spin_lock_irqsave(fence->lock, flags);
183 if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {
184 if (ktime_before(cmp1: deadline, cmp2: pt->deadline))
185 pt->deadline = deadline;
186 } else {
187 pt->deadline = deadline;
188 __set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags);
189 }
190 spin_unlock_irqrestore(lock: fence->lock, flags);
191}
192
193static const struct dma_fence_ops timeline_fence_ops = {
194 .get_driver_name = timeline_fence_get_driver_name,
195 .get_timeline_name = timeline_fence_get_timeline_name,
196 .signaled = timeline_fence_signaled,
197 .release = timeline_fence_release,
198 .set_deadline = timeline_fence_set_deadline,
199};
200
201/**
202 * sync_timeline_signal() - signal a status change on a sync_timeline
203 * @obj: sync_timeline to signal
204 * @inc: num to increment on timeline->value
205 *
206 * A sync implementation should call this any time one of it's fences
207 * has signaled or has an error condition.
208 */
209static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
210{
211 LIST_HEAD(signalled);
212 struct sync_pt *pt, *next;
213
214 trace_sync_timeline(timeline: obj);
215
216 spin_lock_irq(lock: &obj->lock);
217
218 obj->value += inc;
219
220 list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
221 if (!timeline_fence_signaled(fence: &pt->base))
222 break;
223
224 dma_fence_get(fence: &pt->base);
225
226 list_move_tail(list: &pt->link, head: &signalled);
227 rb_erase(&pt->node, &obj->pt_tree);
228
229 dma_fence_signal_locked(fence: &pt->base);
230 }
231
232 spin_unlock_irq(lock: &obj->lock);
233
234 list_for_each_entry_safe(pt, next, &signalled, link) {
235 list_del_init(entry: &pt->link);
236 dma_fence_put(fence: &pt->base);
237 }
238}
239
240/**
241 * sync_pt_create() - creates a sync pt
242 * @obj: parent sync_timeline
243 * @value: value of the fence
244 *
245 * Creates a new sync_pt (fence) as a child of @parent. @size bytes will be
246 * allocated allowing for implementation specific data to be kept after
247 * the generic sync_timeline struct. Returns the sync_pt object or
248 * NULL in case of error.
249 */
250static struct sync_pt *sync_pt_create(struct sync_timeline *obj,
251 unsigned int value)
252{
253 struct sync_pt *pt;
254
255 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
256 if (!pt)
257 return NULL;
258
259 sync_timeline_get(obj);
260 dma_fence_init(fence: &pt->base, ops: &timeline_fence_ops, lock: &obj->lock,
261 context: obj->context, seqno: value);
262 INIT_LIST_HEAD(list: &pt->link);
263
264 spin_lock_irq(lock: &obj->lock);
265 if (!dma_fence_is_signaled_locked(fence: &pt->base)) {
266 struct rb_node **p = &obj->pt_tree.rb_node;
267 struct rb_node *parent = NULL;
268
269 while (*p) {
270 struct sync_pt *other;
271 int cmp;
272
273 parent = *p;
274 other = rb_entry(parent, typeof(*pt), node);
275 cmp = value - other->base.seqno;
276 if (cmp > 0) {
277 p = &parent->rb_right;
278 } else if (cmp < 0) {
279 p = &parent->rb_left;
280 } else {
281 if (dma_fence_get_rcu(fence: &other->base)) {
282 sync_timeline_put(obj);
283 kfree(objp: pt);
284 pt = other;
285 goto unlock;
286 }
287 p = &parent->rb_left;
288 }
289 }
290 rb_link_node(node: &pt->node, parent, rb_link: p);
291 rb_insert_color(&pt->node, &obj->pt_tree);
292
293 parent = rb_next(&pt->node);
294 list_add_tail(new: &pt->link,
295 head: parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list);
296 }
297unlock:
298 spin_unlock_irq(lock: &obj->lock);
299
300 return pt;
301}
302
303/*
304 * *WARNING*
305 *
306 * improper use of this can result in deadlocking kernel drivers from userspace.
307 */
308
309/* opening sw_sync create a new sync obj */
310static int sw_sync_debugfs_open(struct inode *inode, struct file *file)
311{
312 struct sync_timeline *obj;
313 char task_comm[TASK_COMM_LEN];
314
315 get_task_comm(task_comm, current);
316
317 obj = sync_timeline_create(name: task_comm);
318 if (!obj)
319 return -ENOMEM;
320
321 file->private_data = obj;
322
323 return 0;
324}
325
326static int sw_sync_debugfs_release(struct inode *inode, struct file *file)
327{
328 struct sync_timeline *obj = file->private_data;
329 struct sync_pt *pt, *next;
330
331 spin_lock_irq(lock: &obj->lock);
332
333 list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
334 dma_fence_set_error(fence: &pt->base, error: -ENOENT);
335 dma_fence_signal_locked(fence: &pt->base);
336 }
337
338 spin_unlock_irq(lock: &obj->lock);
339
340 sync_timeline_put(obj);
341 return 0;
342}
343
344static long sw_sync_ioctl_create_fence(struct sync_timeline *obj,
345 unsigned long arg)
346{
347 int fd = get_unused_fd_flags(O_CLOEXEC);
348 int err;
349 struct sync_pt *pt;
350 struct sync_file *sync_file;
351 struct sw_sync_create_fence_data data;
352
353 /* SW sync fence are inherently unsafe and can deadlock the kernel */
354 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
355
356 if (fd < 0)
357 return fd;
358
359 if (copy_from_user(to: &data, from: (void __user *)arg, n: sizeof(data))) {
360 err = -EFAULT;
361 goto err;
362 }
363
364 pt = sync_pt_create(obj, value: data.value);
365 if (!pt) {
366 err = -ENOMEM;
367 goto err;
368 }
369
370 sync_file = sync_file_create(fence: &pt->base);
371 dma_fence_put(fence: &pt->base);
372 if (!sync_file) {
373 err = -ENOMEM;
374 goto err;
375 }
376
377 data.fence = fd;
378 if (copy_to_user(to: (void __user *)arg, from: &data, n: sizeof(data))) {
379 fput(sync_file->file);
380 err = -EFAULT;
381 goto err;
382 }
383
384 fd_install(fd, file: sync_file->file);
385
386 return 0;
387
388err:
389 put_unused_fd(fd);
390 return err;
391}
392
393static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg)
394{
395 u32 value;
396
397 if (copy_from_user(to: &value, from: (void __user *)arg, n: sizeof(value)))
398 return -EFAULT;
399
400 while (value > INT_MAX) {
401 sync_timeline_signal(obj, INT_MAX);
402 value -= INT_MAX;
403 }
404
405 sync_timeline_signal(obj, inc: value);
406
407 return 0;
408}
409
410static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long arg)
411{
412 struct sw_sync_get_deadline data;
413 struct dma_fence *fence;
414 unsigned long flags;
415 struct sync_pt *pt;
416 int ret = 0;
417
418 if (copy_from_user(to: &data, from: (void __user *)arg, n: sizeof(data)))
419 return -EFAULT;
420
421 if (data.deadline_ns || data.pad)
422 return -EINVAL;
423
424 fence = sync_file_get_fence(fd: data.fence_fd);
425 if (!fence)
426 return -EINVAL;
427
428 pt = dma_fence_to_sync_pt(fence);
429 if (!pt) {
430 ret = -EINVAL;
431 goto put_fence;
432 }
433
434 spin_lock_irqsave(fence->lock, flags);
435 if (!test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {
436 ret = -ENOENT;
437 goto unlock;
438 }
439 data.deadline_ns = ktime_to_ns(kt: pt->deadline);
440 spin_unlock_irqrestore(lock: fence->lock, flags);
441
442 dma_fence_put(fence);
443
444 if (ret)
445 return ret;
446
447 if (copy_to_user(to: (void __user *)arg, from: &data, n: sizeof(data)))
448 return -EFAULT;
449
450 return 0;
451
452unlock:
453 spin_unlock_irqrestore(lock: fence->lock, flags);
454put_fence:
455 dma_fence_put(fence);
456
457 return ret;
458}
459
460static long sw_sync_ioctl(struct file *file, unsigned int cmd,
461 unsigned long arg)
462{
463 struct sync_timeline *obj = file->private_data;
464
465 switch (cmd) {
466 case SW_SYNC_IOC_CREATE_FENCE:
467 return sw_sync_ioctl_create_fence(obj, arg);
468
469 case SW_SYNC_IOC_INC:
470 return sw_sync_ioctl_inc(obj, arg);
471
472 case SW_SYNC_GET_DEADLINE:
473 return sw_sync_ioctl_get_deadline(obj, arg);
474
475 default:
476 return -ENOTTY;
477 }
478}
479
480const struct file_operations sw_sync_debugfs_fops = {
481 .open = sw_sync_debugfs_open,
482 .release = sw_sync_debugfs_release,
483 .unlocked_ioctl = sw_sync_ioctl,
484 .compat_ioctl = compat_ptr_ioctl,
485};
486

source code of linux/drivers/dma-buf/sw_sync.c