1	/*
2	* Copyright 2017 Red Hat
3	* Parts ported from amdgpu (fence wait code).
4	* Copyright 2016 Advanced Micro Devices, Inc.
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice (including the next
14	* paragraph) shall be included in all copies or substantial portions of the
15	* Software.
16	*
17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23	* IN THE SOFTWARE.
24	*
25	* Authors:
26	*
27	*/
28
29	/**
30	* DOC: Overview
31	*
32	* DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
33	* container for a synchronization primitive which can be used by userspace
34	* to explicitly synchronize GPU commands, can be shared between userspace
35	* processes, and can be shared between different DRM drivers.
36	* Their primary use-case is to implement Vulkan fences and semaphores.
37	* The syncobj userspace API provides ioctls for several operations:
38	*
39	* - Creation and destruction of syncobjs
40	* - Import and export of syncobjs to/from a syncobj file descriptor
41	* - Import and export a syncobj's underlying fence to/from a sync file
42	* - Reset a syncobj (set its fence to NULL)
43	* - Signal a syncobj (set a trivially signaled fence)
44	* - Wait for a syncobj's fence to appear and be signaled
45	*
46	* The syncobj userspace API also provides operations to manipulate a syncobj
47	* in terms of a timeline of struct &dma_fence_chain rather than a single
48	* struct &dma_fence, through the following operations:
49	*
50	* - Signal a given point on the timeline
51	* - Wait for a given point to appear and/or be signaled
52	* - Import and export from/to a given point of a timeline
53	*
54	* At it's core, a syncobj is simply a wrapper around a pointer to a struct
55	* &dma_fence which may be NULL.
56	* When a syncobj is first created, its pointer is either NULL or a pointer
57	* to an already signaled fence depending on whether the
58	* &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
59	* &DRM_IOCTL_SYNCOBJ_CREATE.
60	*
61	* If the syncobj is considered as a binary (its state is either signaled or
62	* unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
63	* the syncobj, the syncobj's fence is replaced with a fence which will be
64	* signaled by the completion of that work.
65	* If the syncobj is considered as a timeline primitive, when GPU work is
66	* enqueued in a DRM driver to signal the a given point of the syncobj, a new
67	* struct &dma_fence_chain pointing to the DRM driver's fence and also
68	* pointing to the previous fence that was in the syncobj. The new struct
69	* &dma_fence_chain fence replace the syncobj's fence and will be signaled by
70	* completion of the DRM driver's work and also any work associated with the
71	* fence previously in the syncobj.
72	*
73	* When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
74	* time the work is enqueued, it waits on the syncobj's fence before
75	* submitting the work to hardware. That fence is either :
76	*
77	* - The syncobj's current fence if the syncobj is considered as a binary
78	* primitive.
79	* - The struct &dma_fence associated with a given point if the syncobj is
80	* considered as a timeline primitive.
81	*
82	* If the syncobj's fence is NULL or not present in the syncobj's timeline,
83	* the enqueue operation is expected to fail.
84	*
85	* With binary syncobj, all manipulation of the syncobjs's fence happens in
86	* terms of the current fence at the time the ioctl is called by userspace
87	* regardless of whether that operation is an immediate host-side operation
88	* (signal or reset) or or an operation which is enqueued in some driver
89	* queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
90	* to manipulate a syncobj from the host by resetting its pointer to NULL or
91	* setting its pointer to a fence which is already signaled.
92	*
93	* With a timeline syncobj, all manipulation of the synobj's fence happens in
94	* terms of a u64 value referring to point in the timeline. See
95	* dma_fence_chain_find_seqno() to see how a given point is found in the
96	* timeline.
97	*
98	* Note that applications should be careful to always use timeline set of
99	* ioctl() when dealing with syncobj considered as timeline. Using a binary
100	* set of ioctl() with a syncobj considered as timeline could result incorrect
101	* synchronization. The use of binary syncobj is supported through the
102	* timeline set of ioctl() by using a point value of 0, this will reproduce
103	* the behavior of the binary set of ioctl() (for example replace the
104	* syncobj's fence when signaling).
105	*
106	*
107	* Host-side wait on syncobjs
108	* --------------------------
109	*
110	* &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
111	* host-side wait on all of the syncobj fences simultaneously.
112	* If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
113	* all of the syncobj fences to be signaled before it returns.
114	* Otherwise, it returns once at least one syncobj fence has been signaled
115	* and the index of a signaled fence is written back to the client.
116	*
117	* Unlike the enqueued GPU work dependencies which fail if they see a NULL
118	* fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
119	* the host-side wait will first wait for the syncobj to receive a non-NULL
120	* fence and then wait on that fence.
121	* If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
122	* syncobjs in the array has a NULL fence, -EINVAL will be returned.
123	* Assuming the syncobj starts off with a NULL fence, this allows a client
124	* to do a host wait in one thread (or process) which waits on GPU work
125	* submitted in another thread (or process) without having to manually
126	* synchronize between the two.
127	* This requirement is inherited from the Vulkan fence API.
128	*
129	* If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set
130	* a fence deadline hint on the backing fences before waiting, to provide the
131	* fence signaler with an appropriate sense of urgency. The deadline is
132	* specified as an absolute &CLOCK_MONOTONIC value in units of ns.
133	*
134	* Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
135	* handles as well as an array of u64 points and does a host-side wait on all
136	* of syncobj fences at the given points simultaneously.
137	*
138	* &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
139	* fence to materialize on the timeline without waiting for the fence to be
140	* signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
141	* requirement is inherited from the wait-before-signal behavior required by
142	* the Vulkan timeline semaphore API.
143	*
144	* Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
145	* blocking: an eventfd will be signaled when the syncobj is. This is useful to
146	* integrate the wait in an event loop.
147	*
148	*
149	* Import/export of syncobjs
150	* -------------------------
151	*
152	* &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
153	* provide two mechanisms for import/export of syncobjs.
154	*
155	* The first lets the client import or export an entire syncobj to a file
156	* descriptor.
157	* These fd's are opaque and have no other use case, except passing the
158	* syncobj between processes.
159	* All exported file descriptors and any syncobj handles created as a
160	* result of importing those file descriptors own a reference to the
161	* same underlying struct &drm_syncobj and the syncobj can be used
162	* persistently across all the processes with which it is shared.
163	* The syncobj is freed only once the last reference is dropped.
164	* Unlike dma-buf, importing a syncobj creates a new handle (with its own
165	* reference) for every import instead of de-duplicating.
166	* The primary use-case of this persistent import/export is for shared
167	* Vulkan fences and semaphores.
168	*
169	* The second import/export mechanism, which is indicated by
170	* &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
171	* &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
172	* import/export the syncobj's current fence from/to a &sync_file.
173	* When a syncobj is exported to a sync file, that sync file wraps the
174	* sycnobj's fence at the time of export and any later signal or reset
175	* operations on the syncobj will not affect the exported sync file.
176	* When a sync file is imported into a syncobj, the syncobj's fence is set
177	* to the fence wrapped by that sync file.
178	* Because sync files are immutable, resetting or signaling the syncobj
179	* will not affect any sync files whose fences have been imported into the
180	* syncobj.
181	*
182	*
183	* Import/export of timeline points in timeline syncobjs
184	* -----------------------------------------------------
185	*
186	* &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
187	* &dma_fence_chain of a syncobj at a given u64 point to another u64 point
188	* into another syncobj.
189	*
190	* Note that if you want to transfer a struct &dma_fence_chain from a given
191	* point on a timeline syncobj from/into a binary syncobj, you can use the
192	* point 0 to mean take/replace the fence in the syncobj.
193	*/
194
195	#include <linux/anon_inodes.h>
196	#include <linux/dma-fence-unwrap.h>
197	#include <linux/eventfd.h>
198	#include <linux/export.h>
199	#include <linux/file.h>
200	#include <linux/fs.h>
201	#include <linux/sched/signal.h>
202	#include <linux/sync_file.h>
203	#include <linux/uaccess.h>
204
205	#include <drm/drm.h>
206	#include <drm/drm_drv.h>
207	#include <drm/drm_file.h>
208	#include <drm/drm_gem.h>
209	#include <drm/drm_print.h>
210	#include <drm/drm_syncobj.h>
211	#include <drm/drm_utils.h>
212
213	#include "drm_internal.h"
214
215	struct syncobj_wait_entry {
216	struct list_head node;
217	struct task_struct *task;
218	struct dma_fence *fence;
219	struct dma_fence_cb fence_cb;
220	u64 point;
221	};
222
223	static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
224	struct syncobj_wait_entry *wait);
225
226	struct syncobj_eventfd_entry {
227	struct list_head node;
228	struct dma_fence *fence;
229	struct dma_fence_cb fence_cb;
230	struct drm_syncobj *syncobj;
231	struct eventfd_ctx *ev_fd_ctx;
232	u64 point;
233	u32 flags;
234	};
235
236	static void
237	syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
238	struct syncobj_eventfd_entry *entry);
239
240	/**
241	* drm_syncobj_find - lookup and reference a sync object.
242	* @file_private: drm file private pointer
243	* @handle: sync object handle to lookup.
244	*
245	* Returns a reference to the syncobj pointed to by handle or NULL. The
246	* reference must be released by calling drm_syncobj_put().
247	*/
248	struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
249	u32 handle)
250	{
251	struct drm_syncobj *syncobj;
252
253	spin_lock(lock: &file_private->syncobj_table_lock);
254
255	/* Check if we currently have a reference on the object */
256	syncobj = idr_find(&file_private->syncobj_idr, id: handle);
257	if (syncobj)
258	drm_syncobj_get(obj: syncobj);
259
260	spin_unlock(lock: &file_private->syncobj_table_lock);
261
262	return syncobj;
263	}
264	EXPORT_SYMBOL(drm_syncobj_find);
265
266	static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
267	struct syncobj_wait_entry *wait)
268	{
269	struct dma_fence *fence;
270
271	if (wait->fence)
272	return;
273
274	spin_lock(lock: &syncobj->lock);
275	/* We've already tried once to get a fence and failed. Now that we
276	* have the lock, try one more time just to be sure we don't add a
277	* callback when a fence has already been set.
278	*/
279	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
280	if (!fence || dma_fence_chain_find_seqno(pfence: &fence, seqno: wait->point)) {
281	dma_fence_put(fence);
282	list_add_tail(new: &wait->node, head: &syncobj->cb_list);
283	} else if (!fence) {
284	wait->fence = dma_fence_get_stub();
285	} else {
286	wait->fence = fence;
287	}
288	spin_unlock(lock: &syncobj->lock);
289	}
290
291	static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
292	struct syncobj_wait_entry *wait)
293	{
294	if (!wait->node.next)
295	return;
296
297	spin_lock(lock: &syncobj->lock);
298	list_del_init(entry: &wait->node);
299	spin_unlock(lock: &syncobj->lock);
300	}
301
302	static void
303	syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
304	{
305	eventfd_ctx_put(ctx: entry->ev_fd_ctx);
306	dma_fence_put(fence: entry->fence);
307	/* This happens either inside the syncobj lock, or after the node has
308	* already been removed from the list.
309	*/
310	list_del(entry: &entry->node);
311	kfree(objp: entry);
312	}
313
314	static void
315	drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
316	struct syncobj_eventfd_entry *entry)
317	{
318	spin_lock(lock: &syncobj->lock);
319	list_add_tail(new: &entry->node, head: &syncobj->ev_fd_list);
320	syncobj_eventfd_entry_func(syncobj, entry);
321	spin_unlock(lock: &syncobj->lock);
322	}
323
324	/**
325	* drm_syncobj_add_point - add new timeline point to the syncobj
326	* @syncobj: sync object to add timeline point do
327	* @chain: chain node to use to add the point
328	* @fence: fence to encapsulate in the chain node
329	* @point: sequence number to use for the point
330	*
331	* Add the chain node as new timeline point to the syncobj.
332	*/
333	void drm_syncobj_add_point(struct drm_syncobj *syncobj,
334	struct dma_fence_chain *chain,
335	struct dma_fence *fence,
336	uint64_t point)
337	{
338	struct syncobj_wait_entry *wait_cur, *wait_tmp;
339	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
340	struct dma_fence *prev;
341
342	dma_fence_get(fence);
343
344	spin_lock(lock: &syncobj->lock);
345
346	prev = drm_syncobj_fence_get(syncobj);
347	/* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
348	if (prev && prev->seqno >= point)
349	DRM_DEBUG("You are adding an unorder point to timeline!\n");
350	dma_fence_chain_init(chain, prev, fence, seqno: point);
351	rcu_assign_pointer(syncobj->fence, &chain->base);
352
353	list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
354	syncobj_wait_syncobj_func(syncobj, wait: wait_cur);
355	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
356	syncobj_eventfd_entry_func(syncobj, entry: ev_fd_cur);
357	spin_unlock(lock: &syncobj->lock);
358
359	/* Walk the chain once to trigger garbage collection */
360	dma_fence_chain_for_each(fence, prev);
361	dma_fence_put(fence: prev);
362	}
363	EXPORT_SYMBOL(drm_syncobj_add_point);
364
365	/**
366	* drm_syncobj_replace_fence - replace fence in a sync object.
367	* @syncobj: Sync object to replace fence in
368	* @fence: fence to install in sync file.
369	*
370	* This replaces the fence on a sync object.
371	*/
372	void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
373	struct dma_fence *fence)
374	{
375	struct dma_fence *old_fence;
376	struct syncobj_wait_entry *wait_cur, *wait_tmp;
377	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
378
379	if (fence)
380	dma_fence_get(fence);
381
382	spin_lock(lock: &syncobj->lock);
383
384	old_fence = rcu_dereference_protected(syncobj->fence,
385	lockdep_is_held(&syncobj->lock));
386	rcu_assign_pointer(syncobj->fence, fence);
387
388	if (fence != old_fence) {
389	list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
390	syncobj_wait_syncobj_func(syncobj, wait: wait_cur);
391	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
392	syncobj_eventfd_entry_func(syncobj, entry: ev_fd_cur);
393	}
394
395	spin_unlock(lock: &syncobj->lock);
396
397	dma_fence_put(fence: old_fence);
398	}
399	EXPORT_SYMBOL(drm_syncobj_replace_fence);
400
401	/**
402	* drm_syncobj_assign_null_handle - assign a stub fence to the sync object
403	* @syncobj: sync object to assign the fence on
404	*
405	* Assign a already signaled stub fence to the sync object.
406	*/
407	static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
408	{
409	struct dma_fence *fence = dma_fence_allocate_private_stub(timestamp: ktime_get());
410
411	if (!fence)
412	return -ENOMEM;
413
414	drm_syncobj_replace_fence(syncobj, fence);
415	dma_fence_put(fence);
416	return 0;
417	}
418
419	/* 5s default for wait submission */
420	#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
421	/**
422	* drm_syncobj_find_fence - lookup and reference the fence in a sync object
423	* @file_private: drm file private pointer
424	* @handle: sync object handle to lookup.
425	* @point: timeline point
426	* @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
427	* @fence: out parameter for the fence
428	*
429	* This is just a convenience function that combines drm_syncobj_find() and
430	* drm_syncobj_fence_get().
431	*
432	* Returns 0 on success or a negative error value on failure. On success @fence
433	* contains a reference to the fence, which must be released by calling
434	* dma_fence_put().
435	*/
436	int drm_syncobj_find_fence(struct drm_file *file_private,
437	u32 handle, u64 point, u64 flags,
438	struct dma_fence **fence)
439	{
440	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
441	struct syncobj_wait_entry wait;
442	u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
443	int ret;
444
445	if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
446	return -EINVAL;
447
448	if (!syncobj)
449	return -ENOENT;
450
451	/* Waiting for userspace with locks help is illegal cause that can
452	* trivial deadlock with page faults for example. Make lockdep complain
453	* about it early on.
454	*/
455	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
456	might_sleep();
457	lockdep_assert_none_held_once();
458	}
459
460	*fence = drm_syncobj_fence_get(syncobj);
461
462	if (*fence) {
463	ret = dma_fence_chain_find_seqno(pfence: fence, seqno: point);
464	if (!ret) {
465	/* If the requested seqno is already signaled
466	* drm_syncobj_find_fence may return a NULL
467	* fence. To make sure the recipient gets
468	* signalled, use a new fence instead.
469	*/
470	if (!*fence)
471	*fence = dma_fence_get_stub();
472
473	goto out;
474	}
475	dma_fence_put(fence: *fence);
476	} else {
477	ret = -EINVAL;
478	}
479
480	if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
481	goto out;
482
483	memset(&wait, 0, sizeof(wait));
484	wait.task = current;
485	wait.point = point;
486	drm_syncobj_fence_add_wait(syncobj, wait: &wait);
487
488	do {
489	set_current_state(TASK_INTERRUPTIBLE);
490	if (wait.fence) {
491	ret = 0;
492	break;
493	}
494	if (timeout == 0) {
495	ret = -ETIME;
496	break;
497	}
498
499	if (signal_pending(current)) {
500	ret = -ERESTARTSYS;
501	break;
502	}
503
504	timeout = schedule_timeout(timeout);
505	} while (1);
506
507	__set_current_state(TASK_RUNNING);
508	*fence = wait.fence;
509
510	if (wait.node.next)
511	drm_syncobj_remove_wait(syncobj, wait: &wait);
512
513	out:
514	drm_syncobj_put(obj: syncobj);
515
516	return ret;
517	}
518	EXPORT_SYMBOL(drm_syncobj_find_fence);
519
520	/**
521	* drm_syncobj_free - free a sync object.
522	* @kref: kref to free.
523	*
524	* Only to be called from kref_put in drm_syncobj_put.
525	*/
526	void drm_syncobj_free(struct kref *kref)
527	{
528	struct drm_syncobj *syncobj = container_of(kref,
529	struct drm_syncobj,
530	refcount);
531	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
532
533	drm_syncobj_replace_fence(syncobj, NULL);
534
535	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
536	syncobj_eventfd_entry_free(entry: ev_fd_cur);
537
538	kfree(objp: syncobj);
539	}
540	EXPORT_SYMBOL(drm_syncobj_free);
541
542	/**
543	* drm_syncobj_create - create a new syncobj
544	* @out_syncobj: returned syncobj
545	* @flags: DRM_SYNCOBJ_* flags
546	* @fence: if non-NULL, the syncobj will represent this fence
547	*
548	* This is the first function to create a sync object. After creating, drivers
549	* probably want to make it available to userspace, either through
550	* drm_syncobj_get_handle() or drm_syncobj_get_fd().
551	*
552	* Returns 0 on success or a negative error value on failure.
553	*/
554	int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
555	struct dma_fence *fence)
556	{
557	int ret;
558	struct drm_syncobj *syncobj;
559
560	syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
561	if (!syncobj)
562	return -ENOMEM;
563
564	kref_init(kref: &syncobj->refcount);
565	INIT_LIST_HEAD(list: &syncobj->cb_list);
566	INIT_LIST_HEAD(list: &syncobj->ev_fd_list);
567	spin_lock_init(&syncobj->lock);
568
569	if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
570	ret = drm_syncobj_assign_null_handle(syncobj);
571	if (ret < 0) {
572	drm_syncobj_put(obj: syncobj);
573	return ret;
574	}
575	}
576
577	if (fence)
578	drm_syncobj_replace_fence(syncobj, fence);
579
580	*out_syncobj = syncobj;
581	return 0;
582	}
583	EXPORT_SYMBOL(drm_syncobj_create);
584
585	/**
586	* drm_syncobj_get_handle - get a handle from a syncobj
587	* @file_private: drm file private pointer
588	* @syncobj: Sync object to export
589	* @handle: out parameter with the new handle
590	*
591	* Exports a sync object created with drm_syncobj_create() as a handle on
592	* @file_private to userspace.
593	*
594	* Returns 0 on success or a negative error value on failure.
595	*/
596	int drm_syncobj_get_handle(struct drm_file *file_private,
597	struct drm_syncobj *syncobj, u32 *handle)
598	{
599	int ret;
600
601	/* take a reference to put in the idr */
602	drm_syncobj_get(obj: syncobj);
603
604	idr_preload(GFP_KERNEL);
605	spin_lock(lock: &file_private->syncobj_table_lock);
606	ret = idr_alloc(&file_private->syncobj_idr, ptr: syncobj, start: 1, end: 0, GFP_NOWAIT);
607	spin_unlock(lock: &file_private->syncobj_table_lock);
608
609	idr_preload_end();
610
611	if (ret < 0) {
612	drm_syncobj_put(obj: syncobj);
613	return ret;
614	}
615
616	*handle = ret;
617	return 0;
618	}
619	EXPORT_SYMBOL(drm_syncobj_get_handle);
620
621	static int drm_syncobj_create_as_handle(struct drm_file *file_private,
622	u32 *handle, uint32_t flags)
623	{
624	int ret;
625	struct drm_syncobj *syncobj;
626
627	ret = drm_syncobj_create(&syncobj, flags, NULL);
628	if (ret)
629	return ret;
630
631	ret = drm_syncobj_get_handle(file_private, syncobj, handle);
632	drm_syncobj_put(obj: syncobj);
633	return ret;
634	}
635
636	static int drm_syncobj_destroy(struct drm_file *file_private,
637	u32 handle)
638	{
639	struct drm_syncobj *syncobj;
640
641	spin_lock(lock: &file_private->syncobj_table_lock);
642	syncobj = idr_remove(&file_private->syncobj_idr, id: handle);
643	spin_unlock(lock: &file_private->syncobj_table_lock);
644
645	if (!syncobj)
646	return -EINVAL;
647
648	drm_syncobj_put(obj: syncobj);
649	return 0;
650	}
651
652	static int drm_syncobj_file_release(struct inode *inode, struct file *file)
653	{
654	struct drm_syncobj *syncobj = file->private_data;
655
656	drm_syncobj_put(obj: syncobj);
657	return 0;
658	}
659
660	static const struct file_operations drm_syncobj_file_fops = {
661	.release = drm_syncobj_file_release,
662	};
663
664	/**
665	* drm_syncobj_get_fd - get a file descriptor from a syncobj
666	* @syncobj: Sync object to export
667	* @p_fd: out parameter with the new file descriptor
668	*
669	* Exports a sync object created with drm_syncobj_create() as a file descriptor.
670	*
671	* Returns 0 on success or a negative error value on failure.
672	*/
673	int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
674	{
675	struct file *file;
676	int fd;
677
678	fd = get_unused_fd_flags(O_CLOEXEC);
679	if (fd < 0)
680	return fd;
681
682	file = anon_inode_getfile(name: "syncobj_file",
683	fops: &drm_syncobj_file_fops,
684	priv: syncobj, flags: 0);
685	if (IS_ERR(ptr: file)) {
686	put_unused_fd(fd);
687	return PTR_ERR(ptr: file);
688	}
689
690	drm_syncobj_get(obj: syncobj);
691	fd_install(fd, file);
692
693	*p_fd = fd;
694	return 0;
695	}
696	EXPORT_SYMBOL(drm_syncobj_get_fd);
697
698	static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
699	u32 handle, int *p_fd)
700	{
701	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
702	int ret;
703
704	if (!syncobj)
705	return -EINVAL;
706
707	ret = drm_syncobj_get_fd(syncobj, p_fd);
708	drm_syncobj_put(obj: syncobj);
709	return ret;
710	}
711
712	static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
713	int fd, u32 *handle)
714	{
715	struct drm_syncobj *syncobj;
716	CLASS(fd, f)(fd);
717	int ret;
718
719	if (fd_empty(f))
720	return -EINVAL;
721
722	if (fd_file(f)->f_op != &drm_syncobj_file_fops)
723	return -EINVAL;
724
725	/* take a reference to put in the idr */
726	syncobj = fd_file(f)->private_data;
727	drm_syncobj_get(obj: syncobj);
728
729	idr_preload(GFP_KERNEL);
730	spin_lock(lock: &file_private->syncobj_table_lock);
731	ret = idr_alloc(&file_private->syncobj_idr, ptr: syncobj, start: 1, end: 0, GFP_NOWAIT);
732	spin_unlock(lock: &file_private->syncobj_table_lock);
733	idr_preload_end();
734
735	if (ret > 0) {
736	*handle = ret;
737	ret = 0;
738	} else
739	drm_syncobj_put(obj: syncobj);
740
741	return ret;
742	}
743
744	static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
745	int fd, int handle, u64 point)
746	{
747	struct dma_fence *fence = sync_file_get_fence(fd);
748	struct drm_syncobj *syncobj;
749
750	if (!fence)
751	return -EINVAL;
752
753	syncobj = drm_syncobj_find(file_private, handle);
754	if (!syncobj) {
755	dma_fence_put(fence);
756	return -ENOENT;
757	}
758
759	if (point) {
760	struct dma_fence_chain *chain = dma_fence_chain_alloc();
761
762	if (!chain)
763	return -ENOMEM;
764
765	drm_syncobj_add_point(syncobj, chain, fence, point);
766	} else {
767	drm_syncobj_replace_fence(syncobj, fence);
768	}
769
770	dma_fence_put(fence);
771	drm_syncobj_put(obj: syncobj);
772	return 0;
773	}
774
775	static int drm_syncobj_export_sync_file(struct drm_file *file_private,
776	int handle, u64 point, int *p_fd)
777	{
778	int ret;
779	struct dma_fence *fence;
780	struct sync_file *sync_file;
781	int fd = get_unused_fd_flags(O_CLOEXEC);
782
783	if (fd < 0)
784	return fd;
785
786	ret = drm_syncobj_find_fence(file_private, handle, point, 0, &fence);
787	if (ret)
788	goto err_put_fd;
789
790	sync_file = sync_file_create(fence);
791
792	dma_fence_put(fence);
793
794	if (!sync_file) {
795	ret = -EINVAL;
796	goto err_put_fd;
797	}
798
799	fd_install(fd, file: sync_file->file);
800
801	*p_fd = fd;
802	return 0;
803	err_put_fd:
804	put_unused_fd(fd);
805	return ret;
806	}
807	/**
808	* drm_syncobj_open - initializes syncobj file-private structures at devnode open time
809	* @file_private: drm file-private structure to set up
810	*
811	* Called at device open time, sets up the structure for handling refcounting
812	* of sync objects.
813	*/
814	void
815	drm_syncobj_open(struct drm_file *file_private)
816	{
817	idr_init_base(idr: &file_private->syncobj_idr, base: 1);
818	spin_lock_init(&file_private->syncobj_table_lock);
819	}
820
821	static int
822	drm_syncobj_release_handle(int id, void *ptr, void *data)
823	{
824	struct drm_syncobj *syncobj = ptr;
825
826	drm_syncobj_put(obj: syncobj);
827	return 0;
828	}
829
830	/**
831	* drm_syncobj_release - release file-private sync object resources
832	* @file_private: drm file-private structure to clean up
833	*
834	* Called at close time when the filp is going away.
835	*
836	* Releases any remaining references on objects by this filp.
837	*/
838	void
839	drm_syncobj_release(struct drm_file *file_private)
840	{
841	idr_for_each(&file_private->syncobj_idr,
842	fn: &drm_syncobj_release_handle, data: file_private);
843	idr_destroy(&file_private->syncobj_idr);
844	}
845
846	int
847	drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
848	struct drm_file *file_private)
849	{
850	struct drm_syncobj_create *args = data;
851
852	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
853	return -EOPNOTSUPP;
854
855	/* no valid flags yet */
856	if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
857	return -EINVAL;
858
859	return drm_syncobj_create_as_handle(file_private,
860	handle: &args->handle, flags: args->flags);
861	}
862
863	int
864	drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
865	struct drm_file *file_private)
866	{
867	struct drm_syncobj_destroy *args = data;
868
869	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
870	return -EOPNOTSUPP;
871
872	/* make sure padding is empty */
873	if (args->pad)
874	return -EINVAL;
875	return drm_syncobj_destroy(file_private, handle: args->handle);
876	}
877
878	int
879	drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
880	struct drm_file *file_private)
881	{
882	struct drm_syncobj_handle *args = data;
883	unsigned int valid_flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE |
884	DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE;
885	u64 point = 0;
886
887	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
888	return -EOPNOTSUPP;
889
890	if (args->pad)
891	return -EINVAL;
892
893	if (args->flags & ~valid_flags)
894	return -EINVAL;
895
896	if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE)
897	point = args->point;
898
899	if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
900	return drm_syncobj_export_sync_file(file_private, handle: args->handle,
901	point, p_fd: &args->fd);
902
903	if (args->point)
904	return -EINVAL;
905
906	return drm_syncobj_handle_to_fd(file_private, handle: args->handle,
907	p_fd: &args->fd);
908	}
909
910	int
911	drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
912	struct drm_file *file_private)
913	{
914	struct drm_syncobj_handle *args = data;
915	unsigned int valid_flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE |
916	DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE;
917	u64 point = 0;
918
919	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
920	return -EOPNOTSUPP;
921
922	if (args->pad)
923	return -EINVAL;
924
925	if (args->flags & ~valid_flags)
926	return -EINVAL;
927
928	if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE)
929	point = args->point;
930
931	if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
932	return drm_syncobj_import_sync_file_fence(file_private,
933	fd: args->fd,
934	handle: args->handle,
935	point);
936
937	if (args->point)
938	return -EINVAL;
939
940	return drm_syncobj_fd_to_handle(file_private, fd: args->fd,
941	handle: &args->handle);
942	}
943
944	static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
945	struct drm_syncobj_transfer *args)
946	{
947	struct drm_syncobj *timeline_syncobj = NULL;
948	struct dma_fence *fence, *tmp;
949	struct dma_fence_chain *chain;
950	int ret;
951
952	timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
953	if (!timeline_syncobj) {
954	return -ENOENT;
955	}
956	ret = drm_syncobj_find_fence(file_private, args->src_handle,
957	args->src_point, args->flags,
958	&tmp);
959	if (ret)
960	goto err_put_timeline;
961
962	fence = dma_fence_unwrap_merge(tmp);
963	dma_fence_put(fence: tmp);
964	if (!fence) {
965	ret = -ENOMEM;
966	goto err_put_timeline;
967	}
968
969	chain = dma_fence_chain_alloc();
970	if (!chain) {
971	ret = -ENOMEM;
972	goto err_free_fence;
973	}
974
975	drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
976	err_free_fence:
977	dma_fence_put(fence);
978	err_put_timeline:
979	drm_syncobj_put(obj: timeline_syncobj);
980
981	return ret;
982	}
983
984	static int
985	drm_syncobj_transfer_to_binary(struct drm_file *file_private,
986	struct drm_syncobj_transfer *args)
987	{
988	struct drm_syncobj *binary_syncobj = NULL;
989	struct dma_fence *fence;
990	int ret;
991
992	binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
993	if (!binary_syncobj)
994	return -ENOENT;
995	ret = drm_syncobj_find_fence(file_private, args->src_handle,
996	args->src_point, args->flags, &fence);
997	if (ret)
998	goto err;
999	drm_syncobj_replace_fence(binary_syncobj, fence);
1000	dma_fence_put(fence);
1001	err:
1002	drm_syncobj_put(obj: binary_syncobj);
1003
1004	return ret;
1005	}
1006	int
1007	drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
1008	struct drm_file *file_private)
1009	{
1010	struct drm_syncobj_transfer *args = data;
1011	int ret;
1012
1013	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1014	return -EOPNOTSUPP;
1015
1016	if (args->pad)
1017	return -EINVAL;
1018
1019	if (args->dst_point)
1020	ret = drm_syncobj_transfer_to_timeline(file_private, args);
1021	else
1022	ret = drm_syncobj_transfer_to_binary(file_private, args);
1023
1024	return ret;
1025	}
1026
1027	static void syncobj_wait_fence_func(struct dma_fence *fence,
1028	struct dma_fence_cb *cb)
1029	{
1030	struct syncobj_wait_entry *wait =
1031	container_of(cb, struct syncobj_wait_entry, fence_cb);
1032
1033	wake_up_process(tsk: wait->task);
1034	}
1035
1036	static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1037	struct syncobj_wait_entry *wait)
1038	{
1039	struct dma_fence *fence;
1040
1041	/* This happens inside the syncobj lock */
1042	fence = rcu_dereference_protected(syncobj->fence,
1043	lockdep_is_held(&syncobj->lock));
1044	dma_fence_get(fence);
1045	if (!fence || dma_fence_chain_find_seqno(pfence: &fence, seqno: wait->point)) {
1046	dma_fence_put(fence);
1047	return;
1048	} else if (!fence) {
1049	wait->fence = dma_fence_get_stub();
1050	} else {
1051	wait->fence = fence;
1052	}
1053
1054	wake_up_process(tsk: wait->task);
1055	list_del_init(entry: &wait->node);
1056	}
1057
1058	static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1059	void __user *user_points,
1060	uint32_t count,
1061	uint32_t flags,
1062	signed long timeout,
1063	uint32_t *idx,
1064	ktime_t *deadline)
1065	{
1066	struct syncobj_wait_entry *entries;
1067	struct dma_fence *fence;
1068	uint64_t *points;
1069	uint32_t signaled_count, i;
1070
1071	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1072	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1073	might_sleep();
1074	lockdep_assert_none_held_once();
1075	}
1076
1077	points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1078	if (points == NULL)
1079	return -ENOMEM;
1080
1081	if (!user_points) {
1082	memset(points, 0, count * sizeof(uint64_t));
1083
1084	} else if (copy_from_user(to: points, from: user_points,
1085	n: sizeof(uint64_t) * count)) {
1086	timeout = -EFAULT;
1087	goto err_free_points;
1088	}
1089
1090	entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1091	if (!entries) {
1092	timeout = -ENOMEM;
1093	goto err_free_points;
1094	}
1095	/* Walk the list of sync objects and initialize entries. We do
1096	* this up-front so that we can properly return -EINVAL if there is
1097	* a syncobj with a missing fence and then never have the chance of
1098	* returning -EINVAL again.
1099	*/
1100	signaled_count = 0;
1101	for (i = 0; i < count; ++i) {
1102	struct dma_fence *fence;
1103
1104	entries[i].task = current;
1105	entries[i].point = points[i];
1106	fence = drm_syncobj_fence_get(syncobj: syncobjs[i]);
1107	if (!fence || dma_fence_chain_find_seqno(pfence: &fence, seqno: points[i])) {
1108	dma_fence_put(fence);
1109	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1110	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1111	continue;
1112	} else {
1113	timeout = -EINVAL;
1114	goto cleanup_entries;
1115	}
1116	}
1117
1118	if (fence)
1119	entries[i].fence = fence;
1120	else
1121	entries[i].fence = dma_fence_get_stub();
1122
1123	if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1124	dma_fence_is_signaled(fence: entries[i].fence)) {
1125	if (signaled_count == 0 && idx)
1126	*idx = i;
1127	signaled_count++;
1128	}
1129	}
1130
1131	if (signaled_count == count ||
1132	(signaled_count > 0 &&
1133	!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1134	goto cleanup_entries;
1135
1136	/* There's a very annoying laxness in the dma_fence API here, in
1137	* that backends are not required to automatically report when a
1138	* fence is signaled prior to fence->ops->enable_signaling() being
1139	* called. So here if we fail to match signaled_count, we need to
1140	* fallthough and try a 0 timeout wait!
1141	*/
1142
1143	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1144	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1145	for (i = 0; i < count; ++i)
1146	drm_syncobj_fence_add_wait(syncobj: syncobjs[i], wait: &entries[i]);
1147	}
1148
1149	if (deadline) {
1150	for (i = 0; i < count; ++i) {
1151	fence = entries[i].fence;
1152	if (!fence)
1153	continue;
1154	dma_fence_set_deadline(fence, deadline: *deadline);
1155	}
1156	}
1157
1158	do {
1159	set_current_state(TASK_INTERRUPTIBLE);
1160
1161	signaled_count = 0;
1162	for (i = 0; i < count; ++i) {
1163	fence = entries[i].fence;
1164	if (!fence)
1165	continue;
1166
1167	if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1168	dma_fence_is_signaled(fence) ||
1169	(!entries[i].fence_cb.func &&
1170	dma_fence_add_callback(fence,
1171	cb: &entries[i].fence_cb,
1172	func: syncobj_wait_fence_func))) {
1173	/* The fence has been signaled */
1174	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1175	signaled_count++;
1176	} else {
1177	if (idx)
1178	*idx = i;
1179	goto done_waiting;
1180	}
1181	}
1182	}
1183
1184	if (signaled_count == count)
1185	goto done_waiting;
1186
1187	if (timeout == 0) {
1188	timeout = -ETIME;
1189	goto done_waiting;
1190	}
1191
1192	if (signal_pending(current)) {
1193	timeout = -ERESTARTSYS;
1194	goto done_waiting;
1195	}
1196
1197	timeout = schedule_timeout(timeout);
1198	} while (1);
1199
1200	done_waiting:
1201	__set_current_state(TASK_RUNNING);
1202
1203	cleanup_entries:
1204	for (i = 0; i < count; ++i) {
1205	drm_syncobj_remove_wait(syncobj: syncobjs[i], wait: &entries[i]);
1206	if (entries[i].fence_cb.func)
1207	dma_fence_remove_callback(fence: entries[i].fence,
1208	cb: &entries[i].fence_cb);
1209	dma_fence_put(fence: entries[i].fence);
1210	}
1211	kfree(objp: entries);
1212
1213	err_free_points:
1214	kfree(objp: points);
1215
1216	return timeout;
1217	}
1218
1219	/**
1220	* drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1221	*
1222	* @timeout_nsec: timeout nsec component in ns, 0 for poll
1223	*
1224	* Calculate the timeout in jiffies from an absolute time in sec/nsec.
1225	*/
1226	signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1227	{
1228	ktime_t abs_timeout, now;
1229	u64 timeout_ns, timeout_jiffies64;
1230
1231	/* make 0 timeout means poll - absolute 0 doesn't seem valid */
1232	if (timeout_nsec == 0)
1233	return 0;
1234
1235	abs_timeout = ns_to_ktime(ns: timeout_nsec);
1236	now = ktime_get();
1237
1238	if (!ktime_after(cmp1: abs_timeout, cmp2: now))
1239	return 0;
1240
1241	timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1242
1243	timeout_jiffies64 = nsecs_to_jiffies64(n: timeout_ns);
1244	/* clamp timeout to avoid infinite timeout */
1245	if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1246	return MAX_SCHEDULE_TIMEOUT - 1;
1247
1248	return timeout_jiffies64 + 1;
1249	}
1250	EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1251
1252	static int drm_syncobj_array_wait(struct drm_device *dev,
1253	struct drm_file *file_private,
1254	struct drm_syncobj_wait *wait,
1255	struct drm_syncobj_timeline_wait *timeline_wait,
1256	struct drm_syncobj **syncobjs, bool timeline,
1257	ktime_t *deadline)
1258	{
1259	signed long timeout = 0;
1260	uint32_t first = ~0;
1261
1262	if (!timeline) {
1263	timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1264	timeout = drm_syncobj_array_wait_timeout(syncobjs,
1265	NULL,
1266	count: wait->count_handles,
1267	flags: wait->flags,
1268	timeout, idx: &first,
1269	deadline);
1270	if (timeout < 0)
1271	return timeout;
1272	wait->first_signaled = first;
1273	} else {
1274	timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1275	timeout = drm_syncobj_array_wait_timeout(syncobjs,
1276	u64_to_user_ptr(timeline_wait->points),
1277	count: timeline_wait->count_handles,
1278	flags: timeline_wait->flags,
1279	timeout, idx: &first,
1280	deadline);
1281	if (timeout < 0)
1282	return timeout;
1283	timeline_wait->first_signaled = first;
1284	}
1285	return 0;
1286	}
1287
1288	static int drm_syncobj_array_find(struct drm_file *file_private,
1289	void __user *user_handles,
1290	uint32_t count_handles,
1291	struct drm_syncobj ***syncobjs_out)
1292	{
1293	uint32_t i, *handles;
1294	struct drm_syncobj **syncobjs;
1295	int ret;
1296
1297	handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1298	if (handles == NULL)
1299	return -ENOMEM;
1300
1301	if (copy_from_user(to: handles, from: user_handles,
1302	n: sizeof(uint32_t) * count_handles)) {
1303	ret = -EFAULT;
1304	goto err_free_handles;
1305	}
1306
1307	syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1308	if (syncobjs == NULL) {
1309	ret = -ENOMEM;
1310	goto err_free_handles;
1311	}
1312
1313	for (i = 0; i < count_handles; i++) {
1314	syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1315	if (!syncobjs[i]) {
1316	ret = -ENOENT;
1317	goto err_put_syncobjs;
1318	}
1319	}
1320
1321	kfree(objp: handles);
1322	*syncobjs_out = syncobjs;
1323	return 0;
1324
1325	err_put_syncobjs:
1326	while (i-- > 0)
1327	drm_syncobj_put(obj: syncobjs[i]);
1328	kfree(objp: syncobjs);
1329	err_free_handles:
1330	kfree(objp: handles);
1331
1332	return ret;
1333	}
1334
1335	static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1336	uint32_t count)
1337	{
1338	uint32_t i;
1339
1340	for (i = 0; i < count; i++)
1341	drm_syncobj_put(obj: syncobjs[i]);
1342	kfree(objp: syncobjs);
1343	}
1344
1345	int
1346	drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1347	struct drm_file *file_private)
1348	{
1349	struct drm_syncobj_wait *args = data;
1350	struct drm_syncobj **syncobjs;
1351	unsigned int possible_flags;
1352	ktime_t t, *tp = NULL;
1353	int ret = 0;
1354
1355	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
1356	return -EOPNOTSUPP;
1357
1358	possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1359	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1360	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1361
1362	if (args->flags & ~possible_flags)
1363	return -EINVAL;
1364
1365	if (args->count_handles == 0)
1366	return 0;
1367
1368	ret = drm_syncobj_array_find(file_private,
1369	u64_to_user_ptr(args->handles),
1370	count_handles: args->count_handles,
1371	syncobjs_out: &syncobjs);
1372	if (ret < 0)
1373	return ret;
1374
1375	if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1376	t = ns_to_ktime(ns: args->deadline_nsec);
1377	tp = &t;
1378	}
1379
1380	ret = drm_syncobj_array_wait(dev, file_private,
1381	wait: args, NULL, syncobjs, timeline: false, deadline: tp);
1382
1383	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1384
1385	return ret;
1386	}
1387
1388	int
1389	drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1390	struct drm_file *file_private)
1391	{
1392	struct drm_syncobj_timeline_wait *args = data;
1393	struct drm_syncobj **syncobjs;
1394	unsigned int possible_flags;
1395	ktime_t t, *tp = NULL;
1396	int ret = 0;
1397
1398	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1399	return -EOPNOTSUPP;
1400
1401	possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1402	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1403	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE |
1404	DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1405
1406	if (args->flags & ~possible_flags)
1407	return -EINVAL;
1408
1409	if (args->count_handles == 0)
1410	return 0;
1411
1412	ret = drm_syncobj_array_find(file_private,
1413	u64_to_user_ptr(args->handles),
1414	count_handles: args->count_handles,
1415	syncobjs_out: &syncobjs);
1416	if (ret < 0)
1417	return ret;
1418
1419	if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1420	t = ns_to_ktime(ns: args->deadline_nsec);
1421	tp = &t;
1422	}
1423
1424	ret = drm_syncobj_array_wait(dev, file_private,
1425	NULL, timeline_wait: args, syncobjs, timeline: true, deadline: tp);
1426
1427	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1428
1429	return ret;
1430	}
1431
1432	static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1433	struct dma_fence_cb *cb)
1434	{
1435	struct syncobj_eventfd_entry *entry =
1436	container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1437
1438	eventfd_signal(ctx: entry->ev_fd_ctx);
1439	syncobj_eventfd_entry_free(entry);
1440	}
1441
1442	static void
1443	syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1444	struct syncobj_eventfd_entry *entry)
1445	{
1446	int ret;
1447	struct dma_fence *fence;
1448
1449	/* This happens inside the syncobj lock */
1450	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1451	if (!fence)
1452	return;
1453
1454	ret = dma_fence_chain_find_seqno(pfence: &fence, seqno: entry->point);
1455	if (ret != 0) {
1456	/* The given seqno has not been submitted yet. */
1457	dma_fence_put(fence);
1458	return;
1459	} else if (!fence) {
1460	/* If dma_fence_chain_find_seqno returns 0 but sets the fence
1461	* to NULL, it implies that the given seqno is signaled and a
1462	* later seqno has already been submitted. Assign a stub fence
1463	* so that the eventfd still gets signaled below.
1464	*/
1465	fence = dma_fence_get_stub();
1466	}
1467
1468	list_del_init(entry: &entry->node);
1469	entry->fence = fence;
1470
1471	if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1472	eventfd_signal(ctx: entry->ev_fd_ctx);
1473	syncobj_eventfd_entry_free(entry);
1474	} else {
1475	ret = dma_fence_add_callback(fence, cb: &entry->fence_cb,
1476	func: syncobj_eventfd_entry_fence_func);
1477	if (ret == -ENOENT) {
1478	eventfd_signal(ctx: entry->ev_fd_ctx);
1479	syncobj_eventfd_entry_free(entry);
1480	}
1481	}
1482	}
1483
1484	int
1485	drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1486	struct drm_file *file_private)
1487	{
1488	struct drm_syncobj_eventfd *args = data;
1489	struct drm_syncobj *syncobj;
1490	struct eventfd_ctx *ev_fd_ctx;
1491	struct syncobj_eventfd_entry *entry;
1492	int ret;
1493
1494	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1495	return -EOPNOTSUPP;
1496
1497	if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1498	return -EINVAL;
1499
1500	if (args->pad)
1501	return -EINVAL;
1502
1503	syncobj = drm_syncobj_find(file_private, args->handle);
1504	if (!syncobj)
1505	return -ENOENT;
1506
1507	ev_fd_ctx = eventfd_ctx_fdget(fd: args->fd);
1508	if (IS_ERR(ptr: ev_fd_ctx)) {
1509	ret = PTR_ERR(ptr: ev_fd_ctx);
1510	goto err_fdget;
1511	}
1512
1513	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1514	if (!entry) {
1515	ret = -ENOMEM;
1516	goto err_kzalloc;
1517	}
1518	entry->syncobj = syncobj;
1519	entry->ev_fd_ctx = ev_fd_ctx;
1520	entry->point = args->point;
1521	entry->flags = args->flags;
1522
1523	drm_syncobj_add_eventfd(syncobj, entry);
1524	drm_syncobj_put(obj: syncobj);
1525
1526	return 0;
1527
1528	err_kzalloc:
1529	eventfd_ctx_put(ctx: ev_fd_ctx);
1530	err_fdget:
1531	drm_syncobj_put(obj: syncobj);
1532	return ret;
1533	}
1534
1535	int
1536	drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1537	struct drm_file *file_private)
1538	{
1539	struct drm_syncobj_array *args = data;
1540	struct drm_syncobj **syncobjs;
1541	uint32_t i;
1542	int ret;
1543
1544	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
1545	return -EOPNOTSUPP;
1546
1547	if (args->pad != 0)
1548	return -EINVAL;
1549
1550	if (args->count_handles == 0)
1551	return -EINVAL;
1552
1553	ret = drm_syncobj_array_find(file_private,
1554	u64_to_user_ptr(args->handles),
1555	count_handles: args->count_handles,
1556	syncobjs_out: &syncobjs);
1557	if (ret < 0)
1558	return ret;
1559
1560	for (i = 0; i < args->count_handles; i++)
1561	drm_syncobj_replace_fence(syncobjs[i], NULL);
1562
1563	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1564
1565	return 0;
1566	}
1567
1568	int
1569	drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1570	struct drm_file *file_private)
1571	{
1572	struct drm_syncobj_array *args = data;
1573	struct drm_syncobj **syncobjs;
1574	uint32_t i;
1575	int ret;
1576
1577	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ))
1578	return -EOPNOTSUPP;
1579
1580	if (args->pad != 0)
1581	return -EINVAL;
1582
1583	if (args->count_handles == 0)
1584	return -EINVAL;
1585
1586	ret = drm_syncobj_array_find(file_private,
1587	u64_to_user_ptr(args->handles),
1588	count_handles: args->count_handles,
1589	syncobjs_out: &syncobjs);
1590	if (ret < 0)
1591	return ret;
1592
1593	for (i = 0; i < args->count_handles; i++) {
1594	ret = drm_syncobj_assign_null_handle(syncobj: syncobjs[i]);
1595	if (ret < 0)
1596	break;
1597	}
1598
1599	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1600
1601	return ret;
1602	}
1603
1604	int
1605	drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1606	struct drm_file *file_private)
1607	{
1608	struct drm_syncobj_timeline_array *args = data;
1609	struct drm_syncobj **syncobjs;
1610	struct dma_fence_chain **chains;
1611	uint64_t *points;
1612	uint32_t i, j;
1613	int ret;
1614
1615	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1616	return -EOPNOTSUPP;
1617
1618	if (args->flags != 0)
1619	return -EINVAL;
1620
1621	if (args->count_handles == 0)
1622	return -EINVAL;
1623
1624	ret = drm_syncobj_array_find(file_private,
1625	u64_to_user_ptr(args->handles),
1626	count_handles: args->count_handles,
1627	syncobjs_out: &syncobjs);
1628	if (ret < 0)
1629	return ret;
1630
1631	points = kmalloc_array(args->count_handles, sizeof(*points),
1632	GFP_KERNEL);
1633	if (!points) {
1634	ret = -ENOMEM;
1635	goto out;
1636	}
1637	if (!u64_to_user_ptr(args->points)) {
1638	memset(points, 0, args->count_handles * sizeof(uint64_t));
1639	} else if (copy_from_user(to: points, u64_to_user_ptr(args->points),
1640	n: sizeof(uint64_t) * args->count_handles)) {
1641	ret = -EFAULT;
1642	goto err_points;
1643	}
1644
1645	chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1646	if (!chains) {
1647	ret = -ENOMEM;
1648	goto err_points;
1649	}
1650	for (i = 0; i < args->count_handles; i++) {
1651	chains[i] = dma_fence_chain_alloc();
1652	if (!chains[i]) {
1653	for (j = 0; j < i; j++)
1654	dma_fence_chain_free(chain: chains[j]);
1655	ret = -ENOMEM;
1656	goto err_chains;
1657	}
1658	}
1659
1660	for (i = 0; i < args->count_handles; i++) {
1661	struct dma_fence *fence = dma_fence_get_stub();
1662
1663	drm_syncobj_add_point(syncobjs[i], chains[i],
1664	fence, points[i]);
1665	dma_fence_put(fence);
1666	}
1667	err_chains:
1668	kfree(objp: chains);
1669	err_points:
1670	kfree(objp: points);
1671	out:
1672	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1673
1674	return ret;
1675	}
1676
1677	int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1678	struct drm_file *file_private)
1679	{
1680	struct drm_syncobj_timeline_array *args = data;
1681	struct drm_syncobj **syncobjs;
1682	uint64_t __user *points = u64_to_user_ptr(args->points);
1683	uint32_t i;
1684	int ret;
1685
1686	if (!drm_core_check_feature(dev, feature: DRIVER_SYNCOBJ_TIMELINE))
1687	return -EOPNOTSUPP;
1688
1689	if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1690	return -EINVAL;
1691
1692	if (args->count_handles == 0)
1693	return -EINVAL;
1694
1695	ret = drm_syncobj_array_find(file_private,
1696	u64_to_user_ptr(args->handles),
1697	count_handles: args->count_handles,
1698	syncobjs_out: &syncobjs);
1699	if (ret < 0)
1700	return ret;
1701
1702	for (i = 0; i < args->count_handles; i++) {
1703	struct dma_fence_chain *chain;
1704	struct dma_fence *fence;
1705	uint64_t point;
1706
1707	fence = drm_syncobj_fence_get(syncobj: syncobjs[i]);
1708	chain = to_dma_fence_chain(fence);
1709	if (chain) {
1710	struct dma_fence *iter, *last_signaled =
1711	dma_fence_get(fence);
1712
1713	if (args->flags &
1714	DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1715	point = fence->seqno;
1716	} else {
1717	dma_fence_chain_for_each(iter, fence) {
1718	if (iter->context != fence->context) {
1719	dma_fence_put(fence: iter);
1720	/* It is most likely that timeline has
1721	* unorder points. */
1722	break;
1723	}
1724	dma_fence_put(fence: last_signaled);
1725	last_signaled = dma_fence_get(fence: iter);
1726	}
1727	point = dma_fence_is_signaled(fence: last_signaled) ?
1728	last_signaled->seqno :
1729	to_dma_fence_chain(fence: last_signaled)->prev_seqno;
1730	}
1731	dma_fence_put(fence: last_signaled);
1732	} else {
1733	point = 0;
1734	}
1735	dma_fence_put(fence);
1736	ret = copy_to_user(to: &points[i], from: &point, n: sizeof(uint64_t));
1737	ret = ret ? -EFAULT : 0;
1738	if (ret)
1739	break;
1740	}
1741	drm_syncobj_array_free(syncobjs, count: args->count_handles);
1742
1743	return ret;
1744	}
1745