1	/*
2	* drm_irq.c IRQ and vblank support
3	*
4	* \author Rickard E. (Rik) Faith <faith@valinux.com>
5	* \author Gareth Hughes <gareth@valinux.com>
6	*
7	* Permission is hereby granted, free of charge, to any person obtaining a
8	* copy of this software and associated documentation files (the "Software"),
9	* to deal in the Software without restriction, including without limitation
10	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
11	* and/or sell copies of the Software, and to permit persons to whom the
12	* Software is furnished to do so, subject to the following conditions:
13	*
14	* The above copyright notice and this permission notice (including the next
15	* paragraph) shall be included in all copies or substantial portions of the
16	* Software.
17	*
18	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21	* VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24	* OTHER DEALINGS IN THE SOFTWARE.
25	*/
26
27	#include <linux/export.h>
28	#include <linux/kthread.h>
29	#include <linux/moduleparam.h>
30
31	#include <drm/drm_crtc.h>
32	#include <drm/drm_drv.h>
33	#include <drm/drm_framebuffer.h>
34	#include <drm/drm_managed.h>
35	#include <drm/drm_modeset_helper_vtables.h>
36	#include <drm/drm_print.h>
37	#include <drm/drm_vblank.h>
38
39	#include "drm_internal.h"
40	#include "drm_trace.h"
41
42	/**
43	* DOC: vblank handling
44	*
45	* From the computer's perspective, every time the monitor displays
46	* a new frame the scanout engine has "scanned out" the display image
47	* from top to bottom, one row of pixels at a time. The current row
48	* of pixels is referred to as the current scanline.
49	*
50	* In addition to the display's visible area, there's usually a couple of
51	* extra scanlines which aren't actually displayed on the screen.
52	* These extra scanlines don't contain image data and are occasionally used
53	* for features like audio and infoframes. The region made up of these
54	* scanlines is referred to as the vertical blanking region, or vblank for
55	* short.
56	*
57	* For historical reference, the vertical blanking period was designed to
58	* give the electron gun (on CRTs) enough time to move back to the top of
59	* the screen to start scanning out the next frame. Similar for horizontal
60	* blanking periods. They were designed to give the electron gun enough
61	* time to move back to the other side of the screen to start scanning the
62	* next scanline.
63	*
64	* ::
65	*
66	*
67	* physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
68	* top of | |
69	* display | |
70	* | New frame |
71	* | |
72	* |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
73	* |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
74	* |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
75	* | | frame as it
76	* | | travels down
77	* | | ("scan out")
78	* | Old frame |
79	* | |
80	* | |
81	* | |
82	* | | physical
83	* | | bottom of
84	* vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
85	* blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86	* region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87	* ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
88	* start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
89	* new frame
90	*
91	* "Physical top of display" is the reference point for the high-precision/
92	* corrected timestamp.
93	*
94	* On a lot of display hardware, programming needs to take effect during the
95	* vertical blanking period so that settings like gamma, the image buffer
96	* buffer to be scanned out, etc. can safely be changed without showing
97	* any visual artifacts on the screen. In some unforgiving hardware, some of
98	* this programming has to both start and end in the same vblank. To help
99	* with the timing of the hardware programming, an interrupt is usually
100	* available to notify the driver when it can start the updating of registers.
101	* The interrupt is in this context named the vblank interrupt.
102	*
103	* The vblank interrupt may be fired at different points depending on the
104	* hardware. Some hardware implementations will fire the interrupt when the
105	* new frame start, other implementations will fire the interrupt at different
106	* points in time.
107	*
108	* Vertical blanking plays a major role in graphics rendering. To achieve
109	* tear-free display, users must synchronize page flips and/or rendering to
110	* vertical blanking. The DRM API offers ioctls to perform page flips
111	* synchronized to vertical blanking and wait for vertical blanking.
112	*
113	* The DRM core handles most of the vertical blanking management logic, which
114	* involves filtering out spurious interrupts, keeping race-free blanking
115	* counters, coping with counter wrap-around and resets and keeping use counts.
116	* It relies on the driver to generate vertical blanking interrupts and
117	* optionally provide a hardware vertical blanking counter.
118	*
119	* Drivers must initialize the vertical blanking handling core with a call to
120	* drm_vblank_init(). Minimally, a driver needs to implement
121	* &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
122	* drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
123	* support.
124	*
125	* Vertical blanking interrupts can be enabled by the DRM core or by drivers
126	* themselves (for instance to handle page flipping operations). The DRM core
127	* maintains a vertical blanking use count to ensure that the interrupts are not
128	* disabled while a user still needs them. To increment the use count, drivers
129	* call drm_crtc_vblank_get() and release the vblank reference again with
130	* drm_crtc_vblank_put(). In between these two calls vblank interrupts are
131	* guaranteed to be enabled.
132	*
133	* On many hardware disabling the vblank interrupt cannot be done in a race-free
134	* manner, see &drm_vblank_crtc_config.disable_immediate and
135	* &drm_driver.max_vblank_count. In that case the vblank core only disables the
136	* vblanks after a timer has expired, which can be configured through the
137	* ``vblankoffdelay`` module parameter.
138	*
139	* Drivers for hardware without support for vertical-blanking interrupts can
140	* use DRM vblank timers to send vblank events at the rate of the current
141	* display mode's refresh. While not synchronized to the hardware's
142	* vertical-blanking regions, the timer helps DRM clients and compositors to
143	* adapt their update cycle to the display output. Drivers should set up
144	* vblanking as usual, but call drm_crtc_vblank_start_timer() and
145	* drm_crtc_vblank_cancel_timer() as part of their atomic mode setting.
146	* See also DRM vblank helpers for more information.
147	*
148	* Drivers without support for vertical-blanking interrupts nor timers must
149	* not call drm_vblank_init(). For these drivers, atomic helpers will
150	* automatically generate fake vblank events as part of the display update.
151	* This functionality also can be controlled by the driver by enabling and
152	* disabling struct drm_crtc_state.no_vblank.
153	*/
154
155	/* Retry timestamp calculation up to 3 times to satisfy
156	* drm_timestamp_precision before giving up.
157	*/
158	#define DRM_TIMESTAMP_MAXRETRIES 3
159
160	/* Threshold in nanoseconds for detection of redundant
161	* vblank irq in drm_handle_vblank(). 1 msec should be ok.
162	*/
163	#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
164
165	static bool
166	drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
167	ktime_t *tvblank, bool in_vblank_irq);
168
169	static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
170
171	static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
172
173	module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
174	module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
175	MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
176	MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
177
178	static struct drm_vblank_crtc *
179	drm_vblank_crtc(struct drm_device *dev, unsigned int pipe)
180	{
181	return &dev->vblank[pipe];
182	}
183
184	struct drm_vblank_crtc *
185	drm_crtc_vblank_crtc(struct drm_crtc *crtc)
186	{
187	return drm_vblank_crtc(dev: crtc->dev, pipe: drm_crtc_index(crtc));
188	}
189	EXPORT_SYMBOL(drm_crtc_vblank_crtc);
190
191	static void store_vblank(struct drm_device *dev, unsigned int pipe,
192	u32 vblank_count_inc,
193	ktime_t t_vblank, u32 last)
194	{
195	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
196
197	assert_spin_locked(&dev->vblank_time_lock);
198
199	vblank->last = last;
200
201	write_seqlock(sl: &vblank->seqlock);
202	vblank->time = t_vblank;
203	atomic64_add(i: vblank_count_inc, v: &vblank->count);
204	write_sequnlock(sl: &vblank->seqlock);
205	}
206
207	static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
208	{
209	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
210
211	return vblank->max_vblank_count ?: dev->max_vblank_count;
212	}
213
214	/*
215	* "No hw counter" fallback implementation of .get_vblank_counter() hook,
216	* if there is no usable hardware frame counter available.
217	*/
218	static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
219	{
220	drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
221	return 0;
222	}
223
224	static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
225	{
226	if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
227	struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
228
229	if (drm_WARN_ON(dev, !crtc))
230	return 0;
231
232	if (crtc->funcs->get_vblank_counter)
233	return crtc->funcs->get_vblank_counter(crtc);
234	}
235
236	return drm_vblank_no_hw_counter(dev, pipe);
237	}
238
239	/*
240	* Reset the stored timestamp for the current vblank count to correspond
241	* to the last vblank occurred.
242	*
243	* Only to be called from drm_crtc_vblank_on().
244	*
245	* Note: caller must hold &drm_device.vbl_lock since this reads & writes
246	* device vblank fields.
247	*/
248	static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
249	{
250	u32 cur_vblank;
251	bool rc;
252	ktime_t t_vblank;
253	int count = DRM_TIMESTAMP_MAXRETRIES;
254
255	spin_lock(lock: &dev->vblank_time_lock);
256
257	/*
258	* sample the current counter to avoid random jumps
259	* when drm_vblank_enable() applies the diff
260	*/
261	do {
262	cur_vblank = __get_vblank_counter(dev, pipe);
263	rc = drm_get_last_vbltimestamp(dev, pipe, tvblank: &t_vblank, in_vblank_irq: false);
264	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
265
266	/*
267	* Only reinitialize corresponding vblank timestamp if high-precision query
268	* available and didn't fail. Otherwise reinitialize delayed at next vblank
269	* interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
270	*/
271	if (!rc)
272	t_vblank = 0;
273
274	/*
275	* +1 to make sure user will never see the same
276	* vblank counter value before and after a modeset
277	*/
278	store_vblank(dev, pipe, vblank_count_inc: 1, t_vblank, last: cur_vblank);
279
280	spin_unlock(lock: &dev->vblank_time_lock);
281	}
282
283	/*
284	* Call back into the driver to update the appropriate vblank counter
285	* (specified by @pipe). Deal with wraparound, if it occurred, and
286	* update the last read value so we can deal with wraparound on the next
287	* call if necessary.
288	*
289	* Only necessary when going from off->on, to account for frames we
290	* didn't get an interrupt for.
291	*
292	* Note: caller must hold &drm_device.vbl_lock since this reads & writes
293	* device vblank fields.
294	*/
295	static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
296	bool in_vblank_irq)
297	{
298	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
299	u32 cur_vblank, diff;
300	bool rc;
301	ktime_t t_vblank;
302	int count = DRM_TIMESTAMP_MAXRETRIES;
303	int framedur_ns = vblank->framedur_ns;
304	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
305
306	/*
307	* Interrupts were disabled prior to this call, so deal with counter
308	* wrap if needed.
309	* NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
310	* here if the register is small or we had vblank interrupts off for
311	* a long time.
312	*
313	* We repeat the hardware vblank counter & timestamp query until
314	* we get consistent results. This to prevent races between gpu
315	* updating its hardware counter while we are retrieving the
316	* corresponding vblank timestamp.
317	*/
318	do {
319	cur_vblank = __get_vblank_counter(dev, pipe);
320	rc = drm_get_last_vbltimestamp(dev, pipe, tvblank: &t_vblank, in_vblank_irq);
321	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
322
323	if (max_vblank_count) {
324	/* trust the hw counter when it's around */
325	diff = (cur_vblank - vblank->last) & max_vblank_count;
326	} else if (rc && framedur_ns) {
327	u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
328
329	/*
330	* Figure out how many vblanks we've missed based
331	* on the difference in the timestamps and the
332	* frame/field duration.
333	*/
334
335	drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
336	" diff_ns = %lld, framedur_ns = %d)\n",
337	pipe, (long long)diff_ns, framedur_ns);
338
339	diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
340
341	if (diff == 0 && in_vblank_irq)
342	drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
343	pipe);
344	} else {
345	/* some kind of default for drivers w/o accurate vbl timestamping */
346	diff = in_vblank_irq ? 1 : 0;
347	}
348
349	/*
350	* Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
351	* interval? If so then vblank irqs keep running and it will likely
352	* happen that the hardware vblank counter is not trustworthy as it
353	* might reset at some point in that interval and vblank timestamps
354	* are not trustworthy either in that interval. Iow. this can result
355	* in a bogus diff >> 1 which must be avoided as it would cause
356	* random large forward jumps of the software vblank counter.
357	*/
358	if (diff > 1 && (vblank->inmodeset & 0x2)) {
359	drm_dbg_vbl(dev,
360	"clamping vblank bump to 1 on crtc %u: diffr=%u"
361	" due to pre-modeset.\n", pipe, diff);
362	diff = 1;
363	}
364
365	drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
366	" current=%llu, diff=%u, hw=%u hw_last=%u\n",
367	pipe, (unsigned long long)atomic64_read(&vblank->count),
368	diff, cur_vblank, vblank->last);
369
370	if (diff == 0) {
371	drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
372	return;
373	}
374
375	/*
376	* Only reinitialize corresponding vblank timestamp if high-precision query
377	* available and didn't fail, or we were called from the vblank interrupt.
378	* Otherwise reinitialize delayed at next vblank interrupt and assign 0
379	* for now, to mark the vblanktimestamp as invalid.
380	*/
381	if (!rc && !in_vblank_irq)
382	t_vblank = 0;
383
384	store_vblank(dev, pipe, vblank_count_inc: diff, t_vblank, last: cur_vblank);
385	}
386
387	u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
388	{
389	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
390	u64 count;
391
392	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
393	return 0;
394
395	count = atomic64_read(v: &vblank->count);
396
397	/*
398	* This read barrier corresponds to the implicit write barrier of the
399	* write seqlock in store_vblank(). Note that this is the only place
400	* where we need an explicit barrier, since all other access goes
401	* through drm_vblank_count_and_time(), which already has the required
402	* read barrier curtesy of the read seqlock.
403	*/
404	smp_rmb();
405
406	return count;
407	}
408
409	/**
410	* drm_crtc_accurate_vblank_count - retrieve the master vblank counter
411	* @crtc: which counter to retrieve
412	*
413	* This function is similar to drm_crtc_vblank_count() but this function
414	* interpolates to handle a race with vblank interrupts using the high precision
415	* timestamping support.
416	*
417	* This is mostly useful for hardware that can obtain the scanout position, but
418	* doesn't have a hardware frame counter.
419	*/
420	u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
421	{
422	struct drm_device *dev = crtc->dev;
423	unsigned int pipe = drm_crtc_index(crtc);
424	u64 vblank;
425	unsigned long flags;
426
427	drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
428	!crtc->funcs->get_vblank_timestamp,
429	"This function requires support for accurate vblank timestamps.");
430
431	spin_lock_irqsave(&dev->vblank_time_lock, flags);
432
433	drm_update_vblank_count(dev, pipe, in_vblank_irq: false);
434	vblank = drm_vblank_count(dev, pipe);
435
436	spin_unlock_irqrestore(lock: &dev->vblank_time_lock, flags);
437
438	return vblank;
439	}
440	EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
441
442	static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
443	{
444	if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
445	struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
446
447	if (drm_WARN_ON(dev, !crtc))
448	return;
449
450	if (crtc->funcs->disable_vblank)
451	crtc->funcs->disable_vblank(crtc);
452	}
453	}
454
455	/*
456	* Disable vblank irq's on crtc, make sure that last vblank count
457	* of hardware and corresponding consistent software vblank counter
458	* are preserved, even if there are any spurious vblank irq's after
459	* disable.
460	*/
461	void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
462	{
463	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
464	unsigned long irqflags;
465
466	assert_spin_locked(&dev->vbl_lock);
467
468	/* Prevent vblank irq processing while disabling vblank irqs,
469	* so no updates of timestamps or count can happen after we've
470	* disabled. Needed to prevent races in case of delayed irq's.
471	*/
472	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
473
474	/*
475	* Update vblank count and disable vblank interrupts only if the
476	* interrupts were enabled. This avoids calling the ->disable_vblank()
477	* operation in atomic context with the hardware potentially runtime
478	* suspended.
479	*/
480	if (!vblank->enabled)
481	goto out;
482
483	/*
484	* Update the count and timestamp to maintain the
485	* appearance that the counter has been ticking all along until
486	* this time. This makes the count account for the entire time
487	* between drm_crtc_vblank_on() and drm_crtc_vblank_off().
488	*/
489	drm_update_vblank_count(dev, pipe, in_vblank_irq: false);
490	__disable_vblank(dev, pipe);
491	vblank->enabled = false;
492
493	out:
494	spin_unlock_irqrestore(lock: &dev->vblank_time_lock, flags: irqflags);
495	}
496
497	static void vblank_disable_fn(struct timer_list *t)
498	{
499	struct drm_vblank_crtc *vblank = timer_container_of(vblank, t,
500	disable_timer);
501	struct drm_device *dev = vblank->dev;
502	unsigned int pipe = vblank->pipe;
503	unsigned long irqflags;
504
505	spin_lock_irqsave(&dev->vbl_lock, irqflags);
506	if (atomic_read(v: &vblank->refcount) == 0 && vblank->enabled) {
507	drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
508	drm_vblank_disable_and_save(dev, pipe);
509	}
510	spin_unlock_irqrestore(lock: &dev->vbl_lock, flags: irqflags);
511	}
512
513	static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
514	{
515	struct drm_vblank_crtc *vblank = ptr;
516
517	drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
518	drm_core_check_feature(dev, DRIVER_MODESET));
519
520	if (vblank->vblank_timer.crtc)
521	hrtimer_cancel(timer: &vblank->vblank_timer.timer);
522
523	drm_vblank_destroy_worker(vblank);
524	timer_delete_sync(timer: &vblank->disable_timer);
525	}
526
527	/**
528	* drm_vblank_init - initialize vblank support
529	* @dev: DRM device
530	* @num_crtcs: number of CRTCs supported by @dev
531	*
532	* This function initializes vblank support for @num_crtcs display pipelines.
533	* Cleanup is handled automatically through a cleanup function added with
534	* drmm_add_action_or_reset().
535	*
536	* Returns:
537	* Zero on success or a negative error code on failure.
538	*/
539	int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
540	{
541	int ret;
542	unsigned int i;
543
544	spin_lock_init(&dev->vbl_lock);
545	spin_lock_init(&dev->vblank_time_lock);
546
547	dev->vblank = drmm_kcalloc(dev, n: num_crtcs, size: sizeof(*dev->vblank), GFP_KERNEL);
548	if (!dev->vblank)
549	return -ENOMEM;
550
551	dev->num_crtcs = num_crtcs;
552
553	for (i = 0; i < num_crtcs; i++) {
554	struct drm_vblank_crtc *vblank = &dev->vblank[i];
555
556	vblank->dev = dev;
557	vblank->pipe = i;
558	init_waitqueue_head(&vblank->queue);
559	timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
560	seqlock_init(&vblank->seqlock);
561
562	ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
563	vblank);
564	if (ret)
565	return ret;
566
567	ret = drm_vblank_worker_init(vblank);
568	if (ret)
569	return ret;
570	}
571
572	return 0;
573	}
574	EXPORT_SYMBOL(drm_vblank_init);
575
576	/**
577	* drm_dev_has_vblank - test if vblanking has been initialized for
578	* a device
579	* @dev: the device
580	*
581	* Drivers may call this function to test if vblank support is
582	* initialized for a device. For most hardware this means that vblanking
583	* can also be enabled.
584	*
585	* Atomic helpers use this function to initialize
586	* &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
587	*
588	* Returns:
589	* True if vblanking has been initialized for the given device, false
590	* otherwise.
591	*/
592	bool drm_dev_has_vblank(const struct drm_device *dev)
593	{
594	return dev->num_crtcs != 0;
595	}
596	EXPORT_SYMBOL(drm_dev_has_vblank);
597
598	/**
599	* drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
600	* @crtc: which CRTC's vblank waitqueue to retrieve
601	*
602	* This function returns a pointer to the vblank waitqueue for the CRTC.
603	* Drivers can use this to implement vblank waits using wait_event() and related
604	* functions.
605	*/
606	wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
607	{
608	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
609	}
610	EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
611
612
613	/**
614	* drm_calc_timestamping_constants - calculate vblank timestamp constants
615	* @crtc: drm_crtc whose timestamp constants should be updated.
616	* @mode: display mode containing the scanout timings
617	*
618	* Calculate and store various constants which are later needed by vblank and
619	* swap-completion timestamping, e.g, by
620	* drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
621	* CRTC's true scanout timing, so they take things like panel scaling or
622	* other adjustments into account.
623	*/
624	void drm_calc_timestamping_constants(struct drm_crtc *crtc,
625	const struct drm_display_mode *mode)
626	{
627	struct drm_device *dev = crtc->dev;
628	unsigned int pipe = drm_crtc_index(crtc);
629	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
630	int linedur_ns = 0, framedur_ns = 0;
631	int dotclock = mode->crtc_clock;
632
633	if (!drm_dev_has_vblank(dev))
634	return;
635
636	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
637	return;
638
639	/* Valid dotclock? */
640	if (dotclock > 0) {
641	int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
642
643	/*
644	* Convert scanline length in pixels and video
645	* dot clock to line duration and frame duration
646	* in nanoseconds:
647	*/
648	linedur_ns = div_u64(dividend: (u64) mode->crtc_htotal * 1000000, divisor: dotclock);
649	framedur_ns = div_u64(dividend: (u64) frame_size * 1000000, divisor: dotclock);
650
651	/*
652	* Fields of interlaced scanout modes are only half a frame duration.
653	*/
654	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
655	framedur_ns /= 2;
656	} else {
657	drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
658	crtc->base.id);
659	}
660
661	vblank->linedur_ns = linedur_ns;
662	vblank->framedur_ns = framedur_ns;
663	drm_mode_copy(dst: &vblank->hwmode, src: mode);
664
665	drm_dbg_core(dev,
666	"crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
667	crtc->base.id, mode->crtc_htotal,
668	mode->crtc_vtotal, mode->crtc_vdisplay);
669	drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
670	crtc->base.id, dotclock, framedur_ns, linedur_ns);
671	}
672	EXPORT_SYMBOL(drm_calc_timestamping_constants);
673
674	/**
675	* drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
676	* timestamp helper
677	* @crtc: CRTC whose vblank timestamp to retrieve
678	* @max_error: Desired maximum allowable error in timestamps (nanosecs)
679	* On return contains true maximum error of timestamp
680	* @vblank_time: Pointer to time which should receive the timestamp
681	* @in_vblank_irq:
682	* True when called from drm_crtc_handle_vblank(). Some drivers
683	* need to apply some workarounds for gpu-specific vblank irq quirks
684	* if flag is set.
685	* @get_scanout_position:
686	* Callback function to retrieve the scanout position. See
687	* @struct drm_crtc_helper_funcs.get_scanout_position.
688	*
689	* Implements calculation of exact vblank timestamps from given drm_display_mode
690	* timings and current video scanout position of a CRTC.
691	*
692	* The current implementation only handles standard video modes. For double scan
693	* and interlaced modes the driver is supposed to adjust the hardware mode
694	* (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
695	* match the scanout position reported.
696	*
697	* Note that atomic drivers must call drm_calc_timestamping_constants() before
698	* enabling a CRTC. The atomic helpers already take care of that in
699	* drm_atomic_helper_calc_timestamping_constants().
700	*
701	* Returns:
702	* Returns true on success, and false on failure, i.e. when no accurate
703	* timestamp could be acquired.
704	*/
705	bool
706	drm_crtc_vblank_helper_get_vblank_timestamp_internal(
707	struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
708	bool in_vblank_irq,
709	drm_vblank_get_scanout_position_func get_scanout_position)
710	{
711	struct drm_device *dev = crtc->dev;
712	unsigned int pipe = crtc->index;
713	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
714	struct timespec64 ts_etime, ts_vblank_time;
715	ktime_t stime, etime;
716	bool vbl_status;
717	const struct drm_display_mode *mode;
718	int vpos, hpos, i;
719	int delta_ns, duration_ns;
720
721	if (pipe >= dev->num_crtcs) {
722	drm_err(dev, "Invalid crtc %u\n", pipe);
723	return false;
724	}
725
726	/* Scanout position query not supported? Should not happen. */
727	if (!get_scanout_position) {
728	drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
729	return false;
730	}
731
732	if (drm_drv_uses_atomic_modeset(dev))
733	mode = &vblank->hwmode;
734	else
735	mode = &crtc->hwmode;
736
737	/* If mode timing undefined, just return as no-op:
738	* Happens during initial modesetting of a crtc.
739	*/
740	if (mode->crtc_clock == 0) {
741	drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
742	pipe);
743	drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
744	return false;
745	}
746
747	/* Get current scanout position with system timestamp.
748	* Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
749	* if single query takes longer than max_error nanoseconds.
750	*
751	* This guarantees a tight bound on maximum error if
752	* code gets preempted or delayed for some reason.
753	*/
754	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
755	/*
756	* Get vertical and horizontal scanout position vpos, hpos,
757	* and bounding timestamps stime, etime, pre/post query.
758	*/
759	vbl_status = get_scanout_position(crtc, in_vblank_irq,
760	&vpos, &hpos,
761	&stime, &etime,
762	mode);
763
764	/* Return as no-op if scanout query unsupported or failed. */
765	if (!vbl_status) {
766	drm_dbg_core(dev,
767	"crtc %u : scanoutpos query failed.\n",
768	pipe);
769	return false;
770	}
771
772	/* Compute uncertainty in timestamp of scanout position query. */
773	duration_ns = ktime_to_ns(kt: etime) - ktime_to_ns(kt: stime);
774
775	/* Accept result with < max_error nsecs timing uncertainty. */
776	if (duration_ns <= *max_error)
777	break;
778	}
779
780	/* Noisy system timing? */
781	if (i == DRM_TIMESTAMP_MAXRETRIES) {
782	drm_dbg_core(dev,
783	"crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
784	pipe, duration_ns / 1000, *max_error / 1000, i);
785	}
786
787	/* Return upper bound of timestamp precision error. */
788	*max_error = duration_ns;
789
790	/* Convert scanout position into elapsed time at raw_time query
791	* since start of scanout at first display scanline. delta_ns
792	* can be negative if start of scanout hasn't happened yet.
793	*/
794	delta_ns = div_s64(dividend: 1000000LL * (vpos * mode->crtc_htotal + hpos),
795	divisor: mode->crtc_clock);
796
797	/* Subtract time delta from raw timestamp to get final
798	* vblank_time timestamp for end of vblank.
799	*/
800	*vblank_time = ktime_sub_ns(etime, delta_ns);
801
802	if (!drm_debug_enabled(DRM_UT_VBL))
803	return true;
804
805	ts_etime = ktime_to_timespec64(etime);
806	ts_vblank_time = ktime_to_timespec64(*vblank_time);
807
808	drm_dbg_vbl(dev,
809	"crtc %u : v p(%d,%d)@ %ptSp -> %ptSp [e %d us, %d rep]\n",
810	pipe, hpos, vpos, &ts_etime, &ts_vblank_time,
811	duration_ns / 1000, i);
812
813	return true;
814	}
815	EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
816
817	/**
818	* drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
819	* helper
820	* @crtc: CRTC whose vblank timestamp to retrieve
821	* @max_error: Desired maximum allowable error in timestamps (nanosecs)
822	* On return contains true maximum error of timestamp
823	* @vblank_time: Pointer to time which should receive the timestamp
824	* @in_vblank_irq:
825	* True when called from drm_crtc_handle_vblank(). Some drivers
826	* need to apply some workarounds for gpu-specific vblank irq quirks
827	* if flag is set.
828	*
829	* Implements calculation of exact vblank timestamps from given drm_display_mode
830	* timings and current video scanout position of a CRTC. This can be directly
831	* used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
832	* driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
833	*
834	* The current implementation only handles standard video modes. For double scan
835	* and interlaced modes the driver is supposed to adjust the hardware mode
836	* (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
837	* match the scanout position reported.
838	*
839	* Note that atomic drivers must call drm_calc_timestamping_constants() before
840	* enabling a CRTC. The atomic helpers already take care of that in
841	* drm_atomic_helper_calc_timestamping_constants().
842	*
843	* Returns:
844	* Returns true on success, and false on failure, i.e. when no accurate
845	* timestamp could be acquired.
846	*/
847	bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
848	int *max_error,
849	ktime_t *vblank_time,
850	bool in_vblank_irq)
851	{
852	return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
853	crtc, max_error, vblank_time, in_vblank_irq,
854	crtc->helper_private->get_scanout_position);
855	}
856	EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
857
858	/**
859	* drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
860	* recent vblank interval
861	* @crtc: CRTC whose vblank timestamp to retrieve
862	* @tvblank: Pointer to target time which should receive the timestamp
863	* @in_vblank_irq:
864	* True when called from drm_crtc_handle_vblank(). Some drivers
865	* need to apply some workarounds for gpu-specific vblank irq quirks
866	* if flag is set.
867	*
868	* Fetches the system timestamp corresponding to the time of the most recent
869	* vblank interval on specified CRTC. May call into kms-driver to
870	* compute the timestamp with a high-precision GPU specific method.
871	*
872	* Returns zero if timestamp originates from uncorrected do_gettimeofday()
873	* call, i.e., it isn't very precisely locked to the true vblank.
874	*
875	* Returns:
876	* True if timestamp is considered to be very precise, false otherwise.
877	*/
878	static bool
879	drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
880	bool in_vblank_irq)
881	{
882	bool ret = false;
883
884	/* Define requested maximum error on timestamps (nanoseconds). */
885	int max_error = (int) drm_timestamp_precision * 1000;
886
887	/* Query driver if possible and precision timestamping enabled. */
888	if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
889	ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
890	tvblank, in_vblank_irq);
891	}
892
893	/* GPU high precision timestamp query unsupported or failed.
894	* Return current monotonic/gettimeofday timestamp as best estimate.
895	*/
896	if (!ret)
897	*tvblank = ktime_get();
898
899	return ret;
900	}
901
902	static bool
903	drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
904	ktime_t *tvblank, bool in_vblank_irq)
905	{
906	struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
907
908	return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
909	}
910
911	/**
912	* drm_crtc_vblank_count - retrieve "cooked" vblank counter value
913	* @crtc: which counter to retrieve
914	*
915	* Fetches the "cooked" vblank count value that represents the number of
916	* vblank events since the system was booted, including lost events due to
917	* modesetting activity. Note that this timer isn't correct against a racing
918	* vblank interrupt (since it only reports the software vblank counter), see
919	* drm_crtc_accurate_vblank_count() for such use-cases.
920	*
921	* Note that for a given vblank counter value drm_crtc_handle_vblank()
922	* and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
923	* provide a barrier: Any writes done before calling
924	* drm_crtc_handle_vblank() will be visible to callers of the later
925	* functions, if the vblank count is the same or a later one.
926	*
927	* See also &drm_vblank_crtc.count.
928	*
929	* Returns:
930	* The software vblank counter.
931	*/
932	u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
933	{
934	return drm_vblank_count(dev: crtc->dev, pipe: drm_crtc_index(crtc));
935	}
936	EXPORT_SYMBOL(drm_crtc_vblank_count);
937
938	/**
939	* drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
940	* system timestamp corresponding to that vblank counter value.
941	* @dev: DRM device
942	* @pipe: index of CRTC whose counter to retrieve
943	* @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
944	*
945	* Fetches the "cooked" vblank count value that represents the number of
946	* vblank events since the system was booted, including lost events due to
947	* modesetting activity. Returns corresponding system timestamp of the time
948	* of the vblank interval that corresponds to the current vblank counter value.
949	*
950	* This is the legacy version of drm_crtc_vblank_count_and_time().
951	*/
952	static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
953	ktime_t *vblanktime)
954	{
955	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
956	u64 vblank_count;
957	unsigned int seq;
958
959	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
960	*vblanktime = 0;
961	return 0;
962	}
963
964	do {
965	seq = read_seqbegin(sl: &vblank->seqlock);
966	vblank_count = atomic64_read(v: &vblank->count);
967	*vblanktime = vblank->time;
968	} while (read_seqretry(sl: &vblank->seqlock, start: seq));
969
970	return vblank_count;
971	}
972
973	/**
974	* drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
975	* and the system timestamp corresponding to that vblank counter value
976	* @crtc: which counter to retrieve
977	* @vblanktime: Pointer to time to receive the vblank timestamp.
978	*
979	* Fetches the "cooked" vblank count value that represents the number of
980	* vblank events since the system was booted, including lost events due to
981	* modesetting activity. Returns corresponding system timestamp of the time
982	* of the vblank interval that corresponds to the current vblank counter value.
983	*
984	* Note that for a given vblank counter value drm_crtc_handle_vblank()
985	* and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
986	* provide a barrier: Any writes done before calling
987	* drm_crtc_handle_vblank() will be visible to callers of the later
988	* functions, if the vblank count is the same or a later one.
989	*
990	* See also &drm_vblank_crtc.count.
991	*/
992	u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
993	ktime_t *vblanktime)
994	{
995	return drm_vblank_count_and_time(dev: crtc->dev, pipe: drm_crtc_index(crtc),
996	vblanktime);
997	}
998	EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
999
1000	/**
1001	* drm_crtc_next_vblank_start - calculate the time of the next vblank
1002	* @crtc: the crtc for which to calculate next vblank time
1003	* @vblanktime: pointer to time to receive the next vblank timestamp.
1004	*
1005	* Calculate the expected time of the start of the next vblank period,
1006	* based on time of previous vblank and frame duration
1007	*/
1008	int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
1009	{
1010	struct drm_vblank_crtc *vblank;
1011	struct drm_display_mode *mode;
1012	u64 vblank_start;
1013
1014	if (!drm_dev_has_vblank(crtc->dev))
1015	return -EINVAL;
1016
1017	vblank = drm_crtc_vblank_crtc(crtc);
1018	mode = &vblank->hwmode;
1019
1020	if (!vblank->framedur_ns || !vblank->linedur_ns)
1021	return -EINVAL;
1022
1023	if (!drm_crtc_get_last_vbltimestamp(crtc, tvblank: vblanktime, in_vblank_irq: false))
1024	return -EINVAL;
1025
1026	vblank_start = DIV_ROUND_DOWN_ULL(
1027	(u64)vblank->framedur_ns * mode->crtc_vblank_start,
1028	mode->crtc_vtotal);
1029	*vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
1030
1031	return 0;
1032	}
1033	EXPORT_SYMBOL(drm_crtc_next_vblank_start);
1034
1035	static void send_vblank_event(struct drm_device *dev,
1036	struct drm_pending_vblank_event *e,
1037	u64 seq, ktime_t now)
1038	{
1039	struct timespec64 tv;
1040
1041	switch (e->event.base.type) {
1042	case DRM_EVENT_VBLANK:
1043	case DRM_EVENT_FLIP_COMPLETE:
1044	tv = ktime_to_timespec64(now);
1045	e->event.vbl.sequence = seq;
1046	/*
1047	* e->event is a user space structure, with hardcoded unsigned
1048	* 32-bit seconds/microseconds. This is safe as we always use
1049	* monotonic timestamps since linux-4.15
1050	*/
1051	e->event.vbl.tv_sec = tv.tv_sec;
1052	e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1053	break;
1054	case DRM_EVENT_CRTC_SEQUENCE:
1055	if (seq)
1056	e->event.seq.sequence = seq;
1057	e->event.seq.time_ns = ktime_to_ns(kt: now);
1058	break;
1059	}
1060	trace_drm_vblank_event_delivered(file: e->base.file_priv, crtc: e->pipe, seq);
1061	/*
1062	* Use the same timestamp for any associated fence signal to avoid
1063	* mismatch in timestamps for vsync & fence events triggered by the
1064	* same HW event. Frameworks like SurfaceFlinger in Android expects the
1065	* retire-fence timestamp to match exactly with HW vsync as it uses it
1066	* for its software vsync modeling.
1067	*/
1068	drm_send_event_timestamp_locked(dev, e: &e->base, timestamp: now);
1069	}
1070
1071	/**
1072	* drm_crtc_arm_vblank_event - arm vblank event after pageflip
1073	* @crtc: the source CRTC of the vblank event
1074	* @e: the event to send
1075	*
1076	* A lot of drivers need to generate vblank events for the very next vblank
1077	* interrupt. For example when the page flip interrupt happens when the page
1078	* flip gets armed, but not when it actually executes within the next vblank
1079	* period. This helper function implements exactly the required vblank arming
1080	* behaviour.
1081	*
1082	* NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1083	* atomic commit must ensure that the next vblank happens at exactly the same
1084	* time as the atomic commit is committed to the hardware. This function itself
1085	* does **not** protect against the next vblank interrupt racing with either this
1086	* function call or the atomic commit operation. A possible sequence could be:
1087	*
1088	* 1. Driver commits new hardware state into vblank-synchronized registers.
1089	* 2. A vblank happens, committing the hardware state. Also the corresponding
1090	* vblank interrupt is fired off and fully processed by the interrupt
1091	* handler.
1092	* 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1093	* 4. The event is only send out for the next vblank, which is wrong.
1094	*
1095	* An equivalent race can happen when the driver calls
1096	* drm_crtc_arm_vblank_event() before writing out the new hardware state.
1097	*
1098	* The only way to make this work safely is to prevent the vblank from firing
1099	* (and the hardware from committing anything else) until the entire atomic
1100	* commit sequence has run to completion. If the hardware does not have such a
1101	* feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1102	* Instead drivers need to manually send out the event from their interrupt
1103	* handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1104	* possible race with the hardware committing the atomic update.
1105	*
1106	* Caller must hold a vblank reference for the event @e acquired by a
1107	* drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1108	*/
1109	void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1110	struct drm_pending_vblank_event *e)
1111	{
1112	struct drm_device *dev = crtc->dev;
1113	unsigned int pipe = drm_crtc_index(crtc);
1114
1115	assert_spin_locked(&dev->event_lock);
1116
1117	e->pipe = pipe;
1118	e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1119	list_add_tail(new: &e->base.link, head: &dev->vblank_event_list);
1120	}
1121	EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1122
1123	/**
1124	* drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1125	* @crtc: the source CRTC of the vblank event
1126	* @e: the event to send
1127	*
1128	* Updates sequence # and timestamp on event for the most recently processed
1129	* vblank, and sends it to userspace. Caller must hold event lock.
1130	*
1131	* See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1132	* situation, especially to send out events for atomic commit operations.
1133	*/
1134	void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1135	struct drm_pending_vblank_event *e)
1136	{
1137	struct drm_device *dev = crtc->dev;
1138	u64 seq;
1139	unsigned int pipe = drm_crtc_index(crtc);
1140	ktime_t now;
1141
1142	if (drm_dev_has_vblank(dev)) {
1143	seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1144	} else {
1145	seq = 0;
1146
1147	now = ktime_get();
1148	}
1149	e->pipe = pipe;
1150	send_vblank_event(dev, e, seq, now);
1151	}
1152	EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1153
1154	static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1155	{
1156	if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
1157	struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
1158
1159	if (drm_WARN_ON(dev, !crtc))
1160	return 0;
1161
1162	if (crtc->funcs->enable_vblank)
1163	return crtc->funcs->enable_vblank(crtc);
1164	}
1165
1166	return -EINVAL;
1167	}
1168
1169	static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1170	{
1171	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1172	int ret = 0;
1173
1174	assert_spin_locked(&dev->vbl_lock);
1175
1176	spin_lock(lock: &dev->vblank_time_lock);
1177
1178	if (!vblank->enabled) {
1179	/*
1180	* Enable vblank irqs under vblank_time_lock protection.
1181	* All vblank count & timestamp updates are held off
1182	* until we are done reinitializing master counter and
1183	* timestamps. Filtercode in drm_handle_vblank() will
1184	* prevent double-accounting of same vblank interval.
1185	*/
1186	ret = __enable_vblank(dev, pipe);
1187	drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1188	pipe, ret);
1189	if (ret) {
1190	atomic_dec(v: &vblank->refcount);
1191	} else {
1192	drm_update_vblank_count(dev, pipe, in_vblank_irq: 0);
1193	/* drm_update_vblank_count() includes a wmb so we just
1194	* need to ensure that the compiler emits the write
1195	* to mark the vblank as enabled after the call
1196	* to drm_update_vblank_count().
1197	*/
1198	WRITE_ONCE(vblank->enabled, true);
1199	}
1200	}
1201
1202	spin_unlock(lock: &dev->vblank_time_lock);
1203
1204	return ret;
1205	}
1206
1207	int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1208	{
1209	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1210	unsigned long irqflags;
1211	int ret = 0;
1212
1213	if (!drm_dev_has_vblank(dev))
1214	return -EINVAL;
1215
1216	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1217	return -EINVAL;
1218
1219	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1220	/* Going from 0->1 means we have to enable interrupts again */
1221	if (atomic_add_return(i: 1, v: &vblank->refcount) == 1) {
1222	ret = drm_vblank_enable(dev, pipe);
1223	} else {
1224	if (!vblank->enabled) {
1225	atomic_dec(v: &vblank->refcount);
1226	ret = -EINVAL;
1227	}
1228	}
1229	spin_unlock_irqrestore(lock: &dev->vbl_lock, flags: irqflags);
1230
1231	return ret;
1232	}
1233
1234	/**
1235	* drm_crtc_vblank_get - get a reference count on vblank events
1236	* @crtc: which CRTC to own
1237	*
1238	* Acquire a reference count on vblank events to avoid having them disabled
1239	* while in use.
1240	*
1241	* Returns:
1242	* Zero on success or a negative error code on failure.
1243	*/
1244	int drm_crtc_vblank_get(struct drm_crtc *crtc)
1245	{
1246	return drm_vblank_get(dev: crtc->dev, pipe: drm_crtc_index(crtc));
1247	}
1248	EXPORT_SYMBOL(drm_crtc_vblank_get);
1249
1250	void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1251	{
1252	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1253	int vblank_offdelay = vblank->config.offdelay_ms;
1254
1255	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1256	return;
1257
1258	if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1259	return;
1260
1261	/* Last user schedules interrupt disable */
1262	if (atomic_dec_and_test(v: &vblank->refcount)) {
1263	if (!vblank_offdelay)
1264	return;
1265	else if (vblank_offdelay < 0)
1266	vblank_disable_fn(t: &vblank->disable_timer);
1267	else if (!vblank->config.disable_immediate)
1268	mod_timer(timer: &vblank->disable_timer,
1269	expires: jiffies + ((vblank_offdelay * HZ) / 1000));
1270	}
1271	}
1272
1273	/**
1274	* drm_crtc_vblank_put - give up ownership of vblank events
1275	* @crtc: which counter to give up
1276	*
1277	* Release ownership of a given vblank counter, turning off interrupts
1278	* if possible. Disable interrupts after &drm_vblank_crtc_config.offdelay_ms
1279	* milliseconds.
1280	*/
1281	void drm_crtc_vblank_put(struct drm_crtc *crtc)
1282	{
1283	drm_vblank_put(dev: crtc->dev, pipe: drm_crtc_index(crtc));
1284	}
1285	EXPORT_SYMBOL(drm_crtc_vblank_put);
1286
1287	/**
1288	* drm_wait_one_vblank - wait for one vblank
1289	* @dev: DRM device
1290	* @pipe: CRTC index
1291	*
1292	* This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1293	* It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1294	* due to lack of driver support or because the crtc is off.
1295	*
1296	* This is the legacy version of drm_crtc_wait_one_vblank().
1297	*/
1298	void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1299	{
1300	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1301	int ret;
1302	u64 last;
1303
1304	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1305	return;
1306
1307	ret = drm_vblank_get(dev, pipe);
1308	if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1309	pipe, ret))
1310	return;
1311
1312	last = drm_vblank_count(dev, pipe);
1313
1314	ret = wait_event_timeout(vblank->queue,
1315	last != drm_vblank_count(dev, pipe),
1316	msecs_to_jiffies(1000));
1317
1318	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1319
1320	drm_vblank_put(dev, pipe);
1321	}
1322	EXPORT_SYMBOL(drm_wait_one_vblank);
1323
1324	/**
1325	* drm_crtc_wait_one_vblank - wait for one vblank
1326	* @crtc: DRM crtc
1327	*
1328	* This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1329	* It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1330	* due to lack of driver support or because the crtc is off.
1331	*/
1332	void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1333	{
1334	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1335	}
1336	EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1337
1338	/**
1339	* drm_crtc_vblank_off - disable vblank events on a CRTC
1340	* @crtc: CRTC in question
1341	*
1342	* Drivers can use this function to shut down the vblank interrupt handling when
1343	* disabling a crtc. This function ensures that the latest vblank frame count is
1344	* stored so that drm_vblank_on can restore it again.
1345	*
1346	* Drivers must use this function when the hardware vblank counter can get
1347	* reset, e.g. when suspending or disabling the @crtc in general.
1348	*/
1349	void drm_crtc_vblank_off(struct drm_crtc *crtc)
1350	{
1351	struct drm_device *dev = crtc->dev;
1352	unsigned int pipe = drm_crtc_index(crtc);
1353	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
1354	struct drm_pending_vblank_event *e, *t;
1355	ktime_t now;
1356	u64 seq;
1357
1358	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1359	return;
1360
1361	/*
1362	* Grab event_lock early to prevent vblank work from being scheduled
1363	* while we're in the middle of shutting down vblank interrupts
1364	*/
1365	spin_lock_irq(lock: &dev->event_lock);
1366
1367	spin_lock(lock: &dev->vbl_lock);
1368	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1369	pipe, vblank->enabled, vblank->inmodeset);
1370
1371	/* Avoid redundant vblank disables without previous
1372	* drm_crtc_vblank_on(). */
1373	if (drm_core_check_feature(dev, feature: DRIVER_ATOMIC) || !vblank->inmodeset)
1374	drm_vblank_disable_and_save(dev, pipe);
1375
1376	wake_up(&vblank->queue);
1377
1378	/*
1379	* Prevent subsequent drm_vblank_get() from re-enabling
1380	* the vblank interrupt by bumping the refcount.
1381	*/
1382	if (!vblank->inmodeset) {
1383	atomic_inc(v: &vblank->refcount);
1384	vblank->inmodeset = 1;
1385	}
1386	spin_unlock(lock: &dev->vbl_lock);
1387
1388	/* Send any queued vblank events, lest the natives grow disquiet */
1389	seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1390
1391	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1392	if (e->pipe != pipe)
1393	continue;
1394	drm_dbg_core(dev, "Sending premature vblank event on disable: "
1395	"wanted %llu, current %llu\n",
1396	e->sequence, seq);
1397	list_del(entry: &e->base.link);
1398	drm_vblank_put(dev, pipe);
1399	send_vblank_event(dev, e, seq, now);
1400	}
1401
1402	/* Cancel any leftover pending vblank work */
1403	drm_vblank_cancel_pending_works(vblank);
1404
1405	spin_unlock_irq(lock: &dev->event_lock);
1406
1407	/* Will be reset by the modeset helpers when re-enabling the crtc by
1408	* calling drm_calc_timestamping_constants(). */
1409	vblank->hwmode.crtc_clock = 0;
1410
1411	/* Wait for any vblank work that's still executing to finish */
1412	drm_vblank_flush_worker(vblank);
1413	}
1414	EXPORT_SYMBOL(drm_crtc_vblank_off);
1415
1416	/**
1417	* drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1418	* @crtc: CRTC in question
1419	*
1420	* Drivers can use this function to reset the vblank state to off at load time.
1421	* Drivers should use this together with the drm_crtc_vblank_off() and
1422	* drm_crtc_vblank_on() functions. The difference compared to
1423	* drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1424	* and hence doesn't need to call any driver hooks.
1425	*
1426	* This is useful for recovering driver state e.g. on driver load, or on resume.
1427	*/
1428	void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1429	{
1430	struct drm_device *dev = crtc->dev;
1431	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
1432
1433	spin_lock_irq(lock: &dev->vbl_lock);
1434	/*
1435	* Prevent subsequent drm_vblank_get() from enabling the vblank
1436	* interrupt by bumping the refcount.
1437	*/
1438	if (!vblank->inmodeset) {
1439	atomic_inc(v: &vblank->refcount);
1440	vblank->inmodeset = 1;
1441	}
1442	spin_unlock_irq(lock: &dev->vbl_lock);
1443
1444	drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1445	drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1446	}
1447	EXPORT_SYMBOL(drm_crtc_vblank_reset);
1448
1449	/**
1450	* drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1451	* @crtc: CRTC in question
1452	* @max_vblank_count: max hardware vblank counter value
1453	*
1454	* Update the maximum hardware vblank counter value for @crtc
1455	* at runtime. Useful for hardware where the operation of the
1456	* hardware vblank counter depends on the currently active
1457	* display configuration.
1458	*
1459	* For example, if the hardware vblank counter does not work
1460	* when a specific connector is active the maximum can be set
1461	* to zero. And when that specific connector isn't active the
1462	* maximum can again be set to the appropriate non-zero value.
1463	*
1464	* If used, must be called before drm_vblank_on().
1465	*/
1466	void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1467	u32 max_vblank_count)
1468	{
1469	struct drm_device *dev = crtc->dev;
1470	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
1471
1472	drm_WARN_ON(dev, dev->max_vblank_count);
1473	drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1474
1475	vblank->max_vblank_count = max_vblank_count;
1476	}
1477	EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1478
1479	/**
1480	* drm_crtc_vblank_on_config - enable vblank events on a CRTC with custom
1481	* configuration options
1482	* @crtc: CRTC in question
1483	* @config: Vblank configuration value
1484	*
1485	* See drm_crtc_vblank_on(). In addition, this function allows you to provide a
1486	* custom vblank configuration for a given CRTC.
1487	*
1488	* Note that @config is copied, the pointer does not need to stay valid beyond
1489	* this function call. For details of the parameters see
1490	* struct drm_vblank_crtc_config.
1491	*/
1492	void drm_crtc_vblank_on_config(struct drm_crtc *crtc,
1493	const struct drm_vblank_crtc_config *config)
1494	{
1495	struct drm_device *dev = crtc->dev;
1496	unsigned int pipe = drm_crtc_index(crtc);
1497	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
1498
1499	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1500	return;
1501
1502	spin_lock_irq(lock: &dev->vbl_lock);
1503	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1504	pipe, vblank->enabled, vblank->inmodeset);
1505
1506	vblank->config = *config;
1507
1508	/* Drop our private "prevent drm_vblank_get" refcount */
1509	if (vblank->inmodeset) {
1510	atomic_dec(v: &vblank->refcount);
1511	vblank->inmodeset = 0;
1512	}
1513
1514	drm_reset_vblank_timestamp(dev, pipe);
1515
1516	/*
1517	* re-enable interrupts if there are users left, or the
1518	* user wishes vblank interrupts to be enabled all the time.
1519	*/
1520	if (atomic_read(v: &vblank->refcount) != 0 || !vblank->config.offdelay_ms)
1521	drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1522	spin_unlock_irq(lock: &dev->vbl_lock);
1523	}
1524	EXPORT_SYMBOL(drm_crtc_vblank_on_config);
1525
1526	/**
1527	* drm_crtc_vblank_on - enable vblank events on a CRTC
1528	* @crtc: CRTC in question
1529	*
1530	* This functions restores the vblank interrupt state captured with
1531	* drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1532	* that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1533	* unbalanced and so can also be unconditionally called in driver load code to
1534	* reflect the current hardware state of the crtc.
1535	*
1536	* Note that unlike in drm_crtc_vblank_on_config(), default values are used.
1537	*/
1538	void drm_crtc_vblank_on(struct drm_crtc *crtc)
1539	{
1540	const struct drm_vblank_crtc_config config = {
1541	.offdelay_ms = drm_vblank_offdelay,
1542	.disable_immediate = crtc->dev->vblank_disable_immediate
1543	};
1544
1545	drm_crtc_vblank_on_config(crtc, &config);
1546	}
1547	EXPORT_SYMBOL(drm_crtc_vblank_on);
1548
1549	static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1550	{
1551	ktime_t t_vblank;
1552	struct drm_vblank_crtc *vblank;
1553	int framedur_ns;
1554	u64 diff_ns;
1555	u32 cur_vblank, diff = 1;
1556	int count = DRM_TIMESTAMP_MAXRETRIES;
1557	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1558
1559	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1560	return;
1561
1562	assert_spin_locked(&dev->vbl_lock);
1563	assert_spin_locked(&dev->vblank_time_lock);
1564
1565	vblank = drm_vblank_crtc(dev, pipe);
1566	drm_WARN_ONCE(dev,
1567	drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1568	"Cannot compute missed vblanks without frame duration\n");
1569	framedur_ns = vblank->framedur_ns;
1570
1571	do {
1572	cur_vblank = __get_vblank_counter(dev, pipe);
1573	drm_get_last_vbltimestamp(dev, pipe, tvblank: &t_vblank, in_vblank_irq: false);
1574	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1575
1576	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1577	if (framedur_ns)
1578	diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1579
1580
1581	drm_dbg_vbl(dev,
1582	"missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1583	diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1584	vblank->last = (cur_vblank - diff) & max_vblank_count;
1585	}
1586
1587	/**
1588	* drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1589	* @crtc: CRTC in question
1590	*
1591	* Power manamement features can cause frame counter resets between vblank
1592	* disable and enable. Drivers can use this function in their
1593	* &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1594	* the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1595	* vblank counter.
1596	*
1597	* Note that drivers must have race-free high-precision timestamping support,
1598	* i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1599	* &drm_vblank_crtc_config.disable_immediate must be set to indicate the
1600	* time-stamping functions are race-free against vblank hardware counter
1601	* increments.
1602	*/
1603	void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1604	{
1605	struct drm_device *dev = crtc->dev;
1606	unsigned int pipe = drm_crtc_index(crtc);
1607	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1608
1609	drm_WARN_ON_ONCE(dev, !crtc->funcs->get_vblank_timestamp);
1610	drm_WARN_ON_ONCE(dev, vblank->inmodeset);
1611	drm_WARN_ON_ONCE(dev, !vblank->config.disable_immediate);
1612
1613	drm_vblank_restore(dev, pipe);
1614	}
1615	EXPORT_SYMBOL(drm_crtc_vblank_restore);
1616
1617	static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1618	u64 req_seq,
1619	union drm_wait_vblank *vblwait,
1620	struct drm_file *file_priv)
1621	{
1622	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1623	struct drm_pending_vblank_event *e;
1624	ktime_t now;
1625	u64 seq;
1626	int ret;
1627
1628	e = kzalloc(sizeof(*e), GFP_KERNEL);
1629	if (e == NULL) {
1630	ret = -ENOMEM;
1631	goto err_put;
1632	}
1633
1634	e->pipe = pipe;
1635	e->event.base.type = DRM_EVENT_VBLANK;
1636	e->event.base.length = sizeof(e->event.vbl);
1637	e->event.vbl.user_data = vblwait->request.signal;
1638	e->event.vbl.crtc_id = 0;
1639	if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
1640	struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
1641
1642	if (crtc)
1643	e->event.vbl.crtc_id = crtc->base.id;
1644	}
1645
1646	spin_lock_irq(lock: &dev->event_lock);
1647
1648	/*
1649	* drm_crtc_vblank_off() might have been called after we called
1650	* drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1651	* vblank disable, so no need for further locking. The reference from
1652	* drm_vblank_get() protects against vblank disable from another source.
1653	*/
1654	if (!READ_ONCE(vblank->enabled)) {
1655	ret = -EINVAL;
1656	goto err_unlock;
1657	}
1658
1659	ret = drm_event_reserve_init_locked(dev, file_priv, p: &e->base,
1660	e: &e->event.base);
1661
1662	if (ret)
1663	goto err_unlock;
1664
1665	seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1666
1667	drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1668	req_seq, seq, pipe);
1669
1670	trace_drm_vblank_event_queued(file: file_priv, crtc: pipe, seq: req_seq);
1671
1672	e->sequence = req_seq;
1673	if (drm_vblank_passed(seq, ref: req_seq)) {
1674	drm_vblank_put(dev, pipe);
1675	send_vblank_event(dev, e, seq, now);
1676	vblwait->reply.sequence = seq;
1677	} else {
1678	/* drm_handle_vblank_events will call drm_vblank_put */
1679	list_add_tail(new: &e->base.link, head: &dev->vblank_event_list);
1680	vblwait->reply.sequence = req_seq;
1681	}
1682
1683	spin_unlock_irq(lock: &dev->event_lock);
1684
1685	return 0;
1686
1687	err_unlock:
1688	spin_unlock_irq(lock: &dev->event_lock);
1689	kfree(objp: e);
1690	err_put:
1691	drm_vblank_put(dev, pipe);
1692	return ret;
1693	}
1694
1695	static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1696	{
1697	if (vblwait->request.sequence)
1698	return false;
1699
1700	return _DRM_VBLANK_RELATIVE ==
1701	(vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1702	_DRM_VBLANK_EVENT |
1703	_DRM_VBLANK_NEXTONMISS));
1704	}
1705
1706	/*
1707	* Widen a 32-bit param to 64-bits.
1708	*
1709	* \param narrow 32-bit value (missing upper 32 bits)
1710	* \param near 64-bit value that should be 'close' to near
1711	*
1712	* This function returns a 64-bit value using the lower 32-bits from
1713	* 'narrow' and constructing the upper 32-bits so that the result is
1714	* as close as possible to 'near'.
1715	*/
1716
1717	static u64 widen_32_to_64(u32 narrow, u64 near)
1718	{
1719	return near + (s32) (narrow - near);
1720	}
1721
1722	static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1723	struct drm_wait_vblank_reply *reply)
1724	{
1725	ktime_t now;
1726	struct timespec64 ts;
1727
1728	/*
1729	* drm_wait_vblank_reply is a UAPI structure that uses 'long'
1730	* to store the seconds. This is safe as we always use monotonic
1731	* timestamps since linux-4.15.
1732	*/
1733	reply->sequence = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1734	ts = ktime_to_timespec64(now);
1735	reply->tval_sec = (u32)ts.tv_sec;
1736	reply->tval_usec = ts.tv_nsec / 1000;
1737	}
1738
1739	static bool drm_wait_vblank_supported(struct drm_device *dev)
1740	{
1741	return drm_dev_has_vblank(dev);
1742	}
1743
1744	int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1745	struct drm_file *file_priv)
1746	{
1747	struct drm_crtc *crtc;
1748	struct drm_vblank_crtc *vblank;
1749	union drm_wait_vblank *vblwait = data;
1750	int ret;
1751	u64 req_seq, seq;
1752	unsigned int pipe_index;
1753	unsigned int flags, pipe, high_pipe;
1754
1755	if (!drm_wait_vblank_supported(dev))
1756	return -EOPNOTSUPP;
1757
1758	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1759	return -EINVAL;
1760
1761	if (vblwait->request.type &
1762	~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1763	_DRM_VBLANK_HIGH_CRTC_MASK)) {
1764	drm_dbg_core(dev,
1765	"Unsupported type value 0x%x, supported mask 0x%x\n",
1766	vblwait->request.type,
1767	(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1768	_DRM_VBLANK_HIGH_CRTC_MASK));
1769	return -EINVAL;
1770	}
1771
1772	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1773	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1774	if (high_pipe)
1775	pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1776	else
1777	pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1778
1779	/* Convert lease-relative crtc index into global crtc index */
1780	if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
1781	pipe = 0;
1782	drm_for_each_crtc(crtc, dev) {
1783	if (drm_lease_held(file_priv, id: crtc->base.id)) {
1784	if (pipe_index == 0)
1785	break;
1786	pipe_index--;
1787	}
1788	pipe++;
1789	}
1790	} else {
1791	pipe = pipe_index;
1792	}
1793
1794	if (pipe >= dev->num_crtcs)
1795	return -EINVAL;
1796
1797	vblank = &dev->vblank[pipe];
1798
1799	/* If the counter is currently enabled and accurate, short-circuit
1800	* queries to return the cached timestamp of the last vblank.
1801	*/
1802	if (vblank->config.disable_immediate &&
1803	drm_wait_vblank_is_query(vblwait) &&
1804	READ_ONCE(vblank->enabled)) {
1805	drm_wait_vblank_reply(dev, pipe, reply: &vblwait->reply);
1806	return 0;
1807	}
1808
1809	ret = drm_vblank_get(dev, pipe);
1810	if (ret) {
1811	drm_dbg_core(dev,
1812	"crtc %d failed to acquire vblank counter, %d\n",
1813	pipe, ret);
1814	return ret;
1815	}
1816	seq = drm_vblank_count(dev, pipe);
1817
1818	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1819	case _DRM_VBLANK_RELATIVE:
1820	req_seq = seq + vblwait->request.sequence;
1821	vblwait->request.sequence = req_seq;
1822	vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1823	break;
1824	case _DRM_VBLANK_ABSOLUTE:
1825	req_seq = widen_32_to_64(narrow: vblwait->request.sequence, near: seq);
1826	break;
1827	default:
1828	ret = -EINVAL;
1829	goto done;
1830	}
1831
1832	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1833	drm_vblank_passed(seq, ref: req_seq)) {
1834	req_seq = seq + 1;
1835	vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1836	vblwait->request.sequence = req_seq;
1837	}
1838
1839	if (flags & _DRM_VBLANK_EVENT) {
1840	/* must hold on to the vblank ref until the event fires
1841	* drm_vblank_put will be called asynchronously
1842	*/
1843	return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1844	}
1845
1846	if (req_seq != seq) {
1847	int wait;
1848
1849	drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1850	req_seq, pipe);
1851	wait = wait_event_interruptible_timeout(vblank->queue,
1852	drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1853	!READ_ONCE(vblank->enabled),
1854	msecs_to_jiffies(3000));
1855
1856	switch (wait) {
1857	case 0:
1858	/* timeout */
1859	ret = -EBUSY;
1860	break;
1861	case -ERESTARTSYS:
1862	/* interrupted by signal */
1863	ret = -EINTR;
1864	break;
1865	default:
1866	ret = 0;
1867	break;
1868	}
1869	}
1870
1871	if (ret != -EINTR) {
1872	drm_wait_vblank_reply(dev, pipe, reply: &vblwait->reply);
1873
1874	drm_dbg_core(dev, "crtc %d returning %u to client\n",
1875	pipe, vblwait->reply.sequence);
1876	} else {
1877	drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1878	pipe);
1879	}
1880
1881	done:
1882	drm_vblank_put(dev, pipe);
1883	return ret;
1884	}
1885
1886	static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1887	{
1888	struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
1889	bool high_prec = false;
1890	struct drm_pending_vblank_event *e, *t;
1891	ktime_t now;
1892	u64 seq;
1893
1894	assert_spin_locked(&dev->event_lock);
1895
1896	seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1897
1898	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1899	if (e->pipe != pipe)
1900	continue;
1901	if (!drm_vblank_passed(seq, ref: e->sequence))
1902	continue;
1903
1904	drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1905	e->sequence, seq);
1906
1907	list_del(entry: &e->base.link);
1908	drm_vblank_put(dev, pipe);
1909	send_vblank_event(dev, e, seq, now);
1910	}
1911
1912	if (crtc && crtc->funcs->get_vblank_timestamp)
1913	high_prec = true;
1914
1915	trace_drm_vblank_event(crtc: pipe, seq, time: now, high_prec);
1916	}
1917
1918	/**
1919	* drm_handle_vblank - handle a vblank event
1920	* @dev: DRM device
1921	* @pipe: index of CRTC where this event occurred
1922	*
1923	* Drivers should call this routine in their vblank interrupt handlers to
1924	* update the vblank counter and send any signals that may be pending.
1925	*
1926	* This is the legacy version of drm_crtc_handle_vblank().
1927	*/
1928	bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1929	{
1930	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1931	unsigned long irqflags;
1932	bool disable_irq;
1933
1934	if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1935	return false;
1936
1937	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1938	return false;
1939
1940	spin_lock_irqsave(&dev->event_lock, irqflags);
1941
1942	/* Need timestamp lock to prevent concurrent execution with
1943	* vblank enable/disable, as this would cause inconsistent
1944	* or corrupted timestamps and vblank counts.
1945	*/
1946	spin_lock(lock: &dev->vblank_time_lock);
1947
1948	/* Vblank irq handling disabled. Nothing to do. */
1949	if (!vblank->enabled) {
1950	spin_unlock(lock: &dev->vblank_time_lock);
1951	spin_unlock_irqrestore(lock: &dev->event_lock, flags: irqflags);
1952	return false;
1953	}
1954
1955	drm_update_vblank_count(dev, pipe, in_vblank_irq: true);
1956
1957	spin_unlock(lock: &dev->vblank_time_lock);
1958
1959	wake_up(&vblank->queue);
1960
1961	/* With instant-off, we defer disabling the interrupt until after
1962	* we finish processing the following vblank after all events have
1963	* been signaled. The disable has to be last (after
1964	* drm_handle_vblank_events) so that the timestamp is always accurate.
1965	*/
1966	disable_irq = (vblank->config.disable_immediate &&
1967	vblank->config.offdelay_ms > 0 &&
1968	!atomic_read(v: &vblank->refcount));
1969
1970	drm_handle_vblank_events(dev, pipe);
1971	drm_handle_vblank_works(vblank);
1972
1973	spin_unlock_irqrestore(lock: &dev->event_lock, flags: irqflags);
1974
1975	if (disable_irq)
1976	vblank_disable_fn(t: &vblank->disable_timer);
1977
1978	return true;
1979	}
1980	EXPORT_SYMBOL(drm_handle_vblank);
1981
1982	/**
1983	* drm_crtc_handle_vblank - handle a vblank event
1984	* @crtc: where this event occurred
1985	*
1986	* Drivers should call this routine in their vblank interrupt handlers to
1987	* update the vblank counter and send any signals that may be pending.
1988	*
1989	* This is the native KMS version of drm_handle_vblank().
1990	*
1991	* Note that for a given vblank counter value drm_crtc_handle_vblank()
1992	* and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1993	* provide a barrier: Any writes done before calling
1994	* drm_crtc_handle_vblank() will be visible to callers of the later
1995	* functions, if the vblank count is the same or a later one.
1996	*
1997	* See also &drm_vblank_crtc.count.
1998	*
1999	* Returns:
2000	* True if the event was successfully handled, false on failure.
2001	*/
2002	bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
2003	{
2004	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
2005	}
2006	EXPORT_SYMBOL(drm_crtc_handle_vblank);
2007
2008	/*
2009	* Get crtc VBLANK count.
2010	*
2011	* \param dev DRM device
2012	* \param data user argument, pointing to a drm_crtc_get_sequence structure.
2013	* \param file_priv drm file private for the user's open file descriptor
2014	*/
2015
2016	int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2017	struct drm_file *file_priv)
2018	{
2019	struct drm_crtc *crtc;
2020	struct drm_vblank_crtc *vblank;
2021	int pipe;
2022	struct drm_crtc_get_sequence *get_seq = data;
2023	ktime_t now;
2024	bool vblank_enabled;
2025	int ret;
2026
2027	if (!drm_core_check_feature(dev, feature: DRIVER_MODESET))
2028	return -EOPNOTSUPP;
2029
2030	if (!drm_dev_has_vblank(dev))
2031	return -EOPNOTSUPP;
2032
2033	crtc = drm_crtc_find(dev, file_priv, id: get_seq->crtc_id);
2034	if (!crtc)
2035	return -ENOENT;
2036
2037	pipe = drm_crtc_index(crtc);
2038
2039	vblank = drm_crtc_vblank_crtc(crtc);
2040	vblank_enabled = READ_ONCE(vblank->config.disable_immediate) &&
2041	READ_ONCE(vblank->enabled);
2042
2043	if (!vblank_enabled) {
2044	ret = drm_crtc_vblank_get(crtc);
2045	if (ret) {
2046	drm_dbg_core(dev,
2047	"crtc %d failed to acquire vblank counter, %d\n",
2048	pipe, ret);
2049	return ret;
2050	}
2051	}
2052	drm_modeset_lock(lock: &crtc->mutex, NULL);
2053	if (crtc->state)
2054	get_seq->active = crtc->state->enable;
2055	else
2056	get_seq->active = crtc->enabled;
2057	drm_modeset_unlock(lock: &crtc->mutex);
2058	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
2059	get_seq->sequence_ns = ktime_to_ns(kt: now);
2060	if (!vblank_enabled)
2061	drm_crtc_vblank_put(crtc);
2062	return 0;
2063	}
2064
2065	/*
2066	* Queue a event for VBLANK sequence
2067	*
2068	* \param dev DRM device
2069	* \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2070	* \param file_priv drm file private for the user's open file descriptor
2071	*/
2072
2073	int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2074	struct drm_file *file_priv)
2075	{
2076	struct drm_crtc *crtc;
2077	struct drm_vblank_crtc *vblank;
2078	int pipe;
2079	struct drm_crtc_queue_sequence *queue_seq = data;
2080	ktime_t now;
2081	struct drm_pending_vblank_event *e;
2082	u32 flags;
2083	u64 seq;
2084	u64 req_seq;
2085	int ret;
2086
2087	if (!drm_core_check_feature(dev, feature: DRIVER_MODESET))
2088	return -EOPNOTSUPP;
2089
2090	if (!drm_dev_has_vblank(dev))
2091	return -EOPNOTSUPP;
2092
2093	crtc = drm_crtc_find(dev, file_priv, id: queue_seq->crtc_id);
2094	if (!crtc)
2095	return -ENOENT;
2096
2097	flags = queue_seq->flags;
2098	/* Check valid flag bits */
2099	if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2100	DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2101	return -EINVAL;
2102
2103	pipe = drm_crtc_index(crtc);
2104
2105	vblank = drm_crtc_vblank_crtc(crtc);
2106
2107	e = kzalloc(sizeof(*e), GFP_KERNEL);
2108	if (e == NULL)
2109	return -ENOMEM;
2110
2111	ret = drm_crtc_vblank_get(crtc);
2112	if (ret) {
2113	drm_dbg_core(dev,
2114	"crtc %d failed to acquire vblank counter, %d\n",
2115	pipe, ret);
2116	goto err_free;
2117	}
2118
2119	seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
2120	req_seq = queue_seq->sequence;
2121
2122	if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2123	req_seq += seq;
2124
2125	if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, ref: req_seq))
2126	req_seq = seq + 1;
2127
2128	e->pipe = pipe;
2129	e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2130	e->event.base.length = sizeof(e->event.seq);
2131	e->event.seq.user_data = queue_seq->user_data;
2132
2133	spin_lock_irq(lock: &dev->event_lock);
2134
2135	/*
2136	* drm_crtc_vblank_off() might have been called after we called
2137	* drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2138	* vblank disable, so no need for further locking. The reference from
2139	* drm_crtc_vblank_get() protects against vblank disable from another source.
2140	*/
2141	if (!READ_ONCE(vblank->enabled)) {
2142	ret = -EINVAL;
2143	goto err_unlock;
2144	}
2145
2146	ret = drm_event_reserve_init_locked(dev, file_priv, p: &e->base,
2147	e: &e->event.base);
2148
2149	if (ret)
2150	goto err_unlock;
2151
2152	e->sequence = req_seq;
2153
2154	if (drm_vblank_passed(seq, ref: req_seq)) {
2155	drm_crtc_vblank_put(crtc);
2156	send_vblank_event(dev, e, seq, now);
2157	queue_seq->sequence = seq;
2158	} else {
2159	/* drm_handle_vblank_events will call drm_vblank_put */
2160	list_add_tail(new: &e->base.link, head: &dev->vblank_event_list);
2161	queue_seq->sequence = req_seq;
2162	}
2163
2164	spin_unlock_irq(lock: &dev->event_lock);
2165	return 0;
2166
2167	err_unlock:
2168	spin_unlock_irq(lock: &dev->event_lock);
2169	drm_crtc_vblank_put(crtc);
2170	err_free:
2171	kfree(objp: e);
2172	return ret;
2173	}
2174
2175	/*
2176	* VBLANK timer
2177	*/
2178
2179	static enum hrtimer_restart drm_vblank_timer_function(struct hrtimer *timer)
2180	{
2181	struct drm_vblank_crtc_timer *vtimer =
2182	container_of(timer, struct drm_vblank_crtc_timer, timer);
2183	struct drm_crtc *crtc = vtimer->crtc;
2184	const struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
2185	struct drm_device *dev = crtc->dev;
2186	unsigned long flags;
2187	ktime_t interval;
2188	u64 ret_overrun;
2189	bool succ;
2190
2191	spin_lock_irqsave(&vtimer->interval_lock, flags);
2192	interval = vtimer->interval;
2193	spin_unlock_irqrestore(lock: &vtimer->interval_lock, flags);
2194
2195	if (!interval)
2196	return HRTIMER_NORESTART;
2197
2198	ret_overrun = hrtimer_forward_now(timer: &vtimer->timer, interval);
2199	if (ret_overrun != 1)
2200	drm_dbg_vbl(dev, "vblank timer overrun\n");
2201
2202	if (crtc_funcs->handle_vblank_timeout)
2203	succ = crtc_funcs->handle_vblank_timeout(crtc);
2204	else
2205	succ = drm_crtc_handle_vblank(crtc);
2206	if (!succ)
2207	return HRTIMER_NORESTART;
2208
2209	return HRTIMER_RESTART;
2210	}
2211
2212	/**
2213	* drm_crtc_vblank_start_timer - Starts the vblank timer on the given CRTC
2214	* @crtc: the CRTC
2215	*
2216	* Drivers should call this function from their CRTC's enable_vblank
2217	* function to start a vblank timer. The timer will fire after the duration
2218	* of a full frame. drm_crtc_vblank_cancel_timer() disables a running timer.
2219	*
2220	* Returns:
2221	* 0 on success, or a negative errno code otherwise.
2222	*/
2223	int drm_crtc_vblank_start_timer(struct drm_crtc *crtc)
2224	{
2225	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
2226	struct drm_vblank_crtc_timer *vtimer = &vblank->vblank_timer;
2227	unsigned long flags;
2228
2229	if (!vtimer->crtc) {
2230	/*
2231	* Set up the data structures on the first invocation.
2232	*/
2233	vtimer->crtc = crtc;
2234	spin_lock_init(&vtimer->interval_lock);
2235	hrtimer_setup(timer: &vtimer->timer, function: drm_vblank_timer_function,
2236	CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
2237	} else {
2238	/*
2239	* Timer should not be active. If it is, wait for the
2240	* previous cancel operations to finish.
2241	*/
2242	while (hrtimer_active(timer: &vtimer->timer))
2243	hrtimer_try_to_cancel(timer: &vtimer->timer);
2244	}
2245
2246	drm_calc_timestamping_constants(crtc, &crtc->mode);
2247
2248	spin_lock_irqsave(&vtimer->interval_lock, flags);
2249	vtimer->interval = ns_to_ktime(ns: vblank->framedur_ns);
2250	spin_unlock_irqrestore(lock: &vtimer->interval_lock, flags);
2251
2252	hrtimer_start(timer: &vtimer->timer, tim: vtimer->interval, mode: HRTIMER_MODE_REL);
2253
2254	return 0;
2255	}
2256	EXPORT_SYMBOL(drm_crtc_vblank_start_timer);
2257
2258	/**
2259	* drm_crtc_vblank_cancel_timer - Cancels the given CRTC's vblank timer
2260	* @crtc: the CRTC
2261	*
2262	* Drivers should call this function from their CRTC's disable_vblank
2263	* function to stop a vblank timer.
2264	*/
2265	void drm_crtc_vblank_cancel_timer(struct drm_crtc *crtc)
2266	{
2267	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
2268	struct drm_vblank_crtc_timer *vtimer = &vblank->vblank_timer;
2269	unsigned long flags;
2270
2271	/*
2272	* Calling hrtimer_cancel() can result in a deadlock with DRM's
2273	* vblank_time_lime_lock and hrtimers' softirq_expiry_lock. So
2274	* clear interval and indicate cancellation. The timer function
2275	* will cancel itself on the next invocation.
2276	*/
2277
2278	spin_lock_irqsave(&vtimer->interval_lock, flags);
2279	vtimer->interval = 0;
2280	spin_unlock_irqrestore(lock: &vtimer->interval_lock, flags);
2281
2282	hrtimer_try_to_cancel(timer: &vtimer->timer);
2283	}
2284	EXPORT_SYMBOL(drm_crtc_vblank_cancel_timer);
2285
2286	/**
2287	* drm_crtc_vblank_get_vblank_timeout - Returns the vblank timeout
2288	* @crtc: The CRTC
2289	* @vblank_time: Returns the next vblank timestamp
2290	*
2291	* The helper drm_crtc_vblank_get_vblank_timeout() returns the next vblank
2292	* timestamp of the CRTC's vblank timer according to the timer's expiry
2293	* time.
2294	*/
2295	void drm_crtc_vblank_get_vblank_timeout(struct drm_crtc *crtc, ktime_t *vblank_time)
2296	{
2297	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
2298	struct drm_vblank_crtc_timer *vtimer = &vblank->vblank_timer;
2299	u64 cur_count;
2300	ktime_t cur_time;
2301
2302	if (!READ_ONCE(vblank->enabled)) {
2303	*vblank_time = ktime_get();
2304	return;
2305	}
2306
2307	/*
2308	* A concurrent vblank timeout could update the expires field before
2309	* we compare it with the vblank time. Hence we'd compare the old
2310	* expiry time to the new vblank time; deducing the timer had already
2311	* expired. Reread until we get consistent values from both fields.
2312	*/
2313	do {
2314	cur_count = drm_crtc_vblank_count_and_time(crtc, &cur_time);
2315	*vblank_time = READ_ONCE(vtimer->timer.node.expires);
2316	} while (cur_count != drm_crtc_vblank_count_and_time(crtc, &cur_time));
2317
2318	if (drm_WARN_ON(crtc->dev, !ktime_compare(*vblank_time, cur_time)))
2319	return; /* Already expired */
2320
2321	/*
2322	* To prevent races we roll the hrtimer forward before we do any
2323	* interrupt processing - this is how real hw works (the interrupt
2324	* is only generated after all the vblank registers are updated)
2325	* and what the vblank core expects. Therefore we need to always
2326	* correct the timestamp by one frame.
2327	*/
2328	*vblank_time = ktime_sub(*vblank_time, vtimer->interval);
2329	}
2330	EXPORT_SYMBOL(drm_crtc_vblank_get_vblank_timeout);
2331