1	/*
2	* Copyright 2007-8 Advanced Micro Devices, Inc.
3	* Copyright 2008 Red Hat Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*
23	* Authors: Dave Airlie
24	* Alex Deucher
25	*/
26
27	#include <linux/pci.h>
28	#include <linux/pm_runtime.h>
29	#include <linux/gcd.h>
30
31	#include <asm/div64.h>
32
33	#include <drm/drm_crtc_helper.h>
34	#include <drm/drm_device.h>
35	#include <drm/drm_drv.h>
36	#include <drm/drm_edid.h>
37	#include <drm/drm_fourcc.h>
38	#include <drm/drm_framebuffer.h>
39	#include <drm/drm_gem_framebuffer_helper.h>
40	#include <drm/drm_modeset_helper.h>
41	#include <drm/drm_probe_helper.h>
42	#include <drm/drm_vblank.h>
43	#include <drm/radeon_drm.h>
44
45	#include "atom.h"
46	#include "radeon.h"
47	#include "radeon_kms.h"
48
49	static void avivo_crtc_load_lut(struct drm_crtc *crtc)
50	{
51	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
52	struct drm_device *dev = crtc->dev;
53	struct radeon_device *rdev = dev->dev_private;
54	u16 *r, *g, *b;
55	int i;
56
57	DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
58	WREG32(AVIVO_DC_LUTA_CONTROL + radeon_crtc->crtc_offset, 0);
59
60	WREG32(AVIVO_DC_LUTA_BLACK_OFFSET_BLUE + radeon_crtc->crtc_offset, 0);
61	WREG32(AVIVO_DC_LUTA_BLACK_OFFSET_GREEN + radeon_crtc->crtc_offset, 0);
62	WREG32(AVIVO_DC_LUTA_BLACK_OFFSET_RED + radeon_crtc->crtc_offset, 0);
63
64	WREG32(AVIVO_DC_LUTA_WHITE_OFFSET_BLUE + radeon_crtc->crtc_offset, 0xffff);
65	WREG32(AVIVO_DC_LUTA_WHITE_OFFSET_GREEN + radeon_crtc->crtc_offset, 0xffff);
66	WREG32(AVIVO_DC_LUTA_WHITE_OFFSET_RED + radeon_crtc->crtc_offset, 0xffff);
67
68	WREG32(AVIVO_DC_LUT_RW_SELECT, radeon_crtc->crtc_id);
69	WREG32(AVIVO_DC_LUT_RW_MODE, 0);
70	WREG32(AVIVO_DC_LUT_WRITE_EN_MASK, 0x0000003f);
71
72	WREG8(AVIVO_DC_LUT_RW_INDEX, 0);
73	r = crtc->gamma_store;
74	g = r + crtc->gamma_size;
75	b = g + crtc->gamma_size;
76	for (i = 0; i < 256; i++) {
77	WREG32(AVIVO_DC_LUT_30_COLOR,
78	((*r++ & 0xffc0) << 14) |
79	((*g++ & 0xffc0) << 4) |
80	(*b++ >> 6));
81	}
82
83	/* Only change bit 0 of LUT_SEL, other bits are set elsewhere */
84	WREG32_P(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id, ~1);
85	}
86
87	static void dce4_crtc_load_lut(struct drm_crtc *crtc)
88	{
89	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
90	struct drm_device *dev = crtc->dev;
91	struct radeon_device *rdev = dev->dev_private;
92	u16 *r, *g, *b;
93	int i;
94
95	DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
96	WREG32(EVERGREEN_DC_LUT_CONTROL + radeon_crtc->crtc_offset, 0);
97
98	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + radeon_crtc->crtc_offset, 0);
99	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + radeon_crtc->crtc_offset, 0);
100	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + radeon_crtc->crtc_offset, 0);
101
102	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + radeon_crtc->crtc_offset, 0xffff);
103	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + radeon_crtc->crtc_offset, 0xffff);
104	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + radeon_crtc->crtc_offset, 0xffff);
105
106	WREG32(EVERGREEN_DC_LUT_RW_MODE + radeon_crtc->crtc_offset, 0);
107	WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
108
109	WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
110	r = crtc->gamma_store;
111	g = r + crtc->gamma_size;
112	b = g + crtc->gamma_size;
113	for (i = 0; i < 256; i++) {
114	WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
115	((*r++ & 0xffc0) << 14) |
116	((*g++ & 0xffc0) << 4) |
117	(*b++ >> 6));
118	}
119	}
120
121	static void dce5_crtc_load_lut(struct drm_crtc *crtc)
122	{
123	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
124	struct drm_device *dev = crtc->dev;
125	struct radeon_device *rdev = dev->dev_private;
126	u16 *r, *g, *b;
127	int i;
128
129	DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
130
131	msleep(msecs: 10);
132
133	WREG32(NI_INPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
134	(NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) |
135	NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS)));
136	WREG32(NI_PRESCALE_GRPH_CONTROL + radeon_crtc->crtc_offset,
137	NI_GRPH_PRESCALE_BYPASS);
138	WREG32(NI_PRESCALE_OVL_CONTROL + radeon_crtc->crtc_offset,
139	NI_OVL_PRESCALE_BYPASS);
140	WREG32(NI_INPUT_GAMMA_CONTROL + radeon_crtc->crtc_offset,
141	(NI_GRPH_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT) |
142	NI_OVL_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT)));
143
144	WREG32(EVERGREEN_DC_LUT_CONTROL + radeon_crtc->crtc_offset, 0);
145
146	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + radeon_crtc->crtc_offset, 0);
147	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + radeon_crtc->crtc_offset, 0);
148	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + radeon_crtc->crtc_offset, 0);
149
150	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + radeon_crtc->crtc_offset, 0xffff);
151	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + radeon_crtc->crtc_offset, 0xffff);
152	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + radeon_crtc->crtc_offset, 0xffff);
153
154	WREG32(EVERGREEN_DC_LUT_RW_MODE + radeon_crtc->crtc_offset, 0);
155	WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
156
157	WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
158	r = crtc->gamma_store;
159	g = r + crtc->gamma_size;
160	b = g + crtc->gamma_size;
161	for (i = 0; i < 256; i++) {
162	WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
163	((*r++ & 0xffc0) << 14) |
164	((*g++ & 0xffc0) << 4) |
165	(*b++ >> 6));
166	}
167
168	WREG32(NI_DEGAMMA_CONTROL + radeon_crtc->crtc_offset,
169	(NI_GRPH_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
170	NI_OVL_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
171	NI_ICON_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
172	NI_CURSOR_DEGAMMA_MODE(NI_DEGAMMA_BYPASS)));
173	WREG32(NI_GAMUT_REMAP_CONTROL + radeon_crtc->crtc_offset,
174	(NI_GRPH_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS) |
175	NI_OVL_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS)));
176	WREG32(NI_REGAMMA_CONTROL + radeon_crtc->crtc_offset,
177	(NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) |
178	NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS)));
179	WREG32(NI_OUTPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
180	(NI_OUTPUT_CSC_GRPH_MODE(radeon_crtc->output_csc) |
181	NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
182	/* XXX match this to the depth of the crtc fmt block, move to modeset? */
183	WREG32(0x6940 + radeon_crtc->crtc_offset, 0);
184	if (ASIC_IS_DCE8(rdev)) {
185	/* XXX this only needs to be programmed once per crtc at startup,
186	* not sure where the best place for it is
187	*/
188	WREG32(CIK_ALPHA_CONTROL + radeon_crtc->crtc_offset,
189	CIK_CURSOR_ALPHA_BLND_ENA);
190	}
191	}
192
193	static void legacy_crtc_load_lut(struct drm_crtc *crtc)
194	{
195	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
196	struct drm_device *dev = crtc->dev;
197	struct radeon_device *rdev = dev->dev_private;
198	u16 *r, *g, *b;
199	int i;
200	uint32_t dac2_cntl;
201
202	dac2_cntl = RREG32(RADEON_DAC_CNTL2);
203	if (radeon_crtc->crtc_id == 0)
204	dac2_cntl &= (uint32_t)~RADEON_DAC2_PALETTE_ACC_CTL;
205	else
206	dac2_cntl |= RADEON_DAC2_PALETTE_ACC_CTL;
207	WREG32(RADEON_DAC_CNTL2, dac2_cntl);
208
209	WREG8(RADEON_PALETTE_INDEX, 0);
210	r = crtc->gamma_store;
211	g = r + crtc->gamma_size;
212	b = g + crtc->gamma_size;
213	for (i = 0; i < 256; i++) {
214	WREG32(RADEON_PALETTE_30_DATA,
215	((*r++ & 0xffc0) << 14) |
216	((*g++ & 0xffc0) << 4) |
217	(*b++ >> 6));
218	}
219	}
220
221	void radeon_crtc_load_lut(struct drm_crtc *crtc)
222	{
223	struct drm_device *dev = crtc->dev;
224	struct radeon_device *rdev = dev->dev_private;
225
226	if (!crtc->enabled)
227	return;
228
229	if (ASIC_IS_DCE5(rdev))
230	dce5_crtc_load_lut(crtc);
231	else if (ASIC_IS_DCE4(rdev))
232	dce4_crtc_load_lut(crtc);
233	else if (ASIC_IS_AVIVO(rdev))
234	avivo_crtc_load_lut(crtc);
235	else
236	legacy_crtc_load_lut(crtc);
237	}
238
239	static int radeon_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
240	u16 *blue, uint32_t size,
241	struct drm_modeset_acquire_ctx *ctx)
242	{
243	radeon_crtc_load_lut(crtc);
244
245	return 0;
246	}
247
248	static void radeon_crtc_destroy(struct drm_crtc *crtc)
249	{
250	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
251
252	drm_crtc_cleanup(crtc);
253	destroy_workqueue(wq: radeon_crtc->flip_queue);
254	kfree(objp: radeon_crtc);
255	}
256
257	/**
258	* radeon_unpin_work_func - unpin old buffer object
259	*
260	* @__work: kernel work item
261	*
262	* Unpin the old frame buffer object outside of the interrupt handler
263	*/
264	static void radeon_unpin_work_func(struct work_struct *__work)
265	{
266	struct radeon_flip_work *work =
267	container_of(__work, struct radeon_flip_work, unpin_work);
268	int r;
269
270	/* unpin of the old buffer */
271	r = radeon_bo_reserve(bo: work->old_rbo, no_intr: false);
272	if (likely(r == 0)) {
273	radeon_bo_unpin(bo: work->old_rbo);
274	radeon_bo_unreserve(bo: work->old_rbo);
275	} else
276	DRM_ERROR("failed to reserve buffer after flip\n");
277
278	drm_gem_object_put(obj: &work->old_rbo->tbo.base);
279	kfree(objp: work);
280	}
281
282	void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id)
283	{
284	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
285	unsigned long flags;
286	u32 update_pending;
287	int vpos, hpos;
288
289	/* can happen during initialization */
290	if (radeon_crtc == NULL)
291	return;
292
293	/* Skip the pageflip completion check below (based on polling) on
294	* asics which reliably support hw pageflip completion irqs. pflip
295	* irqs are a reliable and race-free method of handling pageflip
296	* completion detection. A use_pflipirq module parameter < 2 allows
297	* to override this in case of asics with faulty pflip irqs.
298	* A module parameter of 0 would only use this polling based path,
299	* a parameter of 1 would use pflip irq only as a backup to this
300	* path, as in Linux 3.16.
301	*/
302	if ((radeon_use_pflipirq == 2) && ASIC_IS_DCE4(rdev))
303	return;
304
305	spin_lock_irqsave(&rdev_to_drm(rdev)->event_lock, flags);
306	if (radeon_crtc->flip_status != RADEON_FLIP_SUBMITTED) {
307	DRM_DEBUG_DRIVER("radeon_crtc->flip_status = %d != "
308	"RADEON_FLIP_SUBMITTED(%d)\n",
309	radeon_crtc->flip_status,
310	RADEON_FLIP_SUBMITTED);
311	spin_unlock_irqrestore(lock: &rdev_to_drm(rdev)->event_lock, flags);
312	return;
313	}
314
315	update_pending = radeon_page_flip_pending(rdev, crtc_id);
316
317	/* Has the pageflip already completed in crtc, or is it certain
318	* to complete in this vblank? GET_DISTANCE_TO_VBLANKSTART provides
319	* distance to start of "fudged earlier" vblank in vpos, distance to
320	* start of real vblank in hpos. vpos >= 0 && hpos < 0 means we are in
321	* the last few scanlines before start of real vblank, where the vblank
322	* irq can fire, so we have sampled update_pending a bit too early and
323	* know the flip will complete at leading edge of the upcoming real
324	* vblank. On pre-AVIVO hardware, flips also complete inside the real
325	* vblank, not only at leading edge, so if update_pending for hpos >= 0
326	* == inside real vblank, the flip will complete almost immediately.
327	* Note that this method of completion handling is still not 100% race
328	* free, as we could execute before the radeon_flip_work_func managed
329	* to run and set the RADEON_FLIP_SUBMITTED status, thereby we no-op,
330	* but the flip still gets programmed into hw and completed during
331	* vblank, leading to a delayed emission of the flip completion event.
332	* This applies at least to pre-AVIVO hardware, where flips are always
333	* completing inside vblank, not only at leading edge of vblank.
334	*/
335	if (update_pending &&
336	(DRM_SCANOUTPOS_VALID &
337	radeon_get_crtc_scanoutpos(dev: rdev_to_drm(rdev), pipe: crtc_id,
338	GET_DISTANCE_TO_VBLANKSTART,
339	vpos: &vpos, hpos: &hpos, NULL, NULL,
340	mode: &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) &&
341	((vpos >= 0 && hpos < 0) || (hpos >= 0 && !ASIC_IS_AVIVO(rdev)))) {
342	/* crtc didn't flip in this target vblank interval,
343	* but flip is pending in crtc. Based on the current
344	* scanout position we know that the current frame is
345	* (nearly) complete and the flip will (likely)
346	* complete before the start of the next frame.
347	*/
348	update_pending = 0;
349	}
350	spin_unlock_irqrestore(lock: &rdev_to_drm(rdev)->event_lock, flags);
351	if (!update_pending)
352	radeon_crtc_handle_flip(rdev, crtc_id);
353	}
354
355	/**
356	* radeon_crtc_handle_flip - page flip completed
357	*
358	* @rdev: radeon device pointer
359	* @crtc_id: crtc number this event is for
360	*
361	* Called when we are sure that a page flip for this crtc is completed.
362	*/
363	void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
364	{
365	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
366	struct radeon_flip_work *work;
367	unsigned long flags;
368
369	/* this can happen at init */
370	if (radeon_crtc == NULL)
371	return;
372
373	spin_lock_irqsave(&rdev_to_drm(rdev)->event_lock, flags);
374	work = radeon_crtc->flip_work;
375	if (radeon_crtc->flip_status != RADEON_FLIP_SUBMITTED) {
376	DRM_DEBUG_DRIVER("radeon_crtc->flip_status = %d != "
377	"RADEON_FLIP_SUBMITTED(%d)\n",
378	radeon_crtc->flip_status,
379	RADEON_FLIP_SUBMITTED);
380	spin_unlock_irqrestore(lock: &rdev_to_drm(rdev)->event_lock, flags);
381	return;
382	}
383
384	/* Pageflip completed. Clean up. */
385	radeon_crtc->flip_status = RADEON_FLIP_NONE;
386	radeon_crtc->flip_work = NULL;
387
388	/* wakeup userspace */
389	if (work->event)
390	drm_crtc_send_vblank_event(crtc: &radeon_crtc->base, e: work->event);
391
392	spin_unlock_irqrestore(lock: &rdev_to_drm(rdev)->event_lock, flags);
393
394	drm_crtc_vblank_put(crtc: &radeon_crtc->base);
395	radeon_irq_kms_pflip_irq_put(rdev, crtc: work->crtc_id);
396	queue_work(wq: radeon_crtc->flip_queue, work: &work->unpin_work);
397	}
398
399	/**
400	* radeon_flip_work_func - page flip framebuffer
401	*
402	* @__work: kernel work item
403	*
404	* Wait for the buffer object to become idle and do the actual page flip
405	*/
406	static void radeon_flip_work_func(struct work_struct *__work)
407	{
408	struct radeon_flip_work *work =
409	container_of(__work, struct radeon_flip_work, flip_work);
410	struct radeon_device *rdev = work->rdev;
411	struct drm_device *dev = rdev_to_drm(rdev);
412	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[work->crtc_id];
413
414	struct drm_crtc *crtc = &radeon_crtc->base;
415	unsigned long flags;
416	int r;
417	int vpos, hpos;
418
419	down_read(sem: &rdev->exclusive_lock);
420	if (work->fence) {
421	struct radeon_fence *fence;
422
423	fence = to_radeon_fence(f: work->fence);
424	if (fence && fence->rdev == rdev) {
425	r = radeon_fence_wait(fence, interruptible: false);
426	if (r == -EDEADLK) {
427	up_read(sem: &rdev->exclusive_lock);
428	do {
429	r = radeon_gpu_reset(rdev);
430	} while (r == -EAGAIN);
431	down_read(sem: &rdev->exclusive_lock);
432	}
433	} else
434	r = dma_fence_wait(fence: work->fence, intr: false);
435
436	if (r)
437	DRM_ERROR("failed to wait on page flip fence (%d)!\n", r);
438
439	/* We continue with the page flip even if we failed to wait on
440	* the fence, otherwise the DRM core and userspace will be
441	* confused about which BO the CRTC is scanning out
442	*/
443
444	dma_fence_put(fence: work->fence);
445	work->fence = NULL;
446	}
447
448	/* Wait until we're out of the vertical blank period before the one
449	* targeted by the flip. Always wait on pre DCE4 to avoid races with
450	* flip completion handling from vblank irq, as these old asics don't
451	* have reliable pageflip completion interrupts.
452	*/
453	while (radeon_crtc->enabled &&
454	(radeon_get_crtc_scanoutpos(dev, pipe: work->crtc_id, flags: 0,
455	vpos: &vpos, hpos: &hpos, NULL, NULL,
456	mode: &crtc->hwmode)
457	& (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) ==
458	(DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) &&
459	(!ASIC_IS_AVIVO(rdev) ||
460	((int) (work->target_vblank -
461	crtc->funcs->get_vblank_counter(crtc)) > 0)))
462	usleep_range(min: 1000, max: 2000);
463
464	/* We borrow the event spin lock for protecting flip_status */
465	spin_lock_irqsave(&crtc->dev->event_lock, flags);
466
467	/* set the proper interrupt */
468	radeon_irq_kms_pflip_irq_get(rdev, crtc: radeon_crtc->crtc_id);
469
470	/* do the flip (mmio) */
471	radeon_page_flip(rdev, radeon_crtc->crtc_id, work->base, work->async);
472
473	radeon_crtc->flip_status = RADEON_FLIP_SUBMITTED;
474	spin_unlock_irqrestore(lock: &crtc->dev->event_lock, flags);
475	up_read(sem: &rdev->exclusive_lock);
476	}
477
478	static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
479	struct drm_framebuffer *fb,
480	struct drm_pending_vblank_event *event,
481	uint32_t page_flip_flags,
482	uint32_t target,
483	struct drm_modeset_acquire_ctx *ctx)
484	{
485	struct drm_device *dev = crtc->dev;
486	struct radeon_device *rdev = dev->dev_private;
487	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
488	struct drm_gem_object *obj;
489	struct radeon_flip_work *work;
490	struct radeon_bo *new_rbo;
491	uint32_t tiling_flags, pitch_pixels;
492	uint64_t base;
493	unsigned long flags;
494	int r;
495
496	work = kzalloc(sizeof *work, GFP_KERNEL);
497	if (work == NULL)
498	return -ENOMEM;
499
500	INIT_WORK(&work->flip_work, radeon_flip_work_func);
501	INIT_WORK(&work->unpin_work, radeon_unpin_work_func);
502
503	work->rdev = rdev;
504	work->crtc_id = radeon_crtc->crtc_id;
505	work->event = event;
506	work->async = (page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0;
507
508	/* schedule unpin of the old buffer */
509	obj = crtc->primary->fb->obj[0];
510
511	/* take a reference to the old object */
512	drm_gem_object_get(obj);
513	work->old_rbo = gem_to_radeon_bo(obj);
514
515	obj = fb->obj[0];
516	new_rbo = gem_to_radeon_bo(obj);
517
518	/* pin the new buffer */
519	DRM_DEBUG_DRIVER("flip-ioctl() cur_rbo = %p, new_rbo = %p\n",
520	work->old_rbo, new_rbo);
521
522	r = radeon_bo_reserve(bo: new_rbo, no_intr: false);
523	if (unlikely(r != 0)) {
524	DRM_ERROR("failed to reserve new rbo buffer before flip\n");
525	goto cleanup;
526	}
527	/* Only 27 bit offset for legacy CRTC */
528	r = radeon_bo_pin_restricted(bo: new_rbo, RADEON_GEM_DOMAIN_VRAM,
529	ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27, gpu_addr: &base);
530	if (unlikely(r != 0)) {
531	radeon_bo_unreserve(bo: new_rbo);
532	r = -EINVAL;
533	DRM_ERROR("failed to pin new rbo buffer before flip\n");
534	goto cleanup;
535	}
536	r = dma_resv_get_singleton(obj: new_rbo->tbo.base.resv, usage: DMA_RESV_USAGE_WRITE,
537	fence: &work->fence);
538	if (r) {
539	radeon_bo_unreserve(bo: new_rbo);
540	DRM_ERROR("failed to get new rbo buffer fences\n");
541	goto cleanup;
542	}
543	radeon_bo_get_tiling_flags(bo: new_rbo, tiling_flags: &tiling_flags, NULL);
544	radeon_bo_unreserve(bo: new_rbo);
545
546	if (!ASIC_IS_AVIVO(rdev)) {
547	/* crtc offset is from display base addr not FB location */
548	base -= radeon_crtc->legacy_display_base_addr;
549	pitch_pixels = fb->pitches[0] / fb->format->cpp[0];
550
551	if (tiling_flags & RADEON_TILING_MACRO) {
552	if (ASIC_IS_R300(rdev)) {
553	base &= ~0x7ff;
554	} else {
555	int byteshift = fb->format->cpp[0] * 8 >> 4;
556	int tile_addr = (((crtc->y >> 3) * pitch_pixels + crtc->x) >> (8 - byteshift)) << 11;
557	base += tile_addr + ((crtc->x << byteshift) % 256) + ((crtc->y % 8) << 8);
558	}
559	} else {
560	int offset = crtc->y * pitch_pixels + crtc->x;
561	switch (fb->format->cpp[0] * 8) {
562	case 8:
563	default:
564	offset *= 1;
565	break;
566	case 15:
567	case 16:
568	offset *= 2;
569	break;
570	case 24:
571	offset *= 3;
572	break;
573	case 32:
574	offset *= 4;
575	break;
576	}
577	base += offset;
578	}
579	base &= ~7;
580	}
581	work->base = base;
582	work->target_vblank = target - (uint32_t)drm_crtc_vblank_count(crtc) +
583	crtc->funcs->get_vblank_counter(crtc);
584
585	/* We borrow the event spin lock for protecting flip_work */
586	spin_lock_irqsave(&crtc->dev->event_lock, flags);
587
588	if (radeon_crtc->flip_status != RADEON_FLIP_NONE) {
589	DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
590	spin_unlock_irqrestore(lock: &crtc->dev->event_lock, flags);
591	r = -EBUSY;
592	goto pflip_cleanup;
593	}
594	radeon_crtc->flip_status = RADEON_FLIP_PENDING;
595	radeon_crtc->flip_work = work;
596
597	/* update crtc fb */
598	crtc->primary->fb = fb;
599
600	spin_unlock_irqrestore(lock: &crtc->dev->event_lock, flags);
601
602	queue_work(wq: radeon_crtc->flip_queue, work: &work->flip_work);
603	return 0;
604
605	pflip_cleanup:
606	if (unlikely(radeon_bo_reserve(new_rbo, false) != 0)) {
607	DRM_ERROR("failed to reserve new rbo in error path\n");
608	goto cleanup;
609	}
610	radeon_bo_unpin(bo: new_rbo);
611	radeon_bo_unreserve(bo: new_rbo);
612
613	cleanup:
614	drm_gem_object_put(obj: &work->old_rbo->tbo.base);
615	dma_fence_put(fence: work->fence);
616	kfree(objp: work);
617	return r;
618	}
619
620	static int
621	radeon_crtc_set_config(struct drm_mode_set *set,
622	struct drm_modeset_acquire_ctx *ctx)
623	{
624	struct drm_device *dev;
625	struct radeon_device *rdev;
626	struct drm_crtc *crtc;
627	bool active = false;
628	int ret;
629
630	if (!set || !set->crtc)
631	return -EINVAL;
632
633	dev = set->crtc->dev;
634
635	ret = pm_runtime_get_sync(dev: dev->dev);
636	if (ret < 0) {
637	pm_runtime_put_autosuspend(dev: dev->dev);
638	return ret;
639	}
640
641	ret = drm_crtc_helper_set_config(set, ctx);
642
643	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
644	if (crtc->enabled)
645	active = true;
646
647	rdev = dev->dev_private;
648	/* if we have active crtcs and we don't have a power ref,
649	take the current one */
650	if (active && !rdev->have_disp_power_ref) {
651	rdev->have_disp_power_ref = true;
652	return ret;
653	}
654	/* if we have no active crtcs, then drop the power ref
655	we got before */
656	if (!active && rdev->have_disp_power_ref) {
657	pm_runtime_put_autosuspend(dev: dev->dev);
658	rdev->have_disp_power_ref = false;
659	}
660
661	/* drop the power reference we got coming in here */
662	pm_runtime_put_autosuspend(dev: dev->dev);
663	return ret;
664	}
665
666	static const struct drm_crtc_funcs radeon_crtc_funcs = {
667	.cursor_set2 = radeon_crtc_cursor_set2,
668	.cursor_move = radeon_crtc_cursor_move,
669	.gamma_set = radeon_crtc_gamma_set,
670	.set_config = radeon_crtc_set_config,
671	.destroy = radeon_crtc_destroy,
672	.page_flip_target = radeon_crtc_page_flip_target,
673	.get_vblank_counter = radeon_get_vblank_counter_kms,
674	.enable_vblank = radeon_enable_vblank_kms,
675	.disable_vblank = radeon_disable_vblank_kms,
676	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
677	};
678
679	static void radeon_crtc_init(struct drm_device *dev, int index)
680	{
681	struct radeon_device *rdev = dev->dev_private;
682	struct radeon_crtc *radeon_crtc;
683
684	radeon_crtc = kzalloc(sizeof(*radeon_crtc), GFP_KERNEL);
685	if (radeon_crtc == NULL)
686	return;
687
688	radeon_crtc->flip_queue = alloc_workqueue("radeon-crtc", WQ_HIGHPRI, 0);
689	if (!radeon_crtc->flip_queue) {
690	kfree(objp: radeon_crtc);
691	return;
692	}
693
694	drm_crtc_init(dev, crtc: &radeon_crtc->base, funcs: &radeon_crtc_funcs);
695
696	drm_mode_crtc_set_gamma_size(crtc: &radeon_crtc->base, gamma_size: 256);
697	radeon_crtc->crtc_id = index;
698	rdev->mode_info.crtcs[index] = radeon_crtc;
699
700	if (rdev->family >= CHIP_BONAIRE) {
701	radeon_crtc->max_cursor_width = CIK_CURSOR_WIDTH;
702	radeon_crtc->max_cursor_height = CIK_CURSOR_HEIGHT;
703	} else {
704	radeon_crtc->max_cursor_width = CURSOR_WIDTH;
705	radeon_crtc->max_cursor_height = CURSOR_HEIGHT;
706	}
707	dev->mode_config.cursor_width = radeon_crtc->max_cursor_width;
708	dev->mode_config.cursor_height = radeon_crtc->max_cursor_height;
709
710	if (rdev->is_atom_bios && (ASIC_IS_AVIVO(rdev) || radeon_r4xx_atom))
711	radeon_atombios_init_crtc(dev, radeon_crtc);
712	else
713	radeon_legacy_init_crtc(dev, radeon_crtc);
714	}
715
716	static const char *encoder_names[38] = {
717	"NONE",
718	"INTERNAL_LVDS",
719	"INTERNAL_TMDS1",
720	"INTERNAL_TMDS2",
721	"INTERNAL_DAC1",
722	"INTERNAL_DAC2",
723	"INTERNAL_SDVOA",
724	"INTERNAL_SDVOB",
725	"SI170B",
726	"CH7303",
727	"CH7301",
728	"INTERNAL_DVO1",
729	"EXTERNAL_SDVOA",
730	"EXTERNAL_SDVOB",
731	"TITFP513",
732	"INTERNAL_LVTM1",
733	"VT1623",
734	"HDMI_SI1930",
735	"HDMI_INTERNAL",
736	"INTERNAL_KLDSCP_TMDS1",
737	"INTERNAL_KLDSCP_DVO1",
738	"INTERNAL_KLDSCP_DAC1",
739	"INTERNAL_KLDSCP_DAC2",
740	"SI178",
741	"MVPU_FPGA",
742	"INTERNAL_DDI",
743	"VT1625",
744	"HDMI_SI1932",
745	"DP_AN9801",
746	"DP_DP501",
747	"INTERNAL_UNIPHY",
748	"INTERNAL_KLDSCP_LVTMA",
749	"INTERNAL_UNIPHY1",
750	"INTERNAL_UNIPHY2",
751	"NUTMEG",
752	"TRAVIS",
753	"INTERNAL_VCE",
754	"INTERNAL_UNIPHY3",
755	};
756
757	static const char *hpd_names[6] = {
758	"HPD1",
759	"HPD2",
760	"HPD3",
761	"HPD4",
762	"HPD5",
763	"HPD6",
764	};
765
766	static void radeon_print_display_setup(struct drm_device *dev)
767	{
768	struct drm_connector *connector;
769	struct radeon_connector *radeon_connector;
770	struct drm_encoder *encoder;
771	struct radeon_encoder *radeon_encoder;
772	uint32_t devices;
773	int i = 0;
774
775	DRM_INFO("Radeon Display Connectors\n");
776	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
777	radeon_connector = to_radeon_connector(connector);
778	DRM_INFO("Connector %d:\n", i);
779	DRM_INFO(" %s\n", connector->name);
780	if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
781	DRM_INFO(" %s\n", hpd_names[radeon_connector->hpd.hpd]);
782	if (radeon_connector->ddc_bus) {
783	DRM_INFO(" DDC: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
784	radeon_connector->ddc_bus->rec.mask_clk_reg,
785	radeon_connector->ddc_bus->rec.mask_data_reg,
786	radeon_connector->ddc_bus->rec.a_clk_reg,
787	radeon_connector->ddc_bus->rec.a_data_reg,
788	radeon_connector->ddc_bus->rec.en_clk_reg,
789	radeon_connector->ddc_bus->rec.en_data_reg,
790	radeon_connector->ddc_bus->rec.y_clk_reg,
791	radeon_connector->ddc_bus->rec.y_data_reg);
792	if (radeon_connector->router.ddc_valid)
793	DRM_INFO(" DDC Router 0x%x/0x%x\n",
794	radeon_connector->router.ddc_mux_control_pin,
795	radeon_connector->router.ddc_mux_state);
796	if (radeon_connector->router.cd_valid)
797	DRM_INFO(" Clock/Data Router 0x%x/0x%x\n",
798	radeon_connector->router.cd_mux_control_pin,
799	radeon_connector->router.cd_mux_state);
800	} else {
801	if (connector->connector_type == DRM_MODE_CONNECTOR_VGA ||
802	connector->connector_type == DRM_MODE_CONNECTOR_DVII ||
803	connector->connector_type == DRM_MODE_CONNECTOR_DVID ||
804	connector->connector_type == DRM_MODE_CONNECTOR_DVIA ||
805	connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
806	connector->connector_type == DRM_MODE_CONNECTOR_HDMIB)
807	DRM_INFO(" DDC: no ddc bus - possible BIOS bug - please report to xorg-driver-ati@lists.x.org\n");
808	}
809	DRM_INFO(" Encoders:\n");
810	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
811	radeon_encoder = to_radeon_encoder(encoder);
812	devices = radeon_encoder->devices & radeon_connector->devices;
813	if (devices) {
814	if (devices & ATOM_DEVICE_CRT1_SUPPORT)
815	DRM_INFO(" CRT1: %s\n", encoder_names[radeon_encoder->encoder_id]);
816	if (devices & ATOM_DEVICE_CRT2_SUPPORT)
817	DRM_INFO(" CRT2: %s\n", encoder_names[radeon_encoder->encoder_id]);
818	if (devices & ATOM_DEVICE_LCD1_SUPPORT)
819	DRM_INFO(" LCD1: %s\n", encoder_names[radeon_encoder->encoder_id]);
820	if (devices & ATOM_DEVICE_DFP1_SUPPORT)
821	DRM_INFO(" DFP1: %s\n", encoder_names[radeon_encoder->encoder_id]);
822	if (devices & ATOM_DEVICE_DFP2_SUPPORT)
823	DRM_INFO(" DFP2: %s\n", encoder_names[radeon_encoder->encoder_id]);
824	if (devices & ATOM_DEVICE_DFP3_SUPPORT)
825	DRM_INFO(" DFP3: %s\n", encoder_names[radeon_encoder->encoder_id]);
826	if (devices & ATOM_DEVICE_DFP4_SUPPORT)
827	DRM_INFO(" DFP4: %s\n", encoder_names[radeon_encoder->encoder_id]);
828	if (devices & ATOM_DEVICE_DFP5_SUPPORT)
829	DRM_INFO(" DFP5: %s\n", encoder_names[radeon_encoder->encoder_id]);
830	if (devices & ATOM_DEVICE_DFP6_SUPPORT)
831	DRM_INFO(" DFP6: %s\n", encoder_names[radeon_encoder->encoder_id]);
832	if (devices & ATOM_DEVICE_TV1_SUPPORT)
833	DRM_INFO(" TV1: %s\n", encoder_names[radeon_encoder->encoder_id]);
834	if (devices & ATOM_DEVICE_CV_SUPPORT)
835	DRM_INFO(" CV: %s\n", encoder_names[radeon_encoder->encoder_id]);
836	}
837	}
838	i++;
839	}
840	}
841
842	static bool radeon_setup_enc_conn(struct drm_device *dev)
843	{
844	struct radeon_device *rdev = dev->dev_private;
845	bool ret = false;
846
847	if (rdev->bios) {
848	if (rdev->is_atom_bios) {
849	ret = radeon_get_atom_connector_info_from_supported_devices_table(dev);
850	if (!ret)
851	ret = radeon_get_atom_connector_info_from_object_table(dev);
852	} else {
853	ret = radeon_get_legacy_connector_info_from_bios(dev);
854	if (!ret)
855	ret = radeon_get_legacy_connector_info_from_table(dev);
856	}
857	} else {
858	if (!ASIC_IS_AVIVO(rdev))
859	ret = radeon_get_legacy_connector_info_from_table(dev);
860	}
861	if (ret) {
862	radeon_setup_encoder_clones(dev);
863	radeon_print_display_setup(dev);
864	}
865
866	return ret;
867	}
868
869	/* avivo */
870
871	/**
872	* avivo_reduce_ratio - fractional number reduction
873	*
874	* @nom: nominator
875	* @den: denominator
876	* @nom_min: minimum value for nominator
877	* @den_min: minimum value for denominator
878	*
879	* Find the greatest common divisor and apply it on both nominator and
880	* denominator, but make nominator and denominator are at least as large
881	* as their minimum values.
882	*/
883	static void avivo_reduce_ratio(unsigned *nom, unsigned *den,
884	unsigned nom_min, unsigned den_min)
885	{
886	unsigned tmp;
887
888	/* reduce the numbers to a simpler ratio */
889	tmp = gcd(a: *nom, b: *den);
890	*nom /= tmp;
891	*den /= tmp;
892
893	/* make sure nominator is large enough */
894	if (*nom < nom_min) {
895	tmp = DIV_ROUND_UP(nom_min, *nom);
896	*nom *= tmp;
897	*den *= tmp;
898	}
899
900	/* make sure the denominator is large enough */
901	if (*den < den_min) {
902	tmp = DIV_ROUND_UP(den_min, *den);
903	*nom *= tmp;
904	*den *= tmp;
905	}
906	}
907
908	/**
909	* avivo_get_fb_ref_div - feedback and ref divider calculation
910	*
911	* @nom: nominator
912	* @den: denominator
913	* @post_div: post divider
914	* @fb_div_max: feedback divider maximum
915	* @ref_div_max: reference divider maximum
916	* @fb_div: resulting feedback divider
917	* @ref_div: resulting reference divider
918	*
919	* Calculate feedback and reference divider for a given post divider. Makes
920	* sure we stay within the limits.
921	*/
922	static void avivo_get_fb_ref_div(unsigned nom, unsigned den, unsigned post_div,
923	unsigned fb_div_max, unsigned ref_div_max,
924	unsigned *fb_div, unsigned *ref_div)
925	{
926	/* limit reference * post divider to a maximum */
927	ref_div_max = clamp(100 / post_div, 1u, ref_div_max);
928
929	/* get matching reference and feedback divider */
930	*ref_div = clamp(den / post_div, 1u, ref_div_max);
931	*fb_div = DIV_ROUND_CLOSEST(nom * *ref_div * post_div, den);
932
933	/* limit fb divider to its maximum */
934	if (*fb_div > fb_div_max) {
935	*ref_div = (*ref_div * fb_div_max)/(*fb_div);
936	*fb_div = fb_div_max;
937	}
938	}
939
940	/**
941	* radeon_compute_pll_avivo - compute PLL paramaters
942	*
943	* @pll: information about the PLL
944	* @freq: target frequency
945	* @dot_clock_p: resulting pixel clock
946	* @fb_div_p: resulting feedback divider
947	* @frac_fb_div_p: fractional part of the feedback divider
948	* @ref_div_p: resulting reference divider
949	* @post_div_p: resulting reference divider
950	*
951	* Try to calculate the PLL parameters to generate the given frequency:
952	* dot_clock = (ref_freq * feedback_div) / (ref_div * post_div)
953	*/
954	void radeon_compute_pll_avivo(struct radeon_pll *pll,
955	u32 freq,
956	u32 *dot_clock_p,
957	u32 *fb_div_p,
958	u32 *frac_fb_div_p,
959	u32 *ref_div_p,
960	u32 *post_div_p)
961	{
962	unsigned target_clock = pll->flags & RADEON_PLL_USE_FRAC_FB_DIV ?
963	freq : freq / 10;
964
965	unsigned fb_div_min, fb_div_max, fb_div;
966	unsigned post_div_min, post_div_max, post_div;
967	unsigned ref_div_min, ref_div_max, ref_div;
968	unsigned post_div_best, diff_best;
969	unsigned nom, den;
970
971	/* determine allowed feedback divider range */
972	fb_div_min = pll->min_feedback_div;
973	fb_div_max = pll->max_feedback_div;
974
975	if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
976	fb_div_min *= 10;
977	fb_div_max *= 10;
978	}
979
980	/* determine allowed ref divider range */
981	if (pll->flags & RADEON_PLL_USE_REF_DIV)
982	ref_div_min = pll->reference_div;
983	else
984	ref_div_min = pll->min_ref_div;
985
986	if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV &&
987	pll->flags & RADEON_PLL_USE_REF_DIV)
988	ref_div_max = pll->reference_div;
989	else if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP)
990	/* fix for problems on RS880 */
991	ref_div_max = min(pll->max_ref_div, 7u);
992	else
993	ref_div_max = pll->max_ref_div;
994
995	/* determine allowed post divider range */
996	if (pll->flags & RADEON_PLL_USE_POST_DIV) {
997	post_div_min = pll->post_div;
998	post_div_max = pll->post_div;
999	} else {
1000	unsigned vco_min, vco_max;
1001
1002	if (pll->flags & RADEON_PLL_IS_LCD) {
1003	vco_min = pll->lcd_pll_out_min;
1004	vco_max = pll->lcd_pll_out_max;
1005	} else {
1006	vco_min = pll->pll_out_min;
1007	vco_max = pll->pll_out_max;
1008	}
1009
1010	if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
1011	vco_min *= 10;
1012	vco_max *= 10;
1013	}
1014
1015	post_div_min = vco_min / target_clock;
1016	if ((target_clock * post_div_min) < vco_min)
1017	++post_div_min;
1018	if (post_div_min < pll->min_post_div)
1019	post_div_min = pll->min_post_div;
1020
1021	post_div_max = vco_max / target_clock;
1022	if ((target_clock * post_div_max) > vco_max)
1023	--post_div_max;
1024	if (post_div_max > pll->max_post_div)
1025	post_div_max = pll->max_post_div;
1026	}
1027
1028	/* represent the searched ratio as fractional number */
1029	nom = target_clock;
1030	den = pll->reference_freq;
1031
1032	/* reduce the numbers to a simpler ratio */
1033	avivo_reduce_ratio(nom: &nom, den: &den, nom_min: fb_div_min, den_min: post_div_min);
1034
1035	/* now search for a post divider */
1036	if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP)
1037	post_div_best = post_div_min;
1038	else
1039	post_div_best = post_div_max;
1040	diff_best = ~0;
1041
1042	for (post_div = post_div_min; post_div <= post_div_max; ++post_div) {
1043	unsigned diff;
1044	avivo_get_fb_ref_div(nom, den, post_div, fb_div_max,
1045	ref_div_max, fb_div: &fb_div, ref_div: &ref_div);
1046	diff = abs(target_clock - (pll->reference_freq * fb_div) /
1047	(ref_div * post_div));
1048
1049	if (diff < diff_best || (diff == diff_best &&
1050	!(pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP))) {
1051
1052	post_div_best = post_div;
1053	diff_best = diff;
1054	}
1055	}
1056	post_div = post_div_best;
1057
1058	/* get the feedback and reference divider for the optimal value */
1059	avivo_get_fb_ref_div(nom, den, post_div, fb_div_max, ref_div_max,
1060	fb_div: &fb_div, ref_div: &ref_div);
1061
1062	/* reduce the numbers to a simpler ratio once more */
1063	/* this also makes sure that the reference divider is large enough */
1064	avivo_reduce_ratio(nom: &fb_div, den: &ref_div, nom_min: fb_div_min, den_min: ref_div_min);
1065
1066	/* avoid high jitter with small fractional dividers */
1067	if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV && (fb_div % 10)) {
1068	fb_div_min = max(fb_div_min, (9 - (fb_div % 10)) * 20 + 50);
1069	if (fb_div < fb_div_min) {
1070	unsigned tmp = DIV_ROUND_UP(fb_div_min, fb_div);
1071	fb_div *= tmp;
1072	ref_div *= tmp;
1073	}
1074	}
1075
1076	/* and finally save the result */
1077	if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
1078	*fb_div_p = fb_div / 10;
1079	*frac_fb_div_p = fb_div % 10;
1080	} else {
1081	*fb_div_p = fb_div;
1082	*frac_fb_div_p = 0;
1083	}
1084
1085	*dot_clock_p = ((pll->reference_freq * *fb_div_p * 10) +
1086	(pll->reference_freq * *frac_fb_div_p)) /
1087	(ref_div * post_div * 10);
1088	*ref_div_p = ref_div;
1089	*post_div_p = post_div;
1090
1091	DRM_DEBUG_KMS("%d - %d, pll dividers - fb: %d.%d ref: %d, post %d\n",
1092	freq, *dot_clock_p * 10, *fb_div_p, *frac_fb_div_p,
1093	ref_div, post_div);
1094	}
1095
1096	/* pre-avivo */
1097	static inline uint32_t radeon_div(uint64_t n, uint32_t d)
1098	{
1099	n += d / 2;
1100
1101	do_div(n, d);
1102	return n;
1103	}
1104
1105	void radeon_compute_pll_legacy(struct radeon_pll *pll,
1106	uint64_t freq,
1107	uint32_t *dot_clock_p,
1108	uint32_t *fb_div_p,
1109	uint32_t *frac_fb_div_p,
1110	uint32_t *ref_div_p,
1111	uint32_t *post_div_p)
1112	{
1113	uint32_t min_ref_div = pll->min_ref_div;
1114	uint32_t max_ref_div = pll->max_ref_div;
1115	uint32_t min_post_div = pll->min_post_div;
1116	uint32_t max_post_div = pll->max_post_div;
1117	uint32_t min_fractional_feed_div = 0;
1118	uint32_t max_fractional_feed_div = 0;
1119	uint32_t best_vco = pll->best_vco;
1120	uint32_t best_post_div = 1;
1121	uint32_t best_ref_div = 1;
1122	uint32_t best_feedback_div = 1;
1123	uint32_t best_frac_feedback_div = 0;
1124	uint32_t best_freq = -1;
1125	uint32_t best_error = 0xffffffff;
1126	uint32_t best_vco_diff = 1;
1127	uint32_t post_div;
1128	u32 pll_out_min, pll_out_max;
1129
1130	DRM_DEBUG_KMS("PLL freq %llu %u %u\n", freq, pll->min_ref_div, pll->max_ref_div);
1131	freq = freq * 1000;
1132
1133	if (pll->flags & RADEON_PLL_IS_LCD) {
1134	pll_out_min = pll->lcd_pll_out_min;
1135	pll_out_max = pll->lcd_pll_out_max;
1136	} else {
1137	pll_out_min = pll->pll_out_min;
1138	pll_out_max = pll->pll_out_max;
1139	}
1140
1141	if (pll_out_min > 64800)
1142	pll_out_min = 64800;
1143
1144	if (pll->flags & RADEON_PLL_USE_REF_DIV)
1145	min_ref_div = max_ref_div = pll->reference_div;
1146	else {
1147	while (min_ref_div < max_ref_div-1) {
1148	uint32_t mid = (min_ref_div + max_ref_div) / 2;
1149	uint32_t pll_in = pll->reference_freq / mid;
1150	if (pll_in < pll->pll_in_min)
1151	max_ref_div = mid;
1152	else if (pll_in > pll->pll_in_max)
1153	min_ref_div = mid;
1154	else
1155	break;
1156	}
1157	}
1158
1159	if (pll->flags & RADEON_PLL_USE_POST_DIV)
1160	min_post_div = max_post_div = pll->post_div;
1161
1162	if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
1163	min_fractional_feed_div = pll->min_frac_feedback_div;
1164	max_fractional_feed_div = pll->max_frac_feedback_div;
1165	}
1166
1167	for (post_div = max_post_div; post_div >= min_post_div; --post_div) {
1168	uint32_t ref_div;
1169
1170	if ((pll->flags & RADEON_PLL_NO_ODD_POST_DIV) && (post_div & 1))
1171	continue;
1172
1173	/* legacy radeons only have a few post_divs */
1174	if (pll->flags & RADEON_PLL_LEGACY) {
1175	if ((post_div == 5) ||
1176	(post_div == 7) ||
1177	(post_div == 9) ||
1178	(post_div == 10) ||
1179	(post_div == 11) ||
1180	(post_div == 13) ||
1181	(post_div == 14) ||
1182	(post_div == 15))
1183	continue;
1184	}
1185
1186	for (ref_div = min_ref_div; ref_div <= max_ref_div; ++ref_div) {
1187	uint32_t feedback_div, current_freq = 0, error, vco_diff;
1188	uint32_t pll_in = pll->reference_freq / ref_div;
1189	uint32_t min_feed_div = pll->min_feedback_div;
1190	uint32_t max_feed_div = pll->max_feedback_div + 1;
1191
1192	if (pll_in < pll->pll_in_min || pll_in > pll->pll_in_max)
1193	continue;
1194
1195	while (min_feed_div < max_feed_div) {
1196	uint32_t vco;
1197	uint32_t min_frac_feed_div = min_fractional_feed_div;
1198	uint32_t max_frac_feed_div = max_fractional_feed_div + 1;
1199	uint32_t frac_feedback_div;
1200	uint64_t tmp;
1201
1202	feedback_div = (min_feed_div + max_feed_div) / 2;
1203
1204	tmp = (uint64_t)pll->reference_freq * feedback_div;
1205	vco = radeon_div(n: tmp, d: ref_div);
1206
1207	if (vco < pll_out_min) {
1208	min_feed_div = feedback_div + 1;
1209	continue;
1210	} else if (vco > pll_out_max) {
1211	max_feed_div = feedback_div;
1212	continue;
1213	}
1214
1215	while (min_frac_feed_div < max_frac_feed_div) {
1216	frac_feedback_div = (min_frac_feed_div + max_frac_feed_div) / 2;
1217	tmp = (uint64_t)pll->reference_freq * 10000 * feedback_div;
1218	tmp += (uint64_t)pll->reference_freq * 1000 * frac_feedback_div;
1219	current_freq = radeon_div(n: tmp, d: ref_div * post_div);
1220
1221	if (pll->flags & RADEON_PLL_PREFER_CLOSEST_LOWER) {
1222	if (freq < current_freq)
1223	error = 0xffffffff;
1224	else
1225	error = freq - current_freq;
1226	} else
1227	error = abs(current_freq - freq);
1228	vco_diff = abs(vco - best_vco);
1229
1230	if ((best_vco == 0 && error < best_error) ||
1231	(best_vco != 0 &&
1232	((best_error > 100 && error < best_error - 100) ||
1233	(abs(error - best_error) < 100 && vco_diff < best_vco_diff)))) {
1234	best_post_div = post_div;
1235	best_ref_div = ref_div;
1236	best_feedback_div = feedback_div;
1237	best_frac_feedback_div = frac_feedback_div;
1238	best_freq = current_freq;
1239	best_error = error;
1240	best_vco_diff = vco_diff;
1241	} else if (current_freq == freq) {
1242	if (best_freq == -1) {
1243	best_post_div = post_div;
1244	best_ref_div = ref_div;
1245	best_feedback_div = feedback_div;
1246	best_frac_feedback_div = frac_feedback_div;
1247	best_freq = current_freq;
1248	best_error = error;
1249	best_vco_diff = vco_diff;
1250	} else if (((pll->flags & RADEON_PLL_PREFER_LOW_REF_DIV) && (ref_div < best_ref_div)) ||
1251	((pll->flags & RADEON_PLL_PREFER_HIGH_REF_DIV) && (ref_div > best_ref_div)) ||
1252	((pll->flags & RADEON_PLL_PREFER_LOW_FB_DIV) && (feedback_div < best_feedback_div)) ||
1253	((pll->flags & RADEON_PLL_PREFER_HIGH_FB_DIV) && (feedback_div > best_feedback_div)) ||
1254	((pll->flags & RADEON_PLL_PREFER_LOW_POST_DIV) && (post_div < best_post_div)) ||
1255	((pll->flags & RADEON_PLL_PREFER_HIGH_POST_DIV) && (post_div > best_post_div))) {
1256	best_post_div = post_div;
1257	best_ref_div = ref_div;
1258	best_feedback_div = feedback_div;
1259	best_frac_feedback_div = frac_feedback_div;
1260	best_freq = current_freq;
1261	best_error = error;
1262	best_vco_diff = vco_diff;
1263	}
1264	}
1265	if (current_freq < freq)
1266	min_frac_feed_div = frac_feedback_div + 1;
1267	else
1268	max_frac_feed_div = frac_feedback_div;
1269	}
1270	if (current_freq < freq)
1271	min_feed_div = feedback_div + 1;
1272	else
1273	max_feed_div = feedback_div;
1274	}
1275	}
1276	}
1277
1278	*dot_clock_p = best_freq / 10000;
1279	*fb_div_p = best_feedback_div;
1280	*frac_fb_div_p = best_frac_feedback_div;
1281	*ref_div_p = best_ref_div;
1282	*post_div_p = best_post_div;
1283	DRM_DEBUG_KMS("%lld %d, pll dividers - fb: %d.%d ref: %d, post %d\n",
1284	(long long)freq,
1285	best_freq / 1000, best_feedback_div, best_frac_feedback_div,
1286	best_ref_div, best_post_div);
1287
1288	}
1289
1290	static const struct drm_framebuffer_funcs radeon_fb_funcs = {
1291	.destroy = drm_gem_fb_destroy,
1292	.create_handle = drm_gem_fb_create_handle,
1293	};
1294
1295	int
1296	radeon_framebuffer_init(struct drm_device *dev,
1297	struct drm_framebuffer *fb,
1298	const struct drm_format_info *info,
1299	const struct drm_mode_fb_cmd2 *mode_cmd,
1300	struct drm_gem_object *obj)
1301	{
1302	int ret;
1303	fb->obj[0] = obj;
1304	drm_helper_mode_fill_fb_struct(dev, fb, info, mode_cmd);
1305	ret = drm_framebuffer_init(dev, fb, funcs: &radeon_fb_funcs);
1306	if (ret) {
1307	fb->obj[0] = NULL;
1308	return ret;
1309	}
1310	return 0;
1311	}
1312
1313	static struct drm_framebuffer *
1314	radeon_user_framebuffer_create(struct drm_device *dev,
1315	struct drm_file *file_priv,
1316	const struct drm_format_info *info,
1317	const struct drm_mode_fb_cmd2 *mode_cmd)
1318	{
1319	struct drm_gem_object *obj;
1320	struct drm_framebuffer *fb;
1321	int ret;
1322
1323	obj = drm_gem_object_lookup(filp: file_priv, handle: mode_cmd->handles[0]);
1324	if (obj == NULL) {
1325	dev_err(dev->dev, "No GEM object associated to handle 0x%08X, "
1326	"can't create framebuffer\n", mode_cmd->handles[0]);
1327	return ERR_PTR(error: -ENOENT);
1328	}
1329
1330	/* Handle is imported dma-buf, so cannot be migrated to VRAM for scanout */
1331	if (obj->import_attach) {
1332	DRM_DEBUG_KMS("Cannot create framebuffer from imported dma_buf\n");
1333	drm_gem_object_put(obj);
1334	return ERR_PTR(error: -EINVAL);
1335	}
1336
1337	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
1338	if (fb == NULL) {
1339	drm_gem_object_put(obj);
1340	return ERR_PTR(error: -ENOMEM);
1341	}
1342
1343	ret = radeon_framebuffer_init(dev, fb, info, mode_cmd, obj);
1344	if (ret) {
1345	kfree(objp: fb);
1346	drm_gem_object_put(obj);
1347	return ERR_PTR(error: ret);
1348	}
1349
1350	return fb;
1351	}
1352
1353	static const struct drm_mode_config_funcs radeon_mode_funcs = {
1354	.fb_create = radeon_user_framebuffer_create,
1355	};
1356
1357	static const struct drm_prop_enum_list radeon_tmds_pll_enum_list[] =
1358	{ { 0, "driver" },
1359	{ 1, "bios" },
1360	};
1361
1362	static const struct drm_prop_enum_list radeon_tv_std_enum_list[] =
1363	{ { TV_STD_NTSC, "ntsc" },
1364	{ TV_STD_PAL, "pal" },
1365	{ TV_STD_PAL_M, "pal-m" },
1366	{ TV_STD_PAL_60, "pal-60" },
1367	{ TV_STD_NTSC_J, "ntsc-j" },
1368	{ TV_STD_SCART_PAL, "scart-pal" },
1369	{ TV_STD_PAL_CN, "pal-cn" },
1370	{ TV_STD_SECAM, "secam" },
1371	};
1372
1373	static const struct drm_prop_enum_list radeon_underscan_enum_list[] =
1374	{ { UNDERSCAN_OFF, "off" },
1375	{ UNDERSCAN_ON, "on" },
1376	{ UNDERSCAN_AUTO, "auto" },
1377	};
1378
1379	static const struct drm_prop_enum_list radeon_audio_enum_list[] =
1380	{ { RADEON_AUDIO_DISABLE, "off" },
1381	{ RADEON_AUDIO_ENABLE, "on" },
1382	{ RADEON_AUDIO_AUTO, "auto" },
1383	};
1384
1385	/* XXX support different dither options? spatial, temporal, both, etc. */
1386	static const struct drm_prop_enum_list radeon_dither_enum_list[] =
1387	{ { RADEON_FMT_DITHER_DISABLE, "off" },
1388	{ RADEON_FMT_DITHER_ENABLE, "on" },
1389	};
1390
1391	static const struct drm_prop_enum_list radeon_output_csc_enum_list[] =
1392	{ { RADEON_OUTPUT_CSC_BYPASS, "bypass" },
1393	{ RADEON_OUTPUT_CSC_TVRGB, "tvrgb" },
1394	{ RADEON_OUTPUT_CSC_YCBCR601, "ycbcr601" },
1395	{ RADEON_OUTPUT_CSC_YCBCR709, "ycbcr709" },
1396	};
1397
1398	static int radeon_modeset_create_props(struct radeon_device *rdev)
1399	{
1400	int sz;
1401
1402	if (rdev->is_atom_bios) {
1403	rdev->mode_info.coherent_mode_property =
1404	drm_property_create_range(dev: rdev_to_drm(rdev), flags: 0, name: "coherent", min: 0, max: 1);
1405	if (!rdev->mode_info.coherent_mode_property)
1406	return -ENOMEM;
1407	}
1408
1409	if (!ASIC_IS_AVIVO(rdev)) {
1410	sz = ARRAY_SIZE(radeon_tmds_pll_enum_list);
1411	rdev->mode_info.tmds_pll_property =
1412	drm_property_create_enum(dev: rdev_to_drm(rdev), flags: 0,
1413	name: "tmds_pll",
1414	props: radeon_tmds_pll_enum_list, num_values: sz);
1415	}
1416
1417	rdev->mode_info.load_detect_property =
1418	drm_property_create_range(dev: rdev_to_drm(rdev), flags: 0, name: "load detection", min: 0, max: 1);
1419	if (!rdev->mode_info.load_detect_property)
1420	return -ENOMEM;
1421
1422	drm_mode_create_scaling_mode_property(dev: rdev_to_drm(rdev));
1423
1424	sz = ARRAY_SIZE(radeon_tv_std_enum_list);
1425	rdev->mode_info.tv_std_property =
1426	drm_property_create_enum(dev: rdev_to_drm(rdev), flags: 0,
1427	name: "tv standard",
1428	props: radeon_tv_std_enum_list, num_values: sz);
1429
1430	sz = ARRAY_SIZE(radeon_underscan_enum_list);
1431	rdev->mode_info.underscan_property =
1432	drm_property_create_enum(dev: rdev_to_drm(rdev), flags: 0,
1433	name: "underscan",
1434	props: radeon_underscan_enum_list, num_values: sz);
1435
1436	rdev->mode_info.underscan_hborder_property =
1437	drm_property_create_range(dev: rdev_to_drm(rdev), flags: 0,
1438	name: "underscan hborder", min: 0, max: 128);
1439	if (!rdev->mode_info.underscan_hborder_property)
1440	return -ENOMEM;
1441
1442	rdev->mode_info.underscan_vborder_property =
1443	drm_property_create_range(dev: rdev_to_drm(rdev), flags: 0,
1444	name: "underscan vborder", min: 0, max: 128);
1445	if (!rdev->mode_info.underscan_vborder_property)
1446	return -ENOMEM;
1447
1448	sz = ARRAY_SIZE(radeon_audio_enum_list);
1449	rdev->mode_info.audio_property =
1450	drm_property_create_enum(dev: rdev_to_drm(rdev), flags: 0,
1451	name: "audio",
1452	props: radeon_audio_enum_list, num_values: sz);
1453
1454	sz = ARRAY_SIZE(radeon_dither_enum_list);
1455	rdev->mode_info.dither_property =
1456	drm_property_create_enum(dev: rdev_to_drm(rdev), flags: 0,
1457	name: "dither",
1458	props: radeon_dither_enum_list, num_values: sz);
1459
1460	sz = ARRAY_SIZE(radeon_output_csc_enum_list);
1461	rdev->mode_info.output_csc_property =
1462	drm_property_create_enum(dev: rdev_to_drm(rdev), flags: 0,
1463	name: "output_csc",
1464	props: radeon_output_csc_enum_list, num_values: sz);
1465
1466	return 0;
1467	}
1468
1469	void radeon_update_display_priority(struct radeon_device *rdev)
1470	{
1471	/* adjustment options for the display watermarks */
1472	if ((radeon_disp_priority == 0) || (radeon_disp_priority > 2)) {
1473	/* set display priority to high for r3xx, rv515 chips
1474	* this avoids flickering due to underflow to the
1475	* display controllers during heavy acceleration.
1476	* Don't force high on rs4xx igp chips as it seems to
1477	* affect the sound card. See kernel bug 15982.
1478	*/
1479	if ((ASIC_IS_R300(rdev) || (rdev->family == CHIP_RV515)) &&
1480	!(rdev->flags & RADEON_IS_IGP))
1481	rdev->disp_priority = 2;
1482	else
1483	rdev->disp_priority = 0;
1484	} else
1485	rdev->disp_priority = radeon_disp_priority;
1486
1487	}
1488
1489	/*
1490	* Allocate hdmi structs and determine register offsets
1491	*/
1492	static void radeon_afmt_init(struct radeon_device *rdev)
1493	{
1494	int i;
1495
1496	for (i = 0; i < RADEON_MAX_AFMT_BLOCKS; i++)
1497	rdev->mode_info.afmt[i] = NULL;
1498
1499	if (ASIC_IS_NODCE(rdev)) {
1500	/* nothing to do */
1501	} else if (ASIC_IS_DCE4(rdev)) {
1502	static uint32_t eg_offsets[] = {
1503	EVERGREEN_CRTC0_REGISTER_OFFSET,
1504	EVERGREEN_CRTC1_REGISTER_OFFSET,
1505	EVERGREEN_CRTC2_REGISTER_OFFSET,
1506	EVERGREEN_CRTC3_REGISTER_OFFSET,
1507	EVERGREEN_CRTC4_REGISTER_OFFSET,
1508	EVERGREEN_CRTC5_REGISTER_OFFSET,
1509	0x13830 - 0x7030,
1510	};
1511	int num_afmt;
1512
1513	/* DCE8 has 7 audio blocks tied to DIG encoders */
1514	/* DCE6 has 6 audio blocks tied to DIG encoders */
1515	/* DCE4/5 has 6 audio blocks tied to DIG encoders */
1516	/* DCE4.1 has 2 audio blocks tied to DIG encoders */
1517	if (ASIC_IS_DCE8(rdev))
1518	num_afmt = 7;
1519	else if (ASIC_IS_DCE6(rdev))
1520	num_afmt = 6;
1521	else if (ASIC_IS_DCE5(rdev))
1522	num_afmt = 6;
1523	else if (ASIC_IS_DCE41(rdev))
1524	num_afmt = 2;
1525	else /* DCE4 */
1526	num_afmt = 6;
1527
1528	BUG_ON(num_afmt > ARRAY_SIZE(eg_offsets));
1529	for (i = 0; i < num_afmt; i++) {
1530	rdev->mode_info.afmt[i] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL);
1531	if (rdev->mode_info.afmt[i]) {
1532	rdev->mode_info.afmt[i]->offset = eg_offsets[i];
1533	rdev->mode_info.afmt[i]->id = i;
1534	}
1535	}
1536	} else if (ASIC_IS_DCE3(rdev)) {
1537	/* DCE3.x has 2 audio blocks tied to DIG encoders */
1538	rdev->mode_info.afmt[0] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL);
1539	if (rdev->mode_info.afmt[0]) {
1540	rdev->mode_info.afmt[0]->offset = DCE3_HDMI_OFFSET0;
1541	rdev->mode_info.afmt[0]->id = 0;
1542	}
1543	rdev->mode_info.afmt[1] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL);
1544	if (rdev->mode_info.afmt[1]) {
1545	rdev->mode_info.afmt[1]->offset = DCE3_HDMI_OFFSET1;
1546	rdev->mode_info.afmt[1]->id = 1;
1547	}
1548	} else if (ASIC_IS_DCE2(rdev)) {
1549	/* DCE2 has at least 1 routable audio block */
1550	rdev->mode_info.afmt[0] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL);
1551	if (rdev->mode_info.afmt[0]) {
1552	rdev->mode_info.afmt[0]->offset = DCE2_HDMI_OFFSET0;
1553	rdev->mode_info.afmt[0]->id = 0;
1554	}
1555	/* r6xx has 2 routable audio blocks */
1556	if (rdev->family >= CHIP_R600) {
1557	rdev->mode_info.afmt[1] = kzalloc(sizeof(struct radeon_afmt), GFP_KERNEL);
1558	if (rdev->mode_info.afmt[1]) {
1559	rdev->mode_info.afmt[1]->offset = DCE2_HDMI_OFFSET1;
1560	rdev->mode_info.afmt[1]->id = 1;
1561	}
1562	}
1563	}
1564	}
1565
1566	static void radeon_afmt_fini(struct radeon_device *rdev)
1567	{
1568	int i;
1569
1570	for (i = 0; i < RADEON_MAX_AFMT_BLOCKS; i++) {
1571	kfree(objp: rdev->mode_info.afmt[i]);
1572	rdev->mode_info.afmt[i] = NULL;
1573	}
1574	}
1575
1576	int radeon_modeset_init(struct radeon_device *rdev)
1577	{
1578	int i;
1579	int ret;
1580
1581	drm_mode_config_init(dev: rdev_to_drm(rdev));
1582	rdev->mode_info.mode_config_initialized = true;
1583
1584	rdev_to_drm(rdev)->mode_config.funcs = &radeon_mode_funcs;
1585
1586	if (radeon_use_pflipirq == 2 && rdev->family >= CHIP_R600)
1587	rdev_to_drm(rdev)->mode_config.async_page_flip = true;
1588
1589	if (ASIC_IS_DCE5(rdev)) {
1590	rdev_to_drm(rdev)->mode_config.max_width = 16384;
1591	rdev_to_drm(rdev)->mode_config.max_height = 16384;
1592	} else if (ASIC_IS_AVIVO(rdev)) {
1593	rdev_to_drm(rdev)->mode_config.max_width = 8192;
1594	rdev_to_drm(rdev)->mode_config.max_height = 8192;
1595	} else {
1596	rdev_to_drm(rdev)->mode_config.max_width = 4096;
1597	rdev_to_drm(rdev)->mode_config.max_height = 4096;
1598	}
1599
1600	rdev_to_drm(rdev)->mode_config.preferred_depth = 24;
1601	rdev_to_drm(rdev)->mode_config.prefer_shadow = 1;
1602
1603	rdev_to_drm(rdev)->mode_config.fb_modifiers_not_supported = true;
1604
1605	ret = radeon_modeset_create_props(rdev);
1606	if (ret) {
1607	return ret;
1608	}
1609
1610	/* init i2c buses */
1611	radeon_i2c_init(rdev);
1612
1613	/* check combios for a valid hardcoded EDID - Sun servers */
1614	if (!rdev->is_atom_bios) {
1615	/* check for hardcoded EDID in BIOS */
1616	radeon_combios_check_hardcoded_edid(rdev);
1617	}
1618
1619	/* allocate crtcs */
1620	for (i = 0; i < rdev->num_crtc; i++) {
1621	radeon_crtc_init(dev: rdev_to_drm(rdev), index: i);
1622	}
1623
1624	/* okay we should have all the bios connectors */
1625	ret = radeon_setup_enc_conn(dev: rdev_to_drm(rdev));
1626	if (!ret) {
1627	return ret;
1628	}
1629
1630	/* init dig PHYs, disp eng pll */
1631	if (rdev->is_atom_bios) {
1632	radeon_atom_encoder_init(rdev);
1633	radeon_atom_disp_eng_pll_init(rdev);
1634	}
1635
1636	/* initialize hpd */
1637	radeon_hpd_init(rdev);
1638
1639	/* setup afmt */
1640	radeon_afmt_init(rdev);
1641
1642	drm_kms_helper_poll_init(dev: rdev_to_drm(rdev));
1643
1644	/* do pm late init */
1645	ret = radeon_pm_late_init(rdev);
1646
1647	return 0;
1648	}
1649
1650	void radeon_modeset_fini(struct radeon_device *rdev)
1651	{
1652	if (rdev->mode_info.mode_config_initialized) {
1653	drm_kms_helper_poll_fini(dev: rdev_to_drm(rdev));
1654	radeon_hpd_fini(rdev);
1655	drm_helper_force_disable_all(dev: rdev_to_drm(rdev));
1656	radeon_afmt_fini(rdev);
1657	drm_mode_config_cleanup(dev: rdev_to_drm(rdev));
1658	rdev->mode_info.mode_config_initialized = false;
1659	}
1660
1661	drm_edid_free(drm_edid: rdev->mode_info.bios_hardcoded_edid);
1662
1663	/* free i2c buses */
1664	radeon_i2c_fini(rdev);
1665	}
1666
1667	static bool is_hdtv_mode(const struct drm_display_mode *mode)
1668	{
1669	/* try and guess if this is a tv or a monitor */
1670	if ((mode->vdisplay == 480 && mode->hdisplay == 720) || /* 480p */
1671	(mode->vdisplay == 576) || /* 576p */
1672	(mode->vdisplay == 720) || /* 720p */
1673	(mode->vdisplay == 1080)) /* 1080p */
1674	return true;
1675	else
1676	return false;
1677	}
1678
1679	bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
1680	const struct drm_display_mode *mode,
1681	struct drm_display_mode *adjusted_mode)
1682	{
1683	struct drm_device *dev = crtc->dev;
1684	struct radeon_device *rdev = dev->dev_private;
1685	struct drm_encoder *encoder;
1686	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
1687	struct radeon_encoder *radeon_encoder;
1688	struct drm_connector *connector;
1689	bool first = true;
1690	u32 src_v = 1, dst_v = 1;
1691	u32 src_h = 1, dst_h = 1;
1692
1693	radeon_crtc->h_border = 0;
1694	radeon_crtc->v_border = 0;
1695
1696	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
1697	if (encoder->crtc != crtc)
1698	continue;
1699	radeon_encoder = to_radeon_encoder(encoder);
1700	connector = radeon_get_connector_for_encoder(encoder);
1701
1702	if (first) {
1703	/* set scaling */
1704	if (radeon_encoder->rmx_type == RMX_OFF)
1705	radeon_crtc->rmx_type = RMX_OFF;
1706	else if (mode->hdisplay < radeon_encoder->native_mode.hdisplay ||
1707	mode->vdisplay < radeon_encoder->native_mode.vdisplay)
1708	radeon_crtc->rmx_type = radeon_encoder->rmx_type;
1709	else
1710	radeon_crtc->rmx_type = RMX_OFF;
1711	/* copy native mode */
1712	memcpy(&radeon_crtc->native_mode,
1713	&radeon_encoder->native_mode,
1714	sizeof(struct drm_display_mode));
1715	src_v = crtc->mode.vdisplay;
1716	dst_v = radeon_crtc->native_mode.vdisplay;
1717	src_h = crtc->mode.hdisplay;
1718	dst_h = radeon_crtc->native_mode.hdisplay;
1719
1720	/* fix up for overscan on hdmi */
1721	if (ASIC_IS_AVIVO(rdev) &&
1722	(!(mode->flags & DRM_MODE_FLAG_INTERLACE)) &&
1723	((radeon_encoder->underscan_type == UNDERSCAN_ON) ||
1724	((radeon_encoder->underscan_type == UNDERSCAN_AUTO) &&
1725	connector->display_info.is_hdmi &&
1726	is_hdtv_mode(mode)))) {
1727	if (radeon_encoder->underscan_hborder != 0)
1728	radeon_crtc->h_border = radeon_encoder->underscan_hborder;
1729	else
1730	radeon_crtc->h_border = (mode->hdisplay >> 5) + 16;
1731	if (radeon_encoder->underscan_vborder != 0)
1732	radeon_crtc->v_border = radeon_encoder->underscan_vborder;
1733	else
1734	radeon_crtc->v_border = (mode->vdisplay >> 5) + 16;
1735	radeon_crtc->rmx_type = RMX_FULL;
1736	src_v = crtc->mode.vdisplay;
1737	dst_v = crtc->mode.vdisplay - (radeon_crtc->v_border * 2);
1738	src_h = crtc->mode.hdisplay;
1739	dst_h = crtc->mode.hdisplay - (radeon_crtc->h_border * 2);
1740	}
1741	first = false;
1742	} else {
1743	if (radeon_crtc->rmx_type != radeon_encoder->rmx_type) {
1744	/* WARNING: Right now this can't happen but
1745	* in the future we need to check that scaling
1746	* are consistent across different encoder
1747	* (ie all encoder can work with the same
1748	* scaling).
1749	*/
1750	DRM_ERROR("Scaling not consistent across encoder.\n");
1751	return false;
1752	}
1753	}
1754	}
1755	if (radeon_crtc->rmx_type != RMX_OFF) {
1756	fixed20_12 a, b;
1757	a.full = dfixed_const(src_v);
1758	b.full = dfixed_const(dst_v);
1759	radeon_crtc->vsc.full = dfixed_div(A: a, B: b);
1760	a.full = dfixed_const(src_h);
1761	b.full = dfixed_const(dst_h);
1762	radeon_crtc->hsc.full = dfixed_div(A: a, B: b);
1763	} else {
1764	radeon_crtc->vsc.full = dfixed_const(1);
1765	radeon_crtc->hsc.full = dfixed_const(1);
1766	}
1767	return true;
1768	}
1769
1770	/*
1771	* Retrieve current video scanout position of crtc on a given gpu, and
1772	* an optional accurate timestamp of when query happened.
1773	*
1774	* \param dev Device to query.
1775	* \param crtc Crtc to query.
1776	* \param flags Flags from caller (DRM_CALLED_FROM_VBLIRQ or 0).
1777	* For driver internal use only also supports these flags:
1778	*
1779	* USE_REAL_VBLANKSTART to use the real start of vblank instead
1780	* of a fudged earlier start of vblank.
1781	*
1782	* GET_DISTANCE_TO_VBLANKSTART to return distance to the
1783	* fudged earlier start of vblank in *vpos and the distance
1784	* to true start of vblank in *hpos.
1785	*
1786	* \param *vpos Location where vertical scanout position should be stored.
1787	* \param *hpos Location where horizontal scanout position should go.
1788	* \param *stime Target location for timestamp taken immediately before
1789	* scanout position query. Can be NULL to skip timestamp.
1790	* \param *etime Target location for timestamp taken immediately after
1791	* scanout position query. Can be NULL to skip timestamp.
1792	*
1793	* Returns vpos as a positive number while in active scanout area.
1794	* Returns vpos as a negative number inside vblank, counting the number
1795	* of scanlines to go until end of vblank, e.g., -1 means "one scanline
1796	* until start of active scanout / end of vblank."
1797	*
1798	* \return Flags, or'ed together as follows:
1799	*
1800	* DRM_SCANOUTPOS_VALID = Query successful.
1801	* DRM_SCANOUTPOS_INVBL = Inside vblank.
1802	* DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of
1803	* this flag means that returned position may be offset by a constant but
1804	* unknown small number of scanlines wrt. real scanout position.
1805	*
1806	*/
1807	int radeon_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
1808	unsigned int flags, int *vpos, int *hpos,
1809	ktime_t *stime, ktime_t *etime,
1810	const struct drm_display_mode *mode)
1811	{
1812	u32 stat_crtc = 0, vbl = 0, position = 0;
1813	int vbl_start, vbl_end, vtotal, ret = 0;
1814	bool in_vbl = true;
1815
1816	struct radeon_device *rdev = dev->dev_private;
1817
1818	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
1819
1820	/* Get optional system timestamp before query. */
1821	if (stime)
1822	*stime = ktime_get();
1823
1824	if (ASIC_IS_DCE4(rdev)) {
1825	if (pipe == 0) {
1826	vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1827	EVERGREEN_CRTC0_REGISTER_OFFSET);
1828	position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1829	EVERGREEN_CRTC0_REGISTER_OFFSET);
1830	ret |= DRM_SCANOUTPOS_VALID;
1831	}
1832	if (pipe == 1) {
1833	vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1834	EVERGREEN_CRTC1_REGISTER_OFFSET);
1835	position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1836	EVERGREEN_CRTC1_REGISTER_OFFSET);
1837	ret |= DRM_SCANOUTPOS_VALID;
1838	}
1839	if (pipe == 2) {
1840	vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1841	EVERGREEN_CRTC2_REGISTER_OFFSET);
1842	position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1843	EVERGREEN_CRTC2_REGISTER_OFFSET);
1844	ret |= DRM_SCANOUTPOS_VALID;
1845	}
1846	if (pipe == 3) {
1847	vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1848	EVERGREEN_CRTC3_REGISTER_OFFSET);
1849	position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1850	EVERGREEN_CRTC3_REGISTER_OFFSET);
1851	ret |= DRM_SCANOUTPOS_VALID;
1852	}
1853	if (pipe == 4) {
1854	vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1855	EVERGREEN_CRTC4_REGISTER_OFFSET);
1856	position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1857	EVERGREEN_CRTC4_REGISTER_OFFSET);
1858	ret |= DRM_SCANOUTPOS_VALID;
1859	}
1860	if (pipe == 5) {
1861	vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1862	EVERGREEN_CRTC5_REGISTER_OFFSET);
1863	position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1864	EVERGREEN_CRTC5_REGISTER_OFFSET);
1865	ret |= DRM_SCANOUTPOS_VALID;
1866	}
1867	} else if (ASIC_IS_AVIVO(rdev)) {
1868	if (pipe == 0) {
1869	vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END);
1870	position = RREG32(AVIVO_D1CRTC_STATUS_POSITION);
1871	ret |= DRM_SCANOUTPOS_VALID;
1872	}
1873	if (pipe == 1) {
1874	vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END);
1875	position = RREG32(AVIVO_D2CRTC_STATUS_POSITION);
1876	ret |= DRM_SCANOUTPOS_VALID;
1877	}
1878	} else {
1879	/* Pre-AVIVO: Different encoding of scanout pos and vblank interval. */
1880	if (pipe == 0) {
1881	/* Assume vbl_end == 0, get vbl_start from
1882	* upper 16 bits.
1883	*/
1884	vbl = (RREG32(RADEON_CRTC_V_TOTAL_DISP) &
1885	RADEON_CRTC_V_DISP) >> RADEON_CRTC_V_DISP_SHIFT;
1886	/* Only retrieve vpos from upper 16 bits, set hpos == 0. */
1887	position = (RREG32(RADEON_CRTC_VLINE_CRNT_VLINE) >> 16) & RADEON_CRTC_V_TOTAL;
1888	stat_crtc = RREG32(RADEON_CRTC_STATUS);
1889	if (!(stat_crtc & 1))
1890	in_vbl = false;
1891
1892	ret |= DRM_SCANOUTPOS_VALID;
1893	}
1894	if (pipe == 1) {
1895	vbl = (RREG32(RADEON_CRTC2_V_TOTAL_DISP) &
1896	RADEON_CRTC_V_DISP) >> RADEON_CRTC_V_DISP_SHIFT;
1897	position = (RREG32(RADEON_CRTC2_VLINE_CRNT_VLINE) >> 16) & RADEON_CRTC_V_TOTAL;
1898	stat_crtc = RREG32(RADEON_CRTC2_STATUS);
1899	if (!(stat_crtc & 1))
1900	in_vbl = false;
1901
1902	ret |= DRM_SCANOUTPOS_VALID;
1903	}
1904	}
1905
1906	/* Get optional system timestamp after query. */
1907	if (etime)
1908	*etime = ktime_get();
1909
1910	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
1911
1912	/* Decode into vertical and horizontal scanout position. */
1913	*vpos = position & 0x1fff;
1914	*hpos = (position >> 16) & 0x1fff;
1915
1916	/* Valid vblank area boundaries from gpu retrieved? */
1917	if (vbl > 0) {
1918	/* Yes: Decode. */
1919	ret |= DRM_SCANOUTPOS_ACCURATE;
1920	vbl_start = vbl & 0x1fff;
1921	vbl_end = (vbl >> 16) & 0x1fff;
1922	}
1923	else {
1924	/* No: Fake something reasonable which gives at least ok results. */
1925	vbl_start = mode->crtc_vdisplay;
1926	vbl_end = 0;
1927	}
1928
1929	/* Called from driver internal vblank counter query code? */
1930	if (flags & GET_DISTANCE_TO_VBLANKSTART) {
1931	/* Caller wants distance from real vbl_start in *hpos */
1932	*hpos = *vpos - vbl_start;
1933	}
1934
1935	/* Fudge vblank to start a few scanlines earlier to handle the
1936	* problem that vblank irqs fire a few scanlines before start
1937	* of vblank. Some driver internal callers need the true vblank
1938	* start to be used and signal this via the USE_REAL_VBLANKSTART flag.
1939	*
1940	* The cause of the "early" vblank irq is that the irq is triggered
1941	* by the line buffer logic when the line buffer read position enters
1942	* the vblank, whereas our crtc scanout position naturally lags the
1943	* line buffer read position.
1944	*/
1945	if (!(flags & USE_REAL_VBLANKSTART))
1946	vbl_start -= rdev->mode_info.crtcs[pipe]->lb_vblank_lead_lines;
1947
1948	/* Test scanout position against vblank region. */
1949	if ((*vpos < vbl_start) && (*vpos >= vbl_end))
1950	in_vbl = false;
1951
1952	/* In vblank? */
1953	if (in_vbl)
1954	ret |= DRM_SCANOUTPOS_IN_VBLANK;
1955
1956	/* Called from driver internal vblank counter query code? */
1957	if (flags & GET_DISTANCE_TO_VBLANKSTART) {
1958	/* Caller wants distance from fudged earlier vbl_start */
1959	*vpos -= vbl_start;
1960	return ret;
1961	}
1962
1963	/* Check if inside vblank area and apply corrective offsets:
1964	* vpos will then be >=0 in video scanout area, but negative
1965	* within vblank area, counting down the number of lines until
1966	* start of scanout.
1967	*/
1968
1969	/* Inside "upper part" of vblank area? Apply corrective offset if so: */
1970	if (in_vbl && (*vpos >= vbl_start)) {
1971	vtotal = mode->crtc_vtotal;
1972	*vpos = *vpos - vtotal;
1973	}
1974
1975	/* Correct for shifted end of vbl at vbl_end. */
1976	*vpos = *vpos - vbl_end;
1977
1978	return ret;
1979	}
1980
1981	bool
1982	radeon_get_crtc_scanout_position(struct drm_crtc *crtc,
1983	bool in_vblank_irq, int *vpos, int *hpos,
1984	ktime_t *stime, ktime_t *etime,
1985	const struct drm_display_mode *mode)
1986	{
1987	struct drm_device *dev = crtc->dev;
1988	unsigned int pipe = crtc->index;
1989
1990	return radeon_get_crtc_scanoutpos(dev, pipe, flags: 0, vpos, hpos,
1991	stime, etime, mode);
1992	}
1993