1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2016 Broadcom
4	*/
5
6	/**
7	* DOC: VC4 DSI0/DSI1 module
8	*
9	* BCM2835 contains two DSI modules, DSI0 and DSI1. DSI0 is a
10	* single-lane DSI controller, while DSI1 is a more modern 4-lane DSI
11	* controller.
12	*
13	* Most Raspberry Pi boards expose DSI1 as their "DISPLAY" connector,
14	* while the compute module brings both DSI0 and DSI1 out.
15	*
16	* This driver has been tested for DSI1 video-mode display only
17	* currently, with most of the information necessary for DSI0
18	* hopefully present.
19	*/
20
21	#include <linux/clk-provider.h>
22	#include <linux/clk.h>
23	#include <linux/completion.h>
24	#include <linux/component.h>
25	#include <linux/dma-mapping.h>
26	#include <linux/dmaengine.h>
27	#include <linux/io.h>
28	#include <linux/of.h>
29	#include <linux/of_address.h>
30	#include <linux/platform_device.h>
31	#include <linux/pm_runtime.h>
32
33	#include <drm/drm_atomic_helper.h>
34	#include <drm/drm_bridge.h>
35	#include <drm/drm_edid.h>
36	#include <drm/drm_mipi_dsi.h>
37	#include <drm/drm_of.h>
38	#include <drm/drm_panel.h>
39	#include <drm/drm_print.h>
40	#include <drm/drm_probe_helper.h>
41	#include <drm/drm_simple_kms_helper.h>
42
43	#include "vc4_drv.h"
44	#include "vc4_regs.h"
45
46	#define DSI_CMD_FIFO_DEPTH 16
47	#define DSI_PIX_FIFO_DEPTH 256
48	#define DSI_PIX_FIFO_WIDTH 4
49
50	#define DSI0_CTRL 0x00
51
52	/* Command packet control. */
53	#define DSI0_TXPKT1C 0x04 /* AKA PKTC */
54	#define DSI1_TXPKT1C 0x04
55	# define DSI_TXPKT1C_TRIG_CMD_MASK VC4_MASK(31, 24)
56	# define DSI_TXPKT1C_TRIG_CMD_SHIFT 24
57	# define DSI_TXPKT1C_CMD_REPEAT_MASK VC4_MASK(23, 10)
58	# define DSI_TXPKT1C_CMD_REPEAT_SHIFT 10
59
60	# define DSI_TXPKT1C_DISPLAY_NO_MASK VC4_MASK(9, 8)
61	# define DSI_TXPKT1C_DISPLAY_NO_SHIFT 8
62	/* Short, trigger, BTA, or a long packet that fits all in CMDFIFO. */
63	# define DSI_TXPKT1C_DISPLAY_NO_SHORT 0
64	/* Primary display where cmdfifo provides part of the payload and
65	* pixelvalve the rest.
66	*/
67	# define DSI_TXPKT1C_DISPLAY_NO_PRIMARY 1
68	/* Secondary display where cmdfifo provides part of the payload and
69	* pixfifo the rest.
70	*/
71	# define DSI_TXPKT1C_DISPLAY_NO_SECONDARY 2
72
73	# define DSI_TXPKT1C_CMD_TX_TIME_MASK VC4_MASK(7, 6)
74	# define DSI_TXPKT1C_CMD_TX_TIME_SHIFT 6
75
76	# define DSI_TXPKT1C_CMD_CTRL_MASK VC4_MASK(5, 4)
77	# define DSI_TXPKT1C_CMD_CTRL_SHIFT 4
78	/* Command only. Uses TXPKT1H and DISPLAY_NO */
79	# define DSI_TXPKT1C_CMD_CTRL_TX 0
80	/* Command with BTA for either ack or read data. */
81	# define DSI_TXPKT1C_CMD_CTRL_RX 1
82	/* Trigger according to TRIG_CMD */
83	# define DSI_TXPKT1C_CMD_CTRL_TRIG 2
84	/* BTA alone for getting error status after a command, or a TE trigger
85	* without a previous command.
86	*/
87	# define DSI_TXPKT1C_CMD_CTRL_BTA 3
88
89	# define DSI_TXPKT1C_CMD_MODE_LP BIT(3)
90	# define DSI_TXPKT1C_CMD_TYPE_LONG BIT(2)
91	# define DSI_TXPKT1C_CMD_TE_EN BIT(1)
92	# define DSI_TXPKT1C_CMD_EN BIT(0)
93
94	/* Command packet header. */
95	#define DSI0_TXPKT1H 0x08 /* AKA PKTH */
96	#define DSI1_TXPKT1H 0x08
97	# define DSI_TXPKT1H_BC_CMDFIFO_MASK VC4_MASK(31, 24)
98	# define DSI_TXPKT1H_BC_CMDFIFO_SHIFT 24
99	# define DSI_TXPKT1H_BC_PARAM_MASK VC4_MASK(23, 8)
100	# define DSI_TXPKT1H_BC_PARAM_SHIFT 8
101	# define DSI_TXPKT1H_BC_DT_MASK VC4_MASK(7, 0)
102	# define DSI_TXPKT1H_BC_DT_SHIFT 0
103
104	#define DSI0_RXPKT1H 0x0c /* AKA RX1_PKTH */
105	#define DSI1_RXPKT1H 0x14
106	# define DSI_RXPKT1H_CRC_ERR BIT(31)
107	# define DSI_RXPKT1H_DET_ERR BIT(30)
108	# define DSI_RXPKT1H_ECC_ERR BIT(29)
109	# define DSI_RXPKT1H_COR_ERR BIT(28)
110	# define DSI_RXPKT1H_INCOMP_PKT BIT(25)
111	# define DSI_RXPKT1H_PKT_TYPE_LONG BIT(24)
112	/* Byte count if DSI_RXPKT1H_PKT_TYPE_LONG */
113	# define DSI_RXPKT1H_BC_PARAM_MASK VC4_MASK(23, 8)
114	# define DSI_RXPKT1H_BC_PARAM_SHIFT 8
115	/* Short return bytes if !DSI_RXPKT1H_PKT_TYPE_LONG */
116	# define DSI_RXPKT1H_SHORT_1_MASK VC4_MASK(23, 16)
117	# define DSI_RXPKT1H_SHORT_1_SHIFT 16
118	# define DSI_RXPKT1H_SHORT_0_MASK VC4_MASK(15, 8)
119	# define DSI_RXPKT1H_SHORT_0_SHIFT 8
120	# define DSI_RXPKT1H_DT_LP_CMD_MASK VC4_MASK(7, 0)
121	# define DSI_RXPKT1H_DT_LP_CMD_SHIFT 0
122
123	#define DSI0_RXPKT2H 0x10 /* AKA RX2_PKTH */
124	#define DSI1_RXPKT2H 0x18
125	# define DSI_RXPKT1H_DET_ERR BIT(30)
126	# define DSI_RXPKT1H_ECC_ERR BIT(29)
127	# define DSI_RXPKT1H_COR_ERR BIT(28)
128	# define DSI_RXPKT1H_INCOMP_PKT BIT(25)
129	# define DSI_RXPKT1H_BC_PARAM_MASK VC4_MASK(23, 8)
130	# define DSI_RXPKT1H_BC_PARAM_SHIFT 8
131	# define DSI_RXPKT1H_DT_MASK VC4_MASK(7, 0)
132	# define DSI_RXPKT1H_DT_SHIFT 0
133
134	#define DSI0_TXPKT_CMD_FIFO 0x14 /* AKA CMD_DATAF */
135	#define DSI1_TXPKT_CMD_FIFO 0x1c
136
137	#define DSI0_DISP0_CTRL 0x18
138	# define DSI_DISP0_PIX_CLK_DIV_MASK VC4_MASK(21, 13)
139	# define DSI_DISP0_PIX_CLK_DIV_SHIFT 13
140	# define DSI_DISP0_LP_STOP_CTRL_MASK VC4_MASK(12, 11)
141	# define DSI_DISP0_LP_STOP_CTRL_SHIFT 11
142	# define DSI_DISP0_LP_STOP_DISABLE 0
143	# define DSI_DISP0_LP_STOP_PERLINE 1
144	# define DSI_DISP0_LP_STOP_PERFRAME 2
145
146	/* Transmit RGB pixels and null packets only during HACTIVE, instead
147	* of going to LP-STOP.
148	*/
149	# define DSI_DISP_HACTIVE_NULL BIT(10)
150	/* Transmit blanking packet only during vblank, instead of allowing LP-STOP. */
151	# define DSI_DISP_VBLP_CTRL BIT(9)
152	/* Transmit blanking packet only during HFP, instead of allowing LP-STOP. */
153	# define DSI_DISP_HFP_CTRL BIT(8)
154	/* Transmit blanking packet only during HBP, instead of allowing LP-STOP. */
155	# define DSI_DISP_HBP_CTRL BIT(7)
156	# define DSI_DISP0_CHANNEL_MASK VC4_MASK(6, 5)
157	# define DSI_DISP0_CHANNEL_SHIFT 5
158	/* Enables end events for HSYNC/VSYNC, not just start events. */
159	# define DSI_DISP0_ST_END BIT(4)
160	# define DSI_DISP0_PFORMAT_MASK VC4_MASK(3, 2)
161	# define DSI_DISP0_PFORMAT_SHIFT 2
162	# define DSI_PFORMAT_RGB565 0
163	# define DSI_PFORMAT_RGB666_PACKED 1
164	# define DSI_PFORMAT_RGB666 2
165	# define DSI_PFORMAT_RGB888 3
166	/* Default is VIDEO mode. */
167	# define DSI_DISP0_COMMAND_MODE BIT(1)
168	# define DSI_DISP0_ENABLE BIT(0)
169
170	#define DSI0_DISP1_CTRL 0x1c
171	#define DSI1_DISP1_CTRL 0x2c
172	/* Format of the data written to TXPKT_PIX_FIFO. */
173	# define DSI_DISP1_PFORMAT_MASK VC4_MASK(2, 1)
174	# define DSI_DISP1_PFORMAT_SHIFT 1
175	# define DSI_DISP1_PFORMAT_16BIT 0
176	# define DSI_DISP1_PFORMAT_24BIT 1
177	# define DSI_DISP1_PFORMAT_32BIT_LE 2
178	# define DSI_DISP1_PFORMAT_32BIT_BE 3
179
180	/* DISP1 is always command mode. */
181	# define DSI_DISP1_ENABLE BIT(0)
182
183	#define DSI0_TXPKT_PIX_FIFO 0x20 /* AKA PIX_FIFO */
184
185	#define DSI0_INT_STAT 0x24
186	#define DSI0_INT_EN 0x28
187	# define DSI0_INT_FIFO_ERR BIT(25)
188	# define DSI0_INT_CMDC_DONE_MASK VC4_MASK(24, 23)
189	# define DSI0_INT_CMDC_DONE_SHIFT 23
190	# define DSI0_INT_CMDC_DONE_NO_REPEAT 1
191	# define DSI0_INT_CMDC_DONE_REPEAT 3
192	# define DSI0_INT_PHY_DIR_RTF BIT(22)
193	# define DSI0_INT_PHY_D1_ULPS BIT(21)
194	# define DSI0_INT_PHY_D1_STOP BIT(20)
195	# define DSI0_INT_PHY_RXLPDT BIT(19)
196	# define DSI0_INT_PHY_RXTRIG BIT(18)
197	# define DSI0_INT_PHY_D0_ULPS BIT(17)
198	# define DSI0_INT_PHY_D0_LPDT BIT(16)
199	# define DSI0_INT_PHY_D0_FTR BIT(15)
200	# define DSI0_INT_PHY_D0_STOP BIT(14)
201	/* Signaled when the clock lane enters the given state. */
202	# define DSI0_INT_PHY_CLK_ULPS BIT(13)
203	# define DSI0_INT_PHY_CLK_HS BIT(12)
204	# define DSI0_INT_PHY_CLK_FTR BIT(11)
205	/* Signaled on timeouts */
206	# define DSI0_INT_PR_TO BIT(10)
207	# define DSI0_INT_TA_TO BIT(9)
208	# define DSI0_INT_LPRX_TO BIT(8)
209	# define DSI0_INT_HSTX_TO BIT(7)
210	/* Contention on a line when trying to drive the line low */
211	# define DSI0_INT_ERR_CONT_LP1 BIT(6)
212	# define DSI0_INT_ERR_CONT_LP0 BIT(5)
213	/* Control error: incorrect line state sequence on data lane 0. */
214	# define DSI0_INT_ERR_CONTROL BIT(4)
215	# define DSI0_INT_ERR_SYNC_ESC BIT(3)
216	# define DSI0_INT_RX2_PKT BIT(2)
217	# define DSI0_INT_RX1_PKT BIT(1)
218	# define DSI0_INT_CMD_PKT BIT(0)
219
220	#define DSI0_INTERRUPTS_ALWAYS_ENABLED (DSI0_INT_ERR_SYNC_ESC | \
221	DSI0_INT_ERR_CONTROL | \
222	DSI0_INT_ERR_CONT_LP0 | \
223	DSI0_INT_ERR_CONT_LP1 | \
224	DSI0_INT_HSTX_TO | \
225	DSI0_INT_LPRX_TO | \
226	DSI0_INT_TA_TO | \
227	DSI0_INT_PR_TO)
228
229	# define DSI1_INT_PHY_D3_ULPS BIT(30)
230	# define DSI1_INT_PHY_D3_STOP BIT(29)
231	# define DSI1_INT_PHY_D2_ULPS BIT(28)
232	# define DSI1_INT_PHY_D2_STOP BIT(27)
233	# define DSI1_INT_PHY_D1_ULPS BIT(26)
234	# define DSI1_INT_PHY_D1_STOP BIT(25)
235	# define DSI1_INT_PHY_D0_ULPS BIT(24)
236	# define DSI1_INT_PHY_D0_STOP BIT(23)
237	# define DSI1_INT_FIFO_ERR BIT(22)
238	# define DSI1_INT_PHY_DIR_RTF BIT(21)
239	# define DSI1_INT_PHY_RXLPDT BIT(20)
240	# define DSI1_INT_PHY_RXTRIG BIT(19)
241	# define DSI1_INT_PHY_D0_LPDT BIT(18)
242	# define DSI1_INT_PHY_DIR_FTR BIT(17)
243
244	/* Signaled when the clock lane enters the given state. */
245	# define DSI1_INT_PHY_CLOCK_ULPS BIT(16)
246	# define DSI1_INT_PHY_CLOCK_HS BIT(15)
247	# define DSI1_INT_PHY_CLOCK_STOP BIT(14)
248
249	/* Signaled on timeouts */
250	# define DSI1_INT_PR_TO BIT(13)
251	# define DSI1_INT_TA_TO BIT(12)
252	# define DSI1_INT_LPRX_TO BIT(11)
253	# define DSI1_INT_HSTX_TO BIT(10)
254
255	/* Contention on a line when trying to drive the line low */
256	# define DSI1_INT_ERR_CONT_LP1 BIT(9)
257	# define DSI1_INT_ERR_CONT_LP0 BIT(8)
258
259	/* Control error: incorrect line state sequence on data lane 0. */
260	# define DSI1_INT_ERR_CONTROL BIT(7)
261	/* LPDT synchronization error (bits received not a multiple of 8. */
262
263	# define DSI1_INT_ERR_SYNC_ESC BIT(6)
264	/* Signaled after receiving an error packet from the display in
265	* response to a read.
266	*/
267	# define DSI1_INT_RXPKT2 BIT(5)
268	/* Signaled after receiving a packet. The header and optional short
269	* response will be in RXPKT1H, and a long response will be in the
270	* RXPKT_FIFO.
271	*/
272	# define DSI1_INT_RXPKT1 BIT(4)
273	# define DSI1_INT_TXPKT2_DONE BIT(3)
274	# define DSI1_INT_TXPKT2_END BIT(2)
275	/* Signaled after all repeats of TXPKT1 are transferred. */
276	# define DSI1_INT_TXPKT1_DONE BIT(1)
277	/* Signaled after each TXPKT1 repeat is scheduled. */
278	# define DSI1_INT_TXPKT1_END BIT(0)
279
280	#define DSI1_INTERRUPTS_ALWAYS_ENABLED (DSI1_INT_ERR_SYNC_ESC | \
281	DSI1_INT_ERR_CONTROL | \
282	DSI1_INT_ERR_CONT_LP0 | \
283	DSI1_INT_ERR_CONT_LP1 | \
284	DSI1_INT_HSTX_TO | \
285	DSI1_INT_LPRX_TO | \
286	DSI1_INT_TA_TO | \
287	DSI1_INT_PR_TO)
288
289	#define DSI0_STAT 0x2c
290	#define DSI0_HSTX_TO_CNT 0x30
291	#define DSI0_LPRX_TO_CNT 0x34
292	#define DSI0_TA_TO_CNT 0x38
293	#define DSI0_PR_TO_CNT 0x3c
294	#define DSI0_PHYC 0x40
295	# define DSI1_PHYC_ESC_CLK_LPDT_MASK VC4_MASK(25, 20)
296	# define DSI1_PHYC_ESC_CLK_LPDT_SHIFT 20
297	# define DSI1_PHYC_HS_CLK_CONTINUOUS BIT(18)
298	# define DSI0_PHYC_ESC_CLK_LPDT_MASK VC4_MASK(17, 12)
299	# define DSI0_PHYC_ESC_CLK_LPDT_SHIFT 12
300	# define DSI1_PHYC_CLANE_ULPS BIT(17)
301	# define DSI1_PHYC_CLANE_ENABLE BIT(16)
302	# define DSI_PHYC_DLANE3_ULPS BIT(13)
303	# define DSI_PHYC_DLANE3_ENABLE BIT(12)
304	# define DSI0_PHYC_HS_CLK_CONTINUOUS BIT(10)
305	# define DSI0_PHYC_CLANE_ULPS BIT(9)
306	# define DSI_PHYC_DLANE2_ULPS BIT(9)
307	# define DSI0_PHYC_CLANE_ENABLE BIT(8)
308	# define DSI_PHYC_DLANE2_ENABLE BIT(8)
309	# define DSI_PHYC_DLANE1_ULPS BIT(5)
310	# define DSI_PHYC_DLANE1_ENABLE BIT(4)
311	# define DSI_PHYC_DLANE0_FORCE_STOP BIT(2)
312	# define DSI_PHYC_DLANE0_ULPS BIT(1)
313	# define DSI_PHYC_DLANE0_ENABLE BIT(0)
314
315	#define DSI0_HS_CLT0 0x44
316	#define DSI0_HS_CLT1 0x48
317	#define DSI0_HS_CLT2 0x4c
318	#define DSI0_HS_DLT3 0x50
319	#define DSI0_HS_DLT4 0x54
320	#define DSI0_HS_DLT5 0x58
321	#define DSI0_HS_DLT6 0x5c
322	#define DSI0_HS_DLT7 0x60
323
324	#define DSI0_PHY_AFEC0 0x64
325	# define DSI0_PHY_AFEC0_DDR2CLK_EN BIT(26)
326	# define DSI0_PHY_AFEC0_DDRCLK_EN BIT(25)
327	# define DSI0_PHY_AFEC0_LATCH_ULPS BIT(24)
328	# define DSI1_PHY_AFEC0_IDR_DLANE3_MASK VC4_MASK(31, 29)
329	# define DSI1_PHY_AFEC0_IDR_DLANE3_SHIFT 29
330	# define DSI1_PHY_AFEC0_IDR_DLANE2_MASK VC4_MASK(28, 26)
331	# define DSI1_PHY_AFEC0_IDR_DLANE2_SHIFT 26
332	# define DSI1_PHY_AFEC0_IDR_DLANE1_MASK VC4_MASK(27, 23)
333	# define DSI1_PHY_AFEC0_IDR_DLANE1_SHIFT 23
334	# define DSI1_PHY_AFEC0_IDR_DLANE0_MASK VC4_MASK(22, 20)
335	# define DSI1_PHY_AFEC0_IDR_DLANE0_SHIFT 20
336	# define DSI1_PHY_AFEC0_IDR_CLANE_MASK VC4_MASK(19, 17)
337	# define DSI1_PHY_AFEC0_IDR_CLANE_SHIFT 17
338	# define DSI0_PHY_AFEC0_ACTRL_DLANE1_MASK VC4_MASK(23, 20)
339	# define DSI0_PHY_AFEC0_ACTRL_DLANE1_SHIFT 20
340	# define DSI0_PHY_AFEC0_ACTRL_DLANE0_MASK VC4_MASK(19, 16)
341	# define DSI0_PHY_AFEC0_ACTRL_DLANE0_SHIFT 16
342	# define DSI0_PHY_AFEC0_ACTRL_CLANE_MASK VC4_MASK(15, 12)
343	# define DSI0_PHY_AFEC0_ACTRL_CLANE_SHIFT 12
344	# define DSI1_PHY_AFEC0_DDR2CLK_EN BIT(16)
345	# define DSI1_PHY_AFEC0_DDRCLK_EN BIT(15)
346	# define DSI1_PHY_AFEC0_LATCH_ULPS BIT(14)
347	# define DSI1_PHY_AFEC0_RESET BIT(13)
348	# define DSI1_PHY_AFEC0_PD BIT(12)
349	# define DSI0_PHY_AFEC0_RESET BIT(11)
350	# define DSI1_PHY_AFEC0_PD_BG BIT(11)
351	# define DSI0_PHY_AFEC0_PD BIT(10)
352	# define DSI1_PHY_AFEC0_PD_DLANE1 BIT(10)
353	# define DSI0_PHY_AFEC0_PD_BG BIT(9)
354	# define DSI1_PHY_AFEC0_PD_DLANE2 BIT(9)
355	# define DSI0_PHY_AFEC0_PD_DLANE1 BIT(8)
356	# define DSI1_PHY_AFEC0_PD_DLANE3 BIT(8)
357	# define DSI_PHY_AFEC0_PTATADJ_MASK VC4_MASK(7, 4)
358	# define DSI_PHY_AFEC0_PTATADJ_SHIFT 4
359	# define DSI_PHY_AFEC0_CTATADJ_MASK VC4_MASK(3, 0)
360	# define DSI_PHY_AFEC0_CTATADJ_SHIFT 0
361
362	#define DSI0_PHY_AFEC1 0x68
363	# define DSI0_PHY_AFEC1_IDR_DLANE1_MASK VC4_MASK(10, 8)
364	# define DSI0_PHY_AFEC1_IDR_DLANE1_SHIFT 8
365	# define DSI0_PHY_AFEC1_IDR_DLANE0_MASK VC4_MASK(6, 4)
366	# define DSI0_PHY_AFEC1_IDR_DLANE0_SHIFT 4
367	# define DSI0_PHY_AFEC1_IDR_CLANE_MASK VC4_MASK(2, 0)
368	# define DSI0_PHY_AFEC1_IDR_CLANE_SHIFT 0
369
370	#define DSI0_TST_SEL 0x6c
371	#define DSI0_TST_MON 0x70
372	#define DSI0_ID 0x74
373	# define DSI_ID_VALUE 0x00647369
374
375	#define DSI1_CTRL 0x00
376	# define DSI_CTRL_HS_CLKC_MASK VC4_MASK(15, 14)
377	# define DSI_CTRL_HS_CLKC_SHIFT 14
378	# define DSI_CTRL_HS_CLKC_BYTE 0
379	# define DSI_CTRL_HS_CLKC_DDR2 1
380	# define DSI_CTRL_HS_CLKC_DDR 2
381
382	# define DSI_CTRL_RX_LPDT_EOT_DISABLE BIT(13)
383	# define DSI_CTRL_LPDT_EOT_DISABLE BIT(12)
384	# define DSI_CTRL_HSDT_EOT_DISABLE BIT(11)
385	# define DSI_CTRL_SOFT_RESET_CFG BIT(10)
386	# define DSI_CTRL_CAL_BYTE BIT(9)
387	# define DSI_CTRL_INV_BYTE BIT(8)
388	# define DSI_CTRL_CLR_LDF BIT(7)
389	# define DSI0_CTRL_CLR_PBCF BIT(6)
390	# define DSI1_CTRL_CLR_RXF BIT(6)
391	# define DSI0_CTRL_CLR_CPBCF BIT(5)
392	# define DSI1_CTRL_CLR_PDF BIT(5)
393	# define DSI0_CTRL_CLR_PDF BIT(4)
394	# define DSI1_CTRL_CLR_CDF BIT(4)
395	# define DSI0_CTRL_CLR_CDF BIT(3)
396	# define DSI0_CTRL_CTRL2 BIT(2)
397	# define DSI1_CTRL_DISABLE_DISP_CRCC BIT(2)
398	# define DSI0_CTRL_CTRL1 BIT(1)
399	# define DSI1_CTRL_DISABLE_DISP_ECCC BIT(1)
400	# define DSI0_CTRL_CTRL0 BIT(0)
401	# define DSI1_CTRL_EN BIT(0)
402	# define DSI0_CTRL_RESET_FIFOS (DSI_CTRL_CLR_LDF | \
403	DSI0_CTRL_CLR_PBCF | \
404	DSI0_CTRL_CLR_CPBCF | \
405	DSI0_CTRL_CLR_PDF | \
406	DSI0_CTRL_CLR_CDF)
407	# define DSI1_CTRL_RESET_FIFOS (DSI_CTRL_CLR_LDF | \
408	DSI1_CTRL_CLR_RXF | \
409	DSI1_CTRL_CLR_PDF | \
410	DSI1_CTRL_CLR_CDF)
411
412	#define DSI1_TXPKT2C 0x0c
413	#define DSI1_TXPKT2H 0x10
414	#define DSI1_TXPKT_PIX_FIFO 0x20
415	#define DSI1_RXPKT_FIFO 0x24
416	#define DSI1_DISP0_CTRL 0x28
417	#define DSI1_INT_STAT 0x30
418	#define DSI1_INT_EN 0x34
419	/* State reporting bits. These mostly behave like INT_STAT, where
420	* writing a 1 clears the bit.
421	*/
422	#define DSI1_STAT 0x38
423	# define DSI1_STAT_PHY_D3_ULPS BIT(31)
424	# define DSI1_STAT_PHY_D3_STOP BIT(30)
425	# define DSI1_STAT_PHY_D2_ULPS BIT(29)
426	# define DSI1_STAT_PHY_D2_STOP BIT(28)
427	# define DSI1_STAT_PHY_D1_ULPS BIT(27)
428	# define DSI1_STAT_PHY_D1_STOP BIT(26)
429	# define DSI1_STAT_PHY_D0_ULPS BIT(25)
430	# define DSI1_STAT_PHY_D0_STOP BIT(24)
431	# define DSI1_STAT_FIFO_ERR BIT(23)
432	# define DSI1_STAT_PHY_RXLPDT BIT(22)
433	# define DSI1_STAT_PHY_RXTRIG BIT(21)
434	# define DSI1_STAT_PHY_D0_LPDT BIT(20)
435	/* Set when in forward direction */
436	# define DSI1_STAT_PHY_DIR BIT(19)
437	# define DSI1_STAT_PHY_CLOCK_ULPS BIT(18)
438	# define DSI1_STAT_PHY_CLOCK_HS BIT(17)
439	# define DSI1_STAT_PHY_CLOCK_STOP BIT(16)
440	# define DSI1_STAT_PR_TO BIT(15)
441	# define DSI1_STAT_TA_TO BIT(14)
442	# define DSI1_STAT_LPRX_TO BIT(13)
443	# define DSI1_STAT_HSTX_TO BIT(12)
444	# define DSI1_STAT_ERR_CONT_LP1 BIT(11)
445	# define DSI1_STAT_ERR_CONT_LP0 BIT(10)
446	# define DSI1_STAT_ERR_CONTROL BIT(9)
447	# define DSI1_STAT_ERR_SYNC_ESC BIT(8)
448	# define DSI1_STAT_RXPKT2 BIT(7)
449	# define DSI1_STAT_RXPKT1 BIT(6)
450	# define DSI1_STAT_TXPKT2_BUSY BIT(5)
451	# define DSI1_STAT_TXPKT2_DONE BIT(4)
452	# define DSI1_STAT_TXPKT2_END BIT(3)
453	# define DSI1_STAT_TXPKT1_BUSY BIT(2)
454	# define DSI1_STAT_TXPKT1_DONE BIT(1)
455	# define DSI1_STAT_TXPKT1_END BIT(0)
456
457	#define DSI1_HSTX_TO_CNT 0x3c
458	#define DSI1_LPRX_TO_CNT 0x40
459	#define DSI1_TA_TO_CNT 0x44
460	#define DSI1_PR_TO_CNT 0x48
461	#define DSI1_PHYC 0x4c
462
463	#define DSI1_HS_CLT0 0x50
464	# define DSI_HS_CLT0_CZERO_MASK VC4_MASK(26, 18)
465	# define DSI_HS_CLT0_CZERO_SHIFT 18
466	# define DSI_HS_CLT0_CPRE_MASK VC4_MASK(17, 9)
467	# define DSI_HS_CLT0_CPRE_SHIFT 9
468	# define DSI_HS_CLT0_CPREP_MASK VC4_MASK(8, 0)
469	# define DSI_HS_CLT0_CPREP_SHIFT 0
470
471	#define DSI1_HS_CLT1 0x54
472	# define DSI_HS_CLT1_CTRAIL_MASK VC4_MASK(17, 9)
473	# define DSI_HS_CLT1_CTRAIL_SHIFT 9
474	# define DSI_HS_CLT1_CPOST_MASK VC4_MASK(8, 0)
475	# define DSI_HS_CLT1_CPOST_SHIFT 0
476
477	#define DSI1_HS_CLT2 0x58
478	# define DSI_HS_CLT2_WUP_MASK VC4_MASK(23, 0)
479	# define DSI_HS_CLT2_WUP_SHIFT 0
480
481	#define DSI1_HS_DLT3 0x5c
482	# define DSI_HS_DLT3_EXIT_MASK VC4_MASK(26, 18)
483	# define DSI_HS_DLT3_EXIT_SHIFT 18
484	# define DSI_HS_DLT3_ZERO_MASK VC4_MASK(17, 9)
485	# define DSI_HS_DLT3_ZERO_SHIFT 9
486	# define DSI_HS_DLT3_PRE_MASK VC4_MASK(8, 0)
487	# define DSI_HS_DLT3_PRE_SHIFT 0
488
489	#define DSI1_HS_DLT4 0x60
490	# define DSI_HS_DLT4_ANLAT_MASK VC4_MASK(22, 18)
491	# define DSI_HS_DLT4_ANLAT_SHIFT 18
492	# define DSI_HS_DLT4_TRAIL_MASK VC4_MASK(17, 9)
493	# define DSI_HS_DLT4_TRAIL_SHIFT 9
494	# define DSI_HS_DLT4_LPX_MASK VC4_MASK(8, 0)
495	# define DSI_HS_DLT4_LPX_SHIFT 0
496
497	#define DSI1_HS_DLT5 0x64
498	# define DSI_HS_DLT5_INIT_MASK VC4_MASK(23, 0)
499	# define DSI_HS_DLT5_INIT_SHIFT 0
500
501	#define DSI1_HS_DLT6 0x68
502	# define DSI_HS_DLT6_TA_GET_MASK VC4_MASK(31, 24)
503	# define DSI_HS_DLT6_TA_GET_SHIFT 24
504	# define DSI_HS_DLT6_TA_SURE_MASK VC4_MASK(23, 16)
505	# define DSI_HS_DLT6_TA_SURE_SHIFT 16
506	# define DSI_HS_DLT6_TA_GO_MASK VC4_MASK(15, 8)
507	# define DSI_HS_DLT6_TA_GO_SHIFT 8
508	# define DSI_HS_DLT6_LP_LPX_MASK VC4_MASK(7, 0)
509	# define DSI_HS_DLT6_LP_LPX_SHIFT 0
510
511	#define DSI1_HS_DLT7 0x6c
512	# define DSI_HS_DLT7_LP_WUP_MASK VC4_MASK(23, 0)
513	# define DSI_HS_DLT7_LP_WUP_SHIFT 0
514
515	#define DSI1_PHY_AFEC0 0x70
516
517	#define DSI1_PHY_AFEC1 0x74
518	# define DSI1_PHY_AFEC1_ACTRL_DLANE3_MASK VC4_MASK(19, 16)
519	# define DSI1_PHY_AFEC1_ACTRL_DLANE3_SHIFT 16
520	# define DSI1_PHY_AFEC1_ACTRL_DLANE2_MASK VC4_MASK(15, 12)
521	# define DSI1_PHY_AFEC1_ACTRL_DLANE2_SHIFT 12
522	# define DSI1_PHY_AFEC1_ACTRL_DLANE1_MASK VC4_MASK(11, 8)
523	# define DSI1_PHY_AFEC1_ACTRL_DLANE1_SHIFT 8
524	# define DSI1_PHY_AFEC1_ACTRL_DLANE0_MASK VC4_MASK(7, 4)
525	# define DSI1_PHY_AFEC1_ACTRL_DLANE0_SHIFT 4
526	# define DSI1_PHY_AFEC1_ACTRL_CLANE_MASK VC4_MASK(3, 0)
527	# define DSI1_PHY_AFEC1_ACTRL_CLANE_SHIFT 0
528
529	#define DSI1_TST_SEL 0x78
530	#define DSI1_TST_MON 0x7c
531	#define DSI1_PHY_TST1 0x80
532	#define DSI1_PHY_TST2 0x84
533	#define DSI1_PHY_FIFO_STAT 0x88
534	/* Actually, all registers in the range that aren't otherwise claimed
535	* will return the ID.
536	*/
537	#define DSI1_ID 0x8c
538
539	struct vc4_dsi_variant {
540	/* Whether we're on bcm2835's DSI0 or DSI1. */
541	unsigned int port;
542
543	bool broken_axi_workaround;
544
545	const char *debugfs_name;
546	const struct debugfs_reg32 *regs;
547	size_t nregs;
548
549	};
550
551	/* General DSI hardware state. */
552	struct vc4_dsi {
553	struct vc4_encoder encoder;
554	struct mipi_dsi_host dsi_host;
555
556	struct platform_device *pdev;
557
558	struct drm_bridge *out_bridge;
559	struct drm_bridge bridge;
560
561	void __iomem *regs;
562
563	struct dma_chan *reg_dma_chan;
564	dma_addr_t reg_dma_paddr;
565	u32 *reg_dma_mem;
566	dma_addr_t reg_paddr;
567
568	const struct vc4_dsi_variant *variant;
569
570	/* DSI channel for the panel we're connected to. */
571	u32 channel;
572	u32 lanes;
573	u32 format;
574	u32 divider;
575	u32 mode_flags;
576
577	/* Input clock from CPRMAN to the digital PHY, for the DSI
578	* escape clock.
579	*/
580	struct clk *escape_clock;
581
582	/* Input clock to the analog PHY, used to generate the DSI bit
583	* clock.
584	*/
585	struct clk *pll_phy_clock;
586
587	/* HS Clocks generated within the DSI analog PHY. */
588	struct clk_fixed_factor phy_clocks[3];
589
590	struct clk_hw_onecell_data *clk_onecell;
591
592	/* Pixel clock output to the pixelvalve, generated from the HS
593	* clock.
594	*/
595	struct clk *pixel_clock;
596
597	struct completion xfer_completion;
598	int xfer_result;
599
600	struct debugfs_regset32 regset;
601	};
602
603	#define host_to_dsi(host) \
604	container_of_const(host, struct vc4_dsi, dsi_host)
605
606	#define to_vc4_dsi(_encoder) \
607	container_of_const(_encoder, struct vc4_dsi, encoder.base)
608
609	#define bridge_to_vc4_dsi(_bridge) \
610	container_of_const(_bridge, struct vc4_dsi, bridge)
611
612	static inline void
613	dsi_dma_workaround_write(struct vc4_dsi *dsi, u32 offset, u32 val)
614	{
615	struct drm_device *drm = dsi->bridge.dev;
616	struct dma_chan *chan = dsi->reg_dma_chan;
617	struct dma_async_tx_descriptor *tx;
618	dma_cookie_t cookie;
619	int ret;
620
621	kunit_fail_current_test("Accessing a register in a unit test!\n");
622
623	/* DSI0 should be able to write normally. */
624	if (!chan) {
625	writel(val, addr: dsi->regs + offset);
626	return;
627	}
628
629	*dsi->reg_dma_mem = val;
630
631	tx = chan->device->device_prep_dma_memcpy(chan,
632	dsi->reg_paddr + offset,
633	dsi->reg_dma_paddr,
634	4, 0);
635	if (!tx) {
636	drm_err(drm, "Failed to set up DMA register write\n");
637	return;
638	}
639
640	cookie = tx->tx_submit(tx);
641	ret = dma_submit_error(cookie);
642	if (ret) {
643	drm_err(drm, "Failed to submit DMA: %d\n", ret);
644	return;
645	}
646	ret = dma_sync_wait(chan, cookie);
647	if (ret)
648	drm_err(drm, "Failed to wait for DMA: %d\n", ret);
649	}
650
651	#define DSI_READ(offset) \
652	({ \
653	kunit_fail_current_test("Accessing a register in a unit test!\n"); \
654	readl(dsi->regs + (offset)); \
655	})
656
657	#define DSI_WRITE(offset, val) dsi_dma_workaround_write(dsi, offset, val)
658	#define DSI_PORT_READ(offset) \
659	DSI_READ(dsi->variant->port ? DSI1_##offset : DSI0_##offset)
660	#define DSI_PORT_WRITE(offset, val) \
661	DSI_WRITE(dsi->variant->port ? DSI1_##offset : DSI0_##offset, val)
662	#define DSI_PORT_BIT(bit) (dsi->variant->port ? DSI1_##bit : DSI0_##bit)
663
664	static const struct debugfs_reg32 dsi0_regs[] = {
665	VC4_REG32(DSI0_CTRL),
666	VC4_REG32(DSI0_STAT),
667	VC4_REG32(DSI0_HSTX_TO_CNT),
668	VC4_REG32(DSI0_LPRX_TO_CNT),
669	VC4_REG32(DSI0_TA_TO_CNT),
670	VC4_REG32(DSI0_PR_TO_CNT),
671	VC4_REG32(DSI0_DISP0_CTRL),
672	VC4_REG32(DSI0_DISP1_CTRL),
673	VC4_REG32(DSI0_INT_STAT),
674	VC4_REG32(DSI0_INT_EN),
675	VC4_REG32(DSI0_PHYC),
676	VC4_REG32(DSI0_HS_CLT0),
677	VC4_REG32(DSI0_HS_CLT1),
678	VC4_REG32(DSI0_HS_CLT2),
679	VC4_REG32(DSI0_HS_DLT3),
680	VC4_REG32(DSI0_HS_DLT4),
681	VC4_REG32(DSI0_HS_DLT5),
682	VC4_REG32(DSI0_HS_DLT6),
683	VC4_REG32(DSI0_HS_DLT7),
684	VC4_REG32(DSI0_PHY_AFEC0),
685	VC4_REG32(DSI0_PHY_AFEC1),
686	VC4_REG32(DSI0_ID),
687	};
688
689	static const struct debugfs_reg32 dsi1_regs[] = {
690	VC4_REG32(DSI1_CTRL),
691	VC4_REG32(DSI1_STAT),
692	VC4_REG32(DSI1_HSTX_TO_CNT),
693	VC4_REG32(DSI1_LPRX_TO_CNT),
694	VC4_REG32(DSI1_TA_TO_CNT),
695	VC4_REG32(DSI1_PR_TO_CNT),
696	VC4_REG32(DSI1_DISP0_CTRL),
697	VC4_REG32(DSI1_DISP1_CTRL),
698	VC4_REG32(DSI1_INT_STAT),
699	VC4_REG32(DSI1_INT_EN),
700	VC4_REG32(DSI1_PHYC),
701	VC4_REG32(DSI1_HS_CLT0),
702	VC4_REG32(DSI1_HS_CLT1),
703	VC4_REG32(DSI1_HS_CLT2),
704	VC4_REG32(DSI1_HS_DLT3),
705	VC4_REG32(DSI1_HS_DLT4),
706	VC4_REG32(DSI1_HS_DLT5),
707	VC4_REG32(DSI1_HS_DLT6),
708	VC4_REG32(DSI1_HS_DLT7),
709	VC4_REG32(DSI1_PHY_AFEC0),
710	VC4_REG32(DSI1_PHY_AFEC1),
711	VC4_REG32(DSI1_ID),
712	};
713
714	static void vc4_dsi_latch_ulps(struct vc4_dsi *dsi, bool latch)
715	{
716	u32 afec0 = DSI_PORT_READ(PHY_AFEC0);
717
718	if (latch)
719	afec0 |= DSI_PORT_BIT(PHY_AFEC0_LATCH_ULPS);
720	else
721	afec0 &= ~DSI_PORT_BIT(PHY_AFEC0_LATCH_ULPS);
722
723	DSI_PORT_WRITE(PHY_AFEC0, afec0);
724	}
725
726	/* Enters or exits Ultra Low Power State. */
727	static void vc4_dsi_ulps(struct vc4_dsi *dsi, bool ulps)
728	{
729	bool non_continuous = dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS;
730	u32 phyc_ulps = ((non_continuous ? DSI_PORT_BIT(PHYC_CLANE_ULPS) : 0) |
731	DSI_PHYC_DLANE0_ULPS |
732	(dsi->lanes > 1 ? DSI_PHYC_DLANE1_ULPS : 0) |
733	(dsi->lanes > 2 ? DSI_PHYC_DLANE2_ULPS : 0) |
734	(dsi->lanes > 3 ? DSI_PHYC_DLANE3_ULPS : 0));
735	u32 stat_ulps = ((non_continuous ? DSI1_STAT_PHY_CLOCK_ULPS : 0) |
736	DSI1_STAT_PHY_D0_ULPS |
737	(dsi->lanes > 1 ? DSI1_STAT_PHY_D1_ULPS : 0) |
738	(dsi->lanes > 2 ? DSI1_STAT_PHY_D2_ULPS : 0) |
739	(dsi->lanes > 3 ? DSI1_STAT_PHY_D3_ULPS : 0));
740	u32 stat_stop = ((non_continuous ? DSI1_STAT_PHY_CLOCK_STOP : 0) |
741	DSI1_STAT_PHY_D0_STOP |
742	(dsi->lanes > 1 ? DSI1_STAT_PHY_D1_STOP : 0) |
743	(dsi->lanes > 2 ? DSI1_STAT_PHY_D2_STOP : 0) |
744	(dsi->lanes > 3 ? DSI1_STAT_PHY_D3_STOP : 0));
745	int ret;
746	bool ulps_currently_enabled = (DSI_PORT_READ(PHY_AFEC0) &
747	DSI_PORT_BIT(PHY_AFEC0_LATCH_ULPS));
748
749	if (ulps == ulps_currently_enabled)
750	return;
751
752	DSI_PORT_WRITE(STAT, stat_ulps);
753	DSI_PORT_WRITE(PHYC, DSI_PORT_READ(PHYC) | phyc_ulps);
754	ret = wait_for((DSI_PORT_READ(STAT) & stat_ulps) == stat_ulps, 200);
755	if (ret) {
756	dev_warn(&dsi->pdev->dev,
757	"Timeout waiting for DSI ULPS entry: STAT 0x%08x",
758	DSI_PORT_READ(STAT));
759	DSI_PORT_WRITE(PHYC, DSI_PORT_READ(PHYC) & ~phyc_ulps);
760	vc4_dsi_latch_ulps(dsi, latch: false);
761	return;
762	}
763
764	/* The DSI module can't be disabled while the module is
765	* generating ULPS state. So, to be able to disable the
766	* module, we have the AFE latch the ULPS state and continue
767	* on to having the module enter STOP.
768	*/
769	vc4_dsi_latch_ulps(dsi, latch: ulps);
770
771	DSI_PORT_WRITE(STAT, stat_stop);
772	DSI_PORT_WRITE(PHYC, DSI_PORT_READ(PHYC) & ~phyc_ulps);
773	ret = wait_for((DSI_PORT_READ(STAT) & stat_stop) == stat_stop, 200);
774	if (ret) {
775	dev_warn(&dsi->pdev->dev,
776	"Timeout waiting for DSI STOP entry: STAT 0x%08x",
777	DSI_PORT_READ(STAT));
778	DSI_PORT_WRITE(PHYC, DSI_PORT_READ(PHYC) & ~phyc_ulps);
779	return;
780	}
781	}
782
783	static u32
784	dsi_hs_timing(u32 ui_ns, u32 ns, u32 ui)
785	{
786	/* The HS timings have to be rounded up to a multiple of 8
787	* because we're using the byte clock.
788	*/
789	return roundup(ui + DIV_ROUND_UP(ns, ui_ns), 8);
790	}
791
792	/* ESC always runs at 100Mhz. */
793	#define ESC_TIME_NS 10
794
795	static u32
796	dsi_esc_timing(u32 ns)
797	{
798	return DIV_ROUND_UP(ns, ESC_TIME_NS);
799	}
800
801	static void vc4_dsi_bridge_disable(struct drm_bridge *bridge,
802	struct drm_atomic_state *state)
803	{
804	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
805	u32 disp0_ctrl;
806
807	disp0_ctrl = DSI_PORT_READ(DISP0_CTRL);
808	disp0_ctrl &= ~DSI_DISP0_ENABLE;
809	DSI_PORT_WRITE(DISP0_CTRL, disp0_ctrl);
810	}
811
812	static void vc4_dsi_bridge_post_disable(struct drm_bridge *bridge,
813	struct drm_atomic_state *state)
814	{
815	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
816	struct device *dev = &dsi->pdev->dev;
817
818	clk_disable_unprepare(clk: dsi->pll_phy_clock);
819	clk_disable_unprepare(clk: dsi->escape_clock);
820	clk_disable_unprepare(clk: dsi->pixel_clock);
821
822	pm_runtime_put(dev);
823	}
824
825	/* Extends the mode's blank intervals to handle BCM2835's integer-only
826	* DSI PLL divider.
827	*
828	* On 2835, PLLD is set to 2Ghz, and may not be changed by the display
829	* driver since most peripherals are hanging off of the PLLD_PER
830	* divider. PLLD_DSI1, which drives our DSI bit clock (and therefore
831	* the pixel clock), only has an integer divider off of DSI.
832	*
833	* To get our panel mode to refresh at the expected 60Hz, we need to
834	* extend the horizontal blank time. This means we drive a
835	* higher-than-expected clock rate to the panel, but that's what the
836	* firmware does too.
837	*/
838	static bool vc4_dsi_bridge_mode_fixup(struct drm_bridge *bridge,
839	const struct drm_display_mode *mode,
840	struct drm_display_mode *adjusted_mode)
841	{
842	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
843	struct clk *phy_parent = clk_get_parent(clk: dsi->pll_phy_clock);
844	unsigned long parent_rate = clk_get_rate(clk: phy_parent);
845	unsigned long pixel_clock_hz = mode->clock * 1000;
846	unsigned long pll_clock = pixel_clock_hz * dsi->divider;
847	int divider;
848
849	/* Find what divider gets us a faster clock than the requested
850	* pixel clock.
851	*/
852	for (divider = 1; divider < 255; divider++) {
853	if (parent_rate / (divider + 1) < pll_clock)
854	break;
855	}
856
857	/* Now that we've picked a PLL divider, calculate back to its
858	* pixel clock.
859	*/
860	pll_clock = parent_rate / divider;
861	pixel_clock_hz = pll_clock / dsi->divider;
862
863	adjusted_mode->clock = pixel_clock_hz / 1000;
864
865	/* Given the new pixel clock, adjust HFP to keep vrefresh the same. */
866	adjusted_mode->htotal = adjusted_mode->clock * mode->htotal /
867	mode->clock;
868	adjusted_mode->hsync_end += adjusted_mode->htotal - mode->htotal;
869	adjusted_mode->hsync_start += adjusted_mode->htotal - mode->htotal;
870
871	return true;
872	}
873
874	static void vc4_dsi_bridge_pre_enable(struct drm_bridge *bridge,
875	struct drm_atomic_state *state)
876	{
877	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
878	const struct drm_crtc_state *crtc_state;
879	struct device *dev = &dsi->pdev->dev;
880	const struct drm_display_mode *mode;
881	struct drm_connector *connector;
882	bool debug_dump_regs = false;
883	unsigned long hs_clock;
884	struct drm_crtc *crtc;
885	u32 ui_ns;
886	/* Minimum LP state duration in escape clock cycles. */
887	u32 lpx = dsi_esc_timing(ns: 60);
888	unsigned long pixel_clock_hz;
889	unsigned long dsip_clock;
890	unsigned long phy_clock;
891	int ret;
892
893	ret = pm_runtime_resume_and_get(dev);
894	if (ret) {
895	drm_err(bridge->dev, "Failed to runtime PM enable on DSI%d\n", dsi->variant->port);
896	return;
897	}
898
899	if (debug_dump_regs) {
900	struct drm_printer p = drm_info_printer(dev: &dsi->pdev->dev);
901	dev_info(&dsi->pdev->dev, "DSI regs before:\n");
902	drm_print_regset32(p: &p, regset: &dsi->regset);
903	}
904
905	/*
906	* Retrieve the CRTC adjusted mode. This requires a little dance to go
907	* from the bridge to the encoder, to the connector and to the CRTC.
908	*/
909	connector = drm_atomic_get_new_connector_for_encoder(state,
910	encoder: bridge->encoder);
911	crtc = drm_atomic_get_new_connector_state(state, connector)->crtc;
912	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
913	mode = &crtc_state->adjusted_mode;
914
915	pixel_clock_hz = mode->clock * 1000;
916
917	/* Round up the clk_set_rate() request slightly, since
918	* PLLD_DSI1 is an integer divider and its rate selection will
919	* never round up.
920	*/
921	phy_clock = (pixel_clock_hz + 1000) * dsi->divider;
922	ret = clk_set_rate(clk: dsi->pll_phy_clock, rate: phy_clock);
923	if (ret) {
924	dev_err(&dsi->pdev->dev,
925	"Failed to set phy clock to %ld: %d\n", phy_clock, ret);
926	}
927
928	/* Reset the DSI and all its fifos. */
929	DSI_PORT_WRITE(CTRL,
930	DSI_CTRL_SOFT_RESET_CFG |
931	DSI_PORT_BIT(CTRL_RESET_FIFOS));
932
933	DSI_PORT_WRITE(CTRL,
934	DSI_CTRL_HSDT_EOT_DISABLE |
935	DSI_CTRL_RX_LPDT_EOT_DISABLE);
936
937	/* Clear all stat bits so we see what has happened during enable. */
938	DSI_PORT_WRITE(STAT, DSI_PORT_READ(STAT));
939
940	/* Set AFE CTR00/CTR1 to release powerdown of analog. */
941	if (dsi->variant->port == 0) {
942	u32 afec0 = (VC4_SET_FIELD(7, DSI_PHY_AFEC0_PTATADJ) |
943	VC4_SET_FIELD(7, DSI_PHY_AFEC0_CTATADJ));
944
945	if (dsi->lanes < 2)
946	afec0 |= DSI0_PHY_AFEC0_PD_DLANE1;
947
948	if (!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO))
949	afec0 |= DSI0_PHY_AFEC0_RESET;
950
951	DSI_PORT_WRITE(PHY_AFEC0, afec0);
952
953	/* AFEC reset hold time */
954	mdelay(1);
955
956	DSI_PORT_WRITE(PHY_AFEC1,
957	VC4_SET_FIELD(6, DSI0_PHY_AFEC1_IDR_DLANE1) |
958	VC4_SET_FIELD(6, DSI0_PHY_AFEC1_IDR_DLANE0) |
959	VC4_SET_FIELD(6, DSI0_PHY_AFEC1_IDR_CLANE));
960	} else {
961	u32 afec0 = (VC4_SET_FIELD(7, DSI_PHY_AFEC0_PTATADJ) |
962	VC4_SET_FIELD(7, DSI_PHY_AFEC0_CTATADJ) |
963	VC4_SET_FIELD(6, DSI1_PHY_AFEC0_IDR_CLANE) |
964	VC4_SET_FIELD(6, DSI1_PHY_AFEC0_IDR_DLANE0) |
965	VC4_SET_FIELD(6, DSI1_PHY_AFEC0_IDR_DLANE1) |
966	VC4_SET_FIELD(6, DSI1_PHY_AFEC0_IDR_DLANE2) |
967	VC4_SET_FIELD(6, DSI1_PHY_AFEC0_IDR_DLANE3));
968
969	if (dsi->lanes < 4)
970	afec0 |= DSI1_PHY_AFEC0_PD_DLANE3;
971	if (dsi->lanes < 3)
972	afec0 |= DSI1_PHY_AFEC0_PD_DLANE2;
973	if (dsi->lanes < 2)
974	afec0 |= DSI1_PHY_AFEC0_PD_DLANE1;
975
976	afec0 |= DSI1_PHY_AFEC0_RESET;
977
978	DSI_PORT_WRITE(PHY_AFEC0, afec0);
979
980	DSI_PORT_WRITE(PHY_AFEC1, 0);
981
982	/* AFEC reset hold time */
983	mdelay(1);
984	}
985
986	ret = clk_prepare_enable(clk: dsi->escape_clock);
987	if (ret) {
988	drm_err(bridge->dev, "Failed to turn on DSI escape clock: %d\n",
989	ret);
990	return;
991	}
992
993	ret = clk_prepare_enable(clk: dsi->pll_phy_clock);
994	if (ret) {
995	drm_err(bridge->dev, "Failed to turn on DSI PLL: %d\n", ret);
996	return;
997	}
998
999	hs_clock = clk_get_rate(clk: dsi->pll_phy_clock);
1000
1001	/* Yes, we set the DSI0P/DSI1P pixel clock to the byte rate,
1002	* not the pixel clock rate. DSIxP take from the APHY's byte,
1003	* DDR2, or DDR4 clock (we use byte) and feed into the PV at
1004	* that rate. Separately, a value derived from PIX_CLK_DIV
1005	* and HS_CLKC is fed into the PV to divide down to the actual
1006	* pixel clock for pushing pixels into DSI.
1007	*/
1008	dsip_clock = phy_clock / 8;
1009	ret = clk_set_rate(clk: dsi->pixel_clock, rate: dsip_clock);
1010	if (ret) {
1011	dev_err(dev, "Failed to set pixel clock to %ldHz: %d\n",
1012	dsip_clock, ret);
1013	}
1014
1015	ret = clk_prepare_enable(clk: dsi->pixel_clock);
1016	if (ret) {
1017	drm_err(bridge->dev, "Failed to turn on DSI pixel clock: %d\n", ret);
1018	return;
1019	}
1020
1021	/* How many ns one DSI unit interval is. Note that the clock
1022	* is DDR, so there's an extra divide by 2.
1023	*/
1024	ui_ns = DIV_ROUND_UP(500000000, hs_clock);
1025
1026	DSI_PORT_WRITE(HS_CLT0,
1027	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 262, 0),
1028	DSI_HS_CLT0_CZERO) |
1029	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 0, 8),
1030	DSI_HS_CLT0_CPRE) |
1031	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 38, 0),
1032	DSI_HS_CLT0_CPREP));
1033
1034	DSI_PORT_WRITE(HS_CLT1,
1035	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 60, 0),
1036	DSI_HS_CLT1_CTRAIL) |
1037	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 60, 52),
1038	DSI_HS_CLT1_CPOST));
1039
1040	DSI_PORT_WRITE(HS_CLT2,
1041	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 1000000, 0),
1042	DSI_HS_CLT2_WUP));
1043
1044	DSI_PORT_WRITE(HS_DLT3,
1045	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 100, 0),
1046	DSI_HS_DLT3_EXIT) |
1047	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 105, 6),
1048	DSI_HS_DLT3_ZERO) |
1049	VC4_SET_FIELD(dsi_hs_timing(ui_ns, 40, 4),
1050	DSI_HS_DLT3_PRE));
1051
1052	DSI_PORT_WRITE(HS_DLT4,
1053	VC4_SET_FIELD(dsi_hs_timing(ui_ns, lpx * ESC_TIME_NS, 0),
1054	DSI_HS_DLT4_LPX) |
1055	VC4_SET_FIELD(max(dsi_hs_timing(ui_ns, 0, 8),
1056	dsi_hs_timing(ui_ns, 60, 4)),
1057	DSI_HS_DLT4_TRAIL) |
1058	VC4_SET_FIELD(0, DSI_HS_DLT4_ANLAT));
1059
1060	/* T_INIT is how long STOP is driven after power-up to
1061	* indicate to the slave (also coming out of power-up) that
1062	* master init is complete, and should be greater than the
1063	* maximum of two value: T_INIT,MASTER and T_INIT,SLAVE. The
1064	* D-PHY spec gives a minimum 100us for T_INIT,MASTER and
1065	* T_INIT,SLAVE, while allowing protocols on top of it to give
1066	* greater minimums. The vc4 firmware uses an extremely
1067	* conservative 5ms, and we maintain that here.
1068	*/
1069	DSI_PORT_WRITE(HS_DLT5, VC4_SET_FIELD(dsi_hs_timing(ui_ns,
1070	5 * 1000 * 1000, 0),
1071	DSI_HS_DLT5_INIT));
1072
1073	DSI_PORT_WRITE(HS_DLT6,
1074	VC4_SET_FIELD(lpx * 5, DSI_HS_DLT6_TA_GET) |
1075	VC4_SET_FIELD(lpx, DSI_HS_DLT6_TA_SURE) |
1076	VC4_SET_FIELD(lpx * 4, DSI_HS_DLT6_TA_GO) |
1077	VC4_SET_FIELD(lpx, DSI_HS_DLT6_LP_LPX));
1078
1079	DSI_PORT_WRITE(HS_DLT7,
1080	VC4_SET_FIELD(dsi_esc_timing(1000000),
1081	DSI_HS_DLT7_LP_WUP));
1082
1083	DSI_PORT_WRITE(PHYC,
1084	DSI_PHYC_DLANE0_ENABLE |
1085	(dsi->lanes >= 2 ? DSI_PHYC_DLANE1_ENABLE : 0) |
1086	(dsi->lanes >= 3 ? DSI_PHYC_DLANE2_ENABLE : 0) |
1087	(dsi->lanes >= 4 ? DSI_PHYC_DLANE3_ENABLE : 0) |
1088	DSI_PORT_BIT(PHYC_CLANE_ENABLE) |
1089	((dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS) ?
1090	0 : DSI_PORT_BIT(PHYC_HS_CLK_CONTINUOUS)) |
1091	(dsi->variant->port == 0 ?
1092	VC4_SET_FIELD(lpx - 1, DSI0_PHYC_ESC_CLK_LPDT) :
1093	VC4_SET_FIELD(lpx - 1, DSI1_PHYC_ESC_CLK_LPDT)));
1094
1095	DSI_PORT_WRITE(CTRL,
1096	DSI_PORT_READ(CTRL) |
1097	DSI_CTRL_CAL_BYTE);
1098
1099	/* HS timeout in HS clock cycles: disabled. */
1100	DSI_PORT_WRITE(HSTX_TO_CNT, 0);
1101	/* LP receive timeout in HS clocks. */
1102	DSI_PORT_WRITE(LPRX_TO_CNT, 0xffffff);
1103	/* Bus turnaround timeout */
1104	DSI_PORT_WRITE(TA_TO_CNT, 100000);
1105	/* Display reset sequence timeout */
1106	DSI_PORT_WRITE(PR_TO_CNT, 100000);
1107
1108	/* Set up DISP1 for transferring long command payloads through
1109	* the pixfifo.
1110	*/
1111	DSI_PORT_WRITE(DISP1_CTRL,
1112	VC4_SET_FIELD(DSI_DISP1_PFORMAT_32BIT_LE,
1113	DSI_DISP1_PFORMAT) |
1114	DSI_DISP1_ENABLE);
1115
1116	/* Ungate the block. */
1117	if (dsi->variant->port == 0)
1118	DSI_PORT_WRITE(CTRL, DSI_PORT_READ(CTRL) | DSI0_CTRL_CTRL0);
1119	else
1120	DSI_PORT_WRITE(CTRL, DSI_PORT_READ(CTRL) | DSI1_CTRL_EN);
1121
1122	/* Bring AFE out of reset. */
1123	DSI_PORT_WRITE(PHY_AFEC0,
1124	DSI_PORT_READ(PHY_AFEC0) &
1125	~DSI_PORT_BIT(PHY_AFEC0_RESET));
1126
1127	vc4_dsi_ulps(dsi, ulps: false);
1128
1129	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
1130	DSI_PORT_WRITE(DISP0_CTRL,
1131	VC4_SET_FIELD(dsi->divider,
1132	DSI_DISP0_PIX_CLK_DIV) |
1133	VC4_SET_FIELD(dsi->format, DSI_DISP0_PFORMAT) |
1134	VC4_SET_FIELD(DSI_DISP0_LP_STOP_PERFRAME,
1135	DSI_DISP0_LP_STOP_CTRL) |
1136	DSI_DISP0_ST_END);
1137	} else {
1138	DSI_PORT_WRITE(DISP0_CTRL,
1139	DSI_DISP0_COMMAND_MODE);
1140	}
1141	}
1142
1143	static void vc4_dsi_bridge_enable(struct drm_bridge *bridge,
1144	struct drm_atomic_state *state)
1145	{
1146	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
1147	bool debug_dump_regs = false;
1148	u32 disp0_ctrl;
1149
1150	disp0_ctrl = DSI_PORT_READ(DISP0_CTRL);
1151	disp0_ctrl |= DSI_DISP0_ENABLE;
1152	DSI_PORT_WRITE(DISP0_CTRL, disp0_ctrl);
1153
1154	if (debug_dump_regs) {
1155	struct drm_printer p = drm_info_printer(dev: &dsi->pdev->dev);
1156	dev_info(&dsi->pdev->dev, "DSI regs after:\n");
1157	drm_print_regset32(p: &p, regset: &dsi->regset);
1158	}
1159	}
1160
1161	static int vc4_dsi_bridge_attach(struct drm_bridge *bridge,
1162	struct drm_encoder *encoder,
1163	enum drm_bridge_attach_flags flags)
1164	{
1165	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
1166
1167	/* Attach the panel or bridge to the dsi bridge */
1168	return drm_bridge_attach(encoder, bridge: dsi->out_bridge,
1169	previous: &dsi->bridge, flags);
1170	}
1171
1172	static ssize_t vc4_dsi_host_transfer(struct mipi_dsi_host *host,
1173	const struct mipi_dsi_msg *msg)
1174	{
1175	struct vc4_dsi *dsi = host_to_dsi(host);
1176	struct drm_device *drm = dsi->bridge.dev;
1177	struct mipi_dsi_packet packet;
1178	u32 pkth = 0, pktc = 0;
1179	int i, ret;
1180	bool is_long = mipi_dsi_packet_format_is_long(type: msg->type);
1181	u32 cmd_fifo_len = 0, pix_fifo_len = 0;
1182
1183	mipi_dsi_create_packet(packet: &packet, msg);
1184
1185	pkth |= VC4_SET_FIELD(packet.header[0], DSI_TXPKT1H_BC_DT);
1186	pkth |= VC4_SET_FIELD(packet.header[1] |
1187	(packet.header[2] << 8),
1188	DSI_TXPKT1H_BC_PARAM);
1189	if (is_long) {
1190	/* Divide data across the various FIFOs we have available.
1191	* The command FIFO takes byte-oriented data, but is of
1192	* limited size. The pixel FIFO (never actually used for
1193	* pixel data in reality) is word oriented, and substantially
1194	* larger. So, we use the pixel FIFO for most of the data,
1195	* sending the residual bytes in the command FIFO at the start.
1196	*
1197	* With this arrangement, the command FIFO will never get full.
1198	*/
1199	if (packet.payload_length <= 16) {
1200	cmd_fifo_len = packet.payload_length;
1201	pix_fifo_len = 0;
1202	} else {
1203	cmd_fifo_len = (packet.payload_length %
1204	DSI_PIX_FIFO_WIDTH);
1205	pix_fifo_len = ((packet.payload_length - cmd_fifo_len) /
1206	DSI_PIX_FIFO_WIDTH);
1207	}
1208
1209	WARN_ON_ONCE(pix_fifo_len >= DSI_PIX_FIFO_DEPTH);
1210
1211	pkth |= VC4_SET_FIELD(cmd_fifo_len, DSI_TXPKT1H_BC_CMDFIFO);
1212	}
1213
1214	if (msg->rx_len) {
1215	pktc |= VC4_SET_FIELD(DSI_TXPKT1C_CMD_CTRL_RX,
1216	DSI_TXPKT1C_CMD_CTRL);
1217	} else {
1218	pktc |= VC4_SET_FIELD(DSI_TXPKT1C_CMD_CTRL_TX,
1219	DSI_TXPKT1C_CMD_CTRL);
1220	}
1221
1222	for (i = 0; i < cmd_fifo_len; i++)
1223	DSI_PORT_WRITE(TXPKT_CMD_FIFO, packet.payload[i]);
1224	for (i = 0; i < pix_fifo_len; i++) {
1225	const u8 *pix = packet.payload + cmd_fifo_len + i * 4;
1226
1227	DSI_PORT_WRITE(TXPKT_PIX_FIFO,
1228	pix[0] |
1229	pix[1] << 8 |
1230	pix[2] << 16 |
1231	pix[3] << 24);
1232	}
1233
1234	if (msg->flags & MIPI_DSI_MSG_USE_LPM)
1235	pktc |= DSI_TXPKT1C_CMD_MODE_LP;
1236	if (is_long)
1237	pktc |= DSI_TXPKT1C_CMD_TYPE_LONG;
1238
1239	/* Send one copy of the packet. Larger repeats are used for pixel
1240	* data in command mode.
1241	*/
1242	pktc |= VC4_SET_FIELD(1, DSI_TXPKT1C_CMD_REPEAT);
1243
1244	pktc |= DSI_TXPKT1C_CMD_EN;
1245	if (pix_fifo_len) {
1246	pktc |= VC4_SET_FIELD(DSI_TXPKT1C_DISPLAY_NO_SECONDARY,
1247	DSI_TXPKT1C_DISPLAY_NO);
1248	} else {
1249	pktc |= VC4_SET_FIELD(DSI_TXPKT1C_DISPLAY_NO_SHORT,
1250	DSI_TXPKT1C_DISPLAY_NO);
1251	}
1252
1253	/* Enable the appropriate interrupt for the transfer completion. */
1254	dsi->xfer_result = 0;
1255	reinit_completion(x: &dsi->xfer_completion);
1256	if (dsi->variant->port == 0) {
1257	DSI_PORT_WRITE(INT_STAT,
1258	DSI0_INT_CMDC_DONE_MASK | DSI1_INT_PHY_DIR_RTF);
1259	if (msg->rx_len) {
1260	DSI_PORT_WRITE(INT_EN, (DSI0_INTERRUPTS_ALWAYS_ENABLED |
1261	DSI0_INT_PHY_DIR_RTF));
1262	} else {
1263	DSI_PORT_WRITE(INT_EN,
1264	(DSI0_INTERRUPTS_ALWAYS_ENABLED |
1265	VC4_SET_FIELD(DSI0_INT_CMDC_DONE_NO_REPEAT,
1266	DSI0_INT_CMDC_DONE)));
1267	}
1268	} else {
1269	DSI_PORT_WRITE(INT_STAT,
1270	DSI1_INT_TXPKT1_DONE | DSI1_INT_PHY_DIR_RTF);
1271	if (msg->rx_len) {
1272	DSI_PORT_WRITE(INT_EN, (DSI1_INTERRUPTS_ALWAYS_ENABLED |
1273	DSI1_INT_PHY_DIR_RTF));
1274	} else {
1275	DSI_PORT_WRITE(INT_EN, (DSI1_INTERRUPTS_ALWAYS_ENABLED |
1276	DSI1_INT_TXPKT1_DONE));
1277	}
1278	}
1279
1280	/* Send the packet. */
1281	DSI_PORT_WRITE(TXPKT1H, pkth);
1282	DSI_PORT_WRITE(TXPKT1C, pktc);
1283
1284	if (!wait_for_completion_timeout(x: &dsi->xfer_completion,
1285	timeout: msecs_to_jiffies(m: 1000))) {
1286	dev_err(&dsi->pdev->dev, "transfer interrupt wait timeout");
1287	dev_err(&dsi->pdev->dev, "instat: 0x%08x\n",
1288	DSI_PORT_READ(INT_STAT));
1289	ret = -ETIMEDOUT;
1290	} else {
1291	ret = dsi->xfer_result;
1292	}
1293
1294	DSI_PORT_WRITE(INT_EN, DSI_PORT_BIT(INTERRUPTS_ALWAYS_ENABLED));
1295
1296	if (ret)
1297	goto reset_fifo_and_return;
1298
1299	if (ret == 0 && msg->rx_len) {
1300	u32 rxpkt1h = DSI_PORT_READ(RXPKT1H);
1301	u8 *msg_rx = msg->rx_buf;
1302
1303	if (rxpkt1h & DSI_RXPKT1H_PKT_TYPE_LONG) {
1304	u32 rxlen = VC4_GET_FIELD(rxpkt1h,
1305	DSI_RXPKT1H_BC_PARAM);
1306
1307	if (rxlen != msg->rx_len) {
1308	drm_err(drm, "DSI returned %db, expecting %db\n",
1309	rxlen, (int)msg->rx_len);
1310	ret = -ENXIO;
1311	goto reset_fifo_and_return;
1312	}
1313
1314	for (i = 0; i < msg->rx_len; i++)
1315	msg_rx[i] = DSI_READ(DSI1_RXPKT_FIFO);
1316	} else {
1317	/* FINISHME: Handle AWER */
1318
1319	msg_rx[0] = VC4_GET_FIELD(rxpkt1h,
1320	DSI_RXPKT1H_SHORT_0);
1321	if (msg->rx_len > 1) {
1322	msg_rx[1] = VC4_GET_FIELD(rxpkt1h,
1323	DSI_RXPKT1H_SHORT_1);
1324	}
1325	}
1326	}
1327
1328	return ret;
1329
1330	reset_fifo_and_return:
1331	drm_err(drm, "DSI transfer failed, resetting: %d\n", ret);
1332
1333	DSI_PORT_WRITE(TXPKT1C, DSI_PORT_READ(TXPKT1C) & ~DSI_TXPKT1C_CMD_EN);
1334	udelay(usec: 1);
1335	DSI_PORT_WRITE(CTRL,
1336	DSI_PORT_READ(CTRL) |
1337	DSI_PORT_BIT(CTRL_RESET_FIFOS));
1338
1339	DSI_PORT_WRITE(TXPKT1C, 0);
1340	DSI_PORT_WRITE(INT_EN, DSI_PORT_BIT(INTERRUPTS_ALWAYS_ENABLED));
1341	return ret;
1342	}
1343
1344	static const struct component_ops vc4_dsi_ops;
1345	static int vc4_dsi_host_attach(struct mipi_dsi_host *host,
1346	struct mipi_dsi_device *device)
1347	{
1348	struct vc4_dsi *dsi = host_to_dsi(host);
1349	int ret;
1350
1351	dsi->lanes = device->lanes;
1352	dsi->channel = device->channel;
1353	dsi->mode_flags = device->mode_flags;
1354
1355	switch (device->format) {
1356	case MIPI_DSI_FMT_RGB888:
1357	dsi->format = DSI_PFORMAT_RGB888;
1358	dsi->divider = 24 / dsi->lanes;
1359	break;
1360	case MIPI_DSI_FMT_RGB666:
1361	dsi->format = DSI_PFORMAT_RGB666;
1362	dsi->divider = 24 / dsi->lanes;
1363	break;
1364	case MIPI_DSI_FMT_RGB666_PACKED:
1365	dsi->format = DSI_PFORMAT_RGB666_PACKED;
1366	dsi->divider = 18 / dsi->lanes;
1367	break;
1368	case MIPI_DSI_FMT_RGB565:
1369	dsi->format = DSI_PFORMAT_RGB565;
1370	dsi->divider = 16 / dsi->lanes;
1371	break;
1372	default:
1373	dev_err(&dsi->pdev->dev, "Unknown DSI format: %d.\n",
1374	dsi->format);
1375	return 0;
1376	}
1377
1378	if (!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO)) {
1379	dev_err(&dsi->pdev->dev,
1380	"Only VIDEO mode panels supported currently.\n");
1381	return 0;
1382	}
1383
1384	drm_bridge_add(bridge: &dsi->bridge);
1385
1386	ret = component_add(&dsi->pdev->dev, &vc4_dsi_ops);
1387	if (ret) {
1388	drm_bridge_remove(bridge: &dsi->bridge);
1389	return ret;
1390	}
1391
1392	return 0;
1393	}
1394
1395	static int vc4_dsi_host_detach(struct mipi_dsi_host *host,
1396	struct mipi_dsi_device *device)
1397	{
1398	struct vc4_dsi *dsi = host_to_dsi(host);
1399
1400	component_del(&dsi->pdev->dev, &vc4_dsi_ops);
1401	drm_bridge_remove(bridge: &dsi->bridge);
1402	return 0;
1403	}
1404
1405	static const struct mipi_dsi_host_ops vc4_dsi_host_ops = {
1406	.attach = vc4_dsi_host_attach,
1407	.detach = vc4_dsi_host_detach,
1408	.transfer = vc4_dsi_host_transfer,
1409	};
1410
1411	static const struct drm_bridge_funcs vc4_dsi_bridge_funcs = {
1412	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
1413	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
1414	.atomic_reset = drm_atomic_helper_bridge_reset,
1415	.atomic_pre_enable = vc4_dsi_bridge_pre_enable,
1416	.atomic_enable = vc4_dsi_bridge_enable,
1417	.atomic_disable = vc4_dsi_bridge_disable,
1418	.atomic_post_disable = vc4_dsi_bridge_post_disable,
1419	.attach = vc4_dsi_bridge_attach,
1420	.mode_fixup = vc4_dsi_bridge_mode_fixup,
1421	};
1422
1423	static int vc4_dsi_late_register(struct drm_encoder *encoder)
1424	{
1425	struct drm_device *drm = encoder->dev;
1426	struct vc4_dsi *dsi = to_vc4_dsi(encoder);
1427
1428	vc4_debugfs_add_regset32(drm, filename: dsi->variant->debugfs_name, regset: &dsi->regset);
1429
1430	return 0;
1431	}
1432
1433	static const struct drm_encoder_funcs vc4_dsi_encoder_funcs = {
1434	.late_register = vc4_dsi_late_register,
1435	};
1436
1437	static const struct vc4_dsi_variant bcm2711_dsi1_variant = {
1438	.port = 1,
1439	.debugfs_name = "dsi1_regs",
1440	.regs = dsi1_regs,
1441	.nregs = ARRAY_SIZE(dsi1_regs),
1442	};
1443
1444	static const struct vc4_dsi_variant bcm2835_dsi0_variant = {
1445	.port = 0,
1446	.debugfs_name = "dsi0_regs",
1447	.regs = dsi0_regs,
1448	.nregs = ARRAY_SIZE(dsi0_regs),
1449	};
1450
1451	static const struct vc4_dsi_variant bcm2835_dsi1_variant = {
1452	.port = 1,
1453	.broken_axi_workaround = true,
1454	.debugfs_name = "dsi1_regs",
1455	.regs = dsi1_regs,
1456	.nregs = ARRAY_SIZE(dsi1_regs),
1457	};
1458
1459	static const struct of_device_id vc4_dsi_dt_match[] = {
1460	{ .compatible = "brcm,bcm2711-dsi1", &bcm2711_dsi1_variant },
1461	{ .compatible = "brcm,bcm2835-dsi0", &bcm2835_dsi0_variant },
1462	{ .compatible = "brcm,bcm2835-dsi1", &bcm2835_dsi1_variant },
1463	{}
1464	};
1465
1466	static void dsi_handle_error(struct vc4_dsi *dsi,
1467	irqreturn_t *ret, u32 stat, u32 bit,
1468	const char *type)
1469	{
1470	if (!(stat & bit))
1471	return;
1472
1473	drm_err(dsi->bridge.dev, "DSI%d: %s error\n", dsi->variant->port,
1474	type);
1475	*ret = IRQ_HANDLED;
1476	}
1477
1478	/*
1479	* Initial handler for port 1 where we need the reg_dma workaround.
1480	* The register DMA writes sleep, so we can't do it in the top half.
1481	* Instead we use IRQF_ONESHOT so that the IRQ gets disabled in the
1482	* parent interrupt contrller until our interrupt thread is done.
1483	*/
1484	static irqreturn_t vc4_dsi_irq_defer_to_thread_handler(int irq, void *data)
1485	{
1486	struct vc4_dsi *dsi = data;
1487	u32 stat = DSI_PORT_READ(INT_STAT);
1488
1489	if (!stat)
1490	return IRQ_NONE;
1491
1492	return IRQ_WAKE_THREAD;
1493	}
1494
1495	/*
1496	* Normal IRQ handler for port 0, or the threaded IRQ handler for port
1497	* 1 where we need the reg_dma workaround.
1498	*/
1499	static irqreturn_t vc4_dsi_irq_handler(int irq, void *data)
1500	{
1501	struct vc4_dsi *dsi = data;
1502	u32 stat = DSI_PORT_READ(INT_STAT);
1503	irqreturn_t ret = IRQ_NONE;
1504
1505	DSI_PORT_WRITE(INT_STAT, stat);
1506
1507	dsi_handle_error(dsi, ret: &ret, stat,
1508	DSI_PORT_BIT(INT_ERR_SYNC_ESC), type: "LPDT sync");
1509	dsi_handle_error(dsi, ret: &ret, stat,
1510	DSI_PORT_BIT(INT_ERR_CONTROL), type: "data lane 0 sequence");
1511	dsi_handle_error(dsi, ret: &ret, stat,
1512	DSI_PORT_BIT(INT_ERR_CONT_LP0), type: "LP0 contention");
1513	dsi_handle_error(dsi, ret: &ret, stat,
1514	DSI_PORT_BIT(INT_ERR_CONT_LP1), type: "LP1 contention");
1515	dsi_handle_error(dsi, ret: &ret, stat,
1516	DSI_PORT_BIT(INT_HSTX_TO), type: "HSTX timeout");
1517	dsi_handle_error(dsi, ret: &ret, stat,
1518	DSI_PORT_BIT(INT_LPRX_TO), type: "LPRX timeout");
1519	dsi_handle_error(dsi, ret: &ret, stat,
1520	DSI_PORT_BIT(INT_TA_TO), type: "turnaround timeout");
1521	dsi_handle_error(dsi, ret: &ret, stat,
1522	DSI_PORT_BIT(INT_PR_TO), type: "peripheral reset timeout");
1523
1524	if (stat & ((dsi->variant->port ? DSI1_INT_TXPKT1_DONE :
1525	DSI0_INT_CMDC_DONE_MASK) |
1526	DSI_PORT_BIT(INT_PHY_DIR_RTF))) {
1527	complete(&dsi->xfer_completion);
1528	ret = IRQ_HANDLED;
1529	} else if (stat & DSI_PORT_BIT(INT_HSTX_TO)) {
1530	complete(&dsi->xfer_completion);
1531	dsi->xfer_result = -ETIMEDOUT;
1532	ret = IRQ_HANDLED;
1533	}
1534
1535	return ret;
1536	}
1537
1538	/**
1539	* vc4_dsi_init_phy_clocks - Exposes clocks generated by the analog
1540	* PHY that are consumed by CPRMAN (clk-bcm2835.c).
1541	* @dsi: DSI encoder
1542	*/
1543	static int
1544	vc4_dsi_init_phy_clocks(struct vc4_dsi *dsi)
1545	{
1546	struct device *dev = &dsi->pdev->dev;
1547	const char *parent_name = __clk_get_name(clk: dsi->pll_phy_clock);
1548	static const struct {
1549	const char *name;
1550	int div;
1551	} phy_clocks[] = {
1552	{ "byte", 8 },
1553	{ "ddr2", 4 },
1554	{ "ddr", 2 },
1555	};
1556	int i;
1557
1558	dsi->clk_onecell = devm_kzalloc(dev,
1559	size: sizeof(*dsi->clk_onecell) +
1560	ARRAY_SIZE(phy_clocks) *
1561	sizeof(struct clk_hw *),
1562	GFP_KERNEL);
1563	if (!dsi->clk_onecell)
1564	return -ENOMEM;
1565	dsi->clk_onecell->num = ARRAY_SIZE(phy_clocks);
1566
1567	for (i = 0; i < ARRAY_SIZE(phy_clocks); i++) {
1568	struct clk_fixed_factor *fix = &dsi->phy_clocks[i];
1569	struct clk_init_data init;
1570	char clk_name[16];
1571	int ret;
1572
1573	snprintf(buf: clk_name, size: sizeof(clk_name),
1574	fmt: "dsi%u_%s", dsi->variant->port, phy_clocks[i].name);
1575
1576	/* We just use core fixed factor clock ops for the PHY
1577	* clocks. The clocks are actually gated by the
1578	* PHY_AFEC0_DDRCLK_EN bits, which we should be
1579	* setting if we use the DDR/DDR2 clocks. However,
1580	* vc4_dsi_encoder_enable() is setting up both AFEC0,
1581	* setting both our parent DSI PLL's rate and this
1582	* clock's rate, so it knows if DDR/DDR2 are going to
1583	* be used and could enable the gates itself.
1584	*/
1585	fix->mult = 1;
1586	fix->div = phy_clocks[i].div;
1587	fix->hw.init = &init;
1588
1589	memset(&init, 0, sizeof(init));
1590	init.parent_names = &parent_name;
1591	init.num_parents = 1;
1592	init.name = clk_name;
1593	init.ops = &clk_fixed_factor_ops;
1594
1595	ret = devm_clk_hw_register(dev, hw: &fix->hw);
1596	if (ret)
1597	return ret;
1598
1599	dsi->clk_onecell->hws[i] = &fix->hw;
1600	}
1601
1602	return of_clk_add_hw_provider(np: dev->of_node,
1603	get: of_clk_hw_onecell_get,
1604	data: dsi->clk_onecell);
1605	}
1606
1607	static void vc4_dsi_dma_mem_release(void *ptr)
1608	{
1609	struct vc4_dsi *dsi = ptr;
1610	struct device *dev = &dsi->pdev->dev;
1611
1612	dma_free_coherent(dev, size: 4, cpu_addr: dsi->reg_dma_mem, dma_handle: dsi->reg_dma_paddr);
1613	dsi->reg_dma_mem = NULL;
1614	}
1615
1616	static void vc4_dsi_dma_chan_release(void *ptr)
1617	{
1618	struct vc4_dsi *dsi = ptr;
1619
1620	dma_release_channel(chan: dsi->reg_dma_chan);
1621	dsi->reg_dma_chan = NULL;
1622	}
1623
1624	static void vc4_dsi_release_action(struct drm_device *drm, void *ptr)
1625	{
1626	struct vc4_dsi *dsi = ptr;
1627
1628	drm_bridge_put(bridge: &dsi->bridge);
1629	}
1630
1631	static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
1632	{
1633	struct platform_device *pdev = to_platform_device(dev);
1634	struct drm_device *drm = dev_get_drvdata(dev: master);
1635	struct vc4_dsi *dsi = dev_get_drvdata(dev);
1636	struct drm_encoder *encoder = &dsi->encoder.base;
1637	int ret;
1638
1639	drm_bridge_get(bridge: &dsi->bridge);
1640
1641	ret = drmm_add_action_or_reset(drm, vc4_dsi_release_action, dsi);
1642	if (ret)
1643	return ret;
1644
1645	dsi->variant = of_device_get_match_data(dev);
1646
1647	dsi->encoder.type = dsi->variant->port ?
1648	VC4_ENCODER_TYPE_DSI1 : VC4_ENCODER_TYPE_DSI0;
1649
1650	dsi->regs = vc4_ioremap_regs(dev: pdev, index: 0);
1651	if (IS_ERR(ptr: dsi->regs))
1652	return PTR_ERR(ptr: dsi->regs);
1653
1654	dsi->regset.base = dsi->regs;
1655	dsi->regset.regs = dsi->variant->regs;
1656	dsi->regset.nregs = dsi->variant->nregs;
1657
1658	if (DSI_PORT_READ(ID) != DSI_ID_VALUE) {
1659	dev_err(dev, "Port returned 0x%08x for ID instead of 0x%08x\n",
1660	DSI_PORT_READ(ID), DSI_ID_VALUE);
1661	return -ENODEV;
1662	}
1663
1664	/* DSI1 on BCM2835/6/7 has a broken AXI slave that doesn't respond to
1665	* writes from the ARM. It does handle writes from the DMA engine,
1666	* so set up a channel for talking to it.
1667	*/
1668	if (dsi->variant->broken_axi_workaround) {
1669	dma_cap_mask_t dma_mask;
1670
1671	dsi->reg_dma_mem = dma_alloc_coherent(dev, size: 4,
1672	dma_handle: &dsi->reg_dma_paddr,
1673	GFP_KERNEL);
1674	if (!dsi->reg_dma_mem) {
1675	drm_err(drm, "Failed to get DMA memory\n");
1676	return -ENOMEM;
1677	}
1678
1679	ret = devm_add_action_or_reset(dev, vc4_dsi_dma_mem_release, dsi);
1680	if (ret)
1681	return ret;
1682
1683	dma_cap_zero(dma_mask);
1684	dma_cap_set(DMA_MEMCPY, dma_mask);
1685
1686	dsi->reg_dma_chan = dma_request_chan_by_mask(mask: &dma_mask);
1687	if (IS_ERR(ptr: dsi->reg_dma_chan)) {
1688	ret = PTR_ERR(ptr: dsi->reg_dma_chan);
1689	if (ret != -EPROBE_DEFER)
1690	drm_err(drm, "Failed to get DMA channel: %d\n",
1691	ret);
1692	return ret;
1693	}
1694
1695	ret = devm_add_action_or_reset(dev, vc4_dsi_dma_chan_release, dsi);
1696	if (ret)
1697	return ret;
1698
1699	/* Get the physical address of the device's registers. The
1700	* struct resource for the regs gives us the bus address
1701	* instead.
1702	*/
1703	dsi->reg_paddr = be32_to_cpup(p: of_get_address(dev: dev->of_node,
1704	index: 0, NULL, NULL));
1705	}
1706
1707	init_completion(x: &dsi->xfer_completion);
1708	/* At startup enable error-reporting interrupts and nothing else. */
1709	DSI_PORT_WRITE(INT_EN, DSI1_INTERRUPTS_ALWAYS_ENABLED);
1710	/* Clear any existing interrupt state. */
1711	DSI_PORT_WRITE(INT_STAT, DSI_PORT_READ(INT_STAT));
1712
1713	if (dsi->reg_dma_mem)
1714	ret = devm_request_threaded_irq(dev, irq: platform_get_irq(pdev, 0),
1715	handler: vc4_dsi_irq_defer_to_thread_handler,
1716	thread_fn: vc4_dsi_irq_handler,
1717	IRQF_ONESHOT,
1718	devname: "vc4 dsi", dev_id: dsi);
1719	else
1720	ret = devm_request_irq(dev, irq: platform_get_irq(pdev, 0),
1721	handler: vc4_dsi_irq_handler, irqflags: 0, devname: "vc4 dsi", dev_id: dsi);
1722	if (ret) {
1723	if (ret != -EPROBE_DEFER)
1724	dev_err(dev, "Failed to get interrupt: %d\n", ret);
1725	return ret;
1726	}
1727
1728	dsi->escape_clock = devm_clk_get(dev, id: "escape");
1729	if (IS_ERR(ptr: dsi->escape_clock)) {
1730	ret = PTR_ERR(ptr: dsi->escape_clock);
1731	if (ret != -EPROBE_DEFER)
1732	dev_err(dev, "Failed to get escape clock: %d\n", ret);
1733	return ret;
1734	}
1735
1736	dsi->pll_phy_clock = devm_clk_get(dev, id: "phy");
1737	if (IS_ERR(ptr: dsi->pll_phy_clock)) {
1738	ret = PTR_ERR(ptr: dsi->pll_phy_clock);
1739	if (ret != -EPROBE_DEFER)
1740	dev_err(dev, "Failed to get phy clock: %d\n", ret);
1741	return ret;
1742	}
1743
1744	dsi->pixel_clock = devm_clk_get(dev, id: "pixel");
1745	if (IS_ERR(ptr: dsi->pixel_clock)) {
1746	ret = PTR_ERR(ptr: dsi->pixel_clock);
1747	if (ret != -EPROBE_DEFER)
1748	dev_err(dev, "Failed to get pixel clock: %d\n", ret);
1749	return ret;
1750	}
1751
1752	dsi->out_bridge = drmm_of_get_bridge(drm, node: dev->of_node, port: 0, endpoint: 0);
1753	if (IS_ERR(ptr: dsi->out_bridge))
1754	return PTR_ERR(ptr: dsi->out_bridge);
1755
1756	/* The esc clock rate is supposed to always be 100Mhz. */
1757	ret = clk_set_rate(clk: dsi->escape_clock, rate: 100 * 1000000);
1758	if (ret) {
1759	dev_err(dev, "Failed to set esc clock: %d\n", ret);
1760	return ret;
1761	}
1762
1763	ret = vc4_dsi_init_phy_clocks(dsi);
1764	if (ret)
1765	return ret;
1766
1767	ret = drmm_encoder_init(dev: drm, encoder,
1768	funcs: &vc4_dsi_encoder_funcs,
1769	DRM_MODE_ENCODER_DSI,
1770	NULL);
1771	if (ret)
1772	return ret;
1773
1774	ret = devm_pm_runtime_enable(dev);
1775	if (ret)
1776	return ret;
1777
1778	ret = drm_bridge_attach(encoder, bridge: &dsi->bridge, NULL, flags: 0);
1779	if (ret)
1780	return ret;
1781
1782	return 0;
1783	}
1784
1785	static const struct component_ops vc4_dsi_ops = {
1786	.bind = vc4_dsi_bind,
1787	};
1788
1789	static int vc4_dsi_dev_probe(struct platform_device *pdev)
1790	{
1791	struct device *dev = &pdev->dev;
1792	struct vc4_dsi *dsi;
1793
1794	dsi = devm_drm_bridge_alloc(&pdev->dev, struct vc4_dsi, bridge, &vc4_dsi_bridge_funcs);
1795	if (IS_ERR(ptr: dsi))
1796	return PTR_ERR(ptr: dsi);
1797	dev_set_drvdata(dev, data: dsi);
1798
1799	dsi->pdev = pdev;
1800	#ifdef CONFIG_OF
1801	dsi->bridge.of_node = dev->of_node;
1802	#endif
1803	dsi->bridge.type = DRM_MODE_CONNECTOR_DSI;
1804	dsi->dsi_host.ops = &vc4_dsi_host_ops;
1805	dsi->dsi_host.dev = dev;
1806	mipi_dsi_host_register(host: &dsi->dsi_host);
1807
1808	return 0;
1809	}
1810
1811	static void vc4_dsi_dev_remove(struct platform_device *pdev)
1812	{
1813	struct device *dev = &pdev->dev;
1814	struct vc4_dsi *dsi = dev_get_drvdata(dev);
1815
1816	mipi_dsi_host_unregister(host: &dsi->dsi_host);
1817	}
1818
1819	struct platform_driver vc4_dsi_driver = {
1820	.probe = vc4_dsi_dev_probe,
1821	.remove = vc4_dsi_dev_remove,
1822	.driver = {
1823	.name = "vc4_dsi",
1824	.of_match_table = vc4_dsi_dt_match,
1825	},
1826	};
1827