1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (c) 2006 ARM Ltd.
4	* Copyright (c) 2010 ST-Ericsson SA
5	* Copyright (c) 2017 Linaro Ltd.
6	*
7	* Author: Peter Pearse <peter.pearse@arm.com>
8	* Author: Linus Walleij <linus.walleij@linaro.org>
9	*
10	* Documentation: ARM DDI 0196G == PL080
11	* Documentation: ARM DDI 0218E == PL081
12	* Documentation: S3C6410 User's Manual == PL080S
13	*
14	* PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
15	* channel.
16	*
17	* The PL080 has 8 channels available for simultaneous use, and the PL081
18	* has only two channels. So on these DMA controllers the number of channels
19	* and the number of incoming DMA signals are two totally different things.
20	* It is usually not possible to theoretically handle all physical signals,
21	* so a multiplexing scheme with possible denial of use is necessary.
22	*
23	* The PL080 has a dual bus master, PL081 has a single master.
24	*
25	* PL080S is a version modified by Samsung and used in S3C64xx SoCs.
26	* It differs in following aspects:
27	* - CH_CONFIG register at different offset,
28	* - separate CH_CONTROL2 register for transfer size,
29	* - bigger maximum transfer size,
30	* - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
31	* - no support for peripheral flow control.
32	*
33	* Memory to peripheral transfer may be visualized as
34	* Get data from memory to DMAC
35	* Until no data left
36	* On burst request from peripheral
37	* Destination burst from DMAC to peripheral
38	* Clear burst request
39	* Raise terminal count interrupt
40	*
41	* For peripherals with a FIFO:
42	* Source burst size == half the depth of the peripheral FIFO
43	* Destination burst size == the depth of the peripheral FIFO
44	*
45	* (Bursts are irrelevant for mem to mem transfers - there are no burst
46	* signals, the DMA controller will simply facilitate its AHB master.)
47	*
48	* ASSUMES default (little) endianness for DMA transfers
49	*
50	* The PL08x has two flow control settings:
51	* - DMAC flow control: the transfer size defines the number of transfers
52	* which occur for the current LLI entry, and the DMAC raises TC at the
53	* end of every LLI entry. Observed behaviour shows the DMAC listening
54	* to both the BREQ and SREQ signals (contrary to documented),
55	* transferring data if either is active. The LBREQ and LSREQ signals
56	* are ignored.
57	*
58	* - Peripheral flow control: the transfer size is ignored (and should be
59	* zero). The data is transferred from the current LLI entry, until
60	* after the final transfer signalled by LBREQ or LSREQ. The DMAC
61	* will then move to the next LLI entry. Unsupported by PL080S.
62	*/
63	#include <linux/amba/bus.h>
64	#include <linux/amba/pl08x.h>
65	#include <linux/debugfs.h>
66	#include <linux/delay.h>
67	#include <linux/device.h>
68	#include <linux/dmaengine.h>
69	#include <linux/dmapool.h>
70	#include <linux/dma-mapping.h>
71	#include <linux/export.h>
72	#include <linux/init.h>
73	#include <linux/interrupt.h>
74	#include <linux/module.h>
75	#include <linux/of.h>
76	#include <linux/of_dma.h>
77	#include <linux/pm_runtime.h>
78	#include <linux/seq_file.h>
79	#include <linux/slab.h>
80	#include <linux/amba/pl080.h>
81
82	#include "dmaengine.h"
83	#include "virt-dma.h"
84
85	#define DRIVER_NAME "pl08xdmac"
86
87	#define PL80X_DMA_BUSWIDTHS \
88	BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
89	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
90	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
91	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
92
93	static struct amba_driver pl08x_amba_driver;
94	struct pl08x_driver_data;
95
96	/**
97	* struct vendor_data - vendor-specific config parameters for PL08x derivatives
98	* @config_offset: offset to the configuration register
99	* @channels: the number of channels available in this variant
100	* @signals: the number of request signals available from the hardware
101	* @dualmaster: whether this version supports dual AHB masters or not.
102	* @nomadik: whether this variant is a ST Microelectronics Nomadik, where the
103	* channels have Nomadik security extension bits that need to be checked
104	* for permission before use and some registers are missing
105	* @pl080s: whether this variant is a Samsung PL080S, which has separate
106	* register and LLI word for transfer size.
107	* @ftdmac020: whether this variant is a Faraday Technology FTDMAC020
108	* @max_transfer_size: the maximum single element transfer size for this
109	* PL08x variant.
110	*/
111	struct vendor_data {
112	u8 config_offset;
113	u8 channels;
114	u8 signals;
115	bool dualmaster;
116	bool nomadik;
117	bool pl080s;
118	bool ftdmac020;
119	u32 max_transfer_size;
120	};
121
122	/**
123	* struct pl08x_bus_data - information of source or destination
124	* busses for a transfer
125	* @addr: current address
126	* @maxwidth: the maximum width of a transfer on this bus
127	* @buswidth: the width of this bus in bytes: 1, 2 or 4
128	*/
129	struct pl08x_bus_data {
130	dma_addr_t addr;
131	u8 maxwidth;
132	u8 buswidth;
133	};
134
135	#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
136
137	/**
138	* struct pl08x_phy_chan - holder for the physical channels
139	* @id: physical index to this channel
140	* @base: memory base address for this physical channel
141	* @reg_config: configuration address for this physical channel
142	* @reg_control: control address for this physical channel
143	* @reg_src: transfer source address register
144	* @reg_dst: transfer destination address register
145	* @reg_lli: transfer LLI address register
146	* @reg_busy: if the variant has a special per-channel busy register,
147	* this contains a pointer to it
148	* @lock: a lock to use when altering an instance of this struct
149	* @serving: the virtual channel currently being served by this physical
150	* channel
151	* @locked: channel unavailable for the system, e.g. dedicated to secure
152	* world
153	* @ftdmac020: channel is on a FTDMAC020
154	* @pl080s: channel is on a PL08s
155	*/
156	struct pl08x_phy_chan {
157	unsigned int id;
158	void __iomem *base;
159	void __iomem *reg_config;
160	void __iomem *reg_control;
161	void __iomem *reg_src;
162	void __iomem *reg_dst;
163	void __iomem *reg_lli;
164	void __iomem *reg_busy;
165	spinlock_t lock;
166	struct pl08x_dma_chan *serving;
167	bool locked;
168	bool ftdmac020;
169	bool pl080s;
170	};
171
172	/**
173	* struct pl08x_sg - structure containing data per sg
174	* @src_addr: src address of sg
175	* @dst_addr: dst address of sg
176	* @len: transfer len in bytes
177	* @node: node for txd's dsg_list
178	*/
179	struct pl08x_sg {
180	dma_addr_t src_addr;
181	dma_addr_t dst_addr;
182	size_t len;
183	struct list_head node;
184	};
185
186	/**
187	* struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
188	* @vd: virtual DMA descriptor
189	* @dsg_list: list of children sg's
190	* @llis_bus: DMA memory address (physical) start for the LLIs
191	* @llis_va: virtual memory address start for the LLIs
192	* @cctl: control reg values for current txd
193	* @ccfg: config reg values for current txd
194	* @done: this marks completed descriptors, which should not have their
195	* mux released.
196	* @cyclic: indicate cyclic transfers
197	*/
198	struct pl08x_txd {
199	struct virt_dma_desc vd;
200	struct list_head dsg_list;
201	dma_addr_t llis_bus;
202	u32 *llis_va;
203	/* Default cctl value for LLIs */
204	u32 cctl;
205	/*
206	* Settings to be put into the physical channel when we
207	* trigger this txd. Other registers are in llis_va[0].
208	*/
209	u32 ccfg;
210	bool done;
211	bool cyclic;
212	};
213
214	/**
215	* enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
216	* states
217	* @PL08X_CHAN_IDLE: the channel is idle
218	* @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
219	* channel and is running a transfer on it
220	* @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
221	* channel, but the transfer is currently paused
222	* @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
223	* channel to become available (only pertains to memcpy channels)
224	*/
225	enum pl08x_dma_chan_state {
226	PL08X_CHAN_IDLE,
227	PL08X_CHAN_RUNNING,
228	PL08X_CHAN_PAUSED,
229	PL08X_CHAN_WAITING,
230	};
231
232	/**
233	* struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
234	* @vc: wrapped virtual channel
235	* @phychan: the physical channel utilized by this channel, if there is one
236	* @name: name of channel
237	* @cd: channel platform data
238	* @cfg: slave configuration
239	* @at: active transaction on this channel
240	* @host: a pointer to the host (internal use)
241	* @state: whether the channel is idle, paused, running etc
242	* @slave: whether this channel is a device (slave) or for memcpy
243	* @signal: the physical DMA request signal which this channel is using
244	* @mux_use: count of descriptors using this DMA request signal setting
245	* @waiting_at: time in jiffies when this channel moved to waiting state
246	*/
247	struct pl08x_dma_chan {
248	struct virt_dma_chan vc;
249	struct pl08x_phy_chan *phychan;
250	const char *name;
251	struct pl08x_channel_data *cd;
252	struct dma_slave_config cfg;
253	struct pl08x_txd *at;
254	struct pl08x_driver_data *host;
255	enum pl08x_dma_chan_state state;
256	bool slave;
257	int signal;
258	unsigned mux_use;
259	unsigned long waiting_at;
260	};
261
262	/**
263	* struct pl08x_driver_data - the local state holder for the PL08x
264	* @slave: optional slave engine for this instance
265	* @memcpy: memcpy engine for this instance
266	* @has_slave: the PL08x has a slave engine (routed signals)
267	* @base: virtual memory base (remapped) for the PL08x
268	* @adev: the corresponding AMBA (PrimeCell) bus entry
269	* @vd: vendor data for this PL08x variant
270	* @pd: platform data passed in from the platform/machine
271	* @phy_chans: array of data for the physical channels
272	* @pool: a pool for the LLI descriptors
273	* @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
274	* fetches
275	* @mem_buses: set to indicate memory transfers on AHB2.
276	* @lli_words: how many words are used in each LLI item for this variant
277	*/
278	struct pl08x_driver_data {
279	struct dma_device slave;
280	struct dma_device memcpy;
281	bool has_slave;
282	void __iomem *base;
283	struct amba_device *adev;
284	const struct vendor_data *vd;
285	struct pl08x_platform_data *pd;
286	struct pl08x_phy_chan *phy_chans;
287	struct dma_pool *pool;
288	u8 lli_buses;
289	u8 mem_buses;
290	u8 lli_words;
291	};
292
293	/*
294	* PL08X specific defines
295	*/
296
297	/* The order of words in an LLI. */
298	#define PL080_LLI_SRC 0
299	#define PL080_LLI_DST 1
300	#define PL080_LLI_LLI 2
301	#define PL080_LLI_CCTL 3
302	#define PL080S_LLI_CCTL2 4
303
304	/* Total words in an LLI. */
305	#define PL080_LLI_WORDS 4
306	#define PL080S_LLI_WORDS 8
307
308	/*
309	* Number of LLIs in each LLI buffer allocated for one transfer
310	* (maximum times we call dma_pool_alloc on this pool without freeing)
311	*/
312	#define MAX_NUM_TSFR_LLIS 512
313	#define PL08X_ALIGN 8
314
315	static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
316	{
317	return container_of(chan, struct pl08x_dma_chan, vc.chan);
318	}
319
320	static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
321	{
322	return container_of(tx, struct pl08x_txd, vd.tx);
323	}
324
325	/*
326	* Mux handling.
327	*
328	* This gives us the DMA request input to the PL08x primecell which the
329	* peripheral described by the channel data will be routed to, possibly
330	* via a board/SoC specific external MUX. One important point to note
331	* here is that this does not depend on the physical channel.
332	*/
333	static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
334	{
335	const struct pl08x_platform_data *pd = plchan->host->pd;
336	int ret;
337
338	if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
339	ret = pd->get_xfer_signal(plchan->cd);
340	if (ret < 0) {
341	plchan->mux_use = 0;
342	return ret;
343	}
344
345	plchan->signal = ret;
346	}
347	return 0;
348	}
349
350	static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
351	{
352	const struct pl08x_platform_data *pd = plchan->host->pd;
353
354	if (plchan->signal >= 0) {
355	WARN_ON(plchan->mux_use == 0);
356
357	if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
358	pd->put_xfer_signal(plchan->cd, plchan->signal);
359	plchan->signal = -1;
360	}
361	}
362	}
363
364	/*
365	* Physical channel handling
366	*/
367
368	/* Whether a certain channel is busy or not */
369	static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
370	{
371	unsigned int val;
372
373	/* If we have a special busy register, take a shortcut */
374	if (ch->reg_busy) {
375	val = readl(addr: ch->reg_busy);
376	return !!(val & BIT(ch->id));
377	}
378	val = readl(addr: ch->reg_config);
379	return val & PL080_CONFIG_ACTIVE;
380	}
381
382	/*
383	* pl08x_write_lli() - Write an LLI into the DMA controller.
384	*
385	* The PL08x derivatives support linked lists, but the first item of the
386	* list containing the source, destination, control word and next LLI is
387	* ignored. Instead the driver has to write those values directly into the
388	* SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE
389	* register need to be set up for the first transfer.
390	*/
391	static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
392	struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
393	{
394	if (pl08x->vd->pl080s)
395	dev_vdbg(&pl08x->adev->dev,
396	"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
397	"clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
398	phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
399	lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
400	lli[PL080S_LLI_CCTL2], ccfg);
401	else
402	dev_vdbg(&pl08x->adev->dev,
403	"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
404	"clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
405	phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
406	lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
407
408	writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src);
409	writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst);
410	writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli);
411
412	/*
413	* The FTMAC020 has a different layout in the CCTL word of the LLI
414	* and the CCTL register which is split in CSR and SIZE registers.
415	* Convert the LLI item CCTL into the proper values to write into
416	* the CSR and SIZE registers.
417	*/
418	if (phychan->ftdmac020) {
419	u32 llictl = lli[PL080_LLI_CCTL];
420	u32 val = 0;
421
422	/* Write the transfer size (12 bits) to the size register */
423	writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK,
424	phychan->base + FTDMAC020_CH_SIZE);
425	/*
426	* Then write the control bits 28..16 to the control register
427	* by shuffleing the bits around to where they are in the
428	* main register. The mapping is as follows:
429	* Bit 28: TC_MSK - mask on all except last LLI
430	* Bit 27..25: SRC_WIDTH
431	* Bit 24..22: DST_WIDTH
432	* Bit 21..20: SRCAD_CTRL
433	* Bit 19..17: DSTAD_CTRL
434	* Bit 17: SRC_SEL
435	* Bit 16: DST_SEL
436	*/
437	if (llictl & FTDMAC020_LLI_TC_MSK)
438	val |= FTDMAC020_CH_CSR_TC_MSK;
439	val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
440	(FTDMAC020_LLI_SRC_WIDTH_SHIFT -
441	FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT));
442	val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
443	(FTDMAC020_LLI_DST_WIDTH_SHIFT -
444	FTDMAC020_CH_CSR_DST_WIDTH_SHIFT));
445	val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >>
446	(FTDMAC020_LLI_SRCAD_CTL_SHIFT -
447	FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT));
448	val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >>
449	(FTDMAC020_LLI_DSTAD_CTL_SHIFT -
450	FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT));
451	if (llictl & FTDMAC020_LLI_SRC_SEL)
452	val |= FTDMAC020_CH_CSR_SRC_SEL;
453	if (llictl & FTDMAC020_LLI_DST_SEL)
454	val |= FTDMAC020_CH_CSR_DST_SEL;
455
456	/*
457	* Set up the bits that exist in the CSR but are not
458	* part the LLI, i.e. only gets written to the control
459	* register right here.
460	*
461	* FIXME: do not just handle memcpy, also handle slave DMA.
462	*/
463	switch (pl08x->pd->memcpy_burst_size) {
464	default:
465	case PL08X_BURST_SZ_1:
466	val |= PL080_BSIZE_1 <<
467	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
468	break;
469	case PL08X_BURST_SZ_4:
470	val |= PL080_BSIZE_4 <<
471	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
472	break;
473	case PL08X_BURST_SZ_8:
474	val |= PL080_BSIZE_8 <<
475	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
476	break;
477	case PL08X_BURST_SZ_16:
478	val |= PL080_BSIZE_16 <<
479	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
480	break;
481	case PL08X_BURST_SZ_32:
482	val |= PL080_BSIZE_32 <<
483	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
484	break;
485	case PL08X_BURST_SZ_64:
486	val |= PL080_BSIZE_64 <<
487	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
488	break;
489	case PL08X_BURST_SZ_128:
490	val |= PL080_BSIZE_128 <<
491	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
492	break;
493	case PL08X_BURST_SZ_256:
494	val |= PL080_BSIZE_256 <<
495	FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
496	break;
497	}
498
499	/* Protection flags */
500	if (pl08x->pd->memcpy_prot_buff)
501	val |= FTDMAC020_CH_CSR_PROT2;
502	if (pl08x->pd->memcpy_prot_cache)
503	val |= FTDMAC020_CH_CSR_PROT3;
504	/* We are the kernel, so we are in privileged mode */
505	val |= FTDMAC020_CH_CSR_PROT1;
506
507	writel_relaxed(val, phychan->reg_control);
508	} else {
509	/* Bits are just identical */
510	writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control);
511	}
512
513	/* Second control word on the PL080s */
514	if (pl08x->vd->pl080s)
515	writel_relaxed(lli[PL080S_LLI_CCTL2],
516	phychan->base + PL080S_CH_CONTROL2);
517
518	writel(val: ccfg, addr: phychan->reg_config);
519	}
520
521	/*
522	* Set the initial DMA register values i.e. those for the first LLI
523	* The next LLI pointer and the configuration interrupt bit have
524	* been set when the LLIs were constructed. Poke them into the hardware
525	* and start the transfer.
526	*/
527	static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
528	{
529	struct pl08x_driver_data *pl08x = plchan->host;
530	struct pl08x_phy_chan *phychan = plchan->phychan;
531	struct virt_dma_desc *vd = vchan_next_desc(vc: &plchan->vc);
532	struct pl08x_txd *txd = to_pl08x_txd(tx: &vd->tx);
533	u32 val;
534
535	list_del(entry: &txd->vd.node);
536
537	plchan->at = txd;
538
539	/* Wait for channel inactive */
540	while (pl08x_phy_channel_busy(ch: phychan))
541	cpu_relax();
542
543	pl08x_write_lli(pl08x, phychan, lli: &txd->llis_va[0], ccfg: txd->ccfg);
544
545	/* Enable the DMA channel */
546	/* Do not access config register until channel shows as disabled */
547	while (readl(addr: pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))
548	cpu_relax();
549
550	/* Do not access config register until channel shows as inactive */
551	if (phychan->ftdmac020) {
552	val = readl(addr: phychan->reg_config);
553	while (val & FTDMAC020_CH_CFG_BUSY)
554	val = readl(addr: phychan->reg_config);
555
556	val = readl(addr: phychan->reg_control);
557	while (val & FTDMAC020_CH_CSR_EN)
558	val = readl(addr: phychan->reg_control);
559
560	writel(val: val | FTDMAC020_CH_CSR_EN,
561	addr: phychan->reg_control);
562	} else {
563	val = readl(addr: phychan->reg_config);
564	while ((val & PL080_CONFIG_ACTIVE) ||
565	(val & PL080_CONFIG_ENABLE))
566	val = readl(addr: phychan->reg_config);
567
568	writel(val: val | PL080_CONFIG_ENABLE, addr: phychan->reg_config);
569	}
570	}
571
572	/*
573	* Pause the channel by setting the HALT bit.
574	*
575	* For M->P transfers, pause the DMAC first and then stop the peripheral -
576	* the FIFO can only drain if the peripheral is still requesting data.
577	* (note: this can still timeout if the DMAC FIFO never drains of data.)
578	*
579	* For P->M transfers, disable the peripheral first to stop it filling
580	* the DMAC FIFO, and then pause the DMAC.
581	*/
582	static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
583	{
584	u32 val;
585	int timeout;
586
587	if (ch->ftdmac020) {
588	/* Use the enable bit on the FTDMAC020 */
589	val = readl(addr: ch->reg_control);
590	val &= ~FTDMAC020_CH_CSR_EN;
591	writel(val, addr: ch->reg_control);
592	return;
593	}
594
595	/* Set the HALT bit and wait for the FIFO to drain */
596	val = readl(addr: ch->reg_config);
597	val |= PL080_CONFIG_HALT;
598	writel(val, addr: ch->reg_config);
599
600	/* Wait for channel inactive */
601	for (timeout = 1000; timeout; timeout--) {
602	if (!pl08x_phy_channel_busy(ch))
603	break;
604	udelay(usec: 1);
605	}
606	if (pl08x_phy_channel_busy(ch))
607	pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
608	}
609
610	static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
611	{
612	u32 val;
613
614	/* Use the enable bit on the FTDMAC020 */
615	if (ch->ftdmac020) {
616	val = readl(addr: ch->reg_control);
617	val |= FTDMAC020_CH_CSR_EN;
618	writel(val, addr: ch->reg_control);
619	return;
620	}
621
622	/* Clear the HALT bit */
623	val = readl(addr: ch->reg_config);
624	val &= ~PL080_CONFIG_HALT;
625	writel(val, addr: ch->reg_config);
626	}
627
628	/*
629	* pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
630	* clears any pending interrupt status. This should not be used for
631	* an on-going transfer, but as a method of shutting down a channel
632	* (eg, when it's no longer used) or terminating a transfer.
633	*/
634	static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
635	struct pl08x_phy_chan *ch)
636	{
637	u32 val;
638
639	/* The layout for the FTDMAC020 is different */
640	if (ch->ftdmac020) {
641	/* Disable all interrupts */
642	val = readl(addr: ch->reg_config);
643	val |= (FTDMAC020_CH_CFG_INT_ABT_MASK |
644	FTDMAC020_CH_CFG_INT_ERR_MASK |
645	FTDMAC020_CH_CFG_INT_TC_MASK);
646	writel(val, addr: ch->reg_config);
647
648	/* Abort and disable channel */
649	val = readl(addr: ch->reg_control);
650	val &= ~FTDMAC020_CH_CSR_EN;
651	val |= FTDMAC020_CH_CSR_ABT;
652	writel(val, addr: ch->reg_control);
653
654	/* Clear ABT and ERR interrupt flags */
655	writel(BIT(ch->id) | BIT(ch->id + 16),
656	addr: pl08x->base + PL080_ERR_CLEAR);
657	writel(BIT(ch->id), addr: pl08x->base + PL080_TC_CLEAR);
658
659	return;
660	}
661
662	val = readl(addr: ch->reg_config);
663	val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
664	PL080_CONFIG_TC_IRQ_MASK);
665	writel(val, addr: ch->reg_config);
666
667	writel(BIT(ch->id), addr: pl08x->base + PL080_ERR_CLEAR);
668	writel(BIT(ch->id), addr: pl08x->base + PL080_TC_CLEAR);
669	}
670
671	static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch)
672	{
673	u32 val;
674	u32 bytes;
675
676	if (ch->ftdmac020) {
677	bytes = readl(addr: ch->base + FTDMAC020_CH_SIZE);
678
679	val = readl(addr: ch->reg_control);
680	val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK;
681	val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT;
682	} else if (ch->pl080s) {
683	val = readl(addr: ch->base + PL080S_CH_CONTROL2);
684	bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
685
686	val = readl(addr: ch->reg_control);
687	val &= PL080_CONTROL_SWIDTH_MASK;
688	val >>= PL080_CONTROL_SWIDTH_SHIFT;
689	} else {
690	/* Plain PL08x */
691	val = readl(addr: ch->reg_control);
692	bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
693
694	val &= PL080_CONTROL_SWIDTH_MASK;
695	val >>= PL080_CONTROL_SWIDTH_SHIFT;
696	}
697
698	switch (val) {
699	case PL080_WIDTH_8BIT:
700	break;
701	case PL080_WIDTH_16BIT:
702	bytes *= 2;
703	break;
704	case PL080_WIDTH_32BIT:
705	bytes *= 4;
706	break;
707	}
708	return bytes;
709	}
710
711	static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va)
712	{
713	u32 val;
714	u32 bytes;
715
716	if (ch->ftdmac020) {
717	val = llis_va[PL080_LLI_CCTL];
718	bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK;
719
720	val = llis_va[PL080_LLI_CCTL];
721	val &= FTDMAC020_LLI_SRC_WIDTH_MSK;
722	val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT;
723	} else if (ch->pl080s) {
724	val = llis_va[PL080S_LLI_CCTL2];
725	bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
726
727	val = llis_va[PL080_LLI_CCTL];
728	val &= PL080_CONTROL_SWIDTH_MASK;
729	val >>= PL080_CONTROL_SWIDTH_SHIFT;
730	} else {
731	/* Plain PL08x */
732	val = llis_va[PL080_LLI_CCTL];
733	bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
734
735	val &= PL080_CONTROL_SWIDTH_MASK;
736	val >>= PL080_CONTROL_SWIDTH_SHIFT;
737	}
738
739	switch (val) {
740	case PL080_WIDTH_8BIT:
741	break;
742	case PL080_WIDTH_16BIT:
743	bytes *= 2;
744	break;
745	case PL080_WIDTH_32BIT:
746	bytes *= 4;
747	break;
748	}
749	return bytes;
750	}
751
752	/* The channel should be paused when calling this */
753	static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
754	{
755	struct pl08x_driver_data *pl08x = plchan->host;
756	const u32 *llis_va, *llis_va_limit;
757	struct pl08x_phy_chan *ch;
758	dma_addr_t llis_bus;
759	struct pl08x_txd *txd;
760	u32 llis_max_words;
761	size_t bytes;
762	u32 clli;
763
764	ch = plchan->phychan;
765	txd = plchan->at;
766
767	if (!ch || !txd)
768	return 0;
769
770	/*
771	* Follow the LLIs to get the number of remaining
772	* bytes in the currently active transaction.
773	*/
774	clli = readl(addr: ch->reg_lli) & ~PL080_LLI_LM_AHB2;
775
776	/* First get the remaining bytes in the active transfer */
777	bytes = get_bytes_in_phy_channel(ch);
778
779	if (!clli)
780	return bytes;
781
782	llis_va = txd->llis_va;
783	llis_bus = txd->llis_bus;
784
785	llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
786	BUG_ON(clli < llis_bus || clli >= llis_bus +
787	sizeof(u32) * llis_max_words);
788
789	/*
790	* Locate the next LLI - as this is an array,
791	* it's simple maths to find.
792	*/
793	llis_va += (clli - llis_bus) / sizeof(u32);
794
795	llis_va_limit = llis_va + llis_max_words;
796
797	for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
798	bytes += get_bytes_in_lli(ch, llis_va);
799
800	/*
801	* A LLI pointer going backward terminates the LLI list
802	*/
803	if (llis_va[PL080_LLI_LLI] <= clli)
804	break;
805	}
806
807	return bytes;
808	}
809
810	/*
811	* Allocate a physical channel for a virtual channel
812	*
813	* Try to locate a physical channel to be used for this transfer. If all
814	* are taken return NULL and the requester will have to cope by using
815	* some fallback PIO mode or retrying later.
816	*/
817	static struct pl08x_phy_chan *
818	pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
819	struct pl08x_dma_chan *virt_chan)
820	{
821	struct pl08x_phy_chan *ch = NULL;
822	unsigned long flags;
823	int i;
824
825	for (i = 0; i < pl08x->vd->channels; i++) {
826	ch = &pl08x->phy_chans[i];
827
828	spin_lock_irqsave(&ch->lock, flags);
829
830	if (!ch->locked && !ch->serving) {
831	ch->serving = virt_chan;
832	spin_unlock_irqrestore(lock: &ch->lock, flags);
833	break;
834	}
835
836	spin_unlock_irqrestore(lock: &ch->lock, flags);
837	}
838
839	if (i == pl08x->vd->channels) {
840	/* No physical channel available, cope with it */
841	return NULL;
842	}
843
844	return ch;
845	}
846
847	/* Mark the physical channel as free. Note, this write is atomic. */
848	static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
849	struct pl08x_phy_chan *ch)
850	{
851	ch->serving = NULL;
852	}
853
854	/*
855	* Try to allocate a physical channel. When successful, assign it to
856	* this virtual channel, and initiate the next descriptor. The
857	* virtual channel lock must be held at this point.
858	*/
859	static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
860	{
861	struct pl08x_driver_data *pl08x = plchan->host;
862	struct pl08x_phy_chan *ch;
863
864	ch = pl08x_get_phy_channel(pl08x, virt_chan: plchan);
865	if (!ch) {
866	dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
867	plchan->state = PL08X_CHAN_WAITING;
868	plchan->waiting_at = jiffies;
869	return;
870	}
871
872	dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
873	ch->id, plchan->name);
874
875	plchan->phychan = ch;
876	plchan->state = PL08X_CHAN_RUNNING;
877	pl08x_start_next_txd(plchan);
878	}
879
880	static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
881	struct pl08x_dma_chan *plchan)
882	{
883	struct pl08x_driver_data *pl08x = plchan->host;
884
885	dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
886	ch->id, plchan->name);
887
888	/*
889	* We do this without taking the lock; we're really only concerned
890	* about whether this pointer is NULL or not, and we're guaranteed
891	* that this will only be called when it _already_ is non-NULL.
892	*/
893	ch->serving = plchan;
894	plchan->phychan = ch;
895	plchan->state = PL08X_CHAN_RUNNING;
896	pl08x_start_next_txd(plchan);
897	}
898
899	/*
900	* Free a physical DMA channel, potentially reallocating it to another
901	* virtual channel if we have any pending.
902	*/
903	static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
904	{
905	struct pl08x_driver_data *pl08x = plchan->host;
906	struct pl08x_dma_chan *p, *next;
907	unsigned long waiting_at;
908	retry:
909	next = NULL;
910	waiting_at = jiffies;
911
912	/*
913	* Find a waiting virtual channel for the next transfer.
914	* To be fair, time when each channel reached waiting state is compared
915	* to select channel that is waiting for the longest time.
916	*/
917	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
918	if (p->state == PL08X_CHAN_WAITING &&
919	p->waiting_at <= waiting_at) {
920	next = p;
921	waiting_at = p->waiting_at;
922	}
923
924	if (!next && pl08x->has_slave) {
925	list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
926	if (p->state == PL08X_CHAN_WAITING &&
927	p->waiting_at <= waiting_at) {
928	next = p;
929	waiting_at = p->waiting_at;
930	}
931	}
932
933	/* Ensure that the physical channel is stopped */
934	pl08x_terminate_phy_chan(pl08x, ch: plchan->phychan);
935
936	if (next) {
937	bool success;
938
939	/*
940	* Eww. We know this isn't going to deadlock
941	* but lockdep probably doesn't.
942	*/
943	spin_lock(lock: &next->vc.lock);
944	/* Re-check the state now that we have the lock */
945	success = next->state == PL08X_CHAN_WAITING;
946	if (success)
947	pl08x_phy_reassign_start(ch: plchan->phychan, plchan: next);
948	spin_unlock(lock: &next->vc.lock);
949
950	/* If the state changed, try to find another channel */
951	if (!success)
952	goto retry;
953	} else {
954	/* No more jobs, so free up the physical channel */
955	pl08x_put_phy_channel(pl08x, ch: plchan->phychan);
956	}
957
958	plchan->phychan = NULL;
959	plchan->state = PL08X_CHAN_IDLE;
960	}
961
962	/*
963	* LLI handling
964	*/
965
966	static inline unsigned int
967	pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x,
968	u32 cctl,
969	bool source)
970	{
971	u32 val;
972
973	if (pl08x->vd->ftdmac020) {
974	if (source)
975	val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
976	FTDMAC020_LLI_SRC_WIDTH_SHIFT;
977	else
978	val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
979	FTDMAC020_LLI_DST_WIDTH_SHIFT;
980	} else {
981	if (source)
982	val = (cctl & PL080_CONTROL_SWIDTH_MASK) >>
983	PL080_CONTROL_SWIDTH_SHIFT;
984	else
985	val = (cctl & PL080_CONTROL_DWIDTH_MASK) >>
986	PL080_CONTROL_DWIDTH_SHIFT;
987	}
988
989	switch (val) {
990	case PL080_WIDTH_8BIT:
991	return 1;
992	case PL080_WIDTH_16BIT:
993	return 2;
994	case PL080_WIDTH_32BIT:
995	return 4;
996	default:
997	break;
998	}
999	BUG();
1000	return 0;
1001	}
1002
1003	static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,
1004	u32 cctl,
1005	u8 srcwidth, u8 dstwidth,
1006	size_t tsize)
1007	{
1008	u32 retbits = cctl;
1009
1010	/*
1011	* Remove all src, dst and transfer size bits, then set the
1012	* width and size according to the parameters. The bit offsets
1013	* are different in the FTDMAC020 so we need to accound for this.
1014	*/
1015	if (pl08x->vd->ftdmac020) {
1016	retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;
1017	retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK;
1018	retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK;
1019
1020	switch (srcwidth) {
1021	case 1:
1022	retbits |= PL080_WIDTH_8BIT <<
1023	FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1024	break;
1025	case 2:
1026	retbits |= PL080_WIDTH_16BIT <<
1027	FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1028	break;
1029	case 4:
1030	retbits |= PL080_WIDTH_32BIT <<
1031	FTDMAC020_LLI_SRC_WIDTH_SHIFT;
1032	break;
1033	default:
1034	BUG();
1035	break;
1036	}
1037
1038	switch (dstwidth) {
1039	case 1:
1040	retbits |= PL080_WIDTH_8BIT <<
1041	FTDMAC020_LLI_DST_WIDTH_SHIFT;
1042	break;
1043	case 2:
1044	retbits |= PL080_WIDTH_16BIT <<
1045	FTDMAC020_LLI_DST_WIDTH_SHIFT;
1046	break;
1047	case 4:
1048	retbits |= PL080_WIDTH_32BIT <<
1049	FTDMAC020_LLI_DST_WIDTH_SHIFT;
1050	break;
1051	default:
1052	BUG();
1053	break;
1054	}
1055
1056	tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK;
1057	retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT;
1058	} else {
1059	retbits &= ~PL080_CONTROL_DWIDTH_MASK;
1060	retbits &= ~PL080_CONTROL_SWIDTH_MASK;
1061	retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
1062
1063	switch (srcwidth) {
1064	case 1:
1065	retbits |= PL080_WIDTH_8BIT <<
1066	PL080_CONTROL_SWIDTH_SHIFT;
1067	break;
1068	case 2:
1069	retbits |= PL080_WIDTH_16BIT <<
1070	PL080_CONTROL_SWIDTH_SHIFT;
1071	break;
1072	case 4:
1073	retbits |= PL080_WIDTH_32BIT <<
1074	PL080_CONTROL_SWIDTH_SHIFT;
1075	break;
1076	default:
1077	BUG();
1078	break;
1079	}
1080
1081	switch (dstwidth) {
1082	case 1:
1083	retbits |= PL080_WIDTH_8BIT <<
1084	PL080_CONTROL_DWIDTH_SHIFT;
1085	break;
1086	case 2:
1087	retbits |= PL080_WIDTH_16BIT <<
1088	PL080_CONTROL_DWIDTH_SHIFT;
1089	break;
1090	case 4:
1091	retbits |= PL080_WIDTH_32BIT <<
1092	PL080_CONTROL_DWIDTH_SHIFT;
1093	break;
1094	default:
1095	BUG();
1096	break;
1097	}
1098
1099	tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
1100	retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
1101	}
1102
1103	return retbits;
1104	}
1105
1106	struct pl08x_lli_build_data {
1107	struct pl08x_txd *txd;
1108	struct pl08x_bus_data srcbus;
1109	struct pl08x_bus_data dstbus;
1110	size_t remainder;
1111	u32 lli_bus;
1112	};
1113
1114	/*
1115	* Autoselect a master bus to use for the transfer. Slave will be the chosen as
1116	* victim in case src & dest are not similarly aligned. i.e. If after aligning
1117	* masters address with width requirements of transfer (by sending few byte by
1118	* byte data), slave is still not aligned, then its width will be reduced to
1119	* BYTE.
1120	* - prefers the destination bus if both available
1121	* - prefers bus with fixed address (i.e. peripheral)
1122	*/
1123	static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x,
1124	struct pl08x_lli_build_data *bd,
1125	struct pl08x_bus_data **mbus,
1126	struct pl08x_bus_data **sbus,
1127	u32 cctl)
1128	{
1129	bool dst_incr;
1130	bool src_incr;
1131
1132	/*
1133	* The FTDMAC020 only supports memory-to-memory transfer, so
1134	* source and destination always increase.
1135	*/
1136	if (pl08x->vd->ftdmac020) {
1137	dst_incr = true;
1138	src_incr = true;
1139	} else {
1140	dst_incr = !!(cctl & PL080_CONTROL_DST_INCR);
1141	src_incr = !!(cctl & PL080_CONTROL_SRC_INCR);
1142	}
1143
1144	/*
1145	* If either bus is not advancing, i.e. it is a peripheral, that
1146	* one becomes master
1147	*/
1148	if (!dst_incr) {
1149	*mbus = &bd->dstbus;
1150	*sbus = &bd->srcbus;
1151	} else if (!src_incr) {
1152	*mbus = &bd->srcbus;
1153	*sbus = &bd->dstbus;
1154	} else {
1155	if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
1156	*mbus = &bd->dstbus;
1157	*sbus = &bd->srcbus;
1158	} else {
1159	*mbus = &bd->srcbus;
1160	*sbus = &bd->dstbus;
1161	}
1162	}
1163	}
1164
1165	/*
1166	* Fills in one LLI for a certain transfer descriptor and advance the counter
1167	*/
1168	static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
1169	struct pl08x_lli_build_data *bd,
1170	int num_llis, int len, u32 cctl, u32 cctl2)
1171	{
1172	u32 offset = num_llis * pl08x->lli_words;
1173	u32 *llis_va = bd->txd->llis_va + offset;
1174	dma_addr_t llis_bus = bd->txd->llis_bus;
1175
1176	BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
1177
1178	/* Advance the offset to next LLI. */
1179	offset += pl08x->lli_words;
1180
1181	llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
1182	llis_va[PL080_LLI_DST] = bd->dstbus.addr;
1183	llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
1184	llis_va[PL080_LLI_LLI] |= bd->lli_bus;
1185	llis_va[PL080_LLI_CCTL] = cctl;
1186	if (pl08x->vd->pl080s)
1187	llis_va[PL080S_LLI_CCTL2] = cctl2;
1188
1189	if (pl08x->vd->ftdmac020) {
1190	/* FIXME: only memcpy so far so both increase */
1191	bd->srcbus.addr += len;
1192	bd->dstbus.addr += len;
1193	} else {
1194	if (cctl & PL080_CONTROL_SRC_INCR)
1195	bd->srcbus.addr += len;
1196	if (cctl & PL080_CONTROL_DST_INCR)
1197	bd->dstbus.addr += len;
1198	}
1199
1200	BUG_ON(bd->remainder < len);
1201
1202	bd->remainder -= len;
1203	}
1204
1205	static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
1206	struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
1207	int num_llis, size_t *total_bytes)
1208	{
1209	*cctl = pl08x_lli_control_bits(pl08x, cctl: *cctl, srcwidth: 1, dstwidth: 1, tsize: len);
1210	pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, cctl: *cctl, cctl2: len);
1211	(*total_bytes) += len;
1212	}
1213
1214	#if 1
1215	static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
1216	const u32 *llis_va, int num_llis)
1217	{
1218	int i;
1219
1220	if (pl08x->vd->pl080s) {
1221	dev_vdbg(&pl08x->adev->dev,
1222	"%-3s %-9s %-10s %-10s %-10s %-10s %s\n",
1223	"lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
1224	for (i = 0; i < num_llis; i++) {
1225	dev_vdbg(&pl08x->adev->dev,
1226	"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1227	i, llis_va, llis_va[PL080_LLI_SRC],
1228	llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
1229	llis_va[PL080_LLI_CCTL],
1230	llis_va[PL080S_LLI_CCTL2]);
1231	llis_va += pl08x->lli_words;
1232	}
1233	} else {
1234	dev_vdbg(&pl08x->adev->dev,
1235	"%-3s %-9s %-10s %-10s %-10s %s\n",
1236	"lli", "", "csrc", "cdst", "clli", "cctl");
1237	for (i = 0; i < num_llis; i++) {
1238	dev_vdbg(&pl08x->adev->dev,
1239	"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1240	i, llis_va, llis_va[PL080_LLI_SRC],
1241	llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
1242	llis_va[PL080_LLI_CCTL]);
1243	llis_va += pl08x->lli_words;
1244	}
1245	}
1246	}
1247	#else
1248	static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
1249	const u32 *llis_va, int num_llis) {}
1250	#endif
1251
1252	/*
1253	* This fills in the table of LLIs for the transfer descriptor
1254	* Note that we assume we never have to change the burst sizes
1255	* Return 0 for error
1256	*/
1257	static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
1258	struct pl08x_txd *txd)
1259	{
1260	struct pl08x_bus_data *mbus, *sbus;
1261	struct pl08x_lli_build_data bd;
1262	int num_llis = 0;
1263	u32 cctl, early_bytes = 0;
1264	size_t max_bytes_per_lli, total_bytes;
1265	u32 *llis_va, *last_lli;
1266	struct pl08x_sg *dsg;
1267
1268	txd->llis_va = dma_pool_alloc(pool: pl08x->pool, GFP_NOWAIT, handle: &txd->llis_bus);
1269	if (!txd->llis_va) {
1270	dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
1271	return 0;
1272	}
1273
1274	bd.txd = txd;
1275	bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
1276	cctl = txd->cctl;
1277
1278	/* Find maximum width of the source bus */
1279	bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, source: true);
1280
1281	/* Find maximum width of the destination bus */
1282	bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, source: false);
1283
1284	list_for_each_entry(dsg, &txd->dsg_list, node) {
1285	total_bytes = 0;
1286	cctl = txd->cctl;
1287
1288	bd.srcbus.addr = dsg->src_addr;
1289	bd.dstbus.addr = dsg->dst_addr;
1290	bd.remainder = dsg->len;
1291	bd.srcbus.buswidth = bd.srcbus.maxwidth;
1292	bd.dstbus.buswidth = bd.dstbus.maxwidth;
1293
1294	pl08x_choose_master_bus(pl08x, bd: &bd, mbus: &mbus, sbus: &sbus, cctl);
1295
1296	dev_vdbg(&pl08x->adev->dev,
1297	"src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
1298	(u64)bd.srcbus.addr,
1299	cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
1300	bd.srcbus.buswidth,
1301	(u64)bd.dstbus.addr,
1302	cctl & PL080_CONTROL_DST_INCR ? "+" : "",
1303	bd.dstbus.buswidth,
1304	bd.remainder);
1305	dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
1306	mbus == &bd.srcbus ? "src" : "dst",
1307	sbus == &bd.srcbus ? "src" : "dst");
1308
1309	/*
1310	* Zero length is only allowed if all these requirements are
1311	* met:
1312	* - flow controller is peripheral.
1313	* - src.addr is aligned to src.width
1314	* - dst.addr is aligned to dst.width
1315	*
1316	* sg_len == 1 should be true, as there can be two cases here:
1317	*
1318	* - Memory addresses are contiguous and are not scattered.
1319	* Here, Only one sg will be passed by user driver, with
1320	* memory address and zero length. We pass this to controller
1321	* and after the transfer it will receive the last burst
1322	* request from peripheral and so transfer finishes.
1323	*
1324	* - Memory addresses are scattered and are not contiguous.
1325	* Here, Obviously as DMA controller doesn't know when a lli's
1326	* transfer gets over, it can't load next lli. So in this
1327	* case, there has to be an assumption that only one lli is
1328	* supported. Thus, we can't have scattered addresses.
1329	*/
1330	if (!bd.remainder) {
1331	u32 fc;
1332
1333	/* FTDMAC020 only does memory-to-memory */
1334	if (pl08x->vd->ftdmac020)
1335	fc = PL080_FLOW_MEM2MEM;
1336	else
1337	fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
1338	PL080_CONFIG_FLOW_CONTROL_SHIFT;
1339	if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
1340	(fc <= PL080_FLOW_SRC2DST_SRC))) {
1341	dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
1342	__func__);
1343	return 0;
1344	}
1345
1346	if (!IS_BUS_ALIGNED(&bd.srcbus) ||
1347	!IS_BUS_ALIGNED(&bd.dstbus)) {
1348	dev_err(&pl08x->adev->dev,
1349	"%s src & dst address must be aligned to src"
1350	" & dst width if peripheral is flow controller",
1351	__func__);
1352	return 0;
1353	}
1354
1355	cctl = pl08x_lli_control_bits(pl08x, cctl,
1356	srcwidth: bd.srcbus.buswidth, dstwidth: bd.dstbus.buswidth,
1357	tsize: 0);
1358	pl08x_fill_lli_for_desc(pl08x, bd: &bd, num_llis: num_llis++,
1359	len: 0, cctl, cctl2: 0);
1360	break;
1361	}
1362
1363	/*
1364	* Send byte by byte for following cases
1365	* - Less than a bus width available
1366	* - until master bus is aligned
1367	*/
1368	if (bd.remainder < mbus->buswidth)
1369	early_bytes = bd.remainder;
1370	else if (!IS_BUS_ALIGNED(mbus)) {
1371	early_bytes = mbus->buswidth -
1372	(mbus->addr & (mbus->buswidth - 1));
1373	if ((bd.remainder - early_bytes) < mbus->buswidth)
1374	early_bytes = bd.remainder;
1375	}
1376
1377	if (early_bytes) {
1378	dev_vdbg(&pl08x->adev->dev,
1379	"%s byte width LLIs (remain 0x%08zx)\n",
1380	__func__, bd.remainder);
1381	prep_byte_width_lli(pl08x, bd: &bd, cctl: &cctl, len: early_bytes,
1382	num_llis: num_llis++, total_bytes: &total_bytes);
1383	}
1384
1385	if (bd.remainder) {
1386	/*
1387	* Master now aligned
1388	* - if slave is not then we must set its width down
1389	*/
1390	if (!IS_BUS_ALIGNED(sbus)) {
1391	dev_dbg(&pl08x->adev->dev,
1392	"%s set down bus width to one byte\n",
1393	__func__);
1394
1395	sbus->buswidth = 1;
1396	}
1397
1398	/*
1399	* Bytes transferred = tsize * src width, not
1400	* MIN(buswidths)
1401	*/
1402	max_bytes_per_lli = bd.srcbus.buswidth *
1403	pl08x->vd->max_transfer_size;
1404	dev_vdbg(&pl08x->adev->dev,
1405	"%s max bytes per lli = %zu\n",
1406	__func__, max_bytes_per_lli);
1407
1408	/*
1409	* Make largest possible LLIs until less than one bus
1410	* width left
1411	*/
1412	while (bd.remainder > (mbus->buswidth - 1)) {
1413	size_t lli_len, tsize, width;
1414
1415	/*
1416	* If enough left try to send max possible,
1417	* otherwise try to send the remainder
1418	*/
1419	lli_len = min(bd.remainder, max_bytes_per_lli);
1420
1421	/*
1422	* Check against maximum bus alignment:
1423	* Calculate actual transfer size in relation to
1424	* bus width an get a maximum remainder of the
1425	* highest bus width - 1
1426	*/
1427	width = max(mbus->buswidth, sbus->buswidth);
1428	lli_len = (lli_len / width) * width;
1429	tsize = lli_len / bd.srcbus.buswidth;
1430
1431	dev_vdbg(&pl08x->adev->dev,
1432	"%s fill lli with single lli chunk of "
1433	"size 0x%08zx (remainder 0x%08zx)\n",
1434	__func__, lli_len, bd.remainder);
1435
1436	cctl = pl08x_lli_control_bits(pl08x, cctl,
1437	srcwidth: bd.srcbus.buswidth, dstwidth: bd.dstbus.buswidth,
1438	tsize);
1439	pl08x_fill_lli_for_desc(pl08x, bd: &bd, num_llis: num_llis++,
1440	len: lli_len, cctl, cctl2: tsize);
1441	total_bytes += lli_len;
1442	}
1443
1444	/*
1445	* Send any odd bytes
1446	*/
1447	if (bd.remainder) {
1448	dev_vdbg(&pl08x->adev->dev,
1449	"%s align with boundary, send odd bytes (remain %zu)\n",
1450	__func__, bd.remainder);
1451	prep_byte_width_lli(pl08x, bd: &bd, cctl: &cctl,
1452	len: bd.remainder, num_llis: num_llis++, total_bytes: &total_bytes);
1453	}
1454	}
1455
1456	if (total_bytes != dsg->len) {
1457	dev_err(&pl08x->adev->dev,
1458	"%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
1459	__func__, total_bytes, dsg->len);
1460	return 0;
1461	}
1462
1463	if (num_llis >= MAX_NUM_TSFR_LLIS) {
1464	dev_err(&pl08x->adev->dev,
1465	"%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
1466	__func__, MAX_NUM_TSFR_LLIS);
1467	return 0;
1468	}
1469	}
1470
1471	llis_va = txd->llis_va;
1472	last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
1473
1474	if (txd->cyclic) {
1475	/* Link back to the first LLI. */
1476	last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
1477	} else {
1478	/* The final LLI terminates the LLI. */
1479	last_lli[PL080_LLI_LLI] = 0;
1480	/* The final LLI element shall also fire an interrupt. */
1481	if (pl08x->vd->ftdmac020)
1482	last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK;
1483	else
1484	last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
1485	}
1486
1487	pl08x_dump_lli(pl08x, llis_va, num_llis);
1488
1489	return num_llis;
1490	}
1491
1492	static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1493	struct pl08x_txd *txd)
1494	{
1495	struct pl08x_sg *dsg, *_dsg;
1496
1497	if (txd->llis_va)
1498	dma_pool_free(pool: pl08x->pool, vaddr: txd->llis_va, addr: txd->llis_bus);
1499
1500	list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
1501	list_del(entry: &dsg->node);
1502	kfree(objp: dsg);
1503	}
1504
1505	kfree(objp: txd);
1506	}
1507
1508	static void pl08x_desc_free(struct virt_dma_desc *vd)
1509	{
1510	struct pl08x_txd *txd = to_pl08x_txd(tx: &vd->tx);
1511	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan: vd->tx.chan);
1512
1513	dma_descriptor_unmap(tx: &vd->tx);
1514	if (!txd->done)
1515	pl08x_release_mux(plchan);
1516
1517	pl08x_free_txd(pl08x: plchan->host, txd);
1518	}
1519
1520	static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
1521	struct pl08x_dma_chan *plchan)
1522	{
1523	LIST_HEAD(head);
1524
1525	vchan_get_all_descriptors(vc: &plchan->vc, head: &head);
1526	vchan_dma_desc_free_list(vc: &plchan->vc, head: &head);
1527	}
1528
1529	/*
1530	* The DMA ENGINE API
1531	*/
1532	static void pl08x_free_chan_resources(struct dma_chan *chan)
1533	{
1534	/* Ensure all queued descriptors are freed */
1535	vchan_free_chan_resources(vc: to_virt_chan(chan));
1536	}
1537
1538	/*
1539	* Code accessing dma_async_is_complete() in a tight loop may give problems.
1540	* If slaves are relying on interrupts to signal completion this function
1541	* must not be called with interrupts disabled.
1542	*/
1543	static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
1544	dma_cookie_t cookie, struct dma_tx_state *txstate)
1545	{
1546	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1547	struct virt_dma_desc *vd;
1548	unsigned long flags;
1549	enum dma_status ret;
1550	size_t bytes = 0;
1551
1552	ret = dma_cookie_status(chan, cookie, state: txstate);
1553	if (ret == DMA_COMPLETE)
1554	return ret;
1555
1556	/*
1557	* There's no point calculating the residue if there's
1558	* no txstate to store the value.
1559	*/
1560	if (!txstate) {
1561	if (plchan->state == PL08X_CHAN_PAUSED)
1562	ret = DMA_PAUSED;
1563	return ret;
1564	}
1565
1566	spin_lock_irqsave(&plchan->vc.lock, flags);
1567	ret = dma_cookie_status(chan, cookie, state: txstate);
1568	if (ret != DMA_COMPLETE) {
1569	vd = vchan_find_desc(&plchan->vc, cookie);
1570	if (vd) {
1571	/* On the issued list, so hasn't been processed yet */
1572	struct pl08x_txd *txd = to_pl08x_txd(tx: &vd->tx);
1573	struct pl08x_sg *dsg;
1574
1575	list_for_each_entry(dsg, &txd->dsg_list, node)
1576	bytes += dsg->len;
1577	} else {
1578	bytes = pl08x_getbytes_chan(plchan);
1579	}
1580	}
1581	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
1582
1583	/*
1584	* This cookie not complete yet
1585	* Get number of bytes left in the active transactions and queue
1586	*/
1587	dma_set_residue(state: txstate, residue: bytes);
1588
1589	if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
1590	ret = DMA_PAUSED;
1591
1592	/* Whether waiting or running, we're in progress */
1593	return ret;
1594	}
1595
1596	/* PrimeCell DMA extension */
1597	struct burst_table {
1598	u32 burstwords;
1599	u32 reg;
1600	};
1601
1602	static const struct burst_table burst_sizes[] = {
1603	{
1604	.burstwords = 256,
1605	.reg = PL080_BSIZE_256,
1606	},
1607	{
1608	.burstwords = 128,
1609	.reg = PL080_BSIZE_128,
1610	},
1611	{
1612	.burstwords = 64,
1613	.reg = PL080_BSIZE_64,
1614	},
1615	{
1616	.burstwords = 32,
1617	.reg = PL080_BSIZE_32,
1618	},
1619	{
1620	.burstwords = 16,
1621	.reg = PL080_BSIZE_16,
1622	},
1623	{
1624	.burstwords = 8,
1625	.reg = PL080_BSIZE_8,
1626	},
1627	{
1628	.burstwords = 4,
1629	.reg = PL080_BSIZE_4,
1630	},
1631	{
1632	.burstwords = 0,
1633	.reg = PL080_BSIZE_1,
1634	},
1635	};
1636
1637	/*
1638	* Given the source and destination available bus masks, select which
1639	* will be routed to each port. We try to have source and destination
1640	* on separate ports, but always respect the allowable settings.
1641	*/
1642	static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst)
1643	{
1644	u32 cctl = 0;
1645	u32 dst_ahb2;
1646	u32 src_ahb2;
1647
1648	/* The FTDMAC020 use different bits to indicate src/dst bus */
1649	if (ftdmac020) {
1650	dst_ahb2 = FTDMAC020_LLI_DST_SEL;
1651	src_ahb2 = FTDMAC020_LLI_SRC_SEL;
1652	} else {
1653	dst_ahb2 = PL080_CONTROL_DST_AHB2;
1654	src_ahb2 = PL080_CONTROL_SRC_AHB2;
1655	}
1656
1657	if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1658	cctl |= dst_ahb2;
1659	if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1660	cctl |= src_ahb2;
1661
1662	return cctl;
1663	}
1664
1665	static u32 pl08x_cctl(u32 cctl)
1666	{
1667	cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1668	PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1669	PL080_CONTROL_PROT_MASK);
1670
1671	/* Access the cell in privileged mode, non-bufferable, non-cacheable */
1672	return cctl | PL080_CONTROL_PROT_SYS;
1673	}
1674
1675	static u32 pl08x_width(enum dma_slave_buswidth width)
1676	{
1677	switch (width) {
1678	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1679	return PL080_WIDTH_8BIT;
1680	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1681	return PL080_WIDTH_16BIT;
1682	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1683	return PL080_WIDTH_32BIT;
1684	default:
1685	return ~0;
1686	}
1687	}
1688
1689	static u32 pl08x_burst(u32 maxburst)
1690	{
1691	int i;
1692
1693	for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1694	if (burst_sizes[i].burstwords <= maxburst)
1695	break;
1696
1697	return burst_sizes[i].reg;
1698	}
1699
1700	static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
1701	enum dma_slave_buswidth addr_width, u32 maxburst)
1702	{
1703	u32 width, burst, cctl = 0;
1704
1705	width = pl08x_width(width: addr_width);
1706	if (width == ~0)
1707	return ~0;
1708
1709	cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
1710	cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
1711
1712	/*
1713	* If this channel will only request single transfers, set this
1714	* down to ONE element. Also select one element if no maxburst
1715	* is specified.
1716	*/
1717	if (plchan->cd->single)
1718	maxburst = 1;
1719
1720	burst = pl08x_burst(maxburst);
1721	cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
1722	cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
1723
1724	return pl08x_cctl(cctl);
1725	}
1726
1727	/*
1728	* Slave transactions callback to the slave device to allow
1729	* synchronization of slave DMA signals with the DMAC enable
1730	*/
1731	static void pl08x_issue_pending(struct dma_chan *chan)
1732	{
1733	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1734	unsigned long flags;
1735
1736	spin_lock_irqsave(&plchan->vc.lock, flags);
1737	if (vchan_issue_pending(vc: &plchan->vc)) {
1738	if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
1739	pl08x_phy_alloc_and_start(plchan);
1740	}
1741	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
1742	}
1743
1744	static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
1745	{
1746	struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
1747
1748	if (txd)
1749	INIT_LIST_HEAD(list: &txd->dsg_list);
1750	return txd;
1751	}
1752
1753	static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x)
1754	{
1755	u32 cctl = 0;
1756
1757	/* Conjure cctl */
1758	switch (pl08x->pd->memcpy_burst_size) {
1759	default:
1760	dev_err(&pl08x->adev->dev,
1761	"illegal burst size for memcpy, set to 1\n");
1762	fallthrough;
1763	case PL08X_BURST_SZ_1:
1764	cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |
1765	PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;
1766	break;
1767	case PL08X_BURST_SZ_4:
1768	cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |
1769	PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;
1770	break;
1771	case PL08X_BURST_SZ_8:
1772	cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |
1773	PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;
1774	break;
1775	case PL08X_BURST_SZ_16:
1776	cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |
1777	PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;
1778	break;
1779	case PL08X_BURST_SZ_32:
1780	cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |
1781	PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;
1782	break;
1783	case PL08X_BURST_SZ_64:
1784	cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |
1785	PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;
1786	break;
1787	case PL08X_BURST_SZ_128:
1788	cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |
1789	PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;
1790	break;
1791	case PL08X_BURST_SZ_256:
1792	cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |
1793	PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;
1794	break;
1795	}
1796
1797	switch (pl08x->pd->memcpy_bus_width) {
1798	default:
1799	dev_err(&pl08x->adev->dev,
1800	"illegal bus width for memcpy, set to 8 bits\n");
1801	fallthrough;
1802	case PL08X_BUS_WIDTH_8_BITS:
1803	cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |
1804	PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
1805	break;
1806	case PL08X_BUS_WIDTH_16_BITS:
1807	cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |
1808	PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
1809	break;
1810	case PL08X_BUS_WIDTH_32_BITS:
1811	cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |
1812	PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
1813	break;
1814	}
1815
1816	/* Protection flags */
1817	if (pl08x->pd->memcpy_prot_buff)
1818	cctl |= PL080_CONTROL_PROT_BUFF;
1819	if (pl08x->pd->memcpy_prot_cache)
1820	cctl |= PL080_CONTROL_PROT_CACHE;
1821
1822	/* We are the kernel, so we are in privileged mode */
1823	cctl |= PL080_CONTROL_PROT_SYS;
1824
1825	/* Both to be incremented or the code will break */
1826	cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
1827
1828	if (pl08x->vd->dualmaster)
1829	cctl |= pl08x_select_bus(ftdmac020: false,
1830	src: pl08x->mem_buses,
1831	dst: pl08x->mem_buses);
1832
1833	return cctl;
1834	}
1835
1836	static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x)
1837	{
1838	u32 cctl = 0;
1839
1840	/* Conjure cctl */
1841	switch (pl08x->pd->memcpy_bus_width) {
1842	default:
1843	dev_err(&pl08x->adev->dev,
1844	"illegal bus width for memcpy, set to 8 bits\n");
1845	fallthrough;
1846	case PL08X_BUS_WIDTH_8_BITS:
1847	cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
1848	PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
1849	break;
1850	case PL08X_BUS_WIDTH_16_BITS:
1851	cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
1852	PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
1853	break;
1854	case PL08X_BUS_WIDTH_32_BITS:
1855	cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
1856	PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
1857	break;
1858	}
1859
1860	/*
1861	* By default mask the TC IRQ on all LLIs, it will be unmasked on
1862	* the last LLI item by other code.
1863	*/
1864	cctl |= FTDMAC020_LLI_TC_MSK;
1865
1866	/*
1867	* Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL
1868	* and FTDMAC020_LLI_DSTAD_CTL as zero
1869	*/
1870	if (pl08x->vd->dualmaster)
1871	cctl |= pl08x_select_bus(ftdmac020: true,
1872	src: pl08x->mem_buses,
1873	dst: pl08x->mem_buses);
1874
1875	return cctl;
1876	}
1877
1878	/*
1879	* Initialize a descriptor to be used by memcpy submit
1880	*/
1881	static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1882	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1883	size_t len, unsigned long flags)
1884	{
1885	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1886	struct pl08x_driver_data *pl08x = plchan->host;
1887	struct pl08x_txd *txd;
1888	struct pl08x_sg *dsg;
1889	int ret;
1890
1891	txd = pl08x_get_txd(plchan);
1892	if (!txd) {
1893	dev_err(&pl08x->adev->dev,
1894	"%s no memory for descriptor\n", __func__);
1895	return NULL;
1896	}
1897
1898	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1899	if (!dsg) {
1900	pl08x_free_txd(pl08x, txd);
1901	return NULL;
1902	}
1903	list_add_tail(new: &dsg->node, head: &txd->dsg_list);
1904
1905	dsg->src_addr = src;
1906	dsg->dst_addr = dest;
1907	dsg->len = len;
1908	if (pl08x->vd->ftdmac020) {
1909	/* Writing CCFG zero ENABLES all interrupts */
1910	txd->ccfg = 0;
1911	txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x);
1912	} else {
1913	txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1914	PL080_CONFIG_TC_IRQ_MASK |
1915	PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1916	txd->cctl = pl08x_memcpy_cctl(pl08x);
1917	}
1918
1919	ret = pl08x_fill_llis_for_desc(pl08x: plchan->host, txd);
1920	if (!ret) {
1921	pl08x_free_txd(pl08x, txd);
1922	return NULL;
1923	}
1924
1925	return vchan_tx_prep(vc: &plchan->vc, vd: &txd->vd, tx_flags: flags);
1926	}
1927
1928	static struct pl08x_txd *pl08x_init_txd(
1929	struct dma_chan *chan,
1930	enum dma_transfer_direction direction,
1931	dma_addr_t *slave_addr)
1932	{
1933	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1934	struct pl08x_driver_data *pl08x = plchan->host;
1935	struct pl08x_txd *txd;
1936	enum dma_slave_buswidth addr_width;
1937	int ret, tmp;
1938	u8 src_buses, dst_buses;
1939	u32 maxburst, cctl;
1940
1941	txd = pl08x_get_txd(plchan);
1942	if (!txd) {
1943	dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
1944	return NULL;
1945	}
1946
1947	/*
1948	* Set up addresses, the PrimeCell configured address
1949	* will take precedence since this may configure the
1950	* channel target address dynamically at runtime.
1951	*/
1952	if (direction == DMA_MEM_TO_DEV) {
1953	cctl = PL080_CONTROL_SRC_INCR;
1954	*slave_addr = plchan->cfg.dst_addr;
1955	addr_width = plchan->cfg.dst_addr_width;
1956	maxburst = plchan->cfg.dst_maxburst;
1957	src_buses = pl08x->mem_buses;
1958	dst_buses = plchan->cd->periph_buses;
1959	} else if (direction == DMA_DEV_TO_MEM) {
1960	cctl = PL080_CONTROL_DST_INCR;
1961	*slave_addr = plchan->cfg.src_addr;
1962	addr_width = plchan->cfg.src_addr_width;
1963	maxburst = plchan->cfg.src_maxburst;
1964	src_buses = plchan->cd->periph_buses;
1965	dst_buses = pl08x->mem_buses;
1966	} else {
1967	pl08x_free_txd(pl08x, txd);
1968	dev_err(&pl08x->adev->dev,
1969	"%s direction unsupported\n", __func__);
1970	return NULL;
1971	}
1972
1973	cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
1974	if (cctl == ~0) {
1975	pl08x_free_txd(pl08x, txd);
1976	dev_err(&pl08x->adev->dev,
1977	"DMA slave configuration botched?\n");
1978	return NULL;
1979	}
1980
1981	txd->cctl = cctl | pl08x_select_bus(ftdmac020: false, src: src_buses, dst: dst_buses);
1982
1983	if (plchan->cfg.device_fc)
1984	tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
1985	PL080_FLOW_PER2MEM_PER;
1986	else
1987	tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
1988	PL080_FLOW_PER2MEM;
1989
1990	txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1991	PL080_CONFIG_TC_IRQ_MASK |
1992	tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1993
1994	ret = pl08x_request_mux(plchan);
1995	if (ret < 0) {
1996	pl08x_free_txd(pl08x, txd);
1997	dev_dbg(&pl08x->adev->dev,
1998	"unable to mux for transfer on %s due to platform restrictions\n",
1999	plchan->name);
2000	return NULL;
2001	}
2002
2003	dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
2004	plchan->signal, plchan->name);
2005
2006	/* Assign the flow control signal to this channel */
2007	if (direction == DMA_MEM_TO_DEV)
2008	txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
2009	else
2010	txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
2011
2012	return txd;
2013	}
2014
2015	static int pl08x_tx_add_sg(struct pl08x_txd *txd,
2016	enum dma_transfer_direction direction,
2017	dma_addr_t slave_addr,
2018	dma_addr_t buf_addr,
2019	unsigned int len)
2020	{
2021	struct pl08x_sg *dsg;
2022
2023	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
2024	if (!dsg)
2025	return -ENOMEM;
2026
2027	list_add_tail(new: &dsg->node, head: &txd->dsg_list);
2028
2029	dsg->len = len;
2030	if (direction == DMA_MEM_TO_DEV) {
2031	dsg->src_addr = buf_addr;
2032	dsg->dst_addr = slave_addr;
2033	} else {
2034	dsg->src_addr = slave_addr;
2035	dsg->dst_addr = buf_addr;
2036	}
2037
2038	return 0;
2039	}
2040
2041	static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
2042	struct dma_chan *chan, struct scatterlist *sgl,
2043	unsigned int sg_len, enum dma_transfer_direction direction,
2044	unsigned long flags, void *context)
2045	{
2046	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2047	struct pl08x_driver_data *pl08x = plchan->host;
2048	struct pl08x_txd *txd;
2049	struct scatterlist *sg;
2050	int ret, tmp;
2051	dma_addr_t slave_addr;
2052
2053	dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
2054	__func__, sg_dma_len(sgl), plchan->name);
2055
2056	txd = pl08x_init_txd(chan, direction, slave_addr: &slave_addr);
2057	if (!txd)
2058	return NULL;
2059
2060	for_each_sg(sgl, sg, sg_len, tmp) {
2061	ret = pl08x_tx_add_sg(txd, direction, slave_addr,
2062	sg_dma_address(sg),
2063	sg_dma_len(sg));
2064	if (ret) {
2065	pl08x_release_mux(plchan);
2066	pl08x_free_txd(pl08x, txd);
2067	dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
2068	__func__);
2069	return NULL;
2070	}
2071	}
2072
2073	ret = pl08x_fill_llis_for_desc(pl08x: plchan->host, txd);
2074	if (!ret) {
2075	pl08x_release_mux(plchan);
2076	pl08x_free_txd(pl08x, txd);
2077	return NULL;
2078	}
2079
2080	return vchan_tx_prep(vc: &plchan->vc, vd: &txd->vd, tx_flags: flags);
2081	}
2082
2083	static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
2084	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
2085	size_t period_len, enum dma_transfer_direction direction,
2086	unsigned long flags)
2087	{
2088	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2089	struct pl08x_driver_data *pl08x = plchan->host;
2090	struct pl08x_txd *txd;
2091	int ret, tmp;
2092	dma_addr_t slave_addr;
2093
2094	dev_dbg(&pl08x->adev->dev,
2095	"%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
2096	__func__, period_len, buf_len,
2097	direction == DMA_MEM_TO_DEV ? "to" : "from",
2098	plchan->name);
2099
2100	txd = pl08x_init_txd(chan, direction, slave_addr: &slave_addr);
2101	if (!txd)
2102	return NULL;
2103
2104	txd->cyclic = true;
2105	txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
2106	for (tmp = 0; tmp < buf_len; tmp += period_len) {
2107	ret = pl08x_tx_add_sg(txd, direction, slave_addr,
2108	buf_addr: buf_addr + tmp, len: period_len);
2109	if (ret) {
2110	pl08x_release_mux(plchan);
2111	pl08x_free_txd(pl08x, txd);
2112	return NULL;
2113	}
2114	}
2115
2116	ret = pl08x_fill_llis_for_desc(pl08x: plchan->host, txd);
2117	if (!ret) {
2118	pl08x_release_mux(plchan);
2119	pl08x_free_txd(pl08x, txd);
2120	return NULL;
2121	}
2122
2123	return vchan_tx_prep(vc: &plchan->vc, vd: &txd->vd, tx_flags: flags);
2124	}
2125
2126	static int pl08x_config(struct dma_chan *chan,
2127	struct dma_slave_config *config)
2128	{
2129	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2130	struct pl08x_driver_data *pl08x = plchan->host;
2131
2132	if (!plchan->slave)
2133	return -EINVAL;
2134
2135	/* Reject definitely invalid configurations */
2136	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
2137	config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
2138	return -EINVAL;
2139
2140	if (config->device_fc && pl08x->vd->pl080s) {
2141	dev_err(&pl08x->adev->dev,
2142	"%s: PL080S does not support peripheral flow control\n",
2143	__func__);
2144	return -EINVAL;
2145	}
2146
2147	plchan->cfg = *config;
2148
2149	return 0;
2150	}
2151
2152	static int pl08x_terminate_all(struct dma_chan *chan)
2153	{
2154	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2155	struct pl08x_driver_data *pl08x = plchan->host;
2156	unsigned long flags;
2157
2158	spin_lock_irqsave(&plchan->vc.lock, flags);
2159	if (!plchan->phychan && !plchan->at) {
2160	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
2161	return 0;
2162	}
2163
2164	plchan->state = PL08X_CHAN_IDLE;
2165
2166	if (plchan->phychan) {
2167	/*
2168	* Mark physical channel as free and free any slave
2169	* signal
2170	*/
2171	pl08x_phy_free(plchan);
2172	}
2173	/* Dequeue jobs and free LLIs */
2174	if (plchan->at) {
2175	vchan_terminate_vdesc(vd: &plchan->at->vd);
2176	plchan->at = NULL;
2177	}
2178	/* Dequeue jobs not yet fired as well */
2179	pl08x_free_txd_list(pl08x, plchan);
2180
2181	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
2182
2183	return 0;
2184	}
2185
2186	static void pl08x_synchronize(struct dma_chan *chan)
2187	{
2188	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2189
2190	vchan_synchronize(vc: &plchan->vc);
2191	}
2192
2193	static int pl08x_pause(struct dma_chan *chan)
2194	{
2195	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2196	unsigned long flags;
2197
2198	/*
2199	* Anything succeeds on channels with no physical allocation and
2200	* no queued transfers.
2201	*/
2202	spin_lock_irqsave(&plchan->vc.lock, flags);
2203	if (!plchan->phychan && !plchan->at) {
2204	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
2205	return 0;
2206	}
2207
2208	pl08x_pause_phy_chan(ch: plchan->phychan);
2209	plchan->state = PL08X_CHAN_PAUSED;
2210
2211	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
2212
2213	return 0;
2214	}
2215
2216	static int pl08x_resume(struct dma_chan *chan)
2217	{
2218	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2219	unsigned long flags;
2220
2221	/*
2222	* Anything succeeds on channels with no physical allocation and
2223	* no queued transfers.
2224	*/
2225	spin_lock_irqsave(&plchan->vc.lock, flags);
2226	if (!plchan->phychan && !plchan->at) {
2227	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
2228	return 0;
2229	}
2230
2231	pl08x_resume_phy_chan(ch: plchan->phychan);
2232	plchan->state = PL08X_CHAN_RUNNING;
2233
2234	spin_unlock_irqrestore(lock: &plchan->vc.lock, flags);
2235
2236	return 0;
2237	}
2238
2239	bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
2240	{
2241	struct pl08x_dma_chan *plchan;
2242	const char *name = chan_id;
2243
2244	/* Reject channels for devices not bound to this driver */
2245	if (chan->device->dev->driver != &pl08x_amba_driver.drv)
2246	return false;
2247
2248	plchan = to_pl08x_chan(chan);
2249
2250	/* Check that the channel is not taken! */
2251	if (!strcmp(plchan->name, name))
2252	return true;
2253
2254	return false;
2255	}
2256	EXPORT_SYMBOL_GPL(pl08x_filter_id);
2257
2258	static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)
2259	{
2260	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
2261
2262	return plchan->cd == chan_id;
2263	}
2264
2265	/*
2266	* Just check that the device is there and active
2267	* TODO: turn this bit on/off depending on the number of physical channels
2268	* actually used, if it is zero... well shut it off. That will save some
2269	* power. Cut the clock at the same time.
2270	*/
2271	static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
2272	{
2273	/* The Nomadik variant does not have the config register */
2274	if (pl08x->vd->nomadik)
2275	return;
2276	/* The FTDMAC020 variant does this in another register */
2277	if (pl08x->vd->ftdmac020) {
2278	writel(PL080_CONFIG_ENABLE, addr: pl08x->base + FTDMAC020_CSR);
2279	return;
2280	}
2281	writel(PL080_CONFIG_ENABLE, addr: pl08x->base + PL080_CONFIG);
2282	}
2283
2284	static irqreturn_t pl08x_irq(int irq, void *dev)
2285	{
2286	struct pl08x_driver_data *pl08x = dev;
2287	u32 mask = 0, err, tc, i;
2288
2289	/* check & clear - ERR & TC interrupts */
2290	err = readl(addr: pl08x->base + PL080_ERR_STATUS);
2291	if (err) {
2292	dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
2293	__func__, err);
2294	writel(val: err, addr: pl08x->base + PL080_ERR_CLEAR);
2295	}
2296	tc = readl(addr: pl08x->base + PL080_TC_STATUS);
2297	if (tc)
2298	writel(val: tc, addr: pl08x->base + PL080_TC_CLEAR);
2299
2300	if (!err && !tc)
2301	return IRQ_NONE;
2302
2303	for (i = 0; i < pl08x->vd->channels; i++) {
2304	if ((BIT(i) & err) || (BIT(i) & tc)) {
2305	/* Locate physical channel */
2306	struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
2307	struct pl08x_dma_chan *plchan = phychan->serving;
2308	struct pl08x_txd *tx;
2309
2310	if (!plchan) {
2311	dev_err(&pl08x->adev->dev,
2312	"%s Error TC interrupt on unused channel: 0x%08x\n",
2313	__func__, i);
2314	continue;
2315	}
2316
2317	spin_lock(lock: &plchan->vc.lock);
2318	tx = plchan->at;
2319	if (tx && tx->cyclic) {
2320	vchan_cyclic_callback(vd: &tx->vd);
2321	} else if (tx) {
2322	plchan->at = NULL;
2323	/*
2324	* This descriptor is done, release its mux
2325	* reservation.
2326	*/
2327	pl08x_release_mux(plchan);
2328	tx->done = true;
2329	vchan_cookie_complete(vd: &tx->vd);
2330
2331	/*
2332	* And start the next descriptor (if any),
2333	* otherwise free this channel.
2334	*/
2335	if (vchan_next_desc(vc: &plchan->vc))
2336	pl08x_start_next_txd(plchan);
2337	else
2338	pl08x_phy_free(plchan);
2339	}
2340	spin_unlock(lock: &plchan->vc.lock);
2341
2342	mask |= BIT(i);
2343	}
2344	}
2345
2346	return mask ? IRQ_HANDLED : IRQ_NONE;
2347	}
2348
2349	static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
2350	{
2351	chan->slave = true;
2352	chan->name = chan->cd->bus_id;
2353	chan->cfg.src_addr = chan->cd->addr;
2354	chan->cfg.dst_addr = chan->cd->addr;
2355	}
2356
2357	/*
2358	* Initialise the DMAC memcpy/slave channels.
2359	* Make a local wrapper to hold required data
2360	*/
2361	static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
2362	struct dma_device *dmadev, unsigned int channels, bool slave)
2363	{
2364	struct pl08x_dma_chan *chan;
2365	int i;
2366
2367	INIT_LIST_HEAD(list: &dmadev->channels);
2368
2369	/*
2370	* Register as many memcpy as we have physical channels,
2371	* we won't always be able to use all but the code will have
2372	* to cope with that situation.
2373	*/
2374	for (i = 0; i < channels; i++) {
2375	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
2376	if (!chan)
2377	return -ENOMEM;
2378
2379	chan->host = pl08x;
2380	chan->state = PL08X_CHAN_IDLE;
2381	chan->signal = -1;
2382
2383	if (slave) {
2384	chan->cd = &pl08x->pd->slave_channels[i];
2385	/*
2386	* Some implementations have muxed signals, whereas some
2387	* use a mux in front of the signals and need dynamic
2388	* assignment of signals.
2389	*/
2390	chan->signal = i;
2391	pl08x_dma_slave_init(chan);
2392	} else {
2393	chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL);
2394	if (!chan->cd) {
2395	kfree(objp: chan);
2396	return -ENOMEM;
2397	}
2398	chan->cd->bus_id = "memcpy";
2399	chan->cd->periph_buses = pl08x->pd->mem_buses;
2400	chan->name = kasprintf(GFP_KERNEL, fmt: "memcpy%d", i);
2401	if (!chan->name) {
2402	kfree(objp: chan->cd);
2403	kfree(objp: chan);
2404	return -ENOMEM;
2405	}
2406	}
2407	dev_dbg(&pl08x->adev->dev,
2408	"initialize virtual channel \"%s\"\n",
2409	chan->name);
2410
2411	chan->vc.desc_free = pl08x_desc_free;
2412	vchan_init(vc: &chan->vc, dmadev);
2413	}
2414	dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
2415	i, slave ? "slave" : "memcpy");
2416	return i;
2417	}
2418
2419	static void pl08x_free_virtual_channels(struct dma_device *dmadev)
2420	{
2421	struct pl08x_dma_chan *chan = NULL;
2422	struct pl08x_dma_chan *next;
2423
2424	list_for_each_entry_safe(chan,
2425	next, &dmadev->channels, vc.chan.device_node) {
2426	list_del(entry: &chan->vc.chan.device_node);
2427	kfree(objp: chan);
2428	}
2429	}
2430
2431	#ifdef CONFIG_DEBUG_FS
2432	static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
2433	{
2434	switch (state) {
2435	case PL08X_CHAN_IDLE:
2436	return "idle";
2437	case PL08X_CHAN_RUNNING:
2438	return "running";
2439	case PL08X_CHAN_PAUSED:
2440	return "paused";
2441	case PL08X_CHAN_WAITING:
2442	return "waiting";
2443	default:
2444	break;
2445	}
2446	return "UNKNOWN STATE";
2447	}
2448
2449	static int pl08x_debugfs_show(struct seq_file *s, void *data)
2450	{
2451	struct pl08x_driver_data *pl08x = s->private;
2452	struct pl08x_dma_chan *chan;
2453	struct pl08x_phy_chan *ch;
2454	unsigned long flags;
2455	int i;
2456
2457	seq_printf(m: s, fmt: "PL08x physical channels:\n");
2458	seq_printf(m: s, fmt: "CHANNEL:\tUSER:\n");
2459	seq_printf(m: s, fmt: "--------\t-----\n");
2460	for (i = 0; i < pl08x->vd->channels; i++) {
2461	struct pl08x_dma_chan *virt_chan;
2462
2463	ch = &pl08x->phy_chans[i];
2464
2465	spin_lock_irqsave(&ch->lock, flags);
2466	virt_chan = ch->serving;
2467
2468	seq_printf(m: s, fmt: "%d\t\t%s%s\n",
2469	ch->id,
2470	virt_chan ? virt_chan->name : "(none)",
2471	ch->locked ? " LOCKED" : "");
2472
2473	spin_unlock_irqrestore(lock: &ch->lock, flags);
2474	}
2475
2476	seq_printf(m: s, fmt: "\nPL08x virtual memcpy channels:\n");
2477	seq_printf(m: s, fmt: "CHANNEL:\tSTATE:\n");
2478	seq_printf(m: s, fmt: "--------\t------\n");
2479	list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
2480	seq_printf(m: s, fmt: "%s\t\t%s\n", chan->name,
2481	pl08x_state_str(state: chan->state));
2482	}
2483
2484	if (pl08x->has_slave) {
2485	seq_printf(m: s, fmt: "\nPL08x virtual slave channels:\n");
2486	seq_printf(m: s, fmt: "CHANNEL:\tSTATE:\n");
2487	seq_printf(m: s, fmt: "--------\t------\n");
2488	list_for_each_entry(chan, &pl08x->slave.channels,
2489	vc.chan.device_node) {
2490	seq_printf(m: s, fmt: "%s\t\t%s\n", chan->name,
2491	pl08x_state_str(state: chan->state));
2492	}
2493	}
2494
2495	return 0;
2496	}
2497
2498	DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
2499
2500	static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2501	{
2502	/* Expose a simple debugfs interface to view all clocks */
2503	debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
2504	NULL, pl08x, &pl08x_debugfs_fops);
2505	}
2506
2507	#else
2508	static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2509	{
2510	}
2511	#endif
2512
2513	#ifdef CONFIG_OF
2514	static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,
2515	u32 id)
2516	{
2517	struct pl08x_dma_chan *chan;
2518
2519	/* Trying to get a slave channel from something with no slave support */
2520	if (!pl08x->has_slave)
2521	return NULL;
2522
2523	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
2524	if (chan->signal == id)
2525	return &chan->vc.chan;
2526	}
2527
2528	return NULL;
2529	}
2530
2531	static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
2532	struct of_dma *ofdma)
2533	{
2534	struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
2535	struct dma_chan *dma_chan;
2536	struct pl08x_dma_chan *plchan;
2537
2538	if (!pl08x)
2539	return NULL;
2540
2541	if (dma_spec->args_count != 2) {
2542	dev_err(&pl08x->adev->dev,
2543	"DMA channel translation requires two cells\n");
2544	return NULL;
2545	}
2546
2547	dma_chan = pl08x_find_chan_id(pl08x, id: dma_spec->args[0]);
2548	if (!dma_chan) {
2549	dev_err(&pl08x->adev->dev,
2550	"DMA slave channel not found\n");
2551	return NULL;
2552	}
2553
2554	plchan = to_pl08x_chan(chan: dma_chan);
2555	dev_dbg(&pl08x->adev->dev,
2556	"translated channel for signal %d\n",
2557	dma_spec->args[0]);
2558
2559	/* Augment channel data for applicable AHB buses */
2560	plchan->cd->periph_buses = dma_spec->args[1];
2561	return dma_get_slave_channel(chan: dma_chan);
2562	}
2563
2564	static int pl08x_of_probe(struct amba_device *adev,
2565	struct pl08x_driver_data *pl08x,
2566	struct device_node *np)
2567	{
2568	struct pl08x_platform_data *pd;
2569	struct pl08x_channel_data *chanp = NULL;
2570	u32 val;
2571	int ret;
2572	int i;
2573
2574	pd = devm_kzalloc(dev: &adev->dev, size: sizeof(*pd), GFP_KERNEL);
2575	if (!pd)
2576	return -ENOMEM;
2577
2578	/* Eligible bus masters for fetching LLIs */
2579	if (of_property_read_bool(np, propname: "lli-bus-interface-ahb1"))
2580	pd->lli_buses |= PL08X_AHB1;
2581	if (of_property_read_bool(np, propname: "lli-bus-interface-ahb2"))
2582	pd->lli_buses |= PL08X_AHB2;
2583	if (!pd->lli_buses) {
2584	dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");
2585	pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;
2586	}
2587
2588	/* Eligible bus masters for memory access */
2589	if (of_property_read_bool(np, propname: "mem-bus-interface-ahb1"))
2590	pd->mem_buses |= PL08X_AHB1;
2591	if (of_property_read_bool(np, propname: "mem-bus-interface-ahb2"))
2592	pd->mem_buses |= PL08X_AHB2;
2593	if (!pd->mem_buses) {
2594	dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");
2595	pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;
2596	}
2597
2598	/* Parse the memcpy channel properties */
2599	ret = of_property_read_u32(np, propname: "memcpy-burst-size", out_value: &val);
2600	if (ret) {
2601	dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");
2602	val = 1;
2603	}
2604	switch (val) {
2605	default:
2606	dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");
2607	fallthrough;
2608	case 1:
2609	pd->memcpy_burst_size = PL08X_BURST_SZ_1;
2610	break;
2611	case 4:
2612	pd->memcpy_burst_size = PL08X_BURST_SZ_4;
2613	break;
2614	case 8:
2615	pd->memcpy_burst_size = PL08X_BURST_SZ_8;
2616	break;
2617	case 16:
2618	pd->memcpy_burst_size = PL08X_BURST_SZ_16;
2619	break;
2620	case 32:
2621	pd->memcpy_burst_size = PL08X_BURST_SZ_32;
2622	break;
2623	case 64:
2624	pd->memcpy_burst_size = PL08X_BURST_SZ_64;
2625	break;
2626	case 128:
2627	pd->memcpy_burst_size = PL08X_BURST_SZ_128;
2628	break;
2629	case 256:
2630	pd->memcpy_burst_size = PL08X_BURST_SZ_256;
2631	break;
2632	}
2633
2634	ret = of_property_read_u32(np, propname: "memcpy-bus-width", out_value: &val);
2635	if (ret) {
2636	dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");
2637	val = 8;
2638	}
2639	switch (val) {
2640	default:
2641	dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");
2642	fallthrough;
2643	case 8:
2644	pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS;
2645	break;
2646	case 16:
2647	pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS;
2648	break;
2649	case 32:
2650	pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS;
2651	break;
2652	}
2653
2654	/*
2655	* Allocate channel data for all possible slave channels (one
2656	* for each possible signal), channels will then be allocated
2657	* for a device and have it's AHB interfaces set up at
2658	* translation time.
2659	*/
2660	if (pl08x->vd->signals) {
2661	chanp = devm_kcalloc(dev: &adev->dev,
2662	n: pl08x->vd->signals,
2663	size: sizeof(struct pl08x_channel_data),
2664	GFP_KERNEL);
2665	if (!chanp)
2666	return -ENOMEM;
2667
2668	pd->slave_channels = chanp;
2669	for (i = 0; i < pl08x->vd->signals; i++) {
2670	/*
2671	* chanp->periph_buses will be assigned at translation
2672	*/
2673	chanp->bus_id = kasprintf(GFP_KERNEL, fmt: "slave%d", i);
2674	chanp++;
2675	}
2676	pd->num_slave_channels = pl08x->vd->signals;
2677	}
2678
2679	pl08x->pd = pd;
2680
2681	return of_dma_controller_register(np: adev->dev.of_node, of_dma_xlate: pl08x_of_xlate,
2682	data: pl08x);
2683	}
2684	#else
2685	static inline int pl08x_of_probe(struct amba_device *adev,
2686	struct pl08x_driver_data *pl08x,
2687	struct device_node *np)
2688	{
2689	return -EINVAL;
2690	}
2691	#endif
2692
2693	static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
2694	{
2695	struct pl08x_driver_data *pl08x;
2696	struct vendor_data *vd = id->data;
2697	struct device_node *np = adev->dev.of_node;
2698	u32 tsfr_size;
2699	int ret = 0;
2700	int i;
2701
2702	ret = amba_request_regions(adev, NULL);
2703	if (ret)
2704	return ret;
2705
2706	/* Ensure that we can do DMA */
2707	ret = dma_set_mask_and_coherent(dev: &adev->dev, DMA_BIT_MASK(32));
2708	if (ret)
2709	goto out_no_pl08x;
2710
2711	/* Create the driver state holder */
2712	pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
2713	if (!pl08x) {
2714	ret = -ENOMEM;
2715	goto out_no_pl08x;
2716	}
2717
2718	/* Assign useful pointers to the driver state */
2719	pl08x->adev = adev;
2720	pl08x->vd = vd;
2721
2722	pl08x->base = ioremap(offset: adev->res.start, size: resource_size(res: &adev->res));
2723	if (!pl08x->base) {
2724	ret = -ENOMEM;
2725	goto out_no_ioremap;
2726	}
2727
2728	if (vd->ftdmac020) {
2729	u32 val;
2730
2731	val = readl(addr: pl08x->base + FTDMAC020_REVISION);
2732	dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n",
2733	(val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);
2734	val = readl(addr: pl08x->base + FTDMAC020_FEATURE);
2735	dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, "
2736	"%s built-in bridge, %s, %s linked lists\n",
2737	(val >> 12) & 0x0f,
2738	(val & BIT(10)) ? "no" : "has",
2739	(val & BIT(9)) ? "AHB0 and AHB1" : "AHB0",
2740	(val & BIT(8)) ? "supports" : "does not support");
2741
2742	/* Vendor data from feature register */
2743	if (!(val & BIT(8)))
2744	dev_warn(&pl08x->adev->dev,
2745	"linked lists not supported, required\n");
2746	vd->channels = (val >> 12) & 0x0f;
2747	vd->dualmaster = !!(val & BIT(9));
2748	}
2749
2750	/* Initialize memcpy engine */
2751	dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
2752	pl08x->memcpy.dev = &adev->dev;
2753	pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
2754	pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
2755	pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
2756	pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
2757	pl08x->memcpy.device_config = pl08x_config;
2758	pl08x->memcpy.device_pause = pl08x_pause;
2759	pl08x->memcpy.device_resume = pl08x_resume;
2760	pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
2761	pl08x->memcpy.device_synchronize = pl08x_synchronize;
2762	pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;
2763	pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
2764	pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);
2765	pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
2766	if (vd->ftdmac020)
2767	pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;
2768
2769
2770	/*
2771	* Initialize slave engine, if the block has no signals, that means
2772	* we have no slave support.
2773	*/
2774	if (vd->signals) {
2775	pl08x->has_slave = true;
2776	dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
2777	dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
2778	pl08x->slave.dev = &adev->dev;
2779	pl08x->slave.device_free_chan_resources =
2780	pl08x_free_chan_resources;
2781	pl08x->slave.device_tx_status = pl08x_dma_tx_status;
2782	pl08x->slave.device_issue_pending = pl08x_issue_pending;
2783	pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
2784	pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
2785	pl08x->slave.device_config = pl08x_config;
2786	pl08x->slave.device_pause = pl08x_pause;
2787	pl08x->slave.device_resume = pl08x_resume;
2788	pl08x->slave.device_terminate_all = pl08x_terminate_all;
2789	pl08x->slave.device_synchronize = pl08x_synchronize;
2790	pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;
2791	pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
2792	pl08x->slave.directions =
2793	BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2794	pl08x->slave.residue_granularity =
2795	DMA_RESIDUE_GRANULARITY_SEGMENT;
2796	}
2797
2798	/* Get the platform data */
2799	pl08x->pd = dev_get_platdata(dev: &adev->dev);
2800	if (!pl08x->pd) {
2801	if (np) {
2802	ret = pl08x_of_probe(adev, pl08x, np);
2803	if (ret)
2804	goto out_no_platdata;
2805	} else {
2806	dev_err(&adev->dev, "no platform data supplied\n");
2807	ret = -EINVAL;
2808	goto out_no_platdata;
2809	}
2810	} else {
2811	pl08x->slave.filter.map = pl08x->pd->slave_map;
2812	pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;
2813	pl08x->slave.filter.fn = pl08x_filter_fn;
2814	}
2815
2816	/* By default, AHB1 only. If dualmaster, from platform */
2817	pl08x->lli_buses = PL08X_AHB1;
2818	pl08x->mem_buses = PL08X_AHB1;
2819	if (pl08x->vd->dualmaster) {
2820	pl08x->lli_buses = pl08x->pd->lli_buses;
2821	pl08x->mem_buses = pl08x->pd->mem_buses;
2822	}
2823
2824	if (vd->pl080s)
2825	pl08x->lli_words = PL080S_LLI_WORDS;
2826	else
2827	pl08x->lli_words = PL080_LLI_WORDS;
2828	tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
2829
2830	/* A DMA memory pool for LLIs, align on 1-byte boundary */
2831	pl08x->pool = dma_pool_create(DRIVER_NAME, dev: &pl08x->adev->dev,
2832	size: tsfr_size, PL08X_ALIGN, boundary: 0);
2833	if (!pl08x->pool) {
2834	ret = -ENOMEM;
2835	goto out_no_lli_pool;
2836	}
2837
2838	/* Turn on the PL08x */
2839	pl08x_ensure_on(pl08x);
2840
2841	/* Clear any pending interrupts */
2842	if (vd->ftdmac020)
2843	/* This variant has error IRQs in bits 16-19 */
2844	writel(val: 0x0000FFFF, addr: pl08x->base + PL080_ERR_CLEAR);
2845	else
2846	writel(val: 0x000000FF, addr: pl08x->base + PL080_ERR_CLEAR);
2847	writel(val: 0x000000FF, addr: pl08x->base + PL080_TC_CLEAR);
2848
2849	/* Attach the interrupt handler */
2850	ret = request_irq(irq: adev->irq[0], handler: pl08x_irq, flags: 0, DRIVER_NAME, dev: pl08x);
2851	if (ret) {
2852	dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2853	__func__, adev->irq[0]);
2854	goto out_no_irq;
2855	}
2856
2857	/* Initialize physical channels */
2858	pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans),
2859	GFP_KERNEL);
2860	if (!pl08x->phy_chans) {
2861	ret = -ENOMEM;
2862	goto out_no_phychans;
2863	}
2864
2865	for (i = 0; i < vd->channels; i++) {
2866	struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
2867
2868	ch->id = i;
2869	ch->base = pl08x->base + PL080_Cx_BASE(i);
2870	if (vd->ftdmac020) {
2871	/* FTDMA020 has a special channel busy register */
2872	ch->reg_busy = ch->base + FTDMAC020_CH_BUSY;
2873	ch->reg_config = ch->base + FTDMAC020_CH_CFG;
2874	ch->reg_control = ch->base + FTDMAC020_CH_CSR;
2875	ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR;
2876	ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR;
2877	ch->reg_lli = ch->base + FTDMAC020_CH_LLP;
2878	ch->ftdmac020 = true;
2879	} else {
2880	ch->reg_config = ch->base + vd->config_offset;
2881	ch->reg_control = ch->base + PL080_CH_CONTROL;
2882	ch->reg_src = ch->base + PL080_CH_SRC_ADDR;
2883	ch->reg_dst = ch->base + PL080_CH_DST_ADDR;
2884	ch->reg_lli = ch->base + PL080_CH_LLI;
2885	}
2886	if (vd->pl080s)
2887	ch->pl080s = true;
2888
2889	spin_lock_init(&ch->lock);
2890
2891	/*
2892	* Nomadik variants can have channels that are locked
2893	* down for the secure world only. Lock up these channels
2894	* by perpetually serving a dummy virtual channel.
2895	*/
2896	if (vd->nomadik) {
2897	u32 val;
2898
2899	val = readl(addr: ch->reg_config);
2900	if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
2901	dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
2902	ch->locked = true;
2903	}
2904	}
2905
2906	dev_dbg(&adev->dev, "physical channel %d is %s\n",
2907	i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
2908	}
2909
2910	/* Register as many memcpy channels as there are physical channels */
2911	ret = pl08x_dma_init_virtual_channels(pl08x, dmadev: &pl08x->memcpy,
2912	channels: pl08x->vd->channels, slave: false);
2913	if (ret <= 0) {
2914	dev_warn(&pl08x->adev->dev,
2915	"%s failed to enumerate memcpy channels - %d\n",
2916	__func__, ret);
2917	goto out_no_memcpy;
2918	}
2919
2920	/* Register slave channels */
2921	if (pl08x->has_slave) {
2922	ret = pl08x_dma_init_virtual_channels(pl08x, dmadev: &pl08x->slave,
2923	channels: pl08x->pd->num_slave_channels, slave: true);
2924	if (ret < 0) {
2925	dev_warn(&pl08x->adev->dev,
2926	"%s failed to enumerate slave channels - %d\n",
2927	__func__, ret);
2928	goto out_no_slave;
2929	}
2930	}
2931
2932	ret = dma_async_device_register(device: &pl08x->memcpy);
2933	if (ret) {
2934	dev_warn(&pl08x->adev->dev,
2935	"%s failed to register memcpy as an async device - %d\n",
2936	__func__, ret);
2937	goto out_no_memcpy_reg;
2938	}
2939
2940	if (pl08x->has_slave) {
2941	ret = dma_async_device_register(device: &pl08x->slave);
2942	if (ret) {
2943	dev_warn(&pl08x->adev->dev,
2944	"%s failed to register slave as an async device - %d\n",
2945	__func__, ret);
2946	goto out_no_slave_reg;
2947	}
2948	}
2949
2950	amba_set_drvdata(adev, pl08x);
2951	init_pl08x_debugfs(pl08x);
2952	dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",
2953	amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),
2954	(unsigned long long)adev->res.start, adev->irq[0]);
2955
2956	return 0;
2957
2958	out_no_slave_reg:
2959	dma_async_device_unregister(device: &pl08x->memcpy);
2960	out_no_memcpy_reg:
2961	if (pl08x->has_slave)
2962	pl08x_free_virtual_channels(dmadev: &pl08x->slave);
2963	out_no_slave:
2964	pl08x_free_virtual_channels(dmadev: &pl08x->memcpy);
2965	out_no_memcpy:
2966	kfree(objp: pl08x->phy_chans);
2967	out_no_phychans:
2968	free_irq(adev->irq[0], pl08x);
2969	out_no_irq:
2970	dma_pool_destroy(pool: pl08x->pool);
2971	out_no_lli_pool:
2972	out_no_platdata:
2973	iounmap(addr: pl08x->base);
2974	out_no_ioremap:
2975	kfree(objp: pl08x);
2976	out_no_pl08x:
2977	amba_release_regions(adev);
2978	return ret;
2979	}
2980
2981	/* PL080 has 8 channels and the PL080 have just 2 */
2982	static struct vendor_data vendor_pl080 = {
2983	.config_offset = PL080_CH_CONFIG,
2984	.channels = 8,
2985	.signals = 16,
2986	.dualmaster = true,
2987	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
2988	};
2989
2990	static struct vendor_data vendor_nomadik = {
2991	.config_offset = PL080_CH_CONFIG,
2992	.channels = 8,
2993	.signals = 32,
2994	.dualmaster = true,
2995	.nomadik = true,
2996	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
2997	};
2998
2999	static struct vendor_data vendor_pl080s = {
3000	.config_offset = PL080S_CH_CONFIG,
3001	.channels = 8,
3002	.signals = 32,
3003	.pl080s = true,
3004	.max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
3005	};
3006
3007	static struct vendor_data vendor_pl081 = {
3008	.config_offset = PL080_CH_CONFIG,
3009	.channels = 2,
3010	.signals = 16,
3011	.dualmaster = false,
3012	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
3013	};
3014
3015	static struct vendor_data vendor_ftdmac020 = {
3016	.config_offset = PL080_CH_CONFIG,
3017	.ftdmac020 = true,
3018	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
3019	};
3020
3021	static const struct amba_id pl08x_ids[] = {
3022	/* Samsung PL080S variant */
3023	{
3024	.id = 0x0a141080,
3025	.mask = 0xffffffff,
3026	.data = &vendor_pl080s,
3027	},
3028	/* PL080 */
3029	{
3030	.id = 0x00041080,
3031	.mask = 0x000fffff,
3032	.data = &vendor_pl080,
3033	},
3034	/* PL081 */
3035	{
3036	.id = 0x00041081,
3037	.mask = 0x000fffff,
3038	.data = &vendor_pl081,
3039	},
3040	/* Nomadik 8815 PL080 variant */
3041	{
3042	.id = 0x00280080,
3043	.mask = 0x00ffffff,
3044	.data = &vendor_nomadik,
3045	},
3046	/* Faraday Technology FTDMAC020 */
3047	{
3048	.id = 0x0003b080,
3049	.mask = 0x000fffff,
3050	.data = &vendor_ftdmac020,
3051	},
3052	{ 0, 0 },
3053	};
3054
3055	MODULE_DEVICE_TABLE(amba, pl08x_ids);
3056
3057	static struct amba_driver pl08x_amba_driver = {
3058	.drv.name = DRIVER_NAME,
3059	.id_table = pl08x_ids,
3060	.probe = pl08x_probe,
3061	};
3062
3063	static int __init pl08x_init(void)
3064	{
3065	int retval;
3066	retval = amba_driver_register(&pl08x_amba_driver);
3067	if (retval)
3068	printk(KERN_WARNING DRIVER_NAME
3069	"failed to register as an AMBA device (%d)\n",
3070	retval);
3071	return retval;
3072	}
3073	subsys_initcall(pl08x_init);
3074