1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Driver for OHCI 1394 controllers
4	*
5	* Copyright (C) 2003-2006 Kristian Hoegsberg <krh@bitplanet.net>
6	*/
7
8	#include <linux/bitops.h>
9	#include <linux/bug.h>
10	#include <linux/compiler.h>
11	#include <linux/delay.h>
12	#include <linux/device.h>
13	#include <linux/dma-mapping.h>
14	#include <linux/firewire.h>
15	#include <linux/firewire-constants.h>
16	#include <linux/init.h>
17	#include <linux/interrupt.h>
18	#include <linux/io.h>
19	#include <linux/kernel.h>
20	#include <linux/list.h>
21	#include <linux/mm.h>
22	#include <linux/module.h>
23	#include <linux/moduleparam.h>
24	#include <linux/mutex.h>
25	#include <linux/pci.h>
26	#include <linux/pci_ids.h>
27	#include <linux/slab.h>
28	#include <linux/spinlock.h>
29	#include <linux/string.h>
30	#include <linux/time.h>
31	#include <linux/vmalloc.h>
32	#include <linux/workqueue.h>
33
34	#include <asm/byteorder.h>
35	#include <asm/page.h>
36
37	#ifdef CONFIG_PPC_PMAC
38	#include <asm/pmac_feature.h>
39	#endif
40
41	#include "core.h"
42	#include "ohci.h"
43	#include "packet-header-definitions.h"
44	#include "phy-packet-definitions.h"
45
46	#include <trace/events/firewire.h>
47
48	static u32 cond_le32_to_cpu(__le32 value, bool has_be_header_quirk);
49
50	#define CREATE_TRACE_POINTS
51	#include <trace/events/firewire_ohci.h>
52
53	#define ohci_notice(ohci, f, args...) dev_notice(ohci->card.device, f, ##args)
54	#define ohci_err(ohci, f, args...) dev_err(ohci->card.device, f, ##args)
55
56	#define DESCRIPTOR_OUTPUT_MORE 0
57	#define DESCRIPTOR_OUTPUT_LAST (1 << 12)
58	#define DESCRIPTOR_INPUT_MORE (2 << 12)
59	#define DESCRIPTOR_INPUT_LAST (3 << 12)
60	#define DESCRIPTOR_STATUS (1 << 11)
61	#define DESCRIPTOR_KEY_IMMEDIATE (2 << 8)
62	#define DESCRIPTOR_PING (1 << 7)
63	#define DESCRIPTOR_YY (1 << 6)
64	#define DESCRIPTOR_NO_IRQ (0 << 4)
65	#define DESCRIPTOR_IRQ_ERROR (1 << 4)
66	#define DESCRIPTOR_IRQ_ALWAYS (3 << 4)
67	#define DESCRIPTOR_BRANCH_ALWAYS (3 << 2)
68	#define DESCRIPTOR_WAIT (3 << 0)
69
70	#define DESCRIPTOR_CMD (0xf << 12)
71
72	struct descriptor {
73	__le16 req_count;
74	__le16 control;
75	__le32 data_address;
76	__le32 branch_address;
77	__le16 res_count;
78	__le16 transfer_status;
79	} __aligned(16);
80
81	#define CONTROL_SET(regs) (regs)
82	#define CONTROL_CLEAR(regs) ((regs) + 4)
83	#define COMMAND_PTR(regs) ((regs) + 12)
84	#define CONTEXT_MATCH(regs) ((regs) + 16)
85
86	#define AR_BUFFER_SIZE (32*1024)
87	#define AR_BUFFERS_MIN DIV_ROUND_UP(AR_BUFFER_SIZE, PAGE_SIZE)
88	/* we need at least two pages for proper list management */
89	#define AR_BUFFERS (AR_BUFFERS_MIN >= 2 ? AR_BUFFERS_MIN : 2)
90
91	#define MAX_ASYNC_PAYLOAD 4096
92	#define MAX_AR_PACKET_SIZE (16 + MAX_ASYNC_PAYLOAD + 4)
93	#define AR_WRAPAROUND_PAGES DIV_ROUND_UP(MAX_AR_PACKET_SIZE, PAGE_SIZE)
94
95	struct ar_context {
96	struct fw_ohci *ohci;
97	struct page *pages[AR_BUFFERS];
98	void *buffer;
99	struct descriptor *descriptors;
100	dma_addr_t descriptors_bus;
101	void *pointer;
102	unsigned int last_buffer_index;
103	u32 regs;
104	struct work_struct work;
105	};
106
107	struct context;
108
109	typedef int (*descriptor_callback_t)(struct context *ctx,
110	struct descriptor *d,
111	struct descriptor *last);
112
113	/*
114	* A buffer that contains a block of DMA-able coherent memory used for
115	* storing a portion of a DMA descriptor program.
116	*/
117	struct descriptor_buffer {
118	struct list_head list;
119	dma_addr_t buffer_bus;
120	size_t buffer_size;
121	size_t used;
122	struct descriptor buffer[];
123	};
124
125	struct context {
126	struct fw_ohci *ohci;
127	u32 regs;
128	int total_allocation;
129	u32 current_bus;
130	bool running;
131
132	/*
133	* List of page-sized buffers for storing DMA descriptors.
134	* Head of list contains buffers in use and tail of list contains
135	* free buffers.
136	*/
137	struct list_head buffer_list;
138
139	/*
140	* Pointer to a buffer inside buffer_list that contains the tail
141	* end of the current DMA program.
142	*/
143	struct descriptor_buffer *buffer_tail;
144
145	/*
146	* The descriptor containing the branch address of the first
147	* descriptor that has not yet been filled by the device.
148	*/
149	struct descriptor *last;
150
151	/*
152	* The last descriptor block in the DMA program. It contains the branch
153	* address that must be updated upon appending a new descriptor.
154	*/
155	struct descriptor *prev;
156	int prev_z;
157
158	descriptor_callback_t callback;
159	};
160
161	struct at_context {
162	struct context context;
163	struct work_struct work;
164	bool flushing;
165	};
166
167	struct iso_context {
168	struct fw_iso_context base;
169	struct context context;
170	void *header;
171	size_t header_length;
172	unsigned long flushing_completions;
173	u32 mc_buffer_bus;
174	u16 mc_completed;
175	u16 last_timestamp;
176	u8 sync;
177	u8 tags;
178	};
179
180	#define CONFIG_ROM_SIZE (CSR_CONFIG_ROM_END - CSR_CONFIG_ROM)
181
182	struct fw_ohci {
183	struct fw_card card;
184
185	__iomem char *registers;
186	int node_id;
187	int generation;
188	int request_generation; /* for timestamping incoming requests */
189	unsigned quirks;
190	unsigned int pri_req_max;
191	u32 bus_time;
192	bool bus_time_running;
193	bool is_root;
194	bool csr_state_setclear_abdicate;
195	int n_ir;
196	int n_it;
197	/*
198	* Spinlock for accessing fw_ohci data. Never call out of
199	* this driver with this lock held.
200	*/
201	spinlock_t lock;
202
203	struct mutex phy_reg_mutex;
204
205	void *misc_buffer;
206	dma_addr_t misc_buffer_bus;
207
208	struct ar_context ar_request_ctx;
209	struct ar_context ar_response_ctx;
210	struct at_context at_request_ctx;
211	struct at_context at_response_ctx;
212
213	u32 it_context_support;
214	u32 it_context_mask; /* unoccupied IT contexts */
215	struct iso_context *it_context_list;
216	u64 ir_context_channels; /* unoccupied channels */
217	u32 ir_context_support;
218	u32 ir_context_mask; /* unoccupied IR contexts */
219	struct iso_context *ir_context_list;
220	u64 mc_channels; /* channels in use by the multichannel IR context */
221	bool mc_allocated;
222
223	__be32 *config_rom;
224	dma_addr_t config_rom_bus;
225	__be32 *next_config_rom;
226	dma_addr_t next_config_rom_bus;
227	__be32 next_header;
228
229	__le32 *self_id;
230	dma_addr_t self_id_bus;
231
232	u32 self_id_buffer[512];
233	};
234
235	static inline struct fw_ohci *fw_ohci(struct fw_card *card)
236	{
237	return container_of(card, struct fw_ohci, card);
238	}
239
240	#define IT_CONTEXT_CYCLE_MATCH_ENABLE 0x80000000
241	#define IR_CONTEXT_BUFFER_FILL 0x80000000
242	#define IR_CONTEXT_ISOCH_HEADER 0x40000000
243	#define IR_CONTEXT_CYCLE_MATCH_ENABLE 0x20000000
244	#define IR_CONTEXT_MULTI_CHANNEL_MODE 0x10000000
245	#define IR_CONTEXT_DUAL_BUFFER_MODE 0x08000000
246
247	#define CONTEXT_RUN 0x8000
248	#define CONTEXT_WAKE 0x1000
249	#define CONTEXT_DEAD 0x0800
250	#define CONTEXT_ACTIVE 0x0400
251
252	#define OHCI1394_MAX_AT_REQ_RETRIES 0xf
253	#define OHCI1394_MAX_AT_RESP_RETRIES 0x2
254	#define OHCI1394_MAX_PHYS_RESP_RETRIES 0x8
255
256	#define OHCI1394_REGISTER_SIZE 0x800
257	#define OHCI1394_PCI_HCI_Control 0x40
258	#define SELF_ID_BUF_SIZE 0x800
259	#define OHCI_VERSION_1_1 0x010010
260
261	static char ohci_driver_name[] = KBUILD_MODNAME;
262
263	#define PCI_VENDOR_ID_PINNACLE_SYSTEMS 0x11bd
264	#define PCI_DEVICE_ID_AGERE_FW643 0x5901
265	#define PCI_DEVICE_ID_CREATIVE_SB1394 0x4001
266	#define PCI_DEVICE_ID_JMICRON_JMB38X_FW 0x2380
267	#define PCI_DEVICE_ID_TI_TSB12LV22 0x8009
268	#define PCI_DEVICE_ID_TI_TSB12LV26 0x8020
269	#define PCI_DEVICE_ID_TI_TSB82AA2 0x8025
270	#define PCI_DEVICE_ID_VIA_VT630X 0x3044
271	#define PCI_REV_ID_VIA_VT6306 0x46
272	#define PCI_DEVICE_ID_VIA_VT6315 0x3403
273
274	#define QUIRK_CYCLE_TIMER 0x1
275	#define QUIRK_RESET_PACKET 0x2
276	#define QUIRK_BE_HEADERS 0x4
277	#define QUIRK_NO_1394A 0x8
278	#define QUIRK_NO_MSI 0x10
279	#define QUIRK_TI_SLLZ059 0x20
280	#define QUIRK_IR_WAKE 0x40
281
282	// On PCI Express Root Complex in any type of AMD Ryzen machine, VIA VT6306/6307/6308 with Asmedia
283	// ASM1083/1085 brings an inconvenience that the read accesses to 'Isochronous Cycle Timer' register
284	// (at offset 0xf0 in PCI I/O space) often causes unexpected system reboot. The mechanism is not
285	// clear, since the read access to the other registers is enough safe; e.g. 'Node ID' register,
286	// while it is probable due to detection of any type of PCIe error.
287	#define QUIRK_REBOOT_BY_CYCLE_TIMER_READ 0x80000000
288
289	#if IS_ENABLED(CONFIG_X86)
290
291	static bool has_reboot_by_cycle_timer_read_quirk(const struct fw_ohci *ohci)
292	{
293	return !!(ohci->quirks & QUIRK_REBOOT_BY_CYCLE_TIMER_READ);
294	}
295
296	#define PCI_DEVICE_ID_ASMEDIA_ASM108X 0x1080
297
298	static bool detect_vt630x_with_asm1083_on_amd_ryzen_machine(const struct pci_dev *pdev)
299	{
300	const struct pci_dev *pcie_to_pci_bridge;
301
302	// Detect any type of AMD Ryzen machine.
303	if (!static_cpu_has(X86_FEATURE_ZEN))
304	return false;
305
306	// Detect VIA VT6306/6307/6308.
307	if (pdev->vendor != PCI_VENDOR_ID_VIA)
308	return false;
309	if (pdev->device != PCI_DEVICE_ID_VIA_VT630X)
310	return false;
311
312	// Detect Asmedia ASM1083/1085.
313	pcie_to_pci_bridge = pdev->bus->self;
314	if (pcie_to_pci_bridge->vendor != PCI_VENDOR_ID_ASMEDIA)
315	return false;
316	if (pcie_to_pci_bridge->device != PCI_DEVICE_ID_ASMEDIA_ASM108X)
317	return false;
318
319	return true;
320	}
321
322	#else
323	#define has_reboot_by_cycle_timer_read_quirk(ohci) false
324	#define detect_vt630x_with_asm1083_on_amd_ryzen_machine(pdev) false
325	#endif
326
327	/* In case of multiple matches in ohci_quirks[], only the first one is used. */
328	static const struct {
329	unsigned short vendor, device, revision, flags;
330	} ohci_quirks[] = {
331	{PCI_VENDOR_ID_AL, PCI_ANY_ID, PCI_ANY_ID,
332	QUIRK_CYCLE_TIMER},
333
334	{PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_FW, PCI_ANY_ID,
335	QUIRK_BE_HEADERS},
336
337	{PCI_VENDOR_ID_ATT, PCI_DEVICE_ID_AGERE_FW643, 6,
338	QUIRK_NO_MSI},
339
340	{PCI_VENDOR_ID_CREATIVE, PCI_DEVICE_ID_CREATIVE_SB1394, PCI_ANY_ID,
341	QUIRK_RESET_PACKET},
342
343	{PCI_VENDOR_ID_JMICRON, PCI_DEVICE_ID_JMICRON_JMB38X_FW, PCI_ANY_ID,
344	QUIRK_NO_MSI},
345
346	{PCI_VENDOR_ID_NEC, PCI_ANY_ID, PCI_ANY_ID,
347	QUIRK_CYCLE_TIMER},
348
349	{PCI_VENDOR_ID_O2, PCI_ANY_ID, PCI_ANY_ID,
350	QUIRK_NO_MSI},
351
352	{PCI_VENDOR_ID_RICOH, PCI_ANY_ID, PCI_ANY_ID,
353	QUIRK_CYCLE_TIMER | QUIRK_NO_MSI},
354
355	{PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_TSB12LV22, PCI_ANY_ID,
356	QUIRK_CYCLE_TIMER | QUIRK_RESET_PACKET | QUIRK_NO_1394A},
357
358	{PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_TSB12LV26, PCI_ANY_ID,
359	QUIRK_RESET_PACKET | QUIRK_TI_SLLZ059},
360
361	{PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_TSB82AA2, PCI_ANY_ID,
362	QUIRK_RESET_PACKET | QUIRK_TI_SLLZ059},
363
364	{PCI_VENDOR_ID_TI, PCI_ANY_ID, PCI_ANY_ID,
365	QUIRK_RESET_PACKET},
366
367	{PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_VT630X, PCI_REV_ID_VIA_VT6306,
368	QUIRK_CYCLE_TIMER | QUIRK_IR_WAKE},
369
370	{PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_VT6315, 0,
371	QUIRK_CYCLE_TIMER /* FIXME: necessary? */ | QUIRK_NO_MSI},
372
373	{PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_VT6315, PCI_ANY_ID,
374	QUIRK_NO_MSI},
375
376	{PCI_VENDOR_ID_VIA, PCI_ANY_ID, PCI_ANY_ID,
377	QUIRK_CYCLE_TIMER | QUIRK_NO_MSI},
378	};
379
380	/* This overrides anything that was found in ohci_quirks[]. */
381	static int param_quirks;
382	module_param_named(quirks, param_quirks, int, 0644);
383	MODULE_PARM_DESC(quirks, "Chip quirks (default = 0"
384	", nonatomic cycle timer = " __stringify(QUIRK_CYCLE_TIMER)
385	", reset packet generation = " __stringify(QUIRK_RESET_PACKET)
386	", AR/selfID endianness = " __stringify(QUIRK_BE_HEADERS)
387	", no 1394a enhancements = " __stringify(QUIRK_NO_1394A)
388	", disable MSI = " __stringify(QUIRK_NO_MSI)
389	", TI SLLZ059 erratum = " __stringify(QUIRK_TI_SLLZ059)
390	", IR wake unreliable = " __stringify(QUIRK_IR_WAKE)
391	")");
392
393	static bool param_remote_dma;
394	module_param_named(remote_dma, param_remote_dma, bool, 0444);
395	MODULE_PARM_DESC(remote_dma, "Enable unfiltered remote DMA (default = N)");
396
397	static inline void reg_write(const struct fw_ohci *ohci, int offset, u32 data)
398	{
399	writel(val: data, addr: ohci->registers + offset);
400	}
401
402	static inline u32 reg_read(const struct fw_ohci *ohci, int offset)
403	{
404	return readl(addr: ohci->registers + offset);
405	}
406
407	static inline void flush_writes(const struct fw_ohci *ohci)
408	{
409	/* Do a dummy read to flush writes. */
410	reg_read(ohci, OHCI1394_Version);
411	}
412
413	/*
414	* Beware! read_phy_reg(), write_phy_reg(), update_phy_reg(), and
415	* read_paged_phy_reg() require the caller to hold ohci->phy_reg_mutex.
416	* In other words, only use ohci_read_phy_reg() and ohci_update_phy_reg()
417	* directly. Exceptions are intrinsically serialized contexts like pci_probe.
418	*/
419	static int read_phy_reg(struct fw_ohci *ohci, int addr)
420	{
421	u32 val;
422	int i;
423
424	reg_write(ohci, OHCI1394_PhyControl, OHCI1394_PhyControl_Read(addr));
425	for (i = 0; i < 3 + 100; i++) {
426	val = reg_read(ohci, OHCI1394_PhyControl);
427	if (!~val)
428	return -ENODEV; /* Card was ejected. */
429
430	if (val & OHCI1394_PhyControl_ReadDone)
431	return OHCI1394_PhyControl_ReadData(val);
432
433	/*
434	* Try a few times without waiting. Sleeping is necessary
435	* only when the link/PHY interface is busy.
436	*/
437	if (i >= 3)
438	msleep(msecs: 1);
439	}
440	ohci_err(ohci, "failed to read phy reg %d\n", addr);
441	dump_stack();
442
443	return -EBUSY;
444	}
445
446	static int write_phy_reg(const struct fw_ohci *ohci, int addr, u32 val)
447	{
448	int i;
449
450	reg_write(ohci, OHCI1394_PhyControl,
451	OHCI1394_PhyControl_Write(addr, val));
452	for (i = 0; i < 3 + 100; i++) {
453	val = reg_read(ohci, OHCI1394_PhyControl);
454	if (!~val)
455	return -ENODEV; /* Card was ejected. */
456
457	if (!(val & OHCI1394_PhyControl_WritePending))
458	return 0;
459
460	if (i >= 3)
461	msleep(msecs: 1);
462	}
463	ohci_err(ohci, "failed to write phy reg %d, val %u\n", addr, val);
464	dump_stack();
465
466	return -EBUSY;
467	}
468
469	static int update_phy_reg(struct fw_ohci *ohci, int addr,
470	int clear_bits, int set_bits)
471	{
472	int ret = read_phy_reg(ohci, addr);
473	if (ret < 0)
474	return ret;
475
476	/*
477	* The interrupt status bits are cleared by writing a one bit.
478	* Avoid clearing them unless explicitly requested in set_bits.
479	*/
480	if (addr == 5)
481	clear_bits |= PHY_INT_STATUS_BITS;
482
483	return write_phy_reg(ohci, addr, val: (ret & ~clear_bits) | set_bits);
484	}
485
486	static int read_paged_phy_reg(struct fw_ohci *ohci, int page, int addr)
487	{
488	int ret;
489
490	ret = update_phy_reg(ohci, addr: 7, PHY_PAGE_SELECT, set_bits: page << 5);
491	if (ret < 0)
492	return ret;
493
494	return read_phy_reg(ohci, addr);
495	}
496
497	static int ohci_read_phy_reg(struct fw_card *card, int addr)
498	{
499	struct fw_ohci *ohci = fw_ohci(card);
500
501	guard(mutex)(T: &ohci->phy_reg_mutex);
502
503	return read_phy_reg(ohci, addr);
504	}
505
506	static int ohci_update_phy_reg(struct fw_card *card, int addr,
507	int clear_bits, int set_bits)
508	{
509	struct fw_ohci *ohci = fw_ohci(card);
510
511	guard(mutex)(T: &ohci->phy_reg_mutex);
512
513	return update_phy_reg(ohci, addr, clear_bits, set_bits);
514	}
515
516	static inline dma_addr_t ar_buffer_bus(struct ar_context *ctx, unsigned int i)
517	{
518	return page_private(ctx->pages[i]);
519	}
520
521	static void ar_context_link_page(struct ar_context *ctx, unsigned int index)
522	{
523	struct descriptor *d;
524
525	d = &ctx->descriptors[index];
526	d->branch_address &= cpu_to_le32(~0xf);
527	d->res_count = cpu_to_le16(PAGE_SIZE);
528	d->transfer_status = 0;
529
530	wmb(); /* finish init of new descriptors before branch_address update */
531	d = &ctx->descriptors[ctx->last_buffer_index];
532	d->branch_address |= cpu_to_le32(1);
533
534	ctx->last_buffer_index = index;
535
536	reg_write(ohci: ctx->ohci, CONTROL_SET(ctx->regs), CONTEXT_WAKE);
537	}
538
539	static void ar_context_release(struct ar_context *ctx)
540	{
541	struct device *dev = ctx->ohci->card.device;
542	unsigned int i;
543
544	if (!ctx->buffer)
545	return;
546
547	vunmap(addr: ctx->buffer);
548
549	for (i = 0; i < AR_BUFFERS; i++) {
550	if (ctx->pages[i])
551	dma_free_pages(dev, PAGE_SIZE, page: ctx->pages[i],
552	dma_handle: ar_buffer_bus(ctx, i), dir: DMA_FROM_DEVICE);
553	}
554	}
555
556	static void ar_context_abort(struct ar_context *ctx, const char *error_msg)
557	{
558	struct fw_ohci *ohci = ctx->ohci;
559
560	if (reg_read(ohci, CONTROL_CLEAR(ctx->regs)) & CONTEXT_RUN) {
561	reg_write(ohci, CONTROL_CLEAR(ctx->regs), CONTEXT_RUN);
562	flush_writes(ohci);
563
564	ohci_err(ohci, "AR error: %s; DMA stopped\n", error_msg);
565	}
566	/* FIXME: restart? */
567	}
568
569	static inline unsigned int ar_next_buffer_index(unsigned int index)
570	{
571	return (index + 1) % AR_BUFFERS;
572	}
573
574	static inline unsigned int ar_first_buffer_index(struct ar_context *ctx)
575	{
576	return ar_next_buffer_index(index: ctx->last_buffer_index);
577	}
578
579	/*
580	* We search for the buffer that contains the last AR packet DMA data written
581	* by the controller.
582	*/
583	static unsigned int ar_search_last_active_buffer(struct ar_context *ctx,
584	unsigned int *buffer_offset)
585	{
586	unsigned int i, next_i, last = ctx->last_buffer_index;
587	__le16 res_count, next_res_count;
588
589	i = ar_first_buffer_index(ctx);
590	res_count = READ_ONCE(ctx->descriptors[i].res_count);
591
592	/* A buffer that is not yet completely filled must be the last one. */
593	while (i != last && res_count == 0) {
594
595	/* Peek at the next descriptor. */
596	next_i = ar_next_buffer_index(index: i);
597	rmb(); /* read descriptors in order */
598	next_res_count = READ_ONCE(ctx->descriptors[next_i].res_count);
599	/*
600	* If the next descriptor is still empty, we must stop at this
601	* descriptor.
602	*/
603	if (next_res_count == cpu_to_le16(PAGE_SIZE)) {
604	/*
605	* The exception is when the DMA data for one packet is
606	* split over three buffers; in this case, the middle
607	* buffer's descriptor might be never updated by the
608	* controller and look still empty, and we have to peek
609	* at the third one.
610	*/
611	if (MAX_AR_PACKET_SIZE > PAGE_SIZE && i != last) {
612	next_i = ar_next_buffer_index(index: next_i);
613	rmb();
614	next_res_count = READ_ONCE(ctx->descriptors[next_i].res_count);
615	if (next_res_count != cpu_to_le16(PAGE_SIZE))
616	goto next_buffer_is_active;
617	}
618
619	break;
620	}
621
622	next_buffer_is_active:
623	i = next_i;
624	res_count = next_res_count;
625	}
626
627	rmb(); /* read res_count before the DMA data */
628
629	*buffer_offset = PAGE_SIZE - le16_to_cpu(res_count);
630	if (*buffer_offset > PAGE_SIZE) {
631	*buffer_offset = 0;
632	ar_context_abort(ctx, error_msg: "corrupted descriptor");
633	}
634
635	return i;
636	}
637
638	static void ar_sync_buffers_for_cpu(struct ar_context *ctx,
639	unsigned int end_buffer_index,
640	unsigned int end_buffer_offset)
641	{
642	unsigned int i;
643
644	i = ar_first_buffer_index(ctx);
645	while (i != end_buffer_index) {
646	dma_sync_single_for_cpu(dev: ctx->ohci->card.device,
647	addr: ar_buffer_bus(ctx, i),
648	PAGE_SIZE, dir: DMA_FROM_DEVICE);
649	i = ar_next_buffer_index(index: i);
650	}
651	if (end_buffer_offset > 0)
652	dma_sync_single_for_cpu(dev: ctx->ohci->card.device,
653	addr: ar_buffer_bus(ctx, i),
654	size: end_buffer_offset, dir: DMA_FROM_DEVICE);
655	}
656
657	#if defined(CONFIG_PPC_PMAC) && defined(CONFIG_PPC32)
658	static u32 cond_le32_to_cpu(__le32 value, bool has_be_header_quirk)
659	{
660	return has_be_header_quirk ? (__force __u32)value : le32_to_cpu(value);
661	}
662
663	static bool has_be_header_quirk(const struct fw_ohci *ohci)
664	{
665	return !!(ohci->quirks & QUIRK_BE_HEADERS);
666	}
667	#else
668	static u32 cond_le32_to_cpu(__le32 value, bool has_be_header_quirk __maybe_unused)
669	{
670	return le32_to_cpu(value);
671	}
672
673	static bool has_be_header_quirk(const struct fw_ohci *ohci)
674	{
675	return false;
676	}
677	#endif
678
679	static __le32 *handle_ar_packet(struct ar_context *ctx, __le32 *buffer)
680	{
681	struct fw_ohci *ohci = ctx->ohci;
682	struct fw_packet p;
683	u32 status, length, tcode;
684	int evt;
685
686	p.header[0] = cond_le32_to_cpu(value: buffer[0], has_be_header_quirk: has_be_header_quirk(ohci));
687	p.header[1] = cond_le32_to_cpu(value: buffer[1], has_be_header_quirk: has_be_header_quirk(ohci));
688	p.header[2] = cond_le32_to_cpu(value: buffer[2], has_be_header_quirk: has_be_header_quirk(ohci));
689
690	tcode = async_header_get_tcode(header: p.header);
691	switch (tcode) {
692	case TCODE_WRITE_QUADLET_REQUEST:
693	case TCODE_READ_QUADLET_RESPONSE:
694	p.header[3] = (__force __u32) buffer[3];
695	p.header_length = 16;
696	p.payload_length = 0;
697	break;
698
699	case TCODE_READ_BLOCK_REQUEST :
700	p.header[3] = cond_le32_to_cpu(value: buffer[3], has_be_header_quirk: has_be_header_quirk(ohci));
701	p.header_length = 16;
702	p.payload_length = 0;
703	break;
704
705	case TCODE_WRITE_BLOCK_REQUEST:
706	case TCODE_READ_BLOCK_RESPONSE:
707	case TCODE_LOCK_REQUEST:
708	case TCODE_LOCK_RESPONSE:
709	p.header[3] = cond_le32_to_cpu(value: buffer[3], has_be_header_quirk: has_be_header_quirk(ohci));
710	p.header_length = 16;
711	p.payload_length = async_header_get_data_length(header: p.header);
712	if (p.payload_length > MAX_ASYNC_PAYLOAD) {
713	ar_context_abort(ctx, error_msg: "invalid packet length");
714	return NULL;
715	}
716	break;
717
718	case TCODE_WRITE_RESPONSE:
719	case TCODE_READ_QUADLET_REQUEST:
720	case TCODE_LINK_INTERNAL:
721	p.header_length = 12;
722	p.payload_length = 0;
723	break;
724
725	default:
726	ar_context_abort(ctx, error_msg: "invalid tcode");
727	return NULL;
728	}
729
730	p.payload = (void *) buffer + p.header_length;
731
732	/* FIXME: What to do about evt_* errors? */
733	length = (p.header_length + p.payload_length + 3) / 4;
734	status = cond_le32_to_cpu(value: buffer[length], has_be_header_quirk: has_be_header_quirk(ohci));
735	evt = (status >> 16) & 0x1f;
736
737	p.ack = evt - 16;
738	p.speed = (status >> 21) & 0x7;
739	p.timestamp = status & 0xffff;
740	p.generation = ohci->request_generation;
741
742	/*
743	* Several controllers, notably from NEC and VIA, forget to
744	* write ack_complete status at PHY packet reception.
745	*/
746	if (evt == OHCI1394_evt_no_status && tcode == TCODE_LINK_INTERNAL)
747	p.ack = ACK_COMPLETE;
748
749	/*
750	* The OHCI bus reset handler synthesizes a PHY packet with
751	* the new generation number when a bus reset happens (see
752	* section 8.4.2.3). This helps us determine when a request
753	* was received and make sure we send the response in the same
754	* generation. We only need this for requests; for responses
755	* we use the unique tlabel for finding the matching
756	* request.
757	*
758	* Alas some chips sometimes emit bus reset packets with a
759	* wrong generation. We set the correct generation for these
760	* at a slightly incorrect time (in handle_selfid_complete_event).
761	*/
762	if (evt == OHCI1394_evt_bus_reset) {
763	if (!(ohci->quirks & QUIRK_RESET_PACKET))
764	ohci->request_generation = (p.header[2] >> 16) & 0xff;
765	} else if (ctx == &ohci->ar_request_ctx) {
766	fw_core_handle_request(card: &ohci->card, request: &p);
767	} else {
768	fw_core_handle_response(card: &ohci->card, packet: &p);
769	}
770
771	return buffer + length + 1;
772	}
773
774	static void *handle_ar_packets(struct ar_context *ctx, void *p, void *end)
775	{
776	void *next;
777
778	while (p < end) {
779	next = handle_ar_packet(ctx, buffer: p);
780	if (!next)
781	return p;
782	p = next;
783	}
784
785	return p;
786	}
787
788	static void ar_recycle_buffers(struct ar_context *ctx, unsigned int end_buffer)
789	{
790	unsigned int i;
791
792	i = ar_first_buffer_index(ctx);
793	while (i != end_buffer) {
794	dma_sync_single_for_device(dev: ctx->ohci->card.device,
795	addr: ar_buffer_bus(ctx, i),
796	PAGE_SIZE, dir: DMA_FROM_DEVICE);
797	ar_context_link_page(ctx, index: i);
798	i = ar_next_buffer_index(index: i);
799	}
800	}
801
802	static void ohci_ar_context_work(struct work_struct *work)
803	{
804	struct ar_context *ctx = from_work(ctx, work, work);
805	unsigned int end_buffer_index, end_buffer_offset;
806	void *p, *end;
807
808	p = ctx->pointer;
809	if (!p)
810	return;
811
812	end_buffer_index = ar_search_last_active_buffer(ctx, buffer_offset: &end_buffer_offset);
813	ar_sync_buffers_for_cpu(ctx, end_buffer_index, end_buffer_offset);
814	end = ctx->buffer + end_buffer_index * PAGE_SIZE + end_buffer_offset;
815
816	if (end_buffer_index < ar_first_buffer_index(ctx)) {
817	// The filled part of the overall buffer wraps around; handle all packets up to the
818	// buffer end here. If the last packet wraps around, its tail will be visible after
819	// the buffer end because the buffer start pages are mapped there again.
820	void *buffer_end = ctx->buffer + AR_BUFFERS * PAGE_SIZE;
821	p = handle_ar_packets(ctx, p, end: buffer_end);
822	if (p < buffer_end)
823	goto error;
824	// adjust p to point back into the actual buffer
825	p -= AR_BUFFERS * PAGE_SIZE;
826	}
827
828	p = handle_ar_packets(ctx, p, end);
829	if (p != end) {
830	if (p > end)
831	ar_context_abort(ctx, error_msg: "inconsistent descriptor");
832	goto error;
833	}
834
835	ctx->pointer = p;
836	ar_recycle_buffers(ctx, end_buffer: end_buffer_index);
837
838	return;
839	error:
840	ctx->pointer = NULL;
841	}
842
843	static int ar_context_init(struct ar_context *ctx, struct fw_ohci *ohci,
844	unsigned int descriptors_offset, u32 regs)
845	{
846	struct device *dev = ohci->card.device;
847	unsigned int i;
848	dma_addr_t dma_addr;
849	struct page *pages[AR_BUFFERS + AR_WRAPAROUND_PAGES];
850	struct descriptor *d;
851
852	ctx->regs = regs;
853	ctx->ohci = ohci;
854	INIT_WORK(&ctx->work, ohci_ar_context_work);
855
856	for (i = 0; i < AR_BUFFERS; i++) {
857	ctx->pages[i] = dma_alloc_pages(dev, PAGE_SIZE, dma_handle: &dma_addr,
858	dir: DMA_FROM_DEVICE, GFP_KERNEL);
859	if (!ctx->pages[i])
860	goto out_of_memory;
861	set_page_private(page: ctx->pages[i], private: dma_addr);
862	dma_sync_single_for_device(dev, addr: dma_addr, PAGE_SIZE,
863	dir: DMA_FROM_DEVICE);
864	}
865
866	for (i = 0; i < AR_BUFFERS; i++)
867	pages[i] = ctx->pages[i];
868	for (i = 0; i < AR_WRAPAROUND_PAGES; i++)
869	pages[AR_BUFFERS + i] = ctx->pages[i];
870	ctx->buffer = vmap(pages, ARRAY_SIZE(pages), VM_MAP, PAGE_KERNEL);
871	if (!ctx->buffer)
872	goto out_of_memory;
873
874	ctx->descriptors = ohci->misc_buffer + descriptors_offset;
875	ctx->descriptors_bus = ohci->misc_buffer_bus + descriptors_offset;
876
877	for (i = 0; i < AR_BUFFERS; i++) {
878	d = &ctx->descriptors[i];
879	d->req_count = cpu_to_le16(PAGE_SIZE);
880	d->control = cpu_to_le16(DESCRIPTOR_INPUT_MORE |
881	DESCRIPTOR_STATUS |
882	DESCRIPTOR_BRANCH_ALWAYS);
883	d->data_address = cpu_to_le32(ar_buffer_bus(ctx, i));
884	d->branch_address = cpu_to_le32(ctx->descriptors_bus +
885	ar_next_buffer_index(i) * sizeof(struct descriptor));
886	}
887
888	return 0;
889
890	out_of_memory:
891	ar_context_release(ctx);
892
893	return -ENOMEM;
894	}
895
896	static void ar_context_run(struct ar_context *ctx)
897	{
898	unsigned int i;
899
900	for (i = 0; i < AR_BUFFERS; i++)
901	ar_context_link_page(ctx, index: i);
902
903	ctx->pointer = ctx->buffer;
904
905	reg_write(ohci: ctx->ohci, COMMAND_PTR(ctx->regs), data: ctx->descriptors_bus | 1);
906	reg_write(ohci: ctx->ohci, CONTROL_SET(ctx->regs), CONTEXT_RUN);
907	}
908
909	static struct descriptor *find_branch_descriptor(struct descriptor *d, int z)
910	{
911	__le16 branch;
912
913	branch = d->control & cpu_to_le16(DESCRIPTOR_BRANCH_ALWAYS);
914
915	/* figure out which descriptor the branch address goes in */
916	if (z == 2 && branch == cpu_to_le16(DESCRIPTOR_BRANCH_ALWAYS))
917	return d;
918	else
919	return d + z - 1;
920	}
921
922	static void context_retire_descriptors(struct context *ctx)
923	{
924	struct descriptor *d, *last;
925	u32 address;
926	int z;
927	struct descriptor_buffer *desc;
928
929	desc = list_entry(ctx->buffer_list.next,
930	struct descriptor_buffer, list);
931	last = ctx->last;
932	while (last->branch_address != 0) {
933	struct descriptor_buffer *old_desc = desc;
934	address = le32_to_cpu(last->branch_address);
935	z = address & 0xf;
936	address &= ~0xf;
937	ctx->current_bus = address;
938
939	/* If the branch address points to a buffer outside of the
940	* current buffer, advance to the next buffer. */
941	if (address < desc->buffer_bus ||
942	address >= desc->buffer_bus + desc->used)
943	desc = list_entry(desc->list.next,
944	struct descriptor_buffer, list);
945	d = desc->buffer + (address - desc->buffer_bus) / sizeof(*d);
946	last = find_branch_descriptor(d, z);
947
948	if (!ctx->callback(ctx, d, last))
949	break;
950
951	if (old_desc != desc) {
952	// If we've advanced to the next buffer, move the previous buffer to the
953	// free list.
954	old_desc->used = 0;
955	guard(spinlock_irqsave)(l: &ctx->ohci->lock);
956	list_move_tail(list: &old_desc->list, head: &ctx->buffer_list);
957	}
958	ctx->last = last;
959	}
960	}
961
962	static void ohci_at_context_work(struct work_struct *work)
963	{
964	struct at_context *ctx = from_work(ctx, work, work);
965
966	context_retire_descriptors(ctx: &ctx->context);
967	}
968
969	static void ohci_isoc_context_work(struct work_struct *work)
970	{
971	struct fw_iso_context *base = from_work(base, work, work);
972	struct iso_context *isoc_ctx = container_of(base, struct iso_context, base);
973
974	context_retire_descriptors(ctx: &isoc_ctx->context);
975	}
976
977	/*
978	* Allocate a new buffer and add it to the list of free buffers for this
979	* context. Must be called with ohci->lock held.
980	*/
981	static int context_add_buffer(struct context *ctx)
982	{
983	struct descriptor_buffer *desc;
984	dma_addr_t bus_addr;
985	int offset;
986
987	/*
988	* 16MB of descriptors should be far more than enough for any DMA
989	* program. This will catch run-away userspace or DoS attacks.
990	*/
991	if (ctx->total_allocation >= 16*1024*1024)
992	return -ENOMEM;
993
994	desc = dmam_alloc_coherent(dev: ctx->ohci->card.device, PAGE_SIZE, dma_handle: &bus_addr, GFP_ATOMIC);
995	if (!desc)
996	return -ENOMEM;
997
998	offset = (void *)&desc->buffer - (void *)desc;
999	/*
1000	* Some controllers, like JMicron ones, always issue 0x20-byte DMA reads
1001	* for descriptors, even 0x10-byte ones. This can cause page faults when
1002	* an IOMMU is in use and the oversized read crosses a page boundary.
1003	* Work around this by always leaving at least 0x10 bytes of padding.
1004	*/
1005	desc->buffer_size = PAGE_SIZE - offset - 0x10;
1006	desc->buffer_bus = bus_addr + offset;
1007	desc->used = 0;
1008
1009	list_add_tail(new: &desc->list, head: &ctx->buffer_list);
1010	ctx->total_allocation += PAGE_SIZE;
1011
1012	return 0;
1013	}
1014
1015	static int context_init(struct context *ctx, struct fw_ohci *ohci,
1016	u32 regs, descriptor_callback_t callback)
1017	{
1018	ctx->ohci = ohci;
1019	ctx->regs = regs;
1020	ctx->total_allocation = 0;
1021
1022	INIT_LIST_HEAD(list: &ctx->buffer_list);
1023	if (context_add_buffer(ctx) < 0)
1024	return -ENOMEM;
1025
1026	ctx->buffer_tail = list_entry(ctx->buffer_list.next,
1027	struct descriptor_buffer, list);
1028
1029	ctx->callback = callback;
1030
1031	/*
1032	* We put a dummy descriptor in the buffer that has a NULL
1033	* branch address and looks like it's been sent. That way we
1034	* have a descriptor to append DMA programs to.
1035	*/
1036	memset(ctx->buffer_tail->buffer, 0, sizeof(*ctx->buffer_tail->buffer));
1037	ctx->buffer_tail->buffer->control = cpu_to_le16(DESCRIPTOR_OUTPUT_LAST);
1038	ctx->buffer_tail->buffer->transfer_status = cpu_to_le16(0x8011);
1039	ctx->buffer_tail->used += sizeof(*ctx->buffer_tail->buffer);
1040	ctx->last = ctx->buffer_tail->buffer;
1041	ctx->prev = ctx->buffer_tail->buffer;
1042	ctx->prev_z = 1;
1043
1044	return 0;
1045	}
1046
1047	static void context_release(struct context *ctx)
1048	{
1049	struct fw_card *card = &ctx->ohci->card;
1050	struct descriptor_buffer *desc, *tmp;
1051
1052	list_for_each_entry_safe(desc, tmp, &ctx->buffer_list, list) {
1053	dmam_free_coherent(dev: card->device, PAGE_SIZE, vaddr: desc,
1054	dma_handle: desc->buffer_bus - ((void *)&desc->buffer - (void *)desc));
1055	}
1056	}
1057
1058	/* Must be called with ohci->lock held */
1059	static struct descriptor *context_get_descriptors(struct context *ctx,
1060	int z, dma_addr_t *d_bus)
1061	{
1062	struct descriptor *d = NULL;
1063	struct descriptor_buffer *desc = ctx->buffer_tail;
1064
1065	if (z * sizeof(*d) > desc->buffer_size)
1066	return NULL;
1067
1068	if (z * sizeof(*d) > desc->buffer_size - desc->used) {
1069	/* No room for the descriptor in this buffer, so advance to the
1070	* next one. */
1071
1072	if (desc->list.next == &ctx->buffer_list) {
1073	/* If there is no free buffer next in the list,
1074	* allocate one. */
1075	if (context_add_buffer(ctx) < 0)
1076	return NULL;
1077	}
1078	desc = list_entry(desc->list.next,
1079	struct descriptor_buffer, list);
1080	ctx->buffer_tail = desc;
1081	}
1082
1083	d = desc->buffer + desc->used / sizeof(*d);
1084	memset(d, 0, z * sizeof(*d));
1085	*d_bus = desc->buffer_bus + desc->used;
1086
1087	return d;
1088	}
1089
1090	static void context_run(struct context *ctx, u32 extra)
1091	{
1092	struct fw_ohci *ohci = ctx->ohci;
1093
1094	reg_write(ohci, COMMAND_PTR(ctx->regs),
1095	le32_to_cpu(ctx->last->branch_address));
1096	reg_write(ohci, CONTROL_CLEAR(ctx->regs), data: ~0);
1097	reg_write(ohci, CONTROL_SET(ctx->regs), CONTEXT_RUN | extra);
1098	ctx->running = true;
1099	flush_writes(ohci);
1100	}
1101
1102	static void context_append(struct context *ctx,
1103	struct descriptor *d, int z, int extra)
1104	{
1105	dma_addr_t d_bus;
1106	struct descriptor_buffer *desc = ctx->buffer_tail;
1107	struct descriptor *d_branch;
1108
1109	d_bus = desc->buffer_bus + (d - desc->buffer) * sizeof(*d);
1110
1111	desc->used += (z + extra) * sizeof(*d);
1112
1113	wmb(); /* finish init of new descriptors before branch_address update */
1114
1115	d_branch = find_branch_descriptor(d: ctx->prev, z: ctx->prev_z);
1116	d_branch->branch_address = cpu_to_le32(d_bus | z);
1117
1118	/*
1119	* VT6306 incorrectly checks only the single descriptor at the
1120	* CommandPtr when the wake bit is written, so if it's a
1121	* multi-descriptor block starting with an INPUT_MORE, put a copy of
1122	* the branch address in the first descriptor.
1123	*
1124	* Not doing this for transmit contexts since not sure how it interacts
1125	* with skip addresses.
1126	*/
1127	if (unlikely(ctx->ohci->quirks & QUIRK_IR_WAKE) &&
1128	d_branch != ctx->prev &&
1129	(ctx->prev->control & cpu_to_le16(DESCRIPTOR_CMD)) ==
1130	cpu_to_le16(DESCRIPTOR_INPUT_MORE)) {
1131	ctx->prev->branch_address = cpu_to_le32(d_bus | z);
1132	}
1133
1134	ctx->prev = d;
1135	ctx->prev_z = z;
1136	}
1137
1138	static void context_stop(struct context *ctx)
1139	{
1140	struct fw_ohci *ohci = ctx->ohci;
1141	u32 reg;
1142	int i;
1143
1144	reg_write(ohci, CONTROL_CLEAR(ctx->regs), CONTEXT_RUN);
1145	ctx->running = false;
1146
1147	for (i = 0; i < 1000; i++) {
1148	reg = reg_read(ohci, CONTROL_SET(ctx->regs));
1149	if ((reg & CONTEXT_ACTIVE) == 0)
1150	return;
1151
1152	if (i)
1153	udelay(usec: 10);
1154	}
1155	ohci_err(ohci, "DMA context still active (0x%08x)\n", reg);
1156	}
1157
1158	struct driver_data {
1159	u8 inline_data[8];
1160	struct fw_packet *packet;
1161	};
1162
1163	/*
1164	* This function appends a packet to the DMA queue for transmission.
1165	* Must always be called with the ochi->lock held to ensure proper
1166	* generation handling and locking around packet queue manipulation.
1167	*/
1168	static int at_context_queue_packet(struct at_context *ctx, struct fw_packet *packet)
1169	{
1170	struct context *context = &ctx->context;
1171	struct fw_ohci *ohci = context->ohci;
1172	dma_addr_t d_bus, payload_bus;
1173	struct driver_data *driver_data;
1174	struct descriptor *d, *last;
1175	__le32 *header;
1176	int z, tcode;
1177
1178	d = context_get_descriptors(ctx: context, z: 4, d_bus: &d_bus);
1179	if (d == NULL) {
1180	packet->ack = RCODE_SEND_ERROR;
1181	return -1;
1182	}
1183
1184	d[0].control = cpu_to_le16(DESCRIPTOR_KEY_IMMEDIATE);
1185	d[0].res_count = cpu_to_le16(packet->timestamp);
1186
1187	tcode = async_header_get_tcode(header: packet->header);
1188	header = (__le32 *) &d[1];
1189	switch (tcode) {
1190	case TCODE_WRITE_QUADLET_REQUEST:
1191	case TCODE_WRITE_BLOCK_REQUEST:
1192	case TCODE_WRITE_RESPONSE:
1193	case TCODE_READ_QUADLET_REQUEST:
1194	case TCODE_READ_BLOCK_REQUEST:
1195	case TCODE_READ_QUADLET_RESPONSE:
1196	case TCODE_READ_BLOCK_RESPONSE:
1197	case TCODE_LOCK_REQUEST:
1198	case TCODE_LOCK_RESPONSE:
1199	ohci1394_at_data_set_src_bus_id(data: header, src_bus_id: false);
1200	ohci1394_at_data_set_speed(data: header, scode: packet->speed);
1201	ohci1394_at_data_set_tlabel(data: header, tlabel: async_header_get_tlabel(header: packet->header));
1202	ohci1394_at_data_set_retry(data: header, retry: async_header_get_retry(header: packet->header));
1203	ohci1394_at_data_set_tcode(data: header, tcode);
1204
1205	ohci1394_at_data_set_destination_id(data: header,
1206	destination_id: async_header_get_destination(header: packet->header));
1207
1208	if (ctx == &ohci->at_response_ctx) {
1209	ohci1394_at_data_set_rcode(data: header, rcode: async_header_get_rcode(header: packet->header));
1210	} else {
1211	ohci1394_at_data_set_destination_offset(data: header,
1212	offset: async_header_get_offset(header: packet->header));
1213	}
1214
1215	if (tcode_is_block_packet(tcode))
1216	header[3] = cpu_to_le32(packet->header[3]);
1217	else
1218	header[3] = (__force __le32) packet->header[3];
1219
1220	d[0].req_count = cpu_to_le16(packet->header_length);
1221	break;
1222	case TCODE_LINK_INTERNAL:
1223	ohci1394_at_data_set_speed(data: header, scode: packet->speed);
1224	ohci1394_at_data_set_tcode(data: header, TCODE_LINK_INTERNAL);
1225
1226	header[1] = cpu_to_le32(packet->header[1]);
1227	header[2] = cpu_to_le32(packet->header[2]);
1228	d[0].req_count = cpu_to_le16(12);
1229
1230	if (is_ping_packet(data: &packet->header[1]))
1231	d[0].control |= cpu_to_le16(DESCRIPTOR_PING);
1232	break;
1233
1234	case TCODE_STREAM_DATA:
1235	ohci1394_it_data_set_speed(data: header, scode: packet->speed);
1236	ohci1394_it_data_set_tag(data: header, tag: isoc_header_get_tag(header: packet->header[0]));
1237	ohci1394_it_data_set_channel(data: header, channel: isoc_header_get_channel(header: packet->header[0]));
1238	ohci1394_it_data_set_tcode(data: header, TCODE_STREAM_DATA);
1239	ohci1394_it_data_set_sync(data: header, sync: isoc_header_get_sy(header: packet->header[0]));
1240
1241	ohci1394_it_data_set_data_length(data: header, data_length: isoc_header_get_data_length(header: packet->header[0]));
1242
1243	d[0].req_count = cpu_to_le16(8);
1244	break;
1245
1246	default:
1247	/* BUG(); */
1248	packet->ack = RCODE_SEND_ERROR;
1249	return -1;
1250	}
1251
1252	BUILD_BUG_ON(sizeof(struct driver_data) > sizeof(struct descriptor));
1253	driver_data = (struct driver_data *) &d[3];
1254	driver_data->packet = packet;
1255	packet->driver_data = driver_data;
1256
1257	if (packet->payload_length > 0) {
1258	if (packet->payload_length > sizeof(driver_data->inline_data)) {
1259	payload_bus = dma_map_single(ohci->card.device,
1260	packet->payload,
1261	packet->payload_length,
1262	DMA_TO_DEVICE);
1263	if (dma_mapping_error(dev: ohci->card.device, dma_addr: payload_bus)) {
1264	packet->ack = RCODE_SEND_ERROR;
1265	return -1;
1266	}
1267	packet->payload_bus = payload_bus;
1268	packet->payload_mapped = true;
1269	} else {
1270	memcpy(driver_data->inline_data, packet->payload,
1271	packet->payload_length);
1272	payload_bus = d_bus + 3 * sizeof(*d);
1273	}
1274
1275	d[2].req_count = cpu_to_le16(packet->payload_length);
1276	d[2].data_address = cpu_to_le32(payload_bus);
1277	last = &d[2];
1278	z = 3;
1279	} else {
1280	last = &d[0];
1281	z = 2;
1282	}
1283
1284	last->control |= cpu_to_le16(DESCRIPTOR_OUTPUT_LAST |
1285	DESCRIPTOR_IRQ_ALWAYS |
1286	DESCRIPTOR_BRANCH_ALWAYS);
1287
1288	/* FIXME: Document how the locking works. */
1289	if (ohci->generation != packet->generation) {
1290	if (packet->payload_mapped)
1291	dma_unmap_single(ohci->card.device, payload_bus,
1292	packet->payload_length, DMA_TO_DEVICE);
1293	packet->ack = RCODE_GENERATION;
1294	return -1;
1295	}
1296
1297	context_append(ctx: context, d, z, extra: 4 - z);
1298
1299	if (context->running)
1300	reg_write(ohci, CONTROL_SET(context->regs), CONTEXT_WAKE);
1301	else
1302	context_run(ctx: context, extra: 0);
1303
1304	return 0;
1305	}
1306
1307	static void at_context_flush(struct at_context *ctx)
1308	{
1309	// Avoid dead lock due to programming mistake.
1310	if (WARN_ON_ONCE(current_work() == &ctx->work))
1311	return;
1312
1313	disable_work_sync(work: &ctx->work);
1314
1315	WRITE_ONCE(ctx->flushing, true);
1316	ohci_at_context_work(work: &ctx->work);
1317	WRITE_ONCE(ctx->flushing, false);
1318
1319	enable_work(work: &ctx->work);
1320	}
1321
1322	static int find_fw_device(struct device *dev, const void *data)
1323	{
1324	struct fw_device *device = fw_device(dev);
1325	const u32 *params = data;
1326
1327	return (device->generation == params[0]) && (device->node_id == params[1]);
1328	}
1329
1330	static int handle_at_packet(struct context *context,
1331	struct descriptor *d,
1332	struct descriptor *last)
1333	{
1334	struct at_context *ctx = container_of(context, struct at_context, context);
1335	struct fw_ohci *ohci = ctx->context.ohci;
1336	struct driver_data *driver_data;
1337	struct fw_packet *packet;
1338	int evt;
1339
1340	if (last->transfer_status == 0 && !READ_ONCE(ctx->flushing))
1341	/* This descriptor isn't done yet, stop iteration. */
1342	return 0;
1343
1344	driver_data = (struct driver_data *) &d[3];
1345	packet = driver_data->packet;
1346	if (packet == NULL)
1347	/* This packet was cancelled, just continue. */
1348	return 1;
1349
1350	if (packet->payload_mapped)
1351	dma_unmap_single(ohci->card.device, packet->payload_bus,
1352	packet->payload_length, DMA_TO_DEVICE);
1353
1354	evt = le16_to_cpu(last->transfer_status) & 0x1f;
1355	packet->timestamp = le16_to_cpu(last->res_count);
1356
1357	switch (evt) {
1358	case OHCI1394_evt_timeout:
1359	/* Async response transmit timed out. */
1360	packet->ack = RCODE_CANCELLED;
1361	break;
1362
1363	case OHCI1394_evt_flushed:
1364	/*
1365	* The packet was flushed should give same error as
1366	* when we try to use a stale generation count.
1367	*/
1368	packet->ack = RCODE_GENERATION;
1369	break;
1370
1371	case OHCI1394_evt_missing_ack:
1372	if (READ_ONCE(ctx->flushing))
1373	packet->ack = RCODE_GENERATION;
1374	else {
1375	/*
1376	* Using a valid (current) generation count, but the
1377	* node is not on the bus or not sending acks.
1378	*/
1379	packet->ack = RCODE_NO_ACK;
1380	}
1381	break;
1382
1383	case ACK_COMPLETE + 0x10:
1384	case ACK_PENDING + 0x10:
1385	case ACK_BUSY_X + 0x10:
1386	case ACK_BUSY_A + 0x10:
1387	case ACK_BUSY_B + 0x10:
1388	case ACK_DATA_ERROR + 0x10:
1389	case ACK_TYPE_ERROR + 0x10:
1390	packet->ack = evt - 0x10;
1391	break;
1392
1393	case OHCI1394_evt_no_status:
1394	if (READ_ONCE(ctx->flushing)) {
1395	packet->ack = RCODE_GENERATION;
1396	break;
1397	}
1398	fallthrough;
1399
1400	default:
1401	if (unlikely(evt == 0x10)) {
1402	u32 params[2] = {
1403	packet->generation,
1404	async_header_get_destination(header: packet->header),
1405	};
1406	struct device *dev;
1407
1408	fw_card_get(card: &ohci->card);
1409	dev = device_find_child(parent: ohci->card.device, data: (const void *)params, match: find_fw_device);
1410	fw_card_put(card: &ohci->card);
1411	if (dev) {
1412	struct fw_device *device = fw_device(dev);
1413	int quirks = READ_ONCE(device->quirks);
1414
1415	put_device(dev);
1416	if (quirks & FW_DEVICE_QUIRK_ACK_PACKET_WITH_INVALID_PENDING_CODE) {
1417	packet->ack = ACK_PENDING;
1418	break;
1419	}
1420	}
1421	}
1422	packet->ack = RCODE_SEND_ERROR;
1423	break;
1424	}
1425
1426	packet->callback(packet, &ohci->card, packet->ack);
1427
1428	return 1;
1429	}
1430
1431	static u32 get_cycle_time(struct fw_ohci *ohci);
1432
1433	static void handle_local_rom(struct fw_ohci *ohci,
1434	struct fw_packet *packet, u32 csr)
1435	{
1436	struct fw_packet response;
1437	int tcode, length, i;
1438
1439	tcode = async_header_get_tcode(header: packet->header);
1440	if (tcode_is_block_packet(tcode))
1441	length = async_header_get_data_length(header: packet->header);
1442	else
1443	length = 4;
1444
1445	i = csr - CSR_CONFIG_ROM;
1446	if (i + length > CONFIG_ROM_SIZE) {
1447	fw_fill_response(response: &response, request_header: packet->header,
1448	RCODE_ADDRESS_ERROR, NULL, length: 0);
1449	} else if (!tcode_is_read_request(tcode)) {
1450	fw_fill_response(response: &response, request_header: packet->header,
1451	RCODE_TYPE_ERROR, NULL, length: 0);
1452	} else {
1453	fw_fill_response(response: &response, request_header: packet->header, RCODE_COMPLETE,
1454	payload: (void *) ohci->config_rom + i, length);
1455	}
1456
1457	// Timestamping on behalf of the hardware.
1458	response.timestamp = cycle_time_to_ohci_tstamp(tstamp: get_cycle_time(ohci));
1459	fw_core_handle_response(card: &ohci->card, packet: &response);
1460	}
1461
1462	static void handle_local_lock(struct fw_ohci *ohci,
1463	struct fw_packet *packet, u32 csr)
1464	{
1465	struct fw_packet response;
1466	int tcode, length, ext_tcode, sel, try;
1467	__be32 *payload, lock_old;
1468	u32 lock_arg, lock_data;
1469
1470	tcode = async_header_get_tcode(header: packet->header);
1471	length = async_header_get_data_length(header: packet->header);
1472	payload = packet->payload;
1473	ext_tcode = async_header_get_extended_tcode(header: packet->header);
1474
1475	if (tcode == TCODE_LOCK_REQUEST &&
1476	ext_tcode == EXTCODE_COMPARE_SWAP && length == 8) {
1477	lock_arg = be32_to_cpu(payload[0]);
1478	lock_data = be32_to_cpu(payload[1]);
1479	} else if (tcode == TCODE_READ_QUADLET_REQUEST) {
1480	lock_arg = 0;
1481	lock_data = 0;
1482	} else {
1483	fw_fill_response(response: &response, request_header: packet->header,
1484	RCODE_TYPE_ERROR, NULL, length: 0);
1485	goto out;
1486	}
1487
1488	sel = (csr - CSR_BUS_MANAGER_ID) / 4;
1489	reg_write(ohci, OHCI1394_CSRData, data: lock_data);
1490	reg_write(ohci, OHCI1394_CSRCompareData, data: lock_arg);
1491	reg_write(ohci, OHCI1394_CSRControl, data: sel);
1492
1493	for (try = 0; try < 20; try++)
1494	if (reg_read(ohci, OHCI1394_CSRControl) & 0x80000000) {
1495	lock_old = cpu_to_be32(reg_read(ohci,
1496	OHCI1394_CSRData));
1497	fw_fill_response(response: &response, request_header: packet->header,
1498	RCODE_COMPLETE,
1499	payload: &lock_old, length: sizeof(lock_old));
1500	goto out;
1501	}
1502
1503	ohci_err(ohci, "swap not done (CSR lock timeout)\n");
1504	fw_fill_response(response: &response, request_header: packet->header, RCODE_BUSY, NULL, length: 0);
1505
1506	out:
1507	// Timestamping on behalf of the hardware.
1508	response.timestamp = cycle_time_to_ohci_tstamp(tstamp: get_cycle_time(ohci));
1509	fw_core_handle_response(card: &ohci->card, packet: &response);
1510	}
1511
1512	static void handle_local_request(struct at_context *ctx, struct fw_packet *packet)
1513	{
1514	struct fw_ohci *ohci = ctx->context.ohci;
1515	u64 offset, csr;
1516
1517	if (ctx == &ohci->at_request_ctx) {
1518	packet->ack = ACK_PENDING;
1519	packet->callback(packet, &ohci->card, packet->ack);
1520	}
1521
1522	offset = async_header_get_offset(header: packet->header);
1523	csr = offset - CSR_REGISTER_BASE;
1524
1525	/* Handle config rom reads. */
1526	if (csr >= CSR_CONFIG_ROM && csr < CSR_CONFIG_ROM_END)
1527	handle_local_rom(ohci, packet, csr);
1528	else switch (csr) {
1529	case CSR_BUS_MANAGER_ID:
1530	case CSR_BANDWIDTH_AVAILABLE:
1531	case CSR_CHANNELS_AVAILABLE_HI:
1532	case CSR_CHANNELS_AVAILABLE_LO:
1533	handle_local_lock(ohci, packet, csr);
1534	break;
1535	default:
1536	if (ctx == &ohci->at_request_ctx)
1537	fw_core_handle_request(card: &ohci->card, request: packet);
1538	else
1539	fw_core_handle_response(card: &ohci->card, packet);
1540	break;
1541	}
1542
1543	if (ctx == &ohci->at_response_ctx) {
1544	packet->ack = ACK_COMPLETE;
1545	packet->callback(packet, &ohci->card, packet->ack);
1546	}
1547	}
1548
1549	static void at_context_transmit(struct at_context *ctx, struct fw_packet *packet)
1550	{
1551	struct fw_ohci *ohci = ctx->context.ohci;
1552	unsigned long flags;
1553	int ret;
1554
1555	spin_lock_irqsave(&ohci->lock, flags);
1556
1557	if (async_header_get_destination(header: packet->header) == ohci->node_id &&
1558	ohci->generation == packet->generation) {
1559	spin_unlock_irqrestore(lock: &ohci->lock, flags);
1560
1561	// Timestamping on behalf of the hardware.
1562	packet->timestamp = cycle_time_to_ohci_tstamp(tstamp: get_cycle_time(ohci));
1563
1564	handle_local_request(ctx, packet);
1565	return;
1566	}
1567
1568	ret = at_context_queue_packet(ctx, packet);
1569	spin_unlock_irqrestore(lock: &ohci->lock, flags);
1570
1571	if (ret < 0) {
1572	// Timestamping on behalf of the hardware.
1573	packet->timestamp = cycle_time_to_ohci_tstamp(tstamp: get_cycle_time(ohci));
1574
1575	packet->callback(packet, &ohci->card, packet->ack);
1576	}
1577	}
1578
1579	static void detect_dead_context(struct fw_ohci *ohci,
1580	const char *name, unsigned int regs)
1581	{
1582	static const char *const evts[] = {
1583	[0x00] = "evt_no_status", [0x01] = "-reserved-",
1584	[0x02] = "evt_long_packet", [0x03] = "evt_missing_ack",
1585	[0x04] = "evt_underrun", [0x05] = "evt_overrun",
1586	[0x06] = "evt_descriptor_read", [0x07] = "evt_data_read",
1587	[0x08] = "evt_data_write", [0x09] = "evt_bus_reset",
1588	[0x0a] = "evt_timeout", [0x0b] = "evt_tcode_err",
1589	[0x0c] = "-reserved-", [0x0d] = "-reserved-",
1590	[0x0e] = "evt_unknown", [0x0f] = "evt_flushed",
1591	[0x10] = "-reserved-", [0x11] = "ack_complete",
1592	[0x12] = "ack_pending ", [0x13] = "-reserved-",
1593	[0x14] = "ack_busy_X", [0x15] = "ack_busy_A",
1594	[0x16] = "ack_busy_B", [0x17] = "-reserved-",
1595	[0x18] = "-reserved-", [0x19] = "-reserved-",
1596	[0x1a] = "-reserved-", [0x1b] = "ack_tardy",
1597	[0x1c] = "-reserved-", [0x1d] = "ack_data_error",
1598	[0x1e] = "ack_type_error", [0x1f] = "-reserved-",
1599	[0x20] = "pending/cancelled",
1600	};
1601	u32 ctl;
1602
1603	ctl = reg_read(ohci, CONTROL_SET(regs));
1604	if (ctl & CONTEXT_DEAD)
1605	ohci_err(ohci, "DMA context %s has stopped, error code: %s\n",
1606	name, evts[ctl & 0x1f]);
1607	}
1608
1609	static void handle_dead_contexts(struct fw_ohci *ohci)
1610	{
1611	unsigned int i;
1612	char name[8];
1613
1614	detect_dead_context(ohci, name: "ATReq", OHCI1394_AsReqTrContextBase);
1615	detect_dead_context(ohci, name: "ATRsp", OHCI1394_AsRspTrContextBase);
1616	detect_dead_context(ohci, name: "ARReq", OHCI1394_AsReqRcvContextBase);
1617	detect_dead_context(ohci, name: "ARRsp", OHCI1394_AsRspRcvContextBase);
1618	for (i = 0; i < 32; ++i) {
1619	if (!(ohci->it_context_support & (1 << i)))
1620	continue;
1621	sprintf(buf: name, fmt: "IT%u", i);
1622	detect_dead_context(ohci, name, OHCI1394_IsoXmitContextBase(i));
1623	}
1624	for (i = 0; i < 32; ++i) {
1625	if (!(ohci->ir_context_support & (1 << i)))
1626	continue;
1627	sprintf(buf: name, fmt: "IR%u", i);
1628	detect_dead_context(ohci, name, OHCI1394_IsoRcvContextBase(i));
1629	}
1630	/* TODO: maybe try to flush and restart the dead contexts */
1631	}
1632
1633	static u32 cycle_timer_ticks(u32 cycle_timer)
1634	{
1635	u32 ticks;
1636
1637	ticks = cycle_timer & 0xfff;
1638	ticks += 3072 * ((cycle_timer >> 12) & 0x1fff);
1639	ticks += (3072 * 8000) * (cycle_timer >> 25);
1640
1641	return ticks;
1642	}
1643
1644	/*
1645	* Some controllers exhibit one or more of the following bugs when updating the
1646	* iso cycle timer register:
1647	* - When the lowest six bits are wrapping around to zero, a read that happens
1648	* at the same time will return garbage in the lowest ten bits.
1649	* - When the cycleOffset field wraps around to zero, the cycleCount field is
1650	* not incremented for about 60 ns.
1651	* - Occasionally, the entire register reads zero.
1652	*
1653	* To catch these, we read the register three times and ensure that the
1654	* difference between each two consecutive reads is approximately the same, i.e.
1655	* less than twice the other. Furthermore, any negative difference indicates an
1656	* error. (A PCI read should take at least 20 ticks of the 24.576 MHz timer to
1657	* execute, so we have enough precision to compute the ratio of the differences.)
1658	*/
1659	static u32 get_cycle_time(struct fw_ohci *ohci)
1660	{
1661	u32 c0, c1, c2;
1662	u32 t0, t1, t2;
1663	s32 diff01, diff12;
1664	int i;
1665
1666	if (has_reboot_by_cycle_timer_read_quirk(ohci))
1667	return 0;
1668
1669	c2 = reg_read(ohci, OHCI1394_IsochronousCycleTimer);
1670
1671	if (ohci->quirks & QUIRK_CYCLE_TIMER) {
1672	i = 0;
1673	c1 = c2;
1674	c2 = reg_read(ohci, OHCI1394_IsochronousCycleTimer);
1675	do {
1676	c0 = c1;
1677	c1 = c2;
1678	c2 = reg_read(ohci, OHCI1394_IsochronousCycleTimer);
1679	t0 = cycle_timer_ticks(cycle_timer: c0);
1680	t1 = cycle_timer_ticks(cycle_timer: c1);
1681	t2 = cycle_timer_ticks(cycle_timer: c2);
1682	diff01 = t1 - t0;
1683	diff12 = t2 - t1;
1684	} while ((diff01 <= 0 || diff12 <= 0 ||
1685	diff01 / diff12 >= 2 || diff12 / diff01 >= 2)
1686	&& i++ < 20);
1687	}
1688
1689	return c2;
1690	}
1691
1692	/*
1693	* This function has to be called at least every 64 seconds. The bus_time
1694	* field stores not only the upper 25 bits of the BUS_TIME register but also
1695	* the most significant bit of the cycle timer in bit 6 so that we can detect
1696	* changes in this bit.
1697	*/
1698	static u32 update_bus_time(struct fw_ohci *ohci)
1699	{
1700	u32 cycle_time_seconds = get_cycle_time(ohci) >> 25;
1701
1702	if (unlikely(!ohci->bus_time_running)) {
1703	reg_write(ohci, OHCI1394_IntMaskSet, OHCI1394_cycle64Seconds);
1704	ohci->bus_time = (lower_32_bits(ktime_get_seconds()) & ~0x7f) |
1705	(cycle_time_seconds & 0x40);
1706	ohci->bus_time_running = true;
1707	}
1708
1709	if ((ohci->bus_time & 0x40) != (cycle_time_seconds & 0x40))
1710	ohci->bus_time += 0x40;
1711
1712	return ohci->bus_time | cycle_time_seconds;
1713	}
1714
1715	static int get_status_for_port(struct fw_ohci *ohci, int port_index,
1716	enum phy_packet_self_id_port_status *status)
1717	{
1718	int reg;
1719
1720	scoped_guard(mutex, &ohci->phy_reg_mutex) {
1721	reg = write_phy_reg(ohci, addr: 7, val: port_index);
1722	if (reg < 0)
1723	return reg;
1724
1725	reg = read_phy_reg(ohci, addr: 8);
1726	if (reg < 0)
1727	return reg;
1728	}
1729
1730	switch (reg & 0x0f) {
1731	case 0x06:
1732	// is child node (connected to parent node)
1733	*status = PHY_PACKET_SELF_ID_PORT_STATUS_PARENT;
1734	break;
1735	case 0x0e:
1736	// is parent node (connected to child node)
1737	*status = PHY_PACKET_SELF_ID_PORT_STATUS_CHILD;
1738	break;
1739	default:
1740	// not connected
1741	*status = PHY_PACKET_SELF_ID_PORT_STATUS_NCONN;
1742	break;
1743	}
1744
1745	return 0;
1746	}
1747
1748	static int get_self_id_pos(struct fw_ohci *ohci, u32 self_id,
1749	int self_id_count)
1750	{
1751	unsigned int left_phy_id = phy_packet_self_id_get_phy_id(quadlet: self_id);
1752	int i;
1753
1754	for (i = 0; i < self_id_count; i++) {
1755	u32 entry = ohci->self_id_buffer[i];
1756	unsigned int right_phy_id = phy_packet_self_id_get_phy_id(quadlet: entry);
1757
1758	if (left_phy_id == right_phy_id)
1759	return -1;
1760	if (left_phy_id < right_phy_id)
1761	return i;
1762	}
1763	return i;
1764	}
1765
1766	static int detect_initiated_reset(struct fw_ohci *ohci, bool *is_initiated_reset)
1767	{
1768	int reg;
1769
1770	guard(mutex)(T: &ohci->phy_reg_mutex);
1771
1772	// Select page 7
1773	reg = write_phy_reg(ohci, addr: 7, val: 0xe0);
1774	if (reg < 0)
1775	return reg;
1776
1777	reg = read_phy_reg(ohci, addr: 8);
1778	if (reg < 0)
1779	return reg;
1780
1781	// set PMODE bit
1782	reg |= 0x40;
1783	reg = write_phy_reg(ohci, addr: 8, val: reg);
1784	if (reg < 0)
1785	return reg;
1786
1787	// read register 12
1788	reg = read_phy_reg(ohci, addr: 12);
1789	if (reg < 0)
1790	return reg;
1791
1792	// bit 3 indicates "initiated reset"
1793	*is_initiated_reset = !!((reg & 0x08) == 0x08);
1794
1795	return 0;
1796	}
1797
1798	/*
1799	* TI TSB82AA2B and TSB12LV26 do not receive the selfID of a locally
1800	* attached TSB41BA3D phy; see http://www.ti.com/litv/pdf/sllz059.
1801	* Construct the selfID from phy register contents.
1802	*/
1803	static int find_and_insert_self_id(struct fw_ohci *ohci, int self_id_count)
1804	{
1805	int reg, i, pos, err;
1806	bool is_initiated_reset;
1807	u32 self_id = 0;
1808
1809	// link active 1, speed 3, bridge 0, contender 1, more packets 0.
1810	phy_packet_set_packet_identifier(quadlet: &self_id, PHY_PACKET_PACKET_IDENTIFIER_SELF_ID);
1811	phy_packet_self_id_zero_set_link_active(quadlet: &self_id, is_active: true);
1812	phy_packet_self_id_zero_set_scode(quadlet: &self_id, SCODE_800);
1813	phy_packet_self_id_zero_set_contender(quadlet: &self_id, is_contender: true);
1814
1815	reg = reg_read(ohci, OHCI1394_NodeID);
1816	if (!(reg & OHCI1394_NodeID_idValid)) {
1817	ohci_notice(ohci,
1818	"node ID not valid, new bus reset in progress\n");
1819	return -EBUSY;
1820	}
1821	phy_packet_self_id_set_phy_id(quadlet: &self_id, phy_id: reg & 0x3f);
1822
1823	reg = ohci_read_phy_reg(card: &ohci->card, addr: 4);
1824	if (reg < 0)
1825	return reg;
1826	phy_packet_self_id_zero_set_power_class(quadlet: &self_id, power_class: reg & 0x07);
1827
1828	reg = ohci_read_phy_reg(card: &ohci->card, addr: 1);
1829	if (reg < 0)
1830	return reg;
1831	phy_packet_self_id_zero_set_gap_count(quadlet: &self_id, gap_count: reg & 0x3f);
1832
1833	for (i = 0; i < 3; i++) {
1834	enum phy_packet_self_id_port_status status;
1835
1836	err = get_status_for_port(ohci, port_index: i, status: &status);
1837	if (err < 0)
1838	return err;
1839
1840	self_id_sequence_set_port_status(self_id_sequence: &self_id, quadlet_count: 1, port_index: i, status);
1841	}
1842
1843	err = detect_initiated_reset(ohci, is_initiated_reset: &is_initiated_reset);
1844	if (err < 0)
1845	return err;
1846	phy_packet_self_id_zero_set_initiated_reset(quadlet: &self_id, is_initiated_reset);
1847
1848	pos = get_self_id_pos(ohci, self_id, self_id_count);
1849	if (pos >= 0) {
1850	memmove(&(ohci->self_id_buffer[pos+1]),
1851	&(ohci->self_id_buffer[pos]),
1852	(self_id_count - pos) * sizeof(*ohci->self_id_buffer));
1853	ohci->self_id_buffer[pos] = self_id;
1854	self_id_count++;
1855	}
1856	return self_id_count;
1857	}
1858
1859	static irqreturn_t handle_selfid_complete_event(int irq, void *data)
1860	{
1861	struct fw_ohci *ohci = data;
1862	int self_id_count, generation, new_generation, i, j;
1863	u32 reg, quadlet;
1864	void *free_rom = NULL;
1865	dma_addr_t free_rom_bus = 0;
1866	bool is_new_root;
1867
1868	reg = reg_read(ohci, OHCI1394_NodeID);
1869	if (!(reg & OHCI1394_NodeID_idValid)) {
1870	ohci_notice(ohci,
1871	"node ID not valid, new bus reset in progress\n");
1872	goto end;
1873	}
1874	if ((reg & OHCI1394_NodeID_nodeNumber) == 63) {
1875	ohci_notice(ohci, "malconfigured bus\n");
1876	goto end;
1877	}
1878	ohci->node_id = reg & (OHCI1394_NodeID_busNumber |
1879	OHCI1394_NodeID_nodeNumber);
1880
1881	is_new_root = (reg & OHCI1394_NodeID_root) != 0;
1882	if (!(ohci->is_root && is_new_root))
1883	reg_write(ohci, OHCI1394_LinkControlSet,
1884	OHCI1394_LinkControl_cycleMaster);
1885	ohci->is_root = is_new_root;
1886
1887	reg = reg_read(ohci, OHCI1394_SelfIDCount);
1888	if (ohci1394_self_id_count_is_error(value: reg)) {
1889	ohci_notice(ohci, "self ID receive error\n");
1890	goto end;
1891	}
1892
1893	trace_self_id_complete(card_index: ohci->card.index, reg, self_id_receive: ohci->self_id, has_be_header_quirk: has_be_header_quirk(ohci));
1894
1895	/*
1896	* The count in the SelfIDCount register is the number of
1897	* bytes in the self ID receive buffer. Since we also receive
1898	* the inverted quadlets and a header quadlet, we shift one
1899	* bit extra to get the actual number of self IDs.
1900	*/
1901	self_id_count = ohci1394_self_id_count_get_size(value: reg) >> 1;
1902
1903	if (self_id_count > 252) {
1904	ohci_notice(ohci, "bad selfIDSize (%08x)\n", reg);
1905	goto end;
1906	}
1907
1908	quadlet = cond_le32_to_cpu(value: ohci->self_id[0], has_be_header_quirk: has_be_header_quirk(ohci));
1909	generation = ohci1394_self_id_receive_q0_get_generation(quadlet0: quadlet);
1910	rmb();
1911
1912	for (i = 1, j = 0; j < self_id_count; i += 2, j++) {
1913	u32 id = cond_le32_to_cpu(value: ohci->self_id[i], has_be_header_quirk: has_be_header_quirk(ohci));
1914	u32 id2 = cond_le32_to_cpu(value: ohci->self_id[i + 1], has_be_header_quirk: has_be_header_quirk(ohci));
1915
1916	if (id != ~id2) {
1917	/*
1918	* If the invalid data looks like a cycle start packet,
1919	* it's likely to be the result of the cycle master
1920	* having a wrong gap count. In this case, the self IDs
1921	* so far are valid and should be processed so that the
1922	* bus manager can then correct the gap count.
1923	*/
1924	if (id == 0xffff008f) {
1925	ohci_notice(ohci, "ignoring spurious self IDs\n");
1926	self_id_count = j;
1927	break;
1928	}
1929
1930	ohci_notice(ohci, "bad self ID %d/%d (%08x != ~%08x)\n",
1931	j, self_id_count, id, id2);
1932	goto end;
1933	}
1934	ohci->self_id_buffer[j] = id;
1935	}
1936
1937	if (ohci->quirks & QUIRK_TI_SLLZ059) {
1938	self_id_count = find_and_insert_self_id(ohci, self_id_count);
1939	if (self_id_count < 0) {
1940	ohci_notice(ohci,
1941	"could not construct local self ID\n");
1942	goto end;
1943	}
1944	}
1945
1946	if (self_id_count == 0) {
1947	ohci_notice(ohci, "no self IDs\n");
1948	goto end;
1949	}
1950	rmb();
1951
1952	/*
1953	* Check the consistency of the self IDs we just read. The
1954	* problem we face is that a new bus reset can start while we
1955	* read out the self IDs from the DMA buffer. If this happens,
1956	* the DMA buffer will be overwritten with new self IDs and we
1957	* will read out inconsistent data. The OHCI specification
1958	* (section 11.2) recommends a technique similar to
1959	* linux/seqlock.h, where we remember the generation of the
1960	* self IDs in the buffer before reading them out and compare
1961	* it to the current generation after reading them out. If
1962	* the two generations match we know we have a consistent set
1963	* of self IDs.
1964	*/
1965
1966	reg = reg_read(ohci, OHCI1394_SelfIDCount);
1967	new_generation = ohci1394_self_id_count_get_generation(value: reg);
1968	if (new_generation != generation) {
1969	ohci_notice(ohci, "new bus reset, discarding self ids\n");
1970	goto end;
1971	}
1972
1973	// FIXME: Document how the locking works.
1974	scoped_guard(spinlock_irq, &ohci->lock) {
1975	ohci->generation = -1; // prevent AT packet queueing
1976	context_stop(ctx: &ohci->at_request_ctx.context);
1977	context_stop(ctx: &ohci->at_response_ctx.context);
1978	}
1979
1980	/*
1981	* Per OHCI 1.2 draft, clause 7.2.3.3, hardware may leave unsent
1982	* packets in the AT queues and software needs to drain them.
1983	* Some OHCI 1.1 controllers (JMicron) apparently require this too.
1984	*/
1985	at_context_flush(ctx: &ohci->at_request_ctx);
1986	at_context_flush(ctx: &ohci->at_response_ctx);
1987
1988	scoped_guard(spinlock_irq, &ohci->lock) {
1989	ohci->generation = generation;
1990	reg_write(ohci, OHCI1394_IntEventClear, OHCI1394_busReset);
1991	reg_write(ohci, OHCI1394_IntMaskSet, OHCI1394_busReset);
1992
1993	if (ohci->quirks & QUIRK_RESET_PACKET)
1994	ohci->request_generation = generation;
1995
1996	// This next bit is unrelated to the AT context stuff but we have to do it under the
1997	// spinlock also. If a new config rom was set up before this reset, the old one is
1998	// now no longer in use and we can free it. Update the config rom pointers to point
1999	// to the current config rom and clear the next_config_rom pointer so a new update
2000	// can take place.
2001	if (ohci->next_config_rom != NULL) {
2002	if (ohci->next_config_rom != ohci->config_rom) {
2003	free_rom = ohci->config_rom;
2004	free_rom_bus = ohci->config_rom_bus;
2005	}
2006	ohci->config_rom = ohci->next_config_rom;
2007	ohci->config_rom_bus = ohci->next_config_rom_bus;
2008	ohci->next_config_rom = NULL;
2009
2010	// Restore config_rom image and manually update config_rom registers.
2011	// Writing the header quadlet will indicate that the config rom is ready,
2012	// so we do that last.
2013	reg_write(ohci, OHCI1394_BusOptions, be32_to_cpu(ohci->config_rom[2]));
2014	ohci->config_rom[0] = ohci->next_header;
2015	reg_write(ohci, OHCI1394_ConfigROMhdr, be32_to_cpu(ohci->next_header));
2016	}
2017
2018	if (param_remote_dma) {
2019	reg_write(ohci, OHCI1394_PhyReqFilterHiSet, data: ~0);
2020	reg_write(ohci, OHCI1394_PhyReqFilterLoSet, data: ~0);
2021	}
2022	}
2023
2024	if (free_rom)
2025	dmam_free_coherent(dev: ohci->card.device, CONFIG_ROM_SIZE, vaddr: free_rom, dma_handle: free_rom_bus);
2026
2027	fw_core_handle_bus_reset(card: &ohci->card, node_id: ohci->node_id, generation,
2028	self_id_count, self_ids: ohci->self_id_buffer,
2029	bm_abdicate: ohci->csr_state_setclear_abdicate);
2030	ohci->csr_state_setclear_abdicate = false;
2031	end:
2032	return IRQ_HANDLED;
2033	}
2034
2035	static irqreturn_t irq_handler(int irq, void *data)
2036	{
2037	struct fw_ohci *ohci = data;
2038	u32 event, iso_event;
2039	int i;
2040
2041	event = reg_read(ohci, OHCI1394_IntEventClear);
2042
2043	if (!event || !~event)
2044	return IRQ_NONE;
2045
2046	/*
2047	* busReset and postedWriteErr events must not be cleared yet
2048	* (OHCI 1.1 clauses 7.2.3.2 and 13.2.8.1)
2049	*/
2050	reg_write(ohci, OHCI1394_IntEventClear,
2051	data: event & ~(OHCI1394_busReset | OHCI1394_postedWriteErr));
2052	trace_irqs(card_index: ohci->card.index, events: event);
2053
2054	// The flag is masked again at handle_selfid_complete_event() scheduled by selfID event.
2055	if (event & OHCI1394_busReset)
2056	reg_write(ohci, OHCI1394_IntMaskClear, OHCI1394_busReset);
2057
2058	if (event & OHCI1394_RQPkt)
2059	queue_work(wq: ohci->card.async_wq, work: &ohci->ar_request_ctx.work);
2060
2061	if (event & OHCI1394_RSPkt)
2062	queue_work(wq: ohci->card.async_wq, work: &ohci->ar_response_ctx.work);
2063
2064	if (event & OHCI1394_reqTxComplete)
2065	queue_work(wq: ohci->card.async_wq, work: &ohci->at_request_ctx.work);
2066
2067	if (event & OHCI1394_respTxComplete)
2068	queue_work(wq: ohci->card.async_wq, work: &ohci->at_response_ctx.work);
2069
2070	if (event & OHCI1394_isochRx) {
2071	iso_event = reg_read(ohci, OHCI1394_IsoRecvIntEventClear);
2072	reg_write(ohci, OHCI1394_IsoRecvIntEventClear, data: iso_event);
2073
2074	while (iso_event) {
2075	i = ffs(iso_event) - 1;
2076	fw_iso_context_schedule_flush_completions(ctx: &ohci->ir_context_list[i].base);
2077	iso_event &= ~(1 << i);
2078	}
2079	}
2080
2081	if (event & OHCI1394_isochTx) {
2082	iso_event = reg_read(ohci, OHCI1394_IsoXmitIntEventClear);
2083	reg_write(ohci, OHCI1394_IsoXmitIntEventClear, data: iso_event);
2084
2085	while (iso_event) {
2086	i = ffs(iso_event) - 1;
2087	fw_iso_context_schedule_flush_completions(ctx: &ohci->it_context_list[i].base);
2088	iso_event &= ~(1 << i);
2089	}
2090	}
2091
2092	if (unlikely(event & OHCI1394_regAccessFail))
2093	ohci_err(ohci, "register access failure\n");
2094
2095	if (unlikely(event & OHCI1394_postedWriteErr)) {
2096	reg_read(ohci, OHCI1394_PostedWriteAddressHi);
2097	reg_read(ohci, OHCI1394_PostedWriteAddressLo);
2098	reg_write(ohci, OHCI1394_IntEventClear,
2099	OHCI1394_postedWriteErr);
2100	dev_err_ratelimited(ohci->card.device, "PCI posted write error\n");
2101	}
2102
2103	if (unlikely(event & OHCI1394_cycleTooLong)) {
2104	dev_notice_ratelimited(ohci->card.device, "isochronous cycle too long\n");
2105	reg_write(ohci, OHCI1394_LinkControlSet,
2106	OHCI1394_LinkControl_cycleMaster);
2107	}
2108
2109	if (unlikely(event & OHCI1394_cycleInconsistent)) {
2110	/*
2111	* We need to clear this event bit in order to make
2112	* cycleMatch isochronous I/O work. In theory we should
2113	* stop active cycleMatch iso contexts now and restart
2114	* them at least two cycles later. (FIXME?)
2115	*/
2116	dev_notice_ratelimited(ohci->card.device, "isochronous cycle inconsistent\n");
2117	}
2118
2119	if (unlikely(event & OHCI1394_unrecoverableError))
2120	handle_dead_contexts(ohci);
2121
2122	if (event & OHCI1394_cycle64Seconds) {
2123	guard(spinlock)(l: &ohci->lock);
2124	update_bus_time(ohci);
2125	} else
2126	flush_writes(ohci);
2127
2128	if (event & OHCI1394_selfIDComplete)
2129	return IRQ_WAKE_THREAD;
2130	else
2131	return IRQ_HANDLED;
2132	}
2133
2134	static int software_reset(struct fw_ohci *ohci)
2135	{
2136	u32 val;
2137	int i;
2138
2139	reg_write(ohci, OHCI1394_HCControlSet, OHCI1394_HCControl_softReset);
2140	for (i = 0; i < 500; i++) {
2141	val = reg_read(ohci, OHCI1394_HCControlSet);
2142	if (!~val)
2143	return -ENODEV; /* Card was ejected. */
2144
2145	if (!(val & OHCI1394_HCControl_softReset))
2146	return 0;
2147
2148	msleep(msecs: 1);
2149	}
2150
2151	return -EBUSY;
2152	}
2153
2154	static void copy_config_rom(__be32 *dest, const __be32 *src, size_t length)
2155	{
2156	size_t size = length * 4;
2157
2158	memcpy(dest, src, size);
2159	if (size < CONFIG_ROM_SIZE)
2160	memset(&dest[length], 0, CONFIG_ROM_SIZE - size);
2161	}
2162
2163	static int configure_1394a_enhancements(struct fw_ohci *ohci)
2164	{
2165	bool enable_1394a;
2166	int ret, clear, set, offset;
2167
2168	/* Check if the driver should configure link and PHY. */
2169	if (!(reg_read(ohci, OHCI1394_HCControlSet) &
2170	OHCI1394_HCControl_programPhyEnable))
2171	return 0;
2172
2173	/* Paranoia: check whether the PHY supports 1394a, too. */
2174	enable_1394a = false;
2175	ret = read_phy_reg(ohci, addr: 2);
2176	if (ret < 0)
2177	return ret;
2178	if ((ret & PHY_EXTENDED_REGISTERS) == PHY_EXTENDED_REGISTERS) {
2179	ret = read_paged_phy_reg(ohci, page: 1, addr: 8);
2180	if (ret < 0)
2181	return ret;
2182	if (ret >= 1)
2183	enable_1394a = true;
2184	}
2185
2186	if (ohci->quirks & QUIRK_NO_1394A)
2187	enable_1394a = false;
2188
2189	/* Configure PHY and link consistently. */
2190	if (enable_1394a) {
2191	clear = 0;
2192	set = PHY_ENABLE_ACCEL | PHY_ENABLE_MULTI;
2193	} else {
2194	clear = PHY_ENABLE_ACCEL | PHY_ENABLE_MULTI;
2195	set = 0;
2196	}
2197	ret = update_phy_reg(ohci, addr: 5, clear_bits: clear, set_bits: set);
2198	if (ret < 0)
2199	return ret;
2200
2201	if (enable_1394a)
2202	offset = OHCI1394_HCControlSet;
2203	else
2204	offset = OHCI1394_HCControlClear;
2205	reg_write(ohci, offset, OHCI1394_HCControl_aPhyEnhanceEnable);
2206
2207	/* Clean up: configuration has been taken care of. */
2208	reg_write(ohci, OHCI1394_HCControlClear,
2209	OHCI1394_HCControl_programPhyEnable);
2210
2211	return 0;
2212	}
2213
2214	static int probe_tsb41ba3d(struct fw_ohci *ohci)
2215	{
2216	/* TI vendor ID = 0x080028, TSB41BA3D product ID = 0x833005 (sic) */
2217	static const u8 id[] = { 0x08, 0x00, 0x28, 0x83, 0x30, 0x05, };
2218	int reg, i;
2219
2220	reg = read_phy_reg(ohci, addr: 2);
2221	if (reg < 0)
2222	return reg;
2223	if ((reg & PHY_EXTENDED_REGISTERS) != PHY_EXTENDED_REGISTERS)
2224	return 0;
2225
2226	for (i = ARRAY_SIZE(id) - 1; i >= 0; i--) {
2227	reg = read_paged_phy_reg(ohci, page: 1, addr: i + 10);
2228	if (reg < 0)
2229	return reg;
2230	if (reg != id[i])
2231	return 0;
2232	}
2233	return 1;
2234	}
2235
2236	static int ohci_enable(struct fw_card *card,
2237	const __be32 *config_rom, size_t length)
2238	{
2239	struct fw_ohci *ohci = fw_ohci(card);
2240	u32 lps, version, irqs;
2241	int i, ret;
2242
2243	ret = software_reset(ohci);
2244	if (ret < 0) {
2245	ohci_err(ohci, "failed to reset ohci card\n");
2246	return ret;
2247	}
2248
2249	/*
2250	* Now enable LPS, which we need in order to start accessing
2251	* most of the registers. In fact, on some cards (ALI M5251),
2252	* accessing registers in the SClk domain without LPS enabled
2253	* will lock up the machine. Wait 50msec to make sure we have
2254	* full link enabled. However, with some cards (well, at least
2255	* a JMicron PCIe card), we have to try again sometimes.
2256	*
2257	* TI TSB82AA2 + TSB81BA3(A) cards signal LPS enabled early but
2258	* cannot actually use the phy at that time. These need tens of
2259	* millisecods pause between LPS write and first phy access too.
2260	*/
2261
2262	reg_write(ohci, OHCI1394_HCControlSet,
2263	OHCI1394_HCControl_LPS |
2264	OHCI1394_HCControl_postedWriteEnable);
2265	flush_writes(ohci);
2266
2267	for (lps = 0, i = 0; !lps && i < 3; i++) {
2268	msleep(msecs: 50);
2269	lps = reg_read(ohci, OHCI1394_HCControlSet) &
2270	OHCI1394_HCControl_LPS;
2271	}
2272
2273	if (!lps) {
2274	ohci_err(ohci, "failed to set Link Power Status\n");
2275	return -EIO;
2276	}
2277
2278	if (ohci->quirks & QUIRK_TI_SLLZ059) {
2279	ret = probe_tsb41ba3d(ohci);
2280	if (ret < 0)
2281	return ret;
2282	if (ret)
2283	ohci_notice(ohci, "local TSB41BA3D phy\n");
2284	else
2285	ohci->quirks &= ~QUIRK_TI_SLLZ059;
2286	}
2287
2288	reg_write(ohci, OHCI1394_HCControlClear,
2289	OHCI1394_HCControl_noByteSwapData);
2290
2291	reg_write(ohci, OHCI1394_SelfIDBuffer, data: ohci->self_id_bus);
2292	reg_write(ohci, OHCI1394_LinkControlSet,
2293	OHCI1394_LinkControl_cycleTimerEnable |
2294	OHCI1394_LinkControl_cycleMaster);
2295
2296	reg_write(ohci, OHCI1394_ATRetries,
2297	OHCI1394_MAX_AT_REQ_RETRIES |
2298	(OHCI1394_MAX_AT_RESP_RETRIES << 4) |
2299	(OHCI1394_MAX_PHYS_RESP_RETRIES << 8) |
2300	(200 << 16));
2301
2302	ohci->bus_time_running = false;
2303
2304	for (i = 0; i < 32; i++)
2305	if (ohci->ir_context_support & (1 << i))
2306	reg_write(ohci, OHCI1394_IsoRcvContextControlClear(i),
2307	IR_CONTEXT_MULTI_CHANNEL_MODE);
2308
2309	version = reg_read(ohci, OHCI1394_Version) & 0x00ff00ff;
2310	if (version >= OHCI_VERSION_1_1) {
2311	reg_write(ohci, OHCI1394_InitialChannelsAvailableHi,
2312	data: 0xfffffffe);
2313	card->broadcast_channel_auto_allocated = true;
2314	}
2315
2316	/* Get implemented bits of the priority arbitration request counter. */
2317	reg_write(ohci, OHCI1394_FairnessControl, data: 0x3f);
2318	ohci->pri_req_max = reg_read(ohci, OHCI1394_FairnessControl) & 0x3f;
2319	reg_write(ohci, OHCI1394_FairnessControl, data: 0);
2320	card->priority_budget_implemented = ohci->pri_req_max != 0;
2321
2322	reg_write(ohci, OHCI1394_PhyUpperBound, FW_MAX_PHYSICAL_RANGE >> 16);
2323	reg_write(ohci, OHCI1394_IntEventClear, data: ~0);
2324	reg_write(ohci, OHCI1394_IntMaskClear, data: ~0);
2325
2326	ret = configure_1394a_enhancements(ohci);
2327	if (ret < 0)
2328	return ret;
2329
2330	/* Activate link_on bit and contender bit in our self ID packets.*/
2331	ret = ohci_update_phy_reg(card, addr: 4, clear_bits: 0, PHY_LINK_ACTIVE | PHY_CONTENDER);
2332	if (ret < 0)
2333	return ret;
2334
2335	/*
2336	* When the link is not yet enabled, the atomic config rom
2337	* update mechanism described below in ohci_set_config_rom()
2338	* is not active. We have to update ConfigRomHeader and
2339	* BusOptions manually, and the write to ConfigROMmap takes
2340	* effect immediately. We tie this to the enabling of the
2341	* link, so we have a valid config rom before enabling - the
2342	* OHCI requires that ConfigROMhdr and BusOptions have valid
2343	* values before enabling.
2344	*
2345	* However, when the ConfigROMmap is written, some controllers
2346	* always read back quadlets 0 and 2 from the config rom to
2347	* the ConfigRomHeader and BusOptions registers on bus reset.
2348	* They shouldn't do that in this initial case where the link
2349	* isn't enabled. This means we have to use the same
2350	* workaround here, setting the bus header to 0 and then write
2351	* the right values in the bus reset work item.
2352	*/
2353
2354	if (config_rom) {
2355	ohci->next_config_rom = dmam_alloc_coherent(dev: ohci->card.device, CONFIG_ROM_SIZE,
2356	dma_handle: &ohci->next_config_rom_bus, GFP_KERNEL);
2357	if (ohci->next_config_rom == NULL)
2358	return -ENOMEM;
2359
2360	copy_config_rom(dest: ohci->next_config_rom, src: config_rom, length);
2361	} else {
2362	/*
2363	* In the suspend case, config_rom is NULL, which
2364	* means that we just reuse the old config rom.
2365	*/
2366	ohci->next_config_rom = ohci->config_rom;
2367	ohci->next_config_rom_bus = ohci->config_rom_bus;
2368	}
2369
2370	ohci->next_header = ohci->next_config_rom[0];
2371	ohci->next_config_rom[0] = 0;
2372	reg_write(ohci, OHCI1394_ConfigROMhdr, data: 0);
2373	reg_write(ohci, OHCI1394_BusOptions,
2374	be32_to_cpu(ohci->next_config_rom[2]));
2375	reg_write(ohci, OHCI1394_ConfigROMmap, data: ohci->next_config_rom_bus);
2376
2377	reg_write(ohci, OHCI1394_AsReqFilterHiSet, data: 0x80000000);
2378
2379	irqs = OHCI1394_reqTxComplete | OHCI1394_respTxComplete |
2380	OHCI1394_RQPkt | OHCI1394_RSPkt |
2381	OHCI1394_isochTx | OHCI1394_isochRx |
2382	OHCI1394_postedWriteErr |
2383	OHCI1394_selfIDComplete |
2384	OHCI1394_regAccessFail |
2385	OHCI1394_cycleInconsistent |
2386	OHCI1394_unrecoverableError |
2387	OHCI1394_cycleTooLong |
2388	OHCI1394_masterIntEnable |
2389	OHCI1394_busReset;
2390	reg_write(ohci, OHCI1394_IntMaskSet, data: irqs);
2391
2392	reg_write(ohci, OHCI1394_HCControlSet,
2393	OHCI1394_HCControl_linkEnable |
2394	OHCI1394_HCControl_BIBimageValid);
2395
2396	reg_write(ohci, OHCI1394_LinkControlSet,
2397	OHCI1394_LinkControl_rcvSelfID |
2398	OHCI1394_LinkControl_rcvPhyPkt);
2399
2400	ar_context_run(ctx: &ohci->ar_request_ctx);
2401	ar_context_run(ctx: &ohci->ar_response_ctx);
2402
2403	flush_writes(ohci);
2404
2405	/* We are ready to go, reset bus to finish initialization. */
2406	fw_schedule_bus_reset(card: &ohci->card, delayed: false, short_reset: true);
2407
2408	return 0;
2409	}
2410
2411	static void ohci_disable(struct fw_card *card)
2412	{
2413	struct pci_dev *pdev = to_pci_dev(card->device);
2414	struct fw_ohci *ohci = pci_get_drvdata(pdev);
2415	int i, irq = pci_irq_vector(dev: pdev, nr: 0);
2416
2417	// If the removal is happening from the suspend state, LPS won't be enabled and host
2418	// registers (eg., IntMaskClear) won't be accessible.
2419	if (!(reg_read(ohci, OHCI1394_HCControlSet) & OHCI1394_HCControl_LPS))
2420	return;
2421
2422	reg_write(ohci, OHCI1394_IntMaskClear, data: ~0);
2423	flush_writes(ohci);
2424
2425	if (irq >= 0)
2426	synchronize_irq(irq);
2427
2428	flush_work(work: &ohci->ar_request_ctx.work);
2429	flush_work(work: &ohci->ar_response_ctx.work);
2430	flush_work(work: &ohci->at_request_ctx.work);
2431	flush_work(work: &ohci->at_response_ctx.work);
2432
2433	for (i = 0; i < ohci->n_ir; ++i) {
2434	if (!(ohci->ir_context_mask & BIT(i)))
2435	flush_work(work: &ohci->ir_context_list[i].base.work);
2436	}
2437	for (i = 0; i < ohci->n_it; ++i) {
2438	if (!(ohci->it_context_mask & BIT(i)))
2439	flush_work(work: &ohci->it_context_list[i].base.work);
2440	}
2441
2442	at_context_flush(ctx: &ohci->at_request_ctx);
2443	at_context_flush(ctx: &ohci->at_response_ctx);
2444	}
2445
2446	static int ohci_set_config_rom(struct fw_card *card,
2447	const __be32 *config_rom, size_t length)
2448	{
2449	struct fw_ohci *ohci;
2450	__be32 *next_config_rom;
2451	dma_addr_t next_config_rom_bus;
2452
2453	ohci = fw_ohci(card);
2454
2455	/*
2456	* When the OHCI controller is enabled, the config rom update
2457	* mechanism is a bit tricky, but easy enough to use. See
2458	* section 5.5.6 in the OHCI specification.
2459	*
2460	* The OHCI controller caches the new config rom address in a
2461	* shadow register (ConfigROMmapNext) and needs a bus reset
2462	* for the changes to take place. When the bus reset is
2463	* detected, the controller loads the new values for the
2464	* ConfigRomHeader and BusOptions registers from the specified
2465	* config rom and loads ConfigROMmap from the ConfigROMmapNext
2466	* shadow register. All automatically and atomically.
2467	*
2468	* Now, there's a twist to this story. The automatic load of
2469	* ConfigRomHeader and BusOptions doesn't honor the
2470	* noByteSwapData bit, so with a be32 config rom, the
2471	* controller will load be32 values in to these registers
2472	* during the atomic update, even on little endian
2473	* architectures. The workaround we use is to put a 0 in the
2474	* header quadlet; 0 is endian agnostic and means that the
2475	* config rom isn't ready yet. In the bus reset work item we
2476	* then set up the real values for the two registers.
2477	*
2478	* We use ohci->lock to avoid racing with the code that sets
2479	* ohci->next_config_rom to NULL (see handle_selfid_complete_event).
2480	*/
2481
2482	next_config_rom = dmam_alloc_coherent(dev: ohci->card.device, CONFIG_ROM_SIZE,
2483	dma_handle: &next_config_rom_bus, GFP_KERNEL);
2484	if (next_config_rom == NULL)
2485	return -ENOMEM;
2486
2487	scoped_guard(spinlock_irq, &ohci->lock) {
2488	// If there is not an already pending config_rom update, push our new allocation
2489	// into the ohci->next_config_rom and then mark the local variable as null so that
2490	// we won't deallocate the new buffer.
2491	//
2492	// OTOH, if there is a pending config_rom update, just use that buffer with the new
2493	// config_rom data, and let this routine free the unused DMA allocation.
2494	if (ohci->next_config_rom == NULL) {
2495	ohci->next_config_rom = next_config_rom;
2496	ohci->next_config_rom_bus = next_config_rom_bus;
2497	next_config_rom = NULL;
2498	}
2499
2500	copy_config_rom(dest: ohci->next_config_rom, src: config_rom, length);
2501
2502	ohci->next_header = config_rom[0];
2503	ohci->next_config_rom[0] = 0;
2504
2505	reg_write(ohci, OHCI1394_ConfigROMmap, data: ohci->next_config_rom_bus);
2506	}
2507
2508	/* If we didn't use the DMA allocation, delete it. */
2509	if (next_config_rom != NULL) {
2510	dmam_free_coherent(dev: ohci->card.device, CONFIG_ROM_SIZE, vaddr: next_config_rom,
2511	dma_handle: next_config_rom_bus);
2512	}
2513
2514	/*
2515	* Now initiate a bus reset to have the changes take
2516	* effect. We clean up the old config rom memory and DMA
2517	* mappings in the bus reset work item, since the OHCI
2518	* controller could need to access it before the bus reset
2519	* takes effect.
2520	*/
2521
2522	fw_schedule_bus_reset(card: &ohci->card, delayed: true, short_reset: true);
2523
2524	return 0;
2525	}
2526
2527	static void ohci_send_request(struct fw_card *card, struct fw_packet *packet)
2528	{
2529	struct fw_ohci *ohci = fw_ohci(card);
2530
2531	at_context_transmit(ctx: &ohci->at_request_ctx, packet);
2532	}
2533
2534	static void ohci_send_response(struct fw_card *card, struct fw_packet *packet)
2535	{
2536	struct fw_ohci *ohci = fw_ohci(card);
2537
2538	at_context_transmit(ctx: &ohci->at_response_ctx, packet);
2539	}
2540
2541	static int ohci_cancel_packet(struct fw_card *card, struct fw_packet *packet)
2542	{
2543	struct fw_ohci *ohci = fw_ohci(card);
2544	struct at_context *ctx = &ohci->at_request_ctx;
2545	struct driver_data *driver_data = packet->driver_data;
2546	int ret = -ENOENT;
2547
2548	// Avoid dead lock due to programming mistake.
2549	if (WARN_ON_ONCE(current_work() == &ctx->work))
2550	return 0;
2551	disable_work_sync(work: &ctx->work);
2552
2553	if (packet->ack != 0)
2554	goto out;
2555
2556	if (packet->payload_mapped)
2557	dma_unmap_single(ohci->card.device, packet->payload_bus,
2558	packet->payload_length, DMA_TO_DEVICE);
2559
2560	driver_data->packet = NULL;
2561	packet->ack = RCODE_CANCELLED;
2562
2563	// Timestamping on behalf of the hardware.
2564	packet->timestamp = cycle_time_to_ohci_tstamp(tstamp: get_cycle_time(ohci));
2565
2566	packet->callback(packet, &ohci->card, packet->ack);
2567	ret = 0;
2568	out:
2569	enable_work(work: &ctx->work);
2570
2571	return ret;
2572	}
2573
2574	static int ohci_enable_phys_dma(struct fw_card *card,
2575	int node_id, int generation)
2576	{
2577	struct fw_ohci *ohci = fw_ohci(card);
2578	int n, ret = 0;
2579
2580	if (param_remote_dma)
2581	return 0;
2582
2583	/*
2584	* FIXME: Make sure this bitmask is cleared when we clear the busReset
2585	* interrupt bit. Clear physReqResourceAllBuses on bus reset.
2586	*/
2587
2588	guard(spinlock_irqsave)(l: &ohci->lock);
2589
2590	if (ohci->generation != generation)
2591	return -ESTALE;
2592
2593	/*
2594	* Note, if the node ID contains a non-local bus ID, physical DMA is
2595	* enabled for _all_ nodes on remote buses.
2596	*/
2597
2598	n = (node_id & 0xffc0) == LOCAL_BUS ? node_id & 0x3f : 63;
2599	if (n < 32)
2600	reg_write(ohci, OHCI1394_PhyReqFilterLoSet, data: 1 << n);
2601	else
2602	reg_write(ohci, OHCI1394_PhyReqFilterHiSet, data: 1 << (n - 32));
2603
2604	flush_writes(ohci);
2605
2606	return ret;
2607	}
2608
2609	static u32 ohci_read_csr(struct fw_card *card, int csr_offset)
2610	{
2611	struct fw_ohci *ohci = fw_ohci(card);
2612	u32 value;
2613
2614	switch (csr_offset) {
2615	case CSR_STATE_CLEAR:
2616	case CSR_STATE_SET:
2617	if (ohci->is_root &&
2618	(reg_read(ohci, OHCI1394_LinkControlSet) &
2619	OHCI1394_LinkControl_cycleMaster))
2620	value = CSR_STATE_BIT_CMSTR;
2621	else
2622	value = 0;
2623	if (ohci->csr_state_setclear_abdicate)
2624	value |= CSR_STATE_BIT_ABDICATE;
2625
2626	return value;
2627
2628	case CSR_NODE_IDS:
2629	return reg_read(ohci, OHCI1394_NodeID) << 16;
2630
2631	case CSR_CYCLE_TIME:
2632	return get_cycle_time(ohci);
2633
2634	case CSR_BUS_TIME:
2635	{
2636	// We might be called just after the cycle timer has wrapped around but just before
2637	// the cycle64Seconds handler, so we better check here, too, if the bus time needs
2638	// to be updated.
2639
2640	guard(spinlock_irqsave)(l: &ohci->lock);
2641	return update_bus_time(ohci);
2642	}
2643	case CSR_BUSY_TIMEOUT:
2644	value = reg_read(ohci, OHCI1394_ATRetries);
2645	return (value >> 4) & 0x0ffff00f;
2646
2647	case CSR_PRIORITY_BUDGET:
2648	return (reg_read(ohci, OHCI1394_FairnessControl) & 0x3f) |
2649	(ohci->pri_req_max << 8);
2650
2651	default:
2652	WARN_ON(1);
2653	return 0;
2654	}
2655	}
2656
2657	static void ohci_write_csr(struct fw_card *card, int csr_offset, u32 value)
2658	{
2659	struct fw_ohci *ohci = fw_ohci(card);
2660
2661	switch (csr_offset) {
2662	case CSR_STATE_CLEAR:
2663	if ((value & CSR_STATE_BIT_CMSTR) && ohci->is_root) {
2664	reg_write(ohci, OHCI1394_LinkControlClear,
2665	OHCI1394_LinkControl_cycleMaster);
2666	flush_writes(ohci);
2667	}
2668	if (value & CSR_STATE_BIT_ABDICATE)
2669	ohci->csr_state_setclear_abdicate = false;
2670	break;
2671
2672	case CSR_STATE_SET:
2673	if ((value & CSR_STATE_BIT_CMSTR) && ohci->is_root) {
2674	reg_write(ohci, OHCI1394_LinkControlSet,
2675	OHCI1394_LinkControl_cycleMaster);
2676	flush_writes(ohci);
2677	}
2678	if (value & CSR_STATE_BIT_ABDICATE)
2679	ohci->csr_state_setclear_abdicate = true;
2680	break;
2681
2682	case CSR_NODE_IDS:
2683	reg_write(ohci, OHCI1394_NodeID, data: value >> 16);
2684	flush_writes(ohci);
2685	break;
2686
2687	case CSR_CYCLE_TIME:
2688	reg_write(ohci, OHCI1394_IsochronousCycleTimer, data: value);
2689	reg_write(ohci, OHCI1394_IntEventSet,
2690	OHCI1394_cycleInconsistent);
2691	flush_writes(ohci);
2692	break;
2693
2694	case CSR_BUS_TIME:
2695	{
2696	guard(spinlock_irqsave)(l: &ohci->lock);
2697	ohci->bus_time = (update_bus_time(ohci) & 0x40) | (value & ~0x7f);
2698	break;
2699	}
2700	case CSR_BUSY_TIMEOUT:
2701	value = (value & 0xf) | ((value & 0xf) << 4) |
2702	((value & 0xf) << 8) | ((value & 0x0ffff000) << 4);
2703	reg_write(ohci, OHCI1394_ATRetries, data: value);
2704	flush_writes(ohci);
2705	break;
2706
2707	case CSR_PRIORITY_BUDGET:
2708	reg_write(ohci, OHCI1394_FairnessControl, data: value & 0x3f);
2709	flush_writes(ohci);
2710	break;
2711
2712	default:
2713	WARN_ON(1);
2714	break;
2715	}
2716	}
2717
2718	static void flush_iso_completions(struct iso_context *ctx, enum fw_iso_context_completions_cause cause)
2719	{
2720	trace_isoc_inbound_single_completions(ctx: &ctx->base, timestamp: ctx->last_timestamp, cause, header: ctx->header,
2721	header_length: ctx->header_length);
2722	trace_isoc_outbound_completions(ctx: &ctx->base, timestamp: ctx->last_timestamp, cause, header: ctx->header,
2723	header_length: ctx->header_length);
2724
2725	ctx->base.callback.sc(&ctx->base, ctx->last_timestamp,
2726	ctx->header_length, ctx->header,
2727	ctx->base.callback_data);
2728	ctx->header_length = 0;
2729	}
2730
2731	static void copy_iso_headers(struct iso_context *ctx, const u32 *dma_hdr)
2732	{
2733	u32 *ctx_hdr;
2734
2735	if (ctx->header_length + ctx->base.header_size > PAGE_SIZE) {
2736	if (ctx->base.drop_overflow_headers)
2737	return;
2738	flush_iso_completions(ctx, cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_HEADER_OVERFLOW);
2739	}
2740
2741	ctx_hdr = ctx->header + ctx->header_length;
2742	ctx->last_timestamp = (u16)le32_to_cpu((__force __le32)dma_hdr[0]);
2743
2744	/*
2745	* The two iso header quadlets are byteswapped to little
2746	* endian by the controller, but we want to present them
2747	* as big endian for consistency with the bus endianness.
2748	*/
2749	if (ctx->base.header_size > 0)
2750	ctx_hdr[0] = swab32(dma_hdr[1]); /* iso packet header */
2751	if (ctx->base.header_size > 4)
2752	ctx_hdr[1] = swab32(dma_hdr[0]); /* timestamp */
2753	if (ctx->base.header_size > 8)
2754	memcpy(&ctx_hdr[2], &dma_hdr[2], ctx->base.header_size - 8);
2755	ctx->header_length += ctx->base.header_size;
2756	}
2757
2758	static int handle_ir_packet_per_buffer(struct context *context,
2759	struct descriptor *d,
2760	struct descriptor *last)
2761	{
2762	struct iso_context *ctx =
2763	container_of(context, struct iso_context, context);
2764	struct descriptor *pd;
2765	u32 buffer_dma;
2766
2767	for (pd = d; pd <= last; pd++)
2768	if (pd->transfer_status)
2769	break;
2770	if (pd > last)
2771	/* Descriptor(s) not done yet, stop iteration */
2772	return 0;
2773
2774	while (!(d->control & cpu_to_le16(DESCRIPTOR_BRANCH_ALWAYS))) {
2775	d++;
2776	buffer_dma = le32_to_cpu(d->data_address);
2777	dma_sync_single_range_for_cpu(dev: context->ohci->card.device,
2778	addr: buffer_dma & PAGE_MASK,
2779	offset: buffer_dma & ~PAGE_MASK,
2780	le16_to_cpu(d->req_count),
2781	dir: DMA_FROM_DEVICE);
2782	}
2783
2784	copy_iso_headers(ctx, dma_hdr: (u32 *) (last + 1));
2785
2786	if (last->control & cpu_to_le16(DESCRIPTOR_IRQ_ALWAYS))
2787	flush_iso_completions(ctx, cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_INTERRUPT);
2788
2789	return 1;
2790	}
2791
2792	/* d == last because each descriptor block is only a single descriptor. */
2793	static int handle_ir_buffer_fill(struct context *context,
2794	struct descriptor *d,
2795	struct descriptor *last)
2796	{
2797	struct iso_context *ctx =
2798	container_of(context, struct iso_context, context);
2799	unsigned int req_count, res_count, completed;
2800	u32 buffer_dma;
2801
2802	req_count = le16_to_cpu(last->req_count);
2803	res_count = le16_to_cpu(READ_ONCE(last->res_count));
2804	completed = req_count - res_count;
2805	buffer_dma = le32_to_cpu(last->data_address);
2806
2807	if (completed > 0) {
2808	ctx->mc_buffer_bus = buffer_dma;
2809	ctx->mc_completed = completed;
2810	}
2811
2812	if (res_count != 0)
2813	/* Descriptor(s) not done yet, stop iteration */
2814	return 0;
2815
2816	dma_sync_single_range_for_cpu(dev: context->ohci->card.device,
2817	addr: buffer_dma & PAGE_MASK,
2818	offset: buffer_dma & ~PAGE_MASK,
2819	size: completed, dir: DMA_FROM_DEVICE);
2820
2821	if (last->control & cpu_to_le16(DESCRIPTOR_IRQ_ALWAYS)) {
2822	trace_isoc_inbound_multiple_completions(ctx: &ctx->base, completed,
2823	cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_INTERRUPT);
2824
2825	ctx->base.callback.mc(&ctx->base,
2826	buffer_dma + completed,
2827	ctx->base.callback_data);
2828	ctx->mc_completed = 0;
2829	}
2830
2831	return 1;
2832	}
2833
2834	static void flush_ir_buffer_fill(struct iso_context *ctx)
2835	{
2836	dma_sync_single_range_for_cpu(dev: ctx->context.ohci->card.device,
2837	addr: ctx->mc_buffer_bus & PAGE_MASK,
2838	offset: ctx->mc_buffer_bus & ~PAGE_MASK,
2839	size: ctx->mc_completed, dir: DMA_FROM_DEVICE);
2840
2841	trace_isoc_inbound_multiple_completions(ctx: &ctx->base, completed: ctx->mc_completed,
2842	cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_FLUSH);
2843
2844	ctx->base.callback.mc(&ctx->base,
2845	ctx->mc_buffer_bus + ctx->mc_completed,
2846	ctx->base.callback_data);
2847	ctx->mc_completed = 0;
2848	}
2849
2850	static inline void sync_it_packet_for_cpu(struct context *context,
2851	struct descriptor *pd)
2852	{
2853	__le16 control;
2854	u32 buffer_dma;
2855
2856	/* only packets beginning with OUTPUT_MORE* have data buffers */
2857	if (pd->control & cpu_to_le16(DESCRIPTOR_BRANCH_ALWAYS))
2858	return;
2859
2860	/* skip over the OUTPUT_MORE_IMMEDIATE descriptor */
2861	pd += 2;
2862
2863	/*
2864	* If the packet has a header, the first OUTPUT_MORE/LAST descriptor's
2865	* data buffer is in the context program's coherent page and must not
2866	* be synced.
2867	*/
2868	if ((le32_to_cpu(pd->data_address) & PAGE_MASK) ==
2869	(context->current_bus & PAGE_MASK)) {
2870	if (pd->control & cpu_to_le16(DESCRIPTOR_BRANCH_ALWAYS))
2871	return;
2872	pd++;
2873	}
2874
2875	do {
2876	buffer_dma = le32_to_cpu(pd->data_address);
2877	dma_sync_single_range_for_cpu(dev: context->ohci->card.device,
2878	addr: buffer_dma & PAGE_MASK,
2879	offset: buffer_dma & ~PAGE_MASK,
2880	le16_to_cpu(pd->req_count),
2881	dir: DMA_TO_DEVICE);
2882	control = pd->control;
2883	pd++;
2884	} while (!(control & cpu_to_le16(DESCRIPTOR_BRANCH_ALWAYS)));
2885	}
2886
2887	static int handle_it_packet(struct context *context,
2888	struct descriptor *d,
2889	struct descriptor *last)
2890	{
2891	struct iso_context *ctx =
2892	container_of(context, struct iso_context, context);
2893	struct descriptor *pd;
2894	__be32 *ctx_hdr;
2895
2896	for (pd = d; pd <= last; pd++)
2897	if (pd->transfer_status)
2898	break;
2899	if (pd > last)
2900	/* Descriptor(s) not done yet, stop iteration */
2901	return 0;
2902
2903	sync_it_packet_for_cpu(context, pd: d);
2904
2905	if (ctx->header_length + 4 > PAGE_SIZE) {
2906	if (ctx->base.drop_overflow_headers)
2907	return 1;
2908	flush_iso_completions(ctx, cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_HEADER_OVERFLOW);
2909	}
2910
2911	ctx_hdr = ctx->header + ctx->header_length;
2912	ctx->last_timestamp = le16_to_cpu(last->res_count);
2913	/* Present this value as big-endian to match the receive code */
2914	*ctx_hdr = cpu_to_be32((le16_to_cpu(pd->transfer_status) << 16) |
2915	le16_to_cpu(pd->res_count));
2916	ctx->header_length += 4;
2917
2918	if (last->control & cpu_to_le16(DESCRIPTOR_IRQ_ALWAYS))
2919	flush_iso_completions(ctx, cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_INTERRUPT);
2920
2921	return 1;
2922	}
2923
2924	static void set_multichannel_mask(struct fw_ohci *ohci, u64 channels)
2925	{
2926	u32 hi = channels >> 32, lo = channels;
2927
2928	reg_write(ohci, OHCI1394_IRMultiChanMaskHiClear, data: ~hi);
2929	reg_write(ohci, OHCI1394_IRMultiChanMaskLoClear, data: ~lo);
2930	reg_write(ohci, OHCI1394_IRMultiChanMaskHiSet, data: hi);
2931	reg_write(ohci, OHCI1394_IRMultiChanMaskLoSet, data: lo);
2932	ohci->mc_channels = channels;
2933	}
2934
2935	static struct fw_iso_context *ohci_allocate_iso_context(struct fw_card *card,
2936	int type, int channel, size_t header_size)
2937	{
2938	struct fw_ohci *ohci = fw_ohci(card);
2939	struct iso_context *ctx;
2940	descriptor_callback_t callback;
2941	u64 *channels;
2942	u32 *mask, regs;
2943	int index, ret = -EBUSY;
2944
2945	scoped_guard(spinlock_irq, &ohci->lock) {
2946	switch (type) {
2947	case FW_ISO_CONTEXT_TRANSMIT:
2948	mask = &ohci->it_context_mask;
2949	callback = handle_it_packet;
2950	index = ffs(*mask) - 1;
2951	if (index >= 0) {
2952	*mask &= ~(1 << index);
2953	regs = OHCI1394_IsoXmitContextBase(index);
2954	ctx = &ohci->it_context_list[index];
2955	}
2956	break;
2957
2958	case FW_ISO_CONTEXT_RECEIVE:
2959	channels = &ohci->ir_context_channels;
2960	mask = &ohci->ir_context_mask;
2961	callback = handle_ir_packet_per_buffer;
2962	index = *channels & 1ULL << channel ? ffs(*mask) - 1 : -1;
2963	if (index >= 0) {
2964	*channels &= ~(1ULL << channel);
2965	*mask &= ~(1 << index);
2966	regs = OHCI1394_IsoRcvContextBase(index);
2967	ctx = &ohci->ir_context_list[index];
2968	}
2969	break;
2970
2971	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
2972	mask = &ohci->ir_context_mask;
2973	callback = handle_ir_buffer_fill;
2974	index = !ohci->mc_allocated ? ffs(*mask) - 1 : -1;
2975	if (index >= 0) {
2976	ohci->mc_allocated = true;
2977	*mask &= ~(1 << index);
2978	regs = OHCI1394_IsoRcvContextBase(index);
2979	ctx = &ohci->ir_context_list[index];
2980	}
2981	break;
2982
2983	default:
2984	index = -1;
2985	ret = -ENOSYS;
2986	}
2987
2988	if (index < 0)
2989	return ERR_PTR(error: ret);
2990	}
2991
2992	memset(ctx, 0, sizeof(*ctx));
2993	ctx->header_length = 0;
2994	ctx->header = (void *) __get_free_page(GFP_KERNEL);
2995	if (ctx->header == NULL) {
2996	ret = -ENOMEM;
2997	goto out;
2998	}
2999	ret = context_init(ctx: &ctx->context, ohci, regs, callback);
3000	if (ret < 0)
3001	goto out_with_header;
3002	fw_iso_context_init_work(ctx: &ctx->base, func: ohci_isoc_context_work);
3003
3004	if (type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL) {
3005	set_multichannel_mask(ohci, channels: 0);
3006	ctx->mc_completed = 0;
3007	}
3008
3009	return &ctx->base;
3010
3011	out_with_header:
3012	free_page((unsigned long)ctx->header);
3013	out:
3014	scoped_guard(spinlock_irq, &ohci->lock) {
3015	switch (type) {
3016	case FW_ISO_CONTEXT_RECEIVE:
3017	*channels |= 1ULL << channel;
3018	break;
3019
3020	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3021	ohci->mc_allocated = false;
3022	break;
3023	}
3024	*mask |= 1 << index;
3025	}
3026
3027	return ERR_PTR(error: ret);
3028	}
3029
3030	static int ohci_start_iso(struct fw_iso_context *base,
3031	s32 cycle, u32 sync, u32 tags)
3032	{
3033	struct iso_context *ctx = container_of(base, struct iso_context, base);
3034	struct fw_ohci *ohci = ctx->context.ohci;
3035	u32 control = IR_CONTEXT_ISOCH_HEADER, match;
3036	int index;
3037
3038	/* the controller cannot start without any queued packets */
3039	if (ctx->context.last->branch_address == 0)
3040	return -ENODATA;
3041
3042	switch (ctx->base.type) {
3043	case FW_ISO_CONTEXT_TRANSMIT:
3044	index = ctx - ohci->it_context_list;
3045	match = 0;
3046	if (cycle >= 0)
3047	match = IT_CONTEXT_CYCLE_MATCH_ENABLE |
3048	(cycle & 0x7fff) << 16;
3049
3050	reg_write(ohci, OHCI1394_IsoXmitIntEventClear, data: 1 << index);
3051	reg_write(ohci, OHCI1394_IsoXmitIntMaskSet, data: 1 << index);
3052	context_run(ctx: &ctx->context, extra: match);
3053	break;
3054
3055	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3056	control |= IR_CONTEXT_BUFFER_FILL|IR_CONTEXT_MULTI_CHANNEL_MODE;
3057	fallthrough;
3058	case FW_ISO_CONTEXT_RECEIVE:
3059	index = ctx - ohci->ir_context_list;
3060	match = (tags << 28) | (sync << 8) | ctx->base.channel;
3061	if (cycle >= 0) {
3062	match |= (cycle & 0x07fff) << 12;
3063	control |= IR_CONTEXT_CYCLE_MATCH_ENABLE;
3064	}
3065
3066	reg_write(ohci, OHCI1394_IsoRecvIntEventClear, data: 1 << index);
3067	reg_write(ohci, OHCI1394_IsoRecvIntMaskSet, data: 1 << index);
3068	reg_write(ohci, CONTEXT_MATCH(ctx->context.regs), data: match);
3069	context_run(ctx: &ctx->context, extra: control);
3070
3071	ctx->sync = sync;
3072	ctx->tags = tags;
3073
3074	break;
3075	}
3076
3077	return 0;
3078	}
3079
3080	static int ohci_stop_iso(struct fw_iso_context *base)
3081	{
3082	struct fw_ohci *ohci = fw_ohci(card: base->card);
3083	struct iso_context *ctx = container_of(base, struct iso_context, base);
3084	int index;
3085
3086	switch (ctx->base.type) {
3087	case FW_ISO_CONTEXT_TRANSMIT:
3088	index = ctx - ohci->it_context_list;
3089	reg_write(ohci, OHCI1394_IsoXmitIntMaskClear, data: 1 << index);
3090	break;
3091
3092	case FW_ISO_CONTEXT_RECEIVE:
3093	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3094	index = ctx - ohci->ir_context_list;
3095	reg_write(ohci, OHCI1394_IsoRecvIntMaskClear, data: 1 << index);
3096	break;
3097	}
3098	flush_writes(ohci);
3099	context_stop(ctx: &ctx->context);
3100
3101	return 0;
3102	}
3103
3104	static void ohci_free_iso_context(struct fw_iso_context *base)
3105	{
3106	struct fw_ohci *ohci = fw_ohci(card: base->card);
3107	struct iso_context *ctx = container_of(base, struct iso_context, base);
3108	int index;
3109
3110	ohci_stop_iso(base);
3111	context_release(ctx: &ctx->context);
3112	free_page((unsigned long)ctx->header);
3113
3114	guard(spinlock_irqsave)(l: &ohci->lock);
3115
3116	switch (base->type) {
3117	case FW_ISO_CONTEXT_TRANSMIT:
3118	index = ctx - ohci->it_context_list;
3119	ohci->it_context_mask |= 1 << index;
3120	break;
3121
3122	case FW_ISO_CONTEXT_RECEIVE:
3123	index = ctx - ohci->ir_context_list;
3124	ohci->ir_context_mask |= 1 << index;
3125	ohci->ir_context_channels |= 1ULL << base->channel;
3126	break;
3127
3128	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3129	index = ctx - ohci->ir_context_list;
3130	ohci->ir_context_mask |= 1 << index;
3131	ohci->ir_context_channels |= ohci->mc_channels;
3132	ohci->mc_channels = 0;
3133	ohci->mc_allocated = false;
3134	break;
3135	}
3136	}
3137
3138	static int ohci_set_iso_channels(struct fw_iso_context *base, u64 *channels)
3139	{
3140	struct fw_ohci *ohci = fw_ohci(card: base->card);
3141
3142	switch (base->type) {
3143	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3144	{
3145	guard(spinlock_irqsave)(l: &ohci->lock);
3146
3147	// Don't allow multichannel to grab other contexts' channels.
3148	if (~ohci->ir_context_channels & ~ohci->mc_channels & *channels) {
3149	*channels = ohci->ir_context_channels;
3150	return -EBUSY;
3151	} else {
3152	set_multichannel_mask(ohci, channels: *channels);
3153	return 0;
3154	}
3155	}
3156	default:
3157	return -EINVAL;
3158	}
3159	}
3160
3161	static void __maybe_unused ohci_resume_iso_dma(struct fw_ohci *ohci)
3162	{
3163	int i;
3164	struct iso_context *ctx;
3165
3166	for (i = 0 ; i < ohci->n_ir ; i++) {
3167	ctx = &ohci->ir_context_list[i];
3168	if (ctx->context.running)
3169	ohci_start_iso(base: &ctx->base, cycle: 0, sync: ctx->sync, tags: ctx->tags);
3170	}
3171
3172	for (i = 0 ; i < ohci->n_it ; i++) {
3173	ctx = &ohci->it_context_list[i];
3174	if (ctx->context.running)
3175	ohci_start_iso(base: &ctx->base, cycle: 0, sync: ctx->sync, tags: ctx->tags);
3176	}
3177	}
3178
3179	static int queue_iso_transmit(struct iso_context *ctx,
3180	struct fw_iso_packet *packet,
3181	struct fw_iso_buffer *buffer,
3182	unsigned long payload)
3183	{
3184	struct descriptor *d, *last, *pd;
3185	struct fw_iso_packet *p;
3186	__le32 *header;
3187	dma_addr_t d_bus, page_bus;
3188	u32 z, header_z, payload_z, irq;
3189	u32 payload_index, payload_end_index, next_page_index;
3190	int page, end_page, i, length, offset;
3191
3192	p = packet;
3193	payload_index = payload;
3194
3195	if (p->skip)
3196	z = 1;
3197	else
3198	z = 2;
3199	if (p->header_length > 0)
3200	z++;
3201
3202	/* Determine the first page the payload isn't contained in. */
3203	end_page = PAGE_ALIGN(payload_index + p->payload_length) >> PAGE_SHIFT;
3204	if (p->payload_length > 0)
3205	payload_z = end_page - (payload_index >> PAGE_SHIFT);
3206	else
3207	payload_z = 0;
3208
3209	z += payload_z;
3210
3211	/* Get header size in number of descriptors. */
3212	header_z = DIV_ROUND_UP(p->header_length, sizeof(*d));
3213
3214	d = context_get_descriptors(ctx: &ctx->context, z: z + header_z, d_bus: &d_bus);
3215	if (d == NULL)
3216	return -ENOMEM;
3217
3218	if (!p->skip) {
3219	d[0].control = cpu_to_le16(DESCRIPTOR_KEY_IMMEDIATE);
3220	d[0].req_count = cpu_to_le16(8);
3221	/*
3222	* Link the skip address to this descriptor itself. This causes
3223	* a context to skip a cycle whenever lost cycles or FIFO
3224	* overruns occur, without dropping the data. The application
3225	* should then decide whether this is an error condition or not.
3226	* FIXME: Make the context's cycle-lost behaviour configurable?
3227	*/
3228	d[0].branch_address = cpu_to_le32(d_bus | z);
3229
3230	header = (__le32 *) &d[1];
3231
3232	ohci1394_it_data_set_speed(data: header, scode: ctx->base.speed);
3233	ohci1394_it_data_set_tag(data: header, tag: p->tag);
3234	ohci1394_it_data_set_channel(data: header, channel: ctx->base.channel);
3235	ohci1394_it_data_set_tcode(data: header, TCODE_STREAM_DATA);
3236	ohci1394_it_data_set_sync(data: header, sync: p->sy);
3237
3238	ohci1394_it_data_set_data_length(data: header, data_length: p->header_length + p->payload_length);
3239	}
3240
3241	if (p->header_length > 0) {
3242	d[2].req_count = cpu_to_le16(p->header_length);
3243	d[2].data_address = cpu_to_le32(d_bus + z * sizeof(*d));
3244	memcpy(&d[z], p->header, p->header_length);
3245	}
3246
3247	pd = d + z - payload_z;
3248	payload_end_index = payload_index + p->payload_length;
3249	for (i = 0; i < payload_z; i++) {
3250	page = payload_index >> PAGE_SHIFT;
3251	offset = payload_index & ~PAGE_MASK;
3252	next_page_index = (page + 1) << PAGE_SHIFT;
3253	length =
3254	min(next_page_index, payload_end_index) - payload_index;
3255	pd[i].req_count = cpu_to_le16(length);
3256
3257	page_bus = page_private(buffer->pages[page]);
3258	pd[i].data_address = cpu_to_le32(page_bus + offset);
3259
3260	dma_sync_single_range_for_device(dev: ctx->context.ohci->card.device,
3261	addr: page_bus, offset, size: length,
3262	dir: DMA_TO_DEVICE);
3263
3264	payload_index += length;
3265	}
3266
3267	if (p->interrupt)
3268	irq = DESCRIPTOR_IRQ_ALWAYS;
3269	else
3270	irq = DESCRIPTOR_NO_IRQ;
3271
3272	last = z == 2 ? d : d + z - 1;
3273	last->control |= cpu_to_le16(DESCRIPTOR_OUTPUT_LAST |
3274	DESCRIPTOR_STATUS |
3275	DESCRIPTOR_BRANCH_ALWAYS |
3276	irq);
3277
3278	context_append(ctx: &ctx->context, d, z, extra: header_z);
3279
3280	return 0;
3281	}
3282
3283	static int queue_iso_packet_per_buffer(struct iso_context *ctx,
3284	struct fw_iso_packet *packet,
3285	struct fw_iso_buffer *buffer,
3286	unsigned long payload)
3287	{
3288	struct device *device = ctx->context.ohci->card.device;
3289	struct descriptor *d, *pd;
3290	dma_addr_t d_bus, page_bus;
3291	u32 z, header_z, rest;
3292	int i, j, length;
3293	int page, offset, packet_count, header_size, payload_per_buffer;
3294
3295	/*
3296	* The OHCI controller puts the isochronous header and trailer in the
3297	* buffer, so we need at least 8 bytes.
3298	*/
3299	packet_count = packet->header_length / ctx->base.header_size;
3300	header_size = max(ctx->base.header_size, (size_t)8);
3301
3302	/* Get header size in number of descriptors. */
3303	header_z = DIV_ROUND_UP(header_size, sizeof(*d));
3304	page = payload >> PAGE_SHIFT;
3305	offset = payload & ~PAGE_MASK;
3306	payload_per_buffer = packet->payload_length / packet_count;
3307
3308	for (i = 0; i < packet_count; i++) {
3309	/* d points to the header descriptor */
3310	z = DIV_ROUND_UP(payload_per_buffer + offset, PAGE_SIZE) + 1;
3311	d = context_get_descriptors(ctx: &ctx->context,
3312	z: z + header_z, d_bus: &d_bus);
3313	if (d == NULL)
3314	return -ENOMEM;
3315
3316	d->control = cpu_to_le16(DESCRIPTOR_STATUS |
3317	DESCRIPTOR_INPUT_MORE);
3318	if (packet->skip && i == 0)
3319	d->control |= cpu_to_le16(DESCRIPTOR_WAIT);
3320	d->req_count = cpu_to_le16(header_size);
3321	d->res_count = d->req_count;
3322	d->transfer_status = 0;
3323	d->data_address = cpu_to_le32(d_bus + (z * sizeof(*d)));
3324
3325	rest = payload_per_buffer;
3326	pd = d;
3327	for (j = 1; j < z; j++) {
3328	pd++;
3329	pd->control = cpu_to_le16(DESCRIPTOR_STATUS |
3330	DESCRIPTOR_INPUT_MORE);
3331
3332	if (offset + rest < PAGE_SIZE)
3333	length = rest;
3334	else
3335	length = PAGE_SIZE - offset;
3336	pd->req_count = cpu_to_le16(length);
3337	pd->res_count = pd->req_count;
3338	pd->transfer_status = 0;
3339
3340	page_bus = page_private(buffer->pages[page]);
3341	pd->data_address = cpu_to_le32(page_bus + offset);
3342
3343	dma_sync_single_range_for_device(dev: device, addr: page_bus,
3344	offset, size: length,
3345	dir: DMA_FROM_DEVICE);
3346
3347	offset = (offset + length) & ~PAGE_MASK;
3348	rest -= length;
3349	if (offset == 0)
3350	page++;
3351	}
3352	pd->control = cpu_to_le16(DESCRIPTOR_STATUS |
3353	DESCRIPTOR_INPUT_LAST |
3354	DESCRIPTOR_BRANCH_ALWAYS);
3355	if (packet->interrupt && i == packet_count - 1)
3356	pd->control |= cpu_to_le16(DESCRIPTOR_IRQ_ALWAYS);
3357
3358	context_append(ctx: &ctx->context, d, z, extra: header_z);
3359	}
3360
3361	return 0;
3362	}
3363
3364	static int queue_iso_buffer_fill(struct iso_context *ctx,
3365	struct fw_iso_packet *packet,
3366	struct fw_iso_buffer *buffer,
3367	unsigned long payload)
3368	{
3369	struct descriptor *d;
3370	dma_addr_t d_bus, page_bus;
3371	int page, offset, rest, z, i, length;
3372
3373	page = payload >> PAGE_SHIFT;
3374	offset = payload & ~PAGE_MASK;
3375	rest = packet->payload_length;
3376
3377	/* We need one descriptor for each page in the buffer. */
3378	z = DIV_ROUND_UP(offset + rest, PAGE_SIZE);
3379
3380	if (WARN_ON(offset & 3 || rest & 3 || page + z > buffer->page_count))
3381	return -EFAULT;
3382
3383	for (i = 0; i < z; i++) {
3384	d = context_get_descriptors(ctx: &ctx->context, z: 1, d_bus: &d_bus);
3385	if (d == NULL)
3386	return -ENOMEM;
3387
3388	d->control = cpu_to_le16(DESCRIPTOR_INPUT_MORE |
3389	DESCRIPTOR_BRANCH_ALWAYS);
3390	if (packet->skip && i == 0)
3391	d->control |= cpu_to_le16(DESCRIPTOR_WAIT);
3392	if (packet->interrupt && i == z - 1)
3393	d->control |= cpu_to_le16(DESCRIPTOR_IRQ_ALWAYS);
3394
3395	if (offset + rest < PAGE_SIZE)
3396	length = rest;
3397	else
3398	length = PAGE_SIZE - offset;
3399	d->req_count = cpu_to_le16(length);
3400	d->res_count = d->req_count;
3401	d->transfer_status = 0;
3402
3403	page_bus = page_private(buffer->pages[page]);
3404	d->data_address = cpu_to_le32(page_bus + offset);
3405
3406	dma_sync_single_range_for_device(dev: ctx->context.ohci->card.device,
3407	addr: page_bus, offset, size: length,
3408	dir: DMA_FROM_DEVICE);
3409
3410	rest -= length;
3411	offset = 0;
3412	page++;
3413
3414	context_append(ctx: &ctx->context, d, z: 1, extra: 0);
3415	}
3416
3417	return 0;
3418	}
3419
3420	static int ohci_queue_iso(struct fw_iso_context *base,
3421	struct fw_iso_packet *packet,
3422	struct fw_iso_buffer *buffer,
3423	unsigned long payload)
3424	{
3425	struct iso_context *ctx = container_of(base, struct iso_context, base);
3426
3427	guard(spinlock_irqsave)(l: &ctx->context.ohci->lock);
3428
3429	switch (base->type) {
3430	case FW_ISO_CONTEXT_TRANSMIT:
3431	return queue_iso_transmit(ctx, packet, buffer, payload);
3432	case FW_ISO_CONTEXT_RECEIVE:
3433	return queue_iso_packet_per_buffer(ctx, packet, buffer, payload);
3434	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3435	return queue_iso_buffer_fill(ctx, packet, buffer, payload);
3436	default:
3437	return -ENOSYS;
3438	}
3439	}
3440
3441	static void ohci_flush_queue_iso(struct fw_iso_context *base)
3442	{
3443	struct context *ctx =
3444	&container_of(base, struct iso_context, base)->context;
3445
3446	reg_write(ohci: ctx->ohci, CONTROL_SET(ctx->regs), CONTEXT_WAKE);
3447	}
3448
3449	static int ohci_flush_iso_completions(struct fw_iso_context *base)
3450	{
3451	struct iso_context *ctx = container_of(base, struct iso_context, base);
3452	int ret = 0;
3453
3454	if (!test_and_set_bit_lock(nr: 0, addr: &ctx->flushing_completions)) {
3455	ohci_isoc_context_work(work: &base->work);
3456
3457	switch (base->type) {
3458	case FW_ISO_CONTEXT_TRANSMIT:
3459	case FW_ISO_CONTEXT_RECEIVE:
3460	if (ctx->header_length != 0)
3461	flush_iso_completions(ctx, cause: FW_ISO_CONTEXT_COMPLETIONS_CAUSE_FLUSH);
3462	break;
3463	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
3464	if (ctx->mc_completed != 0)
3465	flush_ir_buffer_fill(ctx);
3466	break;
3467	default:
3468	ret = -ENOSYS;
3469	}
3470
3471	clear_bit_unlock(nr: 0, addr: &ctx->flushing_completions);
3472	smp_mb__after_atomic();
3473	}
3474
3475	return ret;
3476	}
3477
3478	static const struct fw_card_driver ohci_driver = {
3479	.enable = ohci_enable,
3480	.disable = ohci_disable,
3481	.read_phy_reg = ohci_read_phy_reg,
3482	.update_phy_reg = ohci_update_phy_reg,
3483	.set_config_rom = ohci_set_config_rom,
3484	.send_request = ohci_send_request,
3485	.send_response = ohci_send_response,
3486	.cancel_packet = ohci_cancel_packet,
3487	.enable_phys_dma = ohci_enable_phys_dma,
3488	.read_csr = ohci_read_csr,
3489	.write_csr = ohci_write_csr,
3490
3491	.allocate_iso_context = ohci_allocate_iso_context,
3492	.free_iso_context = ohci_free_iso_context,
3493	.set_iso_channels = ohci_set_iso_channels,
3494	.queue_iso = ohci_queue_iso,
3495	.flush_queue_iso = ohci_flush_queue_iso,
3496	.flush_iso_completions = ohci_flush_iso_completions,
3497	.start_iso = ohci_start_iso,
3498	.stop_iso = ohci_stop_iso,
3499	};
3500
3501	#ifdef CONFIG_PPC_PMAC
3502	static void pmac_ohci_on(struct pci_dev *dev)
3503	{
3504	if (machine_is(powermac)) {
3505	struct device_node *ofn = pci_device_to_OF_node(dev);
3506
3507	if (ofn) {
3508	pmac_call_feature(PMAC_FTR_1394_CABLE_POWER, ofn, 0, 1);
3509	pmac_call_feature(PMAC_FTR_1394_ENABLE, ofn, 0, 1);
3510	}
3511	}
3512	}
3513
3514	static void pmac_ohci_off(struct pci_dev *dev)
3515	{
3516	if (machine_is(powermac)) {
3517	struct device_node *ofn = pci_device_to_OF_node(dev);
3518
3519	if (ofn) {
3520	pmac_call_feature(PMAC_FTR_1394_ENABLE, ofn, 0, 0);
3521	pmac_call_feature(PMAC_FTR_1394_CABLE_POWER, ofn, 0, 0);
3522	}
3523	}
3524	}
3525	#else
3526	static inline void pmac_ohci_on(struct pci_dev *dev) {}
3527	static inline void pmac_ohci_off(struct pci_dev *dev) {}
3528	#endif /* CONFIG_PPC_PMAC */
3529
3530	static void release_ohci(struct device *dev, void *data)
3531	{
3532	struct pci_dev *pdev = to_pci_dev(dev);
3533	struct fw_ohci *ohci = pci_get_drvdata(pdev);
3534
3535	pmac_ohci_off(dev: pdev);
3536
3537	ar_context_release(ctx: &ohci->ar_response_ctx);
3538	ar_context_release(ctx: &ohci->ar_request_ctx);
3539
3540	dev_notice(dev, "removed fw-ohci device\n");
3541	}
3542
3543	static int pci_probe(struct pci_dev *dev,
3544	const struct pci_device_id *ent)
3545	{
3546	struct fw_ohci *ohci;
3547	u32 bus_options, max_receive, link_speed, version;
3548	u64 guid;
3549	int i, flags, irq, err;
3550
3551	if (dev->vendor == PCI_VENDOR_ID_PINNACLE_SYSTEMS) {
3552	dev_err(&dev->dev, "Pinnacle MovieBoard is not yet supported\n");
3553	return -ENOSYS;
3554	}
3555
3556	ohci = devres_alloc(release_ohci, sizeof(*ohci), GFP_KERNEL);
3557	if (ohci == NULL)
3558	return -ENOMEM;
3559	fw_card_initialize(card: &ohci->card, driver: &ohci_driver, device: &dev->dev);
3560	pci_set_drvdata(pdev: dev, data: ohci);
3561	pmac_ohci_on(dev);
3562	devres_add(dev: &dev->dev, res: ohci);
3563
3564	err = pcim_enable_device(pdev: dev);
3565	if (err) {
3566	dev_err(&dev->dev, "failed to enable OHCI hardware\n");
3567	return err;
3568	}
3569
3570	pci_set_master(dev);
3571	pci_write_config_dword(dev, OHCI1394_PCI_HCI_Control, val: 0);
3572
3573	spin_lock_init(&ohci->lock);
3574	mutex_init(&ohci->phy_reg_mutex);
3575
3576	if (!(pci_resource_flags(dev, 0) & IORESOURCE_MEM) ||
3577	pci_resource_len(dev, 0) < OHCI1394_REGISTER_SIZE) {
3578	ohci_err(ohci, "invalid MMIO resource\n");
3579	return -ENXIO;
3580	}
3581
3582	ohci->registers = pcim_iomap_region(pdev: dev, bar: 0, name: ohci_driver_name);
3583	if (IS_ERR(ptr: ohci->registers)) {
3584	ohci_err(ohci, "request and map MMIO resource unavailable\n");
3585	return -ENXIO;
3586	}
3587
3588	for (i = 0; i < ARRAY_SIZE(ohci_quirks); i++)
3589	if ((ohci_quirks[i].vendor == dev->vendor) &&
3590	(ohci_quirks[i].device == (unsigned short)PCI_ANY_ID ||
3591	ohci_quirks[i].device == dev->device) &&
3592	(ohci_quirks[i].revision == (unsigned short)PCI_ANY_ID ||
3593	ohci_quirks[i].revision >= dev->revision)) {
3594	ohci->quirks = ohci_quirks[i].flags;
3595	break;
3596	}
3597	if (param_quirks)
3598	ohci->quirks = param_quirks;
3599
3600	if (detect_vt630x_with_asm1083_on_amd_ryzen_machine(pdev: dev))
3601	ohci->quirks |= QUIRK_REBOOT_BY_CYCLE_TIMER_READ;
3602
3603	/*
3604	* Because dma_alloc_coherent() allocates at least one page,
3605	* we save space by using a common buffer for the AR request/
3606	* response descriptors and the self IDs buffer.
3607	*/
3608	BUILD_BUG_ON(AR_BUFFERS * sizeof(struct descriptor) > PAGE_SIZE/4);
3609	BUILD_BUG_ON(SELF_ID_BUF_SIZE > PAGE_SIZE/2);
3610	ohci->misc_buffer = dmam_alloc_coherent(dev: &dev->dev, PAGE_SIZE, dma_handle: &ohci->misc_buffer_bus,
3611	GFP_KERNEL);
3612	if (!ohci->misc_buffer)
3613	return -ENOMEM;
3614
3615	err = ar_context_init(ctx: &ohci->ar_request_ctx, ohci, descriptors_offset: 0,
3616	OHCI1394_AsReqRcvContextControlSet);
3617	if (err < 0)
3618	return err;
3619
3620	err = ar_context_init(ctx: &ohci->ar_response_ctx, ohci, PAGE_SIZE/4,
3621	OHCI1394_AsRspRcvContextControlSet);
3622	if (err < 0)
3623	return err;
3624
3625	err = context_init(ctx: &ohci->at_request_ctx.context, ohci,
3626	OHCI1394_AsReqTrContextControlSet, callback: handle_at_packet);
3627	if (err < 0)
3628	return err;
3629	INIT_WORK(&ohci->at_request_ctx.work, ohci_at_context_work);
3630
3631	err = context_init(ctx: &ohci->at_response_ctx.context, ohci,
3632	OHCI1394_AsRspTrContextControlSet, callback: handle_at_packet);
3633	if (err < 0)
3634	return err;
3635	INIT_WORK(&ohci->at_response_ctx.work, ohci_at_context_work);
3636
3637	reg_write(ohci, OHCI1394_IsoRecvIntMaskSet, data: ~0);
3638	ohci->ir_context_channels = ~0ULL;
3639	ohci->ir_context_support = reg_read(ohci, OHCI1394_IsoRecvIntMaskSet);
3640	reg_write(ohci, OHCI1394_IsoRecvIntMaskClear, data: ~0);
3641	ohci->ir_context_mask = ohci->ir_context_support;
3642	ohci->n_ir = hweight32(ohci->ir_context_mask);
3643	ohci->ir_context_list = devm_kcalloc(dev: &dev->dev, n: ohci->n_ir, size: sizeof(struct iso_context), GFP_KERNEL);
3644	if (!ohci->ir_context_list)
3645	return -ENOMEM;
3646
3647	reg_write(ohci, OHCI1394_IsoXmitIntMaskSet, data: ~0);
3648	ohci->it_context_support = reg_read(ohci, OHCI1394_IsoXmitIntMaskSet);
3649	/* JMicron JMB38x often shows 0 at first read, just ignore it */
3650	if (!ohci->it_context_support) {
3651	ohci_notice(ohci, "overriding IsoXmitIntMask\n");
3652	ohci->it_context_support = 0xf;
3653	}
3654	reg_write(ohci, OHCI1394_IsoXmitIntMaskClear, data: ~0);
3655	ohci->it_context_mask = ohci->it_context_support;
3656	ohci->n_it = hweight32(ohci->it_context_mask);
3657	ohci->it_context_list = devm_kcalloc(dev: &dev->dev, n: ohci->n_it, size: sizeof(struct iso_context), GFP_KERNEL);
3658	if (!ohci->it_context_list)
3659	return -ENOMEM;
3660
3661	ohci->self_id = ohci->misc_buffer + PAGE_SIZE/2;
3662	ohci->self_id_bus = ohci->misc_buffer_bus + PAGE_SIZE/2;
3663
3664	bus_options = reg_read(ohci, OHCI1394_BusOptions);
3665	max_receive = (bus_options >> 12) & 0xf;
3666	link_speed = bus_options & 0x7;
3667	guid = ((u64) reg_read(ohci, OHCI1394_GUIDHi) << 32) |
3668	reg_read(ohci, OHCI1394_GUIDLo);
3669
3670	flags = PCI_IRQ_INTX;
3671	if (!(ohci->quirks & QUIRK_NO_MSI))
3672	flags |= PCI_IRQ_MSI;
3673	err = pci_alloc_irq_vectors(dev, min_vecs: 1, max_vecs: 1, flags);
3674	if (err < 0)
3675	return err;
3676	irq = pci_irq_vector(dev, nr: 0);
3677	if (irq < 0) {
3678	err = irq;
3679	goto fail_msi;
3680	}
3681
3682	// IRQF_ONESHOT is not applied so that any events are handled in the hardIRQ handler during
3683	// invoking the threaded IRQ handler for SelfIDComplete event.
3684	err = request_threaded_irq(irq, handler: irq_handler, thread_fn: handle_selfid_complete_event,
3685	flags: pci_dev_msi_enabled(pci_dev: dev) ? 0 : IRQF_SHARED, name: ohci_driver_name,
3686	dev: ohci);
3687	if (err < 0) {
3688	ohci_err(ohci, "failed to allocate interrupt %d\n", irq);
3689	goto fail_msi;
3690	}
3691
3692	err = fw_card_add(card: &ohci->card, max_receive, link_speed, guid, supported_isoc_contexts: ohci->n_it + ohci->n_ir);
3693	if (err)
3694	goto fail_irq;
3695
3696	version = reg_read(ohci, OHCI1394_Version) & 0x00ff00ff;
3697	ohci_notice(ohci,
3698	"added OHCI v%x.%x device as card %d, "
3699	"%d IR + %d IT contexts, quirks 0x%x%s\n",
3700	version >> 16, version & 0xff, ohci->card.index,
3701	ohci->n_ir, ohci->n_it, ohci->quirks,
3702	reg_read(ohci, OHCI1394_PhyUpperBound) ?
3703	", physUB" : "");
3704
3705	return 0;
3706
3707	fail_irq:
3708	free_irq(irq, ohci);
3709	fail_msi:
3710	pci_free_irq_vectors(dev);
3711
3712	return err;
3713	}
3714
3715	static void pci_remove(struct pci_dev *dev)
3716	{
3717	struct fw_ohci *ohci = pci_get_drvdata(pdev: dev);
3718	int irq;
3719
3720	fw_core_remove_card(card: &ohci->card);
3721
3722	software_reset(ohci);
3723
3724	irq = pci_irq_vector(dev, nr: 0);
3725	if (irq >= 0)
3726	free_irq(irq, ohci);
3727	pci_free_irq_vectors(dev);
3728
3729	dev_notice(&dev->dev, "removing fw-ohci device\n");
3730	}
3731
3732	static int __maybe_unused pci_suspend(struct device *dev)
3733	{
3734	struct pci_dev *pdev = to_pci_dev(dev);
3735	struct fw_ohci *ohci = pci_get_drvdata(pdev);
3736
3737	software_reset(ohci);
3738	pmac_ohci_off(dev: pdev);
3739
3740	return 0;
3741	}
3742
3743
3744	static int __maybe_unused pci_resume(struct device *dev)
3745	{
3746	struct pci_dev *pdev = to_pci_dev(dev);
3747	struct fw_ohci *ohci = pci_get_drvdata(pdev);
3748	int err;
3749
3750	pmac_ohci_on(dev: pdev);
3751
3752	/* Some systems don't setup GUID register on resume from ram */
3753	if (!reg_read(ohci, OHCI1394_GUIDLo) &&
3754	!reg_read(ohci, OHCI1394_GUIDHi)) {
3755	reg_write(ohci, OHCI1394_GUIDLo, data: (u32)ohci->card.guid);
3756	reg_write(ohci, OHCI1394_GUIDHi, data: (u32)(ohci->card.guid >> 32));
3757	}
3758
3759	err = ohci_enable(card: &ohci->card, NULL, length: 0);
3760	if (err)
3761	return err;
3762
3763	ohci_resume_iso_dma(ohci);
3764
3765	return 0;
3766	}
3767
3768	static const struct pci_device_id pci_table[] = {
3769	{ PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_FIREWIRE_OHCI, ~0) },
3770	{ }
3771	};
3772
3773	MODULE_DEVICE_TABLE(pci, pci_table);
3774
3775	static SIMPLE_DEV_PM_OPS(pci_pm_ops, pci_suspend, pci_resume);
3776
3777	static struct pci_driver fw_ohci_pci_driver = {
3778	.name = ohci_driver_name,
3779	.id_table = pci_table,
3780	.probe = pci_probe,
3781	.remove = pci_remove,
3782	.driver.pm = &pci_pm_ops,
3783	};
3784
3785	static int __init fw_ohci_init(void)
3786	{
3787	return pci_register_driver(&fw_ohci_pci_driver);
3788	}
3789
3790	static void __exit fw_ohci_cleanup(void)
3791	{
3792	pci_unregister_driver(dev: &fw_ohci_pci_driver);
3793	}
3794
3795	module_init(fw_ohci_init);
3796	module_exit(fw_ohci_cleanup);
3797
3798	MODULE_AUTHOR("Kristian Hoegsberg <krh@bitplanet.net>");
3799	MODULE_DESCRIPTION("Driver for PCI OHCI IEEE1394 controllers");
3800	MODULE_LICENSE("GPL");
3801
3802	/* Provide a module alias so root-on-sbp2 initrds don't break. */
3803	MODULE_ALIAS("ohci1394");
3804