1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright IBM Corp. 2007, 2009
4	* Author(s): Utz Bacher <utz.bacher@de.ibm.com>,
5	* Frank Pavlic <fpavlic@de.ibm.com>,
6	* Thomas Spatzier <tspat@de.ibm.com>,
7	* Frank Blaschka <frank.blaschka@de.ibm.com>
8	*/
9
10	#define pr_fmt(fmt) "qeth: " fmt
11
12	#include <linux/export.h>
13	#include <linux/module.h>
14	#include <linux/moduleparam.h>
15	#include <linux/string.h>
16	#include <linux/errno.h>
17	#include <linux/kernel.h>
18	#include <linux/log2.h>
19	#include <linux/io.h>
20	#include <linux/ip.h>
21	#include <linux/tcp.h>
22	#include <linux/mii.h>
23	#include <linux/mm.h>
24	#include <linux/kthread.h>
25	#include <linux/slab.h>
26	#include <linux/if_vlan.h>
27	#include <linux/netdevice.h>
28	#include <linux/netdev_features.h>
29	#include <linux/rcutree.h>
30	#include <linux/skbuff.h>
31	#include <linux/vmalloc.h>
32
33	#include <net/iucv/af_iucv.h>
34	#include <net/dsfield.h>
35	#include <net/sock.h>
36
37	#include <asm/ebcdic.h>
38	#include <asm/chpid.h>
39	#include <asm/sysinfo.h>
40	#include <asm/diag.h>
41	#include <asm/cio.h>
42	#include <asm/ccwdev.h>
43	#include <asm/cpcmd.h>
44
45	#include "qeth_core.h"
46
47	struct qeth_dbf_info qeth_dbf[QETH_DBF_INFOS] = {
48	/* define dbf - Name, Pages, Areas, Maxlen, Level, View, Handle */
49	/* N P A M L V H */
50	[QETH_DBF_SETUP] = {"qeth_setup",
51	8, 1, 8, 5, &debug_hex_ascii_view, NULL},
52	[QETH_DBF_MSG] = {"qeth_msg", 8, 1, 11 * sizeof(long), 3,
53	&debug_sprintf_view, NULL},
54	[QETH_DBF_CTRL] = {"qeth_control",
55	8, 1, QETH_DBF_CTRL_LEN, 5, &debug_hex_ascii_view, NULL},
56	};
57	EXPORT_SYMBOL_GPL(qeth_dbf);
58
59	static struct kmem_cache *qeth_core_header_cache;
60	static struct kmem_cache *qeth_qdio_outbuf_cache;
61	static struct kmem_cache *qeth_qaob_cache;
62
63	static struct device *qeth_core_root_dev;
64	static struct dentry *qeth_debugfs_root;
65	static struct lock_class_key qdio_out_skb_queue_key;
66
67	static void qeth_issue_next_read_cb(struct qeth_card *card,
68	struct qeth_cmd_buffer *iob,
69	unsigned int data_length);
70	static int qeth_qdio_establish(struct qeth_card *);
71	static void qeth_free_qdio_queues(struct qeth_card *card);
72
73	static const char *qeth_get_cardname(struct qeth_card *card)
74	{
75	if (IS_VM_NIC(card)) {
76	switch (card->info.type) {
77	case QETH_CARD_TYPE_OSD:
78	return " Virtual NIC QDIO";
79	case QETH_CARD_TYPE_IQD:
80	return " Virtual NIC Hiper";
81	case QETH_CARD_TYPE_OSM:
82	return " Virtual NIC QDIO - OSM";
83	case QETH_CARD_TYPE_OSX:
84	return " Virtual NIC QDIO - OSX";
85	default:
86	return " unknown";
87	}
88	} else {
89	switch (card->info.type) {
90	case QETH_CARD_TYPE_OSD:
91	return " OSD Express";
92	case QETH_CARD_TYPE_IQD:
93	return " HiperSockets";
94	case QETH_CARD_TYPE_OSM:
95	return " OSM QDIO";
96	case QETH_CARD_TYPE_OSX:
97	return " OSX QDIO";
98	default:
99	return " unknown";
100	}
101	}
102	return " n/a";
103	}
104
105	/* max length to be returned: 14 */
106	const char *qeth_get_cardname_short(struct qeth_card *card)
107	{
108	if (IS_VM_NIC(card)) {
109	switch (card->info.type) {
110	case QETH_CARD_TYPE_OSD:
111	return "Virt.NIC QDIO";
112	case QETH_CARD_TYPE_IQD:
113	return "Virt.NIC Hiper";
114	case QETH_CARD_TYPE_OSM:
115	return "Virt.NIC OSM";
116	case QETH_CARD_TYPE_OSX:
117	return "Virt.NIC OSX";
118	default:
119	return "unknown";
120	}
121	} else {
122	switch (card->info.type) {
123	case QETH_CARD_TYPE_OSD:
124	switch (card->info.link_type) {
125	case QETH_LINK_TYPE_FAST_ETH:
126	return "OSD_100";
127	case QETH_LINK_TYPE_HSTR:
128	return "HSTR";
129	case QETH_LINK_TYPE_GBIT_ETH:
130	return "OSD_1000";
131	case QETH_LINK_TYPE_10GBIT_ETH:
132	return "OSD_10GIG";
133	case QETH_LINK_TYPE_25GBIT_ETH:
134	return "OSD_25GIG";
135	case QETH_LINK_TYPE_LANE_ETH100:
136	return "OSD_FE_LANE";
137	case QETH_LINK_TYPE_LANE_TR:
138	return "OSD_TR_LANE";
139	case QETH_LINK_TYPE_LANE_ETH1000:
140	return "OSD_GbE_LANE";
141	case QETH_LINK_TYPE_LANE:
142	return "OSD_ATM_LANE";
143	default:
144	return "OSD_Express";
145	}
146	case QETH_CARD_TYPE_IQD:
147	return "HiperSockets";
148	case QETH_CARD_TYPE_OSM:
149	return "OSM_1000";
150	case QETH_CARD_TYPE_OSX:
151	return "OSX_10GIG";
152	default:
153	return "unknown";
154	}
155	}
156	return "n/a";
157	}
158
159	void qeth_set_allowed_threads(struct qeth_card *card, unsigned long threads,
160	int clear_start_mask)
161	{
162	unsigned long flags;
163
164	spin_lock_irqsave(&card->thread_mask_lock, flags);
165	card->thread_allowed_mask = threads;
166	if (clear_start_mask)
167	card->thread_start_mask &= threads;
168	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
169	wake_up(&card->wait_q);
170	}
171	EXPORT_SYMBOL_GPL(qeth_set_allowed_threads);
172
173	int qeth_threads_running(struct qeth_card *card, unsigned long threads)
174	{
175	unsigned long flags;
176	int rc = 0;
177
178	spin_lock_irqsave(&card->thread_mask_lock, flags);
179	rc = (card->thread_running_mask & threads);
180	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
181	return rc;
182	}
183	EXPORT_SYMBOL_GPL(qeth_threads_running);
184
185	static void qeth_clear_working_pool_list(struct qeth_card *card)
186	{
187	struct qeth_buffer_pool_entry *pool_entry, *tmp;
188	struct qeth_qdio_q *queue = card->qdio.in_q;
189	unsigned int i;
190
191	QETH_CARD_TEXT(card, 5, "clwrklst");
192	list_for_each_entry_safe(pool_entry, tmp,
193	&card->qdio.in_buf_pool.entry_list, list)
194	list_del(entry: &pool_entry->list);
195
196	for (i = 0; i < ARRAY_SIZE(queue->bufs); i++)
197	queue->bufs[i].pool_entry = NULL;
198	}
199
200	static void qeth_free_pool_entry(struct qeth_buffer_pool_entry *entry)
201	{
202	unsigned int i;
203
204	for (i = 0; i < ARRAY_SIZE(entry->elements); i++) {
205	if (entry->elements[i])
206	__free_page(entry->elements[i]);
207	}
208
209	kfree(objp: entry);
210	}
211
212	static void qeth_free_buffer_pool(struct qeth_card *card)
213	{
214	struct qeth_buffer_pool_entry *entry, *tmp;
215
216	list_for_each_entry_safe(entry, tmp, &card->qdio.init_pool.entry_list,
217	init_list) {
218	list_del(entry: &entry->init_list);
219	qeth_free_pool_entry(entry);
220	}
221	}
222
223	static struct qeth_buffer_pool_entry *qeth_alloc_pool_entry(unsigned int pages)
224	{
225	struct qeth_buffer_pool_entry *entry;
226	unsigned int i;
227
228	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
229	if (!entry)
230	return NULL;
231
232	for (i = 0; i < pages; i++) {
233	entry->elements[i] = __dev_alloc_page(GFP_KERNEL);
234
235	if (!entry->elements[i]) {
236	qeth_free_pool_entry(entry);
237	return NULL;
238	}
239	}
240
241	return entry;
242	}
243
244	static int qeth_alloc_buffer_pool(struct qeth_card *card)
245	{
246	unsigned int buf_elements = QETH_MAX_BUFFER_ELEMENTS(card);
247	unsigned int i;
248
249	QETH_CARD_TEXT(card, 5, "alocpool");
250	for (i = 0; i < card->qdio.init_pool.buf_count; ++i) {
251	struct qeth_buffer_pool_entry *entry;
252
253	entry = qeth_alloc_pool_entry(pages: buf_elements);
254	if (!entry) {
255	qeth_free_buffer_pool(card);
256	return -ENOMEM;
257	}
258
259	list_add(new: &entry->init_list, head: &card->qdio.init_pool.entry_list);
260	}
261	return 0;
262	}
263
264	int qeth_resize_buffer_pool(struct qeth_card *card, unsigned int count)
265	{
266	unsigned int buf_elements = QETH_MAX_BUFFER_ELEMENTS(card);
267	struct qeth_qdio_buffer_pool *pool = &card->qdio.init_pool;
268	struct qeth_buffer_pool_entry *entry, *tmp;
269	int delta = count - pool->buf_count;
270	LIST_HEAD(entries);
271
272	QETH_CARD_TEXT(card, 2, "realcbp");
273
274	/* Defer until pool is allocated: */
275	if (list_empty(head: &pool->entry_list))
276	goto out;
277
278	/* Remove entries from the pool: */
279	while (delta < 0) {
280	entry = list_first_entry(&pool->entry_list,
281	struct qeth_buffer_pool_entry,
282	init_list);
283	list_del(entry: &entry->init_list);
284	qeth_free_pool_entry(entry);
285
286	delta++;
287	}
288
289	/* Allocate additional entries: */
290	while (delta > 0) {
291	entry = qeth_alloc_pool_entry(pages: buf_elements);
292	if (!entry) {
293	list_for_each_entry_safe(entry, tmp, &entries,
294	init_list) {
295	list_del(entry: &entry->init_list);
296	qeth_free_pool_entry(entry);
297	}
298
299	return -ENOMEM;
300	}
301
302	list_add(new: &entry->init_list, head: &entries);
303
304	delta--;
305	}
306
307	list_splice(list: &entries, head: &pool->entry_list);
308
309	out:
310	card->qdio.in_buf_pool.buf_count = count;
311	pool->buf_count = count;
312	return 0;
313	}
314	EXPORT_SYMBOL_GPL(qeth_resize_buffer_pool);
315
316	static void qeth_free_qdio_queue(struct qeth_qdio_q *q)
317	{
318	if (!q)
319	return;
320
321	qdio_free_buffers(q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q);
322	kfree(objp: q);
323	}
324
325	static struct qeth_qdio_q *qeth_alloc_qdio_queue(void)
326	{
327	struct qeth_qdio_q *q = kzalloc(sizeof(*q), GFP_KERNEL);
328	int i;
329
330	if (!q)
331	return NULL;
332
333	if (qdio_alloc_buffers(q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q)) {
334	kfree(objp: q);
335	return NULL;
336	}
337
338	for (i = 0; i < QDIO_MAX_BUFFERS_PER_Q; ++i)
339	q->bufs[i].buffer = q->qdio_bufs[i];
340
341	QETH_DBF_HEX(SETUP, 2, &q, sizeof(void *));
342	return q;
343	}
344
345	static int qeth_cq_init(struct qeth_card *card)
346	{
347	int rc;
348
349	if (card->options.cq == QETH_CQ_ENABLED) {
350	QETH_CARD_TEXT(card, 2, "cqinit");
351	qdio_reset_buffers(card->qdio.c_q->qdio_bufs,
352	QDIO_MAX_BUFFERS_PER_Q);
353	card->qdio.c_q->next_buf_to_init = 127;
354
355	rc = qdio_add_bufs_to_input_queue(CARD_DDEV(card), 1, 0, 127);
356	if (rc) {
357	QETH_CARD_TEXT_(card, 2, "1err%d", rc);
358	goto out;
359	}
360	}
361	rc = 0;
362	out:
363	return rc;
364	}
365
366	static void qeth_free_cq(struct qeth_card *card)
367	{
368	if (card->qdio.c_q) {
369	qeth_free_qdio_queue(q: card->qdio.c_q);
370	card->qdio.c_q = NULL;
371	}
372	}
373
374	static int qeth_alloc_cq(struct qeth_card *card)
375	{
376	if (card->options.cq == QETH_CQ_ENABLED) {
377	QETH_CARD_TEXT(card, 2, "cqon");
378	if (!card->qdio.c_q) {
379	card->qdio.c_q = qeth_alloc_qdio_queue();
380	if (!card->qdio.c_q) {
381	dev_err(&card->gdev->dev,
382	"Failed to create completion queue\n");
383	return -ENOMEM;
384	}
385	}
386	} else {
387	QETH_CARD_TEXT(card, 2, "nocq");
388	qeth_free_cq(card);
389	}
390	return 0;
391	}
392
393	static enum iucv_tx_notify qeth_compute_cq_notification(int sbalf15,
394	int delayed)
395	{
396	enum iucv_tx_notify n;
397
398	switch (sbalf15) {
399	case 0:
400	n = delayed ? TX_NOTIFY_DELAYED_OK : TX_NOTIFY_OK;
401	break;
402	case 4:
403	case 16:
404	case 17:
405	case 18:
406	n = delayed ? TX_NOTIFY_DELAYED_UNREACHABLE :
407	TX_NOTIFY_UNREACHABLE;
408	break;
409	default:
410	n = delayed ? TX_NOTIFY_DELAYED_GENERALERROR :
411	TX_NOTIFY_GENERALERROR;
412	break;
413	}
414
415	return n;
416	}
417
418	static void qeth_put_cmd(struct qeth_cmd_buffer *iob)
419	{
420	if (refcount_dec_and_test(r: &iob->ref_count)) {
421	kfree(objp: iob->data);
422	kfree(objp: iob);
423	}
424	}
425	static void qeth_setup_ccw(struct ccw1 *ccw, u8 cmd_code, u8 flags, u32 len,
426	void *data)
427	{
428	ccw->cmd_code = cmd_code;
429	ccw->flags = flags | CCW_FLAG_SLI;
430	ccw->count = len;
431	ccw->cda = virt_to_dma32(data);
432	}
433
434	static int __qeth_issue_next_read(struct qeth_card *card)
435	{
436	struct qeth_cmd_buffer *iob = card->read_cmd;
437	struct qeth_channel *channel = iob->channel;
438	struct ccw1 *ccw = __ccw_from_cmd(iob);
439	int rc;
440
441	QETH_CARD_TEXT(card, 5, "issnxrd");
442	if (channel->state != CH_STATE_UP)
443	return -EIO;
444
445	memset(iob->data, 0, iob->length);
446	qeth_setup_ccw(ccw, CCW_CMD_READ, flags: 0, len: iob->length, data: iob->data);
447	iob->callback = qeth_issue_next_read_cb;
448	/* keep the cmd alive after completion: */
449	qeth_get_cmd(iob);
450
451	QETH_CARD_TEXT(card, 6, "noirqpnd");
452	rc = ccw_device_start(channel->ccwdev, ccw, (addr_t) iob, 0, 0);
453	if (!rc) {
454	channel->active_cmd = iob;
455	} else {
456	QETH_DBF_MESSAGE(2, "error %i on device %x when starting next read ccw!\n",
457	rc, CARD_DEVID(card));
458	qeth_unlock_channel(card, channel);
459	qeth_put_cmd(iob);
460	card->read_or_write_problem = 1;
461	qeth_schedule_recovery(card);
462	}
463	return rc;
464	}
465
466	static int qeth_issue_next_read(struct qeth_card *card)
467	{
468	int ret;
469
470	spin_lock_irq(lock: get_ccwdev_lock(CARD_RDEV(card)));
471	ret = __qeth_issue_next_read(card);
472	spin_unlock_irq(lock: get_ccwdev_lock(CARD_RDEV(card)));
473
474	return ret;
475	}
476
477	static void qeth_enqueue_cmd(struct qeth_card *card,
478	struct qeth_cmd_buffer *iob)
479	{
480	spin_lock_irq(lock: &card->lock);
481	list_add_tail(new: &iob->list_entry, head: &card->cmd_waiter_list);
482	spin_unlock_irq(lock: &card->lock);
483	}
484
485	static void qeth_dequeue_cmd(struct qeth_card *card,
486	struct qeth_cmd_buffer *iob)
487	{
488	spin_lock_irq(lock: &card->lock);
489	list_del(entry: &iob->list_entry);
490	spin_unlock_irq(lock: &card->lock);
491	}
492
493	static void qeth_notify_cmd(struct qeth_cmd_buffer *iob, int reason)
494	{
495	iob->rc = reason;
496	complete(&iob->done);
497	}
498
499	static void qeth_flush_local_addrs4(struct qeth_card *card)
500	{
501	struct qeth_local_addr *addr;
502	struct hlist_node *tmp;
503	unsigned int i;
504
505	spin_lock_irq(lock: &card->local_addrs4_lock);
506	hash_for_each_safe(card->local_addrs4, i, tmp, addr, hnode) {
507	hash_del_rcu(node: &addr->hnode);
508	kfree_rcu(addr, rcu);
509	}
510	spin_unlock_irq(lock: &card->local_addrs4_lock);
511	}
512
513	static void qeth_flush_local_addrs6(struct qeth_card *card)
514	{
515	struct qeth_local_addr *addr;
516	struct hlist_node *tmp;
517	unsigned int i;
518
519	spin_lock_irq(lock: &card->local_addrs6_lock);
520	hash_for_each_safe(card->local_addrs6, i, tmp, addr, hnode) {
521	hash_del_rcu(node: &addr->hnode);
522	kfree_rcu(addr, rcu);
523	}
524	spin_unlock_irq(lock: &card->local_addrs6_lock);
525	}
526
527	static void qeth_flush_local_addrs(struct qeth_card *card)
528	{
529	qeth_flush_local_addrs4(card);
530	qeth_flush_local_addrs6(card);
531	}
532
533	static void qeth_add_local_addrs4(struct qeth_card *card,
534	struct qeth_ipacmd_local_addrs4 *cmd)
535	{
536	unsigned int i;
537
538	if (cmd->addr_length !=
539	sizeof_field(struct qeth_ipacmd_local_addr4, addr)) {
540	dev_err_ratelimited(&card->gdev->dev,
541	"Dropped IPv4 ADD LOCAL ADDR event with bad length %u\n",
542	cmd->addr_length);
543	return;
544	}
545
546	spin_lock(lock: &card->local_addrs4_lock);
547	for (i = 0; i < cmd->count; i++) {
548	unsigned int key = ipv4_addr_hash(ip: cmd->addrs[i].addr);
549	struct qeth_local_addr *addr;
550	bool duplicate = false;
551
552	hash_for_each_possible(card->local_addrs4, addr, hnode, key) {
553	if (addr->addr.s6_addr32[3] == cmd->addrs[i].addr) {
554	duplicate = true;
555	break;
556	}
557	}
558
559	if (duplicate)
560	continue;
561
562	addr = kmalloc(sizeof(*addr), GFP_ATOMIC);
563	if (!addr) {
564	dev_err(&card->gdev->dev,
565	"Failed to allocate local addr object. Traffic to %pI4 might suffer.\n",
566	&cmd->addrs[i].addr);
567	continue;
568	}
569
570	ipv6_addr_set(addr: &addr->addr, w1: 0, w2: 0, w3: 0, w4: cmd->addrs[i].addr);
571	hash_add_rcu(card->local_addrs4, &addr->hnode, key);
572	}
573	spin_unlock(lock: &card->local_addrs4_lock);
574	}
575
576	static void qeth_add_local_addrs6(struct qeth_card *card,
577	struct qeth_ipacmd_local_addrs6 *cmd)
578	{
579	unsigned int i;
580
581	if (cmd->addr_length !=
582	sizeof_field(struct qeth_ipacmd_local_addr6, addr)) {
583	dev_err_ratelimited(&card->gdev->dev,
584	"Dropped IPv6 ADD LOCAL ADDR event with bad length %u\n",
585	cmd->addr_length);
586	return;
587	}
588
589	spin_lock(lock: &card->local_addrs6_lock);
590	for (i = 0; i < cmd->count; i++) {
591	u32 key = ipv6_addr_hash(a: &cmd->addrs[i].addr);
592	struct qeth_local_addr *addr;
593	bool duplicate = false;
594
595	hash_for_each_possible(card->local_addrs6, addr, hnode, key) {
596	if (ipv6_addr_equal(a1: &addr->addr, a2: &cmd->addrs[i].addr)) {
597	duplicate = true;
598	break;
599	}
600	}
601
602	if (duplicate)
603	continue;
604
605	addr = kmalloc(sizeof(*addr), GFP_ATOMIC);
606	if (!addr) {
607	dev_err(&card->gdev->dev,
608	"Failed to allocate local addr object. Traffic to %pI6c might suffer.\n",
609	&cmd->addrs[i].addr);
610	continue;
611	}
612
613	addr->addr = cmd->addrs[i].addr;
614	hash_add_rcu(card->local_addrs6, &addr->hnode, key);
615	}
616	spin_unlock(lock: &card->local_addrs6_lock);
617	}
618
619	static void qeth_del_local_addrs4(struct qeth_card *card,
620	struct qeth_ipacmd_local_addrs4 *cmd)
621	{
622	unsigned int i;
623
624	if (cmd->addr_length !=
625	sizeof_field(struct qeth_ipacmd_local_addr4, addr)) {
626	dev_err_ratelimited(&card->gdev->dev,
627	"Dropped IPv4 DEL LOCAL ADDR event with bad length %u\n",
628	cmd->addr_length);
629	return;
630	}
631
632	spin_lock(lock: &card->local_addrs4_lock);
633	for (i = 0; i < cmd->count; i++) {
634	struct qeth_ipacmd_local_addr4 *addr = &cmd->addrs[i];
635	unsigned int key = ipv4_addr_hash(ip: addr->addr);
636	struct qeth_local_addr *tmp;
637
638	hash_for_each_possible(card->local_addrs4, tmp, hnode, key) {
639	if (tmp->addr.s6_addr32[3] == addr->addr) {
640	hash_del_rcu(node: &tmp->hnode);
641	kfree_rcu(tmp, rcu);
642	break;
643	}
644	}
645	}
646	spin_unlock(lock: &card->local_addrs4_lock);
647	}
648
649	static void qeth_del_local_addrs6(struct qeth_card *card,
650	struct qeth_ipacmd_local_addrs6 *cmd)
651	{
652	unsigned int i;
653
654	if (cmd->addr_length !=
655	sizeof_field(struct qeth_ipacmd_local_addr6, addr)) {
656	dev_err_ratelimited(&card->gdev->dev,
657	"Dropped IPv6 DEL LOCAL ADDR event with bad length %u\n",
658	cmd->addr_length);
659	return;
660	}
661
662	spin_lock(lock: &card->local_addrs6_lock);
663	for (i = 0; i < cmd->count; i++) {
664	struct qeth_ipacmd_local_addr6 *addr = &cmd->addrs[i];
665	u32 key = ipv6_addr_hash(a: &addr->addr);
666	struct qeth_local_addr *tmp;
667
668	hash_for_each_possible(card->local_addrs6, tmp, hnode, key) {
669	if (ipv6_addr_equal(a1: &tmp->addr, a2: &addr->addr)) {
670	hash_del_rcu(node: &tmp->hnode);
671	kfree_rcu(tmp, rcu);
672	break;
673	}
674	}
675	}
676	spin_unlock(lock: &card->local_addrs6_lock);
677	}
678
679	static bool qeth_next_hop_is_local_v4(struct qeth_card *card,
680	struct sk_buff *skb)
681	{
682	struct qeth_local_addr *tmp;
683	bool is_local = false;
684	unsigned int key;
685	__be32 next_hop;
686
687	if (hash_empty(card->local_addrs4))
688	return false;
689
690	rcu_read_lock();
691	next_hop = qeth_next_hop_v4_rcu(skb,
692	dst: qeth_dst_check_rcu(skb, htons(ETH_P_IP)));
693	key = ipv4_addr_hash(ip: next_hop);
694
695	hash_for_each_possible_rcu(card->local_addrs4, tmp, hnode, key) {
696	if (tmp->addr.s6_addr32[3] == next_hop) {
697	is_local = true;
698	break;
699	}
700	}
701	rcu_read_unlock();
702
703	return is_local;
704	}
705
706	static bool qeth_next_hop_is_local_v6(struct qeth_card *card,
707	struct sk_buff *skb)
708	{
709	struct qeth_local_addr *tmp;
710	struct in6_addr *next_hop;
711	bool is_local = false;
712	u32 key;
713
714	if (hash_empty(card->local_addrs6))
715	return false;
716
717	rcu_read_lock();
718	next_hop = qeth_next_hop_v6_rcu(skb,
719	dst: qeth_dst_check_rcu(skb, htons(ETH_P_IPV6)));
720	key = ipv6_addr_hash(a: next_hop);
721
722	hash_for_each_possible_rcu(card->local_addrs6, tmp, hnode, key) {
723	if (ipv6_addr_equal(a1: &tmp->addr, a2: next_hop)) {
724	is_local = true;
725	break;
726	}
727	}
728	rcu_read_unlock();
729
730	return is_local;
731	}
732
733	static int qeth_debugfs_local_addr_show(struct seq_file *m, void *v)
734	{
735	struct qeth_card *card = m->private;
736	struct qeth_local_addr *tmp;
737	unsigned int i;
738
739	rcu_read_lock();
740	hash_for_each_rcu(card->local_addrs4, i, tmp, hnode)
741	seq_printf(m, fmt: "%pI4\n", &tmp->addr.s6_addr32[3]);
742	hash_for_each_rcu(card->local_addrs6, i, tmp, hnode)
743	seq_printf(m, fmt: "%pI6c\n", &tmp->addr);
744	rcu_read_unlock();
745
746	return 0;
747	}
748
749	DEFINE_SHOW_ATTRIBUTE(qeth_debugfs_local_addr);
750
751	static void qeth_issue_ipa_msg(struct qeth_ipa_cmd *cmd, int rc,
752	struct qeth_card *card)
753	{
754	const char *ipa_name;
755	int com = cmd->hdr.command;
756
757	ipa_name = qeth_get_ipa_cmd_name(cmd: com);
758
759	if (rc)
760	QETH_DBF_MESSAGE(2, "IPA: %s(%#x) for device %x returned %#x \"%s\"\n",
761	ipa_name, com, CARD_DEVID(card), rc,
762	qeth_get_ipa_msg(com, rc));
763	else
764	QETH_DBF_MESSAGE(5, "IPA: %s(%#x) for device %x succeeded\n",
765	ipa_name, com, CARD_DEVID(card));
766	}
767
768	static void qeth_default_link_info(struct qeth_card *card)
769	{
770	struct qeth_link_info *link_info = &card->info.link_info;
771
772	QETH_CARD_TEXT(card, 2, "dftlinfo");
773	link_info->duplex = DUPLEX_FULL;
774
775	if (IS_IQD(card) || IS_VM_NIC(card)) {
776	link_info->speed = SPEED_10000;
777	link_info->port = PORT_FIBRE;
778	link_info->link_mode = QETH_LINK_MODE_FIBRE_SHORT;
779	} else {
780	switch (card->info.link_type) {
781	case QETH_LINK_TYPE_FAST_ETH:
782	case QETH_LINK_TYPE_LANE_ETH100:
783	link_info->speed = SPEED_100;
784	link_info->port = PORT_TP;
785	break;
786	case QETH_LINK_TYPE_GBIT_ETH:
787	case QETH_LINK_TYPE_LANE_ETH1000:
788	link_info->speed = SPEED_1000;
789	link_info->port = PORT_FIBRE;
790	break;
791	case QETH_LINK_TYPE_10GBIT_ETH:
792	link_info->speed = SPEED_10000;
793	link_info->port = PORT_FIBRE;
794	break;
795	case QETH_LINK_TYPE_25GBIT_ETH:
796	link_info->speed = SPEED_25000;
797	link_info->port = PORT_FIBRE;
798	break;
799	default:
800	dev_info(&card->gdev->dev,
801	"Unknown link type %x\n",
802	card->info.link_type);
803	link_info->speed = SPEED_UNKNOWN;
804	link_info->port = PORT_OTHER;
805	}
806
807	link_info->link_mode = QETH_LINK_MODE_UNKNOWN;
808	}
809	}
810
811	static struct qeth_ipa_cmd *qeth_check_ipa_data(struct qeth_card *card,
812	struct qeth_ipa_cmd *cmd)
813	{
814	QETH_CARD_TEXT(card, 5, "chkipad");
815
816	if (IS_IPA_REPLY(cmd)) {
817	if (cmd->hdr.command != IPA_CMD_SET_DIAG_ASS)
818	qeth_issue_ipa_msg(cmd, rc: cmd->hdr.return_code, card);
819	return cmd;
820	}
821
822	/* handle unsolicited event: */
823	switch (cmd->hdr.command) {
824	case IPA_CMD_STOPLAN:
825	if (cmd->hdr.return_code == IPA_RC_VEPA_TO_VEB_TRANSITION) {
826	dev_err(&card->gdev->dev,
827	"Adjacent port of interface %s is no longer in reflective relay mode, trigger recovery\n",
828	netdev_name(card->dev));
829	/* Set offline, then probably fail to set online: */
830	qeth_schedule_recovery(card);
831	} else {
832	/* stay online for subsequent STARTLAN */
833	dev_warn(&card->gdev->dev,
834	"The link for interface %s on CHPID 0x%X failed\n",
835	netdev_name(card->dev), card->info.chpid);
836	qeth_issue_ipa_msg(cmd, rc: cmd->hdr.return_code, card);
837	netif_carrier_off(dev: card->dev);
838	qeth_default_link_info(card);
839	}
840	return NULL;
841	case IPA_CMD_STARTLAN:
842	dev_info(&card->gdev->dev,
843	"The link for %s on CHPID 0x%X has been restored\n",
844	netdev_name(card->dev), card->info.chpid);
845	if (card->info.hwtrap)
846	card->info.hwtrap = 2;
847	qeth_schedule_recovery(card);
848	return NULL;
849	case IPA_CMD_SETBRIDGEPORT_IQD:
850	case IPA_CMD_SETBRIDGEPORT_OSA:
851	case IPA_CMD_ADDRESS_CHANGE_NOTIF:
852	if (card->discipline->control_event_handler(card, cmd))
853	return cmd;
854	return NULL;
855	case IPA_CMD_REGISTER_LOCAL_ADDR:
856	if (cmd->hdr.prot_version == QETH_PROT_IPV4)
857	qeth_add_local_addrs4(card, cmd: &cmd->data.local_addrs4);
858	else if (cmd->hdr.prot_version == QETH_PROT_IPV6)
859	qeth_add_local_addrs6(card, cmd: &cmd->data.local_addrs6);
860
861	QETH_CARD_TEXT(card, 3, "irla");
862	return NULL;
863	case IPA_CMD_UNREGISTER_LOCAL_ADDR:
864	if (cmd->hdr.prot_version == QETH_PROT_IPV4)
865	qeth_del_local_addrs4(card, cmd: &cmd->data.local_addrs4);
866	else if (cmd->hdr.prot_version == QETH_PROT_IPV6)
867	qeth_del_local_addrs6(card, cmd: &cmd->data.local_addrs6);
868
869	QETH_CARD_TEXT(card, 3, "urla");
870	return NULL;
871	default:
872	QETH_DBF_MESSAGE(2, "Received data is IPA but not a reply!\n");
873	return cmd;
874	}
875	}
876
877	static void qeth_clear_ipacmd_list(struct qeth_card *card)
878	{
879	struct qeth_cmd_buffer *iob;
880	unsigned long flags;
881
882	QETH_CARD_TEXT(card, 4, "clipalst");
883
884	spin_lock_irqsave(&card->lock, flags);
885	list_for_each_entry(iob, &card->cmd_waiter_list, list_entry)
886	qeth_notify_cmd(iob, reason: -ECANCELED);
887	spin_unlock_irqrestore(lock: &card->lock, flags);
888	}
889
890	static int qeth_check_idx_response(struct qeth_card *card,
891	unsigned char *buffer)
892	{
893	QETH_DBF_HEX(CTRL, 2, buffer, QETH_DBF_CTRL_LEN);
894	if ((buffer[2] & QETH_IDX_TERMINATE_MASK) == QETH_IDX_TERMINATE) {
895	QETH_DBF_MESSAGE(2, "received an IDX TERMINATE with cause code %#04x\n",
896	buffer[4]);
897	QETH_CARD_TEXT(card, 2, "ckidxres");
898	QETH_CARD_TEXT(card, 2, " idxterm");
899	QETH_CARD_TEXT_(card, 2, "rc%x", buffer[4]);
900	if (buffer[4] == QETH_IDX_TERM_BAD_TRANSPORT ||
901	buffer[4] == QETH_IDX_TERM_BAD_TRANSPORT_VM) {
902	dev_err(&card->gdev->dev,
903	"The device does not support the configured transport mode\n");
904	return -EPROTONOSUPPORT;
905	}
906	return -EIO;
907	}
908	return 0;
909	}
910
911	static void qeth_release_buffer_cb(struct qeth_card *card,
912	struct qeth_cmd_buffer *iob,
913	unsigned int data_length)
914	{
915	qeth_put_cmd(iob);
916	}
917
918	static void qeth_cancel_cmd(struct qeth_cmd_buffer *iob, int rc)
919	{
920	qeth_notify_cmd(iob, reason: rc);
921	qeth_put_cmd(iob);
922	}
923
924	static struct qeth_cmd_buffer *qeth_alloc_cmd(struct qeth_channel *channel,
925	unsigned int length,
926	unsigned int ccws, long timeout)
927	{
928	struct qeth_cmd_buffer *iob;
929
930	if (length > QETH_BUFSIZE)
931	return NULL;
932
933	iob = kzalloc(sizeof(*iob), GFP_KERNEL);
934	if (!iob)
935	return NULL;
936
937	iob->data = kzalloc(ALIGN(length, 8) + ccws * sizeof(struct ccw1),
938	GFP_KERNEL | GFP_DMA);
939	if (!iob->data) {
940	kfree(objp: iob);
941	return NULL;
942	}
943
944	init_completion(x: &iob->done);
945	spin_lock_init(&iob->lock);
946	refcount_set(r: &iob->ref_count, n: 1);
947	iob->channel = channel;
948	iob->timeout = timeout;
949	iob->length = length;
950	return iob;
951	}
952
953	static void qeth_issue_next_read_cb(struct qeth_card *card,
954	struct qeth_cmd_buffer *iob,
955	unsigned int data_length)
956	{
957	struct qeth_cmd_buffer *request = NULL;
958	struct qeth_ipa_cmd *cmd = NULL;
959	struct qeth_reply *reply = NULL;
960	struct qeth_cmd_buffer *tmp;
961	unsigned long flags;
962	int rc = 0;
963
964	QETH_CARD_TEXT(card, 4, "sndctlcb");
965	rc = qeth_check_idx_response(card, buffer: iob->data);
966	switch (rc) {
967	case 0:
968	break;
969	case -EIO:
970	qeth_schedule_recovery(card);
971	fallthrough;
972	default:
973	qeth_clear_ipacmd_list(card);
974	goto err_idx;
975	}
976
977	cmd = __ipa_reply(iob);
978	if (cmd) {
979	cmd = qeth_check_ipa_data(card, cmd);
980	if (!cmd)
981	goto out;
982	}
983
984	/* match against pending cmd requests */
985	spin_lock_irqsave(&card->lock, flags);
986	list_for_each_entry(tmp, &card->cmd_waiter_list, list_entry) {
987	if (tmp->match && tmp->match(tmp, iob)) {
988	request = tmp;
989	/* take the object outside the lock */
990	qeth_get_cmd(iob: request);
991	break;
992	}
993	}
994	spin_unlock_irqrestore(lock: &card->lock, flags);
995
996	if (!request)
997	goto out;
998
999	reply = &request->reply;
1000	if (!reply->callback) {
1001	rc = 0;
1002	goto no_callback;
1003	}
1004
1005	spin_lock_irqsave(&request->lock, flags);
1006	if (request->rc)
1007	/* Bail out when the requestor has already left: */
1008	rc = request->rc;
1009	else
1010	rc = reply->callback(card, reply, cmd ? (unsigned long)cmd :
1011	(unsigned long)iob);
1012	spin_unlock_irqrestore(lock: &request->lock, flags);
1013
1014	no_callback:
1015	if (rc <= 0)
1016	qeth_notify_cmd(iob: request, reason: rc);
1017	qeth_put_cmd(iob: request);
1018	out:
1019	memcpy(&card->seqno.pdu_hdr_ack,
1020	QETH_PDU_HEADER_SEQ_NO(iob->data),
1021	QETH_SEQ_NO_LENGTH);
1022	__qeth_issue_next_read(card);
1023	err_idx:
1024	qeth_put_cmd(iob);
1025	}
1026
1027	static int qeth_set_thread_start_bit(struct qeth_card *card,
1028	unsigned long thread)
1029	{
1030	unsigned long flags;
1031	int rc = 0;
1032
1033	spin_lock_irqsave(&card->thread_mask_lock, flags);
1034	if (!(card->thread_allowed_mask & thread))
1035	rc = -EPERM;
1036	else if (card->thread_start_mask & thread)
1037	rc = -EBUSY;
1038	else
1039	card->thread_start_mask |= thread;
1040	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
1041
1042	return rc;
1043	}
1044
1045	static void qeth_clear_thread_start_bit(struct qeth_card *card,
1046	unsigned long thread)
1047	{
1048	unsigned long flags;
1049
1050	spin_lock_irqsave(&card->thread_mask_lock, flags);
1051	card->thread_start_mask &= ~thread;
1052	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
1053	wake_up(&card->wait_q);
1054	}
1055
1056	static void qeth_clear_thread_running_bit(struct qeth_card *card,
1057	unsigned long thread)
1058	{
1059	unsigned long flags;
1060
1061	spin_lock_irqsave(&card->thread_mask_lock, flags);
1062	card->thread_running_mask &= ~thread;
1063	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
1064	wake_up_all(&card->wait_q);
1065	}
1066
1067	static int __qeth_do_run_thread(struct qeth_card *card, unsigned long thread)
1068	{
1069	unsigned long flags;
1070	int rc = 0;
1071
1072	spin_lock_irqsave(&card->thread_mask_lock, flags);
1073	if (card->thread_start_mask & thread) {
1074	if ((card->thread_allowed_mask & thread) &&
1075	!(card->thread_running_mask & thread)) {
1076	rc = 1;
1077	card->thread_start_mask &= ~thread;
1078	card->thread_running_mask |= thread;
1079	} else
1080	rc = -EPERM;
1081	}
1082	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
1083	return rc;
1084	}
1085
1086	static int qeth_do_run_thread(struct qeth_card *card, unsigned long thread)
1087	{
1088	int rc = 0;
1089
1090	wait_event(card->wait_q,
1091	(rc = __qeth_do_run_thread(card, thread)) >= 0);
1092	return rc;
1093	}
1094
1095	int qeth_schedule_recovery(struct qeth_card *card)
1096	{
1097	int rc;
1098
1099	QETH_CARD_TEXT(card, 2, "startrec");
1100
1101	rc = qeth_set_thread_start_bit(card, thread: QETH_RECOVER_THREAD);
1102	if (!rc)
1103	schedule_work(work: &card->kernel_thread_starter);
1104
1105	return rc;
1106	}
1107
1108	static int qeth_get_problem(struct qeth_card *card, struct ccw_device *cdev,
1109	struct irb *irb)
1110	{
1111	int dstat, cstat;
1112	char *sense;
1113
1114	sense = (char *) irb->ecw;
1115	cstat = irb->scsw.cmd.cstat;
1116	dstat = irb->scsw.cmd.dstat;
1117
1118	if (cstat & (SCHN_STAT_CHN_CTRL_CHK | SCHN_STAT_INTF_CTRL_CHK |
1119	SCHN_STAT_CHN_DATA_CHK | SCHN_STAT_CHAIN_CHECK |
1120	SCHN_STAT_PROT_CHECK | SCHN_STAT_PROG_CHECK)) {
1121	QETH_CARD_TEXT(card, 2, "CGENCHK");
1122	dev_warn(&cdev->dev, "The qeth device driver "
1123	"failed to recover an error on the device\n");
1124	QETH_DBF_MESSAGE(2, "check on channel %x with dstat=%#x, cstat=%#x\n",
1125	CCW_DEVID(cdev), dstat, cstat);
1126	print_hex_dump(KERN_WARNING, prefix_str: "qeth: irb ", prefix_type: DUMP_PREFIX_OFFSET,
1127	rowsize: 16, groupsize: 1, buf: irb, len: 64, ascii: 1);
1128	return -EIO;
1129	}
1130
1131	if (dstat & DEV_STAT_UNIT_CHECK) {
1132	if (sense[SENSE_RESETTING_EVENT_BYTE] &
1133	SENSE_RESETTING_EVENT_FLAG) {
1134	QETH_CARD_TEXT(card, 2, "REVIND");
1135	return -EIO;
1136	}
1137	if (sense[SENSE_COMMAND_REJECT_BYTE] &
1138	SENSE_COMMAND_REJECT_FLAG) {
1139	QETH_CARD_TEXT(card, 2, "CMDREJi");
1140	return -EIO;
1141	}
1142	if ((sense[2] == 0xaf) && (sense[3] == 0xfe)) {
1143	QETH_CARD_TEXT(card, 2, "AFFE");
1144	return -EIO;
1145	}
1146	if ((!sense[0]) && (!sense[1]) && (!sense[2]) && (!sense[3])) {
1147	QETH_CARD_TEXT(card, 2, "ZEROSEN");
1148	return 0;
1149	}
1150	QETH_CARD_TEXT(card, 2, "DGENCHK");
1151	return -EIO;
1152	}
1153	return 0;
1154	}
1155
1156	static int qeth_check_irb_error(struct qeth_card *card, struct ccw_device *cdev,
1157	struct irb *irb)
1158	{
1159	if (!IS_ERR(ptr: irb))
1160	return 0;
1161
1162	switch (PTR_ERR(ptr: irb)) {
1163	case -EIO:
1164	QETH_DBF_MESSAGE(2, "i/o-error on channel %x\n",
1165	CCW_DEVID(cdev));
1166	QETH_CARD_TEXT(card, 2, "ckirberr");
1167	QETH_CARD_TEXT_(card, 2, " rc%d", -EIO);
1168	return -EIO;
1169	case -ETIMEDOUT:
1170	dev_warn(&cdev->dev, "A hardware operation timed out"
1171	" on the device\n");
1172	QETH_CARD_TEXT(card, 2, "ckirberr");
1173	QETH_CARD_TEXT_(card, 2, " rc%d", -ETIMEDOUT);
1174	return -ETIMEDOUT;
1175	default:
1176	QETH_DBF_MESSAGE(2, "unknown error %ld on channel %x\n",
1177	PTR_ERR(irb), CCW_DEVID(cdev));
1178	QETH_CARD_TEXT(card, 2, "ckirberr");
1179	QETH_CARD_TEXT(card, 2, " rc???");
1180	return PTR_ERR(ptr: irb);
1181	}
1182	}
1183
1184	/**
1185	* qeth_irq() - qeth interrupt handler
1186	* @cdev: ccw device
1187	* @intparm: expect pointer to iob
1188	* @irb: Interruption Response Block
1189	*
1190	* In the good path:
1191	* corresponding qeth channel is locked with last used iob as active_cmd.
1192	* But this function is also called for error interrupts.
1193	*
1194	* Caller ensures that:
1195	* Interrupts are disabled; ccw device lock is held;
1196	*
1197	*/
1198	static void qeth_irq(struct ccw_device *cdev, unsigned long intparm,
1199	struct irb *irb)
1200	{
1201	int rc;
1202	int cstat, dstat;
1203	struct qeth_cmd_buffer *iob = NULL;
1204	struct ccwgroup_device *gdev;
1205	struct qeth_channel *channel;
1206	struct qeth_card *card;
1207
1208	/* while we hold the ccwdev lock, this stays valid: */
1209	gdev = dev_get_drvdata(dev: &cdev->dev);
1210	card = dev_get_drvdata(dev: &gdev->dev);
1211
1212	QETH_CARD_TEXT(card, 5, "irq");
1213
1214	if (card->read.ccwdev == cdev) {
1215	channel = &card->read;
1216	QETH_CARD_TEXT(card, 5, "read");
1217	} else if (card->write.ccwdev == cdev) {
1218	channel = &card->write;
1219	QETH_CARD_TEXT(card, 5, "write");
1220	} else {
1221	channel = &card->data;
1222	QETH_CARD_TEXT(card, 5, "data");
1223	}
1224
1225	if (intparm == 0) {
1226	QETH_CARD_TEXT(card, 5, "irqunsol");
1227	} else if ((addr_t)intparm != (addr_t)channel->active_cmd) {
1228	QETH_CARD_TEXT(card, 5, "irqunexp");
1229
1230	dev_err(&cdev->dev,
1231	"Received IRQ with intparm %lx, expected %px\n",
1232	intparm, channel->active_cmd);
1233	if (channel->active_cmd)
1234	qeth_cancel_cmd(iob: channel->active_cmd, rc: -EIO);
1235	} else {
1236	iob = (struct qeth_cmd_buffer *) (addr_t)intparm;
1237	}
1238
1239	rc = qeth_check_irb_error(card, cdev, irb);
1240	if (rc) {
1241	/* IO was terminated, free its resources. */
1242	qeth_unlock_channel(card, channel);
1243	if (iob)
1244	qeth_cancel_cmd(iob, rc);
1245	return;
1246	}
1247
1248	if (irb->scsw.cmd.fctl & SCSW_FCTL_CLEAR_FUNC) {
1249	channel->state = CH_STATE_STOPPED;
1250	wake_up(&card->wait_q);
1251	}
1252
1253	if (irb->scsw.cmd.fctl & SCSW_FCTL_HALT_FUNC) {
1254	channel->state = CH_STATE_HALTED;
1255	wake_up(&card->wait_q);
1256	}
1257
1258	if (iob && (irb->scsw.cmd.fctl & (SCSW_FCTL_CLEAR_FUNC |
1259	SCSW_FCTL_HALT_FUNC))) {
1260	qeth_cancel_cmd(iob, rc: -ECANCELED);
1261	iob = NULL;
1262	}
1263
1264	cstat = irb->scsw.cmd.cstat;
1265	dstat = irb->scsw.cmd.dstat;
1266
1267	if ((dstat & DEV_STAT_UNIT_EXCEP) ||
1268	(dstat & DEV_STAT_UNIT_CHECK) ||
1269	(cstat)) {
1270	if (irb->esw.esw0.erw.cons) {
1271	dev_warn(&channel->ccwdev->dev,
1272	"The qeth device driver failed to recover "
1273	"an error on the device\n");
1274	QETH_DBF_MESSAGE(2, "sense data available on channel %x: cstat %#X dstat %#X\n",
1275	CCW_DEVID(channel->ccwdev), cstat,
1276	dstat);
1277	print_hex_dump(KERN_WARNING, prefix_str: "qeth: irb ",
1278	prefix_type: DUMP_PREFIX_OFFSET, rowsize: 16, groupsize: 1, buf: irb, len: 32, ascii: 1);
1279	print_hex_dump(KERN_WARNING, prefix_str: "qeth: sense data ",
1280	prefix_type: DUMP_PREFIX_OFFSET, rowsize: 16, groupsize: 1, buf: irb->ecw, len: 32, ascii: 1);
1281	}
1282
1283	rc = qeth_get_problem(card, cdev, irb);
1284	if (rc) {
1285	card->read_or_write_problem = 1;
1286	qeth_unlock_channel(card, channel);
1287	if (iob)
1288	qeth_cancel_cmd(iob, rc);
1289	qeth_clear_ipacmd_list(card);
1290	qeth_schedule_recovery(card);
1291	return;
1292	}
1293	}
1294
1295	if (scsw_cmd_is_valid_cc(&irb->scsw) && irb->scsw.cmd.cc == 1 && iob) {
1296	/* channel command hasn't started: retry.
1297	* active_cmd is still set to last iob
1298	*/
1299	QETH_CARD_TEXT(card, 2, "irqcc1");
1300	rc = ccw_device_start_timeout(cdev, __ccw_from_cmd(iob),
1301	(addr_t)iob, 0, 0, iob->timeout);
1302	if (rc) {
1303	QETH_DBF_MESSAGE(2,
1304	"ccw retry on %x failed, rc = %i\n",
1305	CARD_DEVID(card), rc);
1306	QETH_CARD_TEXT_(card, 2, " err%d", rc);
1307	qeth_unlock_channel(card, channel);
1308	qeth_cancel_cmd(iob, rc);
1309	}
1310	return;
1311	}
1312
1313	qeth_unlock_channel(card, channel);
1314
1315	if (iob) {
1316	/* sanity check: */
1317	if (irb->scsw.cmd.count > iob->length) {
1318	qeth_cancel_cmd(iob, rc: -EIO);
1319	return;
1320	}
1321	if (iob->callback)
1322	iob->callback(card, iob,
1323	iob->length - irb->scsw.cmd.count);
1324	}
1325	}
1326
1327	static void qeth_notify_skbs(struct qeth_qdio_out_q *q,
1328	struct qeth_qdio_out_buffer *buf,
1329	enum iucv_tx_notify notification)
1330	{
1331	struct sk_buff *skb;
1332
1333	skb_queue_walk(&buf->skb_list, skb) {
1334	struct sock *sk = skb->sk;
1335
1336	QETH_CARD_TEXT_(q->card, 5, "skbn%d", notification);
1337	QETH_CARD_TEXT_(q->card, 5, "%lx", (long) skb);
1338	if (sk && sk->sk_family == PF_IUCV)
1339	iucv_sk(sk)->sk_txnotify(sk, notification);
1340	}
1341	}
1342
1343	static void qeth_tx_complete_buf(struct qeth_qdio_out_q *queue,
1344	struct qeth_qdio_out_buffer *buf, bool error,
1345	int budget)
1346	{
1347	struct sk_buff *skb;
1348
1349	/* Empty buffer? */
1350	if (buf->next_element_to_fill == 0)
1351	return;
1352
1353	QETH_TXQ_STAT_INC(queue, bufs);
1354	QETH_TXQ_STAT_ADD(queue, buf_elements, buf->next_element_to_fill);
1355	if (error) {
1356	QETH_TXQ_STAT_ADD(queue, tx_errors, buf->frames);
1357	} else {
1358	QETH_TXQ_STAT_ADD(queue, tx_packets, buf->frames);
1359	QETH_TXQ_STAT_ADD(queue, tx_bytes, buf->bytes);
1360	}
1361
1362	while ((skb = __skb_dequeue(list: &buf->skb_list)) != NULL) {
1363	unsigned int bytes = qdisc_pkt_len(skb);
1364	bool is_tso = skb_is_gso(skb);
1365	unsigned int packets;
1366
1367	packets = is_tso ? skb_shinfo(skb)->gso_segs : 1;
1368	if (!error) {
1369	if (skb->ip_summed == CHECKSUM_PARTIAL)
1370	QETH_TXQ_STAT_ADD(queue, skbs_csum, packets);
1371	if (skb_is_nonlinear(skb))
1372	QETH_TXQ_STAT_INC(queue, skbs_sg);
1373	if (is_tso) {
1374	QETH_TXQ_STAT_INC(queue, skbs_tso);
1375	QETH_TXQ_STAT_ADD(queue, tso_bytes, bytes);
1376	}
1377	}
1378
1379	napi_consume_skb(skb, budget);
1380	}
1381	}
1382
1383	static void qeth_clear_output_buffer(struct qeth_qdio_out_q *queue,
1384	struct qeth_qdio_out_buffer *buf,
1385	bool error, int budget)
1386	{
1387	int i;
1388
1389	/* is PCI flag set on buffer? */
1390	if (buf->buffer->element[0].sflags & SBAL_SFLAGS0_PCI_REQ) {
1391	atomic_dec(v: &queue->set_pci_flags_count);
1392	QETH_TXQ_STAT_INC(queue, completion_irq);
1393	}
1394
1395	qeth_tx_complete_buf(queue, buf, error, budget);
1396
1397	for (i = 0; i < queue->max_elements; ++i) {
1398	void *data = dma64_to_virt(buf->buffer->element[i].addr);
1399
1400	if (__test_and_clear_bit(i, buf->from_kmem_cache) && data)
1401	kmem_cache_free(s: qeth_core_header_cache, objp: data);
1402	}
1403
1404	qeth_scrub_qdio_buffer(buf: buf->buffer, elements: queue->max_elements);
1405	buf->next_element_to_fill = 0;
1406	buf->frames = 0;
1407	buf->bytes = 0;
1408	atomic_set(v: &buf->state, i: QETH_QDIO_BUF_EMPTY);
1409	}
1410
1411	static void qeth_free_out_buf(struct qeth_qdio_out_buffer *buf)
1412	{
1413	if (buf->aob)
1414	kmem_cache_free(s: qeth_qaob_cache, objp: buf->aob);
1415	kmem_cache_free(s: qeth_qdio_outbuf_cache, objp: buf);
1416	}
1417
1418	static void qeth_tx_complete_pending_bufs(struct qeth_card *card,
1419	struct qeth_qdio_out_q *queue,
1420	bool drain, int budget)
1421	{
1422	struct qeth_qdio_out_buffer *buf, *tmp;
1423
1424	list_for_each_entry_safe(buf, tmp, &queue->pending_bufs, list_entry) {
1425	struct qeth_qaob_priv1 *priv;
1426	struct qaob *aob = buf->aob;
1427	enum iucv_tx_notify notify;
1428	unsigned int i;
1429
1430	priv = (struct qeth_qaob_priv1 *)&aob->user1;
1431	if (drain || READ_ONCE(priv->state) == QETH_QAOB_DONE) {
1432	QETH_CARD_TEXT(card, 5, "fp");
1433	QETH_CARD_TEXT_(card, 5, "%lx", (long) buf);
1434
1435	notify = drain ? TX_NOTIFY_GENERALERROR :
1436	qeth_compute_cq_notification(sbalf15: aob->aorc, delayed: 1);
1437	qeth_notify_skbs(q: queue, buf, notification: notify);
1438	qeth_tx_complete_buf(queue, buf, error: drain, budget);
1439
1440	for (i = 0;
1441	i < aob->sb_count && i < queue->max_elements;
1442	i++) {
1443	void *data = dma64_to_virt(aob->sba[i]);
1444
1445	if (test_bit(i, buf->from_kmem_cache) && data)
1446	kmem_cache_free(s: qeth_core_header_cache,
1447	objp: data);
1448	}
1449
1450	list_del(entry: &buf->list_entry);
1451	qeth_free_out_buf(buf);
1452	}
1453	}
1454	}
1455
1456	static void qeth_drain_output_queue(struct qeth_qdio_out_q *q, bool free)
1457	{
1458	int j;
1459
1460	qeth_tx_complete_pending_bufs(card: q->card, queue: q, drain: true, budget: 0);
1461
1462	for (j = 0; j < QDIO_MAX_BUFFERS_PER_Q; ++j) {
1463	if (!q->bufs[j])
1464	continue;
1465
1466	qeth_clear_output_buffer(q, q->bufs[j], true, 0);
1467	if (free) {
1468	qeth_free_out_buf(q->bufs[j]);
1469	q->bufs[j] = NULL;
1470	}
1471	}
1472	}
1473
1474	static void qeth_drain_output_queues(struct qeth_card *card)
1475	{
1476	int i;
1477
1478	QETH_CARD_TEXT(card, 2, "clearqdbf");
1479	/* clear outbound buffers to free skbs */
1480	for (i = 0; i < card->qdio.no_out_queues; ++i) {
1481	if (card->qdio.out_qs[i])
1482	qeth_drain_output_queue(q: card->qdio.out_qs[i], free: false);
1483	}
1484	}
1485
1486	static void qeth_osa_set_output_queues(struct qeth_card *card, bool single)
1487	{
1488	unsigned int max = single ? 1 : card->dev->num_tx_queues;
1489
1490	if (card->qdio.no_out_queues == max)
1491	return;
1492
1493	if (atomic_read(v: &card->qdio.state) != QETH_QDIO_UNINITIALIZED)
1494	qeth_free_qdio_queues(card);
1495
1496	if (max == 1 && card->qdio.do_prio_queueing != QETH_PRIOQ_DEFAULT)
1497	dev_info(&card->gdev->dev, "Priority Queueing not supported\n");
1498
1499	card->qdio.no_out_queues = max;
1500	}
1501
1502	static int qeth_update_from_chp_desc(struct qeth_card *card)
1503	{
1504	struct ccw_device *ccwdev;
1505	struct channel_path_desc_fmt0 *chp_dsc;
1506
1507	QETH_CARD_TEXT(card, 2, "chp_desc");
1508
1509	ccwdev = card->data.ccwdev;
1510	chp_dsc = ccw_device_get_chp_desc(ccwdev, 0);
1511	if (!chp_dsc)
1512	return -ENOMEM;
1513
1514	card->info.func_level = 0x4100 + chp_dsc->desc;
1515
1516	if (IS_OSD(card) || IS_OSX(card))
1517	/* CHPP field bit 6 == 1 -> single queue */
1518	qeth_osa_set_output_queues(card, single: chp_dsc->chpp & 0x02);
1519
1520	kfree(objp: chp_dsc);
1521	QETH_CARD_TEXT_(card, 2, "nr:%x", card->qdio.no_out_queues);
1522	QETH_CARD_TEXT_(card, 2, "lvl:%02x", card->info.func_level);
1523	return 0;
1524	}
1525
1526	static void qeth_init_qdio_info(struct qeth_card *card)
1527	{
1528	QETH_CARD_TEXT(card, 4, "intqdinf");
1529	atomic_set(v: &card->qdio.state, i: QETH_QDIO_UNINITIALIZED);
1530	card->qdio.do_prio_queueing = QETH_PRIOQ_DEFAULT;
1531	card->qdio.default_out_queue = QETH_DEFAULT_QUEUE;
1532
1533	/* inbound */
1534	card->qdio.in_buf_size = QETH_IN_BUF_SIZE_DEFAULT;
1535	if (IS_IQD(card))
1536	card->qdio.init_pool.buf_count = QETH_IN_BUF_COUNT_HSDEFAULT;
1537	else
1538	card->qdio.init_pool.buf_count = QETH_IN_BUF_COUNT_DEFAULT;
1539	card->qdio.in_buf_pool.buf_count = card->qdio.init_pool.buf_count;
1540	INIT_LIST_HEAD(list: &card->qdio.in_buf_pool.entry_list);
1541	INIT_LIST_HEAD(list: &card->qdio.init_pool.entry_list);
1542	}
1543
1544	static void qeth_set_initial_options(struct qeth_card *card)
1545	{
1546	card->options.route4.type = NO_ROUTER;
1547	card->options.route6.type = NO_ROUTER;
1548	card->options.isolation = ISOLATION_MODE_NONE;
1549	card->options.cq = QETH_CQ_DISABLED;
1550	card->options.layer = QETH_DISCIPLINE_UNDETERMINED;
1551	}
1552
1553	static int qeth_do_start_thread(struct qeth_card *card, unsigned long thread)
1554	{
1555	unsigned long flags;
1556	int rc = 0;
1557
1558	spin_lock_irqsave(&card->thread_mask_lock, flags);
1559	QETH_CARD_TEXT_(card, 4, " %02x%02x%02x",
1560	(u8) card->thread_start_mask,
1561	(u8) card->thread_allowed_mask,
1562	(u8) card->thread_running_mask);
1563	rc = (card->thread_start_mask & thread);
1564	spin_unlock_irqrestore(lock: &card->thread_mask_lock, flags);
1565	return rc;
1566	}
1567
1568	static int qeth_do_reset(void *data);
1569	static void qeth_start_kernel_thread(struct work_struct *work)
1570	{
1571	struct task_struct *ts;
1572	struct qeth_card *card = container_of(work, struct qeth_card,
1573	kernel_thread_starter);
1574	QETH_CARD_TEXT(card, 2, "strthrd");
1575
1576	if (card->read.state != CH_STATE_UP &&
1577	card->write.state != CH_STATE_UP)
1578	return;
1579	if (qeth_do_start_thread(card, thread: QETH_RECOVER_THREAD)) {
1580	ts = kthread_run(qeth_do_reset, card, "qeth_recover");
1581	if (IS_ERR(ptr: ts)) {
1582	qeth_clear_thread_start_bit(card, thread: QETH_RECOVER_THREAD);
1583	qeth_clear_thread_running_bit(card,
1584	thread: QETH_RECOVER_THREAD);
1585	}
1586	}
1587	}
1588
1589	static void qeth_buffer_reclaim_work(struct work_struct *);
1590	static void qeth_setup_card(struct qeth_card *card)
1591	{
1592	QETH_CARD_TEXT(card, 2, "setupcrd");
1593
1594	card->info.type = CARD_RDEV(card)->id.driver_info;
1595	card->state = CARD_STATE_DOWN;
1596	spin_lock_init(&card->lock);
1597	spin_lock_init(&card->thread_mask_lock);
1598	mutex_init(&card->conf_mutex);
1599	mutex_init(&card->discipline_mutex);
1600	INIT_WORK(&card->kernel_thread_starter, qeth_start_kernel_thread);
1601	INIT_LIST_HEAD(list: &card->cmd_waiter_list);
1602	init_waitqueue_head(&card->wait_q);
1603	qeth_set_initial_options(card);
1604	/* IP address takeover */
1605	INIT_LIST_HEAD(list: &card->ipato.entries);
1606	qeth_init_qdio_info(card);
1607	INIT_DELAYED_WORK(&card->buffer_reclaim_work, qeth_buffer_reclaim_work);
1608	hash_init(card->rx_mode_addrs);
1609	hash_init(card->local_addrs4);
1610	hash_init(card->local_addrs6);
1611	spin_lock_init(&card->local_addrs4_lock);
1612	spin_lock_init(&card->local_addrs6_lock);
1613	}
1614
1615	static void qeth_core_sl_print(struct seq_file *m, struct service_level *slr)
1616	{
1617	struct qeth_card *card = container_of(slr, struct qeth_card,
1618	qeth_service_level);
1619	if (card->info.mcl_level[0])
1620	seq_printf(m, fmt: "qeth: %s firmware level %s\n",
1621	CARD_BUS_ID(card), card->info.mcl_level);
1622	}
1623
1624	static struct qeth_card *qeth_alloc_card(struct ccwgroup_device *gdev)
1625	{
1626	struct qeth_card *card;
1627
1628	QETH_DBF_TEXT(SETUP, 2, "alloccrd");
1629	card = kzalloc(sizeof(*card), GFP_KERNEL);
1630	if (!card)
1631	goto out;
1632	QETH_DBF_HEX(SETUP, 2, &card, sizeof(void *));
1633
1634	card->gdev = gdev;
1635	dev_set_drvdata(dev: &gdev->dev, data: card);
1636	CARD_RDEV(card) = gdev->cdev[0];
1637	CARD_WDEV(card) = gdev->cdev[1];
1638	CARD_DDEV(card) = gdev->cdev[2];
1639
1640	card->event_wq = alloc_ordered_workqueue("%s_event", 0,
1641	dev_name(&gdev->dev));
1642	if (!card->event_wq)
1643	goto out_wq;
1644
1645	card->read_cmd = qeth_alloc_cmd(channel: &card->read, QETH_BUFSIZE, ccws: 1, timeout: 0);
1646	if (!card->read_cmd)
1647	goto out_read_cmd;
1648
1649	card->debugfs = debugfs_create_dir(name: dev_name(dev: &gdev->dev),
1650	parent: qeth_debugfs_root);
1651	debugfs_create_file("local_addrs", 0400, card->debugfs, card,
1652	&qeth_debugfs_local_addr_fops);
1653
1654	card->qeth_service_level.seq_print = qeth_core_sl_print;
1655	register_service_level(&card->qeth_service_level);
1656	return card;
1657
1658	out_read_cmd:
1659	destroy_workqueue(wq: card->event_wq);
1660	out_wq:
1661	dev_set_drvdata(dev: &gdev->dev, NULL);
1662	kfree(objp: card);
1663	out:
1664	return NULL;
1665	}
1666
1667	static int qeth_clear_channel(struct qeth_card *card,
1668	struct qeth_channel *channel)
1669	{
1670	int rc;
1671
1672	QETH_CARD_TEXT(card, 3, "clearch");
1673	spin_lock_irq(lock: get_ccwdev_lock(channel->ccwdev));
1674	rc = ccw_device_clear(channel->ccwdev, (addr_t)channel->active_cmd);
1675	spin_unlock_irq(lock: get_ccwdev_lock(channel->ccwdev));
1676
1677	if (rc)
1678	return rc;
1679	rc = wait_event_interruptible_timeout(card->wait_q,
1680	channel->state == CH_STATE_STOPPED, QETH_TIMEOUT);
1681	if (rc == -ERESTARTSYS)
1682	return rc;
1683	if (channel->state != CH_STATE_STOPPED)
1684	return -ETIME;
1685	channel->state = CH_STATE_DOWN;
1686	return 0;
1687	}
1688
1689	static int qeth_halt_channel(struct qeth_card *card,
1690	struct qeth_channel *channel)
1691	{
1692	int rc;
1693
1694	QETH_CARD_TEXT(card, 3, "haltch");
1695	spin_lock_irq(lock: get_ccwdev_lock(channel->ccwdev));
1696	rc = ccw_device_halt(channel->ccwdev, (addr_t)channel->active_cmd);
1697	spin_unlock_irq(lock: get_ccwdev_lock(channel->ccwdev));
1698
1699	if (rc)
1700	return rc;
1701	rc = wait_event_interruptible_timeout(card->wait_q,
1702	channel->state == CH_STATE_HALTED, QETH_TIMEOUT);
1703	if (rc == -ERESTARTSYS)
1704	return rc;
1705	if (channel->state != CH_STATE_HALTED)
1706	return -ETIME;
1707	return 0;
1708	}
1709
1710	static int qeth_stop_channel(struct qeth_channel *channel)
1711	{
1712	struct ccw_device *cdev = channel->ccwdev;
1713	int rc;
1714
1715	rc = ccw_device_set_offline(cdev);
1716
1717	spin_lock_irq(lock: get_ccwdev_lock(cdev));
1718	if (channel->active_cmd)
1719	dev_err(&cdev->dev, "Stopped channel while cmd %px was still active\n",
1720	channel->active_cmd);
1721
1722	cdev->handler = NULL;
1723	spin_unlock_irq(lock: get_ccwdev_lock(cdev));
1724
1725	return rc;
1726	}
1727
1728	static int qeth_start_channel(struct qeth_channel *channel)
1729	{
1730	struct ccw_device *cdev = channel->ccwdev;
1731	int rc;
1732
1733	channel->state = CH_STATE_DOWN;
1734	xchg(&channel->active_cmd, NULL);
1735
1736	spin_lock_irq(lock: get_ccwdev_lock(cdev));
1737	cdev->handler = qeth_irq;
1738	spin_unlock_irq(lock: get_ccwdev_lock(cdev));
1739
1740	rc = ccw_device_set_online(cdev);
1741	if (rc)
1742	goto err;
1743
1744	return 0;
1745
1746	err:
1747	spin_lock_irq(lock: get_ccwdev_lock(cdev));
1748	cdev->handler = NULL;
1749	spin_unlock_irq(lock: get_ccwdev_lock(cdev));
1750	return rc;
1751	}
1752
1753	static int qeth_halt_channels(struct qeth_card *card)
1754	{
1755	int rc1 = 0, rc2 = 0, rc3 = 0;
1756
1757	QETH_CARD_TEXT(card, 3, "haltchs");
1758	rc1 = qeth_halt_channel(card, channel: &card->read);
1759	rc2 = qeth_halt_channel(card, channel: &card->write);
1760	rc3 = qeth_halt_channel(card, channel: &card->data);
1761	if (rc1)
1762	return rc1;
1763	if (rc2)
1764	return rc2;
1765	return rc3;
1766	}
1767
1768	static int qeth_clear_channels(struct qeth_card *card)
1769	{
1770	int rc1 = 0, rc2 = 0, rc3 = 0;
1771
1772	QETH_CARD_TEXT(card, 3, "clearchs");
1773	rc1 = qeth_clear_channel(card, channel: &card->read);
1774	rc2 = qeth_clear_channel(card, channel: &card->write);
1775	rc3 = qeth_clear_channel(card, channel: &card->data);
1776	if (rc1)
1777	return rc1;
1778	if (rc2)
1779	return rc2;
1780	return rc3;
1781	}
1782
1783	static int qeth_clear_halt_card(struct qeth_card *card, int halt)
1784	{
1785	int rc = 0;
1786
1787	QETH_CARD_TEXT(card, 3, "clhacrd");
1788
1789	if (halt)
1790	rc = qeth_halt_channels(card);
1791	if (rc)
1792	return rc;
1793	return qeth_clear_channels(card);
1794	}
1795
1796	static int qeth_qdio_clear_card(struct qeth_card *card, int use_halt)
1797	{
1798	int rc = 0;
1799
1800	QETH_CARD_TEXT(card, 3, "qdioclr");
1801	switch (atomic_cmpxchg(v: &card->qdio.state, old: QETH_QDIO_ESTABLISHED,
1802	new: QETH_QDIO_CLEANING)) {
1803	case QETH_QDIO_ESTABLISHED:
1804	if (IS_IQD(card))
1805	rc = qdio_shutdown(CARD_DDEV(card),
1806	QDIO_FLAG_CLEANUP_USING_HALT);
1807	else
1808	rc = qdio_shutdown(CARD_DDEV(card),
1809	QDIO_FLAG_CLEANUP_USING_CLEAR);
1810	if (rc)
1811	QETH_CARD_TEXT_(card, 3, "1err%d", rc);
1812	atomic_set(v: &card->qdio.state, i: QETH_QDIO_ALLOCATED);
1813	break;
1814	case QETH_QDIO_CLEANING:
1815	return rc;
1816	default:
1817	break;
1818	}
1819	rc = qeth_clear_halt_card(card, halt: use_halt);
1820	if (rc)
1821	QETH_CARD_TEXT_(card, 3, "2err%d", rc);
1822	return rc;
1823	}
1824
1825	static enum qeth_discipline_id qeth_vm_detect_layer(struct qeth_card *card)
1826	{
1827	enum qeth_discipline_id disc = QETH_DISCIPLINE_UNDETERMINED;
1828	struct diag26c_vnic_resp *response = NULL;
1829	struct diag26c_vnic_req *request = NULL;
1830	struct ccw_dev_id id;
1831	char userid[80];
1832	int rc = 0;
1833
1834	QETH_CARD_TEXT(card, 2, "vmlayer");
1835
1836	cpcmd("QUERY USERID", userid, sizeof(userid), &rc);
1837	if (rc)
1838	goto out;
1839
1840	request = kzalloc(sizeof(*request), GFP_KERNEL | GFP_DMA);
1841	response = kzalloc(sizeof(*response), GFP_KERNEL | GFP_DMA);
1842	if (!request || !response) {
1843	rc = -ENOMEM;
1844	goto out;
1845	}
1846
1847	ccw_device_get_id(CARD_RDEV(card), &id);
1848	request->resp_buf_len = sizeof(*response);
1849	request->resp_version = DIAG26C_VERSION6_VM65918;
1850	request->req_format = DIAG26C_VNIC_INFO;
1851	ASCEBC(userid, 8);
1852	memcpy(&request->sys_name, userid, 8);
1853	request->devno = id.devno;
1854
1855	QETH_DBF_HEX(CTRL, 2, request, sizeof(*request));
1856	rc = diag26c(request, response, DIAG26C_PORT_VNIC);
1857	QETH_DBF_HEX(CTRL, 2, request, sizeof(*request));
1858	if (rc)
1859	goto out;
1860	QETH_DBF_HEX(CTRL, 2, response, sizeof(*response));
1861
1862	if (request->resp_buf_len < sizeof(*response) ||
1863	response->version != request->resp_version) {
1864	rc = -EIO;
1865	goto out;
1866	}
1867
1868	if (response->protocol == VNIC_INFO_PROT_L2)
1869	disc = QETH_DISCIPLINE_LAYER2;
1870	else if (response->protocol == VNIC_INFO_PROT_L3)
1871	disc = QETH_DISCIPLINE_LAYER3;
1872
1873	out:
1874	kfree(objp: response);
1875	kfree(objp: request);
1876	if (rc)
1877	QETH_CARD_TEXT_(card, 2, "err%x", rc);
1878	return disc;
1879	}
1880
1881	/* Determine whether the device requires a specific layer discipline */
1882	static enum qeth_discipline_id qeth_enforce_discipline(struct qeth_card *card)
1883	{
1884	enum qeth_discipline_id disc = QETH_DISCIPLINE_UNDETERMINED;
1885
1886	if (IS_OSM(card))
1887	disc = QETH_DISCIPLINE_LAYER2;
1888	else if (IS_VM_NIC(card))
1889	disc = IS_IQD(card) ? QETH_DISCIPLINE_LAYER3 :
1890	qeth_vm_detect_layer(card);
1891
1892	switch (disc) {
1893	case QETH_DISCIPLINE_LAYER2:
1894	QETH_CARD_TEXT(card, 3, "force l2");
1895	break;
1896	case QETH_DISCIPLINE_LAYER3:
1897	QETH_CARD_TEXT(card, 3, "force l3");
1898	break;
1899	default:
1900	QETH_CARD_TEXT(card, 3, "force no");
1901	}
1902
1903	return disc;
1904	}
1905
1906	static void qeth_set_blkt_defaults(struct qeth_card *card)
1907	{
1908	QETH_CARD_TEXT(card, 2, "cfgblkt");
1909
1910	if (card->info.use_v1_blkt) {
1911	card->info.blkt.time_total = 0;
1912	card->info.blkt.inter_packet = 0;
1913	card->info.blkt.inter_packet_jumbo = 0;
1914	} else {
1915	card->info.blkt.time_total = 250;
1916	card->info.blkt.inter_packet = 5;
1917	card->info.blkt.inter_packet_jumbo = 15;
1918	}
1919	}
1920
1921	static void qeth_idx_init(struct qeth_card *card)
1922	{
1923	memset(&card->seqno, 0, sizeof(card->seqno));
1924
1925	card->token.issuer_rm_w = 0x00010103UL;
1926	card->token.cm_filter_w = 0x00010108UL;
1927	card->token.cm_connection_w = 0x0001010aUL;
1928	card->token.ulp_filter_w = 0x0001010bUL;
1929	card->token.ulp_connection_w = 0x0001010dUL;
1930
1931	switch (card->info.type) {
1932	case QETH_CARD_TYPE_IQD:
1933	card->info.func_level = QETH_IDX_FUNC_LEVEL_IQD;
1934	break;
1935	case QETH_CARD_TYPE_OSD:
1936	card->info.func_level = QETH_IDX_FUNC_LEVEL_OSD;
1937	break;
1938	default:
1939	break;
1940	}
1941	}
1942
1943	static void qeth_idx_finalize_cmd(struct qeth_card *card,
1944	struct qeth_cmd_buffer *iob)
1945	{
1946	memcpy(QETH_TRANSPORT_HEADER_SEQ_NO(iob->data), &card->seqno.trans_hdr,
1947	QETH_SEQ_NO_LENGTH);
1948	if (iob->channel == &card->write)
1949	card->seqno.trans_hdr++;
1950	}
1951
1952	static int qeth_peer_func_level(int level)
1953	{
1954	if ((level & 0xff) == 8)
1955	return (level & 0xff) + 0x400;
1956	if (((level >> 8) & 3) == 1)
1957	return (level & 0xff) + 0x200;
1958	return level;
1959	}
1960
1961	static void qeth_mpc_finalize_cmd(struct qeth_card *card,
1962	struct qeth_cmd_buffer *iob)
1963	{
1964	qeth_idx_finalize_cmd(card, iob);
1965
1966	memcpy(QETH_PDU_HEADER_SEQ_NO(iob->data),
1967	&card->seqno.pdu_hdr, QETH_SEQ_NO_LENGTH);
1968	card->seqno.pdu_hdr++;
1969	memcpy(QETH_PDU_HEADER_ACK_SEQ_NO(iob->data),
1970	&card->seqno.pdu_hdr_ack, QETH_SEQ_NO_LENGTH);
1971
1972	iob->callback = qeth_release_buffer_cb;
1973	}
1974
1975	static bool qeth_mpc_match_reply(struct qeth_cmd_buffer *iob,
1976	struct qeth_cmd_buffer *reply)
1977	{
1978	/* MPC cmds are issued strictly in sequence. */
1979	return !IS_IPA(reply->data);
1980	}
1981
1982	static struct qeth_cmd_buffer *qeth_mpc_alloc_cmd(struct qeth_card *card,
1983	const void *data,
1984	unsigned int data_length)
1985	{
1986	struct qeth_cmd_buffer *iob;
1987
1988	iob = qeth_alloc_cmd(channel: &card->write, length: data_length, ccws: 1, QETH_TIMEOUT);
1989	if (!iob)
1990	return NULL;
1991
1992	memcpy(iob->data, data, data_length);
1993	qeth_setup_ccw(ccw: __ccw_from_cmd(iob), CCW_CMD_WRITE, flags: 0, len: data_length,
1994	data: iob->data);
1995	iob->finalize = qeth_mpc_finalize_cmd;
1996	iob->match = qeth_mpc_match_reply;
1997	return iob;
1998	}
1999
2000	/**
2001	* qeth_send_control_data() - send control command to the card
2002	* @card: qeth_card structure pointer
2003	* @iob: qeth_cmd_buffer pointer
2004	* @reply_cb: callback function pointer
2005	* cb_card: pointer to the qeth_card structure
2006	* cb_reply: pointer to the qeth_reply structure
2007	* cb_cmd: pointer to the original iob for non-IPA
2008	* commands, or to the qeth_ipa_cmd structure
2009	* for the IPA commands.
2010	* @reply_param: private pointer passed to the callback
2011	*
2012	* Callback function gets called one or more times, with cb_cmd
2013	* pointing to the response returned by the hardware. Callback
2014	* function must return
2015	* > 0 if more reply blocks are expected,
2016	* 0 if the last or only reply block is received, and
2017	* < 0 on error.
2018	* Callback function can get the value of the reply_param pointer from the
2019	* field 'param' of the structure qeth_reply.
2020	*/
2021
2022	static int qeth_send_control_data(struct qeth_card *card,
2023	struct qeth_cmd_buffer *iob,
2024	int (*reply_cb)(struct qeth_card *cb_card,
2025	struct qeth_reply *cb_reply,
2026	unsigned long cb_cmd),
2027	void *reply_param)
2028	{
2029	struct qeth_channel *channel = iob->channel;
2030	struct qeth_reply *reply = &iob->reply;
2031	long timeout = iob->timeout;
2032	int rc;
2033
2034	QETH_CARD_TEXT(card, 2, "sendctl");
2035
2036	reply->callback = reply_cb;
2037	reply->param = reply_param;
2038
2039	timeout = wait_event_interruptible_timeout(card->wait_q,
2040	qeth_trylock_channel(channel, iob),
2041	timeout);
2042	if (timeout <= 0) {
2043	qeth_put_cmd(iob);
2044	return (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
2045	}
2046
2047	if (iob->finalize)
2048	iob->finalize(card, iob);
2049	QETH_DBF_HEX(CTRL, 2, iob->data, min(iob->length, QETH_DBF_CTRL_LEN));
2050
2051	qeth_enqueue_cmd(card, iob);
2052
2053	/* This pairs with iob->callback, and keeps the iob alive after IO: */
2054	qeth_get_cmd(iob);
2055
2056	QETH_CARD_TEXT(card, 6, "noirqpnd");
2057	spin_lock_irq(lock: get_ccwdev_lock(channel->ccwdev));
2058	rc = ccw_device_start_timeout(channel->ccwdev, __ccw_from_cmd(iob),
2059	(addr_t) iob, 0, 0, timeout);
2060	spin_unlock_irq(lock: get_ccwdev_lock(channel->ccwdev));
2061	if (rc) {
2062	QETH_DBF_MESSAGE(2, "qeth_send_control_data on device %x: ccw_device_start rc = %i\n",
2063	CARD_DEVID(card), rc);
2064	QETH_CARD_TEXT_(card, 2, " err%d", rc);
2065	qeth_dequeue_cmd(card, iob);
2066	qeth_put_cmd(iob);
2067	qeth_unlock_channel(card, channel);
2068	goto out;
2069	}
2070
2071	timeout = wait_for_completion_interruptible_timeout(x: &iob->done,
2072	timeout);
2073	if (timeout <= 0)
2074	rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
2075
2076	qeth_dequeue_cmd(card, iob);
2077
2078	if (reply_cb) {
2079	/* Wait until the callback for a late reply has completed: */
2080	spin_lock_irq(lock: &iob->lock);
2081	if (rc)
2082	/* Zap any callback that's still pending: */
2083	iob->rc = rc;
2084	spin_unlock_irq(lock: &iob->lock);
2085	}
2086
2087	if (!rc)
2088	rc = iob->rc;
2089
2090	out:
2091	qeth_put_cmd(iob);
2092	return rc;
2093	}
2094
2095	struct qeth_node_desc {
2096	struct node_descriptor nd1;
2097	struct node_descriptor nd2;
2098	struct node_descriptor nd3;
2099	};
2100
2101	static void qeth_read_conf_data_cb(struct qeth_card *card,
2102	struct qeth_cmd_buffer *iob,
2103	unsigned int data_length)
2104	{
2105	struct qeth_node_desc *nd = (struct qeth_node_desc *) iob->data;
2106	int rc = 0;
2107	u8 *tag;
2108
2109	QETH_CARD_TEXT(card, 2, "cfgunit");
2110
2111	if (data_length < sizeof(*nd)) {
2112	rc = -EINVAL;
2113	goto out;
2114	}
2115
2116	card->info.is_vm_nic = nd->nd1.plant[0] == _ascebc['V'] &&
2117	nd->nd1.plant[1] == _ascebc['M'];
2118	tag = (u8 *)&nd->nd1.tag;
2119	card->info.chpid = tag[0];
2120	card->info.unit_addr2 = tag[1];
2121
2122	tag = (u8 *)&nd->nd2.tag;
2123	card->info.cula = tag[1];
2124
2125	card->info.use_v1_blkt = nd->nd3.model[0] == 0xF0 &&
2126	nd->nd3.model[1] == 0xF0 &&
2127	nd->nd3.model[2] >= 0xF1 &&
2128	nd->nd3.model[2] <= 0xF4;
2129
2130	out:
2131	qeth_notify_cmd(iob, reason: rc);
2132	qeth_put_cmd(iob);
2133	}
2134
2135	static int qeth_read_conf_data(struct qeth_card *card)
2136	{
2137	struct qeth_channel *channel = &card->data;
2138	struct qeth_cmd_buffer *iob;
2139	struct ciw *ciw;
2140
2141	/* scan for RCD command in extended SenseID data */
2142	ciw = ccw_device_get_ciw(channel->ccwdev, CIW_TYPE_RCD);
2143	if (!ciw || ciw->cmd == 0)
2144	return -EOPNOTSUPP;
2145	if (ciw->count < sizeof(struct qeth_node_desc))
2146	return -EINVAL;
2147
2148	iob = qeth_alloc_cmd(channel, length: ciw->count, ccws: 1, QETH_RCD_TIMEOUT);
2149	if (!iob)
2150	return -ENOMEM;
2151
2152	iob->callback = qeth_read_conf_data_cb;
2153	qeth_setup_ccw(ccw: __ccw_from_cmd(iob), cmd_code: ciw->cmd, flags: 0, len: iob->length,
2154	data: iob->data);
2155
2156	return qeth_send_control_data(card, iob, NULL, NULL);
2157	}
2158
2159	static int qeth_idx_check_activate_response(struct qeth_card *card,
2160	struct qeth_channel *channel,
2161	struct qeth_cmd_buffer *iob)
2162	{
2163	int rc;
2164
2165	rc = qeth_check_idx_response(card, buffer: iob->data);
2166	if (rc)
2167	return rc;
2168
2169	if (QETH_IS_IDX_ACT_POS_REPLY(iob->data))
2170	return 0;
2171
2172	/* negative reply: */
2173	QETH_CARD_TEXT_(card, 2, "idxneg%c",
2174	QETH_IDX_ACT_CAUSE_CODE(iob->data));
2175
2176	switch (QETH_IDX_ACT_CAUSE_CODE(iob->data)) {
2177	case QETH_IDX_ACT_ERR_EXCL:
2178	dev_err(&channel->ccwdev->dev,
2179	"The adapter is used exclusively by another host\n");
2180	return -EBUSY;
2181	case QETH_IDX_ACT_ERR_AUTH:
2182	case QETH_IDX_ACT_ERR_AUTH_USER:
2183	dev_err(&channel->ccwdev->dev,
2184	"Setting the device online failed because of insufficient authorization\n");
2185	return -EPERM;
2186	default:
2187	QETH_DBF_MESSAGE(2, "IDX_ACTIVATE on channel %x: negative reply\n",
2188	CCW_DEVID(channel->ccwdev));
2189	return -EIO;
2190	}
2191	}
2192
2193	static void qeth_idx_activate_read_channel_cb(struct qeth_card *card,
2194	struct qeth_cmd_buffer *iob,
2195	unsigned int data_length)
2196	{
2197	struct qeth_channel *channel = iob->channel;
2198	u16 peer_level;
2199	int rc;
2200
2201	QETH_CARD_TEXT(card, 2, "idxrdcb");
2202
2203	rc = qeth_idx_check_activate_response(card, channel, iob);
2204	if (rc)
2205	goto out;
2206
2207	memcpy(&peer_level, QETH_IDX_ACT_FUNC_LEVEL(iob->data), 2);
2208	if (peer_level != qeth_peer_func_level(level: card->info.func_level)) {
2209	QETH_DBF_MESSAGE(2, "IDX_ACTIVATE on channel %x: function level mismatch (sent: %#x, received: %#x)\n",
2210	CCW_DEVID(channel->ccwdev),
2211	card->info.func_level, peer_level);
2212	rc = -EINVAL;
2213	goto out;
2214	}
2215
2216	memcpy(&card->token.issuer_rm_r,
2217	QETH_IDX_ACT_ISSUER_RM_TOKEN(iob->data),
2218	QETH_MPC_TOKEN_LENGTH);
2219	memcpy(&card->info.mcl_level[0],
2220	QETH_IDX_REPLY_LEVEL(iob->data), QETH_MCL_LENGTH);
2221
2222	out:
2223	qeth_notify_cmd(iob, reason: rc);
2224	qeth_put_cmd(iob);
2225	}
2226
2227	static void qeth_idx_activate_write_channel_cb(struct qeth_card *card,
2228	struct qeth_cmd_buffer *iob,
2229	unsigned int data_length)
2230	{
2231	struct qeth_channel *channel = iob->channel;
2232	u16 peer_level;
2233	int rc;
2234
2235	QETH_CARD_TEXT(card, 2, "idxwrcb");
2236
2237	rc = qeth_idx_check_activate_response(card, channel, iob);
2238	if (rc)
2239	goto out;
2240
2241	memcpy(&peer_level, QETH_IDX_ACT_FUNC_LEVEL(iob->data), 2);
2242	if ((peer_level & ~0x0100) !=
2243	qeth_peer_func_level(level: card->info.func_level)) {
2244	QETH_DBF_MESSAGE(2, "IDX_ACTIVATE on channel %x: function level mismatch (sent: %#x, received: %#x)\n",
2245	CCW_DEVID(channel->ccwdev),
2246	card->info.func_level, peer_level);
2247	rc = -EINVAL;
2248	}
2249
2250	out:
2251	qeth_notify_cmd(iob, reason: rc);
2252	qeth_put_cmd(iob);
2253	}
2254
2255	static void qeth_idx_setup_activate_cmd(struct qeth_card *card,
2256	struct qeth_cmd_buffer *iob)
2257	{
2258	u16 addr = (card->info.cula << 8) + card->info.unit_addr2;
2259	u8 port = ((u8)card->dev->dev_port) | 0x80;
2260	struct ccw1 *ccw = __ccw_from_cmd(iob);
2261
2262	qeth_setup_ccw(&ccw[0], CCW_CMD_WRITE, CCW_FLAG_CC, IDX_ACTIVATE_SIZE,
2263	iob->data);
2264	qeth_setup_ccw(&ccw[1], CCW_CMD_READ, 0, iob->length, iob->data);
2265	iob->finalize = qeth_idx_finalize_cmd;
2266
2267	port |= QETH_IDX_ACT_INVAL_FRAME;
2268	memcpy(QETH_IDX_ACT_PNO(iob->data), &port, 1);
2269	memcpy(QETH_IDX_ACT_ISSUER_RM_TOKEN(iob->data),
2270	&card->token.issuer_rm_w, QETH_MPC_TOKEN_LENGTH);
2271	memcpy(QETH_IDX_ACT_FUNC_LEVEL(iob->data),
2272	&card->info.func_level, 2);
2273	memcpy(QETH_IDX_ACT_QDIO_DEV_CUA(iob->data), &card->info.ddev_devno, 2);
2274	memcpy(QETH_IDX_ACT_QDIO_DEV_REALADDR(iob->data), &addr, 2);
2275	}
2276
2277	static int qeth_idx_activate_read_channel(struct qeth_card *card)
2278	{
2279	struct qeth_channel *channel = &card->read;
2280	struct qeth_cmd_buffer *iob;
2281	int rc;
2282
2283	QETH_CARD_TEXT(card, 2, "idxread");
2284
2285	iob = qeth_alloc_cmd(channel, QETH_BUFSIZE, ccws: 2, QETH_TIMEOUT);
2286	if (!iob)
2287	return -ENOMEM;
2288
2289	memcpy(iob->data, IDX_ACTIVATE_READ, IDX_ACTIVATE_SIZE);
2290	qeth_idx_setup_activate_cmd(card, iob);
2291	iob->callback = qeth_idx_activate_read_channel_cb;
2292
2293	rc = qeth_send_control_data(card, iob, NULL, NULL);
2294	if (rc)
2295	return rc;
2296
2297	channel->state = CH_STATE_UP;
2298	return 0;
2299	}
2300
2301	static int qeth_idx_activate_write_channel(struct qeth_card *card)
2302	{
2303	struct qeth_channel *channel = &card->write;
2304	struct qeth_cmd_buffer *iob;
2305	int rc;
2306
2307	QETH_CARD_TEXT(card, 2, "idxwrite");
2308
2309	iob = qeth_alloc_cmd(channel, QETH_BUFSIZE, ccws: 2, QETH_TIMEOUT);
2310	if (!iob)
2311	return -ENOMEM;
2312
2313	memcpy(iob->data, IDX_ACTIVATE_WRITE, IDX_ACTIVATE_SIZE);
2314	qeth_idx_setup_activate_cmd(card, iob);
2315	iob->callback = qeth_idx_activate_write_channel_cb;
2316
2317	rc = qeth_send_control_data(card, iob, NULL, NULL);
2318	if (rc)
2319	return rc;
2320
2321	channel->state = CH_STATE_UP;
2322	return 0;
2323	}
2324
2325	static int qeth_cm_enable_cb(struct qeth_card *card, struct qeth_reply *reply,
2326	unsigned long data)
2327	{
2328	struct qeth_cmd_buffer *iob;
2329
2330	QETH_CARD_TEXT(card, 2, "cmenblcb");
2331
2332	iob = (struct qeth_cmd_buffer *) data;
2333	memcpy(&card->token.cm_filter_r,
2334	QETH_CM_ENABLE_RESP_FILTER_TOKEN(iob->data),
2335	QETH_MPC_TOKEN_LENGTH);
2336	return 0;
2337	}
2338
2339	static int qeth_cm_enable(struct qeth_card *card)
2340	{
2341	struct qeth_cmd_buffer *iob;
2342
2343	QETH_CARD_TEXT(card, 2, "cmenable");
2344
2345	iob = qeth_mpc_alloc_cmd(card, data: CM_ENABLE, CM_ENABLE_SIZE);
2346	if (!iob)
2347	return -ENOMEM;
2348
2349	memcpy(QETH_CM_ENABLE_ISSUER_RM_TOKEN(iob->data),
2350	&card->token.issuer_rm_r, QETH_MPC_TOKEN_LENGTH);
2351	memcpy(QETH_CM_ENABLE_FILTER_TOKEN(iob->data),
2352	&card->token.cm_filter_w, QETH_MPC_TOKEN_LENGTH);
2353
2354	return qeth_send_control_data(card, iob, reply_cb: qeth_cm_enable_cb, NULL);
2355	}
2356
2357	static int qeth_cm_setup_cb(struct qeth_card *card, struct qeth_reply *reply,
2358	unsigned long data)
2359	{
2360	struct qeth_cmd_buffer *iob;
2361
2362	QETH_CARD_TEXT(card, 2, "cmsetpcb");
2363
2364	iob = (struct qeth_cmd_buffer *) data;
2365	memcpy(&card->token.cm_connection_r,
2366	QETH_CM_SETUP_RESP_DEST_ADDR(iob->data),
2367	QETH_MPC_TOKEN_LENGTH);
2368	return 0;
2369	}
2370
2371	static int qeth_cm_setup(struct qeth_card *card)
2372	{
2373	struct qeth_cmd_buffer *iob;
2374
2375	QETH_CARD_TEXT(card, 2, "cmsetup");
2376
2377	iob = qeth_mpc_alloc_cmd(card, data: CM_SETUP, CM_SETUP_SIZE);
2378	if (!iob)
2379	return -ENOMEM;
2380
2381	memcpy(QETH_CM_SETUP_DEST_ADDR(iob->data),
2382	&card->token.issuer_rm_r, QETH_MPC_TOKEN_LENGTH);
2383	memcpy(QETH_CM_SETUP_CONNECTION_TOKEN(iob->data),
2384	&card->token.cm_connection_w, QETH_MPC_TOKEN_LENGTH);
2385	memcpy(QETH_CM_SETUP_FILTER_TOKEN(iob->data),
2386	&card->token.cm_filter_r, QETH_MPC_TOKEN_LENGTH);
2387	return qeth_send_control_data(card, iob, reply_cb: qeth_cm_setup_cb, NULL);
2388	}
2389
2390	static bool qeth_is_supported_link_type(struct qeth_card *card, u8 link_type)
2391	{
2392	if (link_type == QETH_LINK_TYPE_LANE_TR ||
2393	link_type == QETH_LINK_TYPE_HSTR) {
2394	dev_err(&card->gdev->dev, "Unsupported Token Ring device\n");
2395	return false;
2396	}
2397
2398	return true;
2399	}
2400
2401	static int qeth_update_max_mtu(struct qeth_card *card, unsigned int max_mtu)
2402	{
2403	struct net_device *dev = card->dev;
2404	unsigned int new_mtu;
2405
2406	if (!max_mtu) {
2407	/* IQD needs accurate max MTU to set up its RX buffers: */
2408	if (IS_IQD(card))
2409	return -EINVAL;
2410	/* tolerate quirky HW: */
2411	max_mtu = ETH_MAX_MTU;
2412	}
2413
2414	rtnl_lock();
2415	if (IS_IQD(card)) {
2416	/* move any device with default MTU to new max MTU: */
2417	new_mtu = (dev->mtu == dev->max_mtu) ? max_mtu : dev->mtu;
2418
2419	/* adjust RX buffer size to new max MTU: */
2420	card->qdio.in_buf_size = max_mtu + 2 * PAGE_SIZE;
2421	if (dev->max_mtu && dev->max_mtu != max_mtu)
2422	qeth_free_qdio_queues(card);
2423	} else {
2424	if (dev->mtu)
2425	new_mtu = dev->mtu;
2426	/* default MTUs for first setup: */
2427	else if (IS_LAYER2(card))
2428	new_mtu = ETH_DATA_LEN;
2429	else
2430	new_mtu = ETH_DATA_LEN - 8; /* allow for LLC + SNAP */
2431	}
2432
2433	dev->max_mtu = max_mtu;
2434	dev->mtu = min(new_mtu, max_mtu);
2435	rtnl_unlock();
2436	return 0;
2437	}
2438
2439	static int qeth_get_mtu_outof_framesize(int framesize)
2440	{
2441	switch (framesize) {
2442	case 0x4000:
2443	return 8192;
2444	case 0x6000:
2445	return 16384;
2446	case 0xa000:
2447	return 32768;
2448	case 0xffff:
2449	return 57344;
2450	default:
2451	return 0;
2452	}
2453	}
2454
2455	static int qeth_ulp_enable_cb(struct qeth_card *card, struct qeth_reply *reply,
2456	unsigned long data)
2457	{
2458	__u16 mtu, framesize;
2459	__u16 len;
2460	struct qeth_cmd_buffer *iob;
2461	u8 link_type = 0;
2462
2463	QETH_CARD_TEXT(card, 2, "ulpenacb");
2464
2465	iob = (struct qeth_cmd_buffer *) data;
2466	memcpy(&card->token.ulp_filter_r,
2467	QETH_ULP_ENABLE_RESP_FILTER_TOKEN(iob->data),
2468	QETH_MPC_TOKEN_LENGTH);
2469	if (IS_IQD(card)) {
2470	memcpy(&framesize, QETH_ULP_ENABLE_RESP_MAX_MTU(iob->data), 2);
2471	mtu = qeth_get_mtu_outof_framesize(framesize);
2472	} else {
2473	mtu = *(__u16 *)QETH_ULP_ENABLE_RESP_MAX_MTU(iob->data);
2474	}
2475	*(u16 *)reply->param = mtu;
2476
2477	memcpy(&len, QETH_ULP_ENABLE_RESP_DIFINFO_LEN(iob->data), 2);
2478	if (len >= QETH_MPC_DIFINFO_LEN_INDICATES_LINK_TYPE) {
2479	memcpy(&link_type,
2480	QETH_ULP_ENABLE_RESP_LINK_TYPE(iob->data), 1);
2481	if (!qeth_is_supported_link_type(card, link_type))
2482	return -EPROTONOSUPPORT;
2483	}
2484
2485	card->info.link_type = link_type;
2486	QETH_CARD_TEXT_(card, 2, "link%d", card->info.link_type);
2487	return 0;
2488	}
2489
2490	static u8 qeth_mpc_select_prot_type(struct qeth_card *card)
2491	{
2492	return IS_LAYER2(card) ? QETH_MPC_PROT_L2 : QETH_MPC_PROT_L3;
2493	}
2494
2495	static int qeth_ulp_enable(struct qeth_card *card)
2496	{
2497	u8 prot_type = qeth_mpc_select_prot_type(card);
2498	struct qeth_cmd_buffer *iob;
2499	u16 max_mtu;
2500	int rc;
2501
2502	QETH_CARD_TEXT(card, 2, "ulpenabl");
2503
2504	iob = qeth_mpc_alloc_cmd(card, data: ULP_ENABLE, ULP_ENABLE_SIZE);
2505	if (!iob)
2506	return -ENOMEM;
2507
2508	*(QETH_ULP_ENABLE_LINKNUM(iob->data)) = (u8) card->dev->dev_port;
2509	memcpy(QETH_ULP_ENABLE_PROT_TYPE(iob->data), &prot_type, 1);
2510	memcpy(QETH_ULP_ENABLE_DEST_ADDR(iob->data),
2511	&card->token.cm_connection_r, QETH_MPC_TOKEN_LENGTH);
2512	memcpy(QETH_ULP_ENABLE_FILTER_TOKEN(iob->data),
2513	&card->token.ulp_filter_w, QETH_MPC_TOKEN_LENGTH);
2514	rc = qeth_send_control_data(card, iob, reply_cb: qeth_ulp_enable_cb, reply_param: &max_mtu);
2515	if (rc)
2516	return rc;
2517	return qeth_update_max_mtu(card, max_mtu);
2518	}
2519
2520	static int qeth_ulp_setup_cb(struct qeth_card *card, struct qeth_reply *reply,
2521	unsigned long data)
2522	{
2523	struct qeth_cmd_buffer *iob;
2524
2525	QETH_CARD_TEXT(card, 2, "ulpstpcb");
2526
2527	iob = (struct qeth_cmd_buffer *) data;
2528	memcpy(&card->token.ulp_connection_r,
2529	QETH_ULP_SETUP_RESP_CONNECTION_TOKEN(iob->data),
2530	QETH_MPC_TOKEN_LENGTH);
2531	if (!strncmp("00S", QETH_ULP_SETUP_RESP_CONNECTION_TOKEN(iob->data),
2532	3)) {
2533	QETH_CARD_TEXT(card, 2, "olmlimit");
2534	dev_err(&card->gdev->dev, "A connection could not be "
2535	"established because of an OLM limit\n");
2536	return -EMLINK;
2537	}
2538	return 0;
2539	}
2540
2541	static int qeth_ulp_setup(struct qeth_card *card)
2542	{
2543	__u16 temp;
2544	struct qeth_cmd_buffer *iob;
2545
2546	QETH_CARD_TEXT(card, 2, "ulpsetup");
2547
2548	iob = qeth_mpc_alloc_cmd(card, data: ULP_SETUP, ULP_SETUP_SIZE);
2549	if (!iob)
2550	return -ENOMEM;
2551
2552	memcpy(QETH_ULP_SETUP_DEST_ADDR(iob->data),
2553	&card->token.cm_connection_r, QETH_MPC_TOKEN_LENGTH);
2554	memcpy(QETH_ULP_SETUP_CONNECTION_TOKEN(iob->data),
2555	&card->token.ulp_connection_w, QETH_MPC_TOKEN_LENGTH);
2556	memcpy(QETH_ULP_SETUP_FILTER_TOKEN(iob->data),
2557	&card->token.ulp_filter_r, QETH_MPC_TOKEN_LENGTH);
2558
2559	memcpy(QETH_ULP_SETUP_CUA(iob->data), &card->info.ddev_devno, 2);
2560	temp = (card->info.cula << 8) + card->info.unit_addr2;
2561	memcpy(QETH_ULP_SETUP_REAL_DEVADDR(iob->data), &temp, 2);
2562	return qeth_send_control_data(card, iob, reply_cb: qeth_ulp_setup_cb, NULL);
2563	}
2564
2565	static int qeth_alloc_out_buf(struct qeth_qdio_out_q *q, unsigned int bidx,
2566	gfp_t gfp)
2567	{
2568	struct qeth_qdio_out_buffer *newbuf;
2569
2570	newbuf = kmem_cache_zalloc(qeth_qdio_outbuf_cache, gfp);
2571	if (!newbuf)
2572	return -ENOMEM;
2573
2574	newbuf->buffer = q->qdio_bufs[bidx];
2575	skb_queue_head_init(list: &newbuf->skb_list);
2576	lockdep_set_class(&newbuf->skb_list.lock, &qdio_out_skb_queue_key);
2577	atomic_set(v: &newbuf->state, i: QETH_QDIO_BUF_EMPTY);
2578	q->bufs[bidx] = newbuf;
2579	return 0;
2580	}
2581
2582	static void qeth_free_output_queue(struct qeth_qdio_out_q *q)
2583	{
2584	if (!q)
2585	return;
2586
2587	qeth_drain_output_queue(q, free: true);
2588	qdio_free_buffers(q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q);
2589	kfree(objp: q);
2590	}
2591
2592	static struct qeth_qdio_out_q *qeth_alloc_output_queue(void)
2593	{
2594	struct qeth_qdio_out_q *q = kzalloc(sizeof(*q), GFP_KERNEL);
2595	unsigned int i;
2596
2597	if (!q)
2598	return NULL;
2599
2600	if (qdio_alloc_buffers(q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q))
2601	goto err_qdio_bufs;
2602
2603	for (i = 0; i < QDIO_MAX_BUFFERS_PER_Q; i++) {
2604	if (qeth_alloc_out_buf(q, i, GFP_KERNEL))
2605	goto err_out_bufs;
2606	}
2607
2608	return q;
2609
2610	err_out_bufs:
2611	while (i > 0)
2612	qeth_free_out_buf(buf: q->bufs[--i]);
2613	qdio_free_buffers(q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q);
2614	err_qdio_bufs:
2615	kfree(objp: q);
2616	return NULL;
2617	}
2618
2619	static void qeth_tx_completion_timer(struct timer_list *timer)
2620	{
2621	struct qeth_qdio_out_q *queue = timer_container_of(queue, timer,
2622	timer);
2623
2624	napi_schedule(n: &queue->napi);
2625	QETH_TXQ_STAT_INC(queue, completion_timer);
2626	}
2627
2628	static int qeth_alloc_qdio_queues(struct qeth_card *card)
2629	{
2630	unsigned int i;
2631
2632	QETH_CARD_TEXT(card, 2, "allcqdbf");
2633
2634	/* completion */
2635	if (qeth_alloc_cq(card))
2636	goto out_err;
2637
2638	if (atomic_cmpxchg(v: &card->qdio.state, old: QETH_QDIO_UNINITIALIZED,
2639	new: QETH_QDIO_ALLOCATED) != QETH_QDIO_UNINITIALIZED)
2640	return 0;
2641
2642	/* inbound buffer pool */
2643	if (qeth_alloc_buffer_pool(card))
2644	goto out_buffer_pool;
2645
2646	/* outbound */
2647	for (i = 0; i < card->qdio.no_out_queues; ++i) {
2648	struct qeth_qdio_out_q *queue;
2649
2650	queue = qeth_alloc_output_queue();
2651	if (!queue)
2652	goto out_freeoutq;
2653	QETH_CARD_TEXT_(card, 2, "outq %i", i);
2654	QETH_CARD_HEX(card, 2, &queue, sizeof(void *));
2655	card->qdio.out_qs[i] = queue;
2656	queue->card = card;
2657	queue->queue_no = i;
2658	INIT_LIST_HEAD(list: &queue->pending_bufs);
2659	spin_lock_init(&queue->lock);
2660	timer_setup(&queue->timer, qeth_tx_completion_timer, 0);
2661	if (IS_IQD(card)) {
2662	queue->coalesce_usecs = QETH_TX_COALESCE_USECS;
2663	queue->max_coalesced_frames = QETH_TX_MAX_COALESCED_FRAMES;
2664	queue->rescan_usecs = QETH_TX_TIMER_USECS;
2665	} else {
2666	queue->coalesce_usecs = USEC_PER_SEC;
2667	queue->max_coalesced_frames = 0;
2668	queue->rescan_usecs = 10 * USEC_PER_SEC;
2669	}
2670	queue->priority = QETH_QIB_PQUE_PRIO_DEFAULT;
2671	}
2672
2673	return 0;
2674
2675	out_freeoutq:
2676	while (i > 0) {
2677	qeth_free_output_queue(q: card->qdio.out_qs[--i]);
2678	card->qdio.out_qs[i] = NULL;
2679	}
2680	qeth_free_buffer_pool(card);
2681	out_buffer_pool:
2682	atomic_set(v: &card->qdio.state, i: QETH_QDIO_UNINITIALIZED);
2683	qeth_free_cq(card);
2684	out_err:
2685	return -ENOMEM;
2686	}
2687
2688	static void qeth_free_qdio_queues(struct qeth_card *card)
2689	{
2690	int i, j;
2691
2692	qeth_free_cq(card);
2693
2694	if (atomic_xchg(v: &card->qdio.state, new: QETH_QDIO_UNINITIALIZED) ==
2695	QETH_QDIO_UNINITIALIZED)
2696	return;
2697
2698	for (j = 0; j < QDIO_MAX_BUFFERS_PER_Q; ++j) {
2699	if (card->qdio.in_q->bufs[j].rx_skb) {
2700	consume_skb(card->qdio.in_q->bufs[j].rx_skb);
2701	card->qdio.in_q->bufs[j].rx_skb = NULL;
2702	}
2703	}
2704
2705	/* inbound buffer pool */
2706	qeth_free_buffer_pool(card);
2707	/* free outbound qdio_qs */
2708	for (i = 0; i < card->qdio.no_out_queues; i++) {
2709	qeth_free_output_queue(q: card->qdio.out_qs[i]);
2710	card->qdio.out_qs[i] = NULL;
2711	}
2712	}
2713
2714	static void qeth_fill_qib_parms(struct qeth_card *card,
2715	struct qeth_qib_parms *parms)
2716	{
2717	struct qeth_qdio_out_q *queue;
2718	unsigned int i;
2719
2720	parms->pcit_magic[0] = 'P';
2721	parms->pcit_magic[1] = 'C';
2722	parms->pcit_magic[2] = 'I';
2723	parms->pcit_magic[3] = 'T';
2724	ASCEBC(parms->pcit_magic, sizeof(parms->pcit_magic));
2725	parms->pcit_a = QETH_PCI_THRESHOLD_A(card);
2726	parms->pcit_b = QETH_PCI_THRESHOLD_B(card);
2727	parms->pcit_c = QETH_PCI_TIMER_VALUE(card);
2728
2729	parms->blkt_magic[0] = 'B';
2730	parms->blkt_magic[1] = 'L';
2731	parms->blkt_magic[2] = 'K';
2732	parms->blkt_magic[3] = 'T';
2733	ASCEBC(parms->blkt_magic, sizeof(parms->blkt_magic));
2734	parms->blkt_total = card->info.blkt.time_total;
2735	parms->blkt_inter_packet = card->info.blkt.inter_packet;
2736	parms->blkt_inter_packet_jumbo = card->info.blkt.inter_packet_jumbo;
2737
2738	/* Prio-queueing implicitly uses the default priorities: */
2739	if (qeth_uses_tx_prio_queueing(card) || card->qdio.no_out_queues == 1)
2740	return;
2741
2742	parms->pque_magic[0] = 'P';
2743	parms->pque_magic[1] = 'Q';
2744	parms->pque_magic[2] = 'U';
2745	parms->pque_magic[3] = 'E';
2746	ASCEBC(parms->pque_magic, sizeof(parms->pque_magic));
2747	parms->pque_order = QETH_QIB_PQUE_ORDER_RR;
2748	parms->pque_units = QETH_QIB_PQUE_UNITS_SBAL;
2749
2750	qeth_for_each_output_queue(card, queue, i)
2751	parms->pque_priority[i] = queue->priority;
2752	}
2753
2754	static int qeth_qdio_activate(struct qeth_card *card)
2755	{
2756	QETH_CARD_TEXT(card, 3, "qdioact");
2757	return qdio_activate(CARD_DDEV(card));
2758	}
2759
2760	static int qeth_dm_act(struct qeth_card *card)
2761	{
2762	struct qeth_cmd_buffer *iob;
2763
2764	QETH_CARD_TEXT(card, 2, "dmact");
2765
2766	iob = qeth_mpc_alloc_cmd(card, data: DM_ACT, DM_ACT_SIZE);
2767	if (!iob)
2768	return -ENOMEM;
2769
2770	memcpy(QETH_DM_ACT_DEST_ADDR(iob->data),
2771	&card->token.cm_connection_r, QETH_MPC_TOKEN_LENGTH);
2772	memcpy(QETH_DM_ACT_CONNECTION_TOKEN(iob->data),
2773	&card->token.ulp_connection_r, QETH_MPC_TOKEN_LENGTH);
2774	return qeth_send_control_data(card, iob, NULL, NULL);
2775	}
2776
2777	static int qeth_mpc_initialize(struct qeth_card *card)
2778	{
2779	int rc;
2780
2781	QETH_CARD_TEXT(card, 2, "mpcinit");
2782
2783	rc = qeth_issue_next_read(card);
2784	if (rc) {
2785	QETH_CARD_TEXT_(card, 2, "1err%d", rc);
2786	return rc;
2787	}
2788	rc = qeth_cm_enable(card);
2789	if (rc) {
2790	QETH_CARD_TEXT_(card, 2, "2err%d", rc);
2791	return rc;
2792	}
2793	rc = qeth_cm_setup(card);
2794	if (rc) {
2795	QETH_CARD_TEXT_(card, 2, "3err%d", rc);
2796	return rc;
2797	}
2798	rc = qeth_ulp_enable(card);
2799	if (rc) {
2800	QETH_CARD_TEXT_(card, 2, "4err%d", rc);
2801	return rc;
2802	}
2803	rc = qeth_ulp_setup(card);
2804	if (rc) {
2805	QETH_CARD_TEXT_(card, 2, "5err%d", rc);
2806	return rc;
2807	}
2808	rc = qeth_alloc_qdio_queues(card);
2809	if (rc) {
2810	QETH_CARD_TEXT_(card, 2, "5err%d", rc);
2811	return rc;
2812	}
2813	rc = qeth_qdio_establish(card);
2814	if (rc) {
2815	QETH_CARD_TEXT_(card, 2, "6err%d", rc);
2816	qeth_free_qdio_queues(card);
2817	return rc;
2818	}
2819	rc = qeth_qdio_activate(card);
2820	if (rc) {
2821	QETH_CARD_TEXT_(card, 2, "7err%d", rc);
2822	return rc;
2823	}
2824	rc = qeth_dm_act(card);
2825	if (rc) {
2826	QETH_CARD_TEXT_(card, 2, "8err%d", rc);
2827	return rc;
2828	}
2829
2830	return 0;
2831	}
2832
2833	static void qeth_print_status_message(struct qeth_card *card)
2834	{
2835	switch (card->info.type) {
2836	case QETH_CARD_TYPE_OSD:
2837	case QETH_CARD_TYPE_OSM:
2838	case QETH_CARD_TYPE_OSX:
2839	/* VM will use a non-zero first character
2840	* to indicate a HiperSockets like reporting
2841	* of the level OSA sets the first character to zero
2842	* */
2843	if (!card->info.mcl_level[0]) {
2844	scnprintf(buf: card->info.mcl_level,
2845	size: sizeof(card->info.mcl_level),
2846	fmt: "%02x%02x",
2847	card->info.mcl_level[2],
2848	card->info.mcl_level[3]);
2849	break;
2850	}
2851	fallthrough;
2852	case QETH_CARD_TYPE_IQD:
2853	if (IS_VM_NIC(card) || (card->info.mcl_level[0] & 0x80)) {
2854	card->info.mcl_level[0] = (char) _ebcasc[(__u8)
2855	card->info.mcl_level[0]];
2856	card->info.mcl_level[1] = (char) _ebcasc[(__u8)
2857	card->info.mcl_level[1]];
2858	card->info.mcl_level[2] = (char) _ebcasc[(__u8)
2859	card->info.mcl_level[2]];
2860	card->info.mcl_level[3] = (char) _ebcasc[(__u8)
2861	card->info.mcl_level[3]];
2862	card->info.mcl_level[QETH_MCL_LENGTH] = 0;
2863	}
2864	break;
2865	default:
2866	memset(&card->info.mcl_level[0], 0, QETH_MCL_LENGTH + 1);
2867	}
2868	dev_info(&card->gdev->dev,
2869	"Device is a%s card%s%s%s\nwith link type %s.\n",
2870	qeth_get_cardname(card),
2871	(card->info.mcl_level[0]) ? " (level: " : "",
2872	(card->info.mcl_level[0]) ? card->info.mcl_level : "",
2873	(card->info.mcl_level[0]) ? ")" : "",
2874	qeth_get_cardname_short(card));
2875	}
2876
2877	static void qeth_initialize_working_pool_list(struct qeth_card *card)
2878	{
2879	struct qeth_buffer_pool_entry *entry;
2880
2881	QETH_CARD_TEXT(card, 5, "inwrklst");
2882
2883	list_for_each_entry(entry,
2884	&card->qdio.init_pool.entry_list, init_list) {
2885	qeth_put_buffer_pool_entry(card, entry);
2886	}
2887	}
2888
2889	static struct qeth_buffer_pool_entry *qeth_find_free_buffer_pool_entry(
2890	struct qeth_card *card)
2891	{
2892	struct qeth_buffer_pool_entry *entry;
2893	int i, free;
2894
2895	if (list_empty(head: &card->qdio.in_buf_pool.entry_list))
2896	return NULL;
2897
2898	list_for_each_entry(entry, &card->qdio.in_buf_pool.entry_list, list) {
2899	free = 1;
2900	for (i = 0; i < QETH_MAX_BUFFER_ELEMENTS(card); ++i) {
2901	if (page_count(page: entry->elements[i]) > 1) {
2902	free = 0;
2903	break;
2904	}
2905	}
2906	if (free) {
2907	list_del_init(entry: &entry->list);
2908	return entry;
2909	}
2910	}
2911
2912	/* no free buffer in pool so take first one and swap pages */
2913	entry = list_first_entry(&card->qdio.in_buf_pool.entry_list,
2914	struct qeth_buffer_pool_entry, list);
2915	for (i = 0; i < QETH_MAX_BUFFER_ELEMENTS(card); ++i) {
2916	if (page_count(page: entry->elements[i]) > 1) {
2917	struct page *page = dev_alloc_page();
2918
2919	if (!page)
2920	return NULL;
2921
2922	__free_page(entry->elements[i]);
2923	entry->elements[i] = page;
2924	QETH_CARD_STAT_INC(card, rx_sg_alloc_page);
2925	}
2926	}
2927	list_del_init(entry: &entry->list);
2928	return entry;
2929	}
2930
2931	static int qeth_init_input_buffer(struct qeth_card *card,
2932	struct qeth_qdio_buffer *buf)
2933	{
2934	struct qeth_buffer_pool_entry *pool_entry = buf->pool_entry;
2935	int i;
2936
2937	if ((card->options.cq == QETH_CQ_ENABLED) && (!buf->rx_skb)) {
2938	buf->rx_skb = netdev_alloc_skb(dev: card->dev,
2939	ETH_HLEN +
2940	sizeof(struct ipv6hdr));
2941	if (!buf->rx_skb)
2942	return -ENOMEM;
2943	}
2944
2945	if (!pool_entry) {
2946	pool_entry = qeth_find_free_buffer_pool_entry(card);
2947	if (!pool_entry)
2948	return -ENOBUFS;
2949
2950	buf->pool_entry = pool_entry;
2951	}
2952
2953	/*
2954	* since the buffer is accessed only from the input_tasklet
2955	* there shouldn't be a need to synchronize; also, since we use
2956	* the QETH_IN_BUF_REQUEUE_THRESHOLD we should never run out off
2957	* buffers
2958	*/
2959	for (i = 0; i < QETH_MAX_BUFFER_ELEMENTS(card); ++i) {
2960	buf->buffer->element[i].length = PAGE_SIZE;
2961	buf->buffer->element[i].addr = u64_to_dma64(
2962	page_to_phys(pool_entry->elements[i]));
2963	if (i == QETH_MAX_BUFFER_ELEMENTS(card) - 1)
2964	buf->buffer->element[i].eflags = SBAL_EFLAGS_LAST_ENTRY;
2965	else
2966	buf->buffer->element[i].eflags = 0;
2967	buf->buffer->element[i].sflags = 0;
2968	}
2969	return 0;
2970	}
2971
2972	static unsigned int qeth_tx_select_bulk_max(struct qeth_card *card,
2973	struct qeth_qdio_out_q *queue)
2974	{
2975	if (!IS_IQD(card) ||
2976	qeth_iqd_is_mcast_queue(card, queue) ||
2977	card->options.cq == QETH_CQ_ENABLED ||
2978	qdio_get_ssqd_desc(CARD_DDEV(card), &card->ssqd))
2979	return 1;
2980
2981	return card->ssqd.mmwc ? card->ssqd.mmwc : 1;
2982	}
2983
2984	static int qeth_init_qdio_queues(struct qeth_card *card)
2985	{
2986	unsigned int rx_bufs = card->qdio.in_buf_pool.buf_count;
2987	unsigned int i;
2988	int rc;
2989
2990	QETH_CARD_TEXT(card, 2, "initqdqs");
2991
2992	/* inbound queue */
2993	qdio_reset_buffers(card->qdio.in_q->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q);
2994	memset(&card->rx, 0, sizeof(struct qeth_rx));
2995
2996	qeth_initialize_working_pool_list(card);
2997	/*give only as many buffers to hardware as we have buffer pool entries*/
2998	for (i = 0; i < rx_bufs; i++) {
2999	rc = qeth_init_input_buffer(card, buf: &card->qdio.in_q->bufs[i]);
3000	if (rc)
3001	return rc;
3002	}
3003
3004	card->qdio.in_q->next_buf_to_init = QDIO_BUFNR(rx_bufs);
3005	rc = qdio_add_bufs_to_input_queue(CARD_DDEV(card), 0, 0, rx_bufs);
3006	if (rc) {
3007	QETH_CARD_TEXT_(card, 2, "1err%d", rc);
3008	return rc;
3009	}
3010
3011	/* completion */
3012	rc = qeth_cq_init(card);
3013	if (rc) {
3014	return rc;
3015	}
3016
3017	/* outbound queue */
3018	for (i = 0; i < card->qdio.no_out_queues; ++i) {
3019	struct qeth_qdio_out_q *queue = card->qdio.out_qs[i];
3020
3021	qdio_reset_buffers(queue->qdio_bufs, QDIO_MAX_BUFFERS_PER_Q);
3022	queue->max_elements = QETH_MAX_BUFFER_ELEMENTS(card);
3023	queue->next_buf_to_fill = 0;
3024	queue->do_pack = 0;
3025	queue->prev_hdr = NULL;
3026	queue->coalesced_frames = 0;
3027	queue->bulk_start = 0;
3028	queue->bulk_count = 0;
3029	queue->bulk_max = qeth_tx_select_bulk_max(card, queue);
3030	atomic_set(v: &queue->used_buffers, i: 0);
3031	atomic_set(v: &queue->set_pci_flags_count, i: 0);
3032	netdev_tx_reset_queue(q: netdev_get_tx_queue(dev: card->dev, index: i));
3033	}
3034	return 0;
3035	}
3036
3037	static void qeth_ipa_finalize_cmd(struct qeth_card *card,
3038	struct qeth_cmd_buffer *iob)
3039	{
3040	qeth_mpc_finalize_cmd(card, iob);
3041
3042	/* override with IPA-specific values: */
3043	__ipa_cmd(iob)->hdr.seqno = card->seqno.ipa++;
3044	}
3045
3046	static void qeth_prepare_ipa_cmd(struct qeth_card *card,
3047	struct qeth_cmd_buffer *iob, u16 cmd_length)
3048	{
3049	u8 prot_type = qeth_mpc_select_prot_type(card);
3050	u16 total_length = iob->length;
3051
3052	qeth_setup_ccw(ccw: __ccw_from_cmd(iob), CCW_CMD_WRITE, flags: 0, len: total_length,
3053	data: iob->data);
3054	iob->finalize = qeth_ipa_finalize_cmd;
3055
3056	memcpy(iob->data, IPA_PDU_HEADER, IPA_PDU_HEADER_SIZE);
3057	memcpy(QETH_IPA_PDU_LEN_TOTAL(iob->data), &total_length, 2);
3058	memcpy(QETH_IPA_CMD_PROT_TYPE(iob->data), &prot_type, 1);
3059	memcpy(QETH_IPA_PDU_LEN_PDU1(iob->data), &cmd_length, 2);
3060	memcpy(QETH_IPA_PDU_LEN_PDU2(iob->data), &cmd_length, 2);
3061	memcpy(QETH_IPA_CMD_DEST_ADDR(iob->data),
3062	&card->token.ulp_connection_r, QETH_MPC_TOKEN_LENGTH);
3063	memcpy(QETH_IPA_PDU_LEN_PDU3(iob->data), &cmd_length, 2);
3064	}
3065
3066	static bool qeth_ipa_match_reply(struct qeth_cmd_buffer *iob,
3067	struct qeth_cmd_buffer *reply)
3068	{
3069	struct qeth_ipa_cmd *ipa_reply = __ipa_reply(iob: reply);
3070
3071	return ipa_reply && (__ipa_cmd(iob)->hdr.seqno == ipa_reply->hdr.seqno);
3072	}
3073
3074	struct qeth_cmd_buffer *qeth_ipa_alloc_cmd(struct qeth_card *card,
3075	enum qeth_ipa_cmds cmd_code,
3076	enum qeth_prot_versions prot,
3077	unsigned int data_length)
3078	{
3079	struct qeth_cmd_buffer *iob;
3080	struct qeth_ipacmd_hdr *hdr;
3081
3082	data_length += offsetof(struct qeth_ipa_cmd, data);
3083	iob = qeth_alloc_cmd(channel: &card->write, IPA_PDU_HEADER_SIZE + data_length, ccws: 1,
3084	QETH_IPA_TIMEOUT);
3085	if (!iob)
3086	return NULL;
3087
3088	qeth_prepare_ipa_cmd(card, iob, cmd_length: data_length);
3089	iob->match = qeth_ipa_match_reply;
3090
3091	hdr = &__ipa_cmd(iob)->hdr;
3092	hdr->command = cmd_code;
3093	hdr->initiator = IPA_CMD_INITIATOR_HOST;
3094	/* hdr->seqno is set by qeth_send_control_data() */
3095	hdr->adapter_type = QETH_LINK_TYPE_FAST_ETH;
3096	hdr->rel_adapter_no = (u8) card->dev->dev_port;
3097	hdr->prim_version_no = IS_LAYER2(card) ? 2 : 1;
3098	hdr->param_count = 1;
3099	hdr->prot_version = prot;
3100	return iob;
3101	}
3102	EXPORT_SYMBOL_GPL(qeth_ipa_alloc_cmd);
3103
3104	static int qeth_send_ipa_cmd_cb(struct qeth_card *card,
3105	struct qeth_reply *reply, unsigned long data)
3106	{
3107	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
3108
3109	return (cmd->hdr.return_code) ? -EIO : 0;
3110	}
3111
3112	/*
3113	* qeth_send_ipa_cmd() - send an IPA command
3114	*
3115	* See qeth_send_control_data() for explanation of the arguments.
3116	*/
3117
3118	int qeth_send_ipa_cmd(struct qeth_card *card, struct qeth_cmd_buffer *iob,
3119	int (*reply_cb)(struct qeth_card *, struct qeth_reply*,
3120	unsigned long),
3121	void *reply_param)
3122	{
3123	int rc;
3124
3125	QETH_CARD_TEXT(card, 4, "sendipa");
3126
3127	if (card->read_or_write_problem) {
3128	qeth_put_cmd(iob);
3129	return -EIO;
3130	}
3131
3132	if (reply_cb == NULL)
3133	reply_cb = qeth_send_ipa_cmd_cb;
3134	rc = qeth_send_control_data(card, iob, reply_cb, reply_param);
3135	if (rc == -ETIME) {
3136	qeth_clear_ipacmd_list(card);
3137	qeth_schedule_recovery(card);
3138	}
3139	return rc;
3140	}
3141	EXPORT_SYMBOL_GPL(qeth_send_ipa_cmd);
3142
3143	static int qeth_send_startlan_cb(struct qeth_card *card,
3144	struct qeth_reply *reply, unsigned long data)
3145	{
3146	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
3147
3148	if (cmd->hdr.return_code == IPA_RC_LAN_OFFLINE)
3149	return -ENETDOWN;
3150
3151	return (cmd->hdr.return_code) ? -EIO : 0;
3152	}
3153
3154	static int qeth_send_startlan(struct qeth_card *card)
3155	{
3156	struct qeth_cmd_buffer *iob;
3157
3158	QETH_CARD_TEXT(card, 2, "strtlan");
3159
3160	iob = qeth_ipa_alloc_cmd(card, IPA_CMD_STARTLAN, QETH_PROT_NONE, 0);
3161	if (!iob)
3162	return -ENOMEM;
3163	return qeth_send_ipa_cmd(card, iob, qeth_send_startlan_cb, NULL);
3164	}
3165
3166	static int qeth_setadpparms_inspect_rc(struct qeth_ipa_cmd *cmd)
3167	{
3168	if (!cmd->hdr.return_code)
3169	cmd->hdr.return_code =
3170	cmd->data.setadapterparms.hdr.return_code;
3171	return cmd->hdr.return_code;
3172	}
3173
3174	static int qeth_query_setadapterparms_cb(struct qeth_card *card,
3175	struct qeth_reply *reply, unsigned long data)
3176	{
3177	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
3178	struct qeth_query_cmds_supp *query_cmd;
3179
3180	QETH_CARD_TEXT(card, 3, "quyadpcb");
3181	if (qeth_setadpparms_inspect_rc(cmd))
3182	return -EIO;
3183
3184	query_cmd = &cmd->data.setadapterparms.data.query_cmds_supp;
3185	if (query_cmd->lan_type & 0x7f) {
3186	if (!qeth_is_supported_link_type(card, link_type: query_cmd->lan_type))
3187	return -EPROTONOSUPPORT;
3188
3189	card->info.link_type = query_cmd->lan_type;
3190	QETH_CARD_TEXT_(card, 2, "lnk %d", card->info.link_type);
3191	}
3192
3193	card->options.adp.supported = query_cmd->supported_cmds;
3194	return 0;
3195	}
3196
3197	static struct qeth_cmd_buffer *qeth_get_adapter_cmd(struct qeth_card *card,
3198	enum qeth_ipa_setadp_cmd adp_cmd,
3199	unsigned int data_length)
3200	{
3201	struct qeth_ipacmd_setadpparms_hdr *hdr;
3202	struct qeth_cmd_buffer *iob;
3203
3204	iob = qeth_ipa_alloc_cmd(card, IPA_CMD_SETADAPTERPARMS, QETH_PROT_IPV4,
3205	data_length +
3206	offsetof(struct qeth_ipacmd_setadpparms,
3207	data));
3208	if (!iob)
3209	return NULL;
3210
3211	hdr = &__ipa_cmd(iob)->data.setadapterparms.hdr;
3212	hdr->cmdlength = sizeof(*hdr) + data_length;
3213	hdr->command_code = adp_cmd;
3214	hdr->used_total = 1;
3215	hdr->seq_no = 1;
3216	return iob;
3217	}
3218
3219	static int qeth_query_setadapterparms(struct qeth_card *card)
3220	{
3221	int rc;
3222	struct qeth_cmd_buffer *iob;
3223
3224	QETH_CARD_TEXT(card, 3, "queryadp");
3225	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_QUERY_COMMANDS_SUPPORTED,
3226	SETADP_DATA_SIZEOF(query_cmds_supp));
3227	if (!iob)
3228	return -ENOMEM;
3229	rc = qeth_send_ipa_cmd(card, iob, qeth_query_setadapterparms_cb, NULL);
3230	return rc;
3231	}
3232
3233	static int qeth_query_ipassists_cb(struct qeth_card *card,
3234	struct qeth_reply *reply, unsigned long data)
3235	{
3236	struct qeth_ipa_cmd *cmd;
3237
3238	QETH_CARD_TEXT(card, 2, "qipasscb");
3239
3240	cmd = (struct qeth_ipa_cmd *) data;
3241
3242	switch (cmd->hdr.return_code) {
3243	case IPA_RC_SUCCESS:
3244	break;
3245	case IPA_RC_NOTSUPP:
3246	case IPA_RC_L2_UNSUPPORTED_CMD:
3247	QETH_CARD_TEXT(card, 2, "ipaunsup");
3248	card->options.ipa4.supported |= IPA_SETADAPTERPARMS;
3249	card->options.ipa6.supported |= IPA_SETADAPTERPARMS;
3250	return -EOPNOTSUPP;
3251	default:
3252	QETH_DBF_MESSAGE(1, "IPA_CMD_QIPASSIST on device %x: Unhandled rc=%#x\n",
3253	CARD_DEVID(card), cmd->hdr.return_code);
3254	return -EIO;
3255	}
3256
3257	if (cmd->hdr.prot_version == QETH_PROT_IPV4)
3258	card->options.ipa4 = cmd->hdr.assists;
3259	else if (cmd->hdr.prot_version == QETH_PROT_IPV6)
3260	card->options.ipa6 = cmd->hdr.assists;
3261	else
3262	QETH_DBF_MESSAGE(1, "IPA_CMD_QIPASSIST on device %x: Flawed LIC detected\n",
3263	CARD_DEVID(card));
3264	return 0;
3265	}
3266
3267	static int qeth_query_ipassists(struct qeth_card *card,
3268	enum qeth_prot_versions prot)
3269	{
3270	int rc;
3271	struct qeth_cmd_buffer *iob;
3272
3273	QETH_CARD_TEXT_(card, 2, "qipassi%i", prot);
3274	iob = qeth_ipa_alloc_cmd(card, IPA_CMD_QIPASSIST, prot, 0);
3275	if (!iob)
3276	return -ENOMEM;
3277	rc = qeth_send_ipa_cmd(card, iob, qeth_query_ipassists_cb, NULL);
3278	return rc;
3279	}
3280
3281	static int qeth_query_switch_attributes_cb(struct qeth_card *card,
3282	struct qeth_reply *reply, unsigned long data)
3283	{
3284	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
3285	struct qeth_query_switch_attributes *attrs;
3286	struct qeth_switch_info *sw_info;
3287
3288	QETH_CARD_TEXT(card, 2, "qswiatcb");
3289	if (qeth_setadpparms_inspect_rc(cmd))
3290	return -EIO;
3291
3292	sw_info = (struct qeth_switch_info *)reply->param;
3293	attrs = &cmd->data.setadapterparms.data.query_switch_attributes;
3294	sw_info->capabilities = attrs->capabilities;
3295	sw_info->settings = attrs->settings;
3296	QETH_CARD_TEXT_(card, 2, "%04x%04x", sw_info->capabilities,
3297	sw_info->settings);
3298	return 0;
3299	}
3300
3301	int qeth_query_switch_attributes(struct qeth_card *card,
3302	struct qeth_switch_info *sw_info)
3303	{
3304	struct qeth_cmd_buffer *iob;
3305
3306	QETH_CARD_TEXT(card, 2, "qswiattr");
3307	if (!qeth_adp_supported(card, IPA_SETADP_QUERY_SWITCH_ATTRIBUTES))
3308	return -EOPNOTSUPP;
3309	if (!netif_carrier_ok(dev: card->dev))
3310	return -ENOMEDIUM;
3311	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_QUERY_SWITCH_ATTRIBUTES, data_length: 0);
3312	if (!iob)
3313	return -ENOMEM;
3314	return qeth_send_ipa_cmd(card, iob,
3315	qeth_query_switch_attributes_cb, sw_info);
3316	}
3317
3318	struct qeth_cmd_buffer *qeth_get_diag_cmd(struct qeth_card *card,
3319	enum qeth_diags_cmds sub_cmd,
3320	unsigned int data_length)
3321	{
3322	struct qeth_ipacmd_diagass *cmd;
3323	struct qeth_cmd_buffer *iob;
3324
3325	iob = qeth_ipa_alloc_cmd(card, IPA_CMD_SET_DIAG_ASS, QETH_PROT_NONE,
3326	DIAG_HDR_LEN + data_length);
3327	if (!iob)
3328	return NULL;
3329
3330	cmd = &__ipa_cmd(iob)->data.diagass;
3331	cmd->subcmd_len = DIAG_SUB_HDR_LEN + data_length;
3332	cmd->subcmd = sub_cmd;
3333	return iob;
3334	}
3335	EXPORT_SYMBOL_GPL(qeth_get_diag_cmd);
3336
3337	static int qeth_query_setdiagass_cb(struct qeth_card *card,
3338	struct qeth_reply *reply, unsigned long data)
3339	{
3340	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
3341	u16 rc = cmd->hdr.return_code;
3342
3343	if (rc) {
3344	QETH_CARD_TEXT_(card, 2, "diagq:%x", rc);
3345	return -EIO;
3346	}
3347
3348	card->info.diagass_support = cmd->data.diagass.ext;
3349	return 0;
3350	}
3351
3352	static int qeth_query_setdiagass(struct qeth_card *card)
3353	{
3354	struct qeth_cmd_buffer *iob;
3355
3356	QETH_CARD_TEXT(card, 2, "qdiagass");
3357	iob = qeth_get_diag_cmd(card, QETH_DIAGS_CMD_QUERY, 0);
3358	if (!iob)
3359	return -ENOMEM;
3360	return qeth_send_ipa_cmd(card, iob, qeth_query_setdiagass_cb, NULL);
3361	}
3362
3363	static void qeth_get_trap_id(struct qeth_card *card, struct qeth_trap_id *tid)
3364	{
3365	unsigned long info = get_zeroed_page(GFP_KERNEL);
3366	struct sysinfo_2_2_2 *info222 = (struct sysinfo_2_2_2 *)info;
3367	struct sysinfo_3_2_2 *info322 = (struct sysinfo_3_2_2 *)info;
3368	struct ccw_dev_id ccwid;
3369	int level;
3370
3371	tid->chpid = card->info.chpid;
3372	ccw_device_get_id(CARD_RDEV(card), &ccwid);
3373	tid->ssid = ccwid.ssid;
3374	tid->devno = ccwid.devno;
3375	if (!info)
3376	return;
3377	level = stsi(NULL, 0, 0, 0);
3378	if ((level >= 2) && (stsi(info222, 2, 2, 2) == 0))
3379	tid->lparnr = info222->lpar_number;
3380	if ((level >= 3) && (stsi(info322, 3, 2, 2) == 0)) {
3381	EBCASC(info322->vm[0].name, sizeof(info322->vm[0].name));
3382	memcpy(tid->vmname, info322->vm[0].name, sizeof(tid->vmname));
3383	}
3384	free_page(info);
3385	}
3386
3387	static int qeth_hw_trap_cb(struct qeth_card *card,
3388	struct qeth_reply *reply, unsigned long data)
3389	{
3390	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
3391	u16 rc = cmd->hdr.return_code;
3392
3393	if (rc) {
3394	QETH_CARD_TEXT_(card, 2, "trapc:%x", rc);
3395	return -EIO;
3396	}
3397	return 0;
3398	}
3399
3400	int qeth_hw_trap(struct qeth_card *card, enum qeth_diags_trap_action action)
3401	{
3402	struct qeth_cmd_buffer *iob;
3403	struct qeth_ipa_cmd *cmd;
3404
3405	QETH_CARD_TEXT(card, 2, "diagtrap");
3406	iob = qeth_get_diag_cmd(card, QETH_DIAGS_CMD_TRAP, 64);
3407	if (!iob)
3408	return -ENOMEM;
3409	cmd = __ipa_cmd(iob);
3410	cmd->data.diagass.type = 1;
3411	cmd->data.diagass.action = action;
3412	switch (action) {
3413	case QETH_DIAGS_TRAP_ARM:
3414	cmd->data.diagass.options = 0x0003;
3415	cmd->data.diagass.ext = 0x00010000 +
3416	sizeof(struct qeth_trap_id);
3417	qeth_get_trap_id(card,
3418	tid: (struct qeth_trap_id *)cmd->data.diagass.cdata);
3419	break;
3420	case QETH_DIAGS_TRAP_DISARM:
3421	cmd->data.diagass.options = 0x0001;
3422	break;
3423	case QETH_DIAGS_TRAP_CAPTURE:
3424	break;
3425	}
3426	return qeth_send_ipa_cmd(card, iob, qeth_hw_trap_cb, NULL);
3427	}
3428
3429	static int qeth_check_qdio_errors(struct qeth_card *card,
3430	struct qdio_buffer *buf,
3431	unsigned int qdio_error,
3432	const char *dbftext)
3433	{
3434	if (qdio_error) {
3435	QETH_CARD_TEXT(card, 2, dbftext);
3436	QETH_CARD_TEXT_(card, 2, " F15=%02X",
3437	buf->element[15].sflags);
3438	QETH_CARD_TEXT_(card, 2, " F14=%02X",
3439	buf->element[14].sflags);
3440	QETH_CARD_TEXT_(card, 2, " qerr=%X", qdio_error);
3441	if ((buf->element[15].sflags) == 0x12) {
3442	QETH_CARD_STAT_INC(card, rx_fifo_errors);
3443	return 0;
3444	} else
3445	return 1;
3446	}
3447	return 0;
3448	}
3449
3450	static unsigned int qeth_rx_refill_queue(struct qeth_card *card,
3451	unsigned int count)
3452	{
3453	struct qeth_qdio_q *queue = card->qdio.in_q;
3454	struct list_head *lh;
3455	int i;
3456	int rc;
3457	int newcount = 0;
3458
3459	/* only requeue at a certain threshold to avoid SIGAs */
3460	if (count >= QETH_IN_BUF_REQUEUE_THRESHOLD(card)) {
3461	for (i = queue->next_buf_to_init;
3462	i < queue->next_buf_to_init + count; ++i) {
3463	if (qeth_init_input_buffer(card,
3464	buf: &queue->bufs[QDIO_BUFNR(i)])) {
3465	break;
3466	} else {
3467	newcount++;
3468	}
3469	}
3470
3471	if (newcount < count) {
3472	/* we are in memory shortage so we switch back to
3473	traditional skb allocation and drop packages */
3474	atomic_set(v: &card->force_alloc_skb, i: 3);
3475	count = newcount;
3476	} else {
3477	atomic_add_unless(v: &card->force_alloc_skb, a: -1, u: 0);
3478	}
3479
3480	if (!count) {
3481	i = 0;
3482	list_for_each(lh, &card->qdio.in_buf_pool.entry_list)
3483	i++;
3484	if (i == card->qdio.in_buf_pool.buf_count) {
3485	QETH_CARD_TEXT(card, 2, "qsarbw");
3486	schedule_delayed_work(
3487	dwork: &card->buffer_reclaim_work,
3488	QETH_RECLAIM_WORK_TIME);
3489	}
3490	return 0;
3491	}
3492
3493	rc = qdio_add_bufs_to_input_queue(CARD_DDEV(card), 0,
3494	queue->next_buf_to_init,
3495	count);
3496	if (rc) {
3497	QETH_CARD_TEXT(card, 2, "qinberr");
3498	}
3499	queue->next_buf_to_init = QDIO_BUFNR(queue->next_buf_to_init +
3500	count);
3501	return count;
3502	}
3503
3504	return 0;
3505	}
3506
3507	static void qeth_buffer_reclaim_work(struct work_struct *work)
3508	{
3509	struct qeth_card *card = container_of(to_delayed_work(work),
3510	struct qeth_card,
3511	buffer_reclaim_work);
3512
3513	local_bh_disable();
3514	napi_schedule(n: &card->napi);
3515	/* kick-start the NAPI softirq: */
3516	local_bh_enable();
3517	}
3518
3519	static void qeth_handle_send_error(struct qeth_card *card,
3520	struct qeth_qdio_out_buffer *buffer, unsigned int qdio_err)
3521	{
3522	int sbalf15 = buffer->buffer->element[15].sflags;
3523
3524	QETH_CARD_TEXT(card, 6, "hdsnderr");
3525	qeth_check_qdio_errors(card, buf: buffer->buffer, qdio_error: qdio_err, dbftext: "qouterr");
3526
3527	if (!qdio_err)
3528	return;
3529
3530	if ((sbalf15 >= 15) && (sbalf15 <= 31))
3531	return;
3532
3533	QETH_CARD_TEXT(card, 1, "lnkfail");
3534	QETH_CARD_TEXT_(card, 1, "%04x %02x",
3535	(u16)qdio_err, (u8)sbalf15);
3536	}
3537
3538	/**
3539	* qeth_prep_flush_pack_buffer - Prepares flushing of a packing buffer.
3540	* @queue: queue to check for packing buffer
3541	*
3542	* Returns number of buffers that were prepared for flush.
3543	*/
3544	static int qeth_prep_flush_pack_buffer(struct qeth_qdio_out_q *queue)
3545	{
3546	struct qeth_qdio_out_buffer *buffer;
3547
3548	buffer = queue->bufs[queue->next_buf_to_fill];
3549	if ((atomic_read(v: &buffer->state) == QETH_QDIO_BUF_EMPTY) &&
3550	(buffer->next_element_to_fill > 0)) {
3551	/* it's a packing buffer */
3552	atomic_set(v: &buffer->state, i: QETH_QDIO_BUF_PRIMED);
3553	queue->next_buf_to_fill =
3554	QDIO_BUFNR(queue->next_buf_to_fill + 1);
3555	return 1;
3556	}
3557	return 0;
3558	}
3559
3560	/*
3561	* Switched to packing state if the number of used buffers on a queue
3562	* reaches a certain limit.
3563	*/
3564	static void qeth_switch_to_packing_if_needed(struct qeth_qdio_out_q *queue)
3565	{
3566	if (!queue->do_pack) {
3567	if (atomic_read(v: &queue->used_buffers)
3568	>= QETH_HIGH_WATERMARK_PACK){
3569	/* switch non-PACKING -> PACKING */
3570	QETH_CARD_TEXT(queue->card, 6, "np->pack");
3571	QETH_TXQ_STAT_INC(queue, packing_mode_switch);
3572	queue->do_pack = 1;
3573	}
3574	}
3575	}
3576
3577	/*
3578	* Switches from packing to non-packing mode. If there is a packing
3579	* buffer on the queue this buffer will be prepared to be flushed.
3580	* In that case 1 is returned to inform the caller. If no buffer
3581	* has to be flushed, zero is returned.
3582	*/
3583	static int qeth_switch_to_nonpacking_if_needed(struct qeth_qdio_out_q *queue)
3584	{
3585	if (queue->do_pack) {
3586	if (atomic_read(v: &queue->used_buffers)
3587	<= QETH_LOW_WATERMARK_PACK) {
3588	/* switch PACKING -> non-PACKING */
3589	QETH_CARD_TEXT(queue->card, 6, "pack->np");
3590	QETH_TXQ_STAT_INC(queue, packing_mode_switch);
3591	queue->do_pack = 0;
3592	return qeth_prep_flush_pack_buffer(queue);
3593	}
3594	}
3595	return 0;
3596	}
3597
3598	static void qeth_flush_buffers(struct qeth_qdio_out_q *queue, int index,
3599	int count)
3600	{
3601	struct qeth_qdio_out_buffer *buf = queue->bufs[index];
3602	struct qeth_card *card = queue->card;
3603	unsigned int frames, usecs;
3604	struct qaob *aob = NULL;
3605	int rc;
3606	int i;
3607
3608	for (i = index; i < index + count; ++i) {
3609	unsigned int bidx = QDIO_BUFNR(i);
3610	struct sk_buff *skb;
3611
3612	buf = queue->bufs[bidx];
3613	buf->buffer->element[buf->next_element_to_fill - 1].eflags |=
3614	SBAL_EFLAGS_LAST_ENTRY;
3615	queue->coalesced_frames += buf->frames;
3616
3617	if (IS_IQD(card)) {
3618	skb_queue_walk(&buf->skb_list, skb)
3619	skb_tx_timestamp(skb);
3620	}
3621	}
3622
3623	if (IS_IQD(card)) {
3624	if (card->options.cq == QETH_CQ_ENABLED &&
3625	!qeth_iqd_is_mcast_queue(card, queue) &&
3626	count == 1) {
3627	if (!buf->aob)
3628	buf->aob = kmem_cache_zalloc(qeth_qaob_cache,
3629	GFP_ATOMIC);
3630	if (buf->aob) {
3631	struct qeth_qaob_priv1 *priv;
3632
3633	aob = buf->aob;
3634	priv = (struct qeth_qaob_priv1 *)&aob->user1;
3635	priv->state = QETH_QAOB_ISSUED;
3636	priv->queue_no = queue->queue_no;
3637	}
3638	}
3639	} else {
3640	if (!queue->do_pack) {
3641	if ((atomic_read(v: &queue->used_buffers) >=
3642	(QETH_HIGH_WATERMARK_PACK -
3643	QETH_WATERMARK_PACK_FUZZ)) &&
3644	!atomic_read(v: &queue->set_pci_flags_count)) {
3645	/* it's likely that we'll go to packing
3646	* mode soon */
3647	atomic_inc(v: &queue->set_pci_flags_count);
3648	buf->buffer->element[0].sflags |= SBAL_SFLAGS0_PCI_REQ;
3649	}
3650	} else {
3651	if (!atomic_read(v: &queue->set_pci_flags_count)) {
3652	/*
3653	* there's no outstanding PCI any more, so we
3654	* have to request a PCI to be sure the PCI
3655	* will wake at some time in the future then we
3656	* can flush packed buffers that might still be
3657	* hanging around, which can happen if no
3658	* further send was requested by the stack
3659	*/
3660	atomic_inc(v: &queue->set_pci_flags_count);
3661	buf->buffer->element[0].sflags |= SBAL_SFLAGS0_PCI_REQ;
3662	}
3663	}
3664	}
3665
3666	QETH_TXQ_STAT_INC(queue, doorbell);
3667	rc = qdio_add_bufs_to_output_queue(CARD_DDEV(card), queue->queue_no,
3668	index, count, aob);
3669
3670	switch (rc) {
3671	case 0:
3672	case -ENOBUFS:
3673	/* ignore temporary SIGA errors without busy condition */
3674
3675	/* Fake the TX completion interrupt: */
3676	frames = READ_ONCE(queue->max_coalesced_frames);
3677	usecs = READ_ONCE(queue->coalesce_usecs);
3678
3679	if (frames && queue->coalesced_frames >= frames) {
3680	napi_schedule(n: &queue->napi);
3681	queue->coalesced_frames = 0;
3682	QETH_TXQ_STAT_INC(queue, coal_frames);
3683	} else if (qeth_use_tx_irqs(card) &&
3684	atomic_read(v: &queue->used_buffers) >= 32) {
3685	/* Old behaviour carried over from the qdio layer: */
3686	napi_schedule(n: &queue->napi);
3687	QETH_TXQ_STAT_INC(queue, coal_frames);
3688	} else if (usecs) {
3689	qeth_tx_arm_timer(queue, usecs);
3690	}
3691
3692	break;
3693	default:
3694	QETH_CARD_TEXT(queue->card, 2, "flushbuf");
3695	QETH_CARD_TEXT_(queue->card, 2, " q%d", queue->queue_no);
3696	QETH_CARD_TEXT_(queue->card, 2, " idx%d", index);
3697	QETH_CARD_TEXT_(queue->card, 2, " c%d", count);
3698	QETH_CARD_TEXT_(queue->card, 2, " err%d", rc);
3699
3700	/* this must not happen under normal circumstances. if it
3701	* happens something is really wrong -> recover */
3702	qeth_schedule_recovery(card: queue->card);
3703	}
3704	}
3705
3706	static void qeth_flush_queue(struct qeth_qdio_out_q *queue)
3707	{
3708	qeth_flush_buffers(queue, index: queue->bulk_start, count: queue->bulk_count);
3709
3710	queue->bulk_start = QDIO_BUFNR(queue->bulk_start + queue->bulk_count);
3711	queue->prev_hdr = NULL;
3712	queue->bulk_count = 0;
3713	}
3714
3715	static void qeth_check_outbound_queue(struct qeth_qdio_out_q *queue)
3716	{
3717	/*
3718	* check if we have to switch to non-packing mode or if
3719	* we have to get a pci flag out on the queue
3720	*/
3721	if ((atomic_read(v: &queue->used_buffers) <= QETH_LOW_WATERMARK_PACK) ||
3722	!atomic_read(v: &queue->set_pci_flags_count)) {
3723	unsigned int index, flush_cnt;
3724
3725	spin_lock(lock: &queue->lock);
3726
3727	index = queue->next_buf_to_fill;
3728
3729	flush_cnt = qeth_switch_to_nonpacking_if_needed(queue);
3730	if (!flush_cnt && !atomic_read(v: &queue->set_pci_flags_count))
3731	flush_cnt = qeth_prep_flush_pack_buffer(queue);
3732
3733	if (flush_cnt) {
3734	qeth_flush_buffers(queue, index, count: flush_cnt);
3735	QETH_TXQ_STAT_ADD(queue, bufs_pack, flush_cnt);
3736	}
3737
3738	spin_unlock(lock: &queue->lock);
3739	}
3740	}
3741
3742	static void qeth_qdio_poll(struct ccw_device *cdev, unsigned long card_ptr)
3743	{
3744	struct qeth_card *card = (struct qeth_card *)card_ptr;
3745
3746	napi_schedule_irqoff(n: &card->napi);
3747	}
3748
3749	int qeth_configure_cq(struct qeth_card *card, enum qeth_cq cq)
3750	{
3751	if (card->options.cq == QETH_CQ_NOTAVAILABLE)
3752	return -1;
3753
3754	card->options.cq = cq;
3755	return 0;
3756	}
3757	EXPORT_SYMBOL_GPL(qeth_configure_cq);
3758
3759	static void qeth_qdio_handle_aob(struct qeth_card *card, struct qaob *aob)
3760	{
3761	struct qeth_qaob_priv1 *priv = (struct qeth_qaob_priv1 *)&aob->user1;
3762	unsigned int queue_no = priv->queue_no;
3763
3764	BUILD_BUG_ON(sizeof(*priv) > ARRAY_SIZE(aob->user1));
3765
3766	if (xchg(&priv->state, QETH_QAOB_DONE) == QETH_QAOB_PENDING &&
3767	queue_no < card->qdio.no_out_queues)
3768	napi_schedule(n: &card->qdio.out_qs[queue_no]->napi);
3769	}
3770
3771	static void qeth_qdio_cq_handler(struct qeth_card *card, unsigned int qdio_err,
3772	unsigned int queue, int first_element,
3773	int count)
3774	{
3775	struct qeth_qdio_q *cq = card->qdio.c_q;
3776	int i;
3777	int rc;
3778
3779	QETH_CARD_TEXT_(card, 5, "qcqhe%d", first_element);
3780	QETH_CARD_TEXT_(card, 5, "qcqhc%d", count);
3781	QETH_CARD_TEXT_(card, 5, "qcqherr%d", qdio_err);
3782
3783	if (qdio_err) {
3784	netif_tx_stop_all_queues(dev: card->dev);
3785	qeth_schedule_recovery(card);
3786	return;
3787	}
3788
3789	for (i = first_element; i < first_element + count; ++i) {
3790	struct qdio_buffer *buffer = cq->qdio_bufs[QDIO_BUFNR(i)];
3791	int e = 0;
3792
3793	while ((e < QDIO_MAX_ELEMENTS_PER_BUFFER) &&
3794	buffer->element[e].addr) {
3795	dma64_t phys_aob_addr = buffer->element[e].addr;
3796
3797	qeth_qdio_handle_aob(card, dma64_to_virt(phys_aob_addr));
3798	++e;
3799	}
3800	qeth_scrub_qdio_buffer(buffer, QDIO_MAX_ELEMENTS_PER_BUFFER);
3801	}
3802	rc = qdio_add_bufs_to_input_queue(CARD_DDEV(card), queue,
3803	cq->next_buf_to_init, count);
3804	if (rc) {
3805	dev_warn(&card->gdev->dev,
3806	"QDIO reported an error, rc=%i\n", rc);
3807	QETH_CARD_TEXT(card, 2, "qcqherr");
3808	}
3809
3810	cq->next_buf_to_init = QDIO_BUFNR(cq->next_buf_to_init + count);
3811	}
3812
3813	static void qeth_qdio_input_handler(struct ccw_device *ccwdev,
3814	unsigned int qdio_err, int queue,
3815	int first_elem, int count,
3816	unsigned long card_ptr)
3817	{
3818	struct qeth_card *card = (struct qeth_card *)card_ptr;
3819
3820	QETH_CARD_TEXT_(card, 2, "qihq%d", queue);
3821	QETH_CARD_TEXT_(card, 2, "qiec%d", qdio_err);
3822
3823	if (qdio_err)
3824	qeth_schedule_recovery(card);
3825	}
3826
3827	static void qeth_qdio_output_handler(struct ccw_device *ccwdev,
3828	unsigned int qdio_error, int __queue,
3829	int first_element, int count,
3830	unsigned long card_ptr)
3831	{
3832	struct qeth_card *card = (struct qeth_card *) card_ptr;
3833
3834	QETH_CARD_TEXT(card, 2, "achkcond");
3835	netif_tx_stop_all_queues(dev: card->dev);
3836	qeth_schedule_recovery(card);
3837	}
3838
3839	/*
3840	* Note: Function assumes that we have 4 outbound queues.
3841	*/
3842	static int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb)
3843	{
3844	struct vlan_ethhdr *veth = vlan_eth_hdr(skb);
3845	u8 tos;
3846
3847	switch (card->qdio.do_prio_queueing) {
3848	case QETH_PRIO_Q_ING_TOS:
3849	case QETH_PRIO_Q_ING_PREC:
3850	switch (vlan_get_protocol(skb)) {
3851	case htons(ETH_P_IP):
3852	tos = ipv4_get_dsfield(iph: ip_hdr(skb));
3853	break;
3854	case htons(ETH_P_IPV6):
3855	tos = ipv6_get_dsfield(ipv6h: ipv6_hdr(skb));
3856	break;
3857	default:
3858	return card->qdio.default_out_queue;
3859	}
3860	if (card->qdio.do_prio_queueing == QETH_PRIO_Q_ING_PREC)
3861	return ~tos >> 6 & 3;
3862	if (tos & IPTOS_MINCOST)
3863	return 3;
3864	if (tos & IPTOS_RELIABILITY)
3865	return 2;
3866	if (tos & IPTOS_THROUGHPUT)
3867	return 1;
3868	if (tos & IPTOS_LOWDELAY)
3869	return 0;
3870	break;
3871	case QETH_PRIO_Q_ING_SKB:
3872	if (skb->priority > 5)
3873	return 0;
3874	return ~skb->priority >> 1 & 3;
3875	case QETH_PRIO_Q_ING_VLAN:
3876	if (veth->h_vlan_proto == htons(ETH_P_8021Q))
3877	return ~ntohs(veth->h_vlan_TCI) >>
3878	(VLAN_PRIO_SHIFT + 1) & 3;
3879	break;
3880	case QETH_PRIO_Q_ING_FIXED:
3881	return card->qdio.default_out_queue;
3882	default:
3883	break;
3884	}
3885	return card->qdio.default_out_queue;
3886	}
3887
3888	/**
3889	* qeth_get_elements_for_frags() - find number of SBALEs for skb frags.
3890	* @skb: SKB address
3891	*
3892	* Returns the number of pages, and thus QDIO buffer elements, needed to cover
3893	* fragmented part of the SKB. Returns zero for linear SKB.
3894	*/
3895	static int qeth_get_elements_for_frags(struct sk_buff *skb)
3896	{
3897	int cnt, elements = 0;
3898
3899	for (cnt = 0; cnt < skb_shinfo(skb)->nr_frags; cnt++) {
3900	skb_frag_t *frag = &skb_shinfo(skb)->frags[cnt];
3901
3902	elements += qeth_get_elements_for_range(
3903	(addr_t)skb_frag_address(frag),
3904	(addr_t)skb_frag_address(frag) + skb_frag_size(frag));
3905	}
3906	return elements;
3907	}
3908
3909	/**
3910	* qeth_count_elements() - Counts the number of QDIO buffer elements needed
3911	* to transmit an skb.
3912	* @skb: the skb to operate on.
3913	* @data_offset: skip this part of the skb's linear data
3914	*
3915	* Returns the number of pages, and thus QDIO buffer elements, needed to map the
3916	* skb's data (both its linear part and paged fragments).
3917	*/
3918	static unsigned int qeth_count_elements(struct sk_buff *skb,
3919	unsigned int data_offset)
3920	{
3921	unsigned int elements = qeth_get_elements_for_frags(skb);
3922	addr_t end = (addr_t)skb->data + skb_headlen(skb);
3923	addr_t start = (addr_t)skb->data + data_offset;
3924
3925	if (start != end)
3926	elements += qeth_get_elements_for_range(start, end);
3927	return elements;
3928	}
3929
3930	#define QETH_HDR_CACHE_OBJ_SIZE (sizeof(struct qeth_hdr_tso) + \
3931	MAX_TCP_HEADER)
3932
3933	/**
3934	* qeth_add_hw_header() - add a HW header to an skb.
3935	* @queue: TX queue that the skb will be placed on.
3936	* @skb: skb that the HW header should be added to.
3937	* @hdr: double pointer to a qeth_hdr. When returning with >= 0,
3938	* it contains a valid pointer to a qeth_hdr.
3939	* @hdr_len: length of the HW header.
3940	* @proto_len: length of protocol headers that need to be in same page as the
3941	* HW header.
3942	* @elements: returns the required number of buffer elements for this skb.
3943	*
3944	* Returns the pushed length. If the header can't be pushed on
3945	* (eg. because it would cross a page boundary), it is allocated from
3946	* the cache instead and 0 is returned.
3947	* The number of needed buffer elements is returned in @elements.
3948	* Error to create the hdr is indicated by returning with < 0.
3949	*/
3950	static int qeth_add_hw_header(struct qeth_qdio_out_q *queue,
3951	struct sk_buff *skb, struct qeth_hdr **hdr,
3952	unsigned int hdr_len, unsigned int proto_len,
3953	unsigned int *elements)
3954	{
3955	gfp_t gfp = GFP_ATOMIC | (skb_pfmemalloc(skb) ? __GFP_MEMALLOC : 0);
3956	const unsigned int contiguous = proto_len ? proto_len : 1;
3957	const unsigned int max_elements = queue->max_elements;
3958	unsigned int __elements;
3959	addr_t start, end;
3960	bool push_ok;
3961	int rc;
3962
3963	check_layout:
3964	start = (addr_t)skb->data - hdr_len;
3965	end = (addr_t)skb->data;
3966
3967	if (qeth_get_elements_for_range(start, end + contiguous) == 1) {
3968	/* Push HW header into same page as first protocol header. */
3969	push_ok = true;
3970	/* ... but TSO always needs a separate element for headers: */
3971	if (skb_is_gso(skb))
3972	__elements = 1 + qeth_count_elements(skb, data_offset: proto_len);
3973	else
3974	__elements = qeth_count_elements(skb, data_offset: 0);
3975	} else if (!proto_len && PAGE_ALIGNED(skb->data)) {
3976	/* Push HW header into preceding page, flush with skb->data. */
3977	push_ok = true;
3978	__elements = 1 + qeth_count_elements(skb, data_offset: 0);
3979	} else {
3980	/* Use header cache, copy protocol headers up. */
3981	push_ok = false;
3982	__elements = 1 + qeth_count_elements(skb, data_offset: proto_len);
3983	}
3984
3985	/* Compress skb to fit into one IO buffer: */
3986	if (__elements > max_elements) {
3987	if (!skb_is_nonlinear(skb)) {
3988	/* Drop it, no easy way of shrinking it further. */
3989	QETH_DBF_MESSAGE(2, "Dropped an oversized skb (Max Elements=%u / Actual=%u / Length=%u).\n",
3990	max_elements, __elements, skb->len);
3991	return -E2BIG;
3992	}
3993
3994	rc = skb_linearize(skb);
3995	if (rc) {
3996	QETH_TXQ_STAT_INC(queue, skbs_linearized_fail);
3997	return rc;
3998	}
3999
4000	QETH_TXQ_STAT_INC(queue, skbs_linearized);
4001	/* Linearization changed the layout, re-evaluate: */
4002	goto check_layout;
4003	}
4004
4005	*elements = __elements;
4006	/* Add the header: */
4007	if (push_ok) {
4008	*hdr = skb_push(skb, len: hdr_len);
4009	return hdr_len;
4010	}
4011
4012	/* Fall back to cache element with known-good alignment: */
4013	if (hdr_len + proto_len > QETH_HDR_CACHE_OBJ_SIZE)
4014	return -E2BIG;
4015	*hdr = kmem_cache_alloc(qeth_core_header_cache, gfp);
4016	if (!*hdr)
4017	return -ENOMEM;
4018	/* Copy protocol headers behind HW header: */
4019	skb_copy_from_linear_data(skb, to: ((char *)*hdr) + hdr_len, len: proto_len);
4020	return 0;
4021	}
4022
4023	static bool qeth_iqd_may_bulk(struct qeth_qdio_out_q *queue,
4024	struct sk_buff *curr_skb,
4025	struct qeth_hdr *curr_hdr)
4026	{
4027	struct qeth_qdio_out_buffer *buffer = queue->bufs[queue->bulk_start];
4028	struct qeth_hdr *prev_hdr = queue->prev_hdr;
4029
4030	if (!prev_hdr)
4031	return true;
4032
4033	/* All packets must have the same target: */
4034	if (curr_hdr->hdr.l2.id == QETH_HEADER_TYPE_LAYER2) {
4035	struct sk_buff *prev_skb = skb_peek(list_: &buffer->skb_list);
4036
4037	return ether_addr_equal(addr1: eth_hdr(skb: prev_skb)->h_dest,
4038	addr2: eth_hdr(skb: curr_skb)->h_dest) &&
4039	qeth_l2_same_vlan(h1: &prev_hdr->hdr.l2, h2: &curr_hdr->hdr.l2);
4040	}
4041
4042	return qeth_l3_same_next_hop(h1: &prev_hdr->hdr.l3, h2: &curr_hdr->hdr.l3) &&
4043	qeth_l3_iqd_same_vlan(h1: &prev_hdr->hdr.l3, h2: &curr_hdr->hdr.l3);
4044	}
4045
4046	/**
4047	* qeth_fill_buffer() - map skb into an output buffer
4048	* @buf: buffer to transport the skb
4049	* @skb: skb to map into the buffer
4050	* @hdr: qeth_hdr for this skb. Either at skb->data, or allocated
4051	* from qeth_core_header_cache.
4052	* @offset: when mapping the skb, start at skb->data + offset
4053	* @hd_len: if > 0, build a dedicated header element of this size
4054	*/
4055	static unsigned int qeth_fill_buffer(struct qeth_qdio_out_buffer *buf,
4056	struct sk_buff *skb, struct qeth_hdr *hdr,
4057	unsigned int offset, unsigned int hd_len)
4058	{
4059	struct qdio_buffer *buffer = buf->buffer;
4060	int element = buf->next_element_to_fill;
4061	int length = skb_headlen(skb) - offset;
4062	char *data = skb->data + offset;
4063	unsigned int elem_length, cnt;
4064	bool is_first_elem = true;
4065
4066	__skb_queue_tail(list: &buf->skb_list, newsk: skb);
4067
4068	/* build dedicated element for HW Header */
4069	if (hd_len) {
4070	is_first_elem = false;
4071
4072	buffer->element[element].addr = virt_to_dma64(hdr);
4073	buffer->element[element].length = hd_len;
4074	buffer->element[element].eflags = SBAL_EFLAGS_FIRST_FRAG;
4075
4076	/* HW header is allocated from cache: */
4077	if ((void *)hdr != skb->data)
4078	__set_bit(element, buf->from_kmem_cache);
4079	/* HW header was pushed and is contiguous with linear part: */
4080	else if (length > 0 && !PAGE_ALIGNED(data) &&
4081	(data == (char *)hdr + hd_len))
4082	buffer->element[element].eflags |=
4083	SBAL_EFLAGS_CONTIGUOUS;
4084
4085	element++;
4086	}
4087
4088	/* map linear part into buffer element(s) */
4089	while (length > 0) {
4090	elem_length = min_t(unsigned int, length,
4091	PAGE_SIZE - offset_in_page(data));
4092
4093	buffer->element[element].addr = virt_to_dma64(data);
4094	buffer->element[element].length = elem_length;
4095	length -= elem_length;
4096	if (is_first_elem) {
4097	is_first_elem = false;
4098	if (length || skb_is_nonlinear(skb))
4099	/* skb needs additional elements */
4100	buffer->element[element].eflags =
4101	SBAL_EFLAGS_FIRST_FRAG;
4102	else
4103	buffer->element[element].eflags = 0;
4104	} else {
4105	buffer->element[element].eflags =
4106	SBAL_EFLAGS_MIDDLE_FRAG;
4107	}
4108
4109	data += elem_length;
4110	element++;
4111	}
4112
4113	/* map page frags into buffer element(s) */
4114	for (cnt = 0; cnt < skb_shinfo(skb)->nr_frags; cnt++) {
4115	skb_frag_t *frag = &skb_shinfo(skb)->frags[cnt];
4116
4117	data = skb_frag_address(frag);
4118	length = skb_frag_size(frag);
4119	while (length > 0) {
4120	elem_length = min_t(unsigned int, length,
4121	PAGE_SIZE - offset_in_page(data));
4122
4123	buffer->element[element].addr = virt_to_dma64(data);
4124	buffer->element[element].length = elem_length;
4125	buffer->element[element].eflags =
4126	SBAL_EFLAGS_MIDDLE_FRAG;
4127
4128	length -= elem_length;
4129	data += elem_length;
4130	element++;
4131	}
4132	}
4133
4134	if (buffer->element[element - 1].eflags)
4135	buffer->element[element - 1].eflags = SBAL_EFLAGS_LAST_FRAG;
4136	buf->next_element_to_fill = element;
4137	return element;
4138	}
4139
4140	static int __qeth_xmit(struct qeth_card *card, struct qeth_qdio_out_q *queue,
4141	struct sk_buff *skb, unsigned int elements,
4142	struct qeth_hdr *hdr, unsigned int offset,
4143	unsigned int hd_len)
4144	{
4145	unsigned int bytes = qdisc_pkt_len(skb);
4146	struct qeth_qdio_out_buffer *buffer;
4147	unsigned int next_element;
4148	struct netdev_queue *txq;
4149	bool stopped = false;
4150	bool flush;
4151
4152	buffer = queue->bufs[QDIO_BUFNR(queue->bulk_start + queue->bulk_count)];
4153	txq = netdev_get_tx_queue(dev: card->dev, index: skb_get_queue_mapping(skb));
4154
4155	/* Just a sanity check, the wake/stop logic should ensure that we always
4156	* get a free buffer.
4157	*/
4158	if (atomic_read(v: &buffer->state) != QETH_QDIO_BUF_EMPTY)
4159	return -EBUSY;
4160
4161	flush = !qeth_iqd_may_bulk(queue, curr_skb: skb, curr_hdr: hdr);
4162
4163	if (flush ||
4164	(buffer->next_element_to_fill + elements > queue->max_elements)) {
4165	if (buffer->next_element_to_fill > 0) {
4166	atomic_set(v: &buffer->state, i: QETH_QDIO_BUF_PRIMED);
4167	queue->bulk_count++;
4168	}
4169
4170	if (queue->bulk_count >= queue->bulk_max)
4171	flush = true;
4172
4173	if (flush)
4174	qeth_flush_queue(queue);
4175
4176	buffer = queue->bufs[QDIO_BUFNR(queue->bulk_start +
4177	queue->bulk_count)];
4178
4179	/* Sanity-check again: */
4180	if (atomic_read(v: &buffer->state) != QETH_QDIO_BUF_EMPTY)
4181	return -EBUSY;
4182	}
4183
4184	if (buffer->next_element_to_fill == 0 &&
4185	atomic_inc_return(&queue->used_buffers) >= QDIO_MAX_BUFFERS_PER_Q) {
4186	/* If a TX completion happens right _here_ and misses to wake
4187	* the txq, then our re-check below will catch the race.
4188	*/
4189	QETH_TXQ_STAT_INC(queue, stopped);
4190	netif_tx_stop_queue(dev_queue: txq);
4191	stopped = true;
4192	}
4193
4194	next_element = qeth_fill_buffer(buf: buffer, skb, hdr, offset, hd_len);
4195	buffer->bytes += bytes;
4196	buffer->frames += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
4197	queue->prev_hdr = hdr;
4198
4199	flush = __netdev_tx_sent_queue(dev_queue: txq, bytes,
4200	xmit_more: !stopped && netdev_xmit_more());
4201
4202	if (flush || next_element >= queue->max_elements) {
4203	atomic_set(v: &buffer->state, i: QETH_QDIO_BUF_PRIMED);
4204	queue->bulk_count++;
4205
4206	if (queue->bulk_count >= queue->bulk_max)
4207	flush = true;
4208
4209	if (flush)
4210	qeth_flush_queue(queue);
4211	}
4212
4213	if (stopped && !qeth_out_queue_is_full(queue))
4214	netif_tx_start_queue(dev_queue: txq);
4215	return 0;
4216	}
4217
4218	static int qeth_do_send_packet(struct qeth_card *card,
4219	struct qeth_qdio_out_q *queue,
4220	struct sk_buff *skb, struct qeth_hdr *hdr,
4221	unsigned int offset, unsigned int hd_len,
4222	unsigned int elements_needed)
4223	{
4224	unsigned int start_index = queue->next_buf_to_fill;
4225	struct qeth_qdio_out_buffer *buffer;
4226	unsigned int next_element;
4227	struct netdev_queue *txq;
4228	bool stopped = false;
4229	int flush_count = 0;
4230	int do_pack = 0;
4231	int rc = 0;
4232
4233	buffer = queue->bufs[queue->next_buf_to_fill];
4234
4235	/* Just a sanity check, the wake/stop logic should ensure that we always
4236	* get a free buffer.
4237	*/
4238	if (atomic_read(v: &buffer->state) != QETH_QDIO_BUF_EMPTY)
4239	return -EBUSY;
4240
4241	txq = netdev_get_tx_queue(dev: card->dev, index: skb_get_queue_mapping(skb));
4242
4243	/* check if we need to switch packing state of this queue */
4244	qeth_switch_to_packing_if_needed(queue);
4245	if (queue->do_pack) {
4246	do_pack = 1;
4247	/* does packet fit in current buffer? */
4248	if (buffer->next_element_to_fill + elements_needed >
4249	queue->max_elements) {
4250	/* ... no -> set state PRIMED */
4251	atomic_set(v: &buffer->state, i: QETH_QDIO_BUF_PRIMED);
4252	flush_count++;
4253	queue->next_buf_to_fill =
4254	QDIO_BUFNR(queue->next_buf_to_fill + 1);
4255	buffer = queue->bufs[queue->next_buf_to_fill];
4256
4257	/* We stepped forward, so sanity-check again: */
4258	if (atomic_read(v: &buffer->state) !=
4259	QETH_QDIO_BUF_EMPTY) {
4260	qeth_flush_buffers(queue, index: start_index,
4261	count: flush_count);
4262	rc = -EBUSY;
4263	goto out;
4264	}
4265	}
4266	}
4267
4268	if (buffer->next_element_to_fill == 0 &&
4269	atomic_inc_return(&queue->used_buffers) >= QDIO_MAX_BUFFERS_PER_Q) {
4270	/* If a TX completion happens right _here_ and misses to wake
4271	* the txq, then our re-check below will catch the race.
4272	*/
4273	QETH_TXQ_STAT_INC(queue, stopped);
4274	netif_tx_stop_queue(dev_queue: txq);
4275	stopped = true;
4276	}
4277
4278	next_element = qeth_fill_buffer(buf: buffer, skb, hdr, offset, hd_len);
4279	buffer->bytes += qdisc_pkt_len(skb);
4280	buffer->frames += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
4281
4282	if (queue->do_pack)
4283	QETH_TXQ_STAT_INC(queue, skbs_pack);
4284	if (!queue->do_pack || stopped || next_element >= queue->max_elements) {
4285	flush_count++;
4286	atomic_set(v: &buffer->state, i: QETH_QDIO_BUF_PRIMED);
4287	queue->next_buf_to_fill =
4288	QDIO_BUFNR(queue->next_buf_to_fill + 1);
4289	}
4290
4291	if (flush_count)
4292	qeth_flush_buffers(queue, index: start_index, count: flush_count);
4293
4294	out:
4295	if (do_pack)
4296	QETH_TXQ_STAT_ADD(queue, bufs_pack, flush_count);
4297
4298	if (stopped && !qeth_out_queue_is_full(queue))
4299	netif_tx_start_queue(dev_queue: txq);
4300	return rc;
4301	}
4302
4303	static void qeth_fill_tso_ext(struct qeth_hdr_tso *hdr,
4304	unsigned int payload_len, struct sk_buff *skb,
4305	unsigned int proto_len)
4306	{
4307	struct qeth_hdr_ext_tso *ext = &hdr->ext;
4308
4309	ext->hdr_tot_len = sizeof(*ext);
4310	ext->imb_hdr_no = 1;
4311	ext->hdr_type = 1;
4312	ext->hdr_version = 1;
4313	ext->hdr_len = 28;
4314	ext->payload_len = payload_len;
4315	ext->mss = skb_shinfo(skb)->gso_size;
4316	ext->dg_hdr_len = proto_len;
4317	}
4318
4319	int qeth_xmit(struct qeth_card *card, struct sk_buff *skb,
4320	struct qeth_qdio_out_q *queue, __be16 proto,
4321	void (*fill_header)(struct qeth_qdio_out_q *queue,
4322	struct qeth_hdr *hdr, struct sk_buff *skb,
4323	__be16 proto, unsigned int data_len))
4324	{
4325	unsigned int proto_len, hw_hdr_len;
4326	unsigned int frame_len = skb->len;
4327	bool is_tso = skb_is_gso(skb);
4328	unsigned int data_offset = 0;
4329	struct qeth_hdr *hdr = NULL;
4330	unsigned int hd_len = 0;
4331	unsigned int elements;
4332	int push_len, rc;
4333
4334	if (is_tso) {
4335	hw_hdr_len = sizeof(struct qeth_hdr_tso);
4336	proto_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
4337	} else {
4338	hw_hdr_len = sizeof(struct qeth_hdr);
4339	proto_len = (IS_IQD(card) && IS_LAYER2(card)) ? ETH_HLEN : 0;
4340	}
4341
4342	rc = skb_cow_head(skb, headroom: hw_hdr_len);
4343	if (rc)
4344	return rc;
4345
4346	push_len = qeth_add_hw_header(queue, skb, hdr: &hdr, hdr_len: hw_hdr_len, proto_len,
4347	elements: &elements);
4348	if (push_len < 0)
4349	return push_len;
4350	if (is_tso || !push_len) {
4351	/* HW header needs its own buffer element. */
4352	hd_len = hw_hdr_len + proto_len;
4353	data_offset = push_len + proto_len;
4354	}
4355	memset(hdr, 0, hw_hdr_len);
4356	fill_header(queue, hdr, skb, proto, frame_len);
4357	if (is_tso)
4358	qeth_fill_tso_ext(hdr: (struct qeth_hdr_tso *) hdr,
4359	payload_len: frame_len - proto_len, skb, proto_len);
4360
4361	if (IS_IQD(card)) {
4362	rc = __qeth_xmit(card, queue, skb, elements, hdr, offset: data_offset,
4363	hd_len);
4364	} else {
4365	/* TODO: drop skb_orphan() once TX completion is fast enough */
4366	skb_orphan(skb);
4367	spin_lock(lock: &queue->lock);
4368	rc = qeth_do_send_packet(card, queue, skb, hdr, offset: data_offset,
4369	hd_len, elements_needed: elements);
4370	spin_unlock(lock: &queue->lock);
4371	}
4372
4373	if (rc && !push_len)
4374	kmem_cache_free(s: qeth_core_header_cache, objp: hdr);
4375
4376	return rc;
4377	}
4378	EXPORT_SYMBOL_GPL(qeth_xmit);
4379
4380	static int qeth_setadp_promisc_mode_cb(struct qeth_card *card,
4381	struct qeth_reply *reply, unsigned long data)
4382	{
4383	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
4384	struct qeth_ipacmd_setadpparms *setparms;
4385
4386	QETH_CARD_TEXT(card, 4, "prmadpcb");
4387
4388	setparms = &(cmd->data.setadapterparms);
4389	if (qeth_setadpparms_inspect_rc(cmd)) {
4390	QETH_CARD_TEXT_(card, 4, "prmrc%x", cmd->hdr.return_code);
4391	setparms->data.mode = SET_PROMISC_MODE_OFF;
4392	}
4393	card->info.promisc_mode = setparms->data.mode;
4394	return (cmd->hdr.return_code) ? -EIO : 0;
4395	}
4396
4397	void qeth_setadp_promisc_mode(struct qeth_card *card, bool enable)
4398	{
4399	enum qeth_ipa_promisc_modes mode = enable ? SET_PROMISC_MODE_ON :
4400	SET_PROMISC_MODE_OFF;
4401	struct qeth_cmd_buffer *iob;
4402	struct qeth_ipa_cmd *cmd;
4403
4404	QETH_CARD_TEXT(card, 4, "setprom");
4405	QETH_CARD_TEXT_(card, 4, "mode:%x", mode);
4406
4407	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_SET_PROMISC_MODE,
4408	SETADP_DATA_SIZEOF(mode));
4409	if (!iob)
4410	return;
4411	cmd = __ipa_cmd(iob);
4412	cmd->data.setadapterparms.data.mode = mode;
4413	qeth_send_ipa_cmd(card, iob, qeth_setadp_promisc_mode_cb, NULL);
4414	}
4415	EXPORT_SYMBOL_GPL(qeth_setadp_promisc_mode);
4416
4417	static int qeth_setadpparms_change_macaddr_cb(struct qeth_card *card,
4418	struct qeth_reply *reply, unsigned long data)
4419	{
4420	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
4421	struct qeth_ipacmd_setadpparms *adp_cmd;
4422
4423	QETH_CARD_TEXT(card, 4, "chgmaccb");
4424	if (qeth_setadpparms_inspect_rc(cmd))
4425	return -EIO;
4426
4427	adp_cmd = &cmd->data.setadapterparms;
4428	if (!is_valid_ether_addr(addr: adp_cmd->data.change_addr.addr))
4429	return -EADDRNOTAVAIL;
4430
4431	if (IS_LAYER2(card) && IS_OSD(card) && !IS_VM_NIC(card) &&
4432	!(adp_cmd->hdr.flags & QETH_SETADP_FLAGS_VIRTUAL_MAC))
4433	return -EADDRNOTAVAIL;
4434
4435	eth_hw_addr_set(dev: card->dev, addr: adp_cmd->data.change_addr.addr);
4436	return 0;
4437	}
4438
4439	int qeth_setadpparms_change_macaddr(struct qeth_card *card)
4440	{
4441	int rc;
4442	struct qeth_cmd_buffer *iob;
4443	struct qeth_ipa_cmd *cmd;
4444
4445	QETH_CARD_TEXT(card, 4, "chgmac");
4446
4447	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_ALTER_MAC_ADDRESS,
4448	SETADP_DATA_SIZEOF(change_addr));
4449	if (!iob)
4450	return -ENOMEM;
4451	cmd = __ipa_cmd(iob);
4452	cmd->data.setadapterparms.data.change_addr.cmd = CHANGE_ADDR_READ_MAC;
4453	cmd->data.setadapterparms.data.change_addr.addr_size = ETH_ALEN;
4454	ether_addr_copy(dst: cmd->data.setadapterparms.data.change_addr.addr,
4455	src: card->dev->dev_addr);
4456	rc = qeth_send_ipa_cmd(card, iob, qeth_setadpparms_change_macaddr_cb,
4457	NULL);
4458	return rc;
4459	}
4460	EXPORT_SYMBOL_GPL(qeth_setadpparms_change_macaddr);
4461
4462	static int qeth_setadpparms_set_access_ctrl_cb(struct qeth_card *card,
4463	struct qeth_reply *reply, unsigned long data)
4464	{
4465	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
4466	struct qeth_set_access_ctrl *access_ctrl_req;
4467
4468	QETH_CARD_TEXT(card, 4, "setaccb");
4469
4470	access_ctrl_req = &cmd->data.setadapterparms.data.set_access_ctrl;
4471	QETH_CARD_TEXT_(card, 2, "rc=%d",
4472	cmd->data.setadapterparms.hdr.return_code);
4473	if (cmd->data.setadapterparms.hdr.return_code !=
4474	SET_ACCESS_CTRL_RC_SUCCESS)
4475	QETH_DBF_MESSAGE(3, "ERR:SET_ACCESS_CTRL(%#x) on device %x: %#x\n",
4476	access_ctrl_req->subcmd_code, CARD_DEVID(card),
4477	cmd->data.setadapterparms.hdr.return_code);
4478	switch (qeth_setadpparms_inspect_rc(cmd)) {
4479	case SET_ACCESS_CTRL_RC_SUCCESS:
4480	if (access_ctrl_req->subcmd_code == ISOLATION_MODE_NONE)
4481	dev_info(&card->gdev->dev,
4482	"QDIO data connection isolation is deactivated\n");
4483	else
4484	dev_info(&card->gdev->dev,
4485	"QDIO data connection isolation is activated\n");
4486	return 0;
4487	case SET_ACCESS_CTRL_RC_ALREADY_NOT_ISOLATED:
4488	QETH_DBF_MESSAGE(2, "QDIO data connection isolation on device %x already deactivated\n",
4489	CARD_DEVID(card));
4490	return 0;
4491	case SET_ACCESS_CTRL_RC_ALREADY_ISOLATED:
4492	QETH_DBF_MESSAGE(2, "QDIO data connection isolation on device %x already activated\n",
4493	CARD_DEVID(card));
4494	return 0;
4495	case SET_ACCESS_CTRL_RC_NOT_SUPPORTED:
4496	dev_err(&card->gdev->dev, "Adapter does not "
4497	"support QDIO data connection isolation\n");
4498	return -EOPNOTSUPP;
4499	case SET_ACCESS_CTRL_RC_NONE_SHARED_ADAPTER:
4500	dev_err(&card->gdev->dev,
4501	"Adapter is dedicated. "
4502	"QDIO data connection isolation not supported\n");
4503	return -EOPNOTSUPP;
4504	case SET_ACCESS_CTRL_RC_ACTIVE_CHECKSUM_OFF:
4505	dev_err(&card->gdev->dev,
4506	"TSO does not permit QDIO data connection isolation\n");
4507	return -EPERM;
4508	case SET_ACCESS_CTRL_RC_REFLREL_UNSUPPORTED:
4509	dev_err(&card->gdev->dev, "The adjacent switch port does not "
4510	"support reflective relay mode\n");
4511	return -EOPNOTSUPP;
4512	case SET_ACCESS_CTRL_RC_REFLREL_FAILED:
4513	dev_err(&card->gdev->dev, "The reflective relay mode cannot be "
4514	"enabled at the adjacent switch port");
4515	return -EREMOTEIO;
4516	case SET_ACCESS_CTRL_RC_REFLREL_DEACT_FAILED:
4517	dev_warn(&card->gdev->dev, "Turning off reflective relay mode "
4518	"at the adjacent switch failed\n");
4519	/* benign error while disabling ISOLATION_MODE_FWD */
4520	return 0;
4521	default:
4522	return -EIO;
4523	}
4524	}
4525
4526	int qeth_setadpparms_set_access_ctrl(struct qeth_card *card,
4527	enum qeth_ipa_isolation_modes mode)
4528	{
4529	int rc;
4530	struct qeth_cmd_buffer *iob;
4531	struct qeth_ipa_cmd *cmd;
4532	struct qeth_set_access_ctrl *access_ctrl_req;
4533
4534	QETH_CARD_TEXT(card, 4, "setacctl");
4535
4536	if (!qeth_adp_supported(card, IPA_SETADP_SET_ACCESS_CONTROL)) {
4537	dev_err(&card->gdev->dev,
4538	"Adapter does not support QDIO data connection isolation\n");
4539	return -EOPNOTSUPP;
4540	}
4541
4542	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_SET_ACCESS_CONTROL,
4543	SETADP_DATA_SIZEOF(set_access_ctrl));
4544	if (!iob)
4545	return -ENOMEM;
4546	cmd = __ipa_cmd(iob);
4547	access_ctrl_req = &cmd->data.setadapterparms.data.set_access_ctrl;
4548	access_ctrl_req->subcmd_code = mode;
4549
4550	rc = qeth_send_ipa_cmd(card, iob, qeth_setadpparms_set_access_ctrl_cb,
4551	NULL);
4552	if (rc) {
4553	QETH_CARD_TEXT_(card, 2, "rc=%d", rc);
4554	QETH_DBF_MESSAGE(3, "IPA(SET_ACCESS_CTRL(%d) on device %x: sent failed\n",
4555	rc, CARD_DEVID(card));
4556	}
4557
4558	return rc;
4559	}
4560
4561	void qeth_tx_timeout(struct net_device *dev, unsigned int txqueue)
4562	{
4563	struct qeth_card *card;
4564
4565	card = dev->ml_priv;
4566	QETH_CARD_TEXT(card, 4, "txtimeo");
4567	qeth_schedule_recovery(card);
4568	}
4569	EXPORT_SYMBOL_GPL(qeth_tx_timeout);
4570
4571	static int qeth_mdio_read(struct net_device *dev, int phy_id, int regnum)
4572	{
4573	struct qeth_card *card = dev->ml_priv;
4574	int rc = 0;
4575
4576	switch (regnum) {
4577	case MII_BMCR: /* Basic mode control register */
4578	rc = BMCR_FULLDPLX;
4579	if ((card->info.link_type != QETH_LINK_TYPE_GBIT_ETH) &&
4580	(card->info.link_type != QETH_LINK_TYPE_10GBIT_ETH) &&
4581	(card->info.link_type != QETH_LINK_TYPE_25GBIT_ETH))
4582	rc |= BMCR_SPEED100;
4583	break;
4584	case MII_BMSR: /* Basic mode status register */
4585	rc = BMSR_ERCAP | BMSR_ANEGCOMPLETE | BMSR_LSTATUS |
4586	BMSR_10HALF | BMSR_10FULL | BMSR_100HALF | BMSR_100FULL |
4587	BMSR_100BASE4;
4588	break;
4589	case MII_PHYSID1: /* PHYS ID 1 */
4590	rc = (dev->dev_addr[0] << 16) | (dev->dev_addr[1] << 8) |
4591	dev->dev_addr[2];
4592	rc = (rc >> 5) & 0xFFFF;
4593	break;
4594	case MII_PHYSID2: /* PHYS ID 2 */
4595	rc = (dev->dev_addr[2] << 10) & 0xFFFF;
4596	break;
4597	case MII_ADVERTISE: /* Advertisement control reg */
4598	rc = ADVERTISE_ALL;
4599	break;
4600	case MII_LPA: /* Link partner ability reg */
4601	rc = LPA_10HALF | LPA_10FULL | LPA_100HALF | LPA_100FULL |
4602	LPA_100BASE4 | LPA_LPACK;
4603	break;
4604	case MII_EXPANSION: /* Expansion register */
4605	break;
4606	case MII_DCOUNTER: /* disconnect counter */
4607	break;
4608	case MII_FCSCOUNTER: /* false carrier counter */
4609	break;
4610	case MII_NWAYTEST: /* N-way auto-neg test register */
4611	break;
4612	case MII_RERRCOUNTER: /* rx error counter */
4613	rc = card->stats.rx_length_errors +
4614	card->stats.rx_frame_errors +
4615	card->stats.rx_fifo_errors;
4616	break;
4617	case MII_SREVISION: /* silicon revision */
4618	break;
4619	case MII_RESV1: /* reserved 1 */
4620	break;
4621	case MII_LBRERROR: /* loopback, rx, bypass error */
4622	break;
4623	case MII_PHYADDR: /* physical address */
4624	break;
4625	case MII_RESV2: /* reserved 2 */
4626	break;
4627	case MII_TPISTATUS: /* TPI status for 10mbps */
4628	break;
4629	case MII_NCONFIG: /* network interface config */
4630	break;
4631	default:
4632	break;
4633	}
4634	return rc;
4635	}
4636
4637	static int qeth_snmp_command_cb(struct qeth_card *card,
4638	struct qeth_reply *reply, unsigned long data)
4639	{
4640	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
4641	struct qeth_arp_query_info *qinfo = reply->param;
4642	struct qeth_ipacmd_setadpparms *adp_cmd;
4643	unsigned int data_len;
4644	void *snmp_data;
4645
4646	QETH_CARD_TEXT(card, 3, "snpcmdcb");
4647
4648	if (cmd->hdr.return_code) {
4649	QETH_CARD_TEXT_(card, 4, "scer1%x", cmd->hdr.return_code);
4650	return -EIO;
4651	}
4652	if (cmd->data.setadapterparms.hdr.return_code) {
4653	cmd->hdr.return_code =
4654	cmd->data.setadapterparms.hdr.return_code;
4655	QETH_CARD_TEXT_(card, 4, "scer2%x", cmd->hdr.return_code);
4656	return -EIO;
4657	}
4658
4659	adp_cmd = &cmd->data.setadapterparms;
4660	data_len = adp_cmd->hdr.cmdlength - sizeof(adp_cmd->hdr);
4661	if (adp_cmd->hdr.seq_no == 1) {
4662	snmp_data = &adp_cmd->data.snmp;
4663	} else {
4664	snmp_data = &adp_cmd->data.snmp.request;
4665	data_len -= offsetof(struct qeth_snmp_cmd, request);
4666	}
4667
4668	/* check if there is enough room in userspace */
4669	if ((qinfo->udata_len - qinfo->udata_offset) < data_len) {
4670	QETH_CARD_TEXT_(card, 4, "scer3%i", -ENOSPC);
4671	return -ENOSPC;
4672	}
4673	QETH_CARD_TEXT_(card, 4, "snore%i",
4674	cmd->data.setadapterparms.hdr.used_total);
4675	QETH_CARD_TEXT_(card, 4, "sseqn%i",
4676	cmd->data.setadapterparms.hdr.seq_no);
4677	/*copy entries to user buffer*/
4678	memcpy(qinfo->udata + qinfo->udata_offset, snmp_data, data_len);
4679	qinfo->udata_offset += data_len;
4680
4681	if (cmd->data.setadapterparms.hdr.seq_no <
4682	cmd->data.setadapterparms.hdr.used_total)
4683	return 1;
4684	return 0;
4685	}
4686
4687	static int qeth_snmp_command(struct qeth_card *card, char __user *udata)
4688	{
4689	struct qeth_snmp_ureq __user *ureq;
4690	struct qeth_cmd_buffer *iob;
4691	unsigned int req_len;
4692	struct qeth_arp_query_info qinfo = {0, };
4693	int rc = 0;
4694
4695	QETH_CARD_TEXT(card, 3, "snmpcmd");
4696
4697	if (IS_VM_NIC(card))
4698	return -EOPNOTSUPP;
4699
4700	if ((!qeth_adp_supported(card, IPA_SETADP_SET_SNMP_CONTROL)) &&
4701	IS_LAYER3(card))
4702	return -EOPNOTSUPP;
4703
4704	ureq = (struct qeth_snmp_ureq __user *) udata;
4705	if (get_user(qinfo.udata_len, &ureq->hdr.data_len) ||
4706	get_user(req_len, &ureq->hdr.req_len))
4707	return -EFAULT;
4708
4709	/* Sanitize user input, to avoid overflows in iob size calculation: */
4710	if (req_len > QETH_BUFSIZE)
4711	return -EINVAL;
4712
4713	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_SET_SNMP_CONTROL, data_length: req_len);
4714	if (!iob)
4715	return -ENOMEM;
4716
4717	if (copy_from_user(to: &__ipa_cmd(iob)->data.setadapterparms.data.snmp,
4718	from: &ureq->cmd, n: req_len)) {
4719	qeth_put_cmd(iob);
4720	return -EFAULT;
4721	}
4722
4723	qinfo.udata = kzalloc(qinfo.udata_len, GFP_KERNEL);
4724	if (!qinfo.udata) {
4725	qeth_put_cmd(iob);
4726	return -ENOMEM;
4727	}
4728	qinfo.udata_offset = sizeof(struct qeth_snmp_ureq_hdr);
4729
4730	rc = qeth_send_ipa_cmd(card, iob, qeth_snmp_command_cb, &qinfo);
4731	if (rc)
4732	QETH_DBF_MESSAGE(2, "SNMP command failed on device %x: (%#x)\n",
4733	CARD_DEVID(card), rc);
4734	else {
4735	if (copy_to_user(to: udata, from: qinfo.udata, n: qinfo.udata_len))
4736	rc = -EFAULT;
4737	}
4738
4739	kfree(objp: qinfo.udata);
4740	return rc;
4741	}
4742
4743	static int qeth_setadpparms_query_oat_cb(struct qeth_card *card,
4744	struct qeth_reply *reply,
4745	unsigned long data)
4746	{
4747	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *)data;
4748	struct qeth_qoat_priv *priv = reply->param;
4749	int resdatalen;
4750
4751	QETH_CARD_TEXT(card, 3, "qoatcb");
4752	if (qeth_setadpparms_inspect_rc(cmd))
4753	return -EIO;
4754
4755	resdatalen = cmd->data.setadapterparms.hdr.cmdlength;
4756
4757	if (resdatalen > (priv->buffer_len - priv->response_len))
4758	return -ENOSPC;
4759
4760	memcpy(priv->buffer + priv->response_len,
4761	&cmd->data.setadapterparms.hdr, resdatalen);
4762	priv->response_len += resdatalen;
4763
4764	if (cmd->data.setadapterparms.hdr.seq_no <
4765	cmd->data.setadapterparms.hdr.used_total)
4766	return 1;
4767	return 0;
4768	}
4769
4770	static int qeth_query_oat_command(struct qeth_card *card, char __user *udata)
4771	{
4772	int rc = 0;
4773	struct qeth_cmd_buffer *iob;
4774	struct qeth_ipa_cmd *cmd;
4775	struct qeth_query_oat *oat_req;
4776	struct qeth_query_oat_data oat_data;
4777	struct qeth_qoat_priv priv;
4778	void __user *tmp;
4779
4780	QETH_CARD_TEXT(card, 3, "qoatcmd");
4781
4782	if (!qeth_adp_supported(card, IPA_SETADP_QUERY_OAT))
4783	return -EOPNOTSUPP;
4784
4785	if (copy_from_user(to: &oat_data, from: udata, n: sizeof(oat_data)))
4786	return -EFAULT;
4787
4788	priv.buffer_len = oat_data.buffer_len;
4789	priv.response_len = 0;
4790	priv.buffer = vzalloc(oat_data.buffer_len);
4791	if (!priv.buffer)
4792	return -ENOMEM;
4793
4794	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_QUERY_OAT,
4795	SETADP_DATA_SIZEOF(query_oat));
4796	if (!iob) {
4797	rc = -ENOMEM;
4798	goto out_free;
4799	}
4800	cmd = __ipa_cmd(iob);
4801	oat_req = &cmd->data.setadapterparms.data.query_oat;
4802	oat_req->subcmd_code = oat_data.command;
4803
4804	rc = qeth_send_ipa_cmd(card, iob, qeth_setadpparms_query_oat_cb, &priv);
4805	if (!rc) {
4806	tmp = u64_to_user_ptr(oat_data.ptr);
4807	oat_data.response_len = priv.response_len;
4808
4809	if (copy_to_user(to: tmp, from: priv.buffer, n: priv.response_len) ||
4810	copy_to_user(to: udata, from: &oat_data, n: sizeof(oat_data)))
4811	rc = -EFAULT;
4812	}
4813
4814	out_free:
4815	vfree(addr: priv.buffer);
4816	return rc;
4817	}
4818
4819	static int qeth_init_link_info_oat_cb(struct qeth_card *card,
4820	struct qeth_reply *reply_priv,
4821	unsigned long data)
4822	{
4823	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *)data;
4824	struct qeth_link_info *link_info = reply_priv->param;
4825	struct qeth_query_oat_physical_if *phys_if;
4826	struct qeth_query_oat_reply *reply;
4827
4828	QETH_CARD_TEXT(card, 2, "qoatincb");
4829	if (qeth_setadpparms_inspect_rc(cmd))
4830	return -EIO;
4831
4832	/* Multi-part reply is unexpected, don't bother: */
4833	if (cmd->data.setadapterparms.hdr.used_total > 1)
4834	return -EINVAL;
4835
4836	/* Expect the reply to start with phys_if data: */
4837	reply = &cmd->data.setadapterparms.data.query_oat.reply[0];
4838	if (reply->type != QETH_QOAT_REPLY_TYPE_PHYS_IF ||
4839	reply->length < sizeof(*reply))
4840	return -EINVAL;
4841
4842	phys_if = &reply->phys_if;
4843
4844	switch (phys_if->speed_duplex) {
4845	case QETH_QOAT_PHYS_SPEED_10M_HALF:
4846	link_info->speed = SPEED_10;
4847	link_info->duplex = DUPLEX_HALF;
4848	break;
4849	case QETH_QOAT_PHYS_SPEED_10M_FULL:
4850	link_info->speed = SPEED_10;
4851	link_info->duplex = DUPLEX_FULL;
4852	break;
4853	case QETH_QOAT_PHYS_SPEED_100M_HALF:
4854	link_info->speed = SPEED_100;
4855	link_info->duplex = DUPLEX_HALF;
4856	break;
4857	case QETH_QOAT_PHYS_SPEED_100M_FULL:
4858	link_info->speed = SPEED_100;
4859	link_info->duplex = DUPLEX_FULL;
4860	break;
4861	case QETH_QOAT_PHYS_SPEED_1000M_HALF:
4862	link_info->speed = SPEED_1000;
4863	link_info->duplex = DUPLEX_HALF;
4864	break;
4865	case QETH_QOAT_PHYS_SPEED_1000M_FULL:
4866	link_info->speed = SPEED_1000;
4867	link_info->duplex = DUPLEX_FULL;
4868	break;
4869	case QETH_QOAT_PHYS_SPEED_10G_FULL:
4870	link_info->speed = SPEED_10000;
4871	link_info->duplex = DUPLEX_FULL;
4872	break;
4873	case QETH_QOAT_PHYS_SPEED_25G_FULL:
4874	link_info->speed = SPEED_25000;
4875	link_info->duplex = DUPLEX_FULL;
4876	break;
4877	case QETH_QOAT_PHYS_SPEED_UNKNOWN:
4878	default:
4879	link_info->speed = SPEED_UNKNOWN;
4880	link_info->duplex = DUPLEX_UNKNOWN;
4881	break;
4882	}
4883
4884	switch (phys_if->media_type) {
4885	case QETH_QOAT_PHYS_MEDIA_COPPER:
4886	link_info->port = PORT_TP;
4887	link_info->link_mode = QETH_LINK_MODE_UNKNOWN;
4888	break;
4889	case QETH_QOAT_PHYS_MEDIA_FIBRE_SHORT:
4890	link_info->port = PORT_FIBRE;
4891	link_info->link_mode = QETH_LINK_MODE_FIBRE_SHORT;
4892	break;
4893	case QETH_QOAT_PHYS_MEDIA_FIBRE_LONG:
4894	link_info->port = PORT_FIBRE;
4895	link_info->link_mode = QETH_LINK_MODE_FIBRE_LONG;
4896	break;
4897	default:
4898	link_info->port = PORT_OTHER;
4899	link_info->link_mode = QETH_LINK_MODE_UNKNOWN;
4900	break;
4901	}
4902
4903	return 0;
4904	}
4905
4906	static void qeth_init_link_info(struct qeth_card *card)
4907	{
4908	qeth_default_link_info(card);
4909
4910	/* Get more accurate data via QUERY OAT: */
4911	if (qeth_adp_supported(card, IPA_SETADP_QUERY_OAT)) {
4912	struct qeth_link_info link_info;
4913	struct qeth_cmd_buffer *iob;
4914
4915	iob = qeth_get_adapter_cmd(card, adp_cmd: IPA_SETADP_QUERY_OAT,
4916	SETADP_DATA_SIZEOF(query_oat));
4917	if (iob) {
4918	struct qeth_ipa_cmd *cmd = __ipa_cmd(iob);
4919	struct qeth_query_oat *oat_req;
4920
4921	oat_req = &cmd->data.setadapterparms.data.query_oat;
4922	oat_req->subcmd_code = QETH_QOAT_SCOPE_INTERFACE;
4923
4924	if (!qeth_send_ipa_cmd(card, iob,
4925	qeth_init_link_info_oat_cb,
4926	&link_info)) {
4927	if (link_info.speed != SPEED_UNKNOWN)
4928	card->info.link_info.speed = link_info.speed;
4929	if (link_info.duplex != DUPLEX_UNKNOWN)
4930	card->info.link_info.duplex = link_info.duplex;
4931	if (link_info.port != PORT_OTHER)
4932	card->info.link_info.port = link_info.port;
4933	if (link_info.link_mode != QETH_LINK_MODE_UNKNOWN)
4934	card->info.link_info.link_mode = link_info.link_mode;
4935	}
4936	}
4937	}
4938	}
4939
4940	/**
4941	* qeth_vm_request_mac() - Request a hypervisor-managed MAC address
4942	* @card: pointer to a qeth_card
4943	*
4944	* Returns
4945	* 0, if a MAC address has been set for the card's netdevice
4946	* a return code, for various error conditions
4947	*/
4948	int qeth_vm_request_mac(struct qeth_card *card)
4949	{
4950	struct diag26c_mac_resp *response;
4951	struct diag26c_mac_req *request;
4952	int rc;
4953
4954	QETH_CARD_TEXT(card, 2, "vmreqmac");
4955
4956	request = kzalloc(sizeof(*request), GFP_KERNEL | GFP_DMA);
4957	response = kzalloc(sizeof(*response), GFP_KERNEL | GFP_DMA);
4958	if (!request || !response) {
4959	rc = -ENOMEM;
4960	goto out;
4961	}
4962
4963	request->resp_buf_len = sizeof(*response);
4964	request->resp_version = DIAG26C_VERSION2;
4965	request->op_code = DIAG26C_GET_MAC;
4966	request->devno = card->info.ddev_devno;
4967
4968	QETH_DBF_HEX(CTRL, 2, request, sizeof(*request));
4969	rc = diag26c(request, response, DIAG26C_MAC_SERVICES);
4970	QETH_DBF_HEX(CTRL, 2, request, sizeof(*request));
4971	if (rc)
4972	goto out;
4973	QETH_DBF_HEX(CTRL, 2, response, sizeof(*response));
4974
4975	if (request->resp_buf_len < sizeof(*response) ||
4976	response->version != request->resp_version) {
4977	rc = -EIO;
4978	QETH_CARD_TEXT(card, 2, "badresp");
4979	QETH_CARD_HEX(card, 2, &request->resp_buf_len,
4980	sizeof(request->resp_buf_len));
4981	} else if (!is_valid_ether_addr(addr: response->mac)) {
4982	rc = -EINVAL;
4983	QETH_CARD_TEXT(card, 2, "badmac");
4984	QETH_CARD_HEX(card, 2, response->mac, ETH_ALEN);
4985	} else {
4986	eth_hw_addr_set(dev: card->dev, addr: response->mac);
4987	}
4988
4989	out:
4990	kfree(objp: response);
4991	kfree(objp: request);
4992	return rc;
4993	}
4994	EXPORT_SYMBOL_GPL(qeth_vm_request_mac);
4995
4996	static void qeth_determine_capabilities(struct qeth_card *card)
4997	{
4998	struct qeth_channel *channel = &card->data;
4999	struct ccw_device *ddev = channel->ccwdev;
5000	int rc;
5001	int ddev_offline = 0;
5002
5003	QETH_CARD_TEXT(card, 2, "detcapab");
5004	if (!ddev->online) {
5005	ddev_offline = 1;
5006	rc = qeth_start_channel(channel);
5007	if (rc) {
5008	QETH_CARD_TEXT_(card, 2, "3err%d", rc);
5009	goto out;
5010	}
5011	}
5012
5013	rc = qeth_read_conf_data(card);
5014	if (rc) {
5015	QETH_DBF_MESSAGE(2, "qeth_read_conf_data on device %x returned %i\n",
5016	CARD_DEVID(card), rc);
5017	QETH_CARD_TEXT_(card, 2, "5err%d", rc);
5018	goto out_offline;
5019	}
5020
5021	rc = qdio_get_ssqd_desc(ddev, &card->ssqd);
5022	if (rc)
5023	QETH_CARD_TEXT_(card, 2, "6err%d", rc);
5024
5025	QETH_CARD_TEXT_(card, 2, "qfmt%d", card->ssqd.qfmt);
5026	QETH_CARD_TEXT_(card, 2, "ac1:%02x", card->ssqd.qdioac1);
5027	QETH_CARD_TEXT_(card, 2, "ac2:%04x", card->ssqd.qdioac2);
5028	QETH_CARD_TEXT_(card, 2, "ac3:%04x", card->ssqd.qdioac3);
5029	QETH_CARD_TEXT_(card, 2, "icnt%d", card->ssqd.icnt);
5030	if (!((card->ssqd.qfmt != QDIO_IQDIO_QFMT) ||
5031	((card->ssqd.qdioac1 & CHSC_AC1_INITIATE_INPUTQ) == 0) ||
5032	((card->ssqd.qdioac3 & CHSC_AC3_FORMAT2_CQ_AVAILABLE) == 0))) {
5033	dev_info(&card->gdev->dev,
5034	"Completion Queueing supported\n");
5035	} else {
5036	card->options.cq = QETH_CQ_NOTAVAILABLE;
5037	}
5038
5039	out_offline:
5040	if (ddev_offline == 1)
5041	qeth_stop_channel(channel);
5042	out:
5043	return;
5044	}
5045
5046	static void qeth_read_ccw_conf_data(struct qeth_card *card)
5047	{
5048	struct qeth_card_info *info = &card->info;
5049	struct ccw_device *cdev = CARD_DDEV(card);
5050	struct ccw_dev_id dev_id;
5051
5052	QETH_CARD_TEXT(card, 2, "ccwconfd");
5053	ccw_device_get_id(cdev, &dev_id);
5054
5055	info->ddev_devno = dev_id.devno;
5056	info->ids_valid = !ccw_device_get_cssid(cdev, &info->cssid) &&
5057	!ccw_device_get_iid(cdev, &info->iid) &&
5058	!ccw_device_get_chid(cdev, 0, &info->chid);
5059	info->ssid = dev_id.ssid;
5060
5061	dev_info(&card->gdev->dev, "CHID: %x CHPID: %x\n",
5062	info->chid, info->chpid);
5063
5064	QETH_CARD_TEXT_(card, 3, "devn%x", info->ddev_devno);
5065	QETH_CARD_TEXT_(card, 3, "cssid:%x", info->cssid);
5066	QETH_CARD_TEXT_(card, 3, "iid:%x", info->iid);
5067	QETH_CARD_TEXT_(card, 3, "ssid:%x", info->ssid);
5068	QETH_CARD_TEXT_(card, 3, "chpid:%x", info->chpid);
5069	QETH_CARD_TEXT_(card, 3, "chid:%x", info->chid);
5070	QETH_CARD_TEXT_(card, 3, "idval%x", info->ids_valid);
5071	}
5072
5073	static int qeth_qdio_establish(struct qeth_card *card)
5074	{
5075	struct qdio_buffer **out_sbal_ptrs[QETH_MAX_OUT_QUEUES];
5076	struct qdio_buffer **in_sbal_ptrs[QETH_MAX_IN_QUEUES];
5077	struct qeth_qib_parms *qib_parms = NULL;
5078	struct qdio_initialize init_data;
5079	unsigned int no_input_qs = 1;
5080	unsigned int i;
5081	int rc = 0;
5082
5083	QETH_CARD_TEXT(card, 2, "qdioest");
5084
5085	if (!IS_IQD(card) && !IS_VM_NIC(card)) {
5086	qib_parms = kzalloc(sizeof_field(struct qib, parm), GFP_KERNEL);
5087	if (!qib_parms)
5088	return -ENOMEM;
5089
5090	qeth_fill_qib_parms(card, parms: qib_parms);
5091	}
5092
5093	in_sbal_ptrs[0] = card->qdio.in_q->qdio_bufs;
5094	if (card->options.cq == QETH_CQ_ENABLED) {
5095	in_sbal_ptrs[1] = card->qdio.c_q->qdio_bufs;
5096	no_input_qs++;
5097	}
5098
5099	for (i = 0; i < card->qdio.no_out_queues; i++)
5100	out_sbal_ptrs[i] = card->qdio.out_qs[i]->qdio_bufs;
5101
5102	memset(&init_data, 0, sizeof(struct qdio_initialize));
5103	init_data.q_format = IS_IQD(card) ? QDIO_IQDIO_QFMT :
5104	QDIO_QETH_QFMT;
5105	init_data.qib_param_field_format = 0;
5106	init_data.qib_param_field = (void *)qib_parms;
5107	init_data.no_input_qs = no_input_qs;
5108	init_data.no_output_qs = card->qdio.no_out_queues;
5109	init_data.input_handler = qeth_qdio_input_handler;
5110	init_data.output_handler = qeth_qdio_output_handler;
5111	init_data.irq_poll = qeth_qdio_poll;
5112	init_data.int_parm = (unsigned long) card;
5113	init_data.input_sbal_addr_array = in_sbal_ptrs;
5114	init_data.output_sbal_addr_array = out_sbal_ptrs;
5115
5116	if (atomic_cmpxchg(v: &card->qdio.state, old: QETH_QDIO_ALLOCATED,
5117	new: QETH_QDIO_ESTABLISHED) == QETH_QDIO_ALLOCATED) {
5118	rc = qdio_allocate(CARD_DDEV(card), init_data.no_input_qs,
5119	init_data.no_output_qs);
5120	if (rc) {
5121	atomic_set(v: &card->qdio.state, i: QETH_QDIO_ALLOCATED);
5122	goto out;
5123	}
5124	rc = qdio_establish(CARD_DDEV(card), &init_data);
5125	if (rc) {
5126	atomic_set(v: &card->qdio.state, i: QETH_QDIO_ALLOCATED);
5127	qdio_free(CARD_DDEV(card));
5128	}
5129	}
5130
5131	switch (card->options.cq) {
5132	case QETH_CQ_ENABLED:
5133	dev_info(&card->gdev->dev, "Completion Queue support enabled");
5134	break;
5135	case QETH_CQ_DISABLED:
5136	dev_info(&card->gdev->dev, "Completion Queue support disabled");
5137	break;
5138	default:
5139	break;
5140	}
5141
5142	out:
5143	kfree(objp: qib_parms);
5144	return rc;
5145	}
5146
5147	static void qeth_core_free_card(struct qeth_card *card)
5148	{
5149	QETH_CARD_TEXT(card, 2, "freecrd");
5150
5151	unregister_service_level(&card->qeth_service_level);
5152	debugfs_remove_recursive(dentry: card->debugfs);
5153	qeth_put_cmd(iob: card->read_cmd);
5154	destroy_workqueue(wq: card->event_wq);
5155	dev_set_drvdata(dev: &card->gdev->dev, NULL);
5156	kfree(objp: card);
5157	}
5158
5159	static void qeth_trace_features(struct qeth_card *card)
5160	{
5161	QETH_CARD_TEXT(card, 2, "features");
5162	QETH_CARD_HEX(card, 2, &card->options.ipa4, sizeof(card->options.ipa4));
5163	QETH_CARD_HEX(card, 2, &card->options.ipa6, sizeof(card->options.ipa6));
5164	QETH_CARD_HEX(card, 2, &card->options.adp, sizeof(card->options.adp));
5165	QETH_CARD_HEX(card, 2, &card->info.diagass_support,
5166	sizeof(card->info.diagass_support));
5167	}
5168
5169	static struct ccw_device_id qeth_ids[] = {
5170	{CCW_DEVICE_DEVTYPE(0x1731, 0x01, 0x1732, 0x01),
5171	.driver_info = QETH_CARD_TYPE_OSD},
5172	{CCW_DEVICE_DEVTYPE(0x1731, 0x05, 0x1732, 0x05),
5173	.driver_info = QETH_CARD_TYPE_IQD},
5174	{CCW_DEVICE_DEVTYPE(0x1731, 0x02, 0x1732, 0x03),
5175	.driver_info = QETH_CARD_TYPE_OSM},
5176	#ifdef CONFIG_QETH_OSX
5177	{CCW_DEVICE_DEVTYPE(0x1731, 0x02, 0x1732, 0x02),
5178	.driver_info = QETH_CARD_TYPE_OSX},
5179	#endif
5180	{},
5181	};
5182	MODULE_DEVICE_TABLE(ccw, qeth_ids);
5183
5184	static struct ccw_driver qeth_ccw_driver = {
5185	.driver = {
5186	.owner = THIS_MODULE,
5187	.name = "qeth",
5188	},
5189	.ids = qeth_ids,
5190	.probe = ccwgroup_probe_ccwdev,
5191	.remove = ccwgroup_remove_ccwdev,
5192	};
5193
5194	static int qeth_hardsetup_card(struct qeth_card *card, bool *carrier_ok)
5195	{
5196	int retries = 3;
5197	int rc;
5198
5199	QETH_CARD_TEXT(card, 2, "hrdsetup");
5200	atomic_set(v: &card->force_alloc_skb, i: 0);
5201	rc = qeth_update_from_chp_desc(card);
5202	if (rc)
5203	return rc;
5204	retry:
5205	if (retries < 3)
5206	QETH_DBF_MESSAGE(2, "Retrying to do IDX activates on device %x.\n",
5207	CARD_DEVID(card));
5208	rc = qeth_qdio_clear_card(card, use_halt: !IS_IQD(card));
5209	qeth_stop_channel(channel: &card->data);
5210	qeth_stop_channel(channel: &card->write);
5211	qeth_stop_channel(channel: &card->read);
5212	qdio_free(CARD_DDEV(card));
5213
5214	rc = qeth_start_channel(channel: &card->read);
5215	if (rc)
5216	goto retriable;
5217	rc = qeth_start_channel(channel: &card->write);
5218	if (rc)
5219	goto retriable;
5220	rc = qeth_start_channel(channel: &card->data);
5221	if (rc)
5222	goto retriable;
5223	retriable:
5224	if (rc == -ERESTARTSYS) {
5225	QETH_CARD_TEXT(card, 2, "break1");
5226	return rc;
5227	} else if (rc) {
5228	QETH_CARD_TEXT_(card, 2, "1err%d", rc);
5229	if (--retries < 0)
5230	goto out;
5231	else
5232	goto retry;
5233	}
5234
5235	qeth_determine_capabilities(card);
5236	qeth_read_ccw_conf_data(card);
5237	qeth_idx_init(card);
5238
5239	rc = qeth_idx_activate_read_channel(card);
5240	if (rc == -EINTR) {
5241	QETH_CARD_TEXT(card, 2, "break2");
5242	return rc;
5243	} else if (rc) {
5244	QETH_CARD_TEXT_(card, 2, "3err%d", rc);
5245	if (--retries < 0)
5246	goto out;
5247	else
5248	goto retry;
5249	}
5250
5251	rc = qeth_idx_activate_write_channel(card);
5252	if (rc == -EINTR) {
5253	QETH_CARD_TEXT(card, 2, "break3");
5254	return rc;
5255	} else if (rc) {
5256	QETH_CARD_TEXT_(card, 2, "4err%d", rc);
5257	if (--retries < 0)
5258	goto out;
5259	else
5260	goto retry;
5261	}
5262	card->read_or_write_problem = 0;
5263	rc = qeth_mpc_initialize(card);
5264	if (rc) {
5265	QETH_CARD_TEXT_(card, 2, "5err%d", rc);
5266	goto out;
5267	}
5268
5269	rc = qeth_send_startlan(card);
5270	if (rc) {
5271	QETH_CARD_TEXT_(card, 2, "6err%d", rc);
5272	if (rc == -ENETDOWN) {
5273	dev_warn(&card->gdev->dev, "The LAN is offline\n");
5274	*carrier_ok = false;
5275	} else {
5276	goto out;
5277	}
5278	} else {
5279	*carrier_ok = true;
5280	}
5281
5282	card->options.ipa4.supported = 0;
5283	card->options.ipa6.supported = 0;
5284	card->options.adp.supported = 0;
5285	card->options.sbp.supported_funcs = 0;
5286	card->info.diagass_support = 0;
5287	rc = qeth_query_ipassists(card, prot: QETH_PROT_IPV4);
5288	if (rc == -ENOMEM)
5289	goto out;
5290	if (qeth_is_supported(card, IPA_IPV6)) {
5291	rc = qeth_query_ipassists(card, prot: QETH_PROT_IPV6);
5292	if (rc == -ENOMEM)
5293	goto out;
5294	}
5295	if (qeth_is_supported(card, IPA_SETADAPTERPARMS)) {
5296	rc = qeth_query_setadapterparms(card);
5297	if (rc < 0) {
5298	QETH_CARD_TEXT_(card, 2, "7err%d", rc);
5299	goto out;
5300	}
5301	}
5302	if (qeth_adp_supported(card, IPA_SETADP_SET_DIAG_ASSIST)) {
5303	rc = qeth_query_setdiagass(card);
5304	if (rc)
5305	QETH_CARD_TEXT_(card, 2, "8err%d", rc);
5306	}
5307
5308	qeth_trace_features(card);
5309
5310	if (!qeth_is_diagass_supported(card, cmd: QETH_DIAGS_CMD_TRAP) ||
5311	(card->info.hwtrap && qeth_hw_trap(card, action: QETH_DIAGS_TRAP_ARM)))
5312	card->info.hwtrap = 0;
5313
5314	if (card->options.isolation != ISOLATION_MODE_NONE) {
5315	rc = qeth_setadpparms_set_access_ctrl(card,
5316	mode: card->options.isolation);
5317	if (rc)
5318	goto out;
5319	}
5320
5321	qeth_init_link_info(card);
5322
5323	rc = qeth_init_qdio_queues(card);
5324	if (rc) {
5325	QETH_CARD_TEXT_(card, 2, "9err%d", rc);
5326	goto out;
5327	}
5328
5329	return 0;
5330	out:
5331	dev_warn(&card->gdev->dev, "The qeth device driver failed to recover "
5332	"an error on the device\n");
5333	QETH_DBF_MESSAGE(2, "Initialization for device %x failed in hardsetup! rc=%d\n",
5334	CARD_DEVID(card), rc);
5335	return rc;
5336	}
5337
5338	static int qeth_set_online(struct qeth_card *card,
5339	const struct qeth_discipline *disc)
5340	{
5341	bool carrier_ok;
5342	int rc;
5343
5344	mutex_lock(&card->conf_mutex);
5345	QETH_CARD_TEXT(card, 2, "setonlin");
5346
5347	rc = qeth_hardsetup_card(card, carrier_ok: &carrier_ok);
5348	if (rc) {
5349	QETH_CARD_TEXT_(card, 2, "2err%04x", rc);
5350	rc = -ENODEV;
5351	goto err_hardsetup;
5352	}
5353
5354	qeth_print_status_message(card);
5355
5356	if (card->dev->reg_state != NETREG_REGISTERED)
5357	/* no need for locking / error handling at this early stage: */
5358	qeth_set_real_num_tx_queues(card, count: qeth_tx_actual_queues(card));
5359
5360	rc = disc->set_online(card, carrier_ok);
5361	if (rc)
5362	goto err_online;
5363
5364	/* let user_space know that device is online */
5365	kobject_uevent(kobj: &card->gdev->dev.kobj, action: KOBJ_CHANGE);
5366
5367	mutex_unlock(lock: &card->conf_mutex);
5368	return 0;
5369
5370	err_online:
5371	err_hardsetup:
5372	qeth_qdio_clear_card(card, use_halt: 0);
5373	qeth_clear_working_pool_list(card);
5374	qeth_flush_local_addrs(card);
5375
5376	qeth_stop_channel(channel: &card->data);
5377	qeth_stop_channel(channel: &card->write);
5378	qeth_stop_channel(channel: &card->read);
5379	qdio_free(CARD_DDEV(card));
5380
5381	mutex_unlock(lock: &card->conf_mutex);
5382	return rc;
5383	}
5384
5385	int qeth_set_offline(struct qeth_card *card, const struct qeth_discipline *disc,
5386	bool resetting)
5387	{
5388	int rc, rc2, rc3;
5389
5390	mutex_lock(&card->conf_mutex);
5391	QETH_CARD_TEXT(card, 3, "setoffl");
5392
5393	if ((!resetting && card->info.hwtrap) || card->info.hwtrap == 2) {
5394	qeth_hw_trap(card, action: QETH_DIAGS_TRAP_DISARM);
5395	card->info.hwtrap = 1;
5396	}
5397
5398	/* cancel any stalled cmd that might block the rtnl: */
5399	qeth_clear_ipacmd_list(card);
5400
5401	rtnl_lock();
5402	netif_device_detach(dev: card->dev);
5403	netif_carrier_off(dev: card->dev);
5404	rtnl_unlock();
5405
5406	cancel_work_sync(work: &card->rx_mode_work);
5407
5408	disc->set_offline(card);
5409
5410	qeth_qdio_clear_card(card, use_halt: 0);
5411	qeth_drain_output_queues(card);
5412	qeth_clear_working_pool_list(card);
5413	qeth_flush_local_addrs(card);
5414	card->info.promisc_mode = 0;
5415	qeth_default_link_info(card);
5416
5417	rc = qeth_stop_channel(channel: &card->data);
5418	rc2 = qeth_stop_channel(channel: &card->write);
5419	rc3 = qeth_stop_channel(channel: &card->read);
5420	if (!rc)
5421	rc = (rc2) ? rc2 : rc3;
5422	if (rc)
5423	QETH_CARD_TEXT_(card, 2, "1err%d", rc);
5424	qdio_free(CARD_DDEV(card));
5425
5426	/* let user_space know that device is offline */
5427	kobject_uevent(kobj: &card->gdev->dev.kobj, action: KOBJ_CHANGE);
5428
5429	mutex_unlock(lock: &card->conf_mutex);
5430	return 0;
5431	}
5432	EXPORT_SYMBOL_GPL(qeth_set_offline);
5433
5434	static int qeth_do_reset(void *data)
5435	{
5436	const struct qeth_discipline *disc;
5437	struct qeth_card *card = data;
5438	int rc;
5439
5440	/* Lock-free, other users will block until we are done. */
5441	disc = card->discipline;
5442
5443	QETH_CARD_TEXT(card, 2, "recover1");
5444	if (!qeth_do_run_thread(card, thread: QETH_RECOVER_THREAD))
5445	return 0;
5446	QETH_CARD_TEXT(card, 2, "recover2");
5447	dev_warn(&card->gdev->dev,
5448	"A recovery process has been started for the device\n");
5449
5450	qeth_set_offline(card, disc, true);
5451	rc = qeth_set_online(card, disc);
5452	if (!rc) {
5453	dev_info(&card->gdev->dev,
5454	"Device successfully recovered!\n");
5455	} else {
5456	qeth_set_offline(card, disc, true);
5457	ccwgroup_set_offline(card->gdev, false);
5458	dev_warn(&card->gdev->dev,
5459	"The qeth device driver failed to recover an error on the device\n");
5460	}
5461	qeth_clear_thread_start_bit(card, thread: QETH_RECOVER_THREAD);
5462	qeth_clear_thread_running_bit(card, thread: QETH_RECOVER_THREAD);
5463	return 0;
5464	}
5465
5466	#if IS_ENABLED(CONFIG_QETH_L3)
5467	static void qeth_l3_rebuild_skb(struct qeth_card *card, struct sk_buff *skb,
5468	struct qeth_hdr *hdr)
5469	{
5470	struct af_iucv_trans_hdr *iucv = (struct af_iucv_trans_hdr *) skb->data;
5471	struct qeth_hdr_layer3 *l3_hdr = &hdr->hdr.l3;
5472	struct net_device *dev = skb->dev;
5473
5474	if (IS_IQD(card) && iucv->magic == ETH_P_AF_IUCV) {
5475	dev_hard_header(skb, dev, ETH_P_AF_IUCV, dev->dev_addr,
5476	"FAKELL", skb->len);
5477	return;
5478	}
5479
5480	if (!(l3_hdr->flags & QETH_HDR_PASSTHRU)) {
5481	u16 prot = (l3_hdr->flags & QETH_HDR_IPV6) ? ETH_P_IPV6 :
5482	ETH_P_IP;
5483	unsigned char tg_addr[ETH_ALEN];
5484
5485	skb_reset_network_header(skb);
5486	switch (l3_hdr->flags & QETH_HDR_CAST_MASK) {
5487	case QETH_CAST_MULTICAST:
5488	if (prot == ETH_P_IP)
5489	ip_eth_mc_map(ip_hdr(skb)->daddr, tg_addr);
5490	else
5491	ipv6_eth_mc_map(&ipv6_hdr(skb)->daddr, tg_addr);
5492	QETH_CARD_STAT_INC(card, rx_multicast);
5493	break;
5494	case QETH_CAST_BROADCAST:
5495	ether_addr_copy(tg_addr, dev->broadcast);
5496	QETH_CARD_STAT_INC(card, rx_multicast);
5497	break;
5498	default:
5499	if (card->options.sniffer)
5500	skb->pkt_type = PACKET_OTHERHOST;
5501	ether_addr_copy(tg_addr, dev->dev_addr);
5502	}
5503
5504	if (l3_hdr->ext_flags & QETH_HDR_EXT_SRC_MAC_ADDR)
5505	dev_hard_header(skb, dev, prot, tg_addr,
5506	&l3_hdr->next_hop.rx.src_mac, skb->len);
5507	else
5508	dev_hard_header(skb, dev, prot, tg_addr, "FAKELL",
5509	skb->len);
5510	}
5511
5512	/* copy VLAN tag from hdr into skb */
5513	if (!card->options.sniffer &&
5514	(l3_hdr->ext_flags & (QETH_HDR_EXT_VLAN_FRAME |
5515	QETH_HDR_EXT_INCLUDE_VLAN_TAG))) {
5516	u16 tag = (l3_hdr->ext_flags & QETH_HDR_EXT_VLAN_FRAME) ?
5517	l3_hdr->vlan_id :
5518	l3_hdr->next_hop.rx.vlan_id;
5519
5520	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
5521	}
5522	}
5523	#endif
5524
5525	static void qeth_receive_skb(struct qeth_card *card, struct sk_buff *skb,
5526	bool uses_frags, bool is_cso)
5527	{
5528	struct napi_struct *napi = &card->napi;
5529
5530	if (is_cso && (card->dev->features & NETIF_F_RXCSUM)) {
5531	skb->ip_summed = CHECKSUM_UNNECESSARY;
5532	QETH_CARD_STAT_INC(card, rx_skb_csum);
5533	} else {
5534	skb->ip_summed = CHECKSUM_NONE;
5535	}
5536
5537	QETH_CARD_STAT_ADD(card, rx_bytes, skb->len);
5538	QETH_CARD_STAT_INC(card, rx_packets);
5539	if (skb_is_nonlinear(skb)) {
5540	QETH_CARD_STAT_INC(card, rx_sg_skbs);
5541	QETH_CARD_STAT_ADD(card, rx_sg_frags,
5542	skb_shinfo(skb)->nr_frags);
5543	}
5544
5545	if (uses_frags) {
5546	napi_gro_frags(napi);
5547	} else {
5548	skb->protocol = eth_type_trans(skb, dev: skb->dev);
5549	napi_gro_receive(napi, skb);
5550	}
5551	}
5552
5553	static void qeth_create_skb_frag(struct sk_buff *skb, char *data, int data_len)
5554	{
5555	struct page *page = virt_to_page(data);
5556	unsigned int next_frag;
5557
5558	next_frag = skb_shinfo(skb)->nr_frags;
5559	get_page(page);
5560	skb_add_rx_frag(skb, i: next_frag, page, offset_in_page(data), size: data_len,
5561	truesize: data_len);
5562	}
5563
5564	static inline int qeth_is_last_sbale(struct qdio_buffer_element *sbale)
5565	{
5566	return (sbale->eflags & SBAL_EFLAGS_LAST_ENTRY);
5567	}
5568
5569	static int qeth_extract_skb(struct qeth_card *card,
5570	struct qeth_qdio_buffer *qethbuffer, u8 *element_no,
5571	int *__offset)
5572	{
5573	struct qeth_priv *priv = netdev_priv(dev: card->dev);
5574	struct qdio_buffer *buffer = qethbuffer->buffer;
5575	struct napi_struct *napi = &card->napi;
5576	struct qdio_buffer_element *element;
5577	unsigned int linear_len = 0;
5578	bool uses_frags = false;
5579	int offset = *__offset;
5580	bool use_rx_sg = false;
5581	unsigned int headroom;
5582	struct qeth_hdr *hdr;
5583	struct sk_buff *skb;
5584	int skb_len = 0;
5585	bool is_cso;
5586
5587	element = &buffer->element[*element_no];
5588
5589	next_packet:
5590	/* qeth_hdr must not cross element boundaries */
5591	while (element->length < offset + sizeof(struct qeth_hdr)) {
5592	if (qeth_is_last_sbale(sbale: element))
5593	return -ENODATA;
5594	element++;
5595	offset = 0;
5596	}
5597
5598	hdr = dma64_to_virt(element->addr) + offset;
5599	offset += sizeof(*hdr);
5600	skb = NULL;
5601
5602	switch (hdr->hdr.l2.id) {
5603	case QETH_HEADER_TYPE_LAYER2:
5604	skb_len = hdr->hdr.l2.pkt_length;
5605	is_cso = hdr->hdr.l2.flags[1] & QETH_HDR_EXT_CSUM_TRANSP_REQ;
5606
5607	linear_len = ETH_HLEN;
5608	headroom = 0;
5609	break;
5610	case QETH_HEADER_TYPE_LAYER3:
5611	skb_len = hdr->hdr.l3.length;
5612	is_cso = hdr->hdr.l3.ext_flags & QETH_HDR_EXT_CSUM_TRANSP_REQ;
5613
5614	if (!IS_LAYER3(card)) {
5615	QETH_CARD_STAT_INC(card, rx_dropped_notsupp);
5616	goto walk_packet;
5617	}
5618
5619	if (hdr->hdr.l3.flags & QETH_HDR_PASSTHRU) {
5620	linear_len = ETH_HLEN;
5621	headroom = 0;
5622	break;
5623	}
5624
5625	if (hdr->hdr.l3.flags & QETH_HDR_IPV6)
5626	linear_len = sizeof(struct ipv6hdr);
5627	else
5628	linear_len = sizeof(struct iphdr);
5629	headroom = ETH_HLEN;
5630	break;
5631	default:
5632	if (hdr->hdr.l2.id & QETH_HEADER_MASK_INVAL)
5633	QETH_CARD_STAT_INC(card, rx_frame_errors);
5634	else
5635	QETH_CARD_STAT_INC(card, rx_dropped_notsupp);
5636
5637	/* Can't determine packet length, drop the whole buffer. */
5638	return -EPROTONOSUPPORT;
5639	}
5640
5641	if (skb_len < linear_len) {
5642	QETH_CARD_STAT_INC(card, rx_dropped_runt);
5643	goto walk_packet;
5644	}
5645
5646	use_rx_sg = (card->options.cq == QETH_CQ_ENABLED) ||
5647	(skb_len > READ_ONCE(priv->rx_copybreak) &&
5648	!atomic_read(v: &card->force_alloc_skb));
5649
5650	if (use_rx_sg) {
5651	/* QETH_CQ_ENABLED only: */
5652	if (qethbuffer->rx_skb &&
5653	skb_tailroom(skb: qethbuffer->rx_skb) >= linear_len + headroom) {
5654	skb = qethbuffer->rx_skb;
5655	qethbuffer->rx_skb = NULL;
5656	goto use_skb;
5657	}
5658
5659	skb = napi_get_frags(napi);
5660	if (!skb) {
5661	/* -ENOMEM, no point in falling back further. */
5662	QETH_CARD_STAT_INC(card, rx_dropped_nomem);
5663	goto walk_packet;
5664	}
5665
5666	if (skb_tailroom(skb) >= linear_len + headroom) {
5667	uses_frags = true;
5668	goto use_skb;
5669	}
5670
5671	netdev_info_once(card->dev,
5672	"Insufficient linear space in NAPI frags skb, need %u but have %u\n",
5673	linear_len + headroom, skb_tailroom(skb));
5674	/* Shouldn't happen. Don't optimize, fall back to linear skb. */
5675	}
5676
5677	linear_len = skb_len;
5678	skb = napi_alloc_skb(napi, length: linear_len + headroom);
5679	if (!skb) {
5680	QETH_CARD_STAT_INC(card, rx_dropped_nomem);
5681	goto walk_packet;
5682	}
5683
5684	use_skb:
5685	if (headroom)
5686	skb_reserve(skb, len: headroom);
5687	walk_packet:
5688	while (skb_len) {
5689	int data_len = min(skb_len, (int)(element->length - offset));
5690	char *data = dma64_to_virt(element->addr) + offset;
5691
5692	skb_len -= data_len;
5693	offset += data_len;
5694
5695	/* Extract data from current element: */
5696	if (skb && data_len) {
5697	if (linear_len) {
5698	unsigned int copy_len;
5699
5700	copy_len = min_t(unsigned int, linear_len,
5701	data_len);
5702
5703	skb_put_data(skb, data, len: copy_len);
5704	linear_len -= copy_len;
5705	data_len -= copy_len;
5706	data += copy_len;
5707	}
5708
5709	if (data_len)
5710	qeth_create_skb_frag(skb, data, data_len);
5711	}
5712
5713	/* Step forward to next element: */
5714	if (skb_len) {
5715	if (qeth_is_last_sbale(sbale: element)) {
5716	QETH_CARD_TEXT(card, 4, "unexeob");
5717	QETH_CARD_HEX(card, 2, buffer, sizeof(void *));
5718	if (skb) {
5719	if (uses_frags)
5720	napi_free_frags(napi);
5721	else
5722	kfree_skb(skb);
5723	QETH_CARD_STAT_INC(card,
5724	rx_length_errors);
5725	}
5726	return -EMSGSIZE;
5727	}
5728	element++;
5729	offset = 0;
5730	}
5731	}
5732
5733	/* This packet was skipped, go get another one: */
5734	if (!skb)
5735	goto next_packet;
5736
5737	*element_no = element - &buffer->element[0];
5738	*__offset = offset;
5739
5740	#if IS_ENABLED(CONFIG_QETH_L3)
5741	if (hdr->hdr.l2.id == QETH_HEADER_TYPE_LAYER3)
5742	qeth_l3_rebuild_skb(card, skb, hdr);
5743	#endif
5744
5745	qeth_receive_skb(card, skb, uses_frags, is_cso);
5746	return 0;
5747	}
5748
5749	static unsigned int qeth_extract_skbs(struct qeth_card *card, int budget,
5750	struct qeth_qdio_buffer *buf, bool *done)
5751	{
5752	unsigned int work_done = 0;
5753
5754	while (budget) {
5755	if (qeth_extract_skb(card, qethbuffer: buf, element_no: &card->rx.buf_element,
5756	offset: &card->rx.e_offset)) {
5757	*done = true;
5758	break;
5759	}
5760
5761	work_done++;
5762	budget--;
5763	}
5764
5765	return work_done;
5766	}
5767
5768	static unsigned int qeth_rx_poll(struct qeth_card *card, int budget)
5769	{
5770	struct qeth_rx *ctx = &card->rx;
5771	unsigned int work_done = 0;
5772
5773	while (budget > 0) {
5774	struct qeth_qdio_buffer *buffer;
5775	unsigned int skbs_done = 0;
5776	bool done = false;
5777
5778	/* Fetch completed RX buffers: */
5779	if (!card->rx.b_count) {
5780	card->rx.qdio_err = 0;
5781	card->rx.b_count =
5782	qdio_inspect_input_queue(CARD_DDEV(card), 0,
5783	&card->rx.b_index,
5784	&card->rx.qdio_err);
5785	if (card->rx.b_count <= 0) {
5786	card->rx.b_count = 0;
5787	break;
5788	}
5789	}
5790
5791	/* Process one completed RX buffer: */
5792	buffer = &card->qdio.in_q->bufs[card->rx.b_index];
5793	if (!(card->rx.qdio_err &&
5794	qeth_check_qdio_errors(card, buf: buffer->buffer,
5795	qdio_error: card->rx.qdio_err, dbftext: "qinerr")))
5796	skbs_done = qeth_extract_skbs(card, budget, buf: buffer,
5797	done: &done);
5798	else
5799	done = true;
5800
5801	work_done += skbs_done;
5802	budget -= skbs_done;
5803
5804	if (done) {
5805	QETH_CARD_STAT_INC(card, rx_bufs);
5806	qeth_put_buffer_pool_entry(card, entry: buffer->pool_entry);
5807	buffer->pool_entry = NULL;
5808	card->rx.b_count--;
5809	ctx->bufs_refill++;
5810	ctx->bufs_refill -= qeth_rx_refill_queue(card,
5811	count: ctx->bufs_refill);
5812
5813	/* Step forward to next buffer: */
5814	card->rx.b_index = QDIO_BUFNR(card->rx.b_index + 1);
5815	card->rx.buf_element = 0;
5816	card->rx.e_offset = 0;
5817	}
5818	}
5819
5820	return work_done;
5821	}
5822
5823	static void qeth_cq_poll(struct qeth_card *card)
5824	{
5825	unsigned int work_done = 0;
5826
5827	while (work_done < QDIO_MAX_BUFFERS_PER_Q) {
5828	unsigned int start, error;
5829	int completed;
5830
5831	completed = qdio_inspect_input_queue(CARD_DDEV(card), 1, &start,
5832	&error);
5833	if (completed <= 0)
5834	return;
5835
5836	qeth_qdio_cq_handler(card, qdio_err: error, queue: 1, first_element: start, count: completed);
5837	work_done += completed;
5838	}
5839	}
5840
5841	int qeth_poll(struct napi_struct *napi, int budget)
5842	{
5843	struct qeth_card *card = container_of(napi, struct qeth_card, napi);
5844	unsigned int work_done;
5845
5846	work_done = qeth_rx_poll(card, budget);
5847
5848	if (qeth_use_tx_irqs(card)) {
5849	struct qeth_qdio_out_q *queue;
5850	unsigned int i;
5851
5852	qeth_for_each_output_queue(card, queue, i) {
5853	if (!qeth_out_queue_is_empty(queue))
5854	napi_schedule(n: &queue->napi);
5855	}
5856	}
5857
5858	if (card->options.cq == QETH_CQ_ENABLED)
5859	qeth_cq_poll(card);
5860
5861	if (budget) {
5862	struct qeth_rx *ctx = &card->rx;
5863
5864	/* Process any substantial refill backlog: */
5865	ctx->bufs_refill -= qeth_rx_refill_queue(card, count: ctx->bufs_refill);
5866
5867	/* Exhausted the RX budget. Keep IRQ disabled, we get called again. */
5868	if (work_done >= budget)
5869	return work_done;
5870	}
5871
5872	if (napi_complete_done(n: napi, work_done) &&
5873	qdio_start_irq(CARD_DDEV(card)))
5874	napi_schedule(n: napi);
5875
5876	return work_done;
5877	}
5878	EXPORT_SYMBOL_GPL(qeth_poll);
5879
5880	static void qeth_iqd_tx_complete(struct qeth_qdio_out_q *queue,
5881	unsigned int bidx, unsigned int qdio_error,
5882	int budget)
5883	{
5884	struct qeth_qdio_out_buffer *buffer = queue->bufs[bidx];
5885	u8 sflags = buffer->buffer->element[15].sflags;
5886	struct qeth_card *card = queue->card;
5887	bool error = !!qdio_error;
5888
5889	if (qdio_error == QDIO_ERROR_SLSB_PENDING) {
5890	struct qaob *aob = buffer->aob;
5891	struct qeth_qaob_priv1 *priv;
5892	enum iucv_tx_notify notify;
5893
5894	if (!aob) {
5895	netdev_WARN_ONCE(card->dev,
5896	"Pending TX buffer %#x without QAOB on TX queue %u\n",
5897	bidx, queue->queue_no);
5898	qeth_schedule_recovery(card);
5899	return;
5900	}
5901
5902	QETH_CARD_TEXT_(card, 5, "pel%u", bidx);
5903
5904	priv = (struct qeth_qaob_priv1 *)&aob->user1;
5905	/* QAOB hasn't completed yet: */
5906	if (xchg(&priv->state, QETH_QAOB_PENDING) != QETH_QAOB_DONE) {
5907	qeth_notify_skbs(q: queue, buf: buffer, notification: TX_NOTIFY_PENDING);
5908
5909	/* Prepare the queue slot for immediate re-use: */
5910	qeth_scrub_qdio_buffer(buf: buffer->buffer, elements: queue->max_elements);
5911	if (qeth_alloc_out_buf(q: queue, bidx, GFP_ATOMIC)) {
5912	QETH_CARD_TEXT(card, 2, "outofbuf");
5913	qeth_schedule_recovery(card);
5914	}
5915
5916	list_add(new: &buffer->list_entry, head: &queue->pending_bufs);
5917	/* Skip clearing the buffer: */
5918	return;
5919	}
5920
5921	/* QAOB already completed: */
5922	notify = qeth_compute_cq_notification(sbalf15: aob->aorc, delayed: 0);
5923	qeth_notify_skbs(q: queue, buf: buffer, notification: notify);
5924	error = !!aob->aorc;
5925	memset(aob, 0, sizeof(*aob));
5926	} else if (card->options.cq == QETH_CQ_ENABLED) {
5927	qeth_notify_skbs(q: queue, buf: buffer,
5928	notification: qeth_compute_cq_notification(sbalf15: sflags, delayed: 0));
5929	}
5930
5931	qeth_clear_output_buffer(queue, buf: buffer, error, budget);
5932	}
5933
5934	static int qeth_tx_poll(struct napi_struct *napi, int budget)
5935	{
5936	struct qeth_qdio_out_q *queue = qeth_napi_to_out_queue(napi);
5937	unsigned int queue_no = queue->queue_no;
5938	struct qeth_card *card = queue->card;
5939	struct net_device *dev = card->dev;
5940	unsigned int work_done = 0;
5941	struct netdev_queue *txq;
5942
5943	if (IS_IQD(card))
5944	txq = netdev_get_tx_queue(dev, index: qeth_iqd_translate_txq(dev, txq: queue_no));
5945	else
5946	txq = netdev_get_tx_queue(dev, index: queue_no);
5947
5948	while (1) {
5949	unsigned int start, error, i;
5950	unsigned int packets = 0;
5951	unsigned int bytes = 0;
5952	int completed;
5953
5954	qeth_tx_complete_pending_bufs(card, queue, drain: false, budget);
5955
5956	if (qeth_out_queue_is_empty(queue)) {
5957	napi_complete(n: napi);
5958	return 0;
5959	}
5960
5961	/* Give the CPU a breather: */
5962	if (work_done >= QDIO_MAX_BUFFERS_PER_Q) {
5963	QETH_TXQ_STAT_INC(queue, completion_yield);
5964	if (napi_complete_done(n: napi, work_done: 0))
5965	napi_schedule(n: napi);
5966	return 0;
5967	}
5968
5969	completed = qdio_inspect_output_queue(CARD_DDEV(card), queue_no,
5970	&start, &error);
5971	if (completed <= 0) {
5972	/* Ensure we see TX completion for pending work: */
5973	if (napi_complete_done(n: napi, work_done: 0) &&
5974	!atomic_read(v: &queue->set_pci_flags_count))
5975	qeth_tx_arm_timer(queue, usecs: queue->rescan_usecs);
5976	return 0;
5977	}
5978
5979	for (i = start; i < start + completed; i++) {
5980	struct qeth_qdio_out_buffer *buffer;
5981	unsigned int bidx = QDIO_BUFNR(i);
5982
5983	buffer = queue->bufs[bidx];
5984	packets += buffer->frames;
5985	bytes += buffer->bytes;
5986
5987	qeth_handle_send_error(card, buffer, qdio_err: error);
5988	if (IS_IQD(card))
5989	qeth_iqd_tx_complete(queue, bidx, qdio_error: error, budget);
5990	else
5991	qeth_clear_output_buffer(queue, buf: buffer, error,
5992	budget);
5993	}
5994
5995	atomic_sub(i: completed, v: &queue->used_buffers);
5996	work_done += completed;
5997	if (IS_IQD(card))
5998	netdev_tx_completed_queue(dev_queue: txq, pkts: packets, bytes);
5999	else
6000	qeth_check_outbound_queue(queue);
6001
6002	/* xmit may have observed the full-condition, but not yet
6003	* stopped the txq. In which case the code below won't trigger.
6004	* So before returning, xmit will re-check the txq's fill level
6005	* and wake it up if needed.
6006	*/
6007	if (netif_tx_queue_stopped(dev_queue: txq) &&
6008	!qeth_out_queue_is_full(queue))
6009	netif_tx_wake_queue(dev_queue: txq);
6010	}
6011	}
6012
6013	static int qeth_setassparms_inspect_rc(struct qeth_ipa_cmd *cmd)
6014	{
6015	if (!cmd->hdr.return_code)
6016	cmd->hdr.return_code = cmd->data.setassparms.hdr.return_code;
6017	return cmd->hdr.return_code;
6018	}
6019
6020	static int qeth_setassparms_get_caps_cb(struct qeth_card *card,
6021	struct qeth_reply *reply,
6022	unsigned long data)
6023	{
6024	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
6025	struct qeth_ipa_caps *caps = reply->param;
6026
6027	if (qeth_setassparms_inspect_rc(cmd))
6028	return -EIO;
6029
6030	caps->supported = cmd->data.setassparms.data.caps.supported;
6031	caps->enabled = cmd->data.setassparms.data.caps.enabled;
6032	return 0;
6033	}
6034
6035	int qeth_setassparms_cb(struct qeth_card *card,
6036	struct qeth_reply *reply, unsigned long data)
6037	{
6038	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
6039
6040	QETH_CARD_TEXT(card, 4, "defadpcb");
6041
6042	if (cmd->hdr.return_code)
6043	return -EIO;
6044
6045	cmd->hdr.return_code = cmd->data.setassparms.hdr.return_code;
6046	if (cmd->hdr.prot_version == QETH_PROT_IPV4)
6047	card->options.ipa4.enabled = cmd->hdr.assists.enabled;
6048	if (cmd->hdr.prot_version == QETH_PROT_IPV6)
6049	card->options.ipa6.enabled = cmd->hdr.assists.enabled;
6050	return 0;
6051	}
6052	EXPORT_SYMBOL_GPL(qeth_setassparms_cb);
6053
6054	struct qeth_cmd_buffer *qeth_get_setassparms_cmd(struct qeth_card *card,
6055	enum qeth_ipa_funcs ipa_func,
6056	u16 cmd_code,
6057	unsigned int data_length,
6058	enum qeth_prot_versions prot)
6059	{
6060	struct qeth_ipacmd_setassparms *setassparms;
6061	struct qeth_ipacmd_setassparms_hdr *hdr;
6062	struct qeth_cmd_buffer *iob;
6063
6064	QETH_CARD_TEXT(card, 4, "getasscm");
6065	iob = qeth_ipa_alloc_cmd(card, IPA_CMD_SETASSPARMS, prot,
6066	data_length +
6067	offsetof(struct qeth_ipacmd_setassparms,
6068	data));
6069	if (!iob)
6070	return NULL;
6071
6072	setassparms = &__ipa_cmd(iob)->data.setassparms;
6073	setassparms->assist_no = ipa_func;
6074
6075	hdr = &setassparms->hdr;
6076	hdr->length = sizeof(*hdr) + data_length;
6077	hdr->command_code = cmd_code;
6078	return iob;
6079	}
6080	EXPORT_SYMBOL_GPL(qeth_get_setassparms_cmd);
6081
6082	int qeth_send_simple_setassparms_prot(struct qeth_card *card,
6083	enum qeth_ipa_funcs ipa_func,
6084	u16 cmd_code, u32 *data,
6085	enum qeth_prot_versions prot)
6086	{
6087	unsigned int length = data ? SETASS_DATA_SIZEOF(flags_32bit) : 0;
6088	struct qeth_cmd_buffer *iob;
6089
6090	QETH_CARD_TEXT_(card, 4, "simassp%i", prot);
6091	iob = qeth_get_setassparms_cmd(card, ipa_func, cmd_code, length, prot);
6092	if (!iob)
6093	return -ENOMEM;
6094
6095	if (data)
6096	__ipa_cmd(iob)->data.setassparms.data.flags_32bit = *data;
6097	return qeth_send_ipa_cmd(card, iob, qeth_setassparms_cb, NULL);
6098	}
6099	EXPORT_SYMBOL_GPL(qeth_send_simple_setassparms_prot);
6100
6101	static void qeth_unregister_dbf_views(void)
6102	{
6103	int x;
6104
6105	for (x = 0; x < QETH_DBF_INFOS; x++) {
6106	debug_unregister(qeth_dbf[x].id);
6107	qeth_dbf[x].id = NULL;
6108	}
6109	}
6110
6111	void qeth_dbf_longtext(debug_info_t *id, int level, char *fmt, ...)
6112	{
6113	char dbf_txt_buf[32];
6114	va_list args;
6115
6116	if (!debug_level_enabled(id, level))
6117	return;
6118	va_start(args, fmt);
6119	vscnprintf(buf: dbf_txt_buf, size: sizeof(dbf_txt_buf), fmt, args);
6120	va_end(args);
6121	debug_text_event(id, level, dbf_txt_buf);
6122	}
6123	EXPORT_SYMBOL_GPL(qeth_dbf_longtext);
6124
6125	static int qeth_register_dbf_views(void)
6126	{
6127	int ret;
6128	int x;
6129
6130	for (x = 0; x < QETH_DBF_INFOS; x++) {
6131	/* register the areas */
6132	qeth_dbf[x].id = debug_register(qeth_dbf[x].name,
6133	qeth_dbf[x].pages,
6134	qeth_dbf[x].areas,
6135	qeth_dbf[x].len);
6136	if (qeth_dbf[x].id == NULL) {
6137	qeth_unregister_dbf_views();
6138	return -ENOMEM;
6139	}
6140
6141	/* register a view */
6142	ret = debug_register_view(qeth_dbf[x].id, qeth_dbf[x].view);
6143	if (ret) {
6144	qeth_unregister_dbf_views();
6145	return ret;
6146	}
6147
6148	/* set a passing level */
6149	debug_set_level(qeth_dbf[x].id, qeth_dbf[x].level);
6150	}
6151
6152	return 0;
6153	}
6154
6155	static DEFINE_MUTEX(qeth_mod_mutex); /* for synchronized module loading */
6156
6157	int qeth_setup_discipline(struct qeth_card *card,
6158	enum qeth_discipline_id discipline)
6159	{
6160	int rc;
6161
6162	mutex_lock(&qeth_mod_mutex);
6163	switch (discipline) {
6164	case QETH_DISCIPLINE_LAYER3:
6165	card->discipline = try_then_request_module(
6166	symbol_get(qeth_l3_discipline), "qeth_l3");
6167	break;
6168	case QETH_DISCIPLINE_LAYER2:
6169	card->discipline = try_then_request_module(
6170	symbol_get(qeth_l2_discipline), "qeth_l2");
6171	break;
6172	default:
6173	break;
6174	}
6175	mutex_unlock(lock: &qeth_mod_mutex);
6176
6177	if (!card->discipline) {
6178	dev_err(&card->gdev->dev, "There is no kernel module to "
6179	"support discipline %d\n", discipline);
6180	return -EINVAL;
6181	}
6182
6183	rc = card->discipline->setup(card->gdev);
6184	if (rc) {
6185	if (discipline == QETH_DISCIPLINE_LAYER2)
6186	symbol_put(qeth_l2_discipline);
6187	else
6188	symbol_put(qeth_l3_discipline);
6189	card->discipline = NULL;
6190
6191	return rc;
6192	}
6193
6194	card->options.layer = discipline;
6195	return 0;
6196	}
6197
6198	void qeth_remove_discipline(struct qeth_card *card)
6199	{
6200	card->discipline->remove(card->gdev);
6201
6202	if (IS_LAYER2(card))
6203	symbol_put(qeth_l2_discipline);
6204	else
6205	symbol_put(qeth_l3_discipline);
6206	card->options.layer = QETH_DISCIPLINE_UNDETERMINED;
6207	card->discipline = NULL;
6208	}
6209
6210	static const struct device_type qeth_generic_devtype = {
6211	.name = "qeth_generic",
6212	};
6213
6214	#define DBF_NAME_LEN 20
6215
6216	struct qeth_dbf_entry {
6217	char dbf_name[DBF_NAME_LEN];
6218	debug_info_t *dbf_info;
6219	struct list_head dbf_list;
6220	};
6221
6222	static LIST_HEAD(qeth_dbf_list);
6223	static DEFINE_MUTEX(qeth_dbf_list_mutex);
6224
6225	static debug_info_t *qeth_get_dbf_entry(char *name)
6226	{
6227	struct qeth_dbf_entry *entry;
6228	debug_info_t *rc = NULL;
6229
6230	mutex_lock(&qeth_dbf_list_mutex);
6231	list_for_each_entry(entry, &qeth_dbf_list, dbf_list) {
6232	if (strcmp(entry->dbf_name, name) == 0) {
6233	rc = entry->dbf_info;
6234	break;
6235	}
6236	}
6237	mutex_unlock(lock: &qeth_dbf_list_mutex);
6238	return rc;
6239	}
6240
6241	static int qeth_add_dbf_entry(struct qeth_card *card, char *name)
6242	{
6243	struct qeth_dbf_entry *new_entry;
6244
6245	card->debug = debug_register(name, 2, 1, 8);
6246	if (!card->debug) {
6247	QETH_DBF_TEXT_(SETUP, 2, "%s", "qcdbf");
6248	goto err;
6249	}
6250	if (debug_register_view(card->debug, &debug_hex_ascii_view))
6251	goto err_dbg;
6252	new_entry = kzalloc(sizeof(struct qeth_dbf_entry), GFP_KERNEL);
6253	if (!new_entry)
6254	goto err_dbg;
6255	strscpy(new_entry->dbf_name, name, sizeof(new_entry->dbf_name));
6256	new_entry->dbf_info = card->debug;
6257	mutex_lock(&qeth_dbf_list_mutex);
6258	list_add(new: &new_entry->dbf_list, head: &qeth_dbf_list);
6259	mutex_unlock(lock: &qeth_dbf_list_mutex);
6260
6261	return 0;
6262
6263	err_dbg:
6264	debug_unregister(card->debug);
6265	err:
6266	return -ENOMEM;
6267	}
6268
6269	static void qeth_clear_dbf_list(void)
6270	{
6271	struct qeth_dbf_entry *entry, *tmp;
6272
6273	mutex_lock(&qeth_dbf_list_mutex);
6274	list_for_each_entry_safe(entry, tmp, &qeth_dbf_list, dbf_list) {
6275	list_del(entry: &entry->dbf_list);
6276	debug_unregister(entry->dbf_info);
6277	kfree(objp: entry);
6278	}
6279	mutex_unlock(lock: &qeth_dbf_list_mutex);
6280	}
6281
6282	static struct net_device *qeth_alloc_netdev(struct qeth_card *card)
6283	{
6284	struct net_device *dev;
6285	struct qeth_priv *priv;
6286
6287	switch (card->info.type) {
6288	case QETH_CARD_TYPE_IQD:
6289	dev = alloc_netdev_mqs(sizeof_priv: sizeof(*priv), name: "hsi%d", NET_NAME_UNKNOWN,
6290	setup: ether_setup, QETH_MAX_OUT_QUEUES, rxqs: 1);
6291	break;
6292	case QETH_CARD_TYPE_OSM:
6293	dev = alloc_etherdev(sizeof(*priv));
6294	break;
6295	default:
6296	dev = alloc_etherdev_mqs(sizeof_priv: sizeof(*priv), QETH_MAX_OUT_QUEUES, rxqs: 1);
6297	}
6298
6299	if (!dev)
6300	return NULL;
6301
6302	priv = netdev_priv(dev);
6303	priv->rx_copybreak = QETH_RX_COPYBREAK;
6304	priv->tx_wanted_queues = IS_IQD(card) ? QETH_IQD_MIN_TXQ : 1;
6305
6306	dev->ml_priv = card;
6307	dev->watchdog_timeo = QETH_TX_TIMEOUT;
6308	dev->min_mtu = 576;
6309	/* initialized when device first goes online: */
6310	dev->max_mtu = 0;
6311	dev->mtu = 0;
6312	SET_NETDEV_DEV(dev, &card->gdev->dev);
6313	netif_carrier_off(dev);
6314
6315	dev->ethtool_ops = &qeth_ethtool_ops;
6316	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
6317	dev->hw_features |= NETIF_F_SG;
6318	dev->vlan_features |= NETIF_F_SG;
6319	if (IS_IQD(card))
6320	dev->features |= NETIF_F_SG;
6321
6322	return dev;
6323	}
6324
6325	struct net_device *qeth_clone_netdev(struct net_device *orig)
6326	{
6327	struct net_device *clone = qeth_alloc_netdev(card: orig->ml_priv);
6328
6329	if (!clone)
6330	return NULL;
6331
6332	clone->dev_port = orig->dev_port;
6333	return clone;
6334	}
6335
6336	static int qeth_core_probe_device(struct ccwgroup_device *gdev)
6337	{
6338	struct qeth_card *card;
6339	struct device *dev;
6340	int rc;
6341	enum qeth_discipline_id enforced_disc;
6342	char dbf_name[DBF_NAME_LEN];
6343
6344	QETH_DBF_TEXT(SETUP, 2, "probedev");
6345
6346	dev = &gdev->dev;
6347	if (!get_device(dev))
6348	return -ENODEV;
6349
6350	QETH_DBF_TEXT_(SETUP, 2, "%s", dev_name(&gdev->dev));
6351
6352	card = qeth_alloc_card(gdev);
6353	if (!card) {
6354	QETH_DBF_TEXT_(SETUP, 2, "1err%d", -ENOMEM);
6355	rc = -ENOMEM;
6356	goto err_dev;
6357	}
6358
6359	scnprintf(buf: dbf_name, size: sizeof(dbf_name), fmt: "qeth_card_%s",
6360	dev_name(dev: &gdev->dev));
6361	card->debug = qeth_get_dbf_entry(dbf_name);
6362	if (!card->debug) {
6363	rc = qeth_add_dbf_entry(card, name: dbf_name);
6364	if (rc)
6365	goto err_card;
6366	}
6367
6368	qeth_setup_card(card);
6369	card->dev = qeth_alloc_netdev(card);
6370	if (!card->dev) {
6371	rc = -ENOMEM;
6372	goto err_card;
6373	}
6374
6375	qeth_determine_capabilities(card);
6376	qeth_set_blkt_defaults(card);
6377
6378	card->qdio.in_q = qeth_alloc_qdio_queue();
6379	if (!card->qdio.in_q) {
6380	rc = -ENOMEM;
6381	goto err_rx_queue;
6382	}
6383
6384	card->qdio.no_out_queues = card->dev->num_tx_queues;
6385	rc = qeth_update_from_chp_desc(card);
6386	if (rc)
6387	goto err_chp_desc;
6388
6389	gdev->dev.groups = qeth_dev_groups;
6390
6391	enforced_disc = qeth_enforce_discipline(card);
6392	switch (enforced_disc) {
6393	case QETH_DISCIPLINE_UNDETERMINED:
6394	gdev->dev.type = &qeth_generic_devtype;
6395	break;
6396	default:
6397	card->info.layer_enforced = true;
6398	/* It's so early that we don't need the discipline_mutex yet. */
6399	rc = qeth_setup_discipline(card, discipline: enforced_disc);
6400	if (rc)
6401	goto err_setup_disc;
6402
6403	break;
6404	}
6405
6406	return 0;
6407
6408	err_setup_disc:
6409	err_chp_desc:
6410	qeth_free_qdio_queue(q: card->qdio.in_q);
6411	err_rx_queue:
6412	free_netdev(dev: card->dev);
6413	err_card:
6414	qeth_core_free_card(card);
6415	err_dev:
6416	put_device(dev);
6417	return rc;
6418	}
6419
6420	static void qeth_core_remove_device(struct ccwgroup_device *gdev)
6421	{
6422	struct qeth_card *card = dev_get_drvdata(dev: &gdev->dev);
6423
6424	QETH_CARD_TEXT(card, 2, "removedv");
6425
6426	mutex_lock(&card->discipline_mutex);
6427	if (card->discipline)
6428	qeth_remove_discipline(card);
6429	mutex_unlock(lock: &card->discipline_mutex);
6430
6431	qeth_free_qdio_queues(card);
6432
6433	qeth_free_qdio_queue(q: card->qdio.in_q);
6434	free_netdev(dev: card->dev);
6435	qeth_core_free_card(card);
6436	put_device(dev: &gdev->dev);
6437	}
6438
6439	static int qeth_core_set_online(struct ccwgroup_device *gdev)
6440	{
6441	struct qeth_card *card = dev_get_drvdata(dev: &gdev->dev);
6442	int rc = 0;
6443	enum qeth_discipline_id def_discipline;
6444
6445	mutex_lock(&card->discipline_mutex);
6446	if (!card->discipline) {
6447	def_discipline = IS_IQD(card) ? QETH_DISCIPLINE_LAYER3 :
6448	QETH_DISCIPLINE_LAYER2;
6449	rc = qeth_setup_discipline(card, discipline: def_discipline);
6450	if (rc)
6451	goto err;
6452	}
6453
6454	rc = qeth_set_online(card, disc: card->discipline);
6455
6456	err:
6457	mutex_unlock(lock: &card->discipline_mutex);
6458	return rc;
6459	}
6460
6461	static int qeth_core_set_offline(struct ccwgroup_device *gdev)
6462	{
6463	struct qeth_card *card = dev_get_drvdata(dev: &gdev->dev);
6464	int rc;
6465
6466	mutex_lock(&card->discipline_mutex);
6467	rc = qeth_set_offline(card, card->discipline, false);
6468	mutex_unlock(lock: &card->discipline_mutex);
6469
6470	return rc;
6471	}
6472
6473	static void qeth_core_shutdown(struct ccwgroup_device *gdev)
6474	{
6475	struct qeth_card *card = dev_get_drvdata(dev: &gdev->dev);
6476
6477	qeth_set_allowed_threads(card, 0, 1);
6478	if ((gdev->state == CCWGROUP_ONLINE) && card->info.hwtrap)
6479	qeth_hw_trap(card, action: QETH_DIAGS_TRAP_DISARM);
6480	qeth_qdio_clear_card(card, use_halt: 0);
6481	qeth_drain_output_queues(card);
6482	qdio_free(CARD_DDEV(card));
6483	}
6484
6485	static ssize_t group_store(struct device_driver *ddrv, const char *buf,
6486	size_t count)
6487	{
6488	int err;
6489
6490	err = ccwgroup_create_dev(qeth_core_root_dev, to_ccwgroupdrv(ddrv), 3,
6491	buf);
6492
6493	return err ? err : count;
6494	}
6495	static DRIVER_ATTR_WO(group);
6496
6497	static struct attribute *qeth_drv_attrs[] = {
6498	&driver_attr_group.attr,
6499	NULL,
6500	};
6501	static struct attribute_group qeth_drv_attr_group = {
6502	.attrs = qeth_drv_attrs,
6503	};
6504	static const struct attribute_group *qeth_drv_attr_groups[] = {
6505	&qeth_drv_attr_group,
6506	NULL,
6507	};
6508
6509	static struct ccwgroup_driver qeth_core_ccwgroup_driver = {
6510	.driver = {
6511	.groups = qeth_drv_attr_groups,
6512	.owner = THIS_MODULE,
6513	.name = "qeth",
6514	},
6515	.ccw_driver = &qeth_ccw_driver,
6516	.setup = qeth_core_probe_device,
6517	.remove = qeth_core_remove_device,
6518	.set_online = qeth_core_set_online,
6519	.set_offline = qeth_core_set_offline,
6520	.shutdown = qeth_core_shutdown,
6521	};
6522
6523	int qeth_siocdevprivate(struct net_device *dev, struct ifreq *rq, void __user *data, int cmd)
6524	{
6525	struct qeth_card *card = dev->ml_priv;
6526	int rc = 0;
6527
6528	switch (cmd) {
6529	case SIOC_QETH_ADP_SET_SNMP_CONTROL:
6530	rc = qeth_snmp_command(card, udata: data);
6531	break;
6532	case SIOC_QETH_GET_CARD_TYPE:
6533	if ((IS_OSD(card) || IS_OSM(card) || IS_OSX(card)) &&
6534	!IS_VM_NIC(card))
6535	return 1;
6536	return 0;
6537	case SIOC_QETH_QUERY_OAT:
6538	rc = qeth_query_oat_command(card, udata: data);
6539	break;
6540	default:
6541	rc = -EOPNOTSUPP;
6542	}
6543	if (rc)
6544	QETH_CARD_TEXT_(card, 2, "ioce%x", rc);
6545	return rc;
6546	}
6547	EXPORT_SYMBOL_GPL(qeth_siocdevprivate);
6548
6549	int qeth_do_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
6550	{
6551	struct qeth_card *card = dev->ml_priv;
6552	struct mii_ioctl_data *mii_data;
6553	int rc = 0;
6554
6555	switch (cmd) {
6556	case SIOCGMIIPHY:
6557	mii_data = if_mii(rq);
6558	mii_data->phy_id = 0;
6559	break;
6560	case SIOCGMIIREG:
6561	mii_data = if_mii(rq);
6562	if (mii_data->phy_id != 0)
6563	rc = -EINVAL;
6564	else
6565	mii_data->val_out = qeth_mdio_read(dev,
6566	phy_id: mii_data->phy_id, regnum: mii_data->reg_num);
6567	break;
6568	default:
6569	return -EOPNOTSUPP;
6570	}
6571	if (rc)
6572	QETH_CARD_TEXT_(card, 2, "ioce%x", rc);
6573	return rc;
6574	}
6575	EXPORT_SYMBOL_GPL(qeth_do_ioctl);
6576
6577	static int qeth_start_csum_cb(struct qeth_card *card, struct qeth_reply *reply,
6578	unsigned long data)
6579	{
6580	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
6581	u32 *features = reply->param;
6582
6583	if (qeth_setassparms_inspect_rc(cmd))
6584	return -EIO;
6585
6586	*features = cmd->data.setassparms.data.flags_32bit;
6587	return 0;
6588	}
6589
6590	static int qeth_set_csum_off(struct qeth_card *card, enum qeth_ipa_funcs cstype,
6591	enum qeth_prot_versions prot)
6592	{
6593	return qeth_send_simple_setassparms_prot(card, cstype, IPA_CMD_ASS_STOP,
6594	NULL, prot);
6595	}
6596
6597	static int qeth_set_csum_on(struct qeth_card *card, enum qeth_ipa_funcs cstype,
6598	enum qeth_prot_versions prot, u8 *lp2lp)
6599	{
6600	u32 required_features = QETH_IPA_CHECKSUM_UDP | QETH_IPA_CHECKSUM_TCP;
6601	struct qeth_cmd_buffer *iob;
6602	struct qeth_ipa_caps caps;
6603	u32 features;
6604	int rc;
6605
6606	/* some L3 HW requires combined L3+L4 csum offload: */
6607	if (IS_LAYER3(card) && prot == QETH_PROT_IPV4 &&
6608	cstype == IPA_OUTBOUND_CHECKSUM)
6609	required_features |= QETH_IPA_CHECKSUM_IP_HDR;
6610
6611	iob = qeth_get_setassparms_cmd(card, cstype, IPA_CMD_ASS_START, 0,
6612	prot);
6613	if (!iob)
6614	return -ENOMEM;
6615
6616	rc = qeth_send_ipa_cmd(card, iob, qeth_start_csum_cb, &features);
6617	if (rc)
6618	return rc;
6619
6620	if ((required_features & features) != required_features) {
6621	qeth_set_csum_off(card, cstype, prot);
6622	return -EOPNOTSUPP;
6623	}
6624
6625	iob = qeth_get_setassparms_cmd(card, cstype, IPA_CMD_ASS_ENABLE,
6626	SETASS_DATA_SIZEOF(flags_32bit),
6627	prot);
6628	if (!iob) {
6629	qeth_set_csum_off(card, cstype, prot);
6630	return -ENOMEM;
6631	}
6632
6633	if (features & QETH_IPA_CHECKSUM_LP2LP)
6634	required_features |= QETH_IPA_CHECKSUM_LP2LP;
6635	__ipa_cmd(iob)->data.setassparms.data.flags_32bit = required_features;
6636	rc = qeth_send_ipa_cmd(card, iob, qeth_setassparms_get_caps_cb, &caps);
6637	if (rc) {
6638	qeth_set_csum_off(card, cstype, prot);
6639	return rc;
6640	}
6641
6642	if (!qeth_ipa_caps_supported(caps: &caps, mask: required_features) ||
6643	!qeth_ipa_caps_enabled(caps: &caps, mask: required_features)) {
6644	qeth_set_csum_off(card, cstype, prot);
6645	return -EOPNOTSUPP;
6646	}
6647
6648	dev_info(&card->gdev->dev, "HW Checksumming (%sbound IPv%d) enabled\n",
6649	cstype == IPA_INBOUND_CHECKSUM ? "in" : "out", prot);
6650
6651	if (lp2lp)
6652	*lp2lp = qeth_ipa_caps_enabled(caps: &caps, mask: QETH_IPA_CHECKSUM_LP2LP);
6653
6654	return 0;
6655	}
6656
6657	static int qeth_set_ipa_csum(struct qeth_card *card, bool on, int cstype,
6658	enum qeth_prot_versions prot, u8 *lp2lp)
6659	{
6660	return on ? qeth_set_csum_on(card, cstype, prot, lp2lp) :
6661	qeth_set_csum_off(card, cstype, prot);
6662	}
6663
6664	static int qeth_start_tso_cb(struct qeth_card *card, struct qeth_reply *reply,
6665	unsigned long data)
6666	{
6667	struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data;
6668	struct qeth_tso_start_data *tso_data = reply->param;
6669
6670	if (qeth_setassparms_inspect_rc(cmd))
6671	return -EIO;
6672
6673	tso_data->mss = cmd->data.setassparms.data.tso.mss;
6674	tso_data->supported = cmd->data.setassparms.data.tso.supported;
6675	return 0;
6676	}
6677
6678	static int qeth_set_tso_off(struct qeth_card *card,
6679	enum qeth_prot_versions prot)
6680	{
6681	return qeth_send_simple_setassparms_prot(card, IPA_OUTBOUND_TSO,
6682	IPA_CMD_ASS_STOP, NULL, prot);
6683	}
6684
6685	static int qeth_set_tso_on(struct qeth_card *card,
6686	enum qeth_prot_versions prot)
6687	{
6688	struct qeth_tso_start_data tso_data;
6689	struct qeth_cmd_buffer *iob;
6690	struct qeth_ipa_caps caps;
6691	int rc;
6692
6693	iob = qeth_get_setassparms_cmd(card, IPA_OUTBOUND_TSO,
6694	IPA_CMD_ASS_START, 0, prot);
6695	if (!iob)
6696	return -ENOMEM;
6697
6698	rc = qeth_send_ipa_cmd(card, iob, qeth_start_tso_cb, &tso_data);
6699	if (rc)
6700	return rc;
6701
6702	if (!tso_data.mss || !(tso_data.supported & QETH_IPA_LARGE_SEND_TCP)) {
6703	qeth_set_tso_off(card, prot);
6704	return -EOPNOTSUPP;
6705	}
6706
6707	iob = qeth_get_setassparms_cmd(card, IPA_OUTBOUND_TSO,
6708	IPA_CMD_ASS_ENABLE,
6709	SETASS_DATA_SIZEOF(caps), prot);
6710	if (!iob) {
6711	qeth_set_tso_off(card, prot);
6712	return -ENOMEM;
6713	}
6714
6715	/* enable TSO capability */
6716	__ipa_cmd(iob)->data.setassparms.data.caps.enabled =
6717	QETH_IPA_LARGE_SEND_TCP;
6718	rc = qeth_send_ipa_cmd(card, iob, qeth_setassparms_get_caps_cb, &caps);
6719	if (rc) {
6720	qeth_set_tso_off(card, prot);
6721	return rc;
6722	}
6723
6724	if (!qeth_ipa_caps_supported(caps: &caps, mask: QETH_IPA_LARGE_SEND_TCP) ||
6725	!qeth_ipa_caps_enabled(caps: &caps, mask: QETH_IPA_LARGE_SEND_TCP)) {
6726	qeth_set_tso_off(card, prot);
6727	return -EOPNOTSUPP;
6728	}
6729
6730	dev_info(&card->gdev->dev, "TSOv%u enabled (MSS: %u)\n", prot,
6731	tso_data.mss);
6732	return 0;
6733	}
6734
6735	static int qeth_set_ipa_tso(struct qeth_card *card, bool on,
6736	enum qeth_prot_versions prot)
6737	{
6738	return on ? qeth_set_tso_on(card, prot) : qeth_set_tso_off(card, prot);
6739	}
6740
6741	static int qeth_set_ipa_rx_csum(struct qeth_card *card, bool on)
6742	{
6743	int rc_ipv4 = (on) ? -EOPNOTSUPP : 0;
6744	int rc_ipv6;
6745
6746	if (qeth_is_supported(card, IPA_INBOUND_CHECKSUM))
6747	rc_ipv4 = qeth_set_ipa_csum(card, on, cstype: IPA_INBOUND_CHECKSUM,
6748	prot: QETH_PROT_IPV4, NULL);
6749	if (!qeth_is_supported6(card, IPA_INBOUND_CHECKSUM_V6))
6750	/* no/one Offload Assist available, so the rc is trivial */
6751	return rc_ipv4;
6752
6753	rc_ipv6 = qeth_set_ipa_csum(card, on, cstype: IPA_INBOUND_CHECKSUM,
6754	prot: QETH_PROT_IPV6, NULL);
6755
6756	if (on)
6757	/* enable: success if any Assist is active */
6758	return (rc_ipv6) ? rc_ipv4 : 0;
6759
6760	/* disable: failure if any Assist is still active */
6761	return (rc_ipv6) ? rc_ipv6 : rc_ipv4;
6762	}
6763
6764	/**
6765	* qeth_enable_hw_features() - (Re-)Enable HW functions for device features
6766	* @dev: a net_device
6767	*/
6768	void qeth_enable_hw_features(struct net_device *dev)
6769	{
6770	struct qeth_card *card = dev->ml_priv;
6771	netdev_features_t features;
6772
6773	features = dev->features;
6774	/* force-off any feature that might need an IPA sequence.
6775	* netdev_update_features() will restart them.
6776	*/
6777	dev->features &= ~dev->hw_features;
6778	/* toggle VLAN filter, so that VIDs are re-programmed: */
6779	if (IS_LAYER2(card) && IS_VM_NIC(card)) {
6780	dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
6781	dev->wanted_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
6782	}
6783	netdev_update_features(dev);
6784	if (features != dev->features)
6785	dev_warn(&card->gdev->dev,
6786	"Device recovery failed to restore all offload features\n");
6787	}
6788	EXPORT_SYMBOL_GPL(qeth_enable_hw_features);
6789
6790	static void qeth_check_restricted_features(struct qeth_card *card,
6791	netdev_features_t changed,
6792	netdev_features_t actual)
6793	{
6794	netdev_features_t ipv6_features = NETIF_F_TSO6;
6795	netdev_features_t ipv4_features = NETIF_F_TSO;
6796
6797	if (!card->info.has_lp2lp_cso_v6)
6798	ipv6_features |= NETIF_F_IPV6_CSUM;
6799	if (!card->info.has_lp2lp_cso_v4)
6800	ipv4_features |= NETIF_F_IP_CSUM;
6801
6802	if ((changed & ipv6_features) && !(actual & ipv6_features))
6803	qeth_flush_local_addrs6(card);
6804	if ((changed & ipv4_features) && !(actual & ipv4_features))
6805	qeth_flush_local_addrs4(card);
6806	}
6807
6808	int qeth_set_features(struct net_device *dev, netdev_features_t features)
6809	{
6810	struct qeth_card *card = dev->ml_priv;
6811	netdev_features_t changed = dev->features ^ features;
6812	int rc = 0;
6813
6814	QETH_CARD_TEXT(card, 2, "setfeat");
6815	QETH_CARD_HEX(card, 2, &features, sizeof(features));
6816
6817	if ((changed & NETIF_F_IP_CSUM)) {
6818	rc = qeth_set_ipa_csum(card, on: features & NETIF_F_IP_CSUM,
6819	cstype: IPA_OUTBOUND_CHECKSUM, prot: QETH_PROT_IPV4,
6820	lp2lp: &card->info.has_lp2lp_cso_v4);
6821	if (rc)
6822	changed ^= NETIF_F_IP_CSUM;
6823	}
6824	if (changed & NETIF_F_IPV6_CSUM) {
6825	rc = qeth_set_ipa_csum(card, on: features & NETIF_F_IPV6_CSUM,
6826	cstype: IPA_OUTBOUND_CHECKSUM, prot: QETH_PROT_IPV6,
6827	lp2lp: &card->info.has_lp2lp_cso_v6);
6828	if (rc)
6829	changed ^= NETIF_F_IPV6_CSUM;
6830	}
6831	if (changed & NETIF_F_RXCSUM) {
6832	rc = qeth_set_ipa_rx_csum(card, on: features & NETIF_F_RXCSUM);
6833	if (rc)
6834	changed ^= NETIF_F_RXCSUM;
6835	}
6836	if (changed & NETIF_F_TSO) {
6837	rc = qeth_set_ipa_tso(card, on: features & NETIF_F_TSO,
6838	prot: QETH_PROT_IPV4);
6839	if (rc)
6840	changed ^= NETIF_F_TSO;
6841	}
6842	if (changed & NETIF_F_TSO6) {
6843	rc = qeth_set_ipa_tso(card, on: features & NETIF_F_TSO6,
6844	prot: QETH_PROT_IPV6);
6845	if (rc)
6846	changed ^= NETIF_F_TSO6;
6847	}
6848
6849	qeth_check_restricted_features(card, changed: dev->features ^ features,
6850	actual: dev->features ^ changed);
6851
6852	/* everything changed successfully? */
6853	if ((dev->features ^ features) == changed)
6854	return 0;
6855	/* something went wrong. save changed features and return error */
6856	dev->features ^= changed;
6857	return -EIO;
6858	}
6859	EXPORT_SYMBOL_GPL(qeth_set_features);
6860
6861	netdev_features_t qeth_fix_features(struct net_device *dev,
6862	netdev_features_t features)
6863	{
6864	struct qeth_card *card = dev->ml_priv;
6865
6866	QETH_CARD_TEXT(card, 2, "fixfeat");
6867	if (!qeth_is_supported(card, IPA_OUTBOUND_CHECKSUM))
6868	features &= ~NETIF_F_IP_CSUM;
6869	if (!qeth_is_supported6(card, IPA_OUTBOUND_CHECKSUM_V6))
6870	features &= ~NETIF_F_IPV6_CSUM;
6871	if (!qeth_is_supported(card, IPA_INBOUND_CHECKSUM) &&
6872	!qeth_is_supported6(card, IPA_INBOUND_CHECKSUM_V6))
6873	features &= ~NETIF_F_RXCSUM;
6874	if (!qeth_is_supported(card, IPA_OUTBOUND_TSO))
6875	features &= ~NETIF_F_TSO;
6876	if (!qeth_is_supported6(card, IPA_OUTBOUND_TSO))
6877	features &= ~NETIF_F_TSO6;
6878
6879	QETH_CARD_HEX(card, 2, &features, sizeof(features));
6880	return features;
6881	}
6882	EXPORT_SYMBOL_GPL(qeth_fix_features);
6883
6884	netdev_features_t qeth_features_check(struct sk_buff *skb,
6885	struct net_device *dev,
6886	netdev_features_t features)
6887	{
6888	struct qeth_card *card = dev->ml_priv;
6889
6890	/* Traffic with local next-hop is not eligible for some offloads: */
6891	if (skb->ip_summed == CHECKSUM_PARTIAL &&
6892	READ_ONCE(card->options.isolation) != ISOLATION_MODE_FWD) {
6893	netdev_features_t restricted = 0;
6894
6895	if (skb_is_gso(skb) && !netif_needs_gso(skb, features))
6896	restricted |= NETIF_F_ALL_TSO;
6897
6898	switch (vlan_get_protocol(skb)) {
6899	case htons(ETH_P_IP):
6900	if (!card->info.has_lp2lp_cso_v4)
6901	restricted |= NETIF_F_IP_CSUM;
6902
6903	if (restricted && qeth_next_hop_is_local_v4(card, skb))
6904	features &= ~restricted;
6905	break;
6906	case htons(ETH_P_IPV6):
6907	if (!card->info.has_lp2lp_cso_v6)
6908	restricted |= NETIF_F_IPV6_CSUM;
6909
6910	if (restricted && qeth_next_hop_is_local_v6(card, skb))
6911	features &= ~restricted;
6912	break;
6913	default:
6914	break;
6915	}
6916	}
6917
6918	/* GSO segmentation builds skbs with
6919	* a (small) linear part for the headers, and
6920	* page frags for the data.
6921	* Compared to a linear skb, the header-only part consumes an
6922	* additional buffer element. This reduces buffer utilization, and
6923	* hurts throughput. So compress small segments into one element.
6924	*/
6925	if (netif_needs_gso(skb, features)) {
6926	/* match skb_segment(): */
6927	unsigned int doffset = skb->data - skb_mac_header(skb);
6928	unsigned int hsize = skb_shinfo(skb)->gso_size;
6929	unsigned int hroom = skb_headroom(skb);
6930
6931	/* linearize only if resulting skb allocations are order-0: */
6932	if (SKB_DATA_ALIGN(hroom + doffset + hsize) <= SKB_MAX_HEAD(0))
6933	features &= ~NETIF_F_SG;
6934	}
6935
6936	return vlan_features_check(skb, features);
6937	}
6938	EXPORT_SYMBOL_GPL(qeth_features_check);
6939
6940	void qeth_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
6941	{
6942	struct qeth_card *card = dev->ml_priv;
6943	struct qeth_qdio_out_q *queue;
6944	unsigned int i;
6945
6946	QETH_CARD_TEXT(card, 5, "getstat");
6947
6948	stats->rx_packets = card->stats.rx_packets;
6949	stats->rx_bytes = card->stats.rx_bytes;
6950	stats->rx_errors = card->stats.rx_length_errors +
6951	card->stats.rx_frame_errors +
6952	card->stats.rx_fifo_errors;
6953	stats->rx_dropped = card->stats.rx_dropped_nomem +
6954	card->stats.rx_dropped_notsupp +
6955	card->stats.rx_dropped_runt;
6956	stats->multicast = card->stats.rx_multicast;
6957	stats->rx_length_errors = card->stats.rx_length_errors;
6958	stats->rx_frame_errors = card->stats.rx_frame_errors;
6959	stats->rx_fifo_errors = card->stats.rx_fifo_errors;
6960
6961	for (i = 0; i < card->qdio.no_out_queues; i++) {
6962	queue = card->qdio.out_qs[i];
6963
6964	stats->tx_packets += queue->stats.tx_packets;
6965	stats->tx_bytes += queue->stats.tx_bytes;
6966	stats->tx_errors += queue->stats.tx_errors;
6967	stats->tx_dropped += queue->stats.tx_dropped;
6968	}
6969	}
6970	EXPORT_SYMBOL_GPL(qeth_get_stats64);
6971
6972	#define TC_IQD_UCAST 0
6973	static void qeth_iqd_set_prio_tc_map(struct net_device *dev,
6974	unsigned int ucast_txqs)
6975	{
6976	unsigned int prio;
6977
6978	/* IQD requires mcast traffic to be placed on a dedicated queue, and
6979	* qeth_iqd_select_queue() deals with this.
6980	* For unicast traffic, we defer the queue selection to the stack.
6981	* By installing a trivial prio map that spans over only the unicast
6982	* queues, we can encourage the stack to spread the ucast traffic evenly
6983	* without selecting the mcast queue.
6984	*/
6985
6986	/* One traffic class, spanning over all active ucast queues: */
6987	netdev_set_num_tc(dev, num_tc: 1);
6988	netdev_set_tc_queue(dev, TC_IQD_UCAST, count: ucast_txqs,
6989	QETH_IQD_MIN_UCAST_TXQ);
6990
6991	/* Map all priorities to this traffic class: */
6992	for (prio = 0; prio <= TC_BITMASK; prio++)
6993	netdev_set_prio_tc_map(dev, prio, TC_IQD_UCAST);
6994	}
6995
6996	int qeth_set_real_num_tx_queues(struct qeth_card *card, unsigned int count)
6997	{
6998	struct net_device *dev = card->dev;
6999	int rc;
7000
7001	/* Per netif_setup_tc(), adjust the mapping first: */
7002	if (IS_IQD(card))
7003	qeth_iqd_set_prio_tc_map(dev, ucast_txqs: count - 1);
7004
7005	rc = netif_set_real_num_tx_queues(dev, txq: count);
7006
7007	if (rc && IS_IQD(card))
7008	qeth_iqd_set_prio_tc_map(dev, ucast_txqs: dev->real_num_tx_queues - 1);
7009
7010	return rc;
7011	}
7012	EXPORT_SYMBOL_GPL(qeth_set_real_num_tx_queues);
7013
7014	u16 qeth_iqd_select_queue(struct net_device *dev, struct sk_buff *skb,
7015	u8 cast_type, struct net_device *sb_dev)
7016	{
7017	u16 txq;
7018
7019	if (cast_type != RTN_UNICAST)
7020	return QETH_IQD_MCAST_TXQ;
7021	if (dev->real_num_tx_queues == QETH_IQD_MIN_TXQ)
7022	return QETH_IQD_MIN_UCAST_TXQ;
7023
7024	txq = netdev_pick_tx(dev, skb, sb_dev);
7025	return (txq == QETH_IQD_MCAST_TXQ) ? QETH_IQD_MIN_UCAST_TXQ : txq;
7026	}
7027	EXPORT_SYMBOL_GPL(qeth_iqd_select_queue);
7028
7029	u16 qeth_osa_select_queue(struct net_device *dev, struct sk_buff *skb,
7030	struct net_device *sb_dev)
7031	{
7032	struct qeth_card *card = dev->ml_priv;
7033
7034	if (qeth_uses_tx_prio_queueing(card))
7035	return qeth_get_priority_queue(card, skb);
7036
7037	return netdev_pick_tx(dev, skb, sb_dev);
7038	}
7039	EXPORT_SYMBOL_GPL(qeth_osa_select_queue);
7040
7041	int qeth_open(struct net_device *dev)
7042	{
7043	struct qeth_card *card = dev->ml_priv;
7044	struct qeth_qdio_out_q *queue;
7045	unsigned int i;
7046
7047	QETH_CARD_TEXT(card, 4, "qethopen");
7048
7049	card->data.state = CH_STATE_UP;
7050	netif_tx_start_all_queues(dev);
7051
7052	qeth_for_each_output_queue(card, queue, i) {
7053	netif_napi_add_tx(dev, napi: &queue->napi, poll: qeth_tx_poll);
7054	napi_enable(n: &queue->napi);
7055	}
7056	napi_enable(n: &card->napi);
7057
7058	local_bh_disable();
7059	qeth_for_each_output_queue(card, queue, i) {
7060	napi_schedule(n: &queue->napi);
7061	}
7062	napi_schedule(n: &card->napi);
7063	/* kick-start the NAPI softirq: */
7064	local_bh_enable();
7065
7066	return 0;
7067	}
7068	EXPORT_SYMBOL_GPL(qeth_open);
7069
7070	int qeth_stop(struct net_device *dev)
7071	{
7072	struct qeth_card *card = dev->ml_priv;
7073	struct qeth_qdio_out_q *queue;
7074	unsigned int i;
7075
7076	QETH_CARD_TEXT(card, 4, "qethstop");
7077
7078	napi_disable(n: &card->napi);
7079	cancel_delayed_work_sync(dwork: &card->buffer_reclaim_work);
7080	qdio_stop_irq(CARD_DDEV(card));
7081
7082	/* Quiesce the NAPI instances: */
7083	qeth_for_each_output_queue(card, queue, i)
7084	napi_disable(n: &queue->napi);
7085
7086	/* Stop .ndo_start_xmit, might still access queue->napi. */
7087	netif_tx_disable(dev);
7088
7089	qeth_for_each_output_queue(card, queue, i) {
7090	timer_delete_sync(timer: &queue->timer);
7091	/* Queues may get re-allocated, so remove the NAPIs. */
7092	netif_napi_del(napi: &queue->napi);
7093	}
7094
7095	return 0;
7096	}
7097	EXPORT_SYMBOL_GPL(qeth_stop);
7098
7099	static int __init qeth_core_init(void)
7100	{
7101	int rc;
7102
7103	pr_info("loading core functions\n");
7104
7105	qeth_debugfs_root = debugfs_create_dir(name: "qeth", NULL);
7106
7107	rc = qeth_register_dbf_views();
7108	if (rc)
7109	goto dbf_err;
7110	qeth_core_root_dev = root_device_register("qeth");
7111	rc = PTR_ERR_OR_ZERO(ptr: qeth_core_root_dev);
7112	if (rc)
7113	goto register_err;
7114	qeth_core_header_cache =
7115	kmem_cache_create("qeth_hdr", QETH_HDR_CACHE_OBJ_SIZE,
7116	roundup_pow_of_two(QETH_HDR_CACHE_OBJ_SIZE),
7117	0, NULL);
7118	if (!qeth_core_header_cache) {
7119	rc = -ENOMEM;
7120	goto slab_err;
7121	}
7122	qeth_qdio_outbuf_cache = kmem_cache_create("qeth_buf",
7123	sizeof(struct qeth_qdio_out_buffer), 0, 0, NULL);
7124	if (!qeth_qdio_outbuf_cache) {
7125	rc = -ENOMEM;
7126	goto cqslab_err;
7127	}
7128
7129	qeth_qaob_cache = kmem_cache_create("qeth_qaob",
7130	sizeof(struct qaob),
7131	sizeof(struct qaob),
7132	0, NULL);
7133	if (!qeth_qaob_cache) {
7134	rc = -ENOMEM;
7135	goto qaob_err;
7136	}
7137
7138	rc = ccw_driver_register(&qeth_ccw_driver);
7139	if (rc)
7140	goto ccw_err;
7141	rc = ccwgroup_driver_register(&qeth_core_ccwgroup_driver);
7142	if (rc)
7143	goto ccwgroup_err;
7144
7145	return 0;
7146
7147	ccwgroup_err:
7148	ccw_driver_unregister(&qeth_ccw_driver);
7149	ccw_err:
7150	kmem_cache_destroy(s: qeth_qaob_cache);
7151	qaob_err:
7152	kmem_cache_destroy(s: qeth_qdio_outbuf_cache);
7153	cqslab_err:
7154	kmem_cache_destroy(s: qeth_core_header_cache);
7155	slab_err:
7156	root_device_unregister(root: qeth_core_root_dev);
7157	register_err:
7158	qeth_unregister_dbf_views();
7159	dbf_err:
7160	debugfs_remove_recursive(dentry: qeth_debugfs_root);
7161	pr_err("Initializing the qeth device driver failed\n");
7162	return rc;
7163	}
7164
7165	static void __exit qeth_core_exit(void)
7166	{
7167	qeth_clear_dbf_list();
7168	ccwgroup_driver_unregister(&qeth_core_ccwgroup_driver);
7169	ccw_driver_unregister(&qeth_ccw_driver);
7170	kmem_cache_destroy(s: qeth_qaob_cache);
7171	kmem_cache_destroy(s: qeth_qdio_outbuf_cache);
7172	kmem_cache_destroy(s: qeth_core_header_cache);
7173	root_device_unregister(root: qeth_core_root_dev);
7174	qeth_unregister_dbf_views();
7175	debugfs_remove_recursive(dentry: qeth_debugfs_root);
7176	pr_info("core functions removed\n");
7177	}
7178
7179	module_init(qeth_core_init);
7180	module_exit(qeth_core_exit);
7181	MODULE_AUTHOR("Frank Blaschka <frank.blaschka@de.ibm.com>");
7182	MODULE_DESCRIPTION("qeth core functions");
7183	MODULE_LICENSE("GPL");
7184