1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/* Copyright (c) 2015 - 2021 Intel Corporation */
3	#include "main.h"
4	#include "trace.h"
5
6	static void irdma_cm_post_event(struct irdma_cm_event *event);
7	static void irdma_disconnect_worker(struct work_struct *work);
8
9	/**
10	* irdma_free_sqbuf - put back puda buffer if refcount is 0
11	* @vsi: The VSI structure of the device
12	* @bufp: puda buffer to free
13	*/
14	void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp)
15	{
16	struct irdma_puda_buf *buf = bufp;
17	struct irdma_puda_rsrc *ilq = vsi->ilq;
18
19	if (refcount_dec_and_test(r: &buf->refcount))
20	irdma_puda_ret_bufpool(rsrc: ilq, buf);
21	}
22
23	/**
24	* irdma_record_ird_ord - Record IRD/ORD passed in
25	* @cm_node: connection's node
26	* @conn_ird: connection IRD
27	* @conn_ord: connection ORD
28	*/
29	static void irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird,
30	u32 conn_ord)
31	{
32	if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird)
33	conn_ird = cm_node->dev->hw_attrs.max_hw_ird;
34
35	if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord)
36	conn_ord = cm_node->dev->hw_attrs.max_hw_ord;
37	else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
38	conn_ord = 1;
39	cm_node->ird_size = conn_ird;
40	cm_node->ord_size = conn_ord;
41	}
42
43	/**
44	* irdma_copy_ip_ntohl - copy IP address from network to host
45	* @dst: IP address in host order
46	* @src: IP address in network order (big endian)
47	*/
48	void irdma_copy_ip_ntohl(u32 *dst, __be32 *src)
49	{
50	*dst++ = ntohl(*src++);
51	*dst++ = ntohl(*src++);
52	*dst++ = ntohl(*src++);
53	*dst = ntohl(*src);
54	}
55
56	/**
57	* irdma_copy_ip_htonl - copy IP address from host to network order
58	* @dst: IP address in network order (big endian)
59	* @src: IP address in host order
60	*/
61	void irdma_copy_ip_htonl(__be32 *dst, u32 *src)
62	{
63	*dst++ = htonl(*src++);
64	*dst++ = htonl(*src++);
65	*dst++ = htonl(*src++);
66	*dst = htonl(*src);
67	}
68
69	/**
70	* irdma_get_addr_info
71	* @cm_node: contains ip/tcp info
72	* @cm_info: to get a copy of the cm_node ip/tcp info
73	*/
74	static void irdma_get_addr_info(struct irdma_cm_node *cm_node,
75	struct irdma_cm_info *cm_info)
76	{
77	memset(cm_info, 0, sizeof(*cm_info));
78	cm_info->ipv4 = cm_node->ipv4;
79	cm_info->vlan_id = cm_node->vlan_id;
80	memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr));
81	memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));
82	cm_info->loc_port = cm_node->loc_port;
83	cm_info->rem_port = cm_node->rem_port;
84	}
85
86	/**
87	* irdma_fill_sockaddr4 - fill in addr info for IPv4 connection
88	* @cm_node: connection's node
89	* @event: upper layer's cm event
90	*/
91	static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node,
92	struct iw_cm_event *event)
93	{
94	struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr;
95	struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr;
96
97	laddr->sin_family = AF_INET;
98	raddr->sin_family = AF_INET;
99
100	laddr->sin_port = htons(cm_node->loc_port);
101	raddr->sin_port = htons(cm_node->rem_port);
102
103	laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);
104	raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);
105	}
106
107	/**
108	* irdma_fill_sockaddr6 - fill in addr info for IPv6 connection
109	* @cm_node: connection's node
110	* @event: upper layer's cm event
111	*/
112	static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node,
113	struct iw_cm_event *event)
114	{
115	struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr;
116	struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr;
117
118	laddr6->sin6_family = AF_INET6;
119	raddr6->sin6_family = AF_INET6;
120
121	laddr6->sin6_port = htons(cm_node->loc_port);
122	raddr6->sin6_port = htons(cm_node->rem_port);
123
124	irdma_copy_ip_htonl(dst: laddr6->sin6_addr.in6_u.u6_addr32,
125	src: cm_node->loc_addr);
126	irdma_copy_ip_htonl(dst: raddr6->sin6_addr.in6_u.u6_addr32,
127	src: cm_node->rem_addr);
128	}
129
130	/**
131	* irdma_get_cmevent_info - for cm event upcall
132	* @cm_node: connection's node
133	* @cm_id: upper layers cm struct for the event
134	* @event: upper layer's cm event
135	*/
136	static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node,
137	struct iw_cm_id *cm_id,
138	struct iw_cm_event *event)
139	{
140	memcpy(&event->local_addr, &cm_id->m_local_addr,
141	sizeof(event->local_addr));
142	memcpy(&event->remote_addr, &cm_id->m_remote_addr,
143	sizeof(event->remote_addr));
144	if (cm_node) {
145	event->private_data = cm_node->pdata_buf;
146	event->private_data_len = (u8)cm_node->pdata.size;
147	event->ird = cm_node->ird_size;
148	event->ord = cm_node->ord_size;
149	}
150	}
151
152	/**
153	* irdma_send_cm_event - upcall cm's event handler
154	* @cm_node: connection's node
155	* @cm_id: upper layer's cm info struct
156	* @type: Event type to indicate
157	* @status: status for the event type
158	*/
159	static int irdma_send_cm_event(struct irdma_cm_node *cm_node,
160	struct iw_cm_id *cm_id,
161	enum iw_cm_event_type type, int status)
162	{
163	struct iw_cm_event event = {};
164
165	event.event = type;
166	event.status = status;
167	trace_irdma_send_cm_event(cm_node, cm_id, type, status,
168	caller: __builtin_return_address(0));
169
170	ibdev_dbg(&cm_node->iwdev->ibdev,
171	"CM: cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n",
172	cm_node, cm_id, cm_node->accelerated, cm_node->state, type,
173	status);
174
175	switch (type) {
176	case IW_CM_EVENT_CONNECT_REQUEST:
177	if (cm_node->ipv4)
178	irdma_fill_sockaddr4(cm_node, event: &event);
179	else
180	irdma_fill_sockaddr6(cm_node, event: &event);
181	event.provider_data = cm_node;
182	event.private_data = cm_node->pdata_buf;
183	event.private_data_len = (u8)cm_node->pdata.size;
184	event.ird = cm_node->ird_size;
185	break;
186	case IW_CM_EVENT_CONNECT_REPLY:
187	irdma_get_cmevent_info(cm_node, cm_id, event: &event);
188	break;
189	case IW_CM_EVENT_ESTABLISHED:
190	event.ird = cm_node->ird_size;
191	event.ord = cm_node->ord_size;
192	break;
193	case IW_CM_EVENT_DISCONNECT:
194	case IW_CM_EVENT_CLOSE:
195	/* Wait if we are in RTS but havent issued the iwcm event upcall */
196	if (!cm_node->accelerated)
197	wait_for_completion(&cm_node->establish_comp);
198	break;
199	default:
200	return -EINVAL;
201	}
202
203	return cm_id->event_handler(cm_id, &event);
204	}
205
206	/**
207	* irdma_timer_list_prep - add connection nodes to a list to perform timer tasks
208	* @cm_core: cm's core
209	* @timer_list: a timer list to which cm_node will be selected
210	*/
211	static void irdma_timer_list_prep(struct irdma_cm_core *cm_core,
212	struct list_head *timer_list)
213	{
214	struct irdma_cm_node *cm_node;
215	int bkt;
216
217	hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
218	if ((cm_node->close_entry || cm_node->send_entry) &&
219	refcount_inc_not_zero(r: &cm_node->refcnt))
220	list_add(new: &cm_node->timer_entry, head: timer_list);
221	}
222	}
223
224	/**
225	* irdma_create_event - create cm event
226	* @cm_node: connection's node
227	* @type: Event type to generate
228	*/
229	static struct irdma_cm_event *irdma_create_event(struct irdma_cm_node *cm_node,
230	enum irdma_cm_event_type type)
231	{
232	struct irdma_cm_event *event;
233
234	if (!cm_node->cm_id)
235	return NULL;
236
237	event = kzalloc(sizeof(*event), GFP_ATOMIC);
238
239	if (!event)
240	return NULL;
241
242	event->type = type;
243	event->cm_node = cm_node;
244	memcpy(event->cm_info.rem_addr, cm_node->rem_addr,
245	sizeof(event->cm_info.rem_addr));
246	memcpy(event->cm_info.loc_addr, cm_node->loc_addr,
247	sizeof(event->cm_info.loc_addr));
248	event->cm_info.rem_port = cm_node->rem_port;
249	event->cm_info.loc_port = cm_node->loc_port;
250	event->cm_info.cm_id = cm_node->cm_id;
251	ibdev_dbg(&cm_node->iwdev->ibdev,
252	"CM: node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node,
253	event, type, event->cm_info.loc_addr,
254	event->cm_info.rem_addr);
255	trace_irdma_create_event(cm_node, type, caller: __builtin_return_address(0));
256	irdma_cm_post_event(event);
257
258	return event;
259	}
260
261	/**
262	* irdma_free_retrans_entry - free send entry
263	* @cm_node: connection's node
264	*/
265	static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node)
266	{
267	struct irdma_device *iwdev = cm_node->iwdev;
268	struct irdma_timer_entry *send_entry;
269
270	send_entry = cm_node->send_entry;
271	if (!send_entry)
272	return;
273
274	cm_node->send_entry = NULL;
275	irdma_free_sqbuf(vsi: &iwdev->vsi, bufp: send_entry->sqbuf);
276	kfree(objp: send_entry);
277	refcount_dec(r: &cm_node->refcnt);
278	}
279
280	/**
281	* irdma_cleanup_retrans_entry - free send entry with lock
282	* @cm_node: connection's node
283	*/
284	static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node)
285	{
286	unsigned long flags;
287
288	spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
289	irdma_free_retrans_entry(cm_node);
290	spin_unlock_irqrestore(lock: &cm_node->retrans_list_lock, flags);
291	}
292
293	/**
294	* irdma_form_ah_cm_frame - get a free packet and build frame with address handle
295	* @cm_node: connection's node ionfo to use in frame
296	* @options: pointer to options info
297	* @hdr: pointer mpa header
298	* @pdata: pointer to private data
299	* @flags: indicates FIN or ACK
300	*/
301	static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node,
302	struct irdma_kmem_info *options,
303	struct irdma_kmem_info *hdr,
304	struct irdma_mpa_priv_info *pdata,
305	u8 flags)
306	{
307	struct irdma_puda_buf *sqbuf;
308	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
309	u8 *buf;
310	struct tcphdr *tcph;
311	u16 pktsize;
312	u32 opts_len = 0;
313	u32 pd_len = 0;
314	u32 hdr_len = 0;
315
316	if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) {
317	ibdev_dbg(&cm_node->iwdev->ibdev, "CM: AH invalid\n");
318	return NULL;
319	}
320
321	sqbuf = irdma_puda_get_bufpool(rsrc: vsi->ilq);
322	if (!sqbuf) {
323	ibdev_dbg(&cm_node->iwdev->ibdev, "CM: SQ buf NULL\n");
324	return NULL;
325	}
326
327	sqbuf->ah_id = cm_node->ah->ah_info.ah_idx;
328	buf = sqbuf->mem.va;
329	if (options)
330	opts_len = (u32)options->size;
331
332	if (hdr)
333	hdr_len = hdr->size;
334
335	if (pdata)
336	pd_len = pdata->size;
337
338	pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;
339
340	memset(buf, 0, sizeof(*tcph));
341
342	sqbuf->totallen = pktsize;
343	sqbuf->tcphlen = sizeof(*tcph) + opts_len;
344	sqbuf->scratch = cm_node;
345
346	tcph = (struct tcphdr *)buf;
347	buf += sizeof(*tcph);
348
349	tcph->source = htons(cm_node->loc_port);
350	tcph->dest = htons(cm_node->rem_port);
351	tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
352
353	if (flags & SET_ACK) {
354	cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
355	tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
356	tcph->ack = 1;
357	} else {
358	tcph->ack_seq = 0;
359	}
360
361	if (flags & SET_SYN) {
362	cm_node->tcp_cntxt.loc_seq_num++;
363	tcph->syn = 1;
364	} else {
365	cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
366	}
367
368	if (flags & SET_FIN) {
369	cm_node->tcp_cntxt.loc_seq_num++;
370	tcph->fin = 1;
371	}
372
373	if (flags & SET_RST)
374	tcph->rst = 1;
375
376	tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
377	sqbuf->tcphlen = tcph->doff << 2;
378	tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
379	tcph->urg_ptr = 0;
380
381	if (opts_len) {
382	memcpy(buf, options->addr, opts_len);
383	buf += opts_len;
384	}
385
386	if (hdr_len) {
387	memcpy(buf, hdr->addr, hdr_len);
388	buf += hdr_len;
389	}
390
391	if (pdata && pdata->addr)
392	memcpy(buf, pdata->addr, pdata->size);
393
394	refcount_set(r: &sqbuf->refcount, n: 1);
395
396	print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,
397	16, 8, sqbuf->mem.va, sqbuf->totallen, false);
398
399	return sqbuf;
400	}
401
402	/**
403	* irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet
404	* @cm_node: connection's node ionfo to use in frame
405	* @options: pointer to options info
406	* @hdr: pointer mpa header
407	* @pdata: pointer to private data
408	* @flags: indicates FIN or ACK
409	*/
410	static struct irdma_puda_buf *irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node,
411	struct irdma_kmem_info *options,
412	struct irdma_kmem_info *hdr,
413	struct irdma_mpa_priv_info *pdata,
414	u8 flags)
415	{
416	struct irdma_puda_buf *sqbuf;
417	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
418	u8 *buf;
419
420	struct tcphdr *tcph;
421	struct iphdr *iph;
422	struct ipv6hdr *ip6h;
423	struct ethhdr *ethh;
424	u16 pktsize;
425	u16 eth_hlen = ETH_HLEN;
426	u32 opts_len = 0;
427	u32 pd_len = 0;
428	u32 hdr_len = 0;
429
430	u16 vtag;
431
432	sqbuf = irdma_puda_get_bufpool(rsrc: vsi->ilq);
433	if (!sqbuf)
434	return NULL;
435
436	buf = sqbuf->mem.va;
437
438	if (options)
439	opts_len = (u32)options->size;
440
441	if (hdr)
442	hdr_len = hdr->size;
443
444	if (pdata)
445	pd_len = pdata->size;
446
447	if (cm_node->vlan_id < VLAN_N_VID)
448	eth_hlen += 4;
449
450	if (cm_node->ipv4)
451	pktsize = sizeof(*iph) + sizeof(*tcph);
452	else
453	pktsize = sizeof(*ip6h) + sizeof(*tcph);
454	pktsize += opts_len + hdr_len + pd_len;
455
456	memset(buf, 0, eth_hlen + pktsize);
457
458	sqbuf->totallen = pktsize + eth_hlen;
459	sqbuf->maclen = eth_hlen;
460	sqbuf->tcphlen = sizeof(*tcph) + opts_len;
461	sqbuf->scratch = cm_node;
462
463	ethh = (struct ethhdr *)buf;
464	buf += eth_hlen;
465
466	if (cm_node->do_lpb)
467	sqbuf->do_lpb = true;
468
469	if (cm_node->ipv4) {
470	sqbuf->ipv4 = true;
471
472	iph = (struct iphdr *)buf;
473	buf += sizeof(*iph);
474	tcph = (struct tcphdr *)buf;
475	buf += sizeof(*tcph);
476
477	ether_addr_copy(dst: ethh->h_dest, src: cm_node->rem_mac);
478	ether_addr_copy(dst: ethh->h_source, src: cm_node->loc_mac);
479	if (cm_node->vlan_id < VLAN_N_VID) {
480	((struct vlan_ethhdr *)ethh)->h_vlan_proto =
481	htons(ETH_P_8021Q);
482	vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |
483	cm_node->vlan_id;
484	((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
485
486	((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
487	htons(ETH_P_IP);
488	} else {
489	ethh->h_proto = htons(ETH_P_IP);
490	}
491
492	iph->version = IPVERSION;
493	iph->ihl = 5; /* 5 * 4Byte words, IP headr len */
494	iph->tos = cm_node->tos;
495	iph->tot_len = htons(pktsize);
496	iph->id = htons(++cm_node->tcp_cntxt.loc_id);
497
498	iph->frag_off = htons(0x4000);
499	iph->ttl = 0x40;
500	iph->protocol = IPPROTO_TCP;
501	iph->saddr = htonl(cm_node->loc_addr[0]);
502	iph->daddr = htonl(cm_node->rem_addr[0]);
503	} else {
504	sqbuf->ipv4 = false;
505	ip6h = (struct ipv6hdr *)buf;
506	buf += sizeof(*ip6h);
507	tcph = (struct tcphdr *)buf;
508	buf += sizeof(*tcph);
509
510	ether_addr_copy(dst: ethh->h_dest, src: cm_node->rem_mac);
511	ether_addr_copy(dst: ethh->h_source, src: cm_node->loc_mac);
512	if (cm_node->vlan_id < VLAN_N_VID) {
513	((struct vlan_ethhdr *)ethh)->h_vlan_proto =
514	htons(ETH_P_8021Q);
515	vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |
516	cm_node->vlan_id;
517	((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
518	((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
519	htons(ETH_P_IPV6);
520	} else {
521	ethh->h_proto = htons(ETH_P_IPV6);
522	}
523	ip6h->version = 6;
524	ip6h->priority = cm_node->tos >> 4;
525	ip6h->flow_lbl[0] = cm_node->tos << 4;
526	ip6h->flow_lbl[1] = 0;
527	ip6h->flow_lbl[2] = 0;
528	ip6h->payload_len = htons(pktsize - sizeof(*ip6h));
529	ip6h->nexthdr = 6;
530	ip6h->hop_limit = 128;
531	irdma_copy_ip_htonl(dst: ip6h->saddr.in6_u.u6_addr32,
532	src: cm_node->loc_addr);
533	irdma_copy_ip_htonl(dst: ip6h->daddr.in6_u.u6_addr32,
534	src: cm_node->rem_addr);
535	}
536
537	tcph->source = htons(cm_node->loc_port);
538	tcph->dest = htons(cm_node->rem_port);
539	tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
540
541	if (flags & SET_ACK) {
542	cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
543	tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
544	tcph->ack = 1;
545	} else {
546	tcph->ack_seq = 0;
547	}
548
549	if (flags & SET_SYN) {
550	cm_node->tcp_cntxt.loc_seq_num++;
551	tcph->syn = 1;
552	} else {
553	cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
554	}
555
556	if (flags & SET_FIN) {
557	cm_node->tcp_cntxt.loc_seq_num++;
558	tcph->fin = 1;
559	}
560
561	if (flags & SET_RST)
562	tcph->rst = 1;
563
564	tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
565	sqbuf->tcphlen = tcph->doff << 2;
566	tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
567	tcph->urg_ptr = 0;
568
569	if (opts_len) {
570	memcpy(buf, options->addr, opts_len);
571	buf += opts_len;
572	}
573
574	if (hdr_len) {
575	memcpy(buf, hdr->addr, hdr_len);
576	buf += hdr_len;
577	}
578
579	if (pdata && pdata->addr)
580	memcpy(buf, pdata->addr, pdata->size);
581
582	refcount_set(r: &sqbuf->refcount, n: 1);
583
584	print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,
585	16, 8, sqbuf->mem.va, sqbuf->totallen, false);
586	return sqbuf;
587	}
588
589	/**
590	* irdma_send_reset - Send RST packet
591	* @cm_node: connection's node
592	*/
593	int irdma_send_reset(struct irdma_cm_node *cm_node)
594	{
595	struct irdma_puda_buf *sqbuf;
596	int flags = SET_RST | SET_ACK;
597
598	trace_irdma_send_reset(cm_node, type: 0, caller: __builtin_return_address(0));
599	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
600	flags);
601	if (!sqbuf)
602	return -ENOMEM;
603
604	ibdev_dbg(&cm_node->iwdev->ibdev,
605	"CM: caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
606	__builtin_return_address(0), cm_node, cm_node->cm_id,
607	cm_node->accelerated, cm_node->state, cm_node->rem_port,
608	cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr);
609
610	return irdma_schedule_cm_timer(cm_node, sqbuf, type: IRDMA_TIMER_TYPE_SEND, send_retrans: 0,
611	close_when_complete: 1);
612	}
613
614	/**
615	* irdma_active_open_err - send event for active side cm error
616	* @cm_node: connection's node
617	* @reset: Flag to send reset or not
618	*/
619	static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset)
620	{
621	trace_irdma_active_open_err(cm_node, reset,
622	caller: __builtin_return_address(0));
623	irdma_cleanup_retrans_entry(cm_node);
624	cm_node->cm_core->stats_connect_errs++;
625	if (reset) {
626	ibdev_dbg(&cm_node->iwdev->ibdev,
627	"CM: cm_node=%p state=%d\n", cm_node,
628	cm_node->state);
629	refcount_inc(r: &cm_node->refcnt);
630	irdma_send_reset(cm_node);
631	}
632
633	cm_node->state = IRDMA_CM_STATE_CLOSED;
634	irdma_create_event(cm_node, type: IRDMA_CM_EVENT_ABORTED);
635	}
636
637	/**
638	* irdma_passive_open_err - handle passive side cm error
639	* @cm_node: connection's node
640	* @reset: send reset or just free cm_node
641	*/
642	static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset)
643	{
644	irdma_cleanup_retrans_entry(cm_node);
645	cm_node->cm_core->stats_passive_errs++;
646	cm_node->state = IRDMA_CM_STATE_CLOSED;
647	ibdev_dbg(&cm_node->iwdev->ibdev, "CM: cm_node=%p state =%d\n",
648	cm_node, cm_node->state);
649	trace_irdma_passive_open_err(cm_node, reset,
650	caller: __builtin_return_address(0));
651	if (reset)
652	irdma_send_reset(cm_node);
653	else
654	irdma_rem_ref_cm_node(cm_node);
655	}
656
657	/**
658	* irdma_event_connect_error - to create connect error event
659	* @event: cm information for connect event
660	*/
661	static void irdma_event_connect_error(struct irdma_cm_event *event)
662	{
663	struct irdma_qp *iwqp;
664	struct iw_cm_id *cm_id;
665
666	cm_id = event->cm_node->cm_id;
667	if (!cm_id)
668	return;
669
670	iwqp = cm_id->provider_data;
671
672	if (!iwqp || !iwqp->iwdev)
673	return;
674
675	iwqp->cm_id = NULL;
676	cm_id->provider_data = NULL;
677	irdma_send_cm_event(cm_node: event->cm_node, cm_id, type: IW_CM_EVENT_CONNECT_REPLY,
678	status: -ECONNRESET);
679	irdma_rem_ref_cm_node(cm_node: event->cm_node);
680	}
681
682	/**
683	* irdma_process_options - process options from TCP header
684	* @cm_node: connection's node
685	* @optionsloc: point to start of options
686	* @optionsize: size of all options
687	* @syn_pkt: flag if syn packet
688	*/
689	static int irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc,
690	u32 optionsize, u32 syn_pkt)
691	{
692	u32 tmp;
693	u32 offset = 0;
694	union all_known_options *all_options;
695	char got_mss_option = 0;
696
697	while (offset < optionsize) {
698	all_options = (union all_known_options *)(optionsloc + offset);
699	switch (all_options->base.optionnum) {
700	case OPTION_NUM_EOL:
701	offset = optionsize;
702	break;
703	case OPTION_NUM_NONE:
704	offset += 1;
705	continue;
706	case OPTION_NUM_MSS:
707	ibdev_dbg(&cm_node->iwdev->ibdev,
708	"CM: MSS Length: %d Offset: %d Size: %d\n",
709	all_options->mss.len, offset, optionsize);
710	got_mss_option = 1;
711	if (all_options->mss.len != 4)
712	return -EINVAL;
713	tmp = ntohs(all_options->mss.mss);
714	if ((cm_node->ipv4 &&
715	(tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) ||
716	(!cm_node->ipv4 &&
717	(tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6))
718	return -EINVAL;
719	if (tmp < cm_node->tcp_cntxt.mss)
720	cm_node->tcp_cntxt.mss = tmp;
721	break;
722	case OPTION_NUM_WINDOW_SCALE:
723	cm_node->tcp_cntxt.snd_wscale =
724	all_options->windowscale.shiftcount;
725	break;
726	default:
727	ibdev_dbg(&cm_node->iwdev->ibdev,
728	"CM: Unsupported TCP Option: %x\n",
729	all_options->base.optionnum);
730	break;
731	}
732	offset += all_options->base.len;
733	}
734	if (!got_mss_option && syn_pkt)
735	cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS;
736
737	return 0;
738	}
739
740	/**
741	* irdma_handle_tcp_options - setup TCP context info after parsing TCP options
742	* @cm_node: connection's node
743	* @tcph: pointer tcp header
744	* @optionsize: size of options rcvd
745	* @passive: active or passive flag
746	*/
747	static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node,
748	struct tcphdr *tcph, int optionsize,
749	int passive)
750	{
751	u8 *optionsloc = (u8 *)&tcph[1];
752	int ret;
753
754	if (optionsize) {
755	ret = irdma_process_options(cm_node, optionsloc, optionsize,
756	syn_pkt: (u32)tcph->syn);
757	if (ret) {
758	ibdev_dbg(&cm_node->iwdev->ibdev,
759	"CM: Node %p, Sending Reset\n", cm_node);
760	if (passive)
761	irdma_passive_open_err(cm_node, reset: true);
762	else
763	irdma_active_open_err(cm_node, reset: true);
764	return ret;
765	}
766	}
767
768	cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window)
769	<< cm_node->tcp_cntxt.snd_wscale;
770
771	if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
772	cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
773
774	return 0;
775	}
776
777	/**
778	* irdma_build_mpa_v1 - build a MPA V1 frame
779	* @cm_node: connection's node
780	* @start_addr: address where to build frame
781	* @mpa_key: to do read0 or write0
782	*/
783	static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr,
784	u8 mpa_key)
785	{
786	struct ietf_mpa_v1 *mpa_frame = start_addr;
787
788	switch (mpa_key) {
789	case MPA_KEY_REQUEST:
790	memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
791	break;
792	case MPA_KEY_REPLY:
793	memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
794	break;
795	default:
796	break;
797	}
798	mpa_frame->flags = IETF_MPA_FLAGS_CRC;
799	mpa_frame->rev = cm_node->mpa_frame_rev;
800	mpa_frame->priv_data_len = htons(cm_node->pdata.size);
801	}
802
803	/**
804	* irdma_build_mpa_v2 - build a MPA V2 frame
805	* @cm_node: connection's node
806	* @start_addr: buffer start address
807	* @mpa_key: to do read0 or write0
808	*/
809	static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr,
810	u8 mpa_key)
811	{
812	struct ietf_mpa_v2 *mpa_frame = start_addr;
813	struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
814	u16 ctrl_ird, ctrl_ord;
815
816	/* initialize the upper 5 bytes of the frame */
817	irdma_build_mpa_v1(cm_node, start_addr, mpa_key);
818	mpa_frame->flags |= IETF_MPA_V2_FLAG;
819	if (cm_node->iwdev->iw_ooo) {
820	mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS;
821	cm_node->rcv_mark_en = true;
822	}
823	mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) +
824	IETF_RTR_MSG_SIZE);
825
826	/* initialize RTR msg */
827	if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
828	ctrl_ird = IETF_NO_IRD_ORD;
829	ctrl_ord = IETF_NO_IRD_ORD;
830	} else {
831	ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
832	IETF_NO_IRD_ORD :
833	cm_node->ird_size;
834	ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
835	IETF_NO_IRD_ORD :
836	cm_node->ord_size;
837	}
838	ctrl_ird |= IETF_PEER_TO_PEER;
839
840	switch (mpa_key) {
841	case MPA_KEY_REQUEST:
842	ctrl_ord |= IETF_RDMA0_WRITE;
843	ctrl_ord |= IETF_RDMA0_READ;
844	break;
845	case MPA_KEY_REPLY:
846	switch (cm_node->send_rdma0_op) {
847	case SEND_RDMA_WRITE_ZERO:
848	ctrl_ord |= IETF_RDMA0_WRITE;
849	break;
850	case SEND_RDMA_READ_ZERO:
851	ctrl_ord |= IETF_RDMA0_READ;
852	break;
853	}
854	break;
855	default:
856	break;
857	}
858	rtr_msg->ctrl_ird = htons(ctrl_ird);
859	rtr_msg->ctrl_ord = htons(ctrl_ord);
860	}
861
862	/**
863	* irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2
864	* @cm_node: connection's node
865	* @mpa: mpa: data buffer
866	* @mpa_key: to do read0 or write0
867	*/
868	static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node,
869	struct irdma_kmem_info *mpa, u8 mpa_key)
870	{
871	int hdr_len = 0;
872
873	switch (cm_node->mpa_frame_rev) {
874	case IETF_MPA_V1:
875	hdr_len = sizeof(struct ietf_mpa_v1);
876	irdma_build_mpa_v1(cm_node, start_addr: mpa->addr, mpa_key);
877	break;
878	case IETF_MPA_V2:
879	hdr_len = sizeof(struct ietf_mpa_v2);
880	irdma_build_mpa_v2(cm_node, start_addr: mpa->addr, mpa_key);
881	break;
882	default:
883	break;
884	}
885
886	return hdr_len;
887	}
888
889	/**
890	* irdma_send_mpa_request - active node send mpa request to passive node
891	* @cm_node: connection's node
892	*/
893	static int irdma_send_mpa_request(struct irdma_cm_node *cm_node)
894	{
895	struct irdma_puda_buf *sqbuf;
896
897	cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;
898	cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
899	mpa: &cm_node->mpa_hdr,
900	mpa_key: MPA_KEY_REQUEST);
901	if (!cm_node->mpa_hdr.size) {
902	ibdev_dbg(&cm_node->iwdev->ibdev,
903	"CM: mpa size = %d\n", cm_node->mpa_hdr.size);
904	return -EINVAL;
905	}
906
907	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,
908	&cm_node->mpa_hdr,
909	&cm_node->pdata, SET_ACK);
910	if (!sqbuf)
911	return -ENOMEM;
912
913	return irdma_schedule_cm_timer(cm_node, sqbuf, type: IRDMA_TIMER_TYPE_SEND, send_retrans: 1,
914	close_when_complete: 0);
915	}
916
917	/**
918	* irdma_send_mpa_reject -
919	* @cm_node: connection's node
920	* @pdata: reject data for connection
921	* @plen: length of reject data
922	*/
923	static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node,
924	const void *pdata, u8 plen)
925	{
926	struct irdma_puda_buf *sqbuf;
927	struct irdma_mpa_priv_info priv_info;
928
929	cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;
930	cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
931	mpa: &cm_node->mpa_hdr,
932	mpa_key: MPA_KEY_REPLY);
933
934	cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;
935	priv_info.addr = pdata;
936	priv_info.size = plen;
937
938	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,
939	&cm_node->mpa_hdr, &priv_info,
940	SET_ACK | SET_FIN);
941	if (!sqbuf)
942	return -ENOMEM;
943
944	cm_node->state = IRDMA_CM_STATE_FIN_WAIT1;
945
946	return irdma_schedule_cm_timer(cm_node, sqbuf, type: IRDMA_TIMER_TYPE_SEND, send_retrans: 1,
947	close_when_complete: 0);
948	}
949
950	/**
951	* irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD
952	* @cm_node: connection's node
953	* @buf: Data pointer
954	*/
955	static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node,
956	u8 *buf)
957	{
958	struct ietf_mpa_v2 *mpa_v2_frame;
959	struct ietf_rtr_msg *rtr_msg;
960	u16 ird_size;
961	u16 ord_size;
962	u16 ctrl_ord;
963	u16 ctrl_ird;
964
965	mpa_v2_frame = (struct ietf_mpa_v2 *)buf;
966	rtr_msg = &mpa_v2_frame->rtr_msg;
967
968	/* parse rtr message */
969	ctrl_ord = ntohs(rtr_msg->ctrl_ord);
970	ctrl_ird = ntohs(rtr_msg->ctrl_ird);
971	ird_size = ctrl_ird & IETF_NO_IRD_ORD;
972	ord_size = ctrl_ord & IETF_NO_IRD_ORD;
973
974	if (!(ctrl_ird & IETF_PEER_TO_PEER))
975	return -EOPNOTSUPP;
976
977	if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {
978	cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
979	goto negotiate_done;
980	}
981
982	if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
983	/* responder */
984	if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))
985	cm_node->ird_size = 1;
986	if (cm_node->ord_size > ird_size)
987	cm_node->ord_size = ird_size;
988	} else {
989	/* initiator */
990	if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))
991	/* Remote peer doesn't support RDMA0_READ */
992	return -EOPNOTSUPP;
993
994	if (cm_node->ord_size > ird_size)
995	cm_node->ord_size = ird_size;
996
997	if (cm_node->ird_size < ord_size)
998	/* no resources available */
999	return -EINVAL;
1000	}
1001
1002	negotiate_done:
1003	if (ctrl_ord & IETF_RDMA0_READ)
1004	cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
1005	else if (ctrl_ord & IETF_RDMA0_WRITE)
1006	cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
1007	else
1008	/* Not supported RDMA0 operation */
1009	return -EOPNOTSUPP;
1010
1011	ibdev_dbg(&cm_node->iwdev->ibdev,
1012	"CM: MPAV2 Negotiated ORD: %d, IRD: %d\n",
1013	cm_node->ord_size, cm_node->ird_size);
1014	trace_irdma_negotiate_mpa_v2(cm_node);
1015	return 0;
1016	}
1017
1018	/**
1019	* irdma_parse_mpa - process an IETF MPA frame
1020	* @cm_node: connection's node
1021	* @buf: Data pointer
1022	* @type: to return accept or reject
1023	* @len: Len of mpa buffer
1024	*/
1025	static int irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,
1026	u32 len)
1027	{
1028	struct ietf_mpa_v1 *mpa_frame;
1029	int mpa_hdr_len, priv_data_len, ret;
1030
1031	*type = IRDMA_MPA_REQUEST_ACCEPT;
1032
1033	if (len < sizeof(struct ietf_mpa_v1)) {
1034	ibdev_dbg(&cm_node->iwdev->ibdev,
1035	"CM: ietf buffer small (%x)\n", len);
1036	return -EINVAL;
1037	}
1038
1039	mpa_frame = (struct ietf_mpa_v1 *)buf;
1040	mpa_hdr_len = sizeof(struct ietf_mpa_v1);
1041	priv_data_len = ntohs(mpa_frame->priv_data_len);
1042
1043	if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
1044	ibdev_dbg(&cm_node->iwdev->ibdev,
1045	"CM: private_data too big %d\n", priv_data_len);
1046	return -EOVERFLOW;
1047	}
1048
1049	if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
1050	ibdev_dbg(&cm_node->iwdev->ibdev,
1051	"CM: unsupported mpa rev = %d\n", mpa_frame->rev);
1052	return -EINVAL;
1053	}
1054
1055	if (mpa_frame->rev > cm_node->mpa_frame_rev) {
1056	ibdev_dbg(&cm_node->iwdev->ibdev, "CM: rev %d\n",
1057	mpa_frame->rev);
1058	return -EINVAL;
1059	}
1060
1061	cm_node->mpa_frame_rev = mpa_frame->rev;
1062	if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
1063	if (memcmp(p: mpa_frame->key, IEFT_MPA_KEY_REQ,
1064	IETF_MPA_KEY_SIZE)) {
1065	ibdev_dbg(&cm_node->iwdev->ibdev,
1066	"CM: Unexpected MPA Key received\n");
1067	return -EINVAL;
1068	}
1069	} else {
1070	if (memcmp(p: mpa_frame->key, IEFT_MPA_KEY_REP,
1071	IETF_MPA_KEY_SIZE)) {
1072	ibdev_dbg(&cm_node->iwdev->ibdev,
1073	"CM: Unexpected MPA Key received\n");
1074	return -EINVAL;
1075	}
1076	}
1077
1078	if (priv_data_len + mpa_hdr_len > len) {
1079	ibdev_dbg(&cm_node->iwdev->ibdev,
1080	"CM: ietf buffer len(%x + %x != %x)\n",
1081	priv_data_len, mpa_hdr_len, len);
1082	return -EOVERFLOW;
1083	}
1084
1085	if (len > IRDMA_MAX_CM_BUF) {
1086	ibdev_dbg(&cm_node->iwdev->ibdev,
1087	"CM: ietf buffer large len = %d\n", len);
1088	return -EOVERFLOW;
1089	}
1090
1091	switch (mpa_frame->rev) {
1092	case IETF_MPA_V2:
1093	mpa_hdr_len += IETF_RTR_MSG_SIZE;
1094	ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf);
1095	if (ret)
1096	return ret;
1097	break;
1098	case IETF_MPA_V1:
1099	default:
1100	break;
1101	}
1102
1103	memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len);
1104	cm_node->pdata.size = priv_data_len;
1105
1106	if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
1107	*type = IRDMA_MPA_REQUEST_REJECT;
1108
1109	if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)
1110	cm_node->snd_mark_en = true;
1111
1112	return 0;
1113	}
1114
1115	/**
1116	* irdma_schedule_cm_timer
1117	* @cm_node: connection's node
1118	* @sqbuf: buffer to send
1119	* @type: if it is send or close
1120	* @send_retrans: if rexmits to be done
1121	* @close_when_complete: is cm_node to be removed
1122	*
1123	* note - cm_node needs to be protected before calling this. Encase in:
1124	* irdma_rem_ref_cm_node(cm_core, cm_node);
1125	* irdma_schedule_cm_timer(...)
1126	* refcount_inc(&cm_node->refcnt);
1127	*/
1128	int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
1129	struct irdma_puda_buf *sqbuf,
1130	enum irdma_timer_type type, int send_retrans,
1131	int close_when_complete)
1132	{
1133	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
1134	struct irdma_cm_core *cm_core = cm_node->cm_core;
1135	struct irdma_timer_entry *new_send;
1136	u32 was_timer_set;
1137	unsigned long flags;
1138
1139	new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
1140	if (!new_send) {
1141	if (type != IRDMA_TIMER_TYPE_CLOSE)
1142	irdma_free_sqbuf(vsi, bufp: sqbuf);
1143	return -ENOMEM;
1144	}
1145
1146	new_send->retrycount = IRDMA_DEFAULT_RETRYS;
1147	new_send->retranscount = IRDMA_DEFAULT_RETRANS;
1148	new_send->sqbuf = sqbuf;
1149	new_send->timetosend = jiffies;
1150	new_send->type = type;
1151	new_send->send_retrans = send_retrans;
1152	new_send->close_when_complete = close_when_complete;
1153
1154	if (type == IRDMA_TIMER_TYPE_CLOSE) {
1155	new_send->timetosend += (HZ / 10);
1156	if (cm_node->close_entry) {
1157	kfree(objp: new_send);
1158	ibdev_dbg(&cm_node->iwdev->ibdev,
1159	"CM: already close entry\n");
1160	return -EINVAL;
1161	}
1162
1163	cm_node->close_entry = new_send;
1164	} else { /* type == IRDMA_TIMER_TYPE_SEND */
1165	spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1166	cm_node->send_entry = new_send;
1167	refcount_inc(r: &cm_node->refcnt);
1168	spin_unlock_irqrestore(lock: &cm_node->retrans_list_lock, flags);
1169	new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT;
1170
1171	refcount_inc(r: &sqbuf->refcount);
1172	irdma_puda_send_buf(rsrc: vsi->ilq, buf: sqbuf);
1173	if (!send_retrans) {
1174	irdma_cleanup_retrans_entry(cm_node);
1175	if (close_when_complete)
1176	irdma_rem_ref_cm_node(cm_node);
1177	return 0;
1178	}
1179	}
1180
1181	spin_lock_irqsave(&cm_core->ht_lock, flags);
1182	was_timer_set = timer_pending(timer: &cm_core->tcp_timer);
1183
1184	if (!was_timer_set) {
1185	cm_core->tcp_timer.expires = new_send->timetosend;
1186	add_timer(timer: &cm_core->tcp_timer);
1187	}
1188	spin_unlock_irqrestore(lock: &cm_core->ht_lock, flags);
1189
1190	return 0;
1191	}
1192
1193	/**
1194	* irdma_retrans_expired - Could not rexmit the packet
1195	* @cm_node: connection's node
1196	*/
1197	static void irdma_retrans_expired(struct irdma_cm_node *cm_node)
1198	{
1199	enum irdma_cm_node_state state = cm_node->state;
1200
1201	cm_node->state = IRDMA_CM_STATE_CLOSED;
1202	switch (state) {
1203	case IRDMA_CM_STATE_SYN_RCVD:
1204	case IRDMA_CM_STATE_CLOSING:
1205	irdma_rem_ref_cm_node(cm_node);
1206	break;
1207	case IRDMA_CM_STATE_FIN_WAIT1:
1208	case IRDMA_CM_STATE_LAST_ACK:
1209	irdma_send_reset(cm_node);
1210	break;
1211	default:
1212	refcount_inc(r: &cm_node->refcnt);
1213	irdma_send_reset(cm_node);
1214	irdma_create_event(cm_node, type: IRDMA_CM_EVENT_ABORTED);
1215	break;
1216	}
1217	}
1218
1219	/**
1220	* irdma_handle_close_entry - for handling retry/timeouts
1221	* @cm_node: connection's node
1222	* @rem_node: flag for remove cm_node
1223	*/
1224	static void irdma_handle_close_entry(struct irdma_cm_node *cm_node,
1225	u32 rem_node)
1226	{
1227	struct irdma_timer_entry *close_entry = cm_node->close_entry;
1228	struct irdma_qp *iwqp;
1229	unsigned long flags;
1230
1231	if (!close_entry)
1232	return;
1233	iwqp = (struct irdma_qp *)close_entry->sqbuf;
1234	if (iwqp) {
1235	spin_lock_irqsave(&iwqp->lock, flags);
1236	if (iwqp->cm_id) {
1237	iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
1238	iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR;
1239	iwqp->last_aeq = IRDMA_AE_RESET_SENT;
1240	iwqp->ibqp_state = IB_QPS_ERR;
1241	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1242	irdma_cm_disconn(qp: iwqp);
1243	} else {
1244	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1245	}
1246	} else if (rem_node) {
1247	/* TIME_WAIT state */
1248	irdma_rem_ref_cm_node(cm_node);
1249	}
1250
1251	kfree(objp: close_entry);
1252	cm_node->close_entry = NULL;
1253	}
1254
1255	/**
1256	* irdma_cm_timer_tick - system's timer expired callback
1257	* @t: Pointer to timer_list
1258	*/
1259	static void irdma_cm_timer_tick(struct timer_list *t)
1260	{
1261	unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME;
1262	struct irdma_cm_node *cm_node;
1263	struct irdma_timer_entry *send_entry, *close_entry;
1264	struct list_head *list_core_temp;
1265	struct list_head *list_node;
1266	struct irdma_cm_core *cm_core = timer_container_of(cm_core, t,
1267	tcp_timer);
1268	struct irdma_sc_vsi *vsi;
1269	u32 settimer = 0;
1270	unsigned long timetosend;
1271	unsigned long flags;
1272	struct list_head timer_list;
1273
1274	INIT_LIST_HEAD(list: &timer_list);
1275
1276	rcu_read_lock();
1277	irdma_timer_list_prep(cm_core, timer_list: &timer_list);
1278	rcu_read_unlock();
1279
1280	list_for_each_safe (list_node, list_core_temp, &timer_list) {
1281	cm_node = container_of(list_node, struct irdma_cm_node,
1282	timer_entry);
1283	close_entry = cm_node->close_entry;
1284
1285	if (close_entry) {
1286	if (time_after(close_entry->timetosend, jiffies)) {
1287	if (nexttimeout > close_entry->timetosend ||
1288	!settimer) {
1289	nexttimeout = close_entry->timetosend;
1290	settimer = 1;
1291	}
1292	} else {
1293	irdma_handle_close_entry(cm_node, rem_node: 1);
1294	}
1295	}
1296
1297	spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1298
1299	send_entry = cm_node->send_entry;
1300	if (!send_entry)
1301	goto done;
1302	if (time_after(send_entry->timetosend, jiffies)) {
1303	if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
1304	if (nexttimeout > send_entry->timetosend ||
1305	!settimer) {
1306	nexttimeout = send_entry->timetosend;
1307	settimer = 1;
1308	}
1309	} else {
1310	irdma_free_retrans_entry(cm_node);
1311	}
1312	goto done;
1313	}
1314
1315	if (cm_node->state == IRDMA_CM_STATE_OFFLOADED ||
1316	cm_node->state == IRDMA_CM_STATE_CLOSED) {
1317	irdma_free_retrans_entry(cm_node);
1318	goto done;
1319	}
1320
1321	if (!send_entry->retranscount || !send_entry->retrycount) {
1322	irdma_free_retrans_entry(cm_node);
1323
1324	spin_unlock_irqrestore(lock: &cm_node->retrans_list_lock,
1325	flags);
1326	irdma_retrans_expired(cm_node);
1327	cm_node->state = IRDMA_CM_STATE_CLOSED;
1328	spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1329	goto done;
1330	}
1331	spin_unlock_irqrestore(lock: &cm_node->retrans_list_lock, flags);
1332
1333	vsi = &cm_node->iwdev->vsi;
1334	if (!cm_node->ack_rcvd) {
1335	refcount_inc(r: &send_entry->sqbuf->refcount);
1336	irdma_puda_send_buf(rsrc: vsi->ilq, buf: send_entry->sqbuf);
1337	cm_node->cm_core->stats_pkt_retrans++;
1338	}
1339
1340	spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1341	if (send_entry->send_retrans) {
1342	send_entry->retranscount--;
1343	timetosend = (IRDMA_RETRY_TIMEOUT <<
1344	(IRDMA_DEFAULT_RETRANS -
1345	send_entry->retranscount));
1346
1347	send_entry->timetosend = jiffies +
1348	min(timetosend, IRDMA_MAX_TIMEOUT);
1349	if (nexttimeout > send_entry->timetosend || !settimer) {
1350	nexttimeout = send_entry->timetosend;
1351	settimer = 1;
1352	}
1353	} else {
1354	int close_when_complete;
1355
1356	close_when_complete = send_entry->close_when_complete;
1357	irdma_free_retrans_entry(cm_node);
1358	if (close_when_complete)
1359	irdma_rem_ref_cm_node(cm_node);
1360	}
1361	done:
1362	spin_unlock_irqrestore(lock: &cm_node->retrans_list_lock, flags);
1363	irdma_rem_ref_cm_node(cm_node);
1364	}
1365
1366	if (settimer) {
1367	spin_lock_irqsave(&cm_core->ht_lock, flags);
1368	if (!timer_pending(timer: &cm_core->tcp_timer)) {
1369	cm_core->tcp_timer.expires = nexttimeout;
1370	add_timer(timer: &cm_core->tcp_timer);
1371	}
1372	spin_unlock_irqrestore(lock: &cm_core->ht_lock, flags);
1373	}
1374	}
1375
1376	/**
1377	* irdma_send_syn - send SYN packet
1378	* @cm_node: connection's node
1379	* @sendack: flag to set ACK bit or not
1380	*/
1381	int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)
1382	{
1383	struct irdma_puda_buf *sqbuf;
1384	int flags = SET_SYN;
1385	char optionsbuf[sizeof(struct option_mss) +
1386	sizeof(struct option_windowscale) +
1387	sizeof(struct option_base) + TCP_OPTIONS_PADDING];
1388	struct irdma_kmem_info opts;
1389	int optionssize = 0;
1390	/* Sending MSS option */
1391	union all_known_options *options;
1392
1393	opts.addr = optionsbuf;
1394	if (!cm_node)
1395	return -EINVAL;
1396
1397	options = (union all_known_options *)&optionsbuf[optionssize];
1398	options->mss.optionnum = OPTION_NUM_MSS;
1399	options->mss.len = sizeof(struct option_mss);
1400	options->mss.mss = htons(cm_node->tcp_cntxt.mss);
1401	optionssize += sizeof(struct option_mss);
1402
1403	options = (union all_known_options *)&optionsbuf[optionssize];
1404	options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE;
1405	options->windowscale.len = sizeof(struct option_windowscale);
1406	options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
1407	optionssize += sizeof(struct option_windowscale);
1408	options = (union all_known_options *)&optionsbuf[optionssize];
1409	options->eol = OPTION_NUM_EOL;
1410	optionssize += 1;
1411
1412	if (sendack)
1413	flags |= SET_ACK;
1414
1415	opts.size = optionssize;
1416
1417	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL,
1418	flags);
1419	if (!sqbuf)
1420	return -ENOMEM;
1421
1422	return irdma_schedule_cm_timer(cm_node, sqbuf, type: IRDMA_TIMER_TYPE_SEND, send_retrans: 1,
1423	close_when_complete: 0);
1424	}
1425
1426	/**
1427	* irdma_send_ack - Send ACK packet
1428	* @cm_node: connection's node
1429	*/
1430	void irdma_send_ack(struct irdma_cm_node *cm_node)
1431	{
1432	struct irdma_puda_buf *sqbuf;
1433	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
1434
1435	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
1436	SET_ACK);
1437	if (sqbuf)
1438	irdma_puda_send_buf(rsrc: vsi->ilq, buf: sqbuf);
1439	}
1440
1441	/**
1442	* irdma_send_fin - Send FIN pkt
1443	* @cm_node: connection's node
1444	*/
1445	static int irdma_send_fin(struct irdma_cm_node *cm_node)
1446	{
1447	struct irdma_puda_buf *sqbuf;
1448
1449	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
1450	SET_ACK | SET_FIN);
1451	if (!sqbuf)
1452	return -ENOMEM;
1453
1454	return irdma_schedule_cm_timer(cm_node, sqbuf, type: IRDMA_TIMER_TYPE_SEND, send_retrans: 1,
1455	close_when_complete: 0);
1456	}
1457
1458	/**
1459	* irdma_find_listener - find a cm node listening on this addr-port pair
1460	* @cm_core: cm's core
1461	* @dst_addr: listener ip addr
1462	* @ipv4: flag indicating IPv4 when true
1463	* @dst_port: listener tcp port num
1464	* @vlan_id: virtual LAN ID
1465	* @listener_state: state to match with listen node's
1466	*/
1467	static struct irdma_cm_listener *
1468	irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4,
1469	u16 dst_port, u16 vlan_id,
1470	enum irdma_cm_listener_state listener_state)
1471	{
1472	struct irdma_cm_listener *listen_node;
1473	static const u32 ip_zero[4] = { 0, 0, 0, 0 };
1474	u32 listen_addr[4];
1475	u16 listen_port;
1476	unsigned long flags;
1477
1478	/* walk list and find cm_node associated with this session ID */
1479	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
1480	list_for_each_entry (listen_node, &cm_core->listen_list, list) {
1481	memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));
1482	listen_port = listen_node->loc_port;
1483	if (listen_node->ipv4 != ipv4 || listen_port != dst_port ||
1484	!(listener_state & listen_node->listener_state))
1485	continue;
1486	/* compare node pair, return node handle if a match */
1487	if (!memcmp(p: listen_addr, q: ip_zero, size: sizeof(listen_addr)) ||
1488	(!memcmp(p: listen_addr, q: dst_addr, size: sizeof(listen_addr)) &&
1489	vlan_id == listen_node->vlan_id)) {
1490	refcount_inc(r: &listen_node->refcnt);
1491	spin_unlock_irqrestore(lock: &cm_core->listen_list_lock,
1492	flags);
1493	trace_irdma_find_listener(listener: listen_node);
1494	return listen_node;
1495	}
1496	}
1497	spin_unlock_irqrestore(lock: &cm_core->listen_list_lock, flags);
1498
1499	return NULL;
1500	}
1501
1502	/**
1503	* irdma_del_multiple_qhash - Remove qhash and child listens
1504	* @iwdev: iWarp device
1505	* @cm_info: CM info for parent listen node
1506	* @cm_parent_listen_node: The parent listen node
1507	*/
1508	static int irdma_del_multiple_qhash(struct irdma_device *iwdev,
1509	struct irdma_cm_info *cm_info,
1510	struct irdma_cm_listener *cm_parent_listen_node)
1511	{
1512	struct irdma_cm_listener *child_listen_node;
1513	struct list_head *pos, *tpos;
1514	unsigned long flags;
1515	int ret = -EINVAL;
1516
1517	spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
1518	list_for_each_safe (pos, tpos,
1519	&cm_parent_listen_node->child_listen_list) {
1520	child_listen_node = list_entry(pos, struct irdma_cm_listener,
1521	child_listen_list);
1522	if (child_listen_node->ipv4)
1523	ibdev_dbg(&iwdev->ibdev,
1524	"CM: removing child listen for IP=%pI4, port=%d, vlan=%d\n",
1525	child_listen_node->loc_addr,
1526	child_listen_node->loc_port,
1527	child_listen_node->vlan_id);
1528	else
1529	ibdev_dbg(&iwdev->ibdev,
1530	"CM: removing child listen for IP=%pI6, port=%d, vlan=%d\n",
1531	child_listen_node->loc_addr,
1532	child_listen_node->loc_port,
1533	child_listen_node->vlan_id);
1534	trace_irdma_del_multiple_qhash(listener: child_listen_node);
1535	list_del(entry: pos);
1536	memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1537	sizeof(cm_info->loc_addr));
1538	cm_info->vlan_id = child_listen_node->vlan_id;
1539	if (child_listen_node->qhash_set) {
1540	ret = irdma_manage_qhash(iwdev, cminfo: cm_info,
1541	etype: IRDMA_QHASH_TYPE_TCP_SYN,
1542	mtype: IRDMA_QHASH_MANAGE_TYPE_DELETE,
1543	NULL, wait: false);
1544	child_listen_node->qhash_set = false;
1545	} else {
1546	ret = 0;
1547	}
1548	ibdev_dbg(&iwdev->ibdev,
1549	"CM: Child listen node freed = %p\n",
1550	child_listen_node);
1551	kfree(objp: child_listen_node);
1552	cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
1553	}
1554	spin_unlock_irqrestore(lock: &iwdev->cm_core.listen_list_lock, flags);
1555
1556	return ret;
1557	}
1558
1559	static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
1560	{
1561	struct net_device *ndev = NULL;
1562
1563	rcu_read_lock();
1564	if (ipv4) {
1565	ndev = ip_dev_find(net: &init_net, htonl(loc_addr[0]));
1566	} else if (IS_ENABLED(CONFIG_IPV6)) {
1567	struct net_device *ip_dev;
1568	struct in6_addr laddr6;
1569
1570	irdma_copy_ip_htonl(dst: laddr6.in6_u.u6_addr32, src: loc_addr);
1571
1572	for_each_netdev_rcu (&init_net, ip_dev) {
1573	if (ipv6_chk_addr(net: &init_net, addr: &laddr6, dev: ip_dev, strict: 1)) {
1574	ndev = ip_dev;
1575	break;
1576	}
1577	}
1578	}
1579
1580	if (!ndev)
1581	goto done;
1582	if (is_vlan_dev(dev: ndev))
1583	prio = (vlan_dev_get_egress_qos_mask(dev: ndev, skprio: prio) & VLAN_PRIO_MASK)
1584	>> VLAN_PRIO_SHIFT;
1585	if (ipv4)
1586	dev_put(dev: ndev);
1587
1588	done:
1589	rcu_read_unlock();
1590
1591	return prio;
1592	}
1593
1594	/**
1595	* irdma_get_vlan_mac_ipv6 - Gets the vlan and mac
1596	* @addr: local IPv6 address
1597	* @vlan_id: vlan id for the given IPv6 address
1598	* @mac: mac address for the given IPv6 address
1599	*
1600	* Returns the vlan id and mac for an IPv6 address.
1601	*/
1602	void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)
1603	{
1604	struct net_device *ip_dev = NULL;
1605	struct in6_addr laddr6;
1606
1607	if (!IS_ENABLED(CONFIG_IPV6))
1608	return;
1609
1610	irdma_copy_ip_htonl(dst: laddr6.in6_u.u6_addr32, src: addr);
1611	if (vlan_id)
1612	*vlan_id = 0xFFFF; /* Match rdma_vlan_dev_vlan_id() */
1613	if (mac)
1614	eth_zero_addr(addr: mac);
1615
1616	rcu_read_lock();
1617	for_each_netdev_rcu (&init_net, ip_dev) {
1618	if (ipv6_chk_addr(net: &init_net, addr: &laddr6, dev: ip_dev, strict: 1)) {
1619	if (vlan_id)
1620	*vlan_id = rdma_vlan_dev_vlan_id(dev: ip_dev);
1621	if (ip_dev->dev_addr && mac)
1622	ether_addr_copy(dst: mac, src: ip_dev->dev_addr);
1623	break;
1624	}
1625	}
1626	rcu_read_unlock();
1627	}
1628
1629	/**
1630	* irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address
1631	* @addr: local IPv4 address
1632	*/
1633	u16 irdma_get_vlan_ipv4(u32 *addr)
1634	{
1635	struct net_device *netdev;
1636	u16 vlan_id = 0xFFFF;
1637
1638	netdev = ip_dev_find(net: &init_net, htonl(addr[0]));
1639	if (netdev) {
1640	vlan_id = rdma_vlan_dev_vlan_id(dev: netdev);
1641	dev_put(dev: netdev);
1642	}
1643
1644	return vlan_id;
1645	}
1646
1647	/**
1648	* irdma_add_mqh_6 - Adds multiple qhashes for IPv6
1649	* @iwdev: iWarp device
1650	* @cm_info: CM info for parent listen node
1651	* @cm_parent_listen_node: The parent listen node
1652	*
1653	* Adds a qhash and a child listen node for every IPv6 address
1654	* on the adapter and adds the associated qhash filter
1655	*/
1656	static int irdma_add_mqh_6(struct irdma_device *iwdev,
1657	struct irdma_cm_info *cm_info,
1658	struct irdma_cm_listener *cm_parent_listen_node)
1659	{
1660	struct net_device *ip_dev;
1661	struct inet6_dev *idev;
1662	struct inet6_ifaddr *ifp, *tmp;
1663	struct irdma_cm_listener *child_listen_node;
1664	unsigned long flags;
1665	int ret = 0;
1666
1667	rtnl_lock();
1668	for_each_netdev(&init_net, ip_dev) {
1669	if (!(ip_dev->flags & IFF_UP))
1670	continue;
1671
1672	if (((rdma_vlan_dev_vlan_id(dev: ip_dev) >= VLAN_N_VID) ||
1673	(rdma_vlan_dev_real_dev(dev: ip_dev) != iwdev->netdev)) &&
1674	ip_dev != iwdev->netdev)
1675	continue;
1676
1677	idev = __in6_dev_get(dev: ip_dev);
1678	if (!idev) {
1679	ibdev_dbg(&iwdev->ibdev, "CM: idev == NULL\n");
1680	break;
1681	}
1682	list_for_each_entry_safe (ifp, tmp, &idev->addr_list, if_list) {
1683	ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI6, vlan_id=%d, MAC=%pM\n",
1684	&ifp->addr, rdma_vlan_dev_vlan_id(ip_dev),
1685	ip_dev->dev_addr);
1686	child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
1687	ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",
1688	child_listen_node);
1689	if (!child_listen_node) {
1690	ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
1691	ret = -ENOMEM;
1692	goto exit;
1693	}
1694
1695	cm_info->vlan_id = rdma_vlan_dev_vlan_id(dev: ip_dev);
1696	cm_parent_listen_node->vlan_id = cm_info->vlan_id;
1697	memcpy(child_listen_node, cm_parent_listen_node,
1698	sizeof(*child_listen_node));
1699	irdma_copy_ip_ntohl(dst: child_listen_node->loc_addr,
1700	src: ifp->addr.in6_u.u6_addr32);
1701	memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1702	sizeof(cm_info->loc_addr));
1703	if (!iwdev->vsi.dscp_mode)
1704	cm_info->user_pri =
1705	irdma_iw_get_vlan_prio(loc_addr: child_listen_node->loc_addr,
1706	prio: cm_info->user_pri,
1707	ipv4: false);
1708
1709	ret = irdma_manage_qhash(iwdev, cminfo: cm_info,
1710	etype: IRDMA_QHASH_TYPE_TCP_SYN,
1711	mtype: IRDMA_QHASH_MANAGE_TYPE_ADD,
1712	NULL, wait: true);
1713	if (ret) {
1714	kfree(objp: child_listen_node);
1715	continue;
1716	}
1717
1718	trace_irdma_add_mqh_6(iwdev, listener: child_listen_node,
1719	dev_addr: ip_dev->dev_addr);
1720
1721	child_listen_node->qhash_set = true;
1722	spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
1723	list_add(new: &child_listen_node->child_listen_list,
1724	head: &cm_parent_listen_node->child_listen_list);
1725	spin_unlock_irqrestore(lock: &iwdev->cm_core.listen_list_lock, flags);
1726	cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
1727	}
1728	}
1729	exit:
1730	rtnl_unlock();
1731
1732	return ret;
1733	}
1734
1735	/**
1736	* irdma_add_mqh_4 - Adds multiple qhashes for IPv4
1737	* @iwdev: iWarp device
1738	* @cm_info: CM info for parent listen node
1739	* @cm_parent_listen_node: The parent listen node
1740	*
1741	* Adds a qhash and a child listen node for every IPv4 address
1742	* on the adapter and adds the associated qhash filter
1743	*/
1744	static int irdma_add_mqh_4(struct irdma_device *iwdev,
1745	struct irdma_cm_info *cm_info,
1746	struct irdma_cm_listener *cm_parent_listen_node)
1747	{
1748	struct net_device *ip_dev;
1749	struct in_device *idev;
1750	struct irdma_cm_listener *child_listen_node;
1751	unsigned long flags;
1752	const struct in_ifaddr *ifa;
1753	int ret = 0;
1754
1755	rtnl_lock();
1756	for_each_netdev(&init_net, ip_dev) {
1757	if (!(ip_dev->flags & IFF_UP))
1758	continue;
1759
1760	if (((rdma_vlan_dev_vlan_id(dev: ip_dev) >= VLAN_N_VID) ||
1761	(rdma_vlan_dev_real_dev(dev: ip_dev) != iwdev->netdev)) &&
1762	ip_dev != iwdev->netdev)
1763	continue;
1764
1765	idev = in_dev_get(dev: ip_dev);
1766	if (!idev)
1767	continue;
1768
1769	in_dev_for_each_ifa_rtnl(ifa, idev) {
1770	ibdev_dbg(&iwdev->ibdev,
1771	"CM: Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",
1772	&ifa->ifa_address, rdma_vlan_dev_vlan_id(ip_dev),
1773	ip_dev->dev_addr);
1774	child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
1775	cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
1776	ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",
1777	child_listen_node);
1778	if (!child_listen_node) {
1779	ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
1780	in_dev_put(idev);
1781	ret = -ENOMEM;
1782	goto exit;
1783	}
1784
1785	cm_info->vlan_id = rdma_vlan_dev_vlan_id(dev: ip_dev);
1786	cm_parent_listen_node->vlan_id = cm_info->vlan_id;
1787	memcpy(child_listen_node, cm_parent_listen_node,
1788	sizeof(*child_listen_node));
1789	child_listen_node->loc_addr[0] =
1790	ntohl(ifa->ifa_address);
1791	memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1792	sizeof(cm_info->loc_addr));
1793	if (!iwdev->vsi.dscp_mode)
1794	cm_info->user_pri =
1795	irdma_iw_get_vlan_prio(loc_addr: child_listen_node->loc_addr,
1796	prio: cm_info->user_pri,
1797	ipv4: true);
1798	ret = irdma_manage_qhash(iwdev, cminfo: cm_info,
1799	etype: IRDMA_QHASH_TYPE_TCP_SYN,
1800	mtype: IRDMA_QHASH_MANAGE_TYPE_ADD,
1801	NULL, wait: true);
1802	if (ret) {
1803	kfree(objp: child_listen_node);
1804	cm_parent_listen_node->cm_core
1805	->stats_listen_nodes_created--;
1806	continue;
1807	}
1808
1809	trace_irdma_add_mqh_4(iwdev, listener: child_listen_node,
1810	dev_addr: ip_dev->dev_addr);
1811
1812	child_listen_node->qhash_set = true;
1813	spin_lock_irqsave(&iwdev->cm_core.listen_list_lock,
1814	flags);
1815	list_add(new: &child_listen_node->child_listen_list,
1816	head: &cm_parent_listen_node->child_listen_list);
1817	spin_unlock_irqrestore(lock: &iwdev->cm_core.listen_list_lock, flags);
1818	}
1819	in_dev_put(idev);
1820	}
1821	exit:
1822	rtnl_unlock();
1823
1824	return ret;
1825	}
1826
1827	/**
1828	* irdma_add_mqh - Adds multiple qhashes
1829	* @iwdev: iWarp device
1830	* @cm_info: CM info for parent listen node
1831	* @cm_listen_node: The parent listen node
1832	*/
1833	static int irdma_add_mqh(struct irdma_device *iwdev,
1834	struct irdma_cm_info *cm_info,
1835	struct irdma_cm_listener *cm_listen_node)
1836	{
1837	if (cm_info->ipv4)
1838	return irdma_add_mqh_4(iwdev, cm_info, cm_parent_listen_node: cm_listen_node);
1839	else
1840	return irdma_add_mqh_6(iwdev, cm_info, cm_parent_listen_node: cm_listen_node);
1841	}
1842
1843	/**
1844	* irdma_reset_list_prep - add connection nodes slated for reset to list
1845	* @cm_core: cm's core
1846	* @listener: pointer to listener node
1847	* @reset_list: a list to which cm_node will be selected
1848	*/
1849	static void irdma_reset_list_prep(struct irdma_cm_core *cm_core,
1850	struct irdma_cm_listener *listener,
1851	struct list_head *reset_list)
1852	{
1853	struct irdma_cm_node *cm_node;
1854	int bkt;
1855
1856	hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
1857	if (cm_node->listener == listener &&
1858	!cm_node->accelerated &&
1859	refcount_inc_not_zero(r: &cm_node->refcnt))
1860	list_add(new: &cm_node->reset_entry, head: reset_list);
1861	}
1862	}
1863
1864	/**
1865	* irdma_dec_refcnt_listen - delete listener and associated cm nodes
1866	* @cm_core: cm's core
1867	* @listener: pointer to listener node
1868	* @free_hanging_nodes: to free associated cm_nodes
1869	* @apbvt_del: flag to delete the apbvt
1870	*/
1871	static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core,
1872	struct irdma_cm_listener *listener,
1873	int free_hanging_nodes, bool apbvt_del)
1874	{
1875	int err;
1876	struct list_head *list_pos;
1877	struct list_head *list_temp;
1878	struct irdma_cm_node *cm_node;
1879	struct list_head reset_list;
1880	struct irdma_cm_info nfo;
1881	enum irdma_cm_node_state old_state;
1882	unsigned long flags;
1883
1884	trace_irdma_dec_refcnt_listen(listener, caller: __builtin_return_address(0));
1885	/* free non-accelerated child nodes for this listener */
1886	INIT_LIST_HEAD(list: &reset_list);
1887	if (free_hanging_nodes) {
1888	rcu_read_lock();
1889	irdma_reset_list_prep(cm_core, listener, reset_list: &reset_list);
1890	rcu_read_unlock();
1891	}
1892
1893	list_for_each_safe (list_pos, list_temp, &reset_list) {
1894	cm_node = container_of(list_pos, struct irdma_cm_node,
1895	reset_entry);
1896	if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) {
1897	irdma_rem_ref_cm_node(cm_node);
1898	continue;
1899	}
1900
1901	irdma_cleanup_retrans_entry(cm_node);
1902	err = irdma_send_reset(cm_node);
1903	if (err) {
1904	cm_node->state = IRDMA_CM_STATE_CLOSED;
1905	ibdev_dbg(&cm_node->iwdev->ibdev,
1906	"CM: send reset failed\n");
1907	} else {
1908	old_state = cm_node->state;
1909	cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED;
1910	if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD)
1911	irdma_rem_ref_cm_node(cm_node);
1912	}
1913	}
1914
1915	if (refcount_dec_and_test(r: &listener->refcnt)) {
1916	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
1917	list_del(entry: &listener->list);
1918	spin_unlock_irqrestore(lock: &cm_core->listen_list_lock, flags);
1919
1920	if (apbvt_del)
1921	irdma_del_apbvt(iwdev: listener->iwdev,
1922	entry: listener->apbvt_entry);
1923	memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr));
1924	nfo.loc_port = listener->loc_port;
1925	nfo.ipv4 = listener->ipv4;
1926	nfo.vlan_id = listener->vlan_id;
1927	nfo.user_pri = listener->user_pri;
1928	nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id;
1929
1930	if (!list_empty(head: &listener->child_listen_list)) {
1931	irdma_del_multiple_qhash(iwdev: listener->iwdev, cm_info: &nfo,
1932	cm_parent_listen_node: listener);
1933	} else {
1934	if (listener->qhash_set)
1935	irdma_manage_qhash(iwdev: listener->iwdev,
1936	cminfo: &nfo,
1937	etype: IRDMA_QHASH_TYPE_TCP_SYN,
1938	mtype: IRDMA_QHASH_MANAGE_TYPE_DELETE,
1939	NULL, wait: false);
1940	}
1941
1942	cm_core->stats_listen_destroyed++;
1943	cm_core->stats_listen_nodes_destroyed++;
1944	ibdev_dbg(&listener->iwdev->ibdev,
1945	"CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",
1946	listener->loc_port, listener->loc_addr, listener,
1947	listener->cm_id, listener->qhash_set,
1948	listener->vlan_id, apbvt_del);
1949	kfree(objp: listener);
1950	listener = NULL;
1951	return 0;
1952	}
1953
1954	return -EINVAL;
1955	}
1956
1957	/**
1958	* irdma_cm_del_listen - delete a listener
1959	* @cm_core: cm's core
1960	* @listener: passive connection's listener
1961	* @apbvt_del: flag to delete apbvt
1962	*/
1963	static int irdma_cm_del_listen(struct irdma_cm_core *cm_core,
1964	struct irdma_cm_listener *listener,
1965	bool apbvt_del)
1966	{
1967	listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE;
1968	listener->cm_id = NULL;
1969
1970	return irdma_dec_refcnt_listen(cm_core, listener, free_hanging_nodes: 1, apbvt_del);
1971	}
1972
1973	/**
1974	* irdma_addr_resolve_neigh - resolve neighbor address
1975	* @iwdev: iwarp device structure
1976	* @src_ip: local ip address
1977	* @dst_ip: remote ip address
1978	* @arpindex: if there is an arp entry
1979	*/
1980	static int irdma_addr_resolve_neigh(struct irdma_device *iwdev, u32 src_ip,
1981	u32 dst_ip, int arpindex)
1982	{
1983	struct rtable *rt;
1984	struct neighbour *neigh;
1985	int rc = arpindex;
1986	__be32 dst_ipaddr = htonl(dst_ip);
1987	__be32 src_ipaddr = htonl(src_ip);
1988
1989	rt = ip_route_output(net: &init_net, daddr: dst_ipaddr, saddr: src_ipaddr, dscp: 0, oif: 0,
1990	scope: RT_SCOPE_UNIVERSE);
1991	if (IS_ERR(ptr: rt)) {
1992	ibdev_dbg(&iwdev->ibdev, "CM: ip_route_output fail\n");
1993	return -EINVAL;
1994	}
1995
1996	neigh = dst_neigh_lookup(dst: &rt->dst, daddr: &dst_ipaddr);
1997	if (!neigh)
1998	goto exit;
1999
2000	if (neigh->nud_state & NUD_VALID)
2001	rc = irdma_add_arp(rf: iwdev->rf, ip: &dst_ip, ipv4: true, mac: neigh->ha);
2002	else
2003	neigh_event_send(neigh, NULL);
2004	if (neigh)
2005	neigh_release(neigh);
2006	exit:
2007	ip_rt_put(rt);
2008
2009	return rc;
2010	}
2011
2012	/**
2013	* irdma_get_dst_ipv6 - get destination cache entry via ipv6 lookup
2014	* @src_addr: local ipv6 sock address
2015	* @dst_addr: destination ipv6 sock address
2016	*/
2017	static struct dst_entry *irdma_get_dst_ipv6(struct sockaddr_in6 *src_addr,
2018	struct sockaddr_in6 *dst_addr)
2019	{
2020	struct dst_entry *dst = NULL;
2021
2022	if ((IS_ENABLED(CONFIG_IPV6))) {
2023	struct flowi6 fl6 = {};
2024
2025	fl6.daddr = dst_addr->sin6_addr;
2026	fl6.saddr = src_addr->sin6_addr;
2027	if (ipv6_addr_type(addr: &fl6.daddr) & IPV6_ADDR_LINKLOCAL)
2028	fl6.flowi6_oif = dst_addr->sin6_scope_id;
2029
2030	dst = ip6_route_output(net: &init_net, NULL, fl6: &fl6);
2031	}
2032
2033	return dst;
2034	}
2035
2036	/**
2037	* irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
2038	* @iwdev: iwarp device structure
2039	* @src: local ip address
2040	* @dest: remote ip address
2041	* @arpindex: if there is an arp entry
2042	*/
2043	static int irdma_addr_resolve_neigh_ipv6(struct irdma_device *iwdev, u32 *src,
2044	u32 *dest, int arpindex)
2045	{
2046	struct neighbour *neigh;
2047	int rc = arpindex;
2048	struct dst_entry *dst;
2049	struct sockaddr_in6 dst_addr = {};
2050	struct sockaddr_in6 src_addr = {};
2051
2052	dst_addr.sin6_family = AF_INET6;
2053	irdma_copy_ip_htonl(dst: dst_addr.sin6_addr.in6_u.u6_addr32, src: dest);
2054	src_addr.sin6_family = AF_INET6;
2055	irdma_copy_ip_htonl(dst: src_addr.sin6_addr.in6_u.u6_addr32, src);
2056	dst = irdma_get_dst_ipv6(src_addr: &src_addr, dst_addr: &dst_addr);
2057	if (!dst || dst->error) {
2058	if (dst) {
2059	dst_release(dst);
2060	ibdev_dbg(&iwdev->ibdev,
2061	"CM: ip6_route_output returned dst->error = %d\n",
2062	dst->error);
2063	}
2064	return -EINVAL;
2065	}
2066
2067	neigh = dst_neigh_lookup(dst, daddr: dst_addr.sin6_addr.in6_u.u6_addr32);
2068	if (!neigh)
2069	goto exit;
2070
2071	ibdev_dbg(&iwdev->ibdev, "CM: dst_neigh_lookup MAC=%pM\n",
2072	neigh->ha);
2073
2074	trace_irdma_addr_resolve(iwdev, dev_addr: neigh->ha);
2075
2076	if (neigh->nud_state & NUD_VALID)
2077	rc = irdma_add_arp(rf: iwdev->rf, ip: dest, ipv4: false, mac: neigh->ha);
2078	else
2079	neigh_event_send(neigh, NULL);
2080	if (neigh)
2081	neigh_release(neigh);
2082	exit:
2083	dst_release(dst);
2084
2085	return rc;
2086	}
2087
2088	/**
2089	* irdma_find_node - find a cm node that matches the reference cm node
2090	* @cm_core: cm's core
2091	* @rem_port: remote tcp port num
2092	* @rem_addr: remote ip addr
2093	* @loc_port: local tcp port num
2094	* @loc_addr: local ip addr
2095	* @vlan_id: local VLAN ID
2096	*/
2097	struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,
2098	u16 rem_port, u32 *rem_addr, u16 loc_port,
2099	u32 *loc_addr, u16 vlan_id)
2100	{
2101	struct irdma_cm_node *cm_node;
2102	u32 key = (rem_port << 16) | loc_port;
2103
2104	rcu_read_lock();
2105	hash_for_each_possible_rcu(cm_core->cm_hash_tbl, cm_node, list, key) {
2106	if (cm_node->vlan_id == vlan_id &&
2107	cm_node->loc_port == loc_port && cm_node->rem_port == rem_port &&
2108	!memcmp(p: cm_node->loc_addr, q: loc_addr, size: sizeof(cm_node->loc_addr)) &&
2109	!memcmp(p: cm_node->rem_addr, q: rem_addr, size: sizeof(cm_node->rem_addr))) {
2110	if (!refcount_inc_not_zero(r: &cm_node->refcnt))
2111	goto exit;
2112	rcu_read_unlock();
2113	trace_irdma_find_node(cm_node, type: 0, NULL);
2114	return cm_node;
2115	}
2116	}
2117
2118	exit:
2119	rcu_read_unlock();
2120
2121	/* no owner node */
2122	return NULL;
2123	}
2124
2125	/**
2126	* irdma_add_hte_node - add a cm node to the hash table
2127	* @cm_core: cm's core
2128	* @cm_node: connection's node
2129	*/
2130	static void irdma_add_hte_node(struct irdma_cm_core *cm_core,
2131	struct irdma_cm_node *cm_node)
2132	{
2133	unsigned long flags;
2134	u32 key = (cm_node->rem_port << 16) | cm_node->loc_port;
2135
2136	spin_lock_irqsave(&cm_core->ht_lock, flags);
2137	hash_add_rcu(cm_core->cm_hash_tbl, &cm_node->list, key);
2138	spin_unlock_irqrestore(lock: &cm_core->ht_lock, flags);
2139	}
2140
2141	/**
2142	* irdma_ipv4_is_lpb - check if loopback
2143	* @loc_addr: local addr to compare
2144	* @rem_addr: remote address
2145	*/
2146	bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)
2147	{
2148	return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
2149	}
2150
2151	/**
2152	* irdma_ipv6_is_lpb - check if loopback
2153	* @loc_addr: local addr to compare
2154	* @rem_addr: remote address
2155	*/
2156	bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr)
2157	{
2158	struct in6_addr raddr6;
2159
2160	irdma_copy_ip_htonl(dst: raddr6.in6_u.u6_addr32, src: rem_addr);
2161
2162	return !memcmp(p: loc_addr, q: rem_addr, size: 16) || ipv6_addr_loopback(a: &raddr6);
2163	}
2164
2165	/**
2166	* irdma_cm_create_ah - create a cm address handle
2167	* @cm_node: The connection manager node to create AH for
2168	* @wait: Provides option to wait for ah creation or not
2169	*/
2170	static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)
2171	{
2172	struct irdma_ah_info ah_info = {};
2173	struct irdma_device *iwdev = cm_node->iwdev;
2174
2175	ether_addr_copy(dst: ah_info.mac_addr, src: iwdev->netdev->dev_addr);
2176
2177	ah_info.hop_ttl = 0x40;
2178	ah_info.tc_tos = cm_node->tos;
2179	ah_info.vsi = &iwdev->vsi;
2180
2181	if (cm_node->ipv4) {
2182	ah_info.ipv4_valid = true;
2183	ah_info.dest_ip_addr[0] = cm_node->rem_addr[0];
2184	ah_info.src_ip_addr[0] = cm_node->loc_addr[0];
2185	ah_info.do_lpbk = irdma_ipv4_is_lpb(loc_addr: ah_info.src_ip_addr[0],
2186	rem_addr: ah_info.dest_ip_addr[0]);
2187	} else {
2188	memcpy(ah_info.dest_ip_addr, cm_node->rem_addr,
2189	sizeof(ah_info.dest_ip_addr));
2190	memcpy(ah_info.src_ip_addr, cm_node->loc_addr,
2191	sizeof(ah_info.src_ip_addr));
2192	ah_info.do_lpbk = irdma_ipv6_is_lpb(loc_addr: ah_info.src_ip_addr,
2193	rem_addr: ah_info.dest_ip_addr);
2194	}
2195
2196	ah_info.vlan_tag = cm_node->vlan_id;
2197	if (cm_node->vlan_id < VLAN_N_VID) {
2198	ah_info.insert_vlan_tag = 1;
2199	ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;
2200	}
2201
2202	ah_info.dst_arpindex =
2203	irdma_arp_table(rf: iwdev->rf, ip_addr: ah_info.dest_ip_addr,
2204	ipv4: ah_info.ipv4_valid, NULL, IRDMA_ARP_RESOLVE);
2205
2206	if (irdma_puda_create_ah(dev: &iwdev->rf->sc_dev, ah_info: &ah_info, wait,
2207	type: IRDMA_PUDA_RSRC_TYPE_ILQ, cb_param: cm_node,
2208	ah: &cm_node->ah))
2209	return -ENOMEM;
2210
2211	trace_irdma_create_ah(cm_node);
2212	return 0;
2213	}
2214
2215	/**
2216	* irdma_cm_free_ah - free a cm address handle
2217	* @cm_node: The connection manager node to create AH for
2218	*/
2219	static void irdma_cm_free_ah(struct irdma_cm_node *cm_node)
2220	{
2221	struct irdma_device *iwdev = cm_node->iwdev;
2222
2223	trace_irdma_cm_free_ah(cm_node);
2224	irdma_puda_free_ah(dev: &iwdev->rf->sc_dev, ah: cm_node->ah);
2225	cm_node->ah = NULL;
2226	}
2227
2228	/**
2229	* irdma_make_cm_node - create a new instance of a cm node
2230	* @cm_core: cm's core
2231	* @iwdev: iwarp device structure
2232	* @cm_info: quad info for connection
2233	* @listener: passive connection's listener
2234	*/
2235	static struct irdma_cm_node *
2236	irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,
2237	struct irdma_cm_info *cm_info,
2238	struct irdma_cm_listener *listener)
2239	{
2240	struct irdma_cm_node *cm_node;
2241	int oldarpindex;
2242	int arpindex;
2243	struct net_device *netdev = iwdev->netdev;
2244
2245	/* create an hte and cm_node for this instance */
2246	cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
2247	if (!cm_node)
2248	return NULL;
2249
2250	/* set our node specific transport info */
2251	cm_node->ipv4 = cm_info->ipv4;
2252	cm_node->vlan_id = cm_info->vlan_id;
2253	if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
2254	cm_node->vlan_id = 0;
2255	cm_node->tos = cm_info->tos;
2256	cm_node->user_pri = cm_info->user_pri;
2257	if (listener) {
2258	if (listener->tos != cm_info->tos)
2259	ibdev_warn(ibdev: &iwdev->ibdev,
2260	format: "application TOS[%d] and remote client TOS[%d] mismatch\n",
2261	listener->tos, cm_info->tos);
2262	if (iwdev->vsi.dscp_mode) {
2263	cm_node->user_pri = listener->user_pri;
2264	} else {
2265	cm_node->tos = max(listener->tos, cm_info->tos);
2266	cm_node->user_pri = rt_tos2priority(tos: cm_node->tos);
2267	cm_node->user_pri =
2268	irdma_iw_get_vlan_prio(loc_addr: cm_info->loc_addr,
2269	prio: cm_node->user_pri,
2270	ipv4: cm_info->ipv4);
2271	}
2272	ibdev_dbg(&iwdev->ibdev,
2273	"DCB: listener: TOS:[%d] UP:[%d]\n", cm_node->tos,
2274	cm_node->user_pri);
2275	trace_irdma_listener_tos(iwdev, tos: cm_node->tos,
2276	user_pri: cm_node->user_pri);
2277	}
2278	memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
2279	memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
2280	cm_node->loc_port = cm_info->loc_port;
2281	cm_node->rem_port = cm_info->rem_port;
2282
2283	cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER;
2284	cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
2285	cm_node->iwdev = iwdev;
2286	cm_node->dev = &iwdev->rf->sc_dev;
2287
2288	cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird;
2289	cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord;
2290
2291	cm_node->listener = listener;
2292	cm_node->cm_id = cm_info->cm_id;
2293	ether_addr_copy(dst: cm_node->loc_mac, src: netdev->dev_addr);
2294	spin_lock_init(&cm_node->retrans_list_lock);
2295	cm_node->ack_rcvd = false;
2296
2297	init_completion(x: &cm_node->establish_comp);
2298	refcount_set(r: &cm_node->refcnt, n: 1);
2299	/* associate our parent CM core */
2300	cm_node->cm_core = cm_core;
2301	cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID;
2302	cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale;
2303	cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale;
2304	if (cm_node->ipv4) {
2305	cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]),
2306	htonl(cm_node->rem_addr[0]),
2307	htons(cm_node->loc_port),
2308	htons(cm_node->rem_port));
2309	cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4;
2310	} else if (IS_ENABLED(CONFIG_IPV6)) {
2311	__be32 loc[4] = {
2312	htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]),
2313	htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3])
2314	};
2315	__be32 rem[4] = {
2316	htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]),
2317	htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3])
2318	};
2319	cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(saddr: loc, daddr: rem,
2320	htons(cm_node->loc_port),
2321	htons(cm_node->rem_port));
2322	cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6;
2323	}
2324
2325	if ((cm_node->ipv4 &&
2326	irdma_ipv4_is_lpb(loc_addr: cm_node->loc_addr[0], rem_addr: cm_node->rem_addr[0])) ||
2327	(!cm_node->ipv4 &&
2328	irdma_ipv6_is_lpb(loc_addr: cm_node->loc_addr, rem_addr: cm_node->rem_addr))) {
2329	cm_node->do_lpb = true;
2330	arpindex = irdma_arp_table(rf: iwdev->rf, ip_addr: cm_node->rem_addr,
2331	ipv4: cm_node->ipv4, NULL,
2332	IRDMA_ARP_RESOLVE);
2333	} else {
2334	oldarpindex = irdma_arp_table(rf: iwdev->rf, ip_addr: cm_node->rem_addr,
2335	ipv4: cm_node->ipv4, NULL,
2336	IRDMA_ARP_RESOLVE);
2337	if (cm_node->ipv4)
2338	arpindex = irdma_addr_resolve_neigh(iwdev,
2339	src_ip: cm_info->loc_addr[0],
2340	dst_ip: cm_info->rem_addr[0],
2341	arpindex: oldarpindex);
2342	else if (IS_ENABLED(CONFIG_IPV6))
2343	arpindex = irdma_addr_resolve_neigh_ipv6(iwdev,
2344	src: cm_info->loc_addr,
2345	dest: cm_info->rem_addr,
2346	arpindex: oldarpindex);
2347	else
2348	arpindex = -EINVAL;
2349	}
2350
2351	if (arpindex < 0)
2352	goto err;
2353
2354	ether_addr_copy(dst: cm_node->rem_mac,
2355	src: iwdev->rf->arp_table[arpindex].mac_addr);
2356	irdma_add_hte_node(cm_core, cm_node);
2357	cm_core->stats_nodes_created++;
2358	return cm_node;
2359
2360	err:
2361	kfree(objp: cm_node);
2362
2363	return NULL;
2364	}
2365
2366	static void irdma_destroy_connection(struct irdma_cm_node *cm_node)
2367	{
2368	struct irdma_cm_core *cm_core = cm_node->cm_core;
2369	struct irdma_qp *iwqp;
2370	struct irdma_cm_info nfo;
2371
2372	/* if the node is destroyed before connection was accelerated */
2373	if (!cm_node->accelerated && cm_node->accept_pend) {
2374	ibdev_dbg(&cm_node->iwdev->ibdev,
2375	"CM: node destroyed before established\n");
2376	atomic_dec(v: &cm_node->listener->pend_accepts_cnt);
2377	}
2378	if (cm_node->close_entry)
2379	irdma_handle_close_entry(cm_node, rem_node: 0);
2380	if (cm_node->listener) {
2381	irdma_dec_refcnt_listen(cm_core, listener: cm_node->listener, free_hanging_nodes: 0, apbvt_del: true);
2382	} else {
2383	if (cm_node->apbvt_set) {
2384	irdma_del_apbvt(iwdev: cm_node->iwdev, entry: cm_node->apbvt_entry);
2385	cm_node->apbvt_set = 0;
2386	}
2387	irdma_get_addr_info(cm_node, cm_info: &nfo);
2388	if (cm_node->qhash_set) {
2389	nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2390	irdma_manage_qhash(iwdev: cm_node->iwdev, cminfo: &nfo,
2391	etype: IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2392	mtype: IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL,
2393	wait: false);
2394	cm_node->qhash_set = 0;
2395	}
2396	}
2397
2398	iwqp = cm_node->iwqp;
2399	if (iwqp) {
2400	cm_node->cm_id->rem_ref(cm_node->cm_id);
2401	cm_node->cm_id = NULL;
2402	iwqp->cm_id = NULL;
2403	irdma_qp_rem_ref(ibqp: &iwqp->ibqp);
2404	cm_node->iwqp = NULL;
2405	} else if (cm_node->qhash_set) {
2406	irdma_get_addr_info(cm_node, cm_info: &nfo);
2407	nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2408	irdma_manage_qhash(iwdev: cm_node->iwdev, cminfo: &nfo,
2409	etype: IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2410	mtype: IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, wait: false);
2411	cm_node->qhash_set = 0;
2412	}
2413
2414	cm_core->cm_free_ah(cm_node);
2415	}
2416
2417	/**
2418	* irdma_rem_ref_cm_node - destroy an instance of a cm node
2419	* @cm_node: connection's node
2420	*/
2421	void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)
2422	{
2423	struct irdma_cm_core *cm_core = cm_node->cm_core;
2424	unsigned long flags;
2425
2426	trace_irdma_rem_ref_cm_node(cm_node, type: 0, caller: __builtin_return_address(0));
2427	spin_lock_irqsave(&cm_core->ht_lock, flags);
2428
2429	if (!refcount_dec_and_test(r: &cm_node->refcnt)) {
2430	spin_unlock_irqrestore(lock: &cm_core->ht_lock, flags);
2431	return;
2432	}
2433	if (cm_node->iwqp) {
2434	cm_node->iwqp->cm_node = NULL;
2435	cm_node->iwqp->cm_id = NULL;
2436	}
2437	hash_del_rcu(node: &cm_node->list);
2438	cm_node->cm_core->stats_nodes_destroyed++;
2439
2440	spin_unlock_irqrestore(lock: &cm_core->ht_lock, flags);
2441
2442	irdma_destroy_connection(cm_node);
2443
2444	kfree_rcu(cm_node, rcu_head);
2445	}
2446
2447	/**
2448	* irdma_handle_fin_pkt - FIN packet received
2449	* @cm_node: connection's node
2450	*/
2451	static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node)
2452	{
2453	switch (cm_node->state) {
2454	case IRDMA_CM_STATE_SYN_RCVD:
2455	case IRDMA_CM_STATE_SYN_SENT:
2456	case IRDMA_CM_STATE_ESTABLISHED:
2457	case IRDMA_CM_STATE_MPAREJ_RCVD:
2458	cm_node->tcp_cntxt.rcv_nxt++;
2459	irdma_cleanup_retrans_entry(cm_node);
2460	cm_node->state = IRDMA_CM_STATE_LAST_ACK;
2461	irdma_send_fin(cm_node);
2462	break;
2463	case IRDMA_CM_STATE_MPAREQ_SENT:
2464	irdma_create_event(cm_node, type: IRDMA_CM_EVENT_ABORTED);
2465	cm_node->tcp_cntxt.rcv_nxt++;
2466	irdma_cleanup_retrans_entry(cm_node);
2467	cm_node->state = IRDMA_CM_STATE_CLOSED;
2468	refcount_inc(r: &cm_node->refcnt);
2469	irdma_send_reset(cm_node);
2470	break;
2471	case IRDMA_CM_STATE_FIN_WAIT1:
2472	cm_node->tcp_cntxt.rcv_nxt++;
2473	irdma_cleanup_retrans_entry(cm_node);
2474	cm_node->state = IRDMA_CM_STATE_CLOSING;
2475	irdma_send_ack(cm_node);
2476	/*
2477	* Wait for ACK as this is simultaneous close.
2478	* After we receive ACK, do not send anything.
2479	* Just rm the node.
2480	*/
2481	break;
2482	case IRDMA_CM_STATE_FIN_WAIT2:
2483	cm_node->tcp_cntxt.rcv_nxt++;
2484	irdma_cleanup_retrans_entry(cm_node);
2485	cm_node->state = IRDMA_CM_STATE_TIME_WAIT;
2486	irdma_send_ack(cm_node);
2487	irdma_schedule_cm_timer(cm_node, NULL, type: IRDMA_TIMER_TYPE_CLOSE,
2488	send_retrans: 1, close_when_complete: 0);
2489	break;
2490	case IRDMA_CM_STATE_TIME_WAIT:
2491	cm_node->tcp_cntxt.rcv_nxt++;
2492	irdma_cleanup_retrans_entry(cm_node);
2493	cm_node->state = IRDMA_CM_STATE_CLOSED;
2494	irdma_rem_ref_cm_node(cm_node);
2495	break;
2496	case IRDMA_CM_STATE_OFFLOADED:
2497	default:
2498	ibdev_dbg(&cm_node->iwdev->ibdev,
2499	"CM: bad state node state = %d\n", cm_node->state);
2500	break;
2501	}
2502	}
2503
2504	/**
2505	* irdma_handle_rst_pkt - process received RST packet
2506	* @cm_node: connection's node
2507	* @rbuf: receive buffer
2508	*/
2509	static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node,
2510	struct irdma_puda_buf *rbuf)
2511	{
2512	ibdev_dbg(&cm_node->iwdev->ibdev,
2513	"CM: caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
2514	__builtin_return_address(0), cm_node, cm_node->state,
2515	cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
2516	cm_node->loc_addr);
2517
2518	irdma_cleanup_retrans_entry(cm_node);
2519	switch (cm_node->state) {
2520	case IRDMA_CM_STATE_SYN_SENT:
2521	case IRDMA_CM_STATE_MPAREQ_SENT:
2522	switch (cm_node->mpa_frame_rev) {
2523	case IETF_MPA_V2:
2524	/* Drop down to MPA_V1*/
2525	cm_node->mpa_frame_rev = IETF_MPA_V1;
2526	/* send a syn and goto syn sent state */
2527	cm_node->state = IRDMA_CM_STATE_SYN_SENT;
2528	if (irdma_send_syn(cm_node, sendack: 0))
2529	irdma_active_open_err(cm_node, reset: false);
2530	break;
2531	case IETF_MPA_V1:
2532	default:
2533	irdma_active_open_err(cm_node, reset: false);
2534	break;
2535	}
2536	break;
2537	case IRDMA_CM_STATE_MPAREQ_RCVD:
2538	atomic_inc(v: &cm_node->passive_state);
2539	break;
2540	case IRDMA_CM_STATE_ESTABLISHED:
2541	case IRDMA_CM_STATE_SYN_RCVD:
2542	case IRDMA_CM_STATE_LISTENING:
2543	irdma_passive_open_err(cm_node, reset: false);
2544	break;
2545	case IRDMA_CM_STATE_OFFLOADED:
2546	irdma_active_open_err(cm_node, reset: false);
2547	break;
2548	case IRDMA_CM_STATE_CLOSED:
2549	break;
2550	case IRDMA_CM_STATE_FIN_WAIT2:
2551	case IRDMA_CM_STATE_FIN_WAIT1:
2552	case IRDMA_CM_STATE_LAST_ACK:
2553	case IRDMA_CM_STATE_TIME_WAIT:
2554	cm_node->state = IRDMA_CM_STATE_CLOSED;
2555	irdma_rem_ref_cm_node(cm_node);
2556	break;
2557	default:
2558	break;
2559	}
2560	}
2561
2562	/**
2563	* irdma_handle_rcv_mpa - Process a recv'd mpa buffer
2564	* @cm_node: connection's node
2565	* @rbuf: receive buffer
2566	*/
2567	static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node,
2568	struct irdma_puda_buf *rbuf)
2569	{
2570	int err;
2571	int datasize = rbuf->datalen;
2572	u8 *dataloc = rbuf->data;
2573
2574	enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN;
2575	u32 res_type;
2576
2577	err = irdma_parse_mpa(cm_node, buf: dataloc, type: &res_type, len: datasize);
2578	if (err) {
2579	if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT)
2580	irdma_active_open_err(cm_node, reset: true);
2581	else
2582	irdma_passive_open_err(cm_node, reset: true);
2583	return;
2584	}
2585
2586	switch (cm_node->state) {
2587	case IRDMA_CM_STATE_ESTABLISHED:
2588	if (res_type == IRDMA_MPA_REQUEST_REJECT)
2589	ibdev_dbg(&cm_node->iwdev->ibdev,
2590	"CM: state for reject\n");
2591	cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD;
2592	type = IRDMA_CM_EVENT_MPA_REQ;
2593	irdma_send_ack(cm_node); /* ACK received MPA request */
2594	atomic_set(v: &cm_node->passive_state,
2595	IRDMA_PASSIVE_STATE_INDICATED);
2596	break;
2597	case IRDMA_CM_STATE_MPAREQ_SENT:
2598	irdma_cleanup_retrans_entry(cm_node);
2599	if (res_type == IRDMA_MPA_REQUEST_REJECT) {
2600	type = IRDMA_CM_EVENT_MPA_REJECT;
2601	cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD;
2602	} else {
2603	type = IRDMA_CM_EVENT_CONNECTED;
2604	cm_node->state = IRDMA_CM_STATE_OFFLOADED;
2605	}
2606	irdma_send_ack(cm_node);
2607	break;
2608	default:
2609	ibdev_dbg(&cm_node->iwdev->ibdev,
2610	"CM: wrong cm_node state =%d\n", cm_node->state);
2611	break;
2612	}
2613	irdma_create_event(cm_node, type);
2614	}
2615
2616	/**
2617	* irdma_check_syn - Check for error on received syn ack
2618	* @cm_node: connection's node
2619	* @tcph: pointer tcp header
2620	*/
2621	static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
2622	{
2623	if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) {
2624	irdma_active_open_err(cm_node, reset: true);
2625	return 1;
2626	}
2627
2628	return 0;
2629	}
2630
2631	/**
2632	* irdma_check_seq - check seq numbers if OK
2633	* @cm_node: connection's node
2634	* @tcph: pointer tcp header
2635	*/
2636	static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
2637	{
2638	u32 seq;
2639	u32 ack_seq;
2640	u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
2641	u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
2642	u32 rcv_wnd;
2643	int err = 0;
2644
2645	seq = ntohl(tcph->seq);
2646	ack_seq = ntohl(tcph->ack_seq);
2647	rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
2648	if (ack_seq != loc_seq_num ||
2649	!between(seq1: seq, seq2: rcv_nxt, seq3: (rcv_nxt + rcv_wnd)))
2650	err = -1;
2651	if (err)
2652	ibdev_dbg(&cm_node->iwdev->ibdev,
2653	"CM: seq number err\n");
2654
2655	return err;
2656	}
2657
2658	void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node)
2659	{
2660	struct irdma_cm_info nfo;
2661
2662	irdma_get_addr_info(cm_node, cm_info: &nfo);
2663	nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2664	irdma_manage_qhash(iwdev: cm_node->iwdev, cminfo: &nfo,
2665	etype: IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2666	mtype: IRDMA_QHASH_MANAGE_TYPE_ADD,
2667	cmnode: cm_node, wait: false);
2668	cm_node->qhash_set = true;
2669	}
2670
2671	/**
2672	* irdma_handle_syn_pkt - is for Passive node
2673	* @cm_node: connection's node
2674	* @rbuf: receive buffer
2675	*/
2676	static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,
2677	struct irdma_puda_buf *rbuf)
2678	{
2679	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2680	int err;
2681	u32 inc_sequence;
2682	int optionsize;
2683
2684	optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2685	inc_sequence = ntohl(tcph->seq);
2686
2687	switch (cm_node->state) {
2688	case IRDMA_CM_STATE_SYN_SENT:
2689	case IRDMA_CM_STATE_MPAREQ_SENT:
2690	/* Rcvd syn on active open connection */
2691	irdma_active_open_err(cm_node, reset: 1);
2692	break;
2693	case IRDMA_CM_STATE_LISTENING:
2694	/* Passive OPEN */
2695	if (atomic_read(v: &cm_node->listener->pend_accepts_cnt) >
2696	cm_node->listener->backlog) {
2697	cm_node->cm_core->stats_backlog_drops++;
2698	irdma_passive_open_err(cm_node, reset: false);
2699	break;
2700	}
2701	err = irdma_handle_tcp_options(cm_node, tcph, optionsize, passive: 1);
2702	if (err) {
2703	irdma_passive_open_err(cm_node, reset: false);
2704	/* drop pkt */
2705	break;
2706	}
2707	err = cm_node->cm_core->cm_create_ah(cm_node, false);
2708	if (err) {
2709	irdma_passive_open_err(cm_node, reset: false);
2710	/* drop pkt */
2711	break;
2712	}
2713	cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
2714	cm_node->accept_pend = 1;
2715	atomic_inc(v: &cm_node->listener->pend_accepts_cnt);
2716
2717	cm_node->state = IRDMA_CM_STATE_SYN_RCVD;
2718	break;
2719	case IRDMA_CM_STATE_CLOSED:
2720	irdma_cleanup_retrans_entry(cm_node);
2721	refcount_inc(r: &cm_node->refcnt);
2722	irdma_send_reset(cm_node);
2723	break;
2724	case IRDMA_CM_STATE_OFFLOADED:
2725	case IRDMA_CM_STATE_ESTABLISHED:
2726	case IRDMA_CM_STATE_FIN_WAIT1:
2727	case IRDMA_CM_STATE_FIN_WAIT2:
2728	case IRDMA_CM_STATE_MPAREQ_RCVD:
2729	case IRDMA_CM_STATE_LAST_ACK:
2730	case IRDMA_CM_STATE_CLOSING:
2731	case IRDMA_CM_STATE_UNKNOWN:
2732	default:
2733	break;
2734	}
2735	}
2736
2737	/**
2738	* irdma_handle_synack_pkt - Process SYN+ACK packet (active side)
2739	* @cm_node: connection's node
2740	* @rbuf: receive buffer
2741	*/
2742	static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,
2743	struct irdma_puda_buf *rbuf)
2744	{
2745	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2746	int err;
2747	u32 inc_sequence;
2748	int optionsize;
2749
2750	optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2751	inc_sequence = ntohl(tcph->seq);
2752	switch (cm_node->state) {
2753	case IRDMA_CM_STATE_SYN_SENT:
2754	irdma_cleanup_retrans_entry(cm_node);
2755	/* active open */
2756	if (irdma_check_syn(cm_node, tcph)) {
2757	ibdev_dbg(&cm_node->iwdev->ibdev,
2758	"CM: check syn fail\n");
2759	return;
2760	}
2761	cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2762	/* setup options */
2763	err = irdma_handle_tcp_options(cm_node, tcph, optionsize, passive: 0);
2764	if (err) {
2765	ibdev_dbg(&cm_node->iwdev->ibdev,
2766	"CM: cm_node=%p tcp_options failed\n",
2767	cm_node);
2768	break;
2769	}
2770	irdma_cleanup_retrans_entry(cm_node);
2771	cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
2772	irdma_send_ack(cm_node); /* ACK for the syn_ack */
2773	err = irdma_send_mpa_request(cm_node);
2774	if (err) {
2775	ibdev_dbg(&cm_node->iwdev->ibdev,
2776	"CM: cm_node=%p irdma_send_mpa_request failed\n",
2777	cm_node);
2778	break;
2779	}
2780	cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT;
2781	break;
2782	case IRDMA_CM_STATE_MPAREQ_RCVD:
2783	irdma_passive_open_err(cm_node, reset: true);
2784	break;
2785	case IRDMA_CM_STATE_LISTENING:
2786	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
2787	irdma_cleanup_retrans_entry(cm_node);
2788	cm_node->state = IRDMA_CM_STATE_CLOSED;
2789	irdma_send_reset(cm_node);
2790	break;
2791	case IRDMA_CM_STATE_CLOSED:
2792	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
2793	irdma_cleanup_retrans_entry(cm_node);
2794	refcount_inc(r: &cm_node->refcnt);
2795	irdma_send_reset(cm_node);
2796	break;
2797	case IRDMA_CM_STATE_ESTABLISHED:
2798	case IRDMA_CM_STATE_FIN_WAIT1:
2799	case IRDMA_CM_STATE_FIN_WAIT2:
2800	case IRDMA_CM_STATE_LAST_ACK:
2801	case IRDMA_CM_STATE_OFFLOADED:
2802	case IRDMA_CM_STATE_CLOSING:
2803	case IRDMA_CM_STATE_UNKNOWN:
2804	case IRDMA_CM_STATE_MPAREQ_SENT:
2805	default:
2806	break;
2807	}
2808	}
2809
2810	/**
2811	* irdma_handle_ack_pkt - process packet with ACK
2812	* @cm_node: connection's node
2813	* @rbuf: receive buffer
2814	*/
2815	static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,
2816	struct irdma_puda_buf *rbuf)
2817	{
2818	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2819	u32 inc_sequence;
2820	int ret;
2821	int optionsize;
2822	u32 datasize = rbuf->datalen;
2823
2824	optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2825
2826	if (irdma_check_seq(cm_node, tcph))
2827	return -EINVAL;
2828
2829	inc_sequence = ntohl(tcph->seq);
2830	switch (cm_node->state) {
2831	case IRDMA_CM_STATE_SYN_RCVD:
2832	irdma_cleanup_retrans_entry(cm_node);
2833	ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, passive: 1);
2834	if (ret)
2835	return ret;
2836	cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2837	cm_node->state = IRDMA_CM_STATE_ESTABLISHED;
2838	if (datasize) {
2839	cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2840	irdma_handle_rcv_mpa(cm_node, rbuf);
2841	}
2842	break;
2843	case IRDMA_CM_STATE_ESTABLISHED:
2844	irdma_cleanup_retrans_entry(cm_node);
2845	if (datasize) {
2846	cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2847	irdma_handle_rcv_mpa(cm_node, rbuf);
2848	}
2849	break;
2850	case IRDMA_CM_STATE_MPAREQ_SENT:
2851	cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2852	if (datasize) {
2853	cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2854	cm_node->ack_rcvd = false;
2855	irdma_handle_rcv_mpa(cm_node, rbuf);
2856	} else {
2857	cm_node->ack_rcvd = true;
2858	}
2859	break;
2860	case IRDMA_CM_STATE_LISTENING:
2861	irdma_cleanup_retrans_entry(cm_node);
2862	cm_node->state = IRDMA_CM_STATE_CLOSED;
2863	irdma_send_reset(cm_node);
2864	break;
2865	case IRDMA_CM_STATE_CLOSED:
2866	irdma_cleanup_retrans_entry(cm_node);
2867	refcount_inc(r: &cm_node->refcnt);
2868	irdma_send_reset(cm_node);
2869	break;
2870	case IRDMA_CM_STATE_LAST_ACK:
2871	case IRDMA_CM_STATE_CLOSING:
2872	irdma_cleanup_retrans_entry(cm_node);
2873	cm_node->state = IRDMA_CM_STATE_CLOSED;
2874	irdma_rem_ref_cm_node(cm_node);
2875	break;
2876	case IRDMA_CM_STATE_FIN_WAIT1:
2877	irdma_cleanup_retrans_entry(cm_node);
2878	cm_node->state = IRDMA_CM_STATE_FIN_WAIT2;
2879	break;
2880	case IRDMA_CM_STATE_SYN_SENT:
2881	case IRDMA_CM_STATE_FIN_WAIT2:
2882	case IRDMA_CM_STATE_OFFLOADED:
2883	case IRDMA_CM_STATE_MPAREQ_RCVD:
2884	case IRDMA_CM_STATE_UNKNOWN:
2885	default:
2886	irdma_cleanup_retrans_entry(cm_node);
2887	break;
2888	}
2889
2890	return 0;
2891	}
2892
2893	/**
2894	* irdma_process_pkt - process cm packet
2895	* @cm_node: connection's node
2896	* @rbuf: receive buffer
2897	*/
2898	static void irdma_process_pkt(struct irdma_cm_node *cm_node,
2899	struct irdma_puda_buf *rbuf)
2900	{
2901	enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN;
2902	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2903	u32 fin_set = 0;
2904	int err;
2905
2906	if (tcph->rst) {
2907	pkt_type = IRDMA_PKT_TYPE_RST;
2908	} else if (tcph->syn) {
2909	pkt_type = IRDMA_PKT_TYPE_SYN;
2910	if (tcph->ack)
2911	pkt_type = IRDMA_PKT_TYPE_SYNACK;
2912	} else if (tcph->ack) {
2913	pkt_type = IRDMA_PKT_TYPE_ACK;
2914	}
2915	if (tcph->fin)
2916	fin_set = 1;
2917
2918	switch (pkt_type) {
2919	case IRDMA_PKT_TYPE_SYN:
2920	irdma_handle_syn_pkt(cm_node, rbuf);
2921	break;
2922	case IRDMA_PKT_TYPE_SYNACK:
2923	irdma_handle_synack_pkt(cm_node, rbuf);
2924	break;
2925	case IRDMA_PKT_TYPE_ACK:
2926	err = irdma_handle_ack_pkt(cm_node, rbuf);
2927	if (fin_set && !err)
2928	irdma_handle_fin_pkt(cm_node);
2929	break;
2930	case IRDMA_PKT_TYPE_RST:
2931	irdma_handle_rst_pkt(cm_node, rbuf);
2932	break;
2933	default:
2934	if (fin_set &&
2935	(!irdma_check_seq(cm_node, tcph: (struct tcphdr *)rbuf->tcph)))
2936	irdma_handle_fin_pkt(cm_node);
2937	break;
2938	}
2939	}
2940
2941	/**
2942	* irdma_make_listen_node - create a listen node with params
2943	* @cm_core: cm's core
2944	* @iwdev: iwarp device structure
2945	* @cm_info: quad info for connection
2946	*/
2947	static struct irdma_cm_listener *
2948	irdma_make_listen_node(struct irdma_cm_core *cm_core,
2949	struct irdma_device *iwdev,
2950	struct irdma_cm_info *cm_info)
2951	{
2952	struct irdma_cm_listener *listener;
2953	unsigned long flags;
2954
2955	/* cannot have multiple matching listeners */
2956	listener =
2957	irdma_find_listener(cm_core, dst_addr: cm_info->loc_addr, ipv4: cm_info->ipv4,
2958	dst_port: cm_info->loc_port, vlan_id: cm_info->vlan_id,
2959	listener_state: IRDMA_CM_LISTENER_EITHER_STATE);
2960	if (listener &&
2961	listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) {
2962	refcount_dec(r: &listener->refcnt);
2963	return NULL;
2964	}
2965
2966	if (!listener) {
2967	/* create a CM listen node
2968	* 1/2 node to compare incoming traffic to
2969	*/
2970	listener = kzalloc(sizeof(*listener), GFP_KERNEL);
2971	if (!listener)
2972	return NULL;
2973	cm_core->stats_listen_nodes_created++;
2974	memcpy(listener->loc_addr, cm_info->loc_addr,
2975	sizeof(listener->loc_addr));
2976	listener->loc_port = cm_info->loc_port;
2977
2978	INIT_LIST_HEAD(list: &listener->child_listen_list);
2979
2980	refcount_set(r: &listener->refcnt, n: 1);
2981	} else {
2982	listener->reused_node = 1;
2983	}
2984
2985	listener->cm_id = cm_info->cm_id;
2986	listener->ipv4 = cm_info->ipv4;
2987	listener->vlan_id = cm_info->vlan_id;
2988	atomic_set(v: &listener->pend_accepts_cnt, i: 0);
2989	listener->cm_core = cm_core;
2990	listener->iwdev = iwdev;
2991
2992	listener->backlog = cm_info->backlog;
2993	listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE;
2994
2995	if (!listener->reused_node) {
2996	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
2997	list_add(new: &listener->list, head: &cm_core->listen_list);
2998	spin_unlock_irqrestore(lock: &cm_core->listen_list_lock, flags);
2999	}
3000
3001	return listener;
3002	}
3003
3004	/**
3005	* irdma_create_cm_node - make a connection node with params
3006	* @cm_core: cm's core
3007	* @iwdev: iwarp device structure
3008	* @conn_param: connection parameters
3009	* @cm_info: quad info for connection
3010	* @caller_cm_node: pointer to cm_node structure to return
3011	*/
3012	static int irdma_create_cm_node(struct irdma_cm_core *cm_core,
3013	struct irdma_device *iwdev,
3014	struct iw_cm_conn_param *conn_param,
3015	struct irdma_cm_info *cm_info,
3016	struct irdma_cm_node **caller_cm_node)
3017	{
3018	struct irdma_cm_node *cm_node;
3019	u16 private_data_len = conn_param->private_data_len;
3020	const void *private_data = conn_param->private_data;
3021
3022	/* create a CM connection node */
3023	cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL);
3024	if (!cm_node)
3025	return -ENOMEM;
3026
3027	/* set our node side to client (active) side */
3028	cm_node->tcp_cntxt.client = 1;
3029	cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
3030
3031	irdma_record_ird_ord(cm_node, conn_ird: conn_param->ird, conn_ord: conn_param->ord);
3032
3033	cm_node->pdata.size = private_data_len;
3034	cm_node->pdata.addr = cm_node->pdata_buf;
3035
3036	memcpy(cm_node->pdata_buf, private_data, private_data_len);
3037	*caller_cm_node = cm_node;
3038
3039	return 0;
3040	}
3041
3042	/**
3043	* irdma_cm_reject - reject and teardown a connection
3044	* @cm_node: connection's node
3045	* @pdata: ptr to private data for reject
3046	* @plen: size of private data
3047	*/
3048	static int irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata,
3049	u8 plen)
3050	{
3051	int ret;
3052	int passive_state;
3053
3054	if (cm_node->tcp_cntxt.client)
3055	return 0;
3056
3057	irdma_cleanup_retrans_entry(cm_node);
3058
3059	passive_state = atomic_add_return(i: 1, v: &cm_node->passive_state);
3060	if (passive_state == IRDMA_SEND_RESET_EVENT) {
3061	cm_node->state = IRDMA_CM_STATE_CLOSED;
3062	irdma_rem_ref_cm_node(cm_node);
3063	return 0;
3064	}
3065
3066	if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
3067	irdma_rem_ref_cm_node(cm_node);
3068	return 0;
3069	}
3070
3071	ret = irdma_send_mpa_reject(cm_node, pdata, plen);
3072	if (!ret)
3073	return 0;
3074
3075	cm_node->state = IRDMA_CM_STATE_CLOSED;
3076	if (irdma_send_reset(cm_node))
3077	ibdev_dbg(&cm_node->iwdev->ibdev,
3078	"CM: send reset failed\n");
3079
3080	return ret;
3081	}
3082
3083	/**
3084	* irdma_cm_close - close of cm connection
3085	* @cm_node: connection's node
3086	*/
3087	static int irdma_cm_close(struct irdma_cm_node *cm_node)
3088	{
3089	switch (cm_node->state) {
3090	case IRDMA_CM_STATE_SYN_RCVD:
3091	case IRDMA_CM_STATE_SYN_SENT:
3092	case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:
3093	case IRDMA_CM_STATE_ESTABLISHED:
3094	case IRDMA_CM_STATE_ACCEPTING:
3095	case IRDMA_CM_STATE_MPAREQ_SENT:
3096	case IRDMA_CM_STATE_MPAREQ_RCVD:
3097	irdma_cleanup_retrans_entry(cm_node);
3098	irdma_send_reset(cm_node);
3099	break;
3100	case IRDMA_CM_STATE_CLOSE_WAIT:
3101	cm_node->state = IRDMA_CM_STATE_LAST_ACK;
3102	irdma_send_fin(cm_node);
3103	break;
3104	case IRDMA_CM_STATE_FIN_WAIT1:
3105	case IRDMA_CM_STATE_FIN_WAIT2:
3106	case IRDMA_CM_STATE_LAST_ACK:
3107	case IRDMA_CM_STATE_TIME_WAIT:
3108	case IRDMA_CM_STATE_CLOSING:
3109	return -EINVAL;
3110	case IRDMA_CM_STATE_LISTENING:
3111	irdma_cleanup_retrans_entry(cm_node);
3112	irdma_send_reset(cm_node);
3113	break;
3114	case IRDMA_CM_STATE_MPAREJ_RCVD:
3115	case IRDMA_CM_STATE_UNKNOWN:
3116	case IRDMA_CM_STATE_INITED:
3117	case IRDMA_CM_STATE_CLOSED:
3118	case IRDMA_CM_STATE_LISTENER_DESTROYED:
3119	irdma_rem_ref_cm_node(cm_node);
3120	break;
3121	case IRDMA_CM_STATE_OFFLOADED:
3122	if (cm_node->send_entry)
3123	ibdev_dbg(&cm_node->iwdev->ibdev,
3124	"CM: CM send_entry in OFFLOADED state\n");
3125	irdma_rem_ref_cm_node(cm_node);
3126	break;
3127	}
3128
3129	return 0;
3130	}
3131
3132	/**
3133	* irdma_receive_ilq - recv an ETHERNET packet, and process it
3134	* through CM
3135	* @vsi: VSI structure of dev
3136	* @rbuf: receive buffer
3137	*/
3138	void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf)
3139	{
3140	struct irdma_cm_node *cm_node;
3141	struct irdma_cm_listener *listener;
3142	struct iphdr *iph;
3143	struct ipv6hdr *ip6h;
3144	struct tcphdr *tcph;
3145	struct irdma_cm_info cm_info = {};
3146	struct irdma_device *iwdev = vsi->back_vsi;
3147	struct irdma_cm_core *cm_core = &iwdev->cm_core;
3148	struct vlan_ethhdr *ethh;
3149	u16 vtag;
3150
3151	/* if vlan, then maclen = 18 else 14 */
3152	iph = (struct iphdr *)rbuf->iph;
3153	print_hex_dump_debug("ILQ: RECEIVE ILQ BUFFER", DUMP_PREFIX_OFFSET,
3154	16, 8, rbuf->mem.va, rbuf->totallen, false);
3155	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
3156	if (rbuf->vlan_valid) {
3157	vtag = rbuf->vlan_id;
3158	cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>
3159	VLAN_PRIO_SHIFT;
3160	cm_info.vlan_id = vtag & VLAN_VID_MASK;
3161	} else {
3162	cm_info.vlan_id = 0xFFFF;
3163	}
3164	} else {
3165	ethh = rbuf->mem.va;
3166
3167	if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {
3168	vtag = ntohs(ethh->h_vlan_TCI);
3169	cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>
3170	VLAN_PRIO_SHIFT;
3171	cm_info.vlan_id = vtag & VLAN_VID_MASK;
3172	ibdev_dbg(&cm_core->iwdev->ibdev,
3173	"CM: vlan_id=%d\n", cm_info.vlan_id);
3174	} else {
3175	cm_info.vlan_id = 0xFFFF;
3176	}
3177	}
3178	tcph = (struct tcphdr *)rbuf->tcph;
3179
3180	if (rbuf->ipv4) {
3181	cm_info.loc_addr[0] = ntohl(iph->daddr);
3182	cm_info.rem_addr[0] = ntohl(iph->saddr);
3183	cm_info.ipv4 = true;
3184	cm_info.tos = iph->tos;
3185	} else {
3186	ip6h = (struct ipv6hdr *)rbuf->iph;
3187	irdma_copy_ip_ntohl(dst: cm_info.loc_addr,
3188	src: ip6h->daddr.in6_u.u6_addr32);
3189	irdma_copy_ip_ntohl(dst: cm_info.rem_addr,
3190	src: ip6h->saddr.in6_u.u6_addr32);
3191	cm_info.ipv4 = false;
3192	cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);
3193	}
3194	cm_info.loc_port = ntohs(tcph->dest);
3195	cm_info.rem_port = ntohs(tcph->source);
3196	cm_node = irdma_find_node(cm_core, rem_port: cm_info.rem_port, rem_addr: cm_info.rem_addr,
3197	loc_port: cm_info.loc_port, loc_addr: cm_info.loc_addr, vlan_id: cm_info.vlan_id);
3198
3199	if (!cm_node) {
3200	/* Only type of packet accepted are for the
3201	* PASSIVE open (syn only)
3202	*/
3203	if (!tcph->syn || tcph->ack)
3204	return;
3205
3206	listener = irdma_find_listener(cm_core,
3207	dst_addr: cm_info.loc_addr,
3208	ipv4: cm_info.ipv4,
3209	dst_port: cm_info.loc_port,
3210	vlan_id: cm_info.vlan_id,
3211	listener_state: IRDMA_CM_LISTENER_ACTIVE_STATE);
3212	if (!listener) {
3213	cm_info.cm_id = NULL;
3214	ibdev_dbg(&cm_core->iwdev->ibdev,
3215	"CM: no listener found\n");
3216	return;
3217	}
3218
3219	cm_info.cm_id = listener->cm_id;
3220	cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info: &cm_info,
3221	listener);
3222	if (!cm_node) {
3223	ibdev_dbg(&cm_core->iwdev->ibdev,
3224	"CM: allocate node failed\n");
3225	refcount_dec(r: &listener->refcnt);
3226	return;
3227	}
3228
3229	if (!tcph->rst && !tcph->fin) {
3230	cm_node->state = IRDMA_CM_STATE_LISTENING;
3231	} else {
3232	irdma_rem_ref_cm_node(cm_node);
3233	return;
3234	}
3235
3236	refcount_inc(r: &cm_node->refcnt);
3237	} else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) {
3238	irdma_rem_ref_cm_node(cm_node);
3239	return;
3240	}
3241
3242	irdma_process_pkt(cm_node, rbuf);
3243	irdma_rem_ref_cm_node(cm_node);
3244	}
3245
3246	static int irdma_add_qh(struct irdma_cm_node *cm_node, bool active)
3247	{
3248	if (!active)
3249	irdma_add_conn_est_qh(cm_node);
3250	return 0;
3251	}
3252
3253	static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)
3254	{
3255	}
3256
3257	/**
3258	* irdma_setup_cm_core - setup top level instance of a cm core
3259	* @iwdev: iwarp device structure
3260	* @rdma_ver: HW version
3261	*/
3262	int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)
3263	{
3264	struct irdma_cm_core *cm_core = &iwdev->cm_core;
3265
3266	cm_core->iwdev = iwdev;
3267	cm_core->dev = &iwdev->rf->sc_dev;
3268
3269	/* Handles CM event work items send to Iwarp core */
3270	cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0);
3271	if (!cm_core->event_wq)
3272	return -ENOMEM;
3273
3274	INIT_LIST_HEAD(list: &cm_core->listen_list);
3275
3276	timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0);
3277
3278	spin_lock_init(&cm_core->ht_lock);
3279	spin_lock_init(&cm_core->listen_list_lock);
3280	spin_lock_init(&cm_core->apbvt_lock);
3281	switch (rdma_ver) {
3282	case IRDMA_GEN_1:
3283	cm_core->form_cm_frame = irdma_form_uda_cm_frame;
3284	cm_core->cm_create_ah = irdma_add_qh;
3285	cm_core->cm_free_ah = irdma_cm_free_ah_nop;
3286	break;
3287	case IRDMA_GEN_2:
3288	default:
3289	cm_core->form_cm_frame = irdma_form_ah_cm_frame;
3290	cm_core->cm_create_ah = irdma_cm_create_ah;
3291	cm_core->cm_free_ah = irdma_cm_free_ah;
3292	}
3293
3294	return 0;
3295	}
3296
3297	/**
3298	* irdma_cleanup_cm_core - deallocate a top level instance of a
3299	* cm core
3300	* @cm_core: cm's core
3301	*/
3302	void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)
3303	{
3304	if (!cm_core)
3305	return;
3306
3307	timer_delete_sync(timer: &cm_core->tcp_timer);
3308
3309	destroy_workqueue(wq: cm_core->event_wq);
3310	cm_core->dev->ws_reset(&cm_core->iwdev->vsi);
3311	}
3312
3313	/**
3314	* irdma_init_tcp_ctx - setup qp context
3315	* @cm_node: connection's node
3316	* @tcp_info: offload info for tcp
3317	* @iwqp: associate qp for the connection
3318	*/
3319	static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node,
3320	struct irdma_tcp_offload_info *tcp_info,
3321	struct irdma_qp *iwqp)
3322	{
3323	tcp_info->ipv4 = cm_node->ipv4;
3324	tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo;
3325	tcp_info->wscale = true;
3326	tcp_info->ignore_tcp_opt = true;
3327	tcp_info->ignore_tcp_uns_opt = true;
3328	tcp_info->no_nagle = false;
3329
3330	tcp_info->ttl = IRDMA_DEFAULT_TTL;
3331	tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR;
3332	tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH;
3333	tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH;
3334
3335	tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
3336	tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;
3337	tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;
3338
3339	tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num;
3340	tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd;
3341	tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
3342	tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num;
3343
3344	tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num;
3345	tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss;
3346	tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt;
3347	tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num;
3348	tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd;
3349	tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd
3350	<< cm_node->tcp_cntxt.rcv_wscale;
3351
3352	tcp_info->flow_label = 0;
3353	tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss;
3354	tcp_info->tos = cm_node->tos;
3355	if (cm_node->vlan_id < VLAN_N_VID) {
3356	tcp_info->insert_vlan_tag = true;
3357	tcp_info->vlan_tag = cm_node->vlan_id;
3358	tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;
3359	}
3360	if (cm_node->ipv4) {
3361	tcp_info->src_port = cm_node->loc_port;
3362	tcp_info->dst_port = cm_node->rem_port;
3363
3364	tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0];
3365	tcp_info->local_ipaddr[3] = cm_node->loc_addr[0];
3366	tcp_info->arp_idx = (u16)irdma_arp_table(rf: iwqp->iwdev->rf,
3367	ip_addr: &tcp_info->dest_ip_addr[3],
3368	ipv4: true, NULL,
3369	IRDMA_ARP_RESOLVE);
3370	} else {
3371	tcp_info->src_port = cm_node->loc_port;
3372	tcp_info->dst_port = cm_node->rem_port;
3373	memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr,
3374	sizeof(tcp_info->dest_ip_addr));
3375	memcpy(tcp_info->local_ipaddr, cm_node->loc_addr,
3376	sizeof(tcp_info->local_ipaddr));
3377
3378	tcp_info->arp_idx = (u16)irdma_arp_table(rf: iwqp->iwdev->rf,
3379	ip_addr: &tcp_info->dest_ip_addr[0],
3380	ipv4: false, NULL,
3381	IRDMA_ARP_RESOLVE);
3382	}
3383	}
3384
3385	/**
3386	* irdma_cm_init_tsa_conn - setup qp for RTS
3387	* @iwqp: associate qp for the connection
3388	* @cm_node: connection's node
3389	*/
3390	static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp,
3391	struct irdma_cm_node *cm_node)
3392	{
3393	struct irdma_iwarp_offload_info *iwarp_info;
3394	struct irdma_qp_host_ctx_info *ctx_info;
3395
3396	iwarp_info = &iwqp->iwarp_info;
3397	ctx_info = &iwqp->ctx_info;
3398
3399	ctx_info->tcp_info = &iwqp->tcp_info;
3400	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
3401	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
3402
3403	iwarp_info->ord_size = cm_node->ord_size;
3404	iwarp_info->ird_size = cm_node->ird_size;
3405	iwarp_info->rd_en = true;
3406	iwarp_info->rdmap_ver = 1;
3407	iwarp_info->ddp_ver = 1;
3408	iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;
3409
3410	ctx_info->tcp_info_valid = true;
3411	ctx_info->iwarp_info_valid = true;
3412	ctx_info->user_pri = cm_node->user_pri;
3413
3414	irdma_init_tcp_ctx(cm_node, tcp_info: &iwqp->tcp_info, iwqp);
3415	if (cm_node->snd_mark_en) {
3416	iwarp_info->snd_mark_en = true;
3417	iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) +
3418	cm_node->lsmm_size;
3419	}
3420
3421	cm_node->state = IRDMA_CM_STATE_OFFLOADED;
3422	iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
3423	iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx;
3424
3425	if (cm_node->rcv_mark_en) {
3426	iwarp_info->rcv_mark_en = true;
3427	iwarp_info->align_hdrs = true;
3428	}
3429
3430	irdma_sc_qp_setctx(qp: &iwqp->sc_qp, qp_ctx: iwqp->host_ctx.va, info: ctx_info);
3431
3432	/* once tcp_info is set, no need to do it again */
3433	ctx_info->tcp_info_valid = false;
3434	ctx_info->iwarp_info_valid = false;
3435	}
3436
3437	/**
3438	* irdma_cm_disconn - when a connection is being closed
3439	* @iwqp: associated qp for the connection
3440	*/
3441	void irdma_cm_disconn(struct irdma_qp *iwqp)
3442	{
3443	struct irdma_device *iwdev = iwqp->iwdev;
3444	struct disconn_work *work;
3445	unsigned long flags;
3446
3447	work = kzalloc(sizeof(*work), GFP_ATOMIC);
3448	if (!work)
3449	return;
3450
3451	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
3452	if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) {
3453	spin_unlock_irqrestore(lock: &iwdev->rf->qptable_lock, flags);
3454	ibdev_dbg(&iwdev->ibdev,
3455	"CM: qp_id %d is already freed\n",
3456	iwqp->ibqp.qp_num);
3457	kfree(objp: work);
3458	return;
3459	}
3460	irdma_qp_add_ref(ibqp: &iwqp->ibqp);
3461	spin_unlock_irqrestore(lock: &iwdev->rf->qptable_lock, flags);
3462
3463	work->iwqp = iwqp;
3464	INIT_WORK(&work->work, irdma_disconnect_worker);
3465	queue_work(wq: iwdev->cleanup_wq, work: &work->work);
3466	}
3467
3468	/**
3469	* irdma_qp_disconnect - free qp and close cm
3470	* @iwqp: associate qp for the connection
3471	*/
3472	static void irdma_qp_disconnect(struct irdma_qp *iwqp)
3473	{
3474	struct irdma_device *iwdev = iwqp->iwdev;
3475
3476	iwqp->active_conn = 0;
3477	/* close the CM node down if it is still active */
3478	ibdev_dbg(&iwdev->ibdev, "CM: Call close API\n");
3479	irdma_cm_close(cm_node: iwqp->cm_node);
3480	}
3481
3482	/**
3483	* irdma_cm_disconn_true - called by worker thread to disconnect qp
3484	* @iwqp: associate qp for the connection
3485	*/
3486	static void irdma_cm_disconn_true(struct irdma_qp *iwqp)
3487	{
3488	struct iw_cm_id *cm_id;
3489	struct irdma_device *iwdev;
3490	struct irdma_sc_qp *qp = &iwqp->sc_qp;
3491	u16 last_ae;
3492	u8 original_hw_tcp_state;
3493	u8 original_ibqp_state;
3494	int disconn_status = 0;
3495	int issue_disconn = 0;
3496	int issue_close = 0;
3497	int issue_flush = 0;
3498	unsigned long flags;
3499	int err;
3500
3501	iwdev = iwqp->iwdev;
3502	spin_lock_irqsave(&iwqp->lock, flags);
3503	if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1)) {
3504	struct ib_qp_attr attr;
3505
3506	if (iwqp->flush_issued || iwqp->sc_qp.qp_uk.destroy_pending) {
3507	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
3508	return;
3509	}
3510
3511	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
3512
3513	attr.qp_state = IB_QPS_ERR;
3514	irdma_modify_qp_roce(ibqp: &iwqp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL);
3515	irdma_ib_qp_event(iwqp, event: qp->event_type);
3516	return;
3517	}
3518
3519	cm_id = iwqp->cm_id;
3520	original_hw_tcp_state = iwqp->hw_tcp_state;
3521	original_ibqp_state = iwqp->ibqp_state;
3522	last_ae = iwqp->last_aeq;
3523
3524	if (qp->term_flags) {
3525	issue_disconn = 1;
3526	issue_close = 1;
3527	iwqp->cm_id = NULL;
3528	irdma_terminate_del_timer(qp);
3529	if (!iwqp->flush_issued) {
3530	iwqp->flush_issued = 1;
3531	issue_flush = 1;
3532	}
3533	} else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) ||
3534	((original_ibqp_state == IB_QPS_RTS) &&
3535	(last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) {
3536	issue_disconn = 1;
3537	if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET)
3538	disconn_status = -ECONNRESET;
3539	}
3540
3541	if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
3542	original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
3543	last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
3544	last_ae == IRDMA_AE_BAD_CLOSE ||
3545	last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) {
3546	issue_close = 1;
3547	iwqp->cm_id = NULL;
3548	qp->term_flags = 0;
3549	if (!iwqp->flush_issued) {
3550	iwqp->flush_issued = 1;
3551	issue_flush = 1;
3552	}
3553	}
3554
3555	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
3556	if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) {
3557	irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ |
3558	IRDMA_FLUSH_WAIT);
3559
3560	if (qp->term_flags)
3561	irdma_ib_qp_event(iwqp, event: qp->event_type);
3562	}
3563
3564	if (!cm_id || !cm_id->event_handler)
3565	return;
3566
3567	spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);
3568	if (!iwqp->cm_node) {
3569	spin_unlock_irqrestore(lock: &iwdev->cm_core.ht_lock, flags);
3570	return;
3571	}
3572	refcount_inc(r: &iwqp->cm_node->refcnt);
3573
3574	spin_unlock_irqrestore(lock: &iwdev->cm_core.ht_lock, flags);
3575
3576	if (issue_disconn) {
3577	err = irdma_send_cm_event(cm_node: iwqp->cm_node, cm_id,
3578	type: IW_CM_EVENT_DISCONNECT,
3579	status: disconn_status);
3580	if (err)
3581	ibdev_dbg(&iwdev->ibdev,
3582	"CM: disconnect event failed: - cm_id = %p\n",
3583	cm_id);
3584	}
3585	if (issue_close) {
3586	cm_id->provider_data = iwqp;
3587	err = irdma_send_cm_event(cm_node: iwqp->cm_node, cm_id,
3588	type: IW_CM_EVENT_CLOSE, status: 0);
3589	if (err)
3590	ibdev_dbg(&iwdev->ibdev,
3591	"CM: close event failed: - cm_id = %p\n",
3592	cm_id);
3593	irdma_qp_disconnect(iwqp);
3594	}
3595	irdma_rem_ref_cm_node(cm_node: iwqp->cm_node);
3596	}
3597
3598	/**
3599	* irdma_disconnect_worker - worker for connection close
3600	* @work: points or disconn structure
3601	*/
3602	static void irdma_disconnect_worker(struct work_struct *work)
3603	{
3604	struct disconn_work *dwork = container_of(work, struct disconn_work, work);
3605	struct irdma_qp *iwqp = dwork->iwqp;
3606
3607	kfree(objp: dwork);
3608	irdma_cm_disconn_true(iwqp);
3609	irdma_qp_rem_ref(ibqp: &iwqp->ibqp);
3610	}
3611
3612	/**
3613	* irdma_free_lsmm_rsrc - free lsmm memory and deregister
3614	* @iwqp: associate qp for the connection
3615	*/
3616	void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp)
3617	{
3618	struct irdma_device *iwdev;
3619
3620	iwdev = iwqp->iwdev;
3621
3622	if (iwqp->ietf_mem.va) {
3623	if (iwqp->lsmm_mr)
3624	iwdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr, NULL);
3625	dma_free_coherent(dev: iwdev->rf->sc_dev.hw->device,
3626	size: iwqp->ietf_mem.size, cpu_addr: iwqp->ietf_mem.va,
3627	dma_handle: iwqp->ietf_mem.pa);
3628	iwqp->ietf_mem.va = NULL;
3629	}
3630	}
3631
3632	/**
3633	* irdma_accept - registered call for connection to be accepted
3634	* @cm_id: cm information for passive connection
3635	* @conn_param: accept parameters
3636	*/
3637	int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3638	{
3639	struct ib_qp *ibqp;
3640	struct irdma_qp *iwqp;
3641	struct irdma_device *iwdev;
3642	struct irdma_sc_dev *dev;
3643	struct irdma_cm_node *cm_node;
3644	struct ib_qp_attr attr = {};
3645	int passive_state;
3646	struct ib_mr *ibmr;
3647	struct irdma_pd *iwpd;
3648	u16 buf_len = 0;
3649	struct irdma_kmem_info accept;
3650	u64 tagged_offset;
3651	int wait_ret;
3652	int ret = 0;
3653
3654	ibqp = irdma_get_qp(ibdev: cm_id->device, qpn: conn_param->qpn);
3655	if (!ibqp)
3656	return -EINVAL;
3657
3658	iwqp = to_iwqp(ibqp);
3659	iwdev = iwqp->iwdev;
3660	dev = &iwdev->rf->sc_dev;
3661	cm_node = cm_id->provider_data;
3662
3663	if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {
3664	cm_node->ipv4 = true;
3665	cm_node->vlan_id = irdma_get_vlan_ipv4(addr: cm_node->loc_addr);
3666	} else {
3667	cm_node->ipv4 = false;
3668	irdma_get_vlan_mac_ipv6(addr: cm_node->loc_addr, vlan_id: &cm_node->vlan_id,
3669	NULL);
3670	}
3671	ibdev_dbg(&iwdev->ibdev, "CM: Accept vlan_id=%d\n",
3672	cm_node->vlan_id);
3673
3674	trace_irdma_accept(cm_node, type: 0, NULL);
3675
3676	if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
3677	ret = -EINVAL;
3678	goto error;
3679	}
3680
3681	passive_state = atomic_add_return(i: 1, v: &cm_node->passive_state);
3682	if (passive_state == IRDMA_SEND_RESET_EVENT) {
3683	ret = -ECONNRESET;
3684	goto error;
3685	}
3686
3687	buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE;
3688	iwqp->ietf_mem.size = ALIGN(buf_len, 1);
3689	iwqp->ietf_mem.va = dma_alloc_coherent(dev: dev->hw->device,
3690	size: iwqp->ietf_mem.size,
3691	dma_handle: &iwqp->ietf_mem.pa, GFP_KERNEL);
3692	if (!iwqp->ietf_mem.va) {
3693	ret = -ENOMEM;
3694	goto error;
3695	}
3696
3697	cm_node->pdata.size = conn_param->private_data_len;
3698	accept.addr = iwqp->ietf_mem.va;
3699	accept.size = irdma_cm_build_mpa_frame(cm_node, mpa: &accept, mpa_key: MPA_KEY_REPLY);
3700	memcpy((u8 *)accept.addr + accept.size, conn_param->private_data,
3701	conn_param->private_data_len);
3702
3703	if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) {
3704	ret = -ENOMEM;
3705	goto error;
3706	}
3707	iwqp->sc_qp.user_pri = cm_node->user_pri;
3708	irdma_qp_add_qos(qp: &iwqp->sc_qp);
3709	/* setup our first outgoing iWarp send WQE (the IETF frame response) */
3710	iwpd = iwqp->iwpd;
3711	tagged_offset = (uintptr_t)iwqp->ietf_mem.va;
3712	ibmr = irdma_reg_phys_mr(ib_pd: &iwpd->ibpd, addr: iwqp->ietf_mem.pa, size: buf_len,
3713	acc: IB_ACCESS_LOCAL_WRITE, iova_start: &tagged_offset, dma_mr: false);
3714	if (IS_ERR(ptr: ibmr)) {
3715	ret = -ENOMEM;
3716	goto error;
3717	}
3718
3719	ibmr->pd = &iwpd->ibpd;
3720	ibmr->device = iwpd->ibpd.device;
3721	iwqp->lsmm_mr = ibmr;
3722	if (iwqp->page)
3723	iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(page: iwqp->page);
3724
3725	cm_node->lsmm_size = accept.size + conn_param->private_data_len;
3726	irdma_sc_send_lsmm(qp: &iwqp->sc_qp, lsmm_buf: iwqp->ietf_mem.va, size: cm_node->lsmm_size,
3727	stag: ibmr->lkey);
3728
3729	if (iwqp->page)
3730	kunmap_local(iwqp->sc_qp.qp_uk.sq_base);
3731
3732	iwqp->cm_id = cm_id;
3733	cm_node->cm_id = cm_id;
3734
3735	cm_id->provider_data = iwqp;
3736	iwqp->active_conn = 0;
3737	iwqp->cm_node = cm_node;
3738	cm_node->iwqp = iwqp;
3739	irdma_cm_init_tsa_conn(iwqp, cm_node);
3740	irdma_qp_add_ref(ibqp: &iwqp->ibqp);
3741	cm_id->add_ref(cm_id);
3742
3743	attr.qp_state = IB_QPS_RTS;
3744	cm_node->qhash_set = false;
3745	cm_node->cm_core->cm_free_ah(cm_node);
3746
3747	irdma_modify_qp(ibqp: &iwqp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL);
3748	if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
3749	wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
3750	iwqp->rts_ae_rcvd,
3751	IRDMA_MAX_TIMEOUT);
3752	if (!wait_ret) {
3753	ibdev_dbg(&iwdev->ibdev,
3754	"CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
3755	cm_node, cm_node->loc_port,
3756	cm_node->rem_port, cm_node->cm_id);
3757	ret = -ECONNRESET;
3758	goto error;
3759	}
3760	}
3761
3762	irdma_send_cm_event(cm_node, cm_id, type: IW_CM_EVENT_ESTABLISHED, status: 0);
3763	cm_node->accelerated = true;
3764	complete(&cm_node->establish_comp);
3765
3766	if (cm_node->accept_pend) {
3767	atomic_dec(v: &cm_node->listener->pend_accepts_cnt);
3768	cm_node->accept_pend = 0;
3769	}
3770
3771	ibdev_dbg(&iwdev->ibdev,
3772	"CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
3773	cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
3774	cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
3775	cm_node->cm_core->stats_accepts++;
3776
3777	return 0;
3778	error:
3779	irdma_free_lsmm_rsrc(iwqp);
3780	irdma_rem_ref_cm_node(cm_node);
3781
3782	return ret;
3783	}
3784
3785	/**
3786	* irdma_reject - registered call for connection to be rejected
3787	* @cm_id: cm information for passive connection
3788	* @pdata: private data to be sent
3789	* @pdata_len: private data length
3790	*/
3791	int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
3792	{
3793	struct irdma_device *iwdev;
3794	struct irdma_cm_node *cm_node;
3795
3796	cm_node = cm_id->provider_data;
3797	cm_node->pdata.size = pdata_len;
3798
3799	trace_irdma_reject(cm_node, type: 0, NULL);
3800
3801	iwdev = to_iwdev(ibdev: cm_id->device);
3802	if (!iwdev)
3803	return -EINVAL;
3804
3805	cm_node->cm_core->stats_rejects++;
3806
3807	if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF)
3808	return -EINVAL;
3809
3810	return irdma_cm_reject(cm_node, pdata, plen: pdata_len);
3811	}
3812
3813	/**
3814	* irdma_connect - registered call for connection to be established
3815	* @cm_id: cm information for passive connection
3816	* @conn_param: Information about the connection
3817	*/
3818	int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3819	{
3820	struct ib_qp *ibqp;
3821	struct irdma_qp *iwqp;
3822	struct irdma_device *iwdev;
3823	struct irdma_cm_node *cm_node;
3824	struct irdma_cm_info cm_info;
3825	struct sockaddr_in *laddr;
3826	struct sockaddr_in *raddr;
3827	struct sockaddr_in6 *laddr6;
3828	struct sockaddr_in6 *raddr6;
3829	int ret = 0;
3830
3831	ibqp = irdma_get_qp(ibdev: cm_id->device, qpn: conn_param->qpn);
3832	if (!ibqp)
3833	return -EINVAL;
3834	iwqp = to_iwqp(ibqp);
3835	if (!iwqp)
3836	return -EINVAL;
3837	iwdev = iwqp->iwdev;
3838	if (!iwdev)
3839	return -EINVAL;
3840
3841	laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
3842	raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
3843	laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
3844	raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;
3845
3846	if (!(laddr->sin_port) || !(raddr->sin_port))
3847	return -EINVAL;
3848
3849	iwqp->active_conn = 1;
3850	iwqp->cm_id = NULL;
3851	cm_id->provider_data = iwqp;
3852
3853	/* set up the connection params for the node */
3854	if (cm_id->remote_addr.ss_family == AF_INET) {
3855	if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
3856	return -EINVAL;
3857
3858	cm_info.ipv4 = true;
3859	memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));
3860	memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));
3861	cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
3862	cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);
3863	cm_info.loc_port = ntohs(laddr->sin_port);
3864	cm_info.rem_port = ntohs(raddr->sin_port);
3865	cm_info.vlan_id = irdma_get_vlan_ipv4(addr: cm_info.loc_addr);
3866	} else {
3867	if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
3868	return -EINVAL;
3869
3870	cm_info.ipv4 = false;
3871	irdma_copy_ip_ntohl(dst: cm_info.loc_addr,
3872	src: laddr6->sin6_addr.in6_u.u6_addr32);
3873	irdma_copy_ip_ntohl(dst: cm_info.rem_addr,
3874	src: raddr6->sin6_addr.in6_u.u6_addr32);
3875	cm_info.loc_port = ntohs(laddr6->sin6_port);
3876	cm_info.rem_port = ntohs(raddr6->sin6_port);
3877	irdma_get_vlan_mac_ipv6(addr: cm_info.loc_addr, vlan_id: &cm_info.vlan_id,
3878	NULL);
3879	}
3880	cm_info.cm_id = cm_id;
3881	cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
3882	cm_info.tos = cm_id->tos;
3883	if (iwdev->vsi.dscp_mode) {
3884	cm_info.user_pri =
3885	iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(tos: cm_info.tos)];
3886	} else {
3887	cm_info.user_pri = rt_tos2priority(tos: cm_id->tos);
3888	cm_info.user_pri = irdma_iw_get_vlan_prio(loc_addr: cm_info.loc_addr,
3889	prio: cm_info.user_pri,
3890	ipv4: cm_info.ipv4);
3891	}
3892
3893	if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))
3894	return -ENOMEM;
3895	iwqp->sc_qp.user_pri = cm_info.user_pri;
3896	irdma_qp_add_qos(qp: &iwqp->sc_qp);
3897	ibdev_dbg(&iwdev->ibdev, "DCB: TOS:[%d] UP:[%d]\n", cm_id->tos,
3898	cm_info.user_pri);
3899
3900	trace_irdma_dcb_tos(iwdev, tos: cm_id->tos, user_pri: cm_info.user_pri);
3901
3902	ret = irdma_create_cm_node(cm_core: &iwdev->cm_core, iwdev, conn_param, cm_info: &cm_info,
3903	caller_cm_node: &cm_node);
3904	if (ret)
3905	return ret;
3906	ret = cm_node->cm_core->cm_create_ah(cm_node, true);
3907	if (ret)
3908	goto err;
3909	if (irdma_manage_qhash(iwdev, cminfo: &cm_info,
3910	etype: IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
3911	mtype: IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, wait: true)) {
3912	ret = -EINVAL;
3913	goto err;
3914	}
3915	cm_node->qhash_set = true;
3916
3917	cm_node->apbvt_entry = irdma_add_apbvt(iwdev, port: cm_info.loc_port);
3918	if (!cm_node->apbvt_entry) {
3919	ret = -EINVAL;
3920	goto err;
3921	}
3922
3923	cm_node->apbvt_set = true;
3924	iwqp->cm_node = cm_node;
3925	cm_node->iwqp = iwqp;
3926	iwqp->cm_id = cm_id;
3927	irdma_qp_add_ref(ibqp: &iwqp->ibqp);
3928	cm_id->add_ref(cm_id);
3929
3930	if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
3931	cm_node->state = IRDMA_CM_STATE_SYN_SENT;
3932	ret = irdma_send_syn(cm_node, sendack: 0);
3933	if (ret)
3934	goto err;
3935	}
3936
3937	ibdev_dbg(&iwdev->ibdev,
3938	"CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
3939	cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
3940	cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
3941
3942	trace_irdma_connect(cm_node, type: 0, NULL);
3943
3944	return 0;
3945
3946	err:
3947	if (cm_info.ipv4)
3948	ibdev_dbg(&iwdev->ibdev,
3949	"CM: connect() FAILED: dest addr=%pI4",
3950	cm_info.rem_addr);
3951	else
3952	ibdev_dbg(&iwdev->ibdev,
3953	"CM: connect() FAILED: dest addr=%pI6",
3954	cm_info.rem_addr);
3955	irdma_rem_ref_cm_node(cm_node);
3956	iwdev->cm_core.stats_connect_errs++;
3957
3958	return ret;
3959	}
3960
3961	/**
3962	* irdma_create_listen - registered call creating listener
3963	* @cm_id: cm information for passive connection
3964	* @backlog: to max accept pending count
3965	*/
3966	int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
3967	{
3968	struct irdma_device *iwdev;
3969	struct irdma_cm_listener *cm_listen_node;
3970	struct irdma_cm_info cm_info = {};
3971	struct sockaddr_in *laddr;
3972	struct sockaddr_in6 *laddr6;
3973	bool wildcard = false;
3974	int err;
3975
3976	iwdev = to_iwdev(ibdev: cm_id->device);
3977	if (!iwdev)
3978	return -EINVAL;
3979
3980	laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
3981	laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
3982	cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
3983
3984	if (laddr->sin_family == AF_INET) {
3985	if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
3986	return -EINVAL;
3987
3988	cm_info.ipv4 = true;
3989	cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
3990	cm_info.loc_port = ntohs(laddr->sin_port);
3991
3992	if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) {
3993	cm_info.vlan_id = irdma_get_vlan_ipv4(addr: cm_info.loc_addr);
3994	} else {
3995	cm_info.vlan_id = 0xFFFF;
3996	wildcard = true;
3997	}
3998	} else {
3999	if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
4000	return -EINVAL;
4001
4002	cm_info.ipv4 = false;
4003	irdma_copy_ip_ntohl(dst: cm_info.loc_addr,
4004	src: laddr6->sin6_addr.in6_u.u6_addr32);
4005	cm_info.loc_port = ntohs(laddr6->sin6_port);
4006	if (ipv6_addr_type(addr: &laddr6->sin6_addr) != IPV6_ADDR_ANY) {
4007	irdma_get_vlan_mac_ipv6(addr: cm_info.loc_addr,
4008	vlan_id: &cm_info.vlan_id, NULL);
4009	} else {
4010	cm_info.vlan_id = 0xFFFF;
4011	wildcard = true;
4012	}
4013	}
4014
4015	if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
4016	cm_info.vlan_id = 0;
4017	cm_info.backlog = backlog;
4018	cm_info.cm_id = cm_id;
4019
4020	trace_irdma_create_listen(iwdev, cm_info: &cm_info);
4021
4022	cm_listen_node = irdma_make_listen_node(cm_core: &iwdev->cm_core, iwdev,
4023	cm_info: &cm_info);
4024	if (!cm_listen_node) {
4025	ibdev_dbg(&iwdev->ibdev,
4026	"CM: cm_listen_node == NULL\n");
4027	return -ENOMEM;
4028	}
4029
4030	cm_id->provider_data = cm_listen_node;
4031
4032	cm_listen_node->tos = cm_id->tos;
4033	if (iwdev->vsi.dscp_mode)
4034	cm_listen_node->user_pri =
4035	iwdev->vsi.dscp_map[irdma_tos2dscp(tos: cm_id->tos)];
4036	else
4037	cm_listen_node->user_pri = rt_tos2priority(tos: cm_id->tos);
4038	cm_info.user_pri = cm_listen_node->user_pri;
4039	if (!cm_listen_node->reused_node) {
4040	if (wildcard) {
4041	err = irdma_add_mqh(iwdev, cm_info: &cm_info, cm_listen_node);
4042	if (err)
4043	goto error;
4044	} else {
4045	if (!iwdev->vsi.dscp_mode)
4046	cm_listen_node->user_pri =
4047	irdma_iw_get_vlan_prio(loc_addr: cm_info.loc_addr,
4048	prio: cm_info.user_pri,
4049	ipv4: cm_info.ipv4);
4050	cm_info.user_pri = cm_listen_node->user_pri;
4051	err = irdma_manage_qhash(iwdev, cminfo: &cm_info,
4052	etype: IRDMA_QHASH_TYPE_TCP_SYN,
4053	mtype: IRDMA_QHASH_MANAGE_TYPE_ADD,
4054	NULL, wait: true);
4055	if (err)
4056	goto error;
4057
4058	cm_listen_node->qhash_set = true;
4059	}
4060
4061	cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev,
4062	port: cm_info.loc_port);
4063	if (!cm_listen_node->apbvt_entry)
4064	goto error;
4065	}
4066	cm_id->add_ref(cm_id);
4067	cm_listen_node->cm_core->stats_listen_created++;
4068	ibdev_dbg(&iwdev->ibdev,
4069	"CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",
4070	cm_listen_node->loc_port, cm_listen_node->loc_addr,
4071	cm_listen_node, cm_listen_node->cm_id,
4072	cm_listen_node->qhash_set, cm_listen_node->vlan_id);
4073
4074	return 0;
4075
4076	error:
4077
4078	irdma_cm_del_listen(cm_core: &iwdev->cm_core, listener: cm_listen_node, apbvt_del: false);
4079
4080	return -EINVAL;
4081	}
4082
4083	/**
4084	* irdma_destroy_listen - registered call to destroy listener
4085	* @cm_id: cm information for passive connection
4086	*/
4087	int irdma_destroy_listen(struct iw_cm_id *cm_id)
4088	{
4089	struct irdma_device *iwdev;
4090
4091	iwdev = to_iwdev(ibdev: cm_id->device);
4092	if (cm_id->provider_data)
4093	irdma_cm_del_listen(cm_core: &iwdev->cm_core, listener: cm_id->provider_data,
4094	apbvt_del: true);
4095	else
4096	ibdev_dbg(&iwdev->ibdev,
4097	"CM: cm_id->provider_data was NULL\n");
4098
4099	cm_id->rem_ref(cm_id);
4100
4101	return 0;
4102	}
4103
4104	/**
4105	* irdma_teardown_list_prep - add conn nodes slated for tear down to list
4106	* @cm_core: cm's core
4107	* @teardown_list: a list to which cm_node will be selected
4108	* @ipaddr: pointer to ip address
4109	* @nfo: pointer to cm_info structure instance
4110	* @disconnect_all: flag indicating disconnect all QPs
4111	*/
4112	static void irdma_teardown_list_prep(struct irdma_cm_core *cm_core,
4113	struct list_head *teardown_list,
4114	u32 *ipaddr,
4115	struct irdma_cm_info *nfo,
4116	bool disconnect_all)
4117	{
4118	struct irdma_cm_node *cm_node;
4119	int bkt;
4120
4121	hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
4122	if ((disconnect_all ||
4123	(nfo->vlan_id == cm_node->vlan_id &&
4124	!memcmp(p: cm_node->loc_addr, q: ipaddr, size: nfo->ipv4 ? 4 : 16))) &&
4125	refcount_inc_not_zero(r: &cm_node->refcnt))
4126	list_add(new: &cm_node->teardown_entry, head: teardown_list);
4127	}
4128	}
4129
4130	/**
4131	* irdma_cm_event_connected - handle connected active node
4132	* @event: the info for cm_node of connection
4133	*/
4134	static void irdma_cm_event_connected(struct irdma_cm_event *event)
4135	{
4136	struct irdma_qp *iwqp;
4137	struct irdma_device *iwdev;
4138	struct irdma_cm_node *cm_node;
4139	struct irdma_sc_dev *dev;
4140	struct ib_qp_attr attr = {};
4141	struct iw_cm_id *cm_id;
4142	int status;
4143	bool read0;
4144	int wait_ret = 0;
4145
4146	cm_node = event->cm_node;
4147	cm_id = cm_node->cm_id;
4148	iwqp = cm_id->provider_data;
4149	iwdev = iwqp->iwdev;
4150	dev = &iwdev->rf->sc_dev;
4151	if (iwqp->sc_qp.qp_uk.destroy_pending) {
4152	status = -ETIMEDOUT;
4153	goto error;
4154	}
4155
4156	irdma_cm_init_tsa_conn(iwqp, cm_node);
4157	read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);
4158	if (iwqp->page)
4159	iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(page: iwqp->page);
4160	irdma_sc_send_rtt(qp: &iwqp->sc_qp, read: read0);
4161	if (iwqp->page)
4162	kunmap_local(iwqp->sc_qp.qp_uk.sq_base);
4163
4164	attr.qp_state = IB_QPS_RTS;
4165	cm_node->qhash_set = false;
4166	irdma_modify_qp(ibqp: &iwqp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL);
4167	if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
4168	wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
4169	iwqp->rts_ae_rcvd,
4170	IRDMA_MAX_TIMEOUT);
4171	if (!wait_ret)
4172	ibdev_dbg(&iwdev->ibdev,
4173	"CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
4174	cm_node, cm_node->loc_port,
4175	cm_node->rem_port, cm_node->cm_id);
4176	}
4177
4178	irdma_send_cm_event(cm_node, cm_id, type: IW_CM_EVENT_CONNECT_REPLY, status: 0);
4179	cm_node->accelerated = true;
4180	complete(&cm_node->establish_comp);
4181	cm_node->cm_core->cm_free_ah(cm_node);
4182	return;
4183
4184	error:
4185	iwqp->cm_id = NULL;
4186	cm_id->provider_data = NULL;
4187	irdma_send_cm_event(cm_node: event->cm_node, cm_id, type: IW_CM_EVENT_CONNECT_REPLY,
4188	status);
4189	irdma_rem_ref_cm_node(cm_node: event->cm_node);
4190	}
4191
4192	/**
4193	* irdma_cm_event_reset - handle reset
4194	* @event: the info for cm_node of connection
4195	*/
4196	static void irdma_cm_event_reset(struct irdma_cm_event *event)
4197	{
4198	struct irdma_cm_node *cm_node = event->cm_node;
4199	struct iw_cm_id *cm_id = cm_node->cm_id;
4200	struct irdma_qp *iwqp;
4201
4202	if (!cm_id)
4203	return;
4204
4205	iwqp = cm_id->provider_data;
4206	if (!iwqp)
4207	return;
4208
4209	ibdev_dbg(&cm_node->iwdev->ibdev,
4210	"CM: reset event %p - cm_id = %p\n", event->cm_node, cm_id);
4211	iwqp->cm_id = NULL;
4212
4213	irdma_send_cm_event(cm_node, cm_id: cm_node->cm_id, type: IW_CM_EVENT_DISCONNECT,
4214	status: -ECONNRESET);
4215	irdma_send_cm_event(cm_node, cm_id: cm_node->cm_id, type: IW_CM_EVENT_CLOSE, status: 0);
4216	}
4217
4218	/**
4219	* irdma_cm_event_handler - send event to cm upper layer
4220	* @work: pointer of cm event info.
4221	*/
4222	static void irdma_cm_event_handler(struct work_struct *work)
4223	{
4224	struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work);
4225	struct irdma_cm_node *cm_node;
4226
4227	if (!event || !event->cm_node || !event->cm_node->cm_core)
4228	return;
4229
4230	cm_node = event->cm_node;
4231	trace_irdma_cm_event_handler(cm_node, type: event->type, NULL);
4232
4233	switch (event->type) {
4234	case IRDMA_CM_EVENT_MPA_REQ:
4235	irdma_send_cm_event(cm_node, cm_id: cm_node->cm_id,
4236	type: IW_CM_EVENT_CONNECT_REQUEST, status: 0);
4237	break;
4238	case IRDMA_CM_EVENT_RESET:
4239	irdma_cm_event_reset(event);
4240	break;
4241	case IRDMA_CM_EVENT_CONNECTED:
4242	if (!event->cm_node->cm_id ||
4243	event->cm_node->state != IRDMA_CM_STATE_OFFLOADED)
4244	break;
4245	irdma_cm_event_connected(event);
4246	break;
4247	case IRDMA_CM_EVENT_MPA_REJECT:
4248	if (!event->cm_node->cm_id ||
4249	cm_node->state == IRDMA_CM_STATE_OFFLOADED)
4250	break;
4251	irdma_send_cm_event(cm_node, cm_id: cm_node->cm_id,
4252	type: IW_CM_EVENT_CONNECT_REPLY, status: -ECONNREFUSED);
4253	break;
4254	case IRDMA_CM_EVENT_ABORTED:
4255	if (!event->cm_node->cm_id ||
4256	event->cm_node->state == IRDMA_CM_STATE_OFFLOADED)
4257	break;
4258	irdma_event_connect_error(event);
4259	break;
4260	default:
4261	ibdev_dbg(&cm_node->iwdev->ibdev,
4262	"CM: bad event type = %d\n", event->type);
4263	break;
4264	}
4265
4266	irdma_rem_ref_cm_node(cm_node: event->cm_node);
4267	kfree(objp: event);
4268	}
4269
4270	/**
4271	* irdma_cm_post_event - queue event request for worker thread
4272	* @event: cm node's info for up event call
4273	*/
4274	static void irdma_cm_post_event(struct irdma_cm_event *event)
4275	{
4276	refcount_inc(r: &event->cm_node->refcnt);
4277	INIT_WORK(&event->event_work, irdma_cm_event_handler);
4278	queue_work(wq: event->cm_node->cm_core->event_wq, work: &event->event_work);
4279	}
4280
4281	/**
4282	* irdma_cm_teardown_connections - teardown QPs
4283	* @iwdev: device pointer
4284	* @ipaddr: Pointer to IPv4 or IPv6 address
4285	* @nfo: Connection info
4286	* @disconnect_all: flag indicating disconnect all QPs
4287	*
4288	* teardown QPs where source or destination addr matches ip addr
4289	*/
4290	void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,
4291	struct irdma_cm_info *nfo,
4292	bool disconnect_all)
4293	{
4294	struct irdma_cm_core *cm_core = &iwdev->cm_core;
4295	struct list_head *list_core_temp;
4296	struct list_head *list_node;
4297	struct irdma_cm_node *cm_node;
4298	struct list_head teardown_list;
4299	struct ib_qp_attr attr;
4300
4301	INIT_LIST_HEAD(list: &teardown_list);
4302
4303	rcu_read_lock();
4304	irdma_teardown_list_prep(cm_core, teardown_list: &teardown_list, ipaddr, nfo, disconnect_all);
4305	rcu_read_unlock();
4306
4307	list_for_each_safe (list_node, list_core_temp, &teardown_list) {
4308	cm_node = container_of(list_node, struct irdma_cm_node,
4309	teardown_entry);
4310	attr.qp_state = IB_QPS_ERR;
4311	irdma_modify_qp(ibqp: &cm_node->iwqp->ibqp, attr: &attr, attr_mask: IB_QP_STATE, NULL);
4312	if (iwdev->rf->reset)
4313	irdma_cm_disconn(iwqp: cm_node->iwqp);
4314	irdma_rem_ref_cm_node(cm_node);
4315	}
4316	}
4317
4318	/**
4319	* irdma_qhash_ctrl - enable/disable qhash for list
4320	* @iwdev: device pointer
4321	* @parent_listen_node: parent listen node
4322	* @nfo: cm info node
4323	* @ipaddr: Pointer to IPv4 or IPv6 address
4324	* @ipv4: flag indicating IPv4 when true
4325	* @ifup: flag indicating interface up when true
4326	*
4327	* Enables or disables the qhash for the node in the child
4328	* listen list that matches ipaddr. If no matching IP was found
4329	* it will allocate and add a new child listen node to the
4330	* parent listen node. The listen_list_lock is assumed to be
4331	* held when called.
4332	*/
4333	static void irdma_qhash_ctrl(struct irdma_device *iwdev,
4334	struct irdma_cm_listener *parent_listen_node,
4335	struct irdma_cm_info *nfo, u32 *ipaddr, bool ipv4,
4336	bool ifup)
4337	{
4338	struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
4339	struct irdma_cm_listener *child_listen_node;
4340	struct list_head *pos, *tpos;
4341	bool node_allocated = false;
4342	enum irdma_quad_hash_manage_type op = ifup ?
4343	IRDMA_QHASH_MANAGE_TYPE_ADD :
4344	IRDMA_QHASH_MANAGE_TYPE_DELETE;
4345	int err;
4346
4347	list_for_each_safe (pos, tpos, child_listen_list) {
4348	child_listen_node = list_entry(pos, struct irdma_cm_listener,
4349	child_listen_list);
4350	if (!memcmp(p: child_listen_node->loc_addr, q: ipaddr, size: ipv4 ? 4 : 16))
4351	goto set_qhash;
4352	}
4353
4354	/* if not found then add a child listener if interface is going up */
4355	if (!ifup)
4356	return;
4357	child_listen_node = kmemdup(parent_listen_node,
4358	sizeof(*child_listen_node), GFP_ATOMIC);
4359	if (!child_listen_node)
4360	return;
4361
4362	node_allocated = true;
4363	memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16);
4364
4365	set_qhash:
4366	memcpy(nfo->loc_addr, child_listen_node->loc_addr,
4367	sizeof(nfo->loc_addr));
4368	nfo->vlan_id = child_listen_node->vlan_id;
4369	err = irdma_manage_qhash(iwdev, cminfo: nfo, etype: IRDMA_QHASH_TYPE_TCP_SYN, mtype: op, NULL,
4370	wait: false);
4371	if (!err) {
4372	child_listen_node->qhash_set = ifup;
4373	if (node_allocated)
4374	list_add(new: &child_listen_node->child_listen_list,
4375	head: &parent_listen_node->child_listen_list);
4376	} else if (node_allocated) {
4377	kfree(objp: child_listen_node);
4378	}
4379	}
4380
4381	/**
4382	* irdma_if_notify - process an ifdown on an interface
4383	* @iwdev: device pointer
4384	* @netdev: network device structure
4385	* @ipaddr: Pointer to IPv4 or IPv6 address
4386	* @ipv4: flag indicating IPv4 when true
4387	* @ifup: flag indicating interface up when true
4388	*/
4389	void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,
4390	u32 *ipaddr, bool ipv4, bool ifup)
4391	{
4392	struct irdma_cm_core *cm_core = &iwdev->cm_core;
4393	unsigned long flags;
4394	struct irdma_cm_listener *listen_node;
4395	static const u32 ip_zero[4] = { 0, 0, 0, 0 };
4396	struct irdma_cm_info nfo = {};
4397	u16 vlan_id = rdma_vlan_dev_vlan_id(dev: netdev);
4398	enum irdma_quad_hash_manage_type op = ifup ?
4399	IRDMA_QHASH_MANAGE_TYPE_ADD :
4400	IRDMA_QHASH_MANAGE_TYPE_DELETE;
4401
4402	nfo.vlan_id = vlan_id;
4403	nfo.ipv4 = ipv4;
4404	nfo.qh_qpid = 1;
4405
4406	/* Disable or enable qhash for listeners */
4407	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
4408	list_for_each_entry (listen_node, &cm_core->listen_list, list) {
4409	if (vlan_id != listen_node->vlan_id ||
4410	(memcmp(p: listen_node->loc_addr, q: ipaddr, size: ipv4 ? 4 : 16) &&
4411	memcmp(p: listen_node->loc_addr, q: ip_zero, size: ipv4 ? 4 : 16)))
4412	continue;
4413
4414	memcpy(nfo.loc_addr, listen_node->loc_addr,
4415	sizeof(nfo.loc_addr));
4416	nfo.loc_port = listen_node->loc_port;
4417	nfo.user_pri = listen_node->user_pri;
4418	if (!list_empty(head: &listen_node->child_listen_list)) {
4419	irdma_qhash_ctrl(iwdev, parent_listen_node: listen_node, nfo: &nfo, ipaddr, ipv4,
4420	ifup);
4421	} else if (memcmp(p: listen_node->loc_addr, q: ip_zero,
4422	size: ipv4 ? 4 : 16)) {
4423	if (!irdma_manage_qhash(iwdev, cminfo: &nfo,
4424	etype: IRDMA_QHASH_TYPE_TCP_SYN, mtype: op,
4425	NULL, wait: false))
4426	listen_node->qhash_set = ifup;
4427	}
4428	}
4429	spin_unlock_irqrestore(lock: &cm_core->listen_list_lock, flags);
4430
4431	/* disconnect any connected qp's on ifdown */
4432	if (!ifup)
4433	irdma_cm_teardown_connections(iwdev, ipaddr, nfo: &nfo, disconnect_all: false);
4434	}
4435