1	/*
2	* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32
33	#include <linux/etherdevice.h>
34	#include <rdma/ib_umem.h>
35	#include <rdma/ib_cache.h>
36	#include <rdma/ib_user_verbs.h>
37	#include <rdma/rdma_counter.h>
38	#include <linux/mlx5/fs.h>
39	#include "mlx5_ib.h"
40	#include "ib_rep.h"
41	#include "counters.h"
42	#include "cmd.h"
43	#include "umr.h"
44	#include "qp.h"
45	#include "wr.h"
46
47	enum {
48	MLX5_IB_ACK_REQ_FREQ = 8,
49	};
50
51	enum {
52	MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83,
53	MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
54	MLX5_IB_LINK_TYPE_IB = 0,
55	MLX5_IB_LINK_TYPE_ETH = 1
56	};
57
58	enum raw_qp_set_mask_map {
59	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID = 1UL << 0,
60	MLX5_RAW_QP_RATE_LIMIT = 1UL << 1,
61	};
62
63	enum {
64	MLX5_QP_RM_GO_BACK_N = 0x1,
65	};
66
67	struct mlx5_modify_raw_qp_param {
68	u16 operation;
69
70	u32 set_mask; /* raw_qp_set_mask_map */
71
72	struct mlx5_rate_limit rl;
73
74	u8 rq_q_ctr_id;
75	u32 port;
76	};
77
78	struct mlx5_ib_qp_event_work {
79	struct work_struct work;
80	struct mlx5_core_qp *qp;
81	int type;
82	};
83
84	static struct workqueue_struct *mlx5_ib_qp_event_wq;
85
86	static void get_cqs(enum ib_qp_type qp_type,
87	struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
88	struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
89
90	static int is_qp0(enum ib_qp_type qp_type)
91	{
92	return qp_type == IB_QPT_SMI;
93	}
94
95	static int is_sqp(enum ib_qp_type qp_type)
96	{
97	return is_qp0(qp_type) || is_qp1(qp_type);
98	}
99
100	/**
101	* mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
102	* to kernel buffer
103	*
104	* @umem: User space memory where the WQ is
105	* @buffer: buffer to copy to
106	* @buflen: buffer length
107	* @wqe_index: index of WQE to copy from
108	* @wq_offset: offset to start of WQ
109	* @wq_wqe_cnt: number of WQEs in WQ
110	* @wq_wqe_shift: log2 of WQE size
111	* @bcnt: number of bytes to copy
112	* @bytes_copied: number of bytes to copy (return value)
113	*
114	* Copies from start of WQE bcnt or less bytes.
115	* Does not gurantee to copy the entire WQE.
116	*
117	* Return: zero on success, or an error code.
118	*/
119	static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,
120	size_t buflen, int wqe_index,
121	int wq_offset, int wq_wqe_cnt,
122	int wq_wqe_shift, int bcnt,
123	size_t *bytes_copied)
124	{
125	size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) << wq_wqe_shift);
126	size_t wq_end = wq_offset + (wq_wqe_cnt << wq_wqe_shift);
127	size_t copy_length;
128	int ret;
129
130	/* don't copy more than requested, more than buffer length or
131	* beyond WQ end
132	*/
133	copy_length = min_t(u32, buflen, wq_end - offset);
134	copy_length = min_t(u32, copy_length, bcnt);
135
136	ret = ib_umem_copy_from(dst: buffer, umem, offset, length: copy_length);
137	if (ret)
138	return ret;
139
140	if (!ret && bytes_copied)
141	*bytes_copied = copy_length;
142
143	return 0;
144	}
145
146	static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
147	void *buffer, size_t buflen, size_t *bc)
148	{
149	struct mlx5_wqe_ctrl_seg *ctrl;
150	size_t bytes_copied = 0;
151	size_t wqe_length;
152	void *p;
153	int ds;
154
155	wqe_index = wqe_index & qp->sq.fbc.sz_m1;
156
157	/* read the control segment first */
158	p = mlx5_frag_buf_get_wqe(fbc: &qp->sq.fbc, ix: wqe_index);
159	ctrl = p;
160	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
161	wqe_length = ds * MLX5_WQE_DS_UNITS;
162
163	/* read rest of WQE if it spreads over more than one stride */
164	while (bytes_copied < wqe_length) {
165	size_t copy_length =
166	min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);
167
168	if (!copy_length)
169	break;
170
171	memcpy(buffer + bytes_copied, p, copy_length);
172	bytes_copied += copy_length;
173
174	wqe_index = (wqe_index + 1) & qp->sq.fbc.sz_m1;
175	p = mlx5_frag_buf_get_wqe(fbc: &qp->sq.fbc, ix: wqe_index);
176	}
177	*bc = bytes_copied;
178	return 0;
179	}
180
181	static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
182	void *buffer, size_t buflen, size_t *bc)
183	{
184	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
185	struct ib_umem *umem = base->ubuffer.umem;
186	struct mlx5_ib_wq *wq = &qp->sq;
187	struct mlx5_wqe_ctrl_seg *ctrl;
188	size_t bytes_copied;
189	size_t bytes_copied2;
190	size_t wqe_length;
191	int ret;
192	int ds;
193
194	/* at first read as much as possible */
195	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
196	wq_offset: wq->offset, wq_wqe_cnt: wq->wqe_cnt,
197	wq_wqe_shift: wq->wqe_shift, bcnt: buflen,
198	bytes_copied: &bytes_copied);
199	if (ret)
200	return ret;
201
202	/* we need at least control segment size to proceed */
203	if (bytes_copied < sizeof(*ctrl))
204	return -EINVAL;
205
206	ctrl = buffer;
207	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
208	wqe_length = ds * MLX5_WQE_DS_UNITS;
209
210	/* if we copied enough then we are done */
211	if (bytes_copied >= wqe_length) {
212	*bc = bytes_copied;
213	return 0;
214	}
215
216	/* otherwise this a wrapped around wqe
217	* so read the remaining bytes starting
218	* from wqe_index 0
219	*/
220	ret = mlx5_ib_read_user_wqe_common(umem, buffer: buffer + bytes_copied,
221	buflen: buflen - bytes_copied, wqe_index: 0, wq_offset: wq->offset,
222	wq_wqe_cnt: wq->wqe_cnt, wq_wqe_shift: wq->wqe_shift,
223	bcnt: wqe_length - bytes_copied,
224	bytes_copied: &bytes_copied2);
225
226	if (ret)
227	return ret;
228	*bc = bytes_copied + bytes_copied2;
229	return 0;
230	}
231
232	int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
233	size_t buflen, size_t *bc)
234	{
235	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
236	struct ib_umem *umem = base->ubuffer.umem;
237
238	if (buflen < sizeof(struct mlx5_wqe_ctrl_seg))
239	return -EINVAL;
240
241	if (!umem)
242	return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,
243	buflen, bc);
244
245	return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);
246	}
247
248	static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
249	void *buffer, size_t buflen, size_t *bc)
250	{
251	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
252	struct ib_umem *umem = base->ubuffer.umem;
253	struct mlx5_ib_wq *wq = &qp->rq;
254	size_t bytes_copied;
255	int ret;
256
257	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
258	wq_offset: wq->offset, wq_wqe_cnt: wq->wqe_cnt,
259	wq_wqe_shift: wq->wqe_shift, bcnt: buflen,
260	bytes_copied: &bytes_copied);
261
262	if (ret)
263	return ret;
264	*bc = bytes_copied;
265	return 0;
266	}
267
268	int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
269	size_t buflen, size_t *bc)
270	{
271	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
272	struct ib_umem *umem = base->ubuffer.umem;
273	struct mlx5_ib_wq *wq = &qp->rq;
274	size_t wqe_size = 1 << wq->wqe_shift;
275
276	if (buflen < wqe_size)
277	return -EINVAL;
278
279	if (!umem)
280	return -EOPNOTSUPP;
281
282	return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);
283	}
284
285	static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
286	void *buffer, size_t buflen, size_t *bc)
287	{
288	struct ib_umem *umem = srq->umem;
289	size_t bytes_copied;
290	int ret;
291
292	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, wq_offset: 0,
293	wq_wqe_cnt: srq->msrq.max, wq_wqe_shift: srq->msrq.wqe_shift,
294	bcnt: buflen, bytes_copied: &bytes_copied);
295
296	if (ret)
297	return ret;
298	*bc = bytes_copied;
299	return 0;
300	}
301
302	int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
303	size_t buflen, size_t *bc)
304	{
305	struct ib_umem *umem = srq->umem;
306	size_t wqe_size = 1 << srq->msrq.wqe_shift;
307
308	if (buflen < wqe_size)
309	return -EINVAL;
310
311	if (!umem)
312	return -EOPNOTSUPP;
313
314	return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
315	}
316
317	static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
318	{
319	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibqp->device);
320	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
321	struct mlx5_ib_qp *qp = to_mqp(ibqp);
322	void *pas_ext_union, *err_syn;
323	u32 *outb;
324	int err;
325
326	if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) ||
327	!MLX5_CAP_GEN(dev->mdev, qp_error_syndrome))
328	return;
329
330	outb = kzalloc(outlen, GFP_KERNEL);
331	if (!outb)
332	return;
333
334	err = mlx5_core_qp_query(dev, qp: &qp->trans_qp.base.mqp, outb, outlen,
335	qpc_ext: true);
336	if (err)
337	goto out;
338
339	pas_ext_union =
340	MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);
341	err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,
342	qpc_data_extension.error_syndrome);
343
344	pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",
345	ibqp->device->name, ibqp->port, ibqp->qp_num,
346	ib_wc_status_msg(
347	MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),
348	MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),
349	MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),
350	MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));
351	out:
352	kfree(outb);
353	}
354
355	static void mlx5_ib_handle_qp_event(struct work_struct *_work)
356	{
357	struct mlx5_ib_qp_event_work *qpe_work =
358	container_of(_work, struct mlx5_ib_qp_event_work, work);
359	struct ib_qp *ibqp = &to_mibqp(mqp: qpe_work->qp)->ibqp;
360	struct ib_event event = {};
361
362	event.device = ibqp->device;
363	event.element.qp = ibqp;
364	switch (qpe_work->type) {
365	case MLX5_EVENT_TYPE_PATH_MIG:
366	event.event = IB_EVENT_PATH_MIG;
367	break;
368	case MLX5_EVENT_TYPE_COMM_EST:
369	event.event = IB_EVENT_COMM_EST;
370	break;
371	case MLX5_EVENT_TYPE_SQ_DRAINED:
372	event.event = IB_EVENT_SQ_DRAINED;
373	break;
374	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
375	event.event = IB_EVENT_QP_LAST_WQE_REACHED;
376	break;
377	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
378	event.event = IB_EVENT_QP_FATAL;
379	break;
380	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
381	event.event = IB_EVENT_PATH_MIG_ERR;
382	break;
383	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
384	event.event = IB_EVENT_QP_REQ_ERR;
385	break;
386	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
387	event.event = IB_EVENT_QP_ACCESS_ERR;
388	break;
389	default:
390	pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",
391	qpe_work->type, qpe_work->qp->qpn);
392	goto out;
393	}
394
395	if ((event.event == IB_EVENT_QP_FATAL) ||
396	(event.event == IB_EVENT_QP_ACCESS_ERR))
397	mlx5_ib_qp_err_syndrome(ibqp);
398
399	ibqp->event_handler(&event, ibqp->qp_context);
400
401	out:
402	mlx5_core_res_put(res: &qpe_work->qp->common);
403	kfree(objp: qpe_work);
404	}
405
406	static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
407	{
408	struct ib_qp *ibqp = &to_mibqp(mqp: qp)->ibqp;
409	struct mlx5_ib_qp_event_work *qpe_work;
410
411	if (type == MLX5_EVENT_TYPE_PATH_MIG) {
412	/* This event is only valid for trans_qps */
413	to_mibqp(mqp: qp)->port = to_mibqp(mqp: qp)->trans_qp.alt_port;
414	}
415
416	if (!ibqp->event_handler)
417	goto out_no_handler;
418
419	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
420	if (!qpe_work)
421	goto out_no_handler;
422
423	qpe_work->qp = qp;
424	qpe_work->type = type;
425	INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event);
426	queue_work(wq: mlx5_ib_qp_event_wq, work: &qpe_work->work);
427	return;
428
429	out_no_handler:
430	mlx5_core_res_put(res: &qp->common);
431	}
432
433	static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
434	int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
435	{
436	int wqe_size;
437	int wq_size;
438
439	/* Sanity check RQ size before proceeding */
440	if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
441	return -EINVAL;
442
443	if (!has_rq) {
444	qp->rq.max_gs = 0;
445	qp->rq.wqe_cnt = 0;
446	qp->rq.wqe_shift = 0;
447	cap->max_recv_wr = 0;
448	cap->max_recv_sge = 0;
449	} else {
450	int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
451
452	if (ucmd) {
453	qp->rq.wqe_cnt = ucmd->rq_wqe_count;
454	if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
455	return -EINVAL;
456	qp->rq.wqe_shift = ucmd->rq_wqe_shift;
457	if ((1 << qp->rq.wqe_shift) /
458	sizeof(struct mlx5_wqe_data_seg) <
459	wq_sig)
460	return -EINVAL;
461	qp->rq.max_gs =
462	(1 << qp->rq.wqe_shift) /
463	sizeof(struct mlx5_wqe_data_seg) -
464	wq_sig;
465	qp->rq.max_post = qp->rq.wqe_cnt;
466	} else {
467	wqe_size =
468	wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
469	0;
470	wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
471	wqe_size = roundup_pow_of_two(wqe_size);
472	wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
473	wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
474	qp->rq.wqe_cnt = wq_size / wqe_size;
475	if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
476	mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
477	wqe_size,
478	MLX5_CAP_GEN(dev->mdev,
479	max_wqe_sz_rq));
480	return -EINVAL;
481	}
482	qp->rq.wqe_shift = ilog2(wqe_size);
483	qp->rq.max_gs =
484	(1 << qp->rq.wqe_shift) /
485	sizeof(struct mlx5_wqe_data_seg) -
486	wq_sig;
487	qp->rq.max_post = qp->rq.wqe_cnt;
488	}
489	}
490
491	return 0;
492	}
493
494	static int sq_overhead(struct ib_qp_init_attr *attr)
495	{
496	int size = 0;
497
498	switch (attr->qp_type) {
499	case IB_QPT_XRC_INI:
500	size += sizeof(struct mlx5_wqe_xrc_seg);
501	fallthrough;
502	case IB_QPT_RC:
503	size += sizeof(struct mlx5_wqe_ctrl_seg) +
504	max(sizeof(struct mlx5_wqe_atomic_seg) +
505	sizeof(struct mlx5_wqe_raddr_seg),
506	sizeof(struct mlx5_wqe_umr_ctrl_seg) +
507	sizeof(struct mlx5_mkey_seg) +
508	MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
509	MLX5_IB_UMR_OCTOWORD);
510	break;
511
512	case IB_QPT_XRC_TGT:
513	return 0;
514
515	case IB_QPT_UC:
516	size += sizeof(struct mlx5_wqe_ctrl_seg) +
517	max(sizeof(struct mlx5_wqe_raddr_seg),
518	sizeof(struct mlx5_wqe_umr_ctrl_seg) +
519	sizeof(struct mlx5_mkey_seg));
520	break;
521
522	case IB_QPT_UD:
523	if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
524	size += sizeof(struct mlx5_wqe_eth_pad) +
525	sizeof(struct mlx5_wqe_eth_seg);
526	fallthrough;
527	case IB_QPT_SMI:
528	case MLX5_IB_QPT_HW_GSI:
529	size += sizeof(struct mlx5_wqe_ctrl_seg) +
530	sizeof(struct mlx5_wqe_datagram_seg);
531	break;
532
533	case MLX5_IB_QPT_REG_UMR:
534	size += sizeof(struct mlx5_wqe_ctrl_seg) +
535	sizeof(struct mlx5_wqe_umr_ctrl_seg) +
536	sizeof(struct mlx5_mkey_seg);
537	break;
538
539	default:
540	return -EINVAL;
541	}
542
543	return size;
544	}
545
546	static int calc_send_wqe(struct ib_qp_init_attr *attr)
547	{
548	int inl_size = 0;
549	int size;
550
551	size = sq_overhead(attr);
552	if (size < 0)
553	return size;
554
555	if (attr->cap.max_inline_data) {
556	inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
557	attr->cap.max_inline_data;
558	}
559
560	size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
561	if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
562	ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
563	return MLX5_SIG_WQE_SIZE;
564	else
565	return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
566	}
567
568	static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
569	{
570	int max_sge;
571
572	if (attr->qp_type == IB_QPT_RC)
573	max_sge = (min_t(int, wqe_size, 512) -
574	sizeof(struct mlx5_wqe_ctrl_seg) -
575	sizeof(struct mlx5_wqe_raddr_seg)) /
576	sizeof(struct mlx5_wqe_data_seg);
577	else if (attr->qp_type == IB_QPT_XRC_INI)
578	max_sge = (min_t(int, wqe_size, 512) -
579	sizeof(struct mlx5_wqe_ctrl_seg) -
580	sizeof(struct mlx5_wqe_xrc_seg) -
581	sizeof(struct mlx5_wqe_raddr_seg)) /
582	sizeof(struct mlx5_wqe_data_seg);
583	else
584	max_sge = (wqe_size - sq_overhead(attr)) /
585	sizeof(struct mlx5_wqe_data_seg);
586
587	return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
588	sizeof(struct mlx5_wqe_data_seg));
589	}
590
591	static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
592	struct mlx5_ib_qp *qp)
593	{
594	int wqe_size;
595	int wq_size;
596
597	if (!attr->cap.max_send_wr)
598	return 0;
599
600	wqe_size = calc_send_wqe(attr);
601	mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
602	if (wqe_size < 0)
603	return wqe_size;
604
605	if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
606	mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
607	wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
608	return -EINVAL;
609	}
610
611	qp->max_inline_data = wqe_size - sq_overhead(attr) -
612	sizeof(struct mlx5_wqe_inline_seg);
613	attr->cap.max_inline_data = qp->max_inline_data;
614
615	wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
616	qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
617	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
618	mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
619	attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
620	qp->sq.wqe_cnt,
621	1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
622	return -ENOMEM;
623	}
624	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
625	qp->sq.max_gs = get_send_sge(attr, wqe_size);
626	if (qp->sq.max_gs < attr->cap.max_send_sge)
627	return -ENOMEM;
628
629	attr->cap.max_send_sge = qp->sq.max_gs;
630	qp->sq.max_post = wq_size / wqe_size;
631	attr->cap.max_send_wr = qp->sq.max_post;
632
633	return wq_size;
634	}
635
636	static int set_user_buf_size(struct mlx5_ib_dev *dev,
637	struct mlx5_ib_qp *qp,
638	struct mlx5_ib_create_qp *ucmd,
639	struct mlx5_ib_qp_base *base,
640	struct ib_qp_init_attr *attr)
641	{
642	int desc_sz = 1 << qp->sq.wqe_shift;
643
644	if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
645	mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
646	desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
647	return -EINVAL;
648	}
649
650	if (ucmd->sq_wqe_count && !is_power_of_2(n: ucmd->sq_wqe_count)) {
651	mlx5_ib_warn(dev, "sq_wqe_count %d is not a power of two\n",
652	ucmd->sq_wqe_count);
653	return -EINVAL;
654	}
655
656	qp->sq.wqe_cnt = ucmd->sq_wqe_count;
657
658	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
659	mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
660	qp->sq.wqe_cnt,
661	1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
662	return -EINVAL;
663	}
664
665	if (attr->qp_type == IB_QPT_RAW_PACKET ||
666	qp->flags & IB_QP_CREATE_SOURCE_QPN) {
667	base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
668	qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
669	} else {
670	base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
671	(qp->sq.wqe_cnt << 6);
672	}
673
674	return 0;
675	}
676
677	static int qp_has_rq(struct ib_qp_init_attr *attr)
678	{
679	if (attr->qp_type == IB_QPT_XRC_INI ||
680	attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
681	attr->qp_type == MLX5_IB_QPT_REG_UMR ||
682	!attr->cap.max_recv_wr)
683	return 0;
684
685	return 1;
686	}
687
688	enum {
689	/* this is the first blue flame register in the array of bfregs assigned
690	* to a processes. Since we do not use it for blue flame but rather
691	* regular 64 bit doorbells, we do not need a lock for maintaiing
692	* "odd/even" order
693	*/
694	NUM_NON_BLUE_FLAME_BFREGS = 1,
695	};
696
697	static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
698	{
699	return get_uars_per_sys_page(dev, lib_support: bfregi->lib_uar_4k) *
700	bfregi->num_static_sys_pages * MLX5_NON_FP_BFREGS_PER_UAR;
701	}
702
703	static int num_med_bfreg(struct mlx5_ib_dev *dev,
704	struct mlx5_bfreg_info *bfregi)
705	{
706	int n;
707
708	n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
709	NUM_NON_BLUE_FLAME_BFREGS;
710
711	return n >= 0 ? n : 0;
712	}
713
714	static int first_med_bfreg(struct mlx5_ib_dev *dev,
715	struct mlx5_bfreg_info *bfregi)
716	{
717	return num_med_bfreg(dev, bfregi) ? 1 : -ENOMEM;
718	}
719
720	static int first_hi_bfreg(struct mlx5_ib_dev *dev,
721	struct mlx5_bfreg_info *bfregi)
722	{
723	int med;
724
725	med = num_med_bfreg(dev, bfregi);
726	return ++med;
727	}
728
729	static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
730	struct mlx5_bfreg_info *bfregi)
731	{
732	int i;
733
734	for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
735	if (!bfregi->count[i]) {
736	bfregi->count[i]++;
737	return i;
738	}
739	}
740
741	return -ENOMEM;
742	}
743
744	static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
745	struct mlx5_bfreg_info *bfregi)
746	{
747	int minidx = first_med_bfreg(dev, bfregi);
748	int i;
749
750	if (minidx < 0)
751	return minidx;
752
753	for (i = minidx; i < first_hi_bfreg(dev, bfregi); i++) {
754	if (bfregi->count[i] < bfregi->count[minidx])
755	minidx = i;
756	if (!bfregi->count[minidx])
757	break;
758	}
759
760	bfregi->count[minidx]++;
761	return minidx;
762	}
763
764	static int alloc_bfreg(struct mlx5_ib_dev *dev,
765	struct mlx5_bfreg_info *bfregi)
766	{
767	int bfregn = -ENOMEM;
768
769	if (bfregi->lib_uar_dyn)
770	return -EINVAL;
771
772	mutex_lock(&bfregi->lock);
773	if (bfregi->ver >= 2) {
774	bfregn = alloc_high_class_bfreg(dev, bfregi);
775	if (bfregn < 0)
776	bfregn = alloc_med_class_bfreg(dev, bfregi);
777	}
778
779	if (bfregn < 0) {
780	BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
781	bfregn = 0;
782	bfregi->count[bfregn]++;
783	}
784	mutex_unlock(lock: &bfregi->lock);
785
786	return bfregn;
787	}
788
789	void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
790	{
791	mutex_lock(&bfregi->lock);
792	bfregi->count[bfregn]--;
793	mutex_unlock(lock: &bfregi->lock);
794	}
795
796	static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
797	{
798	switch (state) {
799	case IB_QPS_RESET: return MLX5_QP_STATE_RST;
800	case IB_QPS_INIT: return MLX5_QP_STATE_INIT;
801	case IB_QPS_RTR: return MLX5_QP_STATE_RTR;
802	case IB_QPS_RTS: return MLX5_QP_STATE_RTS;
803	case IB_QPS_SQD: return MLX5_QP_STATE_SQD;
804	case IB_QPS_SQE: return MLX5_QP_STATE_SQER;
805	case IB_QPS_ERR: return MLX5_QP_STATE_ERR;
806	default: return -1;
807	}
808	}
809
810	static int to_mlx5_st(enum ib_qp_type type)
811	{
812	switch (type) {
813	case IB_QPT_RC: return MLX5_QP_ST_RC;
814	case IB_QPT_UC: return MLX5_QP_ST_UC;
815	case IB_QPT_UD: return MLX5_QP_ST_UD;
816	case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR;
817	case IB_QPT_XRC_INI:
818	case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC;
819	case IB_QPT_SMI: return MLX5_QP_ST_QP0;
820	case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1;
821	case MLX5_IB_QPT_DCI: return MLX5_QP_ST_DCI;
822	case IB_QPT_RAW_PACKET: return MLX5_QP_ST_RAW_ETHERTYPE;
823	default: return -EINVAL;
824	}
825	}
826
827	static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
828	struct mlx5_ib_cq *recv_cq);
829	static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
830	struct mlx5_ib_cq *recv_cq);
831
832	int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
833	struct mlx5_bfreg_info *bfregi, u32 bfregn,
834	bool dyn_bfreg)
835	{
836	unsigned int bfregs_per_sys_page;
837	u32 index_of_sys_page;
838	u32 offset;
839
840	if (bfregi->lib_uar_dyn)
841	return -EINVAL;
842
843	bfregs_per_sys_page = get_uars_per_sys_page(dev, lib_support: bfregi->lib_uar_4k) *
844	MLX5_NON_FP_BFREGS_PER_UAR;
845	index_of_sys_page = bfregn / bfregs_per_sys_page;
846
847	if (dyn_bfreg) {
848	index_of_sys_page += bfregi->num_static_sys_pages;
849
850	if (index_of_sys_page >= bfregi->num_sys_pages)
851	return -EINVAL;
852
853	if (bfregn > bfregi->num_dyn_bfregs ||
854	bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
855	mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
856	return -EINVAL;
857	}
858	}
859
860	offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
861	return bfregi->sys_pages[index_of_sys_page] + offset;
862	}
863
864	static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
865	struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
866	{
867	struct mlx5_ib_ucontext *context =
868	rdma_udata_to_drv_context(
869	udata,
870	struct mlx5_ib_ucontext,
871	ibucontext);
872
873	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
874	atomic_dec(v: &dev->delay_drop.rqs_cnt);
875
876	mlx5_ib_db_unmap_user(context, db: &rwq->db);
877	ib_umem_release(umem: rwq->umem);
878	}
879
880	static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
881	struct ib_udata *udata, struct mlx5_ib_rwq *rwq,
882	struct mlx5_ib_create_wq *ucmd)
883	{
884	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
885	udata, struct mlx5_ib_ucontext, ibucontext);
886	unsigned long page_size = 0;
887	u32 offset = 0;
888	int err;
889
890	if (!ucmd->buf_addr)
891	return -EINVAL;
892
893	rwq->umem = ib_umem_get(device: &dev->ib_dev, addr: ucmd->buf_addr, size: rwq->buf_size, access: 0);
894	if (IS_ERR(ptr: rwq->umem)) {
895	mlx5_ib_dbg(dev, "umem_get failed\n");
896	err = PTR_ERR(ptr: rwq->umem);
897	return err;
898	}
899
900	page_size = mlx5_umem_find_best_quantized_pgoff(
901	rwq->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
902	page_offset, 64, &rwq->rq_page_offset);
903	if (!page_size) {
904	mlx5_ib_warn(dev, "bad offset\n");
905	err = -EINVAL;
906	goto err_umem;
907	}
908
909	rwq->rq_num_pas = ib_umem_num_dma_blocks(umem: rwq->umem, pgsz: page_size);
910	rwq->page_shift = order_base_2(page_size);
911	rwq->log_page_size = rwq->page_shift - MLX5_ADAPTER_PAGE_SHIFT;
912	rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
913
914	mlx5_ib_dbg(
915	dev,
916	"addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n",
917	(unsigned long long)ucmd->buf_addr, rwq->buf_size,
918	ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas,
919	offset);
920
921	err = mlx5_ib_db_map_user(context: ucontext, virt: ucmd->db_addr, db: &rwq->db);
922	if (err) {
923	mlx5_ib_dbg(dev, "map failed\n");
924	goto err_umem;
925	}
926
927	return 0;
928
929	err_umem:
930	ib_umem_release(umem: rwq->umem);
931	return err;
932	}
933
934	static int adjust_bfregn(struct mlx5_ib_dev *dev,
935	struct mlx5_bfreg_info *bfregi, int bfregn)
936	{
937	return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
938	bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
939	}
940
941	static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
942	struct mlx5_ib_qp *qp, struct ib_udata *udata,
943	struct ib_qp_init_attr *attr, u32 **in,
944	struct mlx5_ib_create_qp_resp *resp, int *inlen,
945	struct mlx5_ib_qp_base *base,
946	struct mlx5_ib_create_qp *ucmd)
947	{
948	struct mlx5_ib_ucontext *context;
949	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
950	unsigned int page_offset_quantized = 0;
951	unsigned long page_size = 0;
952	int uar_index = 0;
953	int bfregn;
954	int ncont = 0;
955	__be64 *pas;
956	void *qpc;
957	int err;
958	u16 uid;
959	u32 uar_flags;
960
961	context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
962	ibucontext);
963	uar_flags = qp->flags_en &
964	(MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);
965	switch (uar_flags) {
966	case MLX5_QP_FLAG_UAR_PAGE_INDEX:
967	uar_index = ucmd->bfreg_index;
968	bfregn = MLX5_IB_INVALID_BFREG;
969	break;
970	case MLX5_QP_FLAG_BFREG_INDEX:
971	uar_index = bfregn_to_uar_index(dev, bfregi: &context->bfregi,
972	bfregn: ucmd->bfreg_index, dyn_bfreg: true);
973	if (uar_index < 0)
974	return uar_index;
975	bfregn = MLX5_IB_INVALID_BFREG;
976	break;
977	case 0:
978	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
979	return -EINVAL;
980	bfregn = alloc_bfreg(dev, bfregi: &context->bfregi);
981	if (bfregn < 0)
982	return bfregn;
983	break;
984	default:
985	return -EINVAL;
986	}
987
988	mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
989	if (bfregn != MLX5_IB_INVALID_BFREG)
990	uar_index = bfregn_to_uar_index(dev, bfregi: &context->bfregi, bfregn,
991	dyn_bfreg: false);
992
993	qp->rq.offset = 0;
994	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
995	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
996
997	err = set_user_buf_size(dev, qp, ucmd, base, attr);
998	if (err)
999	goto err_bfreg;
1000
1001	if (ucmd->buf_addr && ubuffer->buf_size) {
1002	ubuffer->buf_addr = ucmd->buf_addr;
1003	ubuffer->umem = ib_umem_get(device: &dev->ib_dev, addr: ubuffer->buf_addr,
1004	size: ubuffer->buf_size, access: 0);
1005	if (IS_ERR(ptr: ubuffer->umem)) {
1006	err = PTR_ERR(ptr: ubuffer->umem);
1007	goto err_bfreg;
1008	}
1009	page_size = mlx5_umem_find_best_quantized_pgoff(
1010	ubuffer->umem, qpc, log_page_size,
1011	MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,
1012	&page_offset_quantized);
1013	if (!page_size) {
1014	err = -EINVAL;
1015	goto err_umem;
1016	}
1017	ncont = ib_umem_num_dma_blocks(umem: ubuffer->umem, pgsz: page_size);
1018	} else {
1019	ubuffer->umem = NULL;
1020	}
1021
1022	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1023	MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1024	*in = kvzalloc(*inlen, GFP_KERNEL);
1025	if (!*in) {
1026	err = -ENOMEM;
1027	goto err_umem;
1028	}
1029
1030	uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(ibpd: pd)->uid : 0;
1031	MLX5_SET(create_qp_in, *in, uid, uid);
1032	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1033	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
1034	if (ubuffer->umem) {
1035	mlx5_ib_populate_pas(umem: ubuffer->umem, page_size, pas, access_flags: 0);
1036	MLX5_SET(qpc, qpc, log_page_size,
1037	order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1038	MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);
1039	}
1040	MLX5_SET(qpc, qpc, uar_page, uar_index);
1041	if (bfregn != MLX5_IB_INVALID_BFREG)
1042	resp->bfreg_index = adjust_bfregn(dev, bfregi: &context->bfregi, bfregn);
1043	else
1044	resp->bfreg_index = MLX5_IB_INVALID_BFREG;
1045	qp->bfregn = bfregn;
1046
1047	err = mlx5_ib_db_map_user(context, virt: ucmd->db_addr, db: &qp->db);
1048	if (err) {
1049	mlx5_ib_dbg(dev, "map failed\n");
1050	goto err_free;
1051	}
1052
1053	return 0;
1054
1055	err_free:
1056	kvfree(addr: *in);
1057
1058	err_umem:
1059	ib_umem_release(umem: ubuffer->umem);
1060
1061	err_bfreg:
1062	if (bfregn != MLX5_IB_INVALID_BFREG)
1063	mlx5_ib_free_bfreg(dev, bfregi: &context->bfregi, bfregn);
1064	return err;
1065	}
1066
1067	static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1068	struct mlx5_ib_qp_base *base, struct ib_udata *udata)
1069	{
1070	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1071	udata, struct mlx5_ib_ucontext, ibucontext);
1072
1073	if (udata) {
1074	/* User QP */
1075	mlx5_ib_db_unmap_user(context, db: &qp->db);
1076	ib_umem_release(umem: base->ubuffer.umem);
1077
1078	/*
1079	* Free only the BFREGs which are handled by the kernel.
1080	* BFREGs of UARs allocated dynamically are handled by user.
1081	*/
1082	if (qp->bfregn != MLX5_IB_INVALID_BFREG)
1083	mlx5_ib_free_bfreg(dev, bfregi: &context->bfregi, bfregn: qp->bfregn);
1084	return;
1085	}
1086
1087	/* Kernel QP */
1088	kvfree(addr: qp->sq.wqe_head);
1089	kvfree(addr: qp->sq.w_list);
1090	kvfree(addr: qp->sq.wrid);
1091	kvfree(addr: qp->sq.wr_data);
1092	kvfree(addr: qp->rq.wrid);
1093	if (qp->db.db)
1094	mlx5_db_free(dev: dev->mdev, db: &qp->db);
1095	if (qp->buf.frags)
1096	mlx5_frag_buf_free(dev: dev->mdev, buf: &qp->buf);
1097	}
1098
1099	static int _create_kernel_qp(struct mlx5_ib_dev *dev,
1100	struct ib_qp_init_attr *init_attr,
1101	struct mlx5_ib_qp *qp, u32 **in, int *inlen,
1102	struct mlx5_ib_qp_base *base)
1103	{
1104	int uar_index;
1105	void *qpc;
1106	int err;
1107
1108	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
1109	qp->bf.bfreg = &dev->fp_bfreg;
1110	else
1111	qp->bf.bfreg = &dev->bfreg;
1112
1113	/* We need to divide by two since each register is comprised of
1114	* two buffers of identical size, namely odd and even
1115	*/
1116	qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
1117	uar_index = qp->bf.bfreg->index;
1118
1119	err = calc_sq_size(dev, attr: init_attr, qp);
1120	if (err < 0) {
1121	mlx5_ib_dbg(dev, "err %d\n", err);
1122	return err;
1123	}
1124
1125	qp->rq.offset = 0;
1126	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
1127	base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
1128
1129	err = mlx5_frag_buf_alloc_node(dev: dev->mdev, size: base->ubuffer.buf_size,
1130	buf: &qp->buf, node: dev->mdev->priv.numa_node);
1131	if (err) {
1132	mlx5_ib_dbg(dev, "err %d\n", err);
1133	return err;
1134	}
1135
1136	if (qp->rq.wqe_cnt)
1137	mlx5_init_fbc(frags: qp->buf.frags, log_stride: qp->rq.wqe_shift,
1138	ilog2(qp->rq.wqe_cnt), fbc: &qp->rq.fbc);
1139
1140	if (qp->sq.wqe_cnt) {
1141	int sq_strides_offset = (qp->sq.offset & (PAGE_SIZE - 1)) /
1142	MLX5_SEND_WQE_BB;
1143	mlx5_init_fbc_offset(frags: qp->buf.frags +
1144	(qp->sq.offset / PAGE_SIZE),
1145	ilog2(MLX5_SEND_WQE_BB),
1146	ilog2(qp->sq.wqe_cnt),
1147	strides_offset: sq_strides_offset, fbc: &qp->sq.fbc);
1148
1149	qp->sq.cur_edge = get_sq_edge(sq: &qp->sq, idx: 0);
1150	}
1151
1152	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1153	MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1154	*in = kvzalloc(*inlen, GFP_KERNEL);
1155	if (!*in) {
1156	err = -ENOMEM;
1157	goto err_buf;
1158	}
1159
1160	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1161	MLX5_SET(qpc, qpc, uar_page, uar_index);
1162	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
1163	MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1164
1165	/* Set "fast registration enabled" for all kernel QPs */
1166	MLX5_SET(qpc, qpc, fre, 1);
1167	MLX5_SET(qpc, qpc, rlky, 1);
1168
1169	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
1170	MLX5_SET(qpc, qpc, deth_sqpn, 1);
1171
1172	mlx5_fill_page_frag_array(frag_buf: &qp->buf,
1173	pas: (__be64 *)MLX5_ADDR_OF(create_qp_in,
1174	*in, pas));
1175
1176	err = mlx5_db_alloc(dev: dev->mdev, db: &qp->db);
1177	if (err) {
1178	mlx5_ib_dbg(dev, "err %d\n", err);
1179	goto err_free;
1180	}
1181
1182	qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1183	sizeof(*qp->sq.wrid), GFP_KERNEL);
1184	qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1185	sizeof(*qp->sq.wr_data), GFP_KERNEL);
1186	qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1187	sizeof(*qp->rq.wrid), GFP_KERNEL);
1188	qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1189	sizeof(*qp->sq.w_list), GFP_KERNEL);
1190	qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1191	sizeof(*qp->sq.wqe_head), GFP_KERNEL);
1192
1193	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1194	!qp->sq.w_list || !qp->sq.wqe_head) {
1195	err = -ENOMEM;
1196	goto err_wrid;
1197	}
1198
1199	return 0;
1200
1201	err_wrid:
1202	kvfree(addr: qp->sq.wqe_head);
1203	kvfree(addr: qp->sq.w_list);
1204	kvfree(addr: qp->sq.wrid);
1205	kvfree(addr: qp->sq.wr_data);
1206	kvfree(addr: qp->rq.wrid);
1207	mlx5_db_free(dev: dev->mdev, db: &qp->db);
1208
1209	err_free:
1210	kvfree(addr: *in);
1211
1212	err_buf:
1213	mlx5_frag_buf_free(dev: dev->mdev, buf: &qp->buf);
1214	return err;
1215	}
1216
1217	static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
1218	{
1219	if (attr->srq || (qp->type == IB_QPT_XRC_TGT) ||
1220	(qp->type == MLX5_IB_QPT_DCI) || (qp->type == IB_QPT_XRC_INI))
1221	return MLX5_SRQ_RQ;
1222	else if (!qp->has_rq)
1223	return MLX5_ZERO_LEN_RQ;
1224
1225	return MLX5_NON_ZERO_RQ;
1226	}
1227
1228	static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1229	struct mlx5_ib_qp *qp,
1230	struct mlx5_ib_sq *sq, u32 tdn,
1231	struct ib_pd *pd)
1232	{
1233	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
1234	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1235
1236	MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
1237	MLX5_SET(tisc, tisc, transport_domain, tdn);
1238	if (!mlx5_ib_lag_should_assign_affinity(dev) &&
1239	mlx5_lag_is_lacp_owner(dev: dev->mdev))
1240	MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
1241	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
1242	MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1243
1244	return mlx5_core_create_tis(dev: dev->mdev, in, tisn: &sq->tisn);
1245	}
1246
1247	static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1248	struct mlx5_ib_sq *sq, struct ib_pd *pd)
1249	{
1250	mlx5_cmd_destroy_tis(dev: dev->mdev, tisn: sq->tisn, uid: to_mpd(ibpd: pd)->uid);
1251	}
1252
1253	static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
1254	{
1255	if (sq->flow_rule)
1256	mlx5_del_flow_rules(fr: sq->flow_rule);
1257	sq->flow_rule = NULL;
1258	}
1259
1260	static bool fr_supported(int ts_cap)
1261	{
1262	return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING ||
1263	ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME;
1264	}
1265
1266	static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1267	bool fr_sup, bool rt_sup)
1268	{
1269	if (cq->private_flags & MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS) {
1270	if (!rt_sup) {
1271	mlx5_ib_dbg(dev,
1272	"Real time TS format is not supported\n");
1273	return -EOPNOTSUPP;
1274	}
1275	return MLX5_TIMESTAMP_FORMAT_REAL_TIME;
1276	}
1277	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
1278	if (!fr_sup) {
1279	mlx5_ib_dbg(dev,
1280	"Free running TS format is not supported\n");
1281	return -EOPNOTSUPP;
1282	}
1283	return MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1284	}
1285	return fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1286	MLX5_TIMESTAMP_FORMAT_DEFAULT;
1287	}
1288
1289	static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq)
1290	{
1291	u8 ts_cap = MLX5_CAP_GEN(dev->mdev, rq_ts_format);
1292
1293	return get_ts_format(dev, cq: recv_cq, fr_sup: fr_supported(ts_cap),
1294	rt_sup: rt_supported(ts_cap));
1295	}
1296
1297	static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq)
1298	{
1299	u8 ts_cap = MLX5_CAP_GEN(dev->mdev, sq_ts_format);
1300
1301	return get_ts_format(dev, cq: send_cq, fr_sup: fr_supported(ts_cap),
1302	rt_sup: rt_supported(ts_cap));
1303	}
1304
1305	static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq,
1306	struct mlx5_ib_cq *recv_cq)
1307	{
1308	u8 ts_cap = MLX5_CAP_ROCE(dev->mdev, qp_ts_format);
1309	bool fr_sup = fr_supported(ts_cap);
1310	bool rt_sup = rt_supported(ts_cap);
1311	u8 default_ts = fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1312	MLX5_TIMESTAMP_FORMAT_DEFAULT;
1313	int send_ts_format =
1314	send_cq ? get_ts_format(dev, cq: send_cq, fr_sup, rt_sup) :
1315	default_ts;
1316	int recv_ts_format =
1317	recv_cq ? get_ts_format(dev, cq: recv_cq, fr_sup, rt_sup) :
1318	default_ts;
1319
1320	if (send_ts_format < 0 || recv_ts_format < 0)
1321	return -EOPNOTSUPP;
1322
1323	if (send_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1324	recv_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1325	send_ts_format != recv_ts_format) {
1326	mlx5_ib_dbg(
1327	dev,
1328	"The send ts_format does not match the receive ts_format\n");
1329	return -EOPNOTSUPP;
1330	}
1331
1332	return send_ts_format == default_ts ? recv_ts_format : send_ts_format;
1333	}
1334
1335	static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1336	struct ib_udata *udata,
1337	struct mlx5_ib_sq *sq, void *qpin,
1338	struct ib_pd *pd, struct mlx5_ib_cq *cq)
1339	{
1340	struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1341	__be64 *pas;
1342	void *in;
1343	void *sqc;
1344	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1345	void *wq;
1346	int inlen;
1347	int err;
1348	unsigned int page_offset_quantized;
1349	unsigned long page_size;
1350	int ts_format;
1351
1352	ts_format = get_sq_ts_format(dev, send_cq: cq);
1353	if (ts_format < 0)
1354	return ts_format;
1355
1356	sq->ubuffer.umem = ib_umem_get(device: &dev->ib_dev, addr: ubuffer->buf_addr,
1357	size: ubuffer->buf_size, access: 0);
1358	if (IS_ERR(ptr: sq->ubuffer.umem))
1359	return PTR_ERR(ptr: sq->ubuffer.umem);
1360	page_size = mlx5_umem_find_best_quantized_pgoff(
1361	ubuffer->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
1362	page_offset, 64, &page_offset_quantized);
1363	if (!page_size) {
1364	err = -EINVAL;
1365	goto err_umem;
1366	}
1367
1368	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1369	sizeof(u64) *
1370	ib_umem_num_dma_blocks(umem: sq->ubuffer.umem, pgsz: page_size);
1371	in = kvzalloc(inlen, GFP_KERNEL);
1372	if (!in) {
1373	err = -ENOMEM;
1374	goto err_umem;
1375	}
1376
1377	MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
1378	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1379	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1380	if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1381	MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
1382	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1383	MLX5_SET(sqc, sqc, ts_format, ts_format);
1384	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1385	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1386	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1387	MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
1388	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1389	MLX5_CAP_ETH(dev->mdev, swp))
1390	MLX5_SET(sqc, sqc, allow_swp, 1);
1391
1392	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1393	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1394	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1395	MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1396	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1397	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1398	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1399	MLX5_SET(wq, wq, log_wq_pg_sz,
1400	order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1401	MLX5_SET(wq, wq, page_offset, page_offset_quantized);
1402
1403	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1404	mlx5_ib_populate_pas(umem: sq->ubuffer.umem, page_size, pas, access_flags: 0);
1405
1406	err = mlx5_core_create_sq_tracked(dev, in, inlen, sq: &sq->base.mqp);
1407
1408	kvfree(addr: in);
1409
1410	if (err)
1411	goto err_umem;
1412
1413	return 0;
1414
1415	err_umem:
1416	ib_umem_release(umem: sq->ubuffer.umem);
1417	sq->ubuffer.umem = NULL;
1418
1419	return err;
1420	}
1421
1422	static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1423	struct mlx5_ib_sq *sq)
1424	{
1425	destroy_flow_rule_vport_sq(sq);
1426	mlx5_core_destroy_sq_tracked(dev, sq: &sq->base.mqp);
1427	ib_umem_release(umem: sq->ubuffer.umem);
1428	}
1429
1430	static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1431	struct mlx5_ib_rq *rq, void *qpin,
1432	struct ib_pd *pd, struct mlx5_ib_cq *cq)
1433	{
1434	struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
1435	__be64 *pas;
1436	void *in;
1437	void *rqc;
1438	void *wq;
1439	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1440	struct ib_umem *umem = rq->base.ubuffer.umem;
1441	unsigned int page_offset_quantized;
1442	unsigned long page_size = 0;
1443	int ts_format;
1444	size_t inlen;
1445	int err;
1446
1447	ts_format = get_rq_ts_format(dev, recv_cq: cq);
1448	if (ts_format < 0)
1449	return ts_format;
1450
1451	page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,
1452	MLX5_ADAPTER_PAGE_SHIFT,
1453	page_offset, 64,
1454	&page_offset_quantized);
1455	if (!page_size)
1456	return -EINVAL;
1457
1458	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1459	sizeof(u64) * ib_umem_num_dma_blocks(umem, pgsz: page_size);
1460	in = kvzalloc(inlen, GFP_KERNEL);
1461	if (!in)
1462	return -ENOMEM;
1463
1464	MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
1465	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1466	if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1467	MLX5_SET(rqc, rqc, vsd, 1);
1468	MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1469	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1470	MLX5_SET(rqc, rqc, ts_format, ts_format);
1471	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1472	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1473	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1474
1475	if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
1476	MLX5_SET(rqc, rqc, scatter_fcs, 1);
1477
1478	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1479	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1480	if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1481	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1482	MLX5_SET(wq, wq, page_offset, page_offset_quantized);
1483	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1484	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1485	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1486	MLX5_SET(wq, wq, log_wq_pg_sz,
1487	order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1488	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1489
1490	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1491	mlx5_ib_populate_pas(umem, page_size, pas, access_flags: 0);
1492
1493	err = mlx5_core_create_rq_tracked(dev, in, inlen, rq: &rq->base.mqp);
1494
1495	kvfree(addr: in);
1496
1497	return err;
1498	}
1499
1500	static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1501	struct mlx5_ib_rq *rq)
1502	{
1503	mlx5_core_destroy_rq_tracked(dev, rq: &rq->base.mqp);
1504	}
1505
1506	static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1507	struct mlx5_ib_rq *rq,
1508	u32 qp_flags_en,
1509	struct ib_pd *pd)
1510	{
1511	if (qp_flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1512	MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1513	mlx5_ib_disable_lb(dev, td: false, qp: true);
1514	mlx5_cmd_destroy_tir(dev: dev->mdev, tirn: rq->tirn, uid: to_mpd(ibpd: pd)->uid);
1515	}
1516
1517	static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1518	struct mlx5_ib_rq *rq, u32 tdn,
1519	u32 *qp_flags_en, struct ib_pd *pd,
1520	u32 *out)
1521	{
1522	u8 lb_flag = 0;
1523	u32 *in;
1524	void *tirc;
1525	int inlen;
1526	int err;
1527
1528	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1529	in = kvzalloc(inlen, GFP_KERNEL);
1530	if (!in)
1531	return -ENOMEM;
1532
1533	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1534	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1535	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1536	MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1537	MLX5_SET(tirc, tirc, transport_domain, tdn);
1538	if (*qp_flags_en & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1539	MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1540
1541	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1542	lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1543
1544	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1545	lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1546
1547	if (dev->is_rep) {
1548	lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1549	*qp_flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1550	}
1551
1552	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1553	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1554	err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1555	rq->tirn = MLX5_GET(create_tir_out, out, tirn);
1556	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1557	err = mlx5_ib_enable_lb(dev, td: false, qp: true);
1558
1559	if (err)
1560	destroy_raw_packet_qp_tir(dev, rq, qp_flags_en: 0, pd);
1561	}
1562	kvfree(addr: in);
1563
1564	return err;
1565	}
1566
1567	static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1568	u32 *in, size_t inlen, struct ib_pd *pd,
1569	struct ib_udata *udata,
1570	struct mlx5_ib_create_qp_resp *resp,
1571	struct ib_qp_init_attr *init_attr)
1572	{
1573	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1574	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1575	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1576	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1577	udata, struct mlx5_ib_ucontext, ibucontext);
1578	int err;
1579	u32 tdn = mucontext->tdn;
1580	u16 uid = to_mpd(ibpd: pd)->uid;
1581	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
1582
1583	if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
1584	return -EINVAL;
1585	if (qp->sq.wqe_cnt) {
1586	err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
1587	if (err)
1588	return err;
1589
1590	err = create_raw_packet_qp_sq(dev, udata, sq, qpin: in, pd,
1591	cq: to_mcq(ibcq: init_attr->send_cq));
1592	if (err)
1593	goto err_destroy_tis;
1594
1595	if (uid) {
1596	resp->tisn = sq->tisn;
1597	resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TISN;
1598	resp->sqn = sq->base.mqp.qpn;
1599	resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
1600	}
1601
1602	sq->base.container_mibqp = qp;
1603	sq->base.mqp.event = mlx5_ib_qp_event;
1604	}
1605
1606	if (qp->rq.wqe_cnt) {
1607	rq->base.container_mibqp = qp;
1608
1609	if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
1610	rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
1611	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
1612	rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
1613	err = create_raw_packet_qp_rq(dev, rq, qpin: in, pd,
1614	cq: to_mcq(ibcq: init_attr->recv_cq));
1615	if (err)
1616	goto err_destroy_sq;
1617
1618	err = create_raw_packet_qp_tir(dev, rq, tdn, qp_flags_en: &qp->flags_en, pd,
1619	out);
1620	if (err)
1621	goto err_destroy_rq;
1622
1623	if (uid) {
1624	resp->rqn = rq->base.mqp.qpn;
1625	resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
1626	resp->tirn = rq->tirn;
1627	resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1628	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1629	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1630	resp->tir_icm_addr = MLX5_GET(
1631	create_tir_out, out, icm_address_31_0);
1632	resp->tir_icm_addr |=
1633	(u64)MLX5_GET(create_tir_out, out,
1634	icm_address_39_32)
1635	<< 32;
1636	resp->tir_icm_addr |=
1637	(u64)MLX5_GET(create_tir_out, out,
1638	icm_address_63_40)
1639	<< 40;
1640	resp->comp_mask |=
1641	MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1642	}
1643	}
1644	}
1645
1646	qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1647	rq->base.mqp.qpn;
1648	return 0;
1649
1650	err_destroy_rq:
1651	destroy_raw_packet_qp_rq(dev, rq);
1652	err_destroy_sq:
1653	if (!qp->sq.wqe_cnt)
1654	return err;
1655	destroy_raw_packet_qp_sq(dev, sq);
1656	err_destroy_tis:
1657	destroy_raw_packet_qp_tis(dev, sq, pd);
1658
1659	return err;
1660	}
1661
1662	static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1663	struct mlx5_ib_qp *qp)
1664	{
1665	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1666	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1667	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1668
1669	if (qp->rq.wqe_cnt) {
1670	destroy_raw_packet_qp_tir(dev, rq, qp_flags_en: qp->flags_en, pd: qp->ibqp.pd);
1671	destroy_raw_packet_qp_rq(dev, rq);
1672	}
1673
1674	if (qp->sq.wqe_cnt) {
1675	destroy_raw_packet_qp_sq(dev, sq);
1676	destroy_raw_packet_qp_tis(dev, sq, pd: qp->ibqp.pd);
1677	}
1678	}
1679
1680	static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1681	struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1682	{
1683	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1684	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1685
1686	sq->sq = &qp->sq;
1687	rq->rq = &qp->rq;
1688	sq->doorbell = &qp->db;
1689	rq->doorbell = &qp->db;
1690	}
1691
1692	static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1693	{
1694	if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1695	MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1696	mlx5_ib_disable_lb(dev, td: false, qp: true);
1697	mlx5_cmd_destroy_tir(dev: dev->mdev, tirn: qp->rss_qp.tirn,
1698	uid: to_mpd(ibpd: qp->ibqp.pd)->uid);
1699	}
1700
1701	struct mlx5_create_qp_params {
1702	struct ib_udata *udata;
1703	size_t inlen;
1704	size_t outlen;
1705	size_t ucmd_size;
1706	void *ucmd;
1707	u8 is_rss_raw : 1;
1708	struct ib_qp_init_attr *attr;
1709	u32 uidx;
1710	struct mlx5_ib_create_qp_resp resp;
1711	};
1712
1713	static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1714	struct mlx5_ib_qp *qp,
1715	struct mlx5_create_qp_params *params)
1716	{
1717	struct ib_qp_init_attr *init_attr = params->attr;
1718	struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
1719	struct ib_udata *udata = params->udata;
1720	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1721	udata, struct mlx5_ib_ucontext, ibucontext);
1722	int inlen;
1723	int outlen;
1724	int err;
1725	u32 *in;
1726	u32 *out;
1727	void *tirc;
1728	void *hfso;
1729	u32 selected_fields = 0;
1730	u32 outer_l4;
1731	u32 tdn = mucontext->tdn;
1732	u8 lb_flag = 0;
1733
1734	if (ucmd->comp_mask) {
1735	mlx5_ib_dbg(dev, "invalid comp mask\n");
1736	return -EOPNOTSUPP;
1737	}
1738
1739	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1740	!(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
1741	mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1742	return -EOPNOTSUPP;
1743	}
1744
1745	if (dev->is_rep)
1746	qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1747
1748	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1749	lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1750
1751	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1752	lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1753
1754	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1755	outlen = MLX5_ST_SZ_BYTES(create_tir_out);
1756	in = kvzalloc(inlen + outlen, GFP_KERNEL);
1757	if (!in)
1758	return -ENOMEM;
1759
1760	out = in + MLX5_ST_SZ_DW(create_tir_in);
1761	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1762	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1763	MLX5_SET(tirc, tirc, disp_type,
1764	MLX5_TIRC_DISP_TYPE_INDIRECT);
1765	MLX5_SET(tirc, tirc, indirect_table,
1766	init_attr->rwq_ind_tbl->ind_tbl_num);
1767	MLX5_SET(tirc, tirc, transport_domain, tdn);
1768
1769	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1770
1771	if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1772	MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1773
1774	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1775
1776	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
1777	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1778	else
1779	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1780
1781	switch (ucmd->rx_hash_function) {
1782	case MLX5_RX_HASH_FUNC_TOEPLITZ:
1783	{
1784	void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1785	size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1786
1787	if (len != ucmd->rx_key_len) {
1788	err = -EINVAL;
1789	goto err;
1790	}
1791
1792	MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1793	memcpy(rss_key, ucmd->rx_hash_key, len);
1794	break;
1795	}
1796	default:
1797	err = -EOPNOTSUPP;
1798	goto err;
1799	}
1800
1801	if (!ucmd->rx_hash_fields_mask) {
1802	/* special case when this TIR serves as steering entry without hashing */
1803	if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1804	goto create_tir;
1805	err = -EINVAL;
1806	goto err;
1807	}
1808
1809	if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1810	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1811	((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1812	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1813	err = -EINVAL;
1814	goto err;
1815	}
1816
1817	/* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1818	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1819	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1820	MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1821	MLX5_L3_PROT_TYPE_IPV4);
1822	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1823	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1824	MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1825	MLX5_L3_PROT_TYPE_IPV6);
1826
1827	outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1828	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1829	<< 0 |
1830	((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1831	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1832	<< 1 |
1833	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
1834
1835	/* Check that only one l4 protocol is set */
1836	if (outer_l4 & (outer_l4 - 1)) {
1837	err = -EINVAL;
1838	goto err;
1839	}
1840
1841	/* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1842	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1843	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1844	MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1845	MLX5_L4_PROT_TYPE_TCP);
1846	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1847	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1848	MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1849	MLX5_L4_PROT_TYPE_UDP);
1850
1851	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1852	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1853	selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1854
1855	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1856	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1857	selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1858
1859	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1860	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1861	selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1862
1863	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1864	(ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1865	selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1866
1867	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
1868	selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1869
1870	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1871
1872	create_tir:
1873	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1874	err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1875
1876	qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
1877	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1878	err = mlx5_ib_enable_lb(dev, td: false, qp: true);
1879
1880	if (err)
1881	mlx5_cmd_destroy_tir(dev: dev->mdev, tirn: qp->rss_qp.tirn,
1882	uid: to_mpd(ibpd: pd)->uid);
1883	}
1884
1885	if (err)
1886	goto err;
1887
1888	if (mucontext->devx_uid) {
1889	params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1890	params->resp.tirn = qp->rss_qp.tirn;
1891	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1892	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1893	params->resp.tir_icm_addr =
1894	MLX5_GET(create_tir_out, out, icm_address_31_0);
1895	params->resp.tir_icm_addr |=
1896	(u64)MLX5_GET(create_tir_out, out,
1897	icm_address_39_32)
1898	<< 32;
1899	params->resp.tir_icm_addr |=
1900	(u64)MLX5_GET(create_tir_out, out,
1901	icm_address_63_40)
1902	<< 40;
1903	params->resp.comp_mask |=
1904	MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1905	}
1906	}
1907
1908	kvfree(addr: in);
1909	/* qpn is reserved for that QP */
1910	qp->trans_qp.base.mqp.qpn = 0;
1911	qp->is_rss = true;
1912	return 0;
1913
1914	err:
1915	kvfree(addr: in);
1916	return err;
1917	}
1918
1919	static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
1920	struct mlx5_ib_qp *qp,
1921	struct ib_qp_init_attr *init_attr,
1922	void *qpc)
1923	{
1924	int scqe_sz;
1925	bool allow_scat_cqe = false;
1926
1927	allow_scat_cqe = qp->flags_en & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
1928
1929	if (!allow_scat_cqe && init_attr->sq_sig_type != IB_SIGNAL_ALL_WR)
1930	return;
1931
1932	scqe_sz = mlx5_ib_get_cqe_size(ibcq: init_attr->send_cq);
1933	if (scqe_sz == 128) {
1934	MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1935	return;
1936	}
1937
1938	if (init_attr->qp_type != MLX5_IB_QPT_DCI ||
1939	MLX5_CAP_GEN(dev->mdev, dc_req_scat_data_cqe))
1940	MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1941	}
1942
1943	static int atomic_size_to_mode(int size_mask)
1944	{
1945	/* driver does not support atomic_size > 256B
1946	* and does not know how to translate bigger sizes
1947	*/
1948	int supported_size_mask = size_mask & 0x1ff;
1949	int log_max_size;
1950
1951	if (!supported_size_mask)
1952	return -EOPNOTSUPP;
1953
1954	log_max_size = __fls(word: supported_size_mask);
1955
1956	if (log_max_size > 3)
1957	return log_max_size;
1958
1959	return MLX5_ATOMIC_MODE_8B;
1960	}
1961
1962	static int get_atomic_mode(struct mlx5_ib_dev *dev,
1963	struct mlx5_ib_qp *qp)
1964	{
1965	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
1966	u8 atomic = MLX5_CAP_GEN(dev->mdev, atomic);
1967	int atomic_mode = -EOPNOTSUPP;
1968	int atomic_size_mask;
1969
1970	if (!atomic)
1971	return -EOPNOTSUPP;
1972
1973	if (qp->type == MLX5_IB_QPT_DCT)
1974	atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
1975	else
1976	atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
1977
1978	if ((atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP) ||
1979	(atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD))
1980	atomic_mode = atomic_size_to_mode(size_mask: atomic_size_mask);
1981
1982	if (atomic_mode <= 0 &&
1983	(atomic_operations & MLX5_ATOMIC_OPS_CMP_SWAP &&
1984	atomic_operations & MLX5_ATOMIC_OPS_FETCH_ADD))
1985	atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;
1986
1987	/* OOO DP QPs do not support larger than 8-Bytes atomic operations */
1988	if (atomic_mode > MLX5_ATOMIC_MODE_8B && qp->is_ooo_rq)
1989	atomic_mode = MLX5_ATOMIC_MODE_8B;
1990
1991	return atomic_mode;
1992	}
1993
1994	static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1995	struct mlx5_create_qp_params *params)
1996	{
1997	struct ib_qp_init_attr *attr = params->attr;
1998	u32 uidx = params->uidx;
1999	struct mlx5_ib_resources *devr = &dev->devr;
2000	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2001	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2002	struct mlx5_core_dev *mdev = dev->mdev;
2003	struct mlx5_ib_qp_base *base;
2004	unsigned long flags;
2005	void *qpc;
2006	u32 *in;
2007	int err;
2008
2009	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2010	qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2011
2012	in = kvzalloc(inlen, GFP_KERNEL);
2013	if (!in)
2014	return -ENOMEM;
2015
2016	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2017
2018	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
2019	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2020	MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
2021
2022	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2023	MLX5_SET(qpc, qpc, block_lb_mc, 1);
2024	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2025	MLX5_SET(qpc, qpc, cd_master, 1);
2026	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2027	MLX5_SET(qpc, qpc, cd_slave_send, 1);
2028	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2029	MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2030
2031	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
2032	MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
2033	MLX5_SET(qpc, qpc, no_sq, 1);
2034	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2035	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
2036	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2037	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
2038	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2039
2040	/* 0xffffff means we ask to work with cqe version 0 */
2041	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2042	MLX5_SET(qpc, qpc, user_index, uidx);
2043
2044	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2045	MLX5_SET(qpc, qpc, end_padding_mode,
2046	MLX5_WQ_END_PAD_MODE_ALIGN);
2047	/* Special case to clean flag */
2048	qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2049	}
2050
2051	base = &qp->trans_qp.base;
2052	err = mlx5_qpc_create_qp(dev, qp: &base->mqp, in, inlen, out);
2053	kvfree(addr: in);
2054	if (err)
2055	return err;
2056
2057	base->container_mibqp = qp;
2058	base->mqp.event = mlx5_ib_qp_event;
2059	if (MLX5_CAP_GEN(mdev, ece_support))
2060	params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2061
2062	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2063	list_add_tail(new: &qp->qps_list, head: &dev->qp_list);
2064	spin_unlock_irqrestore(lock: &dev->reset_flow_resource_lock, flags);
2065
2066	qp->trans_qp.xrcdn = to_mxrcd(ibxrcd: attr->xrcd)->xrcdn;
2067	return 0;
2068	}
2069
2070	static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2071	struct mlx5_ib_qp *qp,
2072	struct mlx5_create_qp_params *params)
2073	{
2074	struct ib_qp_init_attr *init_attr = params->attr;
2075	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2076	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2077	struct ib_udata *udata = params->udata;
2078	u32 uidx = params->uidx;
2079	struct mlx5_ib_resources *devr = &dev->devr;
2080	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2081	struct mlx5_core_dev *mdev = dev->mdev;
2082	struct mlx5_ib_cq *send_cq;
2083	struct mlx5_ib_cq *recv_cq;
2084	unsigned long flags;
2085	struct mlx5_ib_qp_base *base;
2086	int ts_format;
2087	int mlx5_st;
2088	void *qpc;
2089	u32 *in;
2090	int err;
2091
2092	spin_lock_init(&qp->sq.lock);
2093	spin_lock_init(&qp->rq.lock);
2094
2095	mlx5_st = to_mlx5_st(type: qp->type);
2096	if (mlx5_st < 0)
2097	return -EINVAL;
2098
2099	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2100	qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2101
2102	base = &qp->trans_qp.base;
2103
2104	qp->has_rq = qp_has_rq(attr: init_attr);
2105	err = set_rq_size(dev, cap: &init_attr->cap, has_rq: qp->has_rq, qp, ucmd);
2106	if (err) {
2107	mlx5_ib_dbg(dev, "err %d\n", err);
2108	return err;
2109	}
2110
2111	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2112	ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2113	return -EINVAL;
2114
2115	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2116	return -EINVAL;
2117
2118	ts_format = get_qp_ts_format(dev, send_cq: to_mcq(ibcq: init_attr->send_cq),
2119	recv_cq: to_mcq(ibcq: init_attr->recv_cq));
2120
2121	if (ts_format < 0)
2122	return ts_format;
2123
2124	err = _create_user_qp(dev, pd, qp, udata, attr: init_attr, in: &in, resp: &params->resp,
2125	inlen: &inlen, base, ucmd);
2126	if (err)
2127	return err;
2128
2129	if (MLX5_CAP_GEN(mdev, ece_support))
2130	MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2131	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2132
2133	MLX5_SET(qpc, qpc, st, mlx5_st);
2134	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2135	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2136
2137	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2138	MLX5_SET(qpc, qpc, wq_signature, 1);
2139
2140	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2141	MLX5_SET(qpc, qpc, cd_master, 1);
2142	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2143	MLX5_SET(qpc, qpc, cd_slave_send, 1);
2144	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE)
2145	configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2146
2147	if (qp->rq.wqe_cnt) {
2148	MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2149	MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2150	}
2151
2152	if (qp->flags_en & MLX5_QP_FLAG_DCI_STREAM) {
2153	MLX5_SET(qpc, qpc, log_num_dci_stream_channels,
2154	ucmd->dci_streams.log_num_concurent);
2155	MLX5_SET(qpc, qpc, log_num_dci_errored_streams,
2156	ucmd->dci_streams.log_num_errored);
2157	}
2158
2159	MLX5_SET(qpc, qpc, ts_format, ts_format);
2160	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2161
2162	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2163
2164	/* Set default resources */
2165	if (init_attr->srq) {
2166	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2167	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2168	to_msrq(init_attr->srq)->msrq.srqn);
2169	} else {
2170	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2171	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2172	to_msrq(devr->s1)->msrq.srqn);
2173	}
2174
2175	if (init_attr->send_cq)
2176	MLX5_SET(qpc, qpc, cqn_snd,
2177	to_mcq(init_attr->send_cq)->mcq.cqn);
2178
2179	if (init_attr->recv_cq)
2180	MLX5_SET(qpc, qpc, cqn_rcv,
2181	to_mcq(init_attr->recv_cq)->mcq.cqn);
2182
2183	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2184
2185	/* 0xffffff means we ask to work with cqe version 0 */
2186	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2187	MLX5_SET(qpc, qpc, user_index, uidx);
2188
2189	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2190	MLX5_SET(qpc, qpc, end_padding_mode,
2191	MLX5_WQ_END_PAD_MODE_ALIGN);
2192	/* Special case to clean flag */
2193	qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2194	}
2195
2196	err = mlx5_qpc_create_qp(dev, qp: &base->mqp, in, inlen, out);
2197
2198	kvfree(addr: in);
2199	if (err)
2200	goto err_create;
2201
2202	base->container_mibqp = qp;
2203	base->mqp.event = mlx5_ib_qp_event;
2204	if (MLX5_CAP_GEN(mdev, ece_support))
2205	params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2206
2207	get_cqs(qp_type: qp->type, ib_send_cq: init_attr->send_cq, ib_recv_cq: init_attr->recv_cq,
2208	send_cq: &send_cq, recv_cq: &recv_cq);
2209	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2210	mlx5_ib_lock_cqs(send_cq, recv_cq);
2211	/* Maintain device to QPs access, needed for further handling via reset
2212	* flow
2213	*/
2214	list_add_tail(new: &qp->qps_list, head: &dev->qp_list);
2215	/* Maintain CQ to QPs access, needed for further handling via reset flow
2216	*/
2217	if (send_cq)
2218	list_add_tail(new: &qp->cq_send_list, head: &send_cq->list_send_qp);
2219	if (recv_cq)
2220	list_add_tail(new: &qp->cq_recv_list, head: &recv_cq->list_recv_qp);
2221	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2222	spin_unlock_irqrestore(lock: &dev->reset_flow_resource_lock, flags);
2223
2224	return 0;
2225
2226	err_create:
2227	destroy_qp(dev, qp, base, udata);
2228	return err;
2229	}
2230
2231	static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2232	struct mlx5_ib_qp *qp,
2233	struct mlx5_create_qp_params *params)
2234	{
2235	struct ib_qp_init_attr *init_attr = params->attr;
2236	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2237	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2238	struct ib_udata *udata = params->udata;
2239	u32 uidx = params->uidx;
2240	struct mlx5_ib_resources *devr = &dev->devr;
2241	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2242	struct mlx5_core_dev *mdev = dev->mdev;
2243	struct mlx5_ib_cq *send_cq;
2244	struct mlx5_ib_cq *recv_cq;
2245	unsigned long flags;
2246	struct mlx5_ib_qp_base *base;
2247	int ts_format;
2248	int mlx5_st;
2249	void *qpc;
2250	u32 *in;
2251	int err;
2252
2253	spin_lock_init(&qp->sq.lock);
2254	spin_lock_init(&qp->rq.lock);
2255
2256	mlx5_st = to_mlx5_st(type: qp->type);
2257	if (mlx5_st < 0)
2258	return -EINVAL;
2259
2260	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2261	qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2262
2263	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
2264	qp->underlay_qpn = init_attr->source_qpn;
2265
2266	base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2267	qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
2268	&qp->raw_packet_qp.rq.base :
2269	&qp->trans_qp.base;
2270
2271	qp->has_rq = qp_has_rq(attr: init_attr);
2272	err = set_rq_size(dev, cap: &init_attr->cap, has_rq: qp->has_rq, qp, ucmd);
2273	if (err) {
2274	mlx5_ib_dbg(dev, "err %d\n", err);
2275	return err;
2276	}
2277
2278	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2279	ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2280	return -EINVAL;
2281
2282	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2283	return -EINVAL;
2284
2285	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
2286	ts_format = get_qp_ts_format(dev, send_cq: to_mcq(ibcq: init_attr->send_cq),
2287	recv_cq: to_mcq(ibcq: init_attr->recv_cq));
2288	if (ts_format < 0)
2289	return ts_format;
2290	}
2291
2292	err = _create_user_qp(dev, pd, qp, udata, attr: init_attr, in: &in, resp: &params->resp,
2293	inlen: &inlen, base, ucmd);
2294	if (err)
2295	return err;
2296
2297	if (is_sqp(qp_type: init_attr->qp_type))
2298	qp->port = init_attr->port_num;
2299
2300	if (MLX5_CAP_GEN(mdev, ece_support))
2301	MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2302	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2303
2304	MLX5_SET(qpc, qpc, st, mlx5_st);
2305	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2306	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2307
2308	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2309	MLX5_SET(qpc, qpc, wq_signature, 1);
2310
2311	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2312	MLX5_SET(qpc, qpc, block_lb_mc, 1);
2313
2314	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2315	MLX5_SET(qpc, qpc, cd_master, 1);
2316	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2317	MLX5_SET(qpc, qpc, cd_slave_send, 1);
2318	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2319	MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2320	if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
2321	MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
2322	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2323	(init_attr->qp_type == IB_QPT_RC ||
2324	init_attr->qp_type == IB_QPT_UC)) {
2325	int rcqe_sz = mlx5_ib_get_cqe_size(ibcq: init_attr->recv_cq);
2326
2327	MLX5_SET(qpc, qpc, cs_res,
2328	rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
2329	MLX5_RES_SCAT_DATA32_CQE);
2330	}
2331	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2332	(qp->type == MLX5_IB_QPT_DCI || qp->type == IB_QPT_RC))
2333	configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2334
2335	if (qp->rq.wqe_cnt) {
2336	MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2337	MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2338	}
2339
2340	if (init_attr->qp_type != IB_QPT_RAW_PACKET)
2341	MLX5_SET(qpc, qpc, ts_format, ts_format);
2342
2343	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2344
2345	if (qp->sq.wqe_cnt) {
2346	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2347	} else {
2348	MLX5_SET(qpc, qpc, no_sq, 1);
2349	if (init_attr->srq &&
2350	init_attr->srq->srq_type == IB_SRQT_TM)
2351	MLX5_SET(qpc, qpc, offload_type,
2352	MLX5_QPC_OFFLOAD_TYPE_RNDV);
2353	}
2354
2355	/* Set default resources */
2356	switch (init_attr->qp_type) {
2357	case IB_QPT_XRC_INI:
2358	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2359	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2360	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2361	break;
2362	default:
2363	if (init_attr->srq) {
2364	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2365	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
2366	} else {
2367	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2368	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
2369	}
2370	}
2371
2372	if (init_attr->send_cq)
2373	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
2374
2375	if (init_attr->recv_cq)
2376	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
2377
2378	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2379
2380	/* 0xffffff means we ask to work with cqe version 0 */
2381	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2382	MLX5_SET(qpc, qpc, user_index, uidx);
2383
2384	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
2385	init_attr->qp_type != IB_QPT_RAW_PACKET) {
2386	MLX5_SET(qpc, qpc, end_padding_mode,
2387	MLX5_WQ_END_PAD_MODE_ALIGN);
2388	/* Special case to clean flag */
2389	qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2390	}
2391
2392	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2393	qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2394	qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
2395	raw_packet_qp_copy_info(qp, raw_packet_qp: &qp->raw_packet_qp);
2396	err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
2397	resp: &params->resp, init_attr);
2398	} else
2399	err = mlx5_qpc_create_qp(dev, qp: &base->mqp, in, inlen, out);
2400
2401	kvfree(addr: in);
2402	if (err)
2403	goto err_create;
2404
2405	base->container_mibqp = qp;
2406	base->mqp.event = mlx5_ib_qp_event;
2407	if (MLX5_CAP_GEN(mdev, ece_support))
2408	params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2409
2410	get_cqs(qp_type: qp->type, ib_send_cq: init_attr->send_cq, ib_recv_cq: init_attr->recv_cq,
2411	send_cq: &send_cq, recv_cq: &recv_cq);
2412	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2413	mlx5_ib_lock_cqs(send_cq, recv_cq);
2414	/* Maintain device to QPs access, needed for further handling via reset
2415	* flow
2416	*/
2417	list_add_tail(new: &qp->qps_list, head: &dev->qp_list);
2418	/* Maintain CQ to QPs access, needed for further handling via reset flow
2419	*/
2420	if (send_cq)
2421	list_add_tail(new: &qp->cq_send_list, head: &send_cq->list_send_qp);
2422	if (recv_cq)
2423	list_add_tail(new: &qp->cq_recv_list, head: &recv_cq->list_recv_qp);
2424	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2425	spin_unlock_irqrestore(lock: &dev->reset_flow_resource_lock, flags);
2426
2427	return 0;
2428
2429	err_create:
2430	destroy_qp(dev, qp, base, udata);
2431	return err;
2432	}
2433
2434	static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2435	struct mlx5_ib_qp *qp,
2436	struct mlx5_create_qp_params *params)
2437	{
2438	struct ib_qp_init_attr *attr = params->attr;
2439	u32 uidx = params->uidx;
2440	struct mlx5_ib_resources *devr = &dev->devr;
2441	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2442	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2443	struct mlx5_core_dev *mdev = dev->mdev;
2444	struct mlx5_ib_cq *send_cq;
2445	struct mlx5_ib_cq *recv_cq;
2446	unsigned long flags;
2447	struct mlx5_ib_qp_base *base;
2448	int mlx5_st;
2449	void *qpc;
2450	u32 *in;
2451	int err;
2452
2453	spin_lock_init(&qp->sq.lock);
2454	spin_lock_init(&qp->rq.lock);
2455
2456	mlx5_st = to_mlx5_st(type: qp->type);
2457	if (mlx5_st < 0)
2458	return -EINVAL;
2459
2460	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2461	qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2462
2463	base = &qp->trans_qp.base;
2464
2465	qp->has_rq = qp_has_rq(attr);
2466	err = set_rq_size(dev, cap: &attr->cap, has_rq: qp->has_rq, qp, NULL);
2467	if (err) {
2468	mlx5_ib_dbg(dev, "err %d\n", err);
2469	return err;
2470	}
2471
2472	err = _create_kernel_qp(dev, init_attr: attr, qp, in: &in, inlen: &inlen, base);
2473	if (err)
2474	return err;
2475
2476	if (is_sqp(qp_type: attr->qp_type))
2477	qp->port = attr->port_num;
2478
2479	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2480
2481	MLX5_SET(qpc, qpc, st, mlx5_st);
2482	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2483
2484	if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
2485	MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
2486	else
2487	MLX5_SET(qpc, qpc, latency_sensitive, 1);
2488
2489
2490	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2491	MLX5_SET(qpc, qpc, block_lb_mc, 1);
2492
2493	if (qp->rq.wqe_cnt) {
2494	MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2495	MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2496	}
2497
2498	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
2499
2500	if (qp->sq.wqe_cnt)
2501	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2502	else
2503	MLX5_SET(qpc, qpc, no_sq, 1);
2504
2505	if (attr->srq) {
2506	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2507	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2508	to_msrq(attr->srq)->msrq.srqn);
2509	} else {
2510	MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2511	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2512	to_msrq(devr->s1)->msrq.srqn);
2513	}
2514
2515	if (attr->send_cq)
2516	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
2517
2518	if (attr->recv_cq)
2519	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
2520
2521	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2522
2523	/* 0xffffff means we ask to work with cqe version 0 */
2524	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2525	MLX5_SET(qpc, qpc, user_index, uidx);
2526
2527	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
2528	if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
2529	MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
2530
2531	if (qp->flags & IB_QP_CREATE_INTEGRITY_EN &&
2532	MLX5_CAP_GEN(mdev, go_back_n))
2533	MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
2534
2535	err = mlx5_qpc_create_qp(dev, qp: &base->mqp, in, inlen, out);
2536	kvfree(addr: in);
2537	if (err)
2538	goto err_create;
2539
2540	base->container_mibqp = qp;
2541	base->mqp.event = mlx5_ib_qp_event;
2542
2543	get_cqs(qp_type: qp->type, ib_send_cq: attr->send_cq, ib_recv_cq: attr->recv_cq,
2544	send_cq: &send_cq, recv_cq: &recv_cq);
2545	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2546	mlx5_ib_lock_cqs(send_cq, recv_cq);
2547	/* Maintain device to QPs access, needed for further handling via reset
2548	* flow
2549	*/
2550	list_add_tail(new: &qp->qps_list, head: &dev->qp_list);
2551	/* Maintain CQ to QPs access, needed for further handling via reset flow
2552	*/
2553	if (send_cq)
2554	list_add_tail(new: &qp->cq_send_list, head: &send_cq->list_send_qp);
2555	if (recv_cq)
2556	list_add_tail(new: &qp->cq_recv_list, head: &recv_cq->list_recv_qp);
2557	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2558	spin_unlock_irqrestore(lock: &dev->reset_flow_resource_lock, flags);
2559
2560	return 0;
2561
2562	err_create:
2563	destroy_qp(dev, qp, base, NULL);
2564	return err;
2565	}
2566
2567	static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2568	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
2569	{
2570	if (send_cq) {
2571	if (recv_cq) {
2572	if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
2573	spin_lock(lock: &send_cq->lock);
2574	spin_lock_nested(&recv_cq->lock,
2575	SINGLE_DEPTH_NESTING);
2576	} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2577	spin_lock(lock: &send_cq->lock);
2578	__acquire(&recv_cq->lock);
2579	} else {
2580	spin_lock(lock: &recv_cq->lock);
2581	spin_lock_nested(&send_cq->lock,
2582	SINGLE_DEPTH_NESTING);
2583	}
2584	} else {
2585	spin_lock(lock: &send_cq->lock);
2586	__acquire(&recv_cq->lock);
2587	}
2588	} else if (recv_cq) {
2589	spin_lock(lock: &recv_cq->lock);
2590	__acquire(&send_cq->lock);
2591	} else {
2592	__acquire(&send_cq->lock);
2593	__acquire(&recv_cq->lock);
2594	}
2595	}
2596
2597	static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2598	__releases(&send_cq->lock) __releases(&recv_cq->lock)
2599	{
2600	if (send_cq) {
2601	if (recv_cq) {
2602	if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
2603	spin_unlock(lock: &recv_cq->lock);
2604	spin_unlock(lock: &send_cq->lock);
2605	} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2606	__release(&recv_cq->lock);
2607	spin_unlock(lock: &send_cq->lock);
2608	} else {
2609	spin_unlock(lock: &send_cq->lock);
2610	spin_unlock(lock: &recv_cq->lock);
2611	}
2612	} else {
2613	__release(&recv_cq->lock);
2614	spin_unlock(lock: &send_cq->lock);
2615	}
2616	} else if (recv_cq) {
2617	__release(&send_cq->lock);
2618	spin_unlock(lock: &recv_cq->lock);
2619	} else {
2620	__release(&recv_cq->lock);
2621	__release(&send_cq->lock);
2622	}
2623	}
2624
2625	static void get_cqs(enum ib_qp_type qp_type,
2626	struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
2627	struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2628	{
2629	switch (qp_type) {
2630	case IB_QPT_XRC_TGT:
2631	*send_cq = NULL;
2632	*recv_cq = NULL;
2633	break;
2634	case MLX5_IB_QPT_REG_UMR:
2635	case IB_QPT_XRC_INI:
2636	*send_cq = ib_send_cq ? to_mcq(ibcq: ib_send_cq) : NULL;
2637	*recv_cq = NULL;
2638	break;
2639
2640	case IB_QPT_SMI:
2641	case MLX5_IB_QPT_HW_GSI:
2642	case IB_QPT_RC:
2643	case IB_QPT_UC:
2644	case IB_QPT_UD:
2645	case IB_QPT_RAW_PACKET:
2646	*send_cq = ib_send_cq ? to_mcq(ibcq: ib_send_cq) : NULL;
2647	*recv_cq = ib_recv_cq ? to_mcq(ibcq: ib_recv_cq) : NULL;
2648	break;
2649	default:
2650	*send_cq = NULL;
2651	*recv_cq = NULL;
2652	break;
2653	}
2654	}
2655
2656	static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2657	const struct mlx5_modify_raw_qp_param *raw_qp_param,
2658	u8 lag_tx_affinity);
2659
2660	static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2661	struct ib_udata *udata)
2662	{
2663	struct mlx5_ib_cq *send_cq, *recv_cq;
2664	struct mlx5_ib_qp_base *base;
2665	unsigned long flags;
2666	int err;
2667
2668	if (qp->is_rss) {
2669	destroy_rss_raw_qp_tir(dev, qp);
2670	return;
2671	}
2672
2673	base = (qp->type == IB_QPT_RAW_PACKET ||
2674	qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
2675	&qp->raw_packet_qp.rq.base :
2676	&qp->trans_qp.base;
2677
2678	if (qp->state != IB_QPS_RESET) {
2679	if (qp->type != IB_QPT_RAW_PACKET &&
2680	!(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
2681	err = mlx5_core_qp_modify(dev, opcode: MLX5_CMD_OP_2RST_QP, opt_param_mask: 0,
2682	NULL, qp: &base->mqp, NULL);
2683	} else {
2684	struct mlx5_modify_raw_qp_param raw_qp_param = {
2685	.operation = MLX5_CMD_OP_2RST_QP
2686	};
2687
2688	err = modify_raw_packet_qp(dev, qp, raw_qp_param: &raw_qp_param, lag_tx_affinity: 0);
2689	}
2690	if (err)
2691	mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
2692	base->mqp.qpn);
2693	}
2694
2695	get_cqs(qp_type: qp->type, ib_send_cq: qp->ibqp.send_cq, ib_recv_cq: qp->ibqp.recv_cq, send_cq: &send_cq,
2696	recv_cq: &recv_cq);
2697
2698	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2699	mlx5_ib_lock_cqs(send_cq, recv_cq);
2700	/* del from lists under both locks above to protect reset flow paths */
2701	list_del(entry: &qp->qps_list);
2702	if (send_cq)
2703	list_del(entry: &qp->cq_send_list);
2704
2705	if (recv_cq)
2706	list_del(entry: &qp->cq_recv_list);
2707
2708	if (!udata) {
2709	__mlx5_ib_cq_clean(cq: recv_cq, qpn: base->mqp.qpn,
2710	srq: qp->ibqp.srq ? to_msrq(ibsrq: qp->ibqp.srq) : NULL);
2711	if (send_cq != recv_cq)
2712	__mlx5_ib_cq_clean(cq: send_cq, qpn: base->mqp.qpn,
2713	NULL);
2714	}
2715	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2716	spin_unlock_irqrestore(lock: &dev->reset_flow_resource_lock, flags);
2717
2718	if (qp->type == IB_QPT_RAW_PACKET ||
2719	qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2720	destroy_raw_packet_qp(dev, qp);
2721	} else {
2722	err = mlx5_core_destroy_qp(dev, qp: &base->mqp);
2723	if (err)
2724	mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2725	base->mqp.qpn);
2726	}
2727
2728	destroy_qp(dev, qp, base, udata);
2729	}
2730
2731	static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2732	struct mlx5_ib_qp *qp,
2733	struct mlx5_create_qp_params *params)
2734	{
2735	struct ib_qp_init_attr *attr = params->attr;
2736	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2737	u32 uidx = params->uidx;
2738	void *dctc;
2739
2740	if (mlx5_lag_is_active(dev: dev->mdev) && !MLX5_CAP_GEN(dev->mdev, lag_dct))
2741	return -EOPNOTSUPP;
2742
2743	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
2744	if (!qp->dct.in)
2745	return -ENOMEM;
2746
2747	MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
2748	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
2749	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2750	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2751	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2752	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2753	MLX5_SET(dctc, dctc, user_index, uidx);
2754	if (MLX5_CAP_GEN(dev->mdev, ece_support))
2755	MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
2756
2757	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
2758	int rcqe_sz = mlx5_ib_get_cqe_size(ibcq: attr->recv_cq);
2759
2760	if (rcqe_sz == 128)
2761	MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
2762	}
2763
2764	qp->state = IB_QPS_RESET;
2765	return 0;
2766	}
2767
2768	static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
2769	enum ib_qp_type *type)
2770	{
2771	if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
2772	goto out;
2773
2774	switch (attr->qp_type) {
2775	case IB_QPT_XRC_TGT:
2776	case IB_QPT_XRC_INI:
2777	if (!MLX5_CAP_GEN(dev->mdev, xrc))
2778	goto out;
2779	fallthrough;
2780	case IB_QPT_RC:
2781	case IB_QPT_UC:
2782	case IB_QPT_SMI:
2783	case MLX5_IB_QPT_HW_GSI:
2784	case IB_QPT_DRIVER:
2785	case IB_QPT_GSI:
2786	case IB_QPT_RAW_PACKET:
2787	case IB_QPT_UD:
2788	case MLX5_IB_QPT_REG_UMR:
2789	break;
2790	default:
2791	goto out;
2792	}
2793
2794	*type = attr->qp_type;
2795	return 0;
2796
2797	out:
2798	mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
2799	return -EOPNOTSUPP;
2800	}
2801
2802	static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2803	struct ib_qp_init_attr *attr,
2804	struct ib_udata *udata)
2805	{
2806	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2807	udata, struct mlx5_ib_ucontext, ibucontext);
2808
2809	if (!udata) {
2810	/* Kernel create_qp callers */
2811	if (attr->rwq_ind_tbl)
2812	return -EOPNOTSUPP;
2813
2814	switch (attr->qp_type) {
2815	case IB_QPT_RAW_PACKET:
2816	case IB_QPT_DRIVER:
2817	return -EOPNOTSUPP;
2818	default:
2819	return 0;
2820	}
2821	}
2822
2823	/* Userspace create_qp callers */
2824	if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
2825	mlx5_ib_dbg(dev,
2826	"Raw Packet QP is only supported for CQE version > 0\n");
2827	return -EINVAL;
2828	}
2829
2830	if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
2831	mlx5_ib_dbg(dev,
2832	"Wrong QP type %d for the RWQ indirect table\n",
2833	attr->qp_type);
2834	return -EINVAL;
2835	}
2836
2837	/*
2838	* We don't need to see this warning, it means that kernel code
2839	* missing ib_pd. Placed here to catch developer's mistakes.
2840	*/
2841	WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
2842	"There is a missing PD pointer assignment\n");
2843	return 0;
2844	}
2845
2846	static bool get_dp_ooo_cap(struct mlx5_core_dev *mdev, enum ib_qp_type qp_type)
2847	{
2848	if (!MLX5_CAP_GEN_2(mdev, dp_ordering_force))
2849	return false;
2850
2851	switch (qp_type) {
2852	case IB_QPT_RC:
2853	return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc);
2854	case IB_QPT_XRC_INI:
2855	case IB_QPT_XRC_TGT:
2856	return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc);
2857	case IB_QPT_UC:
2858	return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc);
2859	case IB_QPT_UD:
2860	return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud);
2861	case MLX5_IB_QPT_DCI:
2862	case MLX5_IB_QPT_DCT:
2863	return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc);
2864	default:
2865	return false;
2866	}
2867	}
2868
2869	static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2870	bool cond, struct mlx5_ib_qp *qp)
2871	{
2872	if (!(*flags & flag))
2873	return;
2874
2875	if (cond) {
2876	qp->flags_en |= flag;
2877	*flags &= ~flag;
2878	return;
2879	}
2880
2881	switch (flag) {
2882	case MLX5_QP_FLAG_SCATTER_CQE:
2883	case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:
2884	/*
2885	* We don't return error if these flags were provided,
2886	* and mlx5 doesn't have right capability.
2887	*/
2888	*flags &= ~(MLX5_QP_FLAG_SCATTER_CQE |
2889	MLX5_QP_FLAG_ALLOW_SCATTER_CQE);
2890	return;
2891	default:
2892	break;
2893	}
2894	mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
2895	}
2896
2897	static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2898	void *ucmd, struct ib_qp_init_attr *attr)
2899	{
2900	struct mlx5_core_dev *mdev = dev->mdev;
2901	bool cond;
2902	int flags;
2903
2904	if (attr->rwq_ind_tbl)
2905	flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
2906	else
2907	flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
2908
2909	switch (flags & (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {
2910	case MLX5_QP_FLAG_TYPE_DCI:
2911	qp->type = MLX5_IB_QPT_DCI;
2912	break;
2913	case MLX5_QP_FLAG_TYPE_DCT:
2914	qp->type = MLX5_IB_QPT_DCT;
2915	break;
2916	default:
2917	if (qp->type != IB_QPT_DRIVER)
2918	break;
2919	/*
2920	* It is IB_QPT_DRIVER and or no subtype or
2921	* wrong subtype were provided.
2922	*/
2923	return -EINVAL;
2924	}
2925
2926	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_TYPE_DCI, cond: true, qp);
2927	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_TYPE_DCT, cond: true, qp);
2928	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_DCI_STREAM,
2929	MLX5_CAP_GEN(mdev, log_max_dci_stream_channels),
2930	qp);
2931
2932	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_SIGNATURE, cond: true, qp);
2933	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_SCATTER_CQE,
2934	MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2935	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_ALLOW_SCATTER_CQE,
2936	MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2937
2938	if (qp->type == IB_QPT_RAW_PACKET) {
2939	cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||
2940	MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||
2941	MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
2942	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_TUNNEL_OFFLOADS,
2943	cond, qp);
2944	process_vendor_flag(dev, flags: &flags,
2945	flag: MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, cond: true,
2946	qp);
2947	process_vendor_flag(dev, flags: &flags,
2948	flag: MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, cond: true,
2949	qp);
2950	}
2951
2952	if (qp->type == IB_QPT_RC)
2953	process_vendor_flag(dev, flags: &flags,
2954	flag: MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
2955	MLX5_CAP_GEN(mdev, qp_packet_based), qp);
2956
2957	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_BFREG_INDEX, cond: true, qp);
2958	process_vendor_flag(dev, flags: &flags, flag: MLX5_QP_FLAG_UAR_PAGE_INDEX, cond: true, qp);
2959
2960	cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
2961	MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
2962	MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
2963	if (attr->rwq_ind_tbl && cond) {
2964	mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
2965	cond);
2966	return -EINVAL;
2967	}
2968
2969	if (flags)
2970	mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
2971
2972	return (flags) ? -EINVAL : 0;
2973	}
2974
2975	static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2976	bool cond, struct mlx5_ib_qp *qp)
2977	{
2978	if (!(*flags & flag))
2979	return;
2980
2981	if (cond) {
2982	qp->flags |= flag;
2983	*flags &= ~flag;
2984	return;
2985	}
2986
2987	mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
2988	}
2989
2990	static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2991	struct ib_qp_init_attr *attr)
2992	{
2993	enum ib_qp_type qp_type = qp->type;
2994	struct mlx5_core_dev *mdev = dev->mdev;
2995	int create_flags = attr->create_flags;
2996	bool cond;
2997
2998	if (qp_type == MLX5_IB_QPT_DCT)
2999	return (create_flags) ? -EINVAL : 0;
3000
3001	if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
3002	return (create_flags) ? -EINVAL : 0;
3003
3004	process_create_flag(dev, flags: &create_flags, flag: IB_QP_CREATE_NETIF_QP,
3005	cond: mlx5_get_flow_namespace(dev: dev->mdev,
3006	type: MLX5_FLOW_NAMESPACE_BYPASS),
3007	qp);
3008	process_create_flag(dev, flags: &create_flags,
3009	flag: IB_QP_CREATE_INTEGRITY_EN,
3010	MLX5_CAP_GEN(mdev, sho), qp);
3011	process_create_flag(dev, flags: &create_flags,
3012	flag: IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
3013	MLX5_CAP_GEN(mdev, block_lb_mc), qp);
3014	process_create_flag(dev, flags: &create_flags, flag: IB_QP_CREATE_CROSS_CHANNEL,
3015	MLX5_CAP_GEN(mdev, cd), qp);
3016	process_create_flag(dev, flags: &create_flags, flag: IB_QP_CREATE_MANAGED_SEND,
3017	MLX5_CAP_GEN(mdev, cd), qp);
3018	process_create_flag(dev, flags: &create_flags, flag: IB_QP_CREATE_MANAGED_RECV,
3019	MLX5_CAP_GEN(mdev, cd), qp);
3020
3021	if (qp_type == IB_QPT_UD) {
3022	process_create_flag(dev, flags: &create_flags,
3023	flag: IB_QP_CREATE_IPOIB_UD_LSO,
3024	MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
3025	qp);
3026	cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
3027	process_create_flag(dev, flags: &create_flags, flag: IB_QP_CREATE_SOURCE_QPN,
3028	cond, qp);
3029	}
3030
3031	if (qp_type == IB_QPT_RAW_PACKET) {
3032	cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3033	MLX5_CAP_ETH(mdev, scatter_fcs);
3034	process_create_flag(dev, flags: &create_flags,
3035	flag: IB_QP_CREATE_SCATTER_FCS, cond, qp);
3036
3037	cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3038	MLX5_CAP_ETH(mdev, vlan_cap);
3039	process_create_flag(dev, flags: &create_flags,
3040	flag: IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
3041	}
3042
3043	process_create_flag(dev, flags: &create_flags,
3044	flag: IB_QP_CREATE_PCI_WRITE_END_PADDING,
3045	MLX5_CAP_GEN(mdev, end_pad), qp);
3046
3047	process_create_flag(dev, flags: &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
3048	cond: true, qp);
3049
3050	if (create_flags) {
3051	mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
3052	create_flags);
3053	return -EOPNOTSUPP;
3054	}
3055	return 0;
3056	}
3057
3058	static int process_udata_size(struct mlx5_ib_dev *dev,
3059	struct mlx5_create_qp_params *params)
3060	{
3061	size_t ucmd = sizeof(struct mlx5_ib_create_qp);
3062	struct ib_udata *udata = params->udata;
3063	size_t outlen = udata->outlen;
3064	size_t inlen = udata->inlen;
3065
3066	params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
3067	params->ucmd_size = ucmd;
3068	if (!params->is_rss_raw) {
3069	/* User has old rdma-core, which doesn't support ECE */
3070	size_t min_inlen =
3071	offsetof(struct mlx5_ib_create_qp, ece_options);
3072
3073	/*
3074	* We will check in check_ucmd_data() that user
3075	* cleared everything after inlen.
3076	*/
3077	params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
3078	goto out;
3079	}
3080
3081	/* RSS RAW QP */
3082	if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
3083	return -EINVAL;
3084
3085	if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
3086	return -EINVAL;
3087
3088	ucmd = sizeof(struct mlx5_ib_create_qp_rss);
3089	params->ucmd_size = ucmd;
3090	if (inlen > ucmd && !ib_is_udata_cleared(udata, offset: ucmd, len: inlen - ucmd))
3091	return -EINVAL;
3092
3093	params->inlen = min(ucmd, inlen);
3094	out:
3095	if (!params->inlen)
3096	mlx5_ib_dbg(dev, "udata is too small\n");
3097
3098	return (params->inlen) ? 0 : -EINVAL;
3099	}
3100
3101	static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
3102	struct mlx5_ib_qp *qp,
3103	struct mlx5_create_qp_params *params)
3104	{
3105	int err;
3106
3107	if (params->is_rss_raw) {
3108	err = create_rss_raw_qp_tir(dev, pd, qp, params);
3109	goto out;
3110	}
3111
3112	switch (qp->type) {
3113	case MLX5_IB_QPT_DCT:
3114	err = create_dct(dev, pd, qp, params);
3115	break;
3116	case MLX5_IB_QPT_DCI:
3117	err = create_dci(dev, pd, qp, params);
3118	break;
3119	case IB_QPT_XRC_TGT:
3120	err = create_xrc_tgt_qp(dev, qp, params);
3121	break;
3122	case IB_QPT_GSI:
3123	err = mlx5_ib_create_gsi(pd, mqp: qp, attr: params->attr);
3124	break;
3125	case MLX5_IB_QPT_HW_GSI:
3126	rdma_restrack_no_track(res: &qp->ibqp.res);
3127	fallthrough;
3128	case MLX5_IB_QPT_REG_UMR:
3129	default:
3130	if (params->udata)
3131	err = create_user_qp(dev, pd, qp, params);
3132	else
3133	err = create_kernel_qp(dev, pd, qp, params);
3134	}
3135
3136	out:
3137	if (err) {
3138	mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
3139	return err;
3140	}
3141
3142	if (is_qp0(qp_type: qp->type))
3143	qp->ibqp.qp_num = 0;
3144	else if (is_qp1(qp_type: qp->type))
3145	qp->ibqp.qp_num = 1;
3146	else
3147	qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
3148
3149	mlx5_ib_dbg(dev,
3150	"QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
3151	qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
3152	params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
3153	-1,
3154	params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
3155	-1,
3156	params->resp.ece_options);
3157
3158	return 0;
3159	}
3160
3161	static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3162	struct ib_qp_init_attr *attr)
3163	{
3164	int ret = 0;
3165
3166	switch (qp->type) {
3167	case MLX5_IB_QPT_DCT:
3168	ret = (!attr->srq || !attr->recv_cq) ? -EINVAL : 0;
3169	break;
3170	case MLX5_IB_QPT_DCI:
3171	ret = (attr->cap.max_recv_wr || attr->cap.max_recv_sge) ?
3172	-EINVAL :
3173	0;
3174	break;
3175	case IB_QPT_RAW_PACKET:
3176	ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
3177	break;
3178	default:
3179	break;
3180	}
3181
3182	if (ret)
3183	mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
3184
3185	return ret;
3186	}
3187
3188	static int get_qp_uidx(struct mlx5_ib_qp *qp,
3189	struct mlx5_create_qp_params *params)
3190	{
3191	struct mlx5_ib_create_qp *ucmd = params->ucmd;
3192	struct ib_udata *udata = params->udata;
3193	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3194	udata, struct mlx5_ib_ucontext, ibucontext);
3195
3196	if (params->is_rss_raw)
3197	return 0;
3198
3199	return get_qp_user_index(ucontext, ucmd, inlen: sizeof(*ucmd), user_index: &params->uidx);
3200	}
3201
3202	static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
3203	{
3204	struct mlx5_ib_dev *dev = to_mdev(ibdev: mqp->ibqp.device);
3205
3206	if (mqp->state == IB_QPS_RTR) {
3207	int err;
3208
3209	err = mlx5_core_destroy_dct(dev, dct: &mqp->dct.mdct);
3210	if (err) {
3211	mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
3212	return err;
3213	}
3214	}
3215
3216	kfree(objp: mqp->dct.in);
3217	return 0;
3218	}
3219
3220	static int check_ucmd_data(struct mlx5_ib_dev *dev,
3221	struct mlx5_create_qp_params *params)
3222	{
3223	struct ib_udata *udata = params->udata;
3224	size_t size, last;
3225	int ret;
3226
3227	if (params->is_rss_raw)
3228	/*
3229	* These QPs don't have "reserved" field in their
3230	* create_qp input struct, so their data is always valid.
3231	*/
3232	last = sizeof(struct mlx5_ib_create_qp_rss);
3233	else
3234	last = offsetof(struct mlx5_ib_create_qp, reserved);
3235
3236	if (udata->inlen <= last)
3237	return 0;
3238
3239	/*
3240	* User provides different create_qp structures based on the
3241	* flow and we need to know if he cleared memory after our
3242	* struct create_qp ends.
3243	*/
3244	size = udata->inlen - last;
3245	ret = ib_is_udata_cleared(udata: params->udata, offset: last, len: size);
3246	if (!ret)
3247	mlx5_ib_dbg(
3248	dev,
3249	"udata is not cleared, inlen = %zu, ucmd = %zu, last = %zu, size = %zu\n",
3250	udata->inlen, params->ucmd_size, last, size);
3251	return ret ? 0 : -EINVAL;
3252	}
3253
3254	int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
3255	struct ib_udata *udata)
3256	{
3257	struct mlx5_create_qp_params params = {};
3258	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibqp->device);
3259	struct mlx5_ib_qp *qp = to_mqp(ibqp);
3260	struct ib_pd *pd = ibqp->pd;
3261	enum ib_qp_type type;
3262	int err;
3263
3264	err = mlx5_ib_dev_res_srq_init(dev);
3265	if (err)
3266	return err;
3267
3268	err = check_qp_type(dev, attr, type: &type);
3269	if (err)
3270	return err;
3271
3272	err = check_valid_flow(dev, pd, attr, udata);
3273	if (err)
3274	return err;
3275
3276	params.udata = udata;
3277	params.uidx = MLX5_IB_DEFAULT_UIDX;
3278	params.attr = attr;
3279	params.is_rss_raw = !!attr->rwq_ind_tbl;
3280
3281	if (udata) {
3282	err = process_udata_size(dev, params: &params);
3283	if (err)
3284	return err;
3285
3286	err = check_ucmd_data(dev, params: &params);
3287	if (err)
3288	return err;
3289
3290	params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
3291	if (!params.ucmd)
3292	return -ENOMEM;
3293
3294	err = ib_copy_from_udata(dest: params.ucmd, udata, len: params.inlen);
3295	if (err)
3296	goto free_ucmd;
3297	}
3298
3299	mutex_init(&qp->mutex);
3300	qp->type = type;
3301	if (udata) {
3302	err = process_vendor_flags(dev, qp, ucmd: params.ucmd, attr);
3303	if (err)
3304	goto free_ucmd;
3305
3306	err = get_qp_uidx(qp, params: &params);
3307	if (err)
3308	goto free_ucmd;
3309	}
3310	err = process_create_flags(dev, qp, attr);
3311	if (err)
3312	goto free_ucmd;
3313
3314	err = check_qp_attr(dev, qp, attr);
3315	if (err)
3316	goto free_ucmd;
3317
3318	err = create_qp(dev, pd, qp, params: &params);
3319	if (err)
3320	goto free_ucmd;
3321
3322	kfree(objp: params.ucmd);
3323	params.ucmd = NULL;
3324
3325	if (udata)
3326	/*
3327	* It is safe to copy response for all user create QP flows,
3328	* including MLX5_IB_QPT_DCT, which doesn't need it.
3329	* In that case, resp will be filled with zeros.
3330	*/
3331	err = ib_copy_to_udata(udata, src: &params.resp, len: params.outlen);
3332	if (err)
3333	goto destroy_qp;
3334
3335	return 0;
3336
3337	destroy_qp:
3338	switch (qp->type) {
3339	case MLX5_IB_QPT_DCT:
3340	mlx5_ib_destroy_dct(mqp: qp);
3341	break;
3342	case IB_QPT_GSI:
3343	mlx5_ib_destroy_gsi(mqp: qp);
3344	break;
3345	default:
3346	destroy_qp_common(dev, qp, udata);
3347	}
3348
3349	free_ucmd:
3350	kfree(objp: params.ucmd);
3351	return err;
3352	}
3353
3354	int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
3355	{
3356	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
3357	struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
3358
3359	if (mqp->type == IB_QPT_GSI)
3360	return mlx5_ib_destroy_gsi(mqp);
3361
3362	if (mqp->type == MLX5_IB_QPT_DCT)
3363	return mlx5_ib_destroy_dct(mqp);
3364
3365	destroy_qp_common(dev, qp: mqp, udata);
3366	return 0;
3367	}
3368
3369	static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
3370	const struct ib_qp_attr *attr, int attr_mask,
3371	void *qpc)
3372	{
3373	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->ibqp.device);
3374	u8 dest_rd_atomic;
3375	u32 access_flags;
3376
3377	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3378	dest_rd_atomic = attr->max_dest_rd_atomic;
3379	else
3380	dest_rd_atomic = qp->trans_qp.resp_depth;
3381
3382	if (attr_mask & IB_QP_ACCESS_FLAGS)
3383	access_flags = attr->qp_access_flags;
3384	else
3385	access_flags = qp->trans_qp.atomic_rd_en;
3386
3387	if (!dest_rd_atomic)
3388	access_flags &= IB_ACCESS_REMOTE_WRITE;
3389
3390	MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));
3391
3392	if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
3393	int atomic_mode;
3394
3395	atomic_mode = get_atomic_mode(dev, qp);
3396	if (atomic_mode < 0)
3397	return -EOPNOTSUPP;
3398
3399	MLX5_SET(qpc, qpc, rae, 1);
3400	MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);
3401	}
3402
3403	MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3404	return 0;
3405	}
3406
3407	enum {
3408	MLX5_PATH_FLAG_FL = 1 << 0,
3409	MLX5_PATH_FLAG_FREE_AR = 1 << 1,
3410	MLX5_PATH_FLAG_COUNTER = 1 << 2,
3411	};
3412
3413	static int mlx5_to_ib_rate_map(u8 rate)
3414	{
3415	static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS,
3416	IB_RATE_25_GBPS, IB_RATE_100_GBPS,
3417	IB_RATE_200_GBPS, IB_RATE_50_GBPS,
3418	IB_RATE_400_GBPS };
3419
3420	if (rate < ARRAY_SIZE(rates))
3421	return rates[rate];
3422
3423	return rate - MLX5_STAT_RATE_OFFSET;
3424	}
3425
3426	static int ib_to_mlx5_rate_map(u8 rate)
3427	{
3428	switch (rate) {
3429	case IB_RATE_PORT_CURRENT:
3430	return 0;
3431	case IB_RATE_56_GBPS:
3432	return 1;
3433	case IB_RATE_25_GBPS:
3434	return 2;
3435	case IB_RATE_100_GBPS:
3436	return 3;
3437	case IB_RATE_200_GBPS:
3438	return 4;
3439	case IB_RATE_50_GBPS:
3440	return 5;
3441	case IB_RATE_400_GBPS:
3442	return 6;
3443	default:
3444	return rate + MLX5_STAT_RATE_OFFSET;
3445	}
3446
3447	return 0;
3448	}
3449
3450	int mlx5r_ib_rate(struct mlx5_ib_dev *dev, u8 rate)
3451	{
3452	u32 stat_rate_support;
3453
3454	if (rate == IB_RATE_PORT_CURRENT || rate == IB_RATE_800_GBPS ||
3455	rate == IB_RATE_1600_GBPS)
3456	return 0;
3457
3458	if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_1600_GBPS)
3459	return -EINVAL;
3460
3461	stat_rate_support = MLX5_CAP_GEN(dev->mdev, stat_rate_support);
3462	while (rate != IB_RATE_PORT_CURRENT &&
3463	!(1 << ib_to_mlx5_rate_map(rate) & stat_rate_support))
3464	--rate;
3465
3466	return ib_to_mlx5_rate_map(rate);
3467	}
3468
3469	static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
3470	struct mlx5_ib_sq *sq, u8 sl,
3471	struct ib_pd *pd)
3472	{
3473	void *in;
3474	void *tisc;
3475	int inlen;
3476	int err;
3477
3478	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
3479	in = kvzalloc(inlen, GFP_KERNEL);
3480	if (!in)
3481	return -ENOMEM;
3482
3483	MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
3484	MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
3485
3486	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3487	MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
3488
3489	err = mlx5_core_modify_tis(dev, tisn: sq->tisn, in);
3490
3491	kvfree(addr: in);
3492
3493	return err;
3494	}
3495
3496	static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
3497	struct mlx5_ib_sq *sq, u8 tx_affinity,
3498	struct ib_pd *pd)
3499	{
3500	void *in;
3501	void *tisc;
3502	int inlen;
3503	int err;
3504
3505	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
3506	in = kvzalloc(inlen, GFP_KERNEL);
3507	if (!in)
3508	return -ENOMEM;
3509
3510	MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
3511	MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
3512
3513	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3514	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
3515
3516	err = mlx5_core_modify_tis(dev, tisn: sq->tisn, in);
3517
3518	kvfree(addr: in);
3519
3520	return err;
3521	}
3522
3523	static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,
3524	u32 lqpn, u32 rqpn)
3525
3526	{
3527	u32 fl = ah->grh.flow_label;
3528
3529	if (!fl)
3530	fl = rdma_calc_flow_label(lqpn, rqpn);
3531
3532	MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));
3533	}
3534
3535	static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3536	const struct rdma_ah_attr *ah, void *path, u8 port,
3537	int attr_mask, u32 path_flags,
3538	const struct ib_qp_attr *attr, bool alt)
3539	{
3540	const struct ib_global_route *grh = rdma_ah_read_grh(attr: ah);
3541	int err;
3542	enum ib_gid_type gid_type;
3543	u8 ah_flags = rdma_ah_get_ah_flags(attr: ah);
3544	u8 sl = rdma_ah_get_sl(attr: ah);
3545
3546	if (attr_mask & IB_QP_PKEY_INDEX)
3547	MLX5_SET(ads, path, pkey_index,
3548	alt ? attr->alt_pkey_index : attr->pkey_index);
3549
3550	if (ah_flags & IB_AH_GRH) {
3551	const struct ib_port_immutable *immutable;
3552
3553	immutable = ib_port_immutable_read(dev: &dev->ib_dev, port);
3554	if (grh->sgid_index >= immutable->gid_tbl_len) {
3555	pr_err("sgid_index (%u) too large. max is %d\n",
3556	grh->sgid_index,
3557	immutable->gid_tbl_len);
3558	return -EINVAL;
3559	}
3560	}
3561
3562	if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
3563	if (!(ah_flags & IB_AH_GRH))
3564	return -EINVAL;
3565
3566	ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),
3567	src: ah->roce.dmac);
3568	if ((qp->type == IB_QPT_RC ||
3569	qp->type == IB_QPT_UC ||
3570	qp->type == IB_QPT_XRC_INI ||
3571	qp->type == IB_QPT_XRC_TGT) &&
3572	(grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
3573	(attr_mask & IB_QP_DEST_QPN))
3574	mlx5_set_path_udp_sport(path, ah,
3575	lqpn: qp->ibqp.qp_num,
3576	rqpn: attr->dest_qp_num);
3577	MLX5_SET(ads, path, eth_prio, sl & 0x7);
3578	gid_type = ah->grh.sgid_attr->gid_type;
3579	if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
3580	MLX5_SET(ads, path, dscp, grh->traffic_class >> 2);
3581	} else {
3582	MLX5_SET(ads, path, fl, !!(path_flags & MLX5_PATH_FLAG_FL));
3583	MLX5_SET(ads, path, free_ar,
3584	!!(path_flags & MLX5_PATH_FLAG_FREE_AR));
3585	MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));
3586	MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));
3587	MLX5_SET(ads, path, grh, !!(ah_flags & IB_AH_GRH));
3588	MLX5_SET(ads, path, sl, sl);
3589	}
3590
3591	if (ah_flags & IB_AH_GRH) {
3592	MLX5_SET(ads, path, src_addr_index, grh->sgid_index);
3593	MLX5_SET(ads, path, hop_limit, grh->hop_limit);
3594	MLX5_SET(ads, path, tclass, grh->traffic_class);
3595	MLX5_SET(ads, path, flow_label, grh->flow_label);
3596	memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh->dgid.raw,
3597	sizeof(grh->dgid.raw));
3598	}
3599
3600	err = mlx5r_ib_rate(dev, rate: rdma_ah_get_static_rate(attr: ah));
3601	if (err < 0)
3602	return err;
3603	MLX5_SET(ads, path, stat_rate, err);
3604	MLX5_SET(ads, path, vhca_port_num, port);
3605
3606	if (attr_mask & IB_QP_TIMEOUT)
3607	MLX5_SET(ads, path, ack_timeout,
3608	alt ? attr->alt_timeout : attr->timeout);
3609
3610	if ((qp->type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
3611	return modify_raw_packet_eth_prio(dev: dev->mdev,
3612	sq: &qp->raw_packet_qp.sq,
3613	sl: sl & 0xf, pd: qp->ibqp.pd);
3614
3615	return 0;
3616	}
3617
3618	static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
3619	[MLX5_QP_STATE_INIT] = {
3620	[MLX5_QP_STATE_INIT] = {
3621	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
3622	MLX5_QP_OPTPAR_RAE |
3623	MLX5_QP_OPTPAR_RWE |
3624	MLX5_QP_OPTPAR_PKEY_INDEX |
3625	MLX5_QP_OPTPAR_PRI_PORT |
3626	MLX5_QP_OPTPAR_LAG_TX_AFF,
3627	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
3628	MLX5_QP_OPTPAR_PKEY_INDEX |
3629	MLX5_QP_OPTPAR_PRI_PORT |
3630	MLX5_QP_OPTPAR_LAG_TX_AFF,
3631	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
3632	MLX5_QP_OPTPAR_Q_KEY |
3633	MLX5_QP_OPTPAR_PRI_PORT,
3634	[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE |
3635	MLX5_QP_OPTPAR_RAE |
3636	MLX5_QP_OPTPAR_RWE |
3637	MLX5_QP_OPTPAR_PKEY_INDEX |
3638	MLX5_QP_OPTPAR_PRI_PORT |
3639	MLX5_QP_OPTPAR_LAG_TX_AFF,
3640	},
3641	[MLX5_QP_STATE_RTR] = {
3642	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3643	MLX5_QP_OPTPAR_RRE |
3644	MLX5_QP_OPTPAR_RAE |
3645	MLX5_QP_OPTPAR_RWE |
3646	MLX5_QP_OPTPAR_PKEY_INDEX |
3647	MLX5_QP_OPTPAR_LAG_TX_AFF,
3648	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3649	MLX5_QP_OPTPAR_RWE |
3650	MLX5_QP_OPTPAR_PKEY_INDEX |
3651	MLX5_QP_OPTPAR_LAG_TX_AFF,
3652	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
3653	MLX5_QP_OPTPAR_Q_KEY,
3654	[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX |
3655	MLX5_QP_OPTPAR_Q_KEY,
3656	[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3657	MLX5_QP_OPTPAR_RRE |
3658	MLX5_QP_OPTPAR_RAE |
3659	MLX5_QP_OPTPAR_RWE |
3660	MLX5_QP_OPTPAR_PKEY_INDEX |
3661	MLX5_QP_OPTPAR_LAG_TX_AFF,
3662	},
3663	},
3664	[MLX5_QP_STATE_RTR] = {
3665	[MLX5_QP_STATE_RTS] = {
3666	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3667	MLX5_QP_OPTPAR_RRE |
3668	MLX5_QP_OPTPAR_RAE |
3669	MLX5_QP_OPTPAR_RWE |
3670	MLX5_QP_OPTPAR_PM_STATE |
3671	MLX5_QP_OPTPAR_RNR_TIMEOUT,
3672	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3673	MLX5_QP_OPTPAR_RWE |
3674	MLX5_QP_OPTPAR_PM_STATE,
3675	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3676	[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3677	MLX5_QP_OPTPAR_RRE |
3678	MLX5_QP_OPTPAR_RAE |
3679	MLX5_QP_OPTPAR_RWE |
3680	MLX5_QP_OPTPAR_PM_STATE |
3681	MLX5_QP_OPTPAR_RNR_TIMEOUT,
3682	},
3683	},
3684	[MLX5_QP_STATE_RTS] = {
3685	[MLX5_QP_STATE_RTS] = {
3686	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
3687	MLX5_QP_OPTPAR_RAE |
3688	MLX5_QP_OPTPAR_RWE |
3689	MLX5_QP_OPTPAR_RNR_TIMEOUT |
3690	MLX5_QP_OPTPAR_PM_STATE |
3691	MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3692	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
3693	MLX5_QP_OPTPAR_PM_STATE |
3694	MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3695	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY |
3696	MLX5_QP_OPTPAR_SRQN |
3697	MLX5_QP_OPTPAR_CQN_RCV,
3698	[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE |
3699	MLX5_QP_OPTPAR_RAE |
3700	MLX5_QP_OPTPAR_RWE |
3701	MLX5_QP_OPTPAR_RNR_TIMEOUT |
3702	MLX5_QP_OPTPAR_PM_STATE |
3703	MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3704	},
3705	},
3706	[MLX5_QP_STATE_SQER] = {
3707	[MLX5_QP_STATE_RTS] = {
3708	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3709	[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3710	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
3711	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3712	MLX5_QP_OPTPAR_RWE |
3713	MLX5_QP_OPTPAR_RAE |
3714	MLX5_QP_OPTPAR_RRE,
3715	[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3716	MLX5_QP_OPTPAR_RWE |
3717	MLX5_QP_OPTPAR_RAE |
3718	MLX5_QP_OPTPAR_RRE,
3719	},
3720	},
3721	[MLX5_QP_STATE_SQD] = {
3722	[MLX5_QP_STATE_RTS] = {
3723	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3724	[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3725	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
3726	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3727	MLX5_QP_OPTPAR_RWE |
3728	MLX5_QP_OPTPAR_RAE |
3729	MLX5_QP_OPTPAR_RRE,
3730	},
3731	},
3732	};
3733
3734	static int ib_nr_to_mlx5_nr(int ib_mask)
3735	{
3736	switch (ib_mask) {
3737	case IB_QP_STATE:
3738	return 0;
3739	case IB_QP_CUR_STATE:
3740	return 0;
3741	case IB_QP_EN_SQD_ASYNC_NOTIFY:
3742	return 0;
3743	case IB_QP_ACCESS_FLAGS:
3744	return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
3745	MLX5_QP_OPTPAR_RAE;
3746	case IB_QP_PKEY_INDEX:
3747	return MLX5_QP_OPTPAR_PKEY_INDEX;
3748	case IB_QP_PORT:
3749	return MLX5_QP_OPTPAR_PRI_PORT;
3750	case IB_QP_QKEY:
3751	return MLX5_QP_OPTPAR_Q_KEY;
3752	case IB_QP_AV:
3753	return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
3754	MLX5_QP_OPTPAR_PRI_PORT;
3755	case IB_QP_PATH_MTU:
3756	return 0;
3757	case IB_QP_TIMEOUT:
3758	return MLX5_QP_OPTPAR_ACK_TIMEOUT;
3759	case IB_QP_RETRY_CNT:
3760	return MLX5_QP_OPTPAR_RETRY_COUNT;
3761	case IB_QP_RNR_RETRY:
3762	return MLX5_QP_OPTPAR_RNR_RETRY;
3763	case IB_QP_RQ_PSN:
3764	return 0;
3765	case IB_QP_MAX_QP_RD_ATOMIC:
3766	return MLX5_QP_OPTPAR_SRA_MAX;
3767	case IB_QP_ALT_PATH:
3768	return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
3769	case IB_QP_MIN_RNR_TIMER:
3770	return MLX5_QP_OPTPAR_RNR_TIMEOUT;
3771	case IB_QP_SQ_PSN:
3772	return 0;
3773	case IB_QP_MAX_DEST_RD_ATOMIC:
3774	return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
3775	MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
3776	case IB_QP_PATH_MIG_STATE:
3777	return MLX5_QP_OPTPAR_PM_STATE;
3778	case IB_QP_CAP:
3779	return 0;
3780	case IB_QP_DEST_QPN:
3781	return 0;
3782	}
3783	return 0;
3784	}
3785
3786	static int ib_mask_to_mlx5_opt(int ib_mask)
3787	{
3788	int result = 0;
3789	int i;
3790
3791	for (i = 0; i < 8 * sizeof(int); i++) {
3792	if ((1 << i) & ib_mask)
3793	result |= ib_nr_to_mlx5_nr(ib_mask: 1 << i);
3794	}
3795
3796	return result;
3797	}
3798
3799	static int modify_raw_packet_qp_rq(
3800	struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
3801	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
3802	{
3803	void *in;
3804	void *rqc;
3805	int inlen;
3806	int err;
3807
3808	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
3809	in = kvzalloc(inlen, GFP_KERNEL);
3810	if (!in)
3811	return -ENOMEM;
3812
3813	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
3814	MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
3815
3816	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
3817	MLX5_SET(rqc, rqc, state, new_state);
3818
3819	if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
3820	if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
3821	MLX5_SET64(modify_rq_in, in, modify_bitmask,
3822	MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
3823	MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
3824	} else
3825	dev_info_once(
3826	&dev->ib_dev.dev,
3827	"RAW PACKET QP counters are not supported on current FW\n");
3828	}
3829
3830	err = mlx5_core_modify_rq(dev: dev->mdev, rqn: rq->base.mqp.qpn, in);
3831	if (err)
3832	goto out;
3833
3834	rq->state = new_state;
3835
3836	out:
3837	kvfree(addr: in);
3838	return err;
3839	}
3840
3841	static int modify_raw_packet_qp_sq(
3842	struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state,
3843	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
3844	{
3845	struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
3846	struct mlx5_rate_limit old_rl = ibqp->rl;
3847	struct mlx5_rate_limit new_rl = old_rl;
3848	bool new_rate_added = false;
3849	u16 rl_index = 0;
3850	void *in;
3851	void *sqc;
3852	int inlen;
3853	int err;
3854
3855	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
3856	in = kvzalloc(inlen, GFP_KERNEL);
3857	if (!in)
3858	return -ENOMEM;
3859
3860	MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
3861	MLX5_SET(modify_sq_in, in, sq_state, sq->state);
3862
3863	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
3864	MLX5_SET(sqc, sqc, state, new_state);
3865
3866	if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
3867	if (new_state != MLX5_SQC_STATE_RDY)
3868	pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
3869	__func__);
3870	else
3871	new_rl = raw_qp_param->rl;
3872	}
3873
3874	if (!mlx5_rl_are_equal(rl_0: &old_rl, rl_1: &new_rl)) {
3875	if (new_rl.rate) {
3876	err = mlx5_rl_add_rate(dev, index: &rl_index, rl: &new_rl);
3877	if (err) {
3878	pr_err("Failed configuring rate limit(err %d): \
3879	rate %u, max_burst_sz %u, typical_pkt_sz %u\n",
3880	err, new_rl.rate, new_rl.max_burst_sz,
3881	new_rl.typical_pkt_sz);
3882
3883	goto out;
3884	}
3885	new_rate_added = true;
3886	}
3887
3888	MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
3889	/* index 0 means no limit */
3890	MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
3891	}
3892
3893	err = mlx5_core_modify_sq(dev, sqn: sq->base.mqp.qpn, in);
3894	if (err) {
3895	/* Remove new rate from table if failed */
3896	if (new_rate_added)
3897	mlx5_rl_remove_rate(dev, rl: &new_rl);
3898	goto out;
3899	}
3900
3901	/* Only remove the old rate after new rate was set */
3902	if ((old_rl.rate && !mlx5_rl_are_equal(rl_0: &old_rl, rl_1: &new_rl)) ||
3903	(new_state != MLX5_SQC_STATE_RDY)) {
3904	mlx5_rl_remove_rate(dev, rl: &old_rl);
3905	if (new_state != MLX5_SQC_STATE_RDY)
3906	memset(&new_rl, 0, sizeof(new_rl));
3907	}
3908
3909	ibqp->rl = new_rl;
3910	sq->state = new_state;
3911
3912	out:
3913	kvfree(addr: in);
3914	return err;
3915	}
3916
3917	static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3918	const struct mlx5_modify_raw_qp_param *raw_qp_param,
3919	u8 tx_affinity)
3920	{
3921	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
3922	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
3923	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
3924	int modify_rq = !!qp->rq.wqe_cnt;
3925	int modify_sq = !!qp->sq.wqe_cnt;
3926	int rq_state;
3927	int sq_state;
3928	int err;
3929
3930	switch (raw_qp_param->operation) {
3931	case MLX5_CMD_OP_RST2INIT_QP:
3932	rq_state = MLX5_RQC_STATE_RDY;
3933	sq_state = MLX5_SQC_STATE_RST;
3934	break;
3935	case MLX5_CMD_OP_2ERR_QP:
3936	rq_state = MLX5_RQC_STATE_ERR;
3937	sq_state = MLX5_SQC_STATE_ERR;
3938	break;
3939	case MLX5_CMD_OP_2RST_QP:
3940	rq_state = MLX5_RQC_STATE_RST;
3941	sq_state = MLX5_SQC_STATE_RST;
3942	break;
3943	case MLX5_CMD_OP_RTR2RTS_QP:
3944	case MLX5_CMD_OP_RTS2RTS_QP:
3945	if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
3946	return -EINVAL;
3947
3948	modify_rq = 0;
3949	sq_state = MLX5_SQC_STATE_RDY;
3950	break;
3951	case MLX5_CMD_OP_INIT2INIT_QP:
3952	case MLX5_CMD_OP_INIT2RTR_QP:
3953	if (raw_qp_param->set_mask)
3954	return -EINVAL;
3955	else
3956	return 0;
3957	default:
3958	WARN_ON(1);
3959	return -EINVAL;
3960	}
3961
3962	if (modify_rq) {
3963	err = modify_raw_packet_qp_rq(dev, rq, new_state: rq_state, raw_qp_param,
3964	pd: qp->ibqp.pd);
3965	if (err)
3966	return err;
3967	}
3968
3969	if (modify_sq) {
3970	struct mlx5_flow_handle *flow_rule;
3971
3972	if (tx_affinity) {
3973	err = modify_raw_packet_tx_affinity(dev: dev->mdev, sq,
3974	tx_affinity,
3975	pd: qp->ibqp.pd);
3976	if (err)
3977	return err;
3978	}
3979
3980	flow_rule = create_flow_rule_vport_sq(dev, sq,
3981	port: raw_qp_param->port);
3982	if (IS_ERR(ptr: flow_rule))
3983	return PTR_ERR(ptr: flow_rule);
3984
3985	err = modify_raw_packet_qp_sq(dev: dev->mdev, sq, new_state: sq_state,
3986	raw_qp_param, pd: qp->ibqp.pd);
3987	if (err) {
3988	if (flow_rule)
3989	mlx5_del_flow_rules(fr: flow_rule);
3990	return err;
3991	}
3992
3993	if (flow_rule) {
3994	destroy_flow_rule_vport_sq(sq);
3995	sq->flow_rule = flow_rule;
3996	}
3997
3998	return err;
3999	}
4000
4001	return 0;
4002	}
4003
4004	static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
4005	struct ib_udata *udata)
4006	{
4007	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
4008	udata, struct mlx5_ib_ucontext, ibucontext);
4009	u8 port_num = mlx5_core_native_port_num(dev: dev->mdev) - 1;
4010	atomic_t *tx_port_affinity;
4011
4012	if (ucontext)
4013	tx_port_affinity = &ucontext->tx_port_affinity;
4014	else
4015	tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
4016
4017	return (unsigned int)atomic_add_return(i: 1, v: tx_port_affinity) %
4018	(dev->lag_active ? dev->lag_ports : MLX5_CAP_GEN(dev->mdev, num_lag_ports)) + 1;
4019	}
4020
4021	static bool qp_supports_affinity(struct mlx5_ib_qp *qp)
4022	{
4023	if ((qp->type == IB_QPT_RC) || (qp->type == IB_QPT_UD) ||
4024	(qp->type == IB_QPT_UC) || (qp->type == IB_QPT_RAW_PACKET) ||
4025	(qp->type == IB_QPT_XRC_INI) || (qp->type == IB_QPT_XRC_TGT) ||
4026	(qp->type == MLX5_IB_QPT_DCI))
4027	return true;
4028	return false;
4029	}
4030
4031	static unsigned int get_tx_affinity(struct ib_qp *qp,
4032	const struct ib_qp_attr *attr,
4033	int attr_mask, u8 init,
4034	struct ib_udata *udata)
4035	{
4036	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
4037	udata, struct mlx5_ib_ucontext, ibucontext);
4038	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
4039	struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
4040	struct mlx5_ib_qp_base *qp_base;
4041	unsigned int tx_affinity;
4042
4043	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
4044	qp_supports_affinity(qp: mqp)))
4045	return 0;
4046
4047	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4048	tx_affinity = mqp->gsi_lag_port;
4049	else if (init)
4050	tx_affinity = get_tx_affinity_rr(dev, udata);
4051	else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
4052	tx_affinity =
4053	mlx5_lag_get_slave_port(dev: dev->mdev, slave: attr->xmit_slave);
4054	else
4055	return 0;
4056
4057	qp_base = &mqp->trans_qp.base;
4058	if (ucontext)
4059	mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
4060	tx_affinity, qp_base->mqp.qpn, ucontext);
4061	else
4062	mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
4063	tx_affinity, qp_base->mqp.qpn);
4064	return tx_affinity;
4065	}
4066
4067	static int __mlx5_ib_qp_set_raw_qp_counter(struct mlx5_ib_qp *qp, u32 set_id,
4068	struct mlx5_core_dev *mdev)
4069	{
4070	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4071	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4072	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
4073	void *rqc;
4074
4075	if (!qp->rq.wqe_cnt)
4076	return 0;
4077
4078	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
4079	MLX5_SET(modify_rq_in, in, uid, to_mpd(qp->ibqp.pd)->uid);
4080
4081	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
4082	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
4083
4084	MLX5_SET64(modify_rq_in, in, modify_bitmask,
4085	MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
4086	MLX5_SET(rqc, rqc, counter_set_id, set_id);
4087
4088	return mlx5_core_modify_rq(dev: mdev, rqn: rq->base.mqp.qpn, in);
4089	}
4090
4091	static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
4092	struct rdma_counter *counter)
4093	{
4094	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
4095	u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};
4096	struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
4097	struct mlx5_ib_qp_base *base;
4098	u32 set_id;
4099	u32 *qpc;
4100
4101	if (counter)
4102	set_id = counter->id;
4103	else
4104	set_id = mlx5_ib_get_counters_id(dev, port_num: mqp->port - 1);
4105
4106	if (mqp->type == IB_QPT_RAW_PACKET)
4107	return __mlx5_ib_qp_set_raw_qp_counter(qp: mqp, set_id, mdev: dev->mdev);
4108
4109	base = &mqp->trans_qp.base;
4110	MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);
4111	MLX5_SET(rts2rts_qp_in, in, qpn, base->mqp.qpn);
4112	MLX5_SET(rts2rts_qp_in, in, uid, base->mqp.uid);
4113	MLX5_SET(rts2rts_qp_in, in, opt_param_mask,
4114	MLX5_QP_OPTPAR_COUNTER_SET_ID);
4115
4116	qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);
4117	MLX5_SET(qpc, qpc, counter_set_id, set_id);
4118	return mlx5_cmd_exec_in(dev->mdev, rts2rts_qp, in);
4119	}
4120
4121	static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
4122	const struct ib_qp_attr *attr, int attr_mask,
4123	enum ib_qp_state cur_state,
4124	enum ib_qp_state new_state,
4125	const struct mlx5_ib_modify_qp *ucmd,
4126	struct mlx5_ib_modify_qp_resp *resp,
4127	struct ib_udata *udata)
4128	{
4129	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
4130	[MLX5_QP_STATE_RST] = {
4131	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4132	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4133	[MLX5_QP_STATE_INIT] = MLX5_CMD_OP_RST2INIT_QP,
4134	},
4135	[MLX5_QP_STATE_INIT] = {
4136	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4137	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4138	[MLX5_QP_STATE_INIT] = MLX5_CMD_OP_INIT2INIT_QP,
4139	[MLX5_QP_STATE_RTR] = MLX5_CMD_OP_INIT2RTR_QP,
4140	},
4141	[MLX5_QP_STATE_RTR] = {
4142	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4143	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4144	[MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTR2RTS_QP,
4145	},
4146	[MLX5_QP_STATE_RTS] = {
4147	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4148	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4149	[MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTS2RTS_QP,
4150	},
4151	[MLX5_QP_STATE_SQD] = {
4152	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4153	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4154	[MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQD_RTS_QP,
4155	},
4156	[MLX5_QP_STATE_SQER] = {
4157	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4158	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4159	[MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQERR2RTS_QP,
4160	},
4161	[MLX5_QP_STATE_ERR] = {
4162	[MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4163	[MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4164	}
4165	};
4166
4167	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibqp->device);
4168	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4169	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
4170	struct mlx5_ib_cq *send_cq, *recv_cq;
4171	struct mlx5_ib_pd *pd;
4172	enum mlx5_qp_state mlx5_cur, mlx5_new;
4173	void *qpc, *pri_path, *alt_path;
4174	enum mlx5_qp_optpar optpar = 0;
4175	u32 set_id = 0;
4176	int mlx5_st;
4177	int err;
4178	u16 op;
4179	u8 tx_affinity = 0;
4180
4181	mlx5_st = to_mlx5_st(type: qp->type);
4182	if (mlx5_st < 0)
4183	return -EINVAL;
4184
4185	qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
4186	if (!qpc)
4187	return -ENOMEM;
4188
4189	pd = to_mpd(ibpd: qp->ibqp.pd);
4190	MLX5_SET(qpc, qpc, st, mlx5_st);
4191
4192	if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
4193	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
4194	} else {
4195	switch (attr->path_mig_state) {
4196	case IB_MIG_MIGRATED:
4197	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
4198	break;
4199	case IB_MIG_REARM:
4200	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);
4201	break;
4202	case IB_MIG_ARMED:
4203	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);
4204	break;
4205	}
4206	}
4207
4208	tx_affinity = get_tx_affinity(qp: ibqp, attr, attr_mask,
4209	init: cur_state == IB_QPS_RESET &&
4210	new_state == IB_QPS_INIT, udata);
4211
4212	MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);
4213	if (tx_affinity && new_state == IB_QPS_RTR &&
4214	MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
4215	optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;
4216
4217	if (is_sqp(qp_type: qp->type)) {
4218	MLX5_SET(qpc, qpc, mtu, IB_MTU_256);
4219	MLX5_SET(qpc, qpc, log_msg_max, 8);
4220	} else if ((qp->type == IB_QPT_UD &&
4221	!(qp->flags & IB_QP_CREATE_SOURCE_QPN)) ||
4222	qp->type == MLX5_IB_QPT_REG_UMR) {
4223	MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
4224	MLX5_SET(qpc, qpc, log_msg_max, 12);
4225	} else if (attr_mask & IB_QP_PATH_MTU) {
4226	if (attr->path_mtu < IB_MTU_256 ||
4227	attr->path_mtu > IB_MTU_4096) {
4228	mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
4229	err = -EINVAL;
4230	goto out;
4231	}
4232	MLX5_SET(qpc, qpc, mtu, attr->path_mtu);
4233	MLX5_SET(qpc, qpc, log_msg_max,
4234	MLX5_CAP_GEN(dev->mdev, log_max_msg));
4235	}
4236
4237	if (attr_mask & IB_QP_DEST_QPN)
4238	MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);
4239
4240	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
4241	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
4242
4243	if (attr_mask & IB_QP_PKEY_INDEX)
4244	MLX5_SET(ads, pri_path, pkey_index, attr->pkey_index);
4245
4246	/* todo implement counter_index functionality */
4247
4248	if (dev->ib_dev.type == RDMA_DEVICE_TYPE_SMI && is_qp0(qp_type: qp->type)) {
4249	MLX5_SET(ads, pri_path, vhca_port_num,
4250	smi_to_native_portnum(dev, qp->port));
4251	if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)
4252	MLX5_SET(ads, pri_path, plane_index, qp->port);
4253	} else if (is_sqp(qp_type: qp->type))
4254	MLX5_SET(ads, pri_path, vhca_port_num, qp->port);
4255
4256	if (attr_mask & IB_QP_PORT)
4257	MLX5_SET(ads, pri_path, vhca_port_num, attr->port_num);
4258
4259	if (attr_mask & IB_QP_AV) {
4260	err = mlx5_set_path(dev, qp, ah: &attr->ah_attr, path: pri_path,
4261	port: attr_mask & IB_QP_PORT ? attr->port_num :
4262	qp->port,
4263	attr_mask, path_flags: 0, attr, alt: false);
4264	if (err)
4265	goto out;
4266	}
4267
4268	if (attr_mask & IB_QP_TIMEOUT)
4269	MLX5_SET(ads, pri_path, ack_timeout, attr->timeout);
4270
4271	if (attr_mask & IB_QP_ALT_PATH) {
4272	err = mlx5_set_path(dev, qp, ah: &attr->alt_ah_attr, path: alt_path,
4273	port: attr->alt_port_num,
4274	attr_mask: attr_mask | IB_QP_PKEY_INDEX |
4275	IB_QP_TIMEOUT,
4276	path_flags: 0, attr, alt: true);
4277	if (err)
4278	goto out;
4279	}
4280
4281	get_cqs(qp_type: qp->type, ib_send_cq: qp->ibqp.send_cq, ib_recv_cq: qp->ibqp.recv_cq,
4282	send_cq: &send_cq, recv_cq: &recv_cq);
4283
4284	MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
4285	if (send_cq)
4286	MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);
4287	if (recv_cq)
4288	MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);
4289
4290	MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);
4291
4292	if (attr_mask & IB_QP_RNR_RETRY)
4293	MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);
4294
4295	if (attr_mask & IB_QP_RETRY_CNT)
4296	MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);
4297
4298	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic)
4299	MLX5_SET(qpc, qpc, log_sra_max, fls(attr->max_rd_atomic - 1));
4300
4301	if (attr_mask & IB_QP_SQ_PSN)
4302	MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);
4303
4304	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic)
4305	MLX5_SET(qpc, qpc, log_rra_max,
4306	fls(attr->max_dest_rd_atomic - 1));
4307
4308	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
4309	err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);
4310	if (err)
4311	goto out;
4312	}
4313
4314	if (attr_mask & IB_QP_MIN_RNR_TIMER)
4315	MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);
4316
4317	if (attr_mask & IB_QP_RQ_PSN)
4318	MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);
4319
4320	if (attr_mask & IB_QP_QKEY)
4321	MLX5_SET(qpc, qpc, q_key, attr->qkey);
4322
4323	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
4324	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
4325
4326	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4327	u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
4328	qp->port) - 1;
4329
4330	/* Underlay port should be used - index 0 function per port */
4331	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
4332	port_num = 0;
4333
4334	if (ibqp->counter)
4335	set_id = ibqp->counter->id;
4336	else
4337	set_id = mlx5_ib_get_counters_id(dev, port_num);
4338	MLX5_SET(qpc, qpc, counter_set_id, set_id);
4339	}
4340
4341	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
4342	MLX5_SET(qpc, qpc, rlky, 1);
4343
4344	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4345	MLX5_SET(qpc, qpc, deth_sqpn, 1);
4346
4347	if (qp->is_ooo_rq && cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4348	MLX5_SET(qpc, qpc, dp_ordering_1, 1);
4349	MLX5_SET(qpc, qpc, dp_ordering_force, 1);
4350	}
4351
4352	mlx5_cur = to_mlx5_state(state: cur_state);
4353	mlx5_new = to_mlx5_state(state: new_state);
4354
4355	if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
4356	!optab[mlx5_cur][mlx5_new]) {
4357	err = -EINVAL;
4358	goto out;
4359	}
4360
4361	op = optab[mlx5_cur][mlx5_new];
4362	optpar |= ib_mask_to_mlx5_opt(ib_mask: attr_mask);
4363	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
4364
4365	if (qp->type == IB_QPT_RAW_PACKET ||
4366	qp->flags & IB_QP_CREATE_SOURCE_QPN) {
4367	struct mlx5_modify_raw_qp_param raw_qp_param = {};
4368
4369	raw_qp_param.operation = op;
4370	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4371	raw_qp_param.rq_q_ctr_id = set_id;
4372	raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
4373	}
4374
4375	if (attr_mask & IB_QP_PORT)
4376	raw_qp_param.port = attr->port_num;
4377
4378	if (attr_mask & IB_QP_RATE_LIMIT) {
4379	raw_qp_param.rl.rate = attr->rate_limit;
4380
4381	if (ucmd->burst_info.max_burst_sz) {
4382	if (attr->rate_limit &&
4383	MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) {
4384	raw_qp_param.rl.max_burst_sz =
4385	ucmd->burst_info.max_burst_sz;
4386	} else {
4387	err = -EINVAL;
4388	goto out;
4389	}
4390	}
4391
4392	if (ucmd->burst_info.typical_pkt_sz) {
4393	if (attr->rate_limit &&
4394	MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) {
4395	raw_qp_param.rl.typical_pkt_sz =
4396	ucmd->burst_info.typical_pkt_sz;
4397	} else {
4398	err = -EINVAL;
4399	goto out;
4400	}
4401	}
4402
4403	raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
4404	}
4405
4406	err = modify_raw_packet_qp(dev, qp, raw_qp_param: &raw_qp_param, tx_affinity);
4407	} else {
4408	if (udata) {
4409	/* For the kernel flows, the resp will stay zero */
4410	resp->ece_options =
4411	MLX5_CAP_GEN(dev->mdev, ece_support) ?
4412	ucmd->ece_options : 0;
4413	resp->response_length = sizeof(*resp);
4414	}
4415	err = mlx5_core_qp_modify(dev, opcode: op, opt_param_mask: optpar, qpc, qp: &base->mqp,
4416	ece: &resp->ece_options);
4417	}
4418
4419	if (err)
4420	goto out;
4421
4422	qp->state = new_state;
4423
4424	if (attr_mask & IB_QP_ACCESS_FLAGS)
4425	qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
4426	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4427	qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
4428	if (attr_mask & IB_QP_PORT)
4429	qp->port = attr->port_num;
4430	if (attr_mask & IB_QP_ALT_PATH)
4431	qp->trans_qp.alt_port = attr->alt_port_num;
4432
4433	/*
4434	* If we moved a kernel QP to RESET, clean up all old CQ
4435	* entries and reinitialize the QP.
4436	*/
4437	if (new_state == IB_QPS_RESET &&
4438	!ibqp->uobject && qp->type != IB_QPT_XRC_TGT) {
4439	mlx5_ib_cq_clean(cq: recv_cq, qpn: base->mqp.qpn,
4440	srq: ibqp->srq ? to_msrq(ibsrq: ibqp->srq) : NULL);
4441	if (send_cq != recv_cq)
4442	mlx5_ib_cq_clean(cq: send_cq, qpn: base->mqp.qpn, NULL);
4443
4444	qp->rq.head = 0;
4445	qp->rq.tail = 0;
4446	qp->sq.head = 0;
4447	qp->sq.tail = 0;
4448	qp->sq.cur_post = 0;
4449	if (qp->sq.wqe_cnt)
4450	qp->sq.cur_edge = get_sq_edge(sq: &qp->sq, idx: 0);
4451	qp->sq.last_poll = 0;
4452	qp->db.db[MLX5_RCV_DBR] = 0;
4453	qp->db.db[MLX5_SND_DBR] = 0;
4454	}
4455
4456	if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
4457	err = __mlx5_ib_qp_set_counter(qp: ibqp, counter: ibqp->counter);
4458	if (!err)
4459	qp->counter_pending = 0;
4460	}
4461
4462	out:
4463	kfree(objp: qpc);
4464	return err;
4465	}
4466
4467	static inline bool is_valid_mask(int mask, int req, int opt)
4468	{
4469	if ((mask & req) != req)
4470	return false;
4471
4472	if (mask & ~(req | opt))
4473	return false;
4474
4475	return true;
4476	}
4477
4478	/* check valid transition for driver QP types
4479	* for now the only QP type that this function supports is DCI
4480	*/
4481	static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
4482	enum ib_qp_attr_mask attr_mask)
4483	{
4484	int req = IB_QP_STATE;
4485	int opt = 0;
4486
4487	if (new_state == IB_QPS_RESET) {
4488	return is_valid_mask(mask: attr_mask, req, opt);
4489	} else if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4490	req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
4491	return is_valid_mask(mask: attr_mask, req, opt);
4492	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4493	opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
4494	return is_valid_mask(mask: attr_mask, req, opt);
4495	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4496	req |= IB_QP_PATH_MTU;
4497	opt = IB_QP_PKEY_INDEX | IB_QP_AV;
4498	return is_valid_mask(mask: attr_mask, req, opt);
4499	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4500	req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4501	IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
4502	opt = IB_QP_MIN_RNR_TIMER;
4503	return is_valid_mask(mask: attr_mask, req, opt);
4504	} else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
4505	opt = IB_QP_MIN_RNR_TIMER;
4506	return is_valid_mask(mask: attr_mask, req, opt);
4507	} else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
4508	return is_valid_mask(mask: attr_mask, req, opt);
4509	}
4510	return false;
4511	}
4512
4513	/* mlx5_ib_modify_dct: modify a DCT QP
4514	* valid transitions are:
4515	* RESET to INIT: must set access_flags, pkey_index and port
4516	* INIT to RTR : must set min_rnr_timer, tclass, flow_label,
4517	* mtu, gid_index and hop_limit
4518	* Other transitions and attributes are illegal
4519	*/
4520	static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4521	int attr_mask, struct mlx5_ib_modify_qp *ucmd,
4522	struct ib_udata *udata)
4523	{
4524	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4525	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibqp->device);
4526	enum ib_qp_state cur_state, new_state;
4527	int required = IB_QP_STATE;
4528	void *dctc;
4529	int err;
4530
4531	if (!(attr_mask & IB_QP_STATE))
4532	return -EINVAL;
4533
4534	cur_state = qp->state;
4535	new_state = attr->qp_state;
4536
4537	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
4538	if (MLX5_CAP_GEN(dev->mdev, ece_support) && ucmd->ece_options)
4539	/*
4540	* DCT doesn't initialize QP till modify command is executed,
4541	* so we need to overwrite previously set ECE field if user
4542	* provided any value except zero, which means not set/not
4543	* valid.
4544	*/
4545	MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
4546
4547	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4548	u16 set_id;
4549
4550	required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
4551	if (!is_valid_mask(mask: attr_mask, req: required, opt: 0))
4552	return -EINVAL;
4553
4554	if (attr->port_num == 0 ||
4555	attr->port_num > dev->num_ports) {
4556	mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4557	attr->port_num, dev->num_ports);
4558	return -EINVAL;
4559	}
4560	if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
4561	MLX5_SET(dctc, dctc, rre, 1);
4562	if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
4563	MLX5_SET(dctc, dctc, rwe, 1);
4564	if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
4565	int atomic_mode;
4566
4567	atomic_mode = get_atomic_mode(dev, qp);
4568	if (atomic_mode < 0)
4569	return -EOPNOTSUPP;
4570
4571	MLX5_SET(dctc, dctc, atomic_mode, atomic_mode);
4572	MLX5_SET(dctc, dctc, rae, 1);
4573	}
4574	MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
4575	if (mlx5_lag_is_active(dev: dev->mdev))
4576	MLX5_SET(dctc, dctc, port,
4577	get_tx_affinity_rr(dev, udata));
4578	else
4579	MLX5_SET(dctc, dctc, port, attr->port_num);
4580
4581	set_id = mlx5_ib_get_counters_id(dev, port_num: attr->port_num - 1);
4582	MLX5_SET(dctc, dctc, counter_set_id, set_id);
4583
4584	qp->port = attr->port_num;
4585	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4586	struct mlx5_ib_modify_qp_resp resp = {};
4587	u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};
4588	u32 min_resp_len = offsetofend(typeof(resp), dctn);
4589
4590	if (udata->outlen < min_resp_len)
4591	return -EINVAL;
4592	/*
4593	* If we don't have enough space for the ECE options,
4594	* simply indicate it with resp.response_length.
4595	*/
4596	resp.response_length = (udata->outlen < sizeof(resp)) ?
4597	min_resp_len :
4598	sizeof(resp);
4599
4600	required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
4601	if (!is_valid_mask(mask: attr_mask, req: required, opt: 0))
4602	return -EINVAL;
4603	MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
4604	MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
4605	MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
4606	MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
4607	MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
4608	MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
4609	if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
4610	MLX5_SET(dctc, dctc, eth_prio, attr->ah_attr.sl & 0x7);
4611	if (qp->is_ooo_rq) {
4612	MLX5_SET(dctc, dctc, dp_ordering_1, 1);
4613	MLX5_SET(dctc, dctc, dp_ordering_force, 1);
4614	}
4615
4616	err = mlx5_core_create_dct(dev, qp: &qp->dct.mdct, in: qp->dct.in,
4617	MLX5_ST_SZ_BYTES(create_dct_in), out,
4618	outlen: sizeof(out));
4619	err = mlx5_cmd_check(dev: dev->mdev, err, in: qp->dct.in, out);
4620	if (err)
4621	return err;
4622	resp.dctn = qp->dct.mdct.mqp.qpn;
4623	if (MLX5_CAP_GEN(dev->mdev, ece_support))
4624	resp.ece_options = MLX5_GET(create_dct_out, out, ece);
4625	err = ib_copy_to_udata(udata, src: &resp, len: resp.response_length);
4626	if (err) {
4627	mlx5_core_destroy_dct(dev, dct: &qp->dct.mdct);
4628	return err;
4629	}
4630	} else {
4631	mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
4632	return -EINVAL;
4633	}
4634
4635	qp->state = new_state;
4636	return 0;
4637	}
4638
4639	static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,
4640	struct mlx5_ib_qp *qp)
4641	{
4642	if (dev->profile != &raw_eth_profile)
4643	return true;
4644
4645	if (qp->type == IB_QPT_RAW_PACKET || qp->type == MLX5_IB_QPT_REG_UMR)
4646	return true;
4647
4648	return false;
4649	}
4650
4651	static int validate_rd_atomic(struct mlx5_ib_dev *dev, struct ib_qp_attr *attr,
4652	int attr_mask, enum ib_qp_type qp_type)
4653	{
4654	int log_max_ra_res;
4655	int log_max_ra_req;
4656
4657	if (qp_type == MLX5_IB_QPT_DCI) {
4658	log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4659	log_max_ra_res_dc);
4660	log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4661	log_max_ra_req_dc);
4662	} else {
4663	log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4664	log_max_ra_res_qp);
4665	log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4666	log_max_ra_req_qp);
4667	}
4668
4669	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
4670	attr->max_rd_atomic > log_max_ra_res) {
4671	mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
4672	attr->max_rd_atomic);
4673	return false;
4674	}
4675
4676	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
4677	attr->max_dest_rd_atomic > log_max_ra_req) {
4678	mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
4679	attr->max_dest_rd_atomic);
4680	return false;
4681	}
4682	return true;
4683	}
4684
4685	int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4686	int attr_mask, struct ib_udata *udata)
4687	{
4688	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibqp->device);
4689	struct mlx5_ib_modify_qp_resp resp = {};
4690	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4691	struct mlx5_ib_modify_qp ucmd = {};
4692	enum ib_qp_type qp_type;
4693	enum ib_qp_state cur_state, new_state;
4694	int err = -EINVAL;
4695
4696	if (!mlx5_ib_modify_qp_allowed(dev, qp))
4697	return -EOPNOTSUPP;
4698
4699	if (attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
4700	return -EOPNOTSUPP;
4701
4702	if (ibqp->rwq_ind_tbl)
4703	return -ENOSYS;
4704
4705	if (udata && udata->inlen) {
4706	if (udata->inlen < offsetofend(typeof(ucmd), ece_options))
4707	return -EINVAL;
4708
4709	if (udata->inlen > sizeof(ucmd) &&
4710	!ib_is_udata_cleared(udata, offset: sizeof(ucmd),
4711	len: udata->inlen - sizeof(ucmd)))
4712	return -EOPNOTSUPP;
4713
4714	if (ib_copy_from_udata(dest: &ucmd, udata,
4715	min(udata->inlen, sizeof(ucmd))))
4716	return -EFAULT;
4717
4718	if (ucmd.comp_mask & ~MLX5_IB_MODIFY_QP_OOO_DP ||
4719	memchr_inv(p: &ucmd.burst_info.reserved, c: 0,
4720	size: sizeof(ucmd.burst_info.reserved)))
4721	return -EOPNOTSUPP;
4722
4723	if (ucmd.comp_mask & MLX5_IB_MODIFY_QP_OOO_DP) {
4724	if (!get_dp_ooo_cap(mdev: dev->mdev, qp_type: qp->type))
4725	return -EOPNOTSUPP;
4726	qp->is_ooo_rq = 1;
4727	}
4728	}
4729
4730	if (qp->type == IB_QPT_GSI)
4731	return mlx5_ib_gsi_modify_qp(qp: ibqp, attr, attr_mask);
4732
4733	qp_type = (qp->type == MLX5_IB_QPT_HW_GSI) ? IB_QPT_GSI : qp->type;
4734
4735	if (qp_type == MLX5_IB_QPT_DCT)
4736	return mlx5_ib_modify_dct(ibqp, attr, attr_mask, ucmd: &ucmd, udata);
4737
4738	mutex_lock(&qp->mutex);
4739
4740	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
4741	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
4742
4743	if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
4744	if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
4745	mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
4746	attr_mask);
4747	goto out;
4748	}
4749	} else if (qp_type != MLX5_IB_QPT_REG_UMR &&
4750	qp_type != MLX5_IB_QPT_DCI &&
4751	!ib_modify_qp_is_ok(cur_state, next_state: new_state, type: qp_type,
4752	mask: attr_mask)) {
4753	mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4754	cur_state, new_state, qp->type, attr_mask);
4755	goto out;
4756	} else if (qp_type == MLX5_IB_QPT_DCI &&
4757	!modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
4758	mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4759	cur_state, new_state, qp_type, attr_mask);
4760	goto out;
4761	}
4762
4763	if ((attr_mask & IB_QP_PORT) &&
4764	(attr->port_num == 0 ||
4765	attr->port_num > dev->num_ports)) {
4766	mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4767	attr->port_num, dev->num_ports);
4768	goto out;
4769	}
4770
4771	if ((attr_mask & IB_QP_PKEY_INDEX) &&
4772	attr->pkey_index >= dev->pkey_table_len) {
4773	mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
4774	goto out;
4775	}
4776
4777	if (!validate_rd_atomic(dev, attr, attr_mask, qp_type))
4778	goto out;
4779
4780	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
4781	err = 0;
4782	goto out;
4783	}
4784
4785	err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
4786	new_state, ucmd: &ucmd, resp: &resp, udata);
4787
4788	/* resp.response_length is set in ECE supported flows only */
4789	if (!err && resp.response_length &&
4790	udata->outlen >= resp.response_length)
4791	/* Return -EFAULT to the user and expect him to destroy QP. */
4792	err = ib_copy_to_udata(udata, src: &resp, len: resp.response_length);
4793
4794	out:
4795	mutex_unlock(lock: &qp->mutex);
4796	return err;
4797	}
4798
4799	static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4800	{
4801	switch (mlx5_state) {
4802	case MLX5_QP_STATE_RST: return IB_QPS_RESET;
4803	case MLX5_QP_STATE_INIT: return IB_QPS_INIT;
4804	case MLX5_QP_STATE_RTR: return IB_QPS_RTR;
4805	case MLX5_QP_STATE_RTS: return IB_QPS_RTS;
4806	case MLX5_QP_STATE_SQ_DRAINING:
4807	case MLX5_QP_STATE_SQD: return IB_QPS_SQD;
4808	case MLX5_QP_STATE_SQER: return IB_QPS_SQE;
4809	case MLX5_QP_STATE_ERR: return IB_QPS_ERR;
4810	default: return -1;
4811	}
4812	}
4813
4814	static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4815	{
4816	switch (mlx5_mig_state) {
4817	case MLX5_QP_PM_ARMED: return IB_MIG_ARMED;
4818	case MLX5_QP_PM_REARM: return IB_MIG_REARM;
4819	case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
4820	default: return -1;
4821	}
4822	}
4823
4824	static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
4825	struct rdma_ah_attr *ah_attr, void *path)
4826	{
4827	int port = MLX5_GET(ads, path, vhca_port_num);
4828	int static_rate;
4829
4830	memset(ah_attr, 0, sizeof(*ah_attr));
4831
4832	if (!port || port > ibdev->num_ports)
4833	return;
4834
4835	ah_attr->type = rdma_ah_find_type(dev: &ibdev->ib_dev, port_num: port);
4836
4837	rdma_ah_set_port_num(attr: ah_attr, port_num: port);
4838	rdma_ah_set_sl(attr: ah_attr, MLX5_GET(ads, path, sl));
4839
4840	rdma_ah_set_dlid(attr: ah_attr, MLX5_GET(ads, path, rlid));
4841	rdma_ah_set_path_bits(attr: ah_attr, MLX5_GET(ads, path, mlid));
4842
4843	static_rate = MLX5_GET(ads, path, stat_rate);
4844	rdma_ah_set_static_rate(attr: ah_attr, static_rate: mlx5_to_ib_rate_map(rate: static_rate));
4845	if (MLX5_GET(ads, path, grh) ||
4846	ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
4847	rdma_ah_set_grh(attr: ah_attr, NULL, MLX5_GET(ads, path, flow_label),
4848	MLX5_GET(ads, path, src_addr_index),
4849	MLX5_GET(ads, path, hop_limit),
4850	MLX5_GET(ads, path, tclass));
4851	rdma_ah_set_dgid_raw(attr: ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip));
4852	}
4853	}
4854
4855	static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4856	struct mlx5_ib_sq *sq,
4857	u8 *sq_state)
4858	{
4859	int err;
4860
4861	err = mlx5_core_query_sq_state(dev: dev->mdev, sqn: sq->base.mqp.qpn, state: sq_state);
4862	if (err)
4863	goto out;
4864	sq->state = *sq_state;
4865
4866	out:
4867	return err;
4868	}
4869
4870	static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4871	struct mlx5_ib_rq *rq,
4872	u8 *rq_state)
4873	{
4874	void *out;
4875	void *rqc;
4876	int inlen;
4877	int err;
4878
4879	inlen = MLX5_ST_SZ_BYTES(query_rq_out);
4880	out = kvzalloc(inlen, GFP_KERNEL);
4881	if (!out)
4882	return -ENOMEM;
4883
4884	err = mlx5_core_query_rq(dev: dev->mdev, rqn: rq->base.mqp.qpn, out);
4885	if (err)
4886	goto out;
4887
4888	rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4889	*rq_state = MLX5_GET(rqc, rqc, state);
4890	rq->state = *rq_state;
4891
4892	out:
4893	kvfree(addr: out);
4894	return err;
4895	}
4896
4897	static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4898	struct mlx5_ib_qp *qp, u8 *qp_state)
4899	{
4900	static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4901	[MLX5_RQC_STATE_RST] = {
4902	[MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4903	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4904	[MLX5_SQC_STATE_ERR] = MLX5_QP_STATE_BAD,
4905	[MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4906	},
4907	[MLX5_RQC_STATE_RDY] = {
4908	[MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
4909	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4910	[MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4911	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
4912	},
4913	[MLX5_RQC_STATE_ERR] = {
4914	[MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4915	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4916	[MLX5_SQC_STATE_ERR] = IB_QPS_ERR,
4917	[MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4918	},
4919	[MLX5_RQ_STATE_NA] = {
4920	[MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
4921	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4922	[MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4923	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
4924	},
4925	};
4926
4927	*qp_state = sqrq_trans[rq_state][sq_state];
4928
4929	if (*qp_state == MLX5_QP_STATE_BAD) {
4930	WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4931	qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4932	qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4933	return -EINVAL;
4934	}
4935
4936	if (*qp_state == MLX5_QP_STATE)
4937	*qp_state = qp->state;
4938
4939	return 0;
4940	}
4941
4942	static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4943	struct mlx5_ib_qp *qp,
4944	u8 *raw_packet_qp_state)
4945	{
4946	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4947	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4948	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4949	int err;
4950	u8 sq_state = MLX5_SQ_STATE_NA;
4951	u8 rq_state = MLX5_RQ_STATE_NA;
4952
4953	if (qp->sq.wqe_cnt) {
4954	err = query_raw_packet_qp_sq_state(dev, sq, sq_state: &sq_state);
4955	if (err)
4956	return err;
4957	}
4958
4959	if (qp->rq.wqe_cnt) {
4960	err = query_raw_packet_qp_rq_state(dev, rq, rq_state: &rq_state);
4961	if (err)
4962	return err;
4963	}
4964
4965	return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4966	qp_state: raw_packet_qp_state);
4967	}
4968
4969	static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4970	struct ib_qp_attr *qp_attr)
4971	{
4972	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
4973	void *qpc, *pri_path, *alt_path;
4974	u32 *outb;
4975	int err;
4976
4977	outb = kzalloc(outlen, GFP_KERNEL);
4978	if (!outb)
4979	return -ENOMEM;
4980
4981	err = mlx5_core_qp_query(dev, qp: &qp->trans_qp.base.mqp, outb, outlen,
4982	qpc_ext: false);
4983	if (err)
4984	goto out;
4985
4986	qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);
4987
4988	qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));
4989	if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)
4990	qp_attr->sq_draining = 1;
4991
4992	qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);
4993	qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));
4994	qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);
4995	qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);
4996	qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);
4997	qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);
4998
4999	if (MLX5_GET(qpc, qpc, rre))
5000	qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ;
5001	if (MLX5_GET(qpc, qpc, rwe))
5002	qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_WRITE;
5003	if (MLX5_GET(qpc, qpc, rae))
5004	qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5005
5006	qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);
5007	qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);
5008	qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);
5009	qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);
5010	qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);
5011
5012	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
5013	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
5014
5015	if (qp->type == IB_QPT_RC || qp->type == IB_QPT_UC ||
5016	qp->type == IB_QPT_XRC_INI || qp->type == IB_QPT_XRC_TGT) {
5017	to_rdma_ah_attr(ibdev: dev, ah_attr: &qp_attr->ah_attr, path: pri_path);
5018	to_rdma_ah_attr(ibdev: dev, ah_attr: &qp_attr->alt_ah_attr, path: alt_path);
5019	qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
5020	qp_attr->alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);
5021	}
5022
5023	qp_attr->pkey_index = MLX5_GET(ads, pri_path, pkey_index);
5024	qp_attr->port_num = MLX5_GET(ads, pri_path, vhca_port_num);
5025	qp_attr->timeout = MLX5_GET(ads, pri_path, ack_timeout);
5026	qp_attr->alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);
5027
5028	out:
5029	kfree(outb);
5030	return err;
5031	}
5032
5033	static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
5034	struct ib_qp_attr *qp_attr, int qp_attr_mask,
5035	struct ib_qp_init_attr *qp_init_attr)
5036	{
5037	struct mlx5_core_dct *dct = &mqp->dct.mdct;
5038	u32 *out;
5039	u32 access_flags = 0;
5040	int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
5041	void *dctc;
5042	int err;
5043	int supported_mask = IB_QP_STATE |
5044	IB_QP_ACCESS_FLAGS |
5045	IB_QP_PORT |
5046	IB_QP_MIN_RNR_TIMER |
5047	IB_QP_AV |
5048	IB_QP_PATH_MTU |
5049	IB_QP_PKEY_INDEX;
5050
5051	if (qp_attr_mask & ~supported_mask)
5052	return -EINVAL;
5053	if (mqp->state != IB_QPS_RTR)
5054	return -EINVAL;
5055
5056	out = kzalloc(outlen, GFP_KERNEL);
5057	if (!out)
5058	return -ENOMEM;
5059
5060	err = mlx5_core_dct_query(dev, dct, out, outlen);
5061	if (err)
5062	goto out;
5063
5064	dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
5065
5066	if (qp_attr_mask & IB_QP_STATE)
5067	qp_attr->qp_state = IB_QPS_RTR;
5068
5069	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
5070	if (MLX5_GET(dctc, dctc, rre))
5071	access_flags |= IB_ACCESS_REMOTE_READ;
5072	if (MLX5_GET(dctc, dctc, rwe))
5073	access_flags |= IB_ACCESS_REMOTE_WRITE;
5074	if (MLX5_GET(dctc, dctc, rae))
5075	access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5076	qp_attr->qp_access_flags = access_flags;
5077	}
5078
5079	if (qp_attr_mask & IB_QP_PORT)
5080	qp_attr->port_num = mqp->port;
5081	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
5082	qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
5083	if (qp_attr_mask & IB_QP_AV) {
5084	qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
5085	qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
5086	qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
5087	qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
5088	}
5089	if (qp_attr_mask & IB_QP_PATH_MTU)
5090	qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
5091	if (qp_attr_mask & IB_QP_PKEY_INDEX)
5092	qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
5093	out:
5094	kfree(objp: out);
5095	return err;
5096	}
5097
5098	int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5099	int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
5100	{
5101	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibqp->device);
5102	struct mlx5_ib_qp *qp = to_mqp(ibqp);
5103	int err = 0;
5104	u8 raw_packet_qp_state;
5105
5106	if (ibqp->rwq_ind_tbl)
5107	return -ENOSYS;
5108
5109	if (qp->type == IB_QPT_GSI)
5110	return mlx5_ib_gsi_query_qp(qp: ibqp, qp_attr, qp_attr_mask,
5111	qp_init_attr);
5112
5113	/* Not all of output fields are applicable, make sure to zero them */
5114	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5115	memset(qp_attr, 0, sizeof(*qp_attr));
5116
5117	if (unlikely(qp->type == MLX5_IB_QPT_DCT))
5118	return mlx5_ib_dct_query_qp(dev, mqp: qp, qp_attr,
5119	qp_attr_mask, qp_init_attr);
5120
5121	mutex_lock(&qp->mutex);
5122
5123	if (qp->type == IB_QPT_RAW_PACKET ||
5124	qp->flags & IB_QP_CREATE_SOURCE_QPN) {
5125	err = query_raw_packet_qp_state(dev, qp, raw_packet_qp_state: &raw_packet_qp_state);
5126	if (err)
5127	goto out;
5128	qp->state = raw_packet_qp_state;
5129	qp_attr->port_num = 1;
5130	} else {
5131	err = query_qp_attr(dev, qp, qp_attr);
5132	if (err)
5133	goto out;
5134	}
5135
5136	qp_attr->qp_state = qp->state;
5137	qp_attr->cur_qp_state = qp_attr->qp_state;
5138	qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
5139	qp_attr->cap.max_recv_sge = qp->rq.max_gs;
5140
5141	if (!ibqp->uobject) {
5142	qp_attr->cap.max_send_wr = qp->sq.max_post;
5143	qp_attr->cap.max_send_sge = qp->sq.max_gs;
5144	qp_init_attr->qp_context = ibqp->qp_context;
5145	} else {
5146	qp_attr->cap.max_send_wr = 0;
5147	qp_attr->cap.max_send_sge = 0;
5148	}
5149
5150	qp_init_attr->qp_type = qp->type;
5151	qp_init_attr->recv_cq = ibqp->recv_cq;
5152	qp_init_attr->send_cq = ibqp->send_cq;
5153	qp_init_attr->srq = ibqp->srq;
5154	qp_attr->cap.max_inline_data = qp->max_inline_data;
5155
5156	qp_init_attr->cap = qp_attr->cap;
5157
5158	qp_init_attr->create_flags = qp->flags;
5159
5160	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5161	IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
5162
5163	out:
5164	mutex_unlock(lock: &qp->mutex);
5165	return err;
5166	}
5167
5168	int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
5169	{
5170	struct mlx5_ib_dev *dev = to_mdev(ibdev: ibxrcd->device);
5171	struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
5172
5173	if (!MLX5_CAP_GEN(dev->mdev, xrc))
5174	return -EOPNOTSUPP;
5175
5176	return mlx5_cmd_xrcd_alloc(dev: dev->mdev, xrcdn: &xrcd->xrcdn, uid: 0);
5177	}
5178
5179	int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
5180	{
5181	struct mlx5_ib_dev *dev = to_mdev(ibdev: xrcd->device);
5182	u32 xrcdn = to_mxrcd(ibxrcd: xrcd)->xrcdn;
5183
5184	return mlx5_cmd_xrcd_dealloc(dev: dev->mdev, xrcdn, uid: 0);
5185	}
5186
5187	static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
5188	{
5189	struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
5190	struct mlx5_ib_dev *dev = to_mdev(ibdev: rwq->ibwq.device);
5191	struct ib_event event;
5192
5193	if (rwq->ibwq.event_handler) {
5194	event.device = rwq->ibwq.device;
5195	event.element.wq = &rwq->ibwq;
5196	switch (type) {
5197	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
5198	event.event = IB_EVENT_WQ_FATAL;
5199	break;
5200	default:
5201	mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
5202	return;
5203	}
5204
5205	rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
5206	}
5207	}
5208
5209	static int set_delay_drop(struct mlx5_ib_dev *dev)
5210	{
5211	int err = 0;
5212
5213	mutex_lock(&dev->delay_drop.lock);
5214	if (dev->delay_drop.activate)
5215	goto out;
5216
5217	err = mlx5_core_set_delay_drop(dev, timeout_usec: dev->delay_drop.timeout);
5218	if (err)
5219	goto out;
5220
5221	dev->delay_drop.activate = true;
5222	out:
5223	mutex_unlock(lock: &dev->delay_drop.lock);
5224
5225	if (!err)
5226	atomic_inc(v: &dev->delay_drop.rqs_cnt);
5227	return err;
5228	}
5229
5230	static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
5231	struct ib_wq_init_attr *init_attr)
5232	{
5233	struct mlx5_ib_dev *dev;
5234	int has_net_offloads;
5235	__be64 *rq_pas0;
5236	int ts_format;
5237	void *in;
5238	void *rqc;
5239	void *wq;
5240	int inlen;
5241	int err;
5242
5243	dev = to_mdev(ibdev: pd->device);
5244
5245	ts_format = get_rq_ts_format(dev, recv_cq: to_mcq(ibcq: init_attr->cq));
5246	if (ts_format < 0)
5247	return ts_format;
5248
5249	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
5250	in = kvzalloc(inlen, GFP_KERNEL);
5251	if (!in)
5252	return -ENOMEM;
5253
5254	MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
5255	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5256	MLX5_SET(rqc, rqc, mem_rq_type,
5257	MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
5258	MLX5_SET(rqc, rqc, ts_format, ts_format);
5259	MLX5_SET(rqc, rqc, user_index, rwq->user_index);
5260	MLX5_SET(rqc, rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
5261	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
5262	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
5263	wq = MLX5_ADDR_OF(rqc, rqc, wq);
5264	MLX5_SET(wq, wq, wq_type,
5265	rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
5266	MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
5267	if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5268	if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
5269	mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
5270	err = -EOPNOTSUPP;
5271	goto out;
5272	} else {
5273	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
5274	}
5275	}
5276	MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
5277	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
5278	/*
5279	* In Firmware number of strides in each WQE is:
5280	* "512 * 2^single_wqe_log_num_of_strides"
5281	* Values 3 to 8 are accepted as 10 to 15, 9 to 18 are
5282	* accepted as 0 to 9
5283	*/
5284	static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,
5285	2, 3, 4, 5, 6, 7, 8, 9 };
5286	MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5287	MLX5_SET(wq, wq, log_wqe_stride_size,
5288	rwq->single_stride_log_num_of_bytes -
5289	MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
5290	MLX5_SET(wq, wq, log_wqe_num_of_strides,
5291	fw_map[rwq->log_num_strides -
5292	MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);
5293	}
5294	MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5295	MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
5296	MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
5297	MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
5298	MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
5299	MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
5300	has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
5301	if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5302	if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5303	mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5304	err = -EOPNOTSUPP;
5305	goto out;
5306	}
5307	} else {
5308	MLX5_SET(rqc, rqc, vsd, 1);
5309	}
5310	if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
5311	if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
5312	mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
5313	err = -EOPNOTSUPP;
5314	goto out;
5315	}
5316	MLX5_SET(rqc, rqc, scatter_fcs, 1);
5317	}
5318	if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5319	if (!(dev->ib_dev.attrs.raw_packet_caps &
5320	IB_RAW_PACKET_CAP_DELAY_DROP)) {
5321	mlx5_ib_dbg(dev, "Delay drop is not supported\n");
5322	err = -EOPNOTSUPP;
5323	goto out;
5324	}
5325	MLX5_SET(rqc, rqc, delay_drop_en, 1);
5326	}
5327	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
5328	mlx5_ib_populate_pas(umem: rwq->umem, page_size: 1UL << rwq->page_shift, pas: rq_pas0, access_flags: 0);
5329	err = mlx5_core_create_rq_tracked(dev, in, inlen, rq: &rwq->core_qp);
5330	if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5331	err = set_delay_drop(dev);
5332	if (err) {
5333	mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5334	err);
5335	mlx5_core_destroy_rq_tracked(dev, rq: &rwq->core_qp);
5336	} else {
5337	rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5338	}
5339	}
5340	out:
5341	kvfree(addr: in);
5342	return err;
5343	}
5344
5345	static int set_user_rq_size(struct mlx5_ib_dev *dev,
5346	struct ib_wq_init_attr *wq_init_attr,
5347	struct mlx5_ib_create_wq *ucmd,
5348	struct mlx5_ib_rwq *rwq)
5349	{
5350	/* Sanity check RQ size before proceeding */
5351	if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
5352	return -EINVAL;
5353
5354	if (!ucmd->rq_wqe_count)
5355	return -EINVAL;
5356
5357	rwq->wqe_count = ucmd->rq_wqe_count;
5358	rwq->wqe_shift = ucmd->rq_wqe_shift;
5359	if (check_shl_overflow(rwq->wqe_count, rwq->wqe_shift, &rwq->buf_size))
5360	return -EINVAL;
5361
5362	rwq->log_rq_stride = rwq->wqe_shift;
5363	rwq->log_rq_size = ilog2(rwq->wqe_count);
5364	return 0;
5365	}
5366
5367	static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
5368	{
5369	if ((log_num_strides > MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
5370	(log_num_strides < MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5371	return false;
5372
5373	if (!MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) &&
5374	(log_num_strides < MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5375	return false;
5376
5377	return true;
5378	}
5379
5380	static int prepare_user_rq(struct ib_pd *pd,
5381	struct ib_wq_init_attr *init_attr,
5382	struct ib_udata *udata,
5383	struct mlx5_ib_rwq *rwq)
5384	{
5385	struct mlx5_ib_dev *dev = to_mdev(ibdev: pd->device);
5386	struct mlx5_ib_create_wq ucmd = {};
5387	int err;
5388	size_t required_cmd_sz;
5389
5390	required_cmd_sz = offsetofend(struct mlx5_ib_create_wq,
5391	single_stride_log_num_of_bytes);
5392	if (udata->inlen < required_cmd_sz) {
5393	mlx5_ib_dbg(dev, "invalid inlen\n");
5394	return -EINVAL;
5395	}
5396
5397	if (udata->inlen > sizeof(ucmd) &&
5398	!ib_is_udata_cleared(udata, offset: sizeof(ucmd),
5399	len: udata->inlen - sizeof(ucmd))) {
5400	mlx5_ib_dbg(dev, "inlen is not supported\n");
5401	return -EOPNOTSUPP;
5402	}
5403
5404	if (ib_copy_from_udata(dest: &ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
5405	mlx5_ib_dbg(dev, "copy failed\n");
5406	return -EFAULT;
5407	}
5408
5409	if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
5410	mlx5_ib_dbg(dev, "invalid comp mask\n");
5411	return -EOPNOTSUPP;
5412	} else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
5413	if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
5414	mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
5415	return -EOPNOTSUPP;
5416	}
5417	if ((ucmd.single_stride_log_num_of_bytes <
5418	MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
5419	(ucmd.single_stride_log_num_of_bytes >
5420	MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
5421	mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
5422	ucmd.single_stride_log_num_of_bytes,
5423	MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
5424	MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5425	return -EINVAL;
5426	}
5427	if (!log_of_strides_valid(dev,
5428	log_num_strides: ucmd.single_wqe_log_num_of_strides)) {
5429	mlx5_ib_dbg(
5430	dev,
5431	"Invalid log num strides (%u. Range is %u - %u)\n",
5432	ucmd.single_wqe_log_num_of_strides,
5433	MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) ?
5434	MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :
5435	MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5436	MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
5437	return -EINVAL;
5438	}
5439	rwq->single_stride_log_num_of_bytes =
5440	ucmd.single_stride_log_num_of_bytes;
5441	rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
5442	rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
5443	rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
5444	}
5445
5446	err = set_user_rq_size(dev, wq_init_attr: init_attr, ucmd: &ucmd, rwq);
5447	if (err) {
5448	mlx5_ib_dbg(dev, "err %d\n", err);
5449	return err;
5450	}
5451
5452	err = create_user_rq(dev, pd, udata, rwq, ucmd: &ucmd);
5453	if (err) {
5454	mlx5_ib_dbg(dev, "err %d\n", err);
5455	return err;
5456	}
5457
5458	rwq->user_index = ucmd.user_index;
5459	return 0;
5460	}
5461
5462	struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
5463	struct ib_wq_init_attr *init_attr,
5464	struct ib_udata *udata)
5465	{
5466	struct mlx5_ib_dev *dev;
5467	struct mlx5_ib_rwq *rwq;
5468	struct mlx5_ib_create_wq_resp resp = {};
5469	size_t min_resp_len;
5470	int err;
5471
5472	if (!udata)
5473	return ERR_PTR(error: -ENOSYS);
5474
5475	min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved);
5476	if (udata->outlen && udata->outlen < min_resp_len)
5477	return ERR_PTR(error: -EINVAL);
5478
5479	if (!capable(CAP_SYS_RAWIO) &&
5480	init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP)
5481	return ERR_PTR(error: -EPERM);
5482
5483	dev = to_mdev(ibdev: pd->device);
5484	switch (init_attr->wq_type) {
5485	case IB_WQT_RQ:
5486	rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5487	if (!rwq)
5488	return ERR_PTR(error: -ENOMEM);
5489	err = prepare_user_rq(pd, init_attr, udata, rwq);
5490	if (err)
5491	goto err;
5492	err = create_rq(rwq, pd, init_attr);
5493	if (err)
5494	goto err_user_rq;
5495	break;
5496	default:
5497	mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5498	init_attr->wq_type);
5499	return ERR_PTR(error: -EINVAL);
5500	}
5501
5502	rwq->ibwq.wq_num = rwq->core_qp.qpn;
5503	rwq->ibwq.state = IB_WQS_RESET;
5504	if (udata->outlen) {
5505	resp.response_length = offsetofend(
5506	struct mlx5_ib_create_wq_resp, response_length);
5507	err = ib_copy_to_udata(udata, src: &resp, len: resp.response_length);
5508	if (err)
5509	goto err_copy;
5510	}
5511
5512	rwq->core_qp.event = mlx5_ib_wq_event;
5513	rwq->ibwq.event_handler = init_attr->event_handler;
5514	return &rwq->ibwq;
5515
5516	err_copy:
5517	mlx5_core_destroy_rq_tracked(dev, rq: &rwq->core_qp);
5518	err_user_rq:
5519	destroy_user_rq(dev, pd, rwq, udata);
5520	err:
5521	kfree(objp: rwq);
5522	return ERR_PTR(error: err);
5523	}
5524
5525	int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
5526	{
5527	struct mlx5_ib_dev *dev = to_mdev(ibdev: wq->device);
5528	struct mlx5_ib_rwq *rwq = to_mrwq(ibwq: wq);
5529	int ret;
5530
5531	ret = mlx5_core_destroy_rq_tracked(dev, rq: &rwq->core_qp);
5532	if (ret)
5533	return ret;
5534	destroy_user_rq(dev, pd: wq->pd, rwq, udata);
5535	kfree(objp: rwq);
5536	return 0;
5537	}
5538
5539	int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
5540	struct ib_rwq_ind_table_init_attr *init_attr,
5541	struct ib_udata *udata)
5542	{
5543	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl =
5544	to_mrwq_ind_table(ib_rwq_ind_tbl: ib_rwq_ind_table);
5545	struct mlx5_ib_dev *dev = to_mdev(ibdev: ib_rwq_ind_table->device);
5546	int sz = 1 << init_attr->log_ind_tbl_size;
5547	struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5548	size_t min_resp_len;
5549	int inlen;
5550	int err;
5551	int i;
5552	u32 *in;
5553	void *rqtc;
5554
5555	if (udata->inlen > 0 &&
5556	!ib_is_udata_cleared(udata, offset: 0,
5557	len: udata->inlen))
5558	return -EOPNOTSUPP;
5559
5560	if (init_attr->log_ind_tbl_size >
5561	MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5562	mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5563	init_attr->log_ind_tbl_size,
5564	MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5565	return -EINVAL;
5566	}
5567
5568	min_resp_len =
5569	offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved);
5570	if (udata->outlen && udata->outlen < min_resp_len)
5571	return -EINVAL;
5572
5573	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
5574	in = kvzalloc(inlen, GFP_KERNEL);
5575	if (!in)
5576	return -ENOMEM;
5577
5578	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5579
5580	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5581	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5582
5583	for (i = 0; i < sz; i++)
5584	MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5585
5586	rwq_ind_tbl->uid = to_mpd(ibpd: init_attr->ind_tbl[0]->pd)->uid;
5587	MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
5588
5589	err = mlx5_core_create_rqt(dev: dev->mdev, in, inlen, rqtn: &rwq_ind_tbl->rqtn);
5590	kvfree(addr: in);
5591	if (err)
5592	return err;
5593
5594	rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5595	if (udata->outlen) {
5596	resp.response_length =
5597	offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,
5598	response_length);
5599	err = ib_copy_to_udata(udata, src: &resp, len: resp.response_length);
5600	if (err)
5601	goto err_copy;
5602	}
5603
5604	return 0;
5605
5606	err_copy:
5607	mlx5_cmd_destroy_rqt(dev: dev->mdev, rqtn: rwq_ind_tbl->rqtn, uid: rwq_ind_tbl->uid);
5608	return err;
5609	}
5610
5611	int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5612	{
5613	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5614	struct mlx5_ib_dev *dev = to_mdev(ibdev: ib_rwq_ind_tbl->device);
5615
5616	return mlx5_cmd_destroy_rqt(dev: dev->mdev, rqtn: rwq_ind_tbl->rqtn, uid: rwq_ind_tbl->uid);
5617	}
5618
5619	int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5620	u32 wq_attr_mask, struct ib_udata *udata)
5621	{
5622	struct mlx5_ib_dev *dev = to_mdev(ibdev: wq->device);
5623	struct mlx5_ib_rwq *rwq = to_mrwq(ibwq: wq);
5624	struct mlx5_ib_modify_wq ucmd = {};
5625	size_t required_cmd_sz;
5626	int curr_wq_state;
5627	int wq_state;
5628	int inlen;
5629	int err;
5630	void *rqc;
5631	void *in;
5632
5633	required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved);
5634	if (udata->inlen < required_cmd_sz)
5635	return -EINVAL;
5636
5637	if (udata->inlen > sizeof(ucmd) &&
5638	!ib_is_udata_cleared(udata, offset: sizeof(ucmd),
5639	len: udata->inlen - sizeof(ucmd)))
5640	return -EOPNOTSUPP;
5641
5642	if (ib_copy_from_udata(dest: &ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5643	return -EFAULT;
5644
5645	if (ucmd.comp_mask || ucmd.reserved)
5646	return -EOPNOTSUPP;
5647
5648	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
5649	in = kvzalloc(inlen, GFP_KERNEL);
5650	if (!in)
5651	return -ENOMEM;
5652
5653	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5654
5655	curr_wq_state = wq_attr->curr_wq_state;
5656	wq_state = wq_attr->wq_state;
5657	if (curr_wq_state == IB_WQS_ERR)
5658	curr_wq_state = MLX5_RQC_STATE_ERR;
5659	if (wq_state == IB_WQS_ERR)
5660	wq_state = MLX5_RQC_STATE_ERR;
5661	MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
5662	MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
5663	MLX5_SET(rqc, rqc, state, wq_state);
5664
5665	if (wq_attr_mask & IB_WQ_FLAGS) {
5666	if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5667	if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5668	MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5669	mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5670	err = -EOPNOTSUPP;
5671	goto out;
5672	}
5673	MLX5_SET64(modify_rq_in, in, modify_bitmask,
5674	MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5675	MLX5_SET(rqc, rqc, vsd,
5676	(wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5677	}
5678
5679	if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5680	mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5681	err = -EOPNOTSUPP;
5682	goto out;
5683	}
5684	}
5685
5686	if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
5687	u16 set_id;
5688
5689	set_id = mlx5_ib_get_counters_id(dev, port_num: 0);
5690	if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5691	MLX5_SET64(modify_rq_in, in, modify_bitmask,
5692	MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
5693	MLX5_SET(rqc, rqc, counter_set_id, set_id);
5694	} else
5695	dev_info_once(
5696	&dev->ib_dev.dev,
5697	"Receive WQ counters are not supported on current FW\n");
5698	}
5699
5700	err = mlx5_core_modify_rq(dev: dev->mdev, rqn: rwq->core_qp.qpn, in);
5701	if (!err)
5702	rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5703
5704	out:
5705	kvfree(addr: in);
5706	return err;
5707	}
5708
5709	struct mlx5_ib_drain_cqe {
5710	struct ib_cqe cqe;
5711	struct completion done;
5712	};
5713
5714	static void mlx5_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
5715	{
5716	struct mlx5_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
5717	struct mlx5_ib_drain_cqe,
5718	cqe);
5719
5720	complete(&cqe->done);
5721	}
5722
5723	/* This function returns only once the drained WR was completed */
5724	static void handle_drain_completion(struct ib_cq *cq,
5725	struct mlx5_ib_drain_cqe *sdrain,
5726	struct mlx5_ib_dev *dev)
5727	{
5728	struct mlx5_core_dev *mdev = dev->mdev;
5729
5730	if (cq->poll_ctx == IB_POLL_DIRECT) {
5731	while (wait_for_completion_timeout(x: &sdrain->done, HZ / 10) <= 0)
5732	ib_process_cq_direct(cq, budget: -1);
5733	return;
5734	}
5735
5736	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5737	struct mlx5_ib_cq *mcq = to_mcq(ibcq: cq);
5738	bool triggered = false;
5739	unsigned long flags;
5740
5741	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
5742	/* Make sure that the CQ handler won't run if wasn't run yet */
5743	if (!mcq->mcq.reset_notify_added)
5744	mcq->mcq.reset_notify_added = 1;
5745	else
5746	triggered = true;
5747	spin_unlock_irqrestore(lock: &dev->reset_flow_resource_lock, flags);
5748
5749	if (triggered) {
5750	/* Wait for any scheduled/running task to be ended */
5751	switch (cq->poll_ctx) {
5752	case IB_POLL_SOFTIRQ:
5753	irq_poll_disable(&cq->iop);
5754	irq_poll_enable(&cq->iop);
5755	break;
5756	case IB_POLL_WORKQUEUE:
5757	cancel_work_sync(work: &cq->work);
5758	break;
5759	default:
5760	WARN_ON_ONCE(1);
5761	}
5762	}
5763
5764	/* Run the CQ handler - this makes sure that the drain WR will
5765	* be processed if wasn't processed yet.
5766	*/
5767	mcq->mcq.comp(&mcq->mcq, NULL);
5768	}
5769
5770	wait_for_completion(&sdrain->done);
5771	}
5772
5773	void mlx5_ib_drain_sq(struct ib_qp *qp)
5774	{
5775	struct ib_cq *cq = qp->send_cq;
5776	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5777	struct mlx5_ib_drain_cqe sdrain;
5778	const struct ib_send_wr *bad_swr;
5779	struct ib_rdma_wr swr = {
5780	.wr = {
5781	.next = NULL,
5782	{ .wr_cqe = &sdrain.cqe, },
5783	.opcode = IB_WR_RDMA_WRITE,
5784	},
5785	};
5786	int ret;
5787	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
5788	struct mlx5_core_dev *mdev = dev->mdev;
5789
5790	ret = ib_modify_qp(qp, qp_attr: &attr, qp_attr_mask: IB_QP_STATE);
5791	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5792	WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5793	return;
5794	}
5795
5796	sdrain.cqe.done = mlx5_ib_drain_qp_done;
5797	init_completion(x: &sdrain.done);
5798
5799	ret = mlx5_ib_post_send_drain(ibqp: qp, wr: &swr.wr, bad_wr: &bad_swr);
5800	if (ret) {
5801	WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5802	return;
5803	}
5804
5805	handle_drain_completion(cq, sdrain: &sdrain, dev);
5806	}
5807
5808	void mlx5_ib_drain_rq(struct ib_qp *qp)
5809	{
5810	struct ib_cq *cq = qp->recv_cq;
5811	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5812	struct mlx5_ib_drain_cqe rdrain;
5813	struct ib_recv_wr rwr = {};
5814	const struct ib_recv_wr *bad_rwr;
5815	int ret;
5816	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
5817	struct mlx5_core_dev *mdev = dev->mdev;
5818
5819	ret = ib_modify_qp(qp, qp_attr: &attr, qp_attr_mask: IB_QP_STATE);
5820	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5821	WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5822	return;
5823	}
5824
5825	rwr.wr_cqe = &rdrain.cqe;
5826	rdrain.cqe.done = mlx5_ib_drain_qp_done;
5827	init_completion(x: &rdrain.done);
5828
5829	ret = mlx5_ib_post_recv_drain(ibqp: qp, wr: &rwr, bad_wr: &bad_rwr);
5830	if (ret) {
5831	WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5832	return;
5833	}
5834
5835	handle_drain_completion(cq, sdrain: &rdrain, dev);
5836	}
5837
5838	/*
5839	* Bind a qp to a counter. If @counter is NULL then bind the qp to
5840	* the default counter
5841	*/
5842	int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)
5843	{
5844	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
5845	struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
5846	int err = 0;
5847
5848	mutex_lock(&mqp->mutex);
5849	if (mqp->state == IB_QPS_RESET) {
5850	qp->counter = counter;
5851	goto out;
5852	}
5853
5854	if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) {
5855	err = -EOPNOTSUPP;
5856	goto out;
5857	}
5858
5859	if (mqp->state == IB_QPS_RTS) {
5860	err = __mlx5_ib_qp_set_counter(qp, counter);
5861	if (!err)
5862	qp->counter = counter;
5863
5864	goto out;
5865	}
5866
5867	mqp->counter_pending = 1;
5868	qp->counter = counter;
5869
5870	out:
5871	mutex_unlock(lock: &mqp->mutex);
5872	return err;
5873	}
5874
5875	int mlx5_ib_qp_event_init(void)
5876	{
5877	mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);
5878	if (!mlx5_ib_qp_event_wq)
5879	return -ENOMEM;
5880
5881	return 0;
5882	}
5883
5884	void mlx5_ib_qp_event_cleanup(void)
5885	{
5886	destroy_workqueue(wq: mlx5_ib_qp_event_wq);
5887	}
5888