1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/*
3	* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4	*/
5
6	#include <rdma/ib_user_verbs.h>
7	#include <rdma/ib_verbs.h>
8	#include <rdma/uverbs_types.h>
9	#include <rdma/uverbs_ioctl.h>
10	#include <rdma/uverbs_std_types.h>
11	#include <rdma/mlx5_user_ioctl_cmds.h>
12	#include <rdma/mlx5_user_ioctl_verbs.h>
13	#include <rdma/ib_hdrs.h>
14	#include <rdma/ib_umem.h>
15	#include <rdma/ib_ucaps.h>
16	#include <linux/mlx5/driver.h>
17	#include <linux/mlx5/fs.h>
18	#include <linux/mlx5/fs_helpers.h>
19	#include <linux/mlx5/eswitch.h>
20	#include <net/inet_ecn.h>
21	#include "mlx5_ib.h"
22	#include "counters.h"
23	#include "devx.h"
24	#include "fs.h"
25
26	#define UVERBS_MODULE_NAME mlx5_ib
27	#include <rdma/uverbs_named_ioctl.h>
28
29	enum {
30	MATCH_CRITERIA_ENABLE_OUTER_BIT,
31	MATCH_CRITERIA_ENABLE_MISC_BIT,
32	MATCH_CRITERIA_ENABLE_INNER_BIT,
33	MATCH_CRITERIA_ENABLE_MISC2_BIT
34	};
35
36
37	struct mlx5_per_qp_opfc {
38	struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];
39	};
40
41	#define HEADER_IS_ZERO(match_criteria, headers) \
42	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
43	0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
44
45	static u8 get_match_criteria_enable(u32 *match_criteria)
46	{
47	u8 match_criteria_enable;
48
49	match_criteria_enable =
50	(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
51	MATCH_CRITERIA_ENABLE_OUTER_BIT;
52	match_criteria_enable |=
53	(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
54	MATCH_CRITERIA_ENABLE_MISC_BIT;
55	match_criteria_enable |=
56	(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
57	MATCH_CRITERIA_ENABLE_INNER_BIT;
58	match_criteria_enable |=
59	(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
60	MATCH_CRITERIA_ENABLE_MISC2_BIT;
61
62	return match_criteria_enable;
63	}
64
65	static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
66	{
67	u8 entry_mask;
68	u8 entry_val;
69	int err = 0;
70
71	if (!mask)
72	goto out;
73
74	entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
75	ip_protocol);
76	entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
77	ip_protocol);
78	if (!entry_mask) {
79	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
80	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
81	goto out;
82	}
83	/* Don't override existing ip protocol */
84	if (mask != entry_mask || val != entry_val)
85	err = -EINVAL;
86	out:
87	return err;
88	}
89
90	static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
91	bool inner)
92	{
93	if (inner) {
94	MLX5_SET(fte_match_set_misc,
95	misc_c, inner_ipv6_flow_label, mask);
96	MLX5_SET(fte_match_set_misc,
97	misc_v, inner_ipv6_flow_label, val);
98	} else {
99	MLX5_SET(fte_match_set_misc,
100	misc_c, outer_ipv6_flow_label, mask);
101	MLX5_SET(fte_match_set_misc,
102	misc_v, outer_ipv6_flow_label, val);
103	}
104	}
105
106	static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
107	{
108	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
109	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
110	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
111	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
112	}
113
114	static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
115	{
116	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
117	!(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
118	return -EOPNOTSUPP;
119
120	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
121	!(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
122	return -EOPNOTSUPP;
123
124	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
125	!(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
126	return -EOPNOTSUPP;
127
128	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
129	!(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
130	return -EOPNOTSUPP;
131
132	return 0;
133	}
134
135	#define LAST_ETH_FIELD vlan_tag
136	#define LAST_IPV4_FIELD tos
137	#define LAST_IPV6_FIELD traffic_class
138	#define LAST_TCP_UDP_FIELD src_port
139	#define LAST_TUNNEL_FIELD tunnel_id
140	#define LAST_FLOW_TAG_FIELD tag_id
141	#define LAST_DROP_FIELD size
142	#define LAST_COUNTERS_FIELD counters
143
144	/* Field is the last supported field */
145	#define FIELDS_NOT_SUPPORTED(filter, field) \
146	memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \
147	sizeof(filter) - offsetofend(typeof(filter), field))
148
149	int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
150	bool is_egress,
151	struct mlx5_flow_act *action)
152	{
153
154	switch (maction->ib_action.type) {
155	case IB_FLOW_ACTION_UNSPECIFIED:
156	if (maction->flow_action_raw.sub_type ==
157	MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
158	if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
159	return -EINVAL;
160	action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
161	action->modify_hdr =
162	maction->flow_action_raw.modify_hdr;
163	return 0;
164	}
165	if (maction->flow_action_raw.sub_type ==
166	MLX5_IB_FLOW_ACTION_DECAP) {
167	if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
168	return -EINVAL;
169	action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
170	return 0;
171	}
172	if (maction->flow_action_raw.sub_type ==
173	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
174	if (action->action &
175	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
176	return -EINVAL;
177	action->action |=
178	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
179	action->pkt_reformat =
180	maction->flow_action_raw.pkt_reformat;
181	return 0;
182	}
183	fallthrough;
184	default:
185	return -EOPNOTSUPP;
186	}
187	}
188
189	static int parse_flow_attr(struct mlx5_core_dev *mdev,
190	struct mlx5_flow_spec *spec,
191	const union ib_flow_spec *ib_spec,
192	const struct ib_flow_attr *flow_attr,
193	struct mlx5_flow_act *action, u32 prev_type)
194	{
195	struct mlx5_flow_context *flow_context = &spec->flow_context;
196	u32 *match_c = spec->match_criteria;
197	u32 *match_v = spec->match_value;
198	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
199	misc_parameters);
200	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
201	misc_parameters);
202	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
203	misc_parameters_2);
204	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
205	misc_parameters_2);
206	void *headers_c;
207	void *headers_v;
208	int match_ipv;
209	int ret;
210
211	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
212	headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
213	inner_headers);
214	headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
215	inner_headers);
216	match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
217	ft_field_support.inner_ip_version);
218	} else {
219	headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
220	outer_headers);
221	headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
222	outer_headers);
223	match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
224	ft_field_support.outer_ip_version);
225	}
226
227	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
228	case IB_FLOW_SPEC_ETH:
229	if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
230	return -EOPNOTSUPP;
231
232	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
233	dmac_47_16),
234	src: ib_spec->eth.mask.dst_mac);
235	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
236	dmac_47_16),
237	src: ib_spec->eth.val.dst_mac);
238
239	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
240	smac_47_16),
241	src: ib_spec->eth.mask.src_mac);
242	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
243	smac_47_16),
244	src: ib_spec->eth.val.src_mac);
245
246	if (ib_spec->eth.mask.vlan_tag) {
247	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
248	cvlan_tag, 1);
249	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
250	cvlan_tag, 1);
251
252	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
253	first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
254	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
255	first_vid, ntohs(ib_spec->eth.val.vlan_tag));
256
257	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
258	first_cfi,
259	ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
260	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
261	first_cfi,
262	ntohs(ib_spec->eth.val.vlan_tag) >> 12);
263
264	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
265	first_prio,
266	ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
267	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
268	first_prio,
269	ntohs(ib_spec->eth.val.vlan_tag) >> 13);
270	}
271	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
272	ethertype, ntohs(ib_spec->eth.mask.ether_type));
273	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
274	ethertype, ntohs(ib_spec->eth.val.ether_type));
275	break;
276	case IB_FLOW_SPEC_IPV4:
277	if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
278	return -EOPNOTSUPP;
279
280	if (match_ipv) {
281	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
282	ip_version, 0xf);
283	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
284	ip_version, MLX5_FS_IPV4_VERSION);
285	} else {
286	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
287	ethertype, 0xffff);
288	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
289	ethertype, ETH_P_IP);
290	}
291
292	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
293	src_ipv4_src_ipv6.ipv4_layout.ipv4),
294	&ib_spec->ipv4.mask.src_ip,
295	sizeof(ib_spec->ipv4.mask.src_ip));
296	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
297	src_ipv4_src_ipv6.ipv4_layout.ipv4),
298	&ib_spec->ipv4.val.src_ip,
299	sizeof(ib_spec->ipv4.val.src_ip));
300	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
301	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
302	&ib_spec->ipv4.mask.dst_ip,
303	sizeof(ib_spec->ipv4.mask.dst_ip));
304	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
305	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
306	&ib_spec->ipv4.val.dst_ip,
307	sizeof(ib_spec->ipv4.val.dst_ip));
308
309	set_tos(outer_c: headers_c, outer_v: headers_v,
310	mask: ib_spec->ipv4.mask.tos, val: ib_spec->ipv4.val.tos);
311
312	if (set_proto(outer_c: headers_c, outer_v: headers_v,
313	mask: ib_spec->ipv4.mask.proto,
314	val: ib_spec->ipv4.val.proto))
315	return -EINVAL;
316	break;
317	case IB_FLOW_SPEC_IPV6:
318	if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
319	return -EOPNOTSUPP;
320
321	if (match_ipv) {
322	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
323	ip_version, 0xf);
324	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
325	ip_version, MLX5_FS_IPV6_VERSION);
326	} else {
327	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
328	ethertype, 0xffff);
329	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
330	ethertype, ETH_P_IPV6);
331	}
332
333	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
334	src_ipv4_src_ipv6.ipv6_layout.ipv6),
335	&ib_spec->ipv6.mask.src_ip,
336	sizeof(ib_spec->ipv6.mask.src_ip));
337	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
338	src_ipv4_src_ipv6.ipv6_layout.ipv6),
339	&ib_spec->ipv6.val.src_ip,
340	sizeof(ib_spec->ipv6.val.src_ip));
341	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
342	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
343	&ib_spec->ipv6.mask.dst_ip,
344	sizeof(ib_spec->ipv6.mask.dst_ip));
345	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
346	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
347	&ib_spec->ipv6.val.dst_ip,
348	sizeof(ib_spec->ipv6.val.dst_ip));
349
350	set_tos(outer_c: headers_c, outer_v: headers_v,
351	mask: ib_spec->ipv6.mask.traffic_class,
352	val: ib_spec->ipv6.val.traffic_class);
353
354	if (set_proto(outer_c: headers_c, outer_v: headers_v,
355	mask: ib_spec->ipv6.mask.next_hdr,
356	val: ib_spec->ipv6.val.next_hdr))
357	return -EINVAL;
358
359	set_flow_label(misc_c: misc_params_c, misc_v: misc_params_v,
360	ntohl(ib_spec->ipv6.mask.flow_label),
361	ntohl(ib_spec->ipv6.val.flow_label),
362	inner: ib_spec->type & IB_FLOW_SPEC_INNER);
363	break;
364	case IB_FLOW_SPEC_ESP:
365	return -EOPNOTSUPP;
366	case IB_FLOW_SPEC_TCP:
367	if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
368	LAST_TCP_UDP_FIELD))
369	return -EOPNOTSUPP;
370
371	if (set_proto(outer_c: headers_c, outer_v: headers_v, mask: 0xff, IPPROTO_TCP))
372	return -EINVAL;
373
374	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
375	ntohs(ib_spec->tcp_udp.mask.src_port));
376	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
377	ntohs(ib_spec->tcp_udp.val.src_port));
378
379	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
380	ntohs(ib_spec->tcp_udp.mask.dst_port));
381	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
382	ntohs(ib_spec->tcp_udp.val.dst_port));
383	break;
384	case IB_FLOW_SPEC_UDP:
385	if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
386	LAST_TCP_UDP_FIELD))
387	return -EOPNOTSUPP;
388
389	if (set_proto(outer_c: headers_c, outer_v: headers_v, mask: 0xff, IPPROTO_UDP))
390	return -EINVAL;
391
392	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
393	ntohs(ib_spec->tcp_udp.mask.src_port));
394	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
395	ntohs(ib_spec->tcp_udp.val.src_port));
396
397	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
398	ntohs(ib_spec->tcp_udp.mask.dst_port));
399	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
400	ntohs(ib_spec->tcp_udp.val.dst_port));
401	break;
402	case IB_FLOW_SPEC_GRE:
403	if (ib_spec->gre.mask.c_ks_res0_ver)
404	return -EOPNOTSUPP;
405
406	if (set_proto(outer_c: headers_c, outer_v: headers_v, mask: 0xff, IPPROTO_GRE))
407	return -EINVAL;
408
409	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
410	0xff);
411	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
412	IPPROTO_GRE);
413
414	MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
415	ntohs(ib_spec->gre.mask.protocol));
416	MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
417	ntohs(ib_spec->gre.val.protocol));
418
419	memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
420	gre_key.nvgre.hi),
421	&ib_spec->gre.mask.key,
422	sizeof(ib_spec->gre.mask.key));
423	memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
424	gre_key.nvgre.hi),
425	&ib_spec->gre.val.key,
426	sizeof(ib_spec->gre.val.key));
427	break;
428	case IB_FLOW_SPEC_MPLS:
429	switch (prev_type) {
430	case IB_FLOW_SPEC_UDP:
431	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
432	ft_field_support.outer_first_mpls_over_udp),
433	set_mask: &ib_spec->mpls.mask.tag))
434	return -EOPNOTSUPP;
435
436	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
437	outer_first_mpls_over_udp),
438	&ib_spec->mpls.val.tag,
439	sizeof(ib_spec->mpls.val.tag));
440	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
441	outer_first_mpls_over_udp),
442	&ib_spec->mpls.mask.tag,
443	sizeof(ib_spec->mpls.mask.tag));
444	break;
445	case IB_FLOW_SPEC_GRE:
446	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
447	ft_field_support.outer_first_mpls_over_gre),
448	set_mask: &ib_spec->mpls.mask.tag))
449	return -EOPNOTSUPP;
450
451	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
452	outer_first_mpls_over_gre),
453	&ib_spec->mpls.val.tag,
454	sizeof(ib_spec->mpls.val.tag));
455	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
456	outer_first_mpls_over_gre),
457	&ib_spec->mpls.mask.tag,
458	sizeof(ib_spec->mpls.mask.tag));
459	break;
460	default:
461	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
462	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
463	ft_field_support.inner_first_mpls),
464	set_mask: &ib_spec->mpls.mask.tag))
465	return -EOPNOTSUPP;
466
467	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
468	inner_first_mpls),
469	&ib_spec->mpls.val.tag,
470	sizeof(ib_spec->mpls.val.tag));
471	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
472	inner_first_mpls),
473	&ib_spec->mpls.mask.tag,
474	sizeof(ib_spec->mpls.mask.tag));
475	} else {
476	if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
477	ft_field_support.outer_first_mpls),
478	set_mask: &ib_spec->mpls.mask.tag))
479	return -EOPNOTSUPP;
480
481	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
482	outer_first_mpls),
483	&ib_spec->mpls.val.tag,
484	sizeof(ib_spec->mpls.val.tag));
485	memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
486	outer_first_mpls),
487	&ib_spec->mpls.mask.tag,
488	sizeof(ib_spec->mpls.mask.tag));
489	}
490	}
491	break;
492	case IB_FLOW_SPEC_VXLAN_TUNNEL:
493	if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
494	LAST_TUNNEL_FIELD))
495	return -EOPNOTSUPP;
496
497	MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
498	ntohl(ib_spec->tunnel.mask.tunnel_id));
499	MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
500	ntohl(ib_spec->tunnel.val.tunnel_id));
501	break;
502	case IB_FLOW_SPEC_ACTION_TAG:
503	if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
504	LAST_FLOW_TAG_FIELD))
505	return -EOPNOTSUPP;
506	if (ib_spec->flow_tag.tag_id >= BIT(24))
507	return -EINVAL;
508
509	flow_context->flow_tag = ib_spec->flow_tag.tag_id;
510	flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
511	break;
512	case IB_FLOW_SPEC_ACTION_DROP:
513	if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
514	LAST_DROP_FIELD))
515	return -EOPNOTSUPP;
516	action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
517	break;
518	case IB_FLOW_SPEC_ACTION_HANDLE:
519	ret = parse_flow_flow_action(maction: to_mflow_act(ibact: ib_spec->action.act),
520	is_egress: flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
521	if (ret)
522	return ret;
523	break;
524	case IB_FLOW_SPEC_ACTION_COUNT:
525	if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
526	LAST_COUNTERS_FIELD))
527	return -EOPNOTSUPP;
528
529	/* for now support only one counters spec per flow */
530	if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
531	return -EINVAL;
532
533	action->counters = ib_spec->flow_count.counters;
534	action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
535	break;
536	default:
537	return -EINVAL;
538	}
539
540	return 0;
541	}
542
543	/* If a flow could catch both multicast and unicast packets,
544	* it won't fall into the multicast flow steering table and this rule
545	* could steal other multicast packets.
546	*/
547	static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
548	{
549	union ib_flow_spec *flow_spec;
550
551	if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
552	ib_attr->num_of_specs < 1)
553	return false;
554
555	flow_spec = (union ib_flow_spec *)(ib_attr + 1);
556	if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
557	struct ib_flow_spec_ipv4 *ipv4_spec;
558
559	ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
560	if (ipv4_is_multicast(addr: ipv4_spec->val.dst_ip))
561	return true;
562
563	return false;
564	}
565
566	if (flow_spec->type == IB_FLOW_SPEC_ETH) {
567	struct ib_flow_spec_eth *eth_spec;
568
569	eth_spec = (struct ib_flow_spec_eth *)flow_spec;
570	return is_multicast_ether_addr(addr: eth_spec->mask.dst_mac) &&
571	is_multicast_ether_addr(addr: eth_spec->val.dst_mac);
572	}
573
574	return false;
575	}
576
577	static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
578	const struct ib_flow_attr *flow_attr,
579	bool check_inner)
580	{
581	union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
582	int match_ipv = check_inner ?
583	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
584	ft_field_support.inner_ip_version) :
585	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
586	ft_field_support.outer_ip_version);
587	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
588	bool ipv4_spec_valid, ipv6_spec_valid;
589	unsigned int ip_spec_type = 0;
590	bool has_ethertype = false;
591	unsigned int spec_index;
592	bool mask_valid = true;
593	u16 eth_type = 0;
594	bool type_valid;
595
596	/* Validate that ethertype is correct */
597	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
598	if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
599	ib_spec->eth.mask.ether_type) {
600	mask_valid = (ib_spec->eth.mask.ether_type ==
601	htons(0xffff));
602	has_ethertype = true;
603	eth_type = ntohs(ib_spec->eth.val.ether_type);
604	} else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
605	(ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
606	ip_spec_type = ib_spec->type;
607	}
608	ib_spec = (void *)ib_spec + ib_spec->size;
609	}
610
611	type_valid = (!has_ethertype) || (!ip_spec_type);
612	if (!type_valid && mask_valid) {
613	ipv4_spec_valid = (eth_type == ETH_P_IP) &&
614	(ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
615	ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
616	(ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
617
618	type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
619	(((eth_type == ETH_P_MPLS_UC) ||
620	(eth_type == ETH_P_MPLS_MC)) && match_ipv);
621	}
622
623	return type_valid;
624	}
625
626	static bool is_valid_attr(struct mlx5_core_dev *mdev,
627	const struct ib_flow_attr *flow_attr)
628	{
629	return is_valid_ethertype(mdev, flow_attr, check_inner: false) &&
630	is_valid_ethertype(mdev, flow_attr, check_inner: true);
631	}
632
633	static void put_flow_table(struct mlx5_ib_dev *dev,
634	struct mlx5_ib_flow_prio *prio, bool ft_added)
635	{
636	prio->refcount -= !!ft_added;
637	if (!prio->refcount) {
638	mlx5_destroy_flow_table(ft: prio->flow_table);
639	prio->flow_table = NULL;
640	}
641	}
642
643	static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
644	{
645	struct mlx5_ib_flow_handler *handler = container_of(flow_id,
646	struct mlx5_ib_flow_handler,
647	ibflow);
648	struct mlx5_ib_flow_handler *iter, *tmp;
649	struct mlx5_ib_dev *dev = handler->dev;
650
651	mutex_lock(&dev->flow_db->lock);
652
653	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
654	mlx5_del_flow_rules(fr: iter->rule);
655	put_flow_table(dev, prio: iter->prio, ft_added: true);
656	list_del(entry: &iter->list);
657	kfree(objp: iter);
658	}
659
660	mlx5_del_flow_rules(fr: handler->rule);
661	put_flow_table(dev, prio: handler->prio, ft_added: true);
662	mlx5_ib_counters_clear_description(counters: handler->ibcounters);
663	mutex_unlock(lock: &dev->flow_db->lock);
664	if (handler->flow_matcher)
665	atomic_dec(v: &handler->flow_matcher->usecnt);
666	kfree(objp: handler);
667
668	return 0;
669	}
670
671	static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
672	{
673	priority *= 2;
674	if (!dont_trap)
675	priority++;
676	return priority;
677	}
678
679	enum flow_table_type {
680	MLX5_IB_FT_RX,
681	MLX5_IB_FT_TX
682	};
683
684	#define MLX5_FS_MAX_TYPES 6
685	#define MLX5_FS_MAX_ENTRIES BIT(16)
686
687	static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device)
688	{
689	struct mlx5_ib_dev *dev = to_mdev(ibdev: device);
690
691	return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed);
692	}
693
694	static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
695	struct mlx5_ib_flow_prio *prio,
696	struct mlx5_flow_table_attr *ft_attr)
697	{
698	struct mlx5_flow_table *ft;
699
700	ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr);
701	if (IS_ERR(ptr: ft))
702	return ERR_CAST(ptr: ft);
703
704	prio->flow_table = ft;
705	prio->refcount = 0;
706	return prio;
707	}
708
709	static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
710	struct ib_flow_attr *flow_attr,
711	enum flow_table_type ft_type)
712	{
713	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
714	struct mlx5_flow_table_attr ft_attr = {};
715	struct mlx5_flow_namespace *ns = NULL;
716	enum mlx5_flow_namespace_type fn_type;
717	struct mlx5_ib_flow_prio *prio;
718	struct mlx5_flow_table *ft;
719	int max_table_size;
720	int num_entries;
721	int num_groups;
722	bool esw_encap;
723	u32 flags = 0;
724	int priority;
725
726	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
727	log_max_ft_size));
728	esw_encap = mlx5_eswitch_get_encap_mode(dev: dev->mdev) !=
729	DEVLINK_ESWITCH_ENCAP_MODE_NONE;
730	switch (flow_attr->type) {
731	case IB_FLOW_ATTR_NORMAL:
732	if (flow_is_multicast_only(ib_attr: flow_attr) && !dont_trap)
733	priority = MLX5_IB_FLOW_MCAST_PRIO;
734	else
735	priority = ib_prio_to_core_prio(priority: flow_attr->priority,
736	dont_trap);
737	if (ft_type == MLX5_IB_FT_RX) {
738	fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
739	prio = &dev->flow_db->prios[priority];
740	if (!dev->is_rep && !esw_encap &&
741	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
742	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
743	if (!dev->is_rep && !esw_encap &&
744	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
745	reformat_l3_tunnel_to_l2))
746	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
747	} else {
748	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
749	dev->mdev, log_max_ft_size));
750	fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
751	prio = &dev->flow_db->egress_prios[priority];
752	if (!dev->is_rep && !esw_encap &&
753	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
754	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
755	}
756	ns = mlx5_get_flow_namespace(dev: dev->mdev, type: fn_type);
757	num_entries = MLX5_FS_MAX_ENTRIES;
758	num_groups = MLX5_FS_MAX_TYPES;
759	break;
760	case IB_FLOW_ATTR_ALL_DEFAULT:
761	case IB_FLOW_ATTR_MC_DEFAULT:
762	ns = mlx5_get_flow_namespace(dev: dev->mdev,
763	type: MLX5_FLOW_NAMESPACE_LEFTOVERS);
764	build_leftovers_ft_param(priority: &priority, n_ent: &num_entries, n_grp: &num_groups);
765	prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
766	break;
767	case IB_FLOW_ATTR_SNIFFER:
768	if (!MLX5_CAP_FLOWTABLE(dev->mdev,
769	allow_sniffer_and_nic_rx_shared_tir))
770	return ERR_PTR(error: -EOPNOTSUPP);
771
772	ns = mlx5_get_flow_namespace(
773	dev: dev->mdev, type: ft_type == MLX5_IB_FT_RX ?
774	MLX5_FLOW_NAMESPACE_SNIFFER_RX :
775	MLX5_FLOW_NAMESPACE_SNIFFER_TX);
776
777	prio = &dev->flow_db->sniffer[ft_type];
778	priority = 0;
779	num_entries = 1;
780	num_groups = 1;
781	break;
782	default:
783	break;
784	}
785
786	if (!ns)
787	return ERR_PTR(error: -EOPNOTSUPP);
788
789	max_table_size = min_t(int, num_entries, max_table_size);
790
791	ft = prio->flow_table;
792	if (ft)
793	return prio;
794
795	ft_attr.prio = priority;
796	ft_attr.max_fte = max_table_size;
797	ft_attr.flags = flags;
798	ft_attr.autogroup.max_num_groups = num_groups;
799	return _get_prio(ns, prio, ft_attr: &ft_attr);
800	}
801
802	enum {
803	RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO,
804	RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO,
805	RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
806	RDMA_RX_ECN_OPCOUNTER_PRIO,
807	RDMA_RX_CNP_OPCOUNTER_PRIO,
808	RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO,
809	};
810
811	enum {
812	RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO,
813	RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
814	RDMA_TX_CNP_OPCOUNTER_PRIO,
815	RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO,
816	};
817
818	static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,
819	struct mlx5_flow_spec *spec)
820	{
821	if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
822	ft_field_support.source_vhca_port) ||
823	!MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
824	ft_field_support.source_vhca_port))
825	return -EOPNOTSUPP;
826
827	MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria,
828	misc_parameters.source_vhca_port);
829	MLX5_SET(fte_match_param, &spec->match_value,
830	misc_parameters.source_vhca_port, port_num);
831
832	return 0;
833	}
834
835	static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,
836	struct mlx5_flow_spec *spec, int ipv)
837	{
838	if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
839	ft_field_support.outer_ip_version))
840	return -EOPNOTSUPP;
841
842	if (mlx5_core_mp_enabled(dev: dev->mdev) &&
843	set_vhca_port_spec(dev, port_num, spec))
844	return -EOPNOTSUPP;
845
846	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
847	outer_headers.ip_ecn);
848	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn,
849	INET_ECN_CE);
850	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
851	outer_headers.ip_version);
852	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
853	ipv);
854
855	spec->match_criteria_enable =
856	get_match_criteria_enable(match_criteria: spec->match_criteria);
857
858	return 0;
859	}
860
861	static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,
862	struct mlx5_flow_spec *spec)
863	{
864	if (mlx5_core_mp_enabled(dev: dev->mdev) &&
865	set_vhca_port_spec(dev, port_num, spec))
866	return -EOPNOTSUPP;
867
868	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
869	misc_parameters.bth_opcode);
870	MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode,
871	IB_BTH_OPCODE_CNP);
872
873	spec->match_criteria_enable =
874	get_match_criteria_enable(match_criteria: spec->match_criteria);
875
876	return 0;
877	}
878
879	/* Returns the prio we should use for the given optional counter type,
880	* whereas for bytes type we use the packet type, since they share the same
881	* resources.
882	*/
883	static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev,
884	u32 type)
885	{
886	u32 prio_type;
887
888	switch (type) {
889	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
890	prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;
891	break;
892	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
893	prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;
894	break;
895	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
896	prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
897	break;
898	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
899	prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
900	break;
901	default:
902	prio_type = type;
903	}
904
905	return &dev->flow_db->opfcs[prio_type];
906	}
907
908	static void put_per_qp_prio(struct mlx5_ib_dev *dev,
909	enum mlx5_ib_optional_counter_type type)
910	{
911	enum mlx5_ib_optional_counter_type per_qp_type;
912	struct mlx5_ib_flow_prio *prio;
913
914	switch (type) {
915	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
916	per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
917	break;
918	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
919	per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
920	break;
921	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
922	per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
923	break;
924	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
925	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
926	break;
927	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
928	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
929	break;
930	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
931	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
932	break;
933	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
934	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
935	break;
936	default:
937	return;
938	}
939
940	prio = get_opfc_prio(dev, type: per_qp_type);
941	put_flow_table(dev, prio, ft_added: true);
942	}
943
944	static int get_per_qp_prio(struct mlx5_ib_dev *dev,
945	enum mlx5_ib_optional_counter_type type)
946	{
947	enum mlx5_ib_optional_counter_type per_qp_type;
948	struct mlx5_flow_table_attr ft_attr = {};
949	enum mlx5_flow_namespace_type fn_type;
950	struct mlx5_flow_namespace *ns;
951	struct mlx5_ib_flow_prio *prio;
952	int priority;
953
954	switch (type) {
955	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
956	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
957	priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO;
958	per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
959	break;
960	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
961	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
962	priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO;
963	per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
964	break;
965	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
966	fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
967	priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO;
968	per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
969	break;
970	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
971	fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
972	priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
973	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
974	break;
975	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
976	fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
977	priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
978	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
979	break;
980	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
981	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
982	priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
983	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
984	break;
985	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
986	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
987	priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
988	per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
989	break;
990	default:
991	return -EINVAL;
992	}
993
994	ns = mlx5_get_flow_namespace(dev: dev->mdev, type: fn_type);
995	if (!ns)
996	return -EOPNOTSUPP;
997
998	prio = get_opfc_prio(dev, type: per_qp_type);
999	if (prio->flow_table)
1000	return 0;
1001
1002	ft_attr.prio = priority;
1003	ft_attr.max_fte = MLX5_FS_MAX_POOL_SIZE;
1004	ft_attr.autogroup.max_num_groups = 1;
1005	prio = _get_prio(ns, prio, ft_attr: &ft_attr);
1006	if (IS_ERR(ptr: prio))
1007	return PTR_ERR(ptr: prio);
1008
1009	prio->refcount = 1;
1010
1011	return 0;
1012	}
1013
1014	static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa,
1015	u32 qp_num, bool *new)
1016	{
1017	struct mlx5_per_qp_opfc *per_qp_opfc;
1018
1019	*new = false;
1020
1021	per_qp_opfc = xa_load(qpn_opfc_xa, index: qp_num);
1022	if (per_qp_opfc)
1023	return per_qp_opfc;
1024	per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);
1025
1026	if (!per_qp_opfc)
1027	return NULL;
1028
1029	*new = true;
1030	return per_qp_opfc;
1031	}
1032
1033	static int add_op_fc_rules(struct mlx5_ib_dev *dev,
1034	struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1035	struct xarray *qpn_opfc_xa,
1036	struct mlx5_per_qp_opfc *per_qp_opfc,
1037	struct mlx5_ib_flow_prio *prio,
1038	enum mlx5_ib_optional_counter_type type,
1039	u32 qp_num, u32 port_num)
1040	{
1041	struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc;
1042	struct mlx5_flow_act flow_act = {};
1043	struct mlx5_flow_destination dst;
1044	struct mlx5_flow_spec *spec;
1045	int i, err, spec_num;
1046	bool is_tx;
1047
1048	if (opfc->fc)
1049	return -EEXIST;
1050
1051	if (mlx5r_is_opfc_shared_and_in_use(opfcs: per_qp_opfc->opfcs, type,
1052	opfc: &in_use_opfc)) {
1053	opfc->fc = in_use_opfc->fc;
1054	opfc->rule[0] = in_use_opfc->rule[0];
1055	return 0;
1056	}
1057
1058	opfc->fc = fc_arr[type];
1059
1060	spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1061	if (!spec) {
1062	err = -ENOMEM;
1063	goto null_fc;
1064	}
1065
1066	switch (type) {
1067	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP:
1068	if (set_ecn_ce_spec(dev, port_num, spec: &spec[0],
1069	MLX5_FS_IPV4_VERSION) ||
1070	set_ecn_ce_spec(dev, port_num, spec: &spec[1],
1071	MLX5_FS_IPV6_VERSION)) {
1072	err = -EOPNOTSUPP;
1073	goto free_spec;
1074	}
1075	spec_num = 2;
1076	is_tx = false;
1077
1078	MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria,
1079	misc_parameters.bth_dst_qp);
1080	MLX5_SET(fte_match_param, spec[1].match_value,
1081	misc_parameters.bth_dst_qp, qp_num);
1082	spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1083	break;
1084	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP:
1085	if (!MLX5_CAP_FLOWTABLE(
1086	dev->mdev,
1087	ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1088	set_cnp_spec(dev, port_num, spec: &spec[0])) {
1089	err = -EOPNOTSUPP;
1090	goto free_spec;
1091	}
1092	spec_num = 1;
1093	is_tx = false;
1094	break;
1095	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP:
1096	if (!MLX5_CAP_FLOWTABLE(
1097	dev->mdev,
1098	ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1099	set_cnp_spec(dev, port_num, spec: &spec[0])) {
1100	err = -EOPNOTSUPP;
1101	goto free_spec;
1102	}
1103	spec_num = 1;
1104	is_tx = true;
1105	break;
1106	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1107	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1108	spec_num = 1;
1109	is_tx = true;
1110	break;
1111	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1112	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1113	spec_num = 1;
1114	is_tx = false;
1115	break;
1116	default:
1117	err = -EINVAL;
1118	goto free_spec;
1119	}
1120
1121	if (is_tx) {
1122	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1123	misc_parameters.source_sqn);
1124	MLX5_SET(fte_match_param, spec->match_value,
1125	misc_parameters.source_sqn, qp_num);
1126	} else {
1127	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1128	misc_parameters.bth_dst_qp);
1129	MLX5_SET(fte_match_param, spec->match_value,
1130	misc_parameters.bth_dst_qp, qp_num);
1131	}
1132
1133	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1134
1135	dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1136	dst.counter = opfc->fc;
1137
1138	flow_act.action =
1139	MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1140
1141	for (i = 0; i < spec_num; i++) {
1142	opfc->rule[i] = mlx5_add_flow_rules(ft: prio->flow_table, spec: &spec[i],
1143	flow_act: &flow_act, dest: &dst, num_dest: 1);
1144	if (IS_ERR(ptr: opfc->rule[i])) {
1145	err = PTR_ERR(ptr: opfc->rule[i]);
1146	goto del_rules;
1147	}
1148	}
1149	prio->refcount += spec_num;
1150
1151	err = xa_err(entry: xa_store(qpn_opfc_xa, index: qp_num, entry: per_qp_opfc, GFP_KERNEL));
1152	if (err)
1153	goto del_rules;
1154
1155	kfree(objp: spec);
1156
1157	return 0;
1158
1159	del_rules:
1160	while (i--)
1161	mlx5_del_flow_rules(fr: opfc->rule[i]);
1162	put_flow_table(dev, prio, ft_added: false);
1163	free_spec:
1164	kfree(objp: spec);
1165	null_fc:
1166	opfc->fc = NULL;
1167	return err;
1168	}
1169
1170	static bool
1171	is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type,
1172	struct mlx5_fc **fc)
1173	{
1174	u32 shared_fc_type;
1175
1176	switch (type) {
1177	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1178	shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
1179	break;
1180	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1181	shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
1182	break;
1183	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1184	shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
1185	break;
1186	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1187	shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
1188	break;
1189	default:
1190	return false;
1191	}
1192
1193	*fc = fc_arr[shared_fc_type];
1194	if (!(*fc))
1195	return false;
1196
1197	return true;
1198	}
1199
1200	void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
1201	struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX])
1202	{
1203	struct mlx5_fc *in_use_fc;
1204	int i;
1205
1206	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1207	i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1208	if (!fc_arr[i])
1209	continue;
1210
1211	if (is_fc_shared_and_in_use(fc_arr, type: i, fc: &in_use_fc)) {
1212	fc_arr[i] = NULL;
1213	continue;
1214	}
1215
1216	mlx5_fc_destroy(dev: dev->mdev, counter: fc_arr[i]);
1217	fc_arr[i] = NULL;
1218	}
1219	}
1220
1221	int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,
1222	struct mlx5_ib_op_fc *opfc,
1223	enum mlx5_ib_optional_counter_type type)
1224	{
1225	struct mlx5_flow_table_attr ft_attr = {};
1226	enum mlx5_flow_namespace_type fn_type;
1227	int priority, i, err, spec_num;
1228	struct mlx5_flow_act flow_act = {};
1229	struct mlx5_flow_destination dst;
1230	struct mlx5_flow_namespace *ns;
1231	struct mlx5_ib_flow_prio *prio;
1232	struct mlx5_flow_spec *spec;
1233
1234	spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1235	if (!spec)
1236	return -ENOMEM;
1237
1238	switch (type) {
1239	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
1240	if (set_ecn_ce_spec(dev, port_num, spec: &spec[0],
1241	MLX5_FS_IPV4_VERSION) ||
1242	set_ecn_ce_spec(dev, port_num, spec: &spec[1],
1243	MLX5_FS_IPV6_VERSION)) {
1244	err = -EOPNOTSUPP;
1245	goto free;
1246	}
1247	spec_num = 2;
1248	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1249	priority = RDMA_RX_ECN_OPCOUNTER_PRIO;
1250	break;
1251
1252	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
1253	if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1254	ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1255	set_cnp_spec(dev, port_num, spec: &spec[0])) {
1256	err = -EOPNOTSUPP;
1257	goto free;
1258	}
1259	spec_num = 1;
1260	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1261	priority = RDMA_RX_CNP_OPCOUNTER_PRIO;
1262	break;
1263
1264	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
1265	if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1266	ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1267	set_cnp_spec(dev, port_num, spec: &spec[0])) {
1268	err = -EOPNOTSUPP;
1269	goto free;
1270	}
1271	spec_num = 1;
1272	fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1273	priority = RDMA_TX_CNP_OPCOUNTER_PRIO;
1274	break;
1275
1276	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
1277	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
1278	spec_num = 1;
1279	fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1280	priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO;
1281	break;
1282
1283	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
1284	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
1285	spec_num = 1;
1286	fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1287	priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO;
1288	break;
1289
1290	default:
1291	err = -EOPNOTSUPP;
1292	goto free;
1293	}
1294
1295	ns = mlx5_get_flow_namespace(dev: dev->mdev, type: fn_type);
1296	if (!ns) {
1297	err = -EOPNOTSUPP;
1298	goto free;
1299	}
1300
1301	prio = get_opfc_prio(dev, type);
1302	if (!prio->flow_table) {
1303	err = get_per_qp_prio(dev, type);
1304	if (err)
1305	goto free;
1306
1307	ft_attr.prio = priority;
1308	ft_attr.max_fte = dev->num_ports * MAX_OPFC_RULES;
1309	ft_attr.autogroup.max_num_groups = 1;
1310	prio = _get_prio(ns, prio, ft_attr: &ft_attr);
1311	if (IS_ERR(ptr: prio)) {
1312	err = PTR_ERR(ptr: prio);
1313	goto put_prio;
1314	}
1315	}
1316
1317	dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1318	dst.counter = opfc->fc;
1319
1320	flow_act.action =
1321	MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1322
1323	for (i = 0; i < spec_num; i++) {
1324	opfc->rule[i] = mlx5_add_flow_rules(ft: prio->flow_table, spec: &spec[i],
1325	flow_act: &flow_act, dest: &dst, num_dest: 1);
1326	if (IS_ERR(ptr: opfc->rule[i])) {
1327	err = PTR_ERR(ptr: opfc->rule[i]);
1328	goto del_rules;
1329	}
1330	}
1331	prio->refcount += spec_num;
1332	kfree(objp: spec);
1333
1334	return 0;
1335
1336	del_rules:
1337	for (i -= 1; i >= 0; i--)
1338	mlx5_del_flow_rules(fr: opfc->rule[i]);
1339	put_flow_table(dev, prio, ft_added: false);
1340	put_prio:
1341	put_per_qp_prio(dev, type);
1342	free:
1343	kfree(objp: spec);
1344	return err;
1345	}
1346
1347	void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
1348	struct mlx5_ib_op_fc *opfc,
1349	enum mlx5_ib_optional_counter_type type)
1350	{
1351	struct mlx5_ib_flow_prio *prio;
1352	int i;
1353
1354	prio = get_opfc_prio(dev, type);
1355
1356	for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) {
1357	mlx5_del_flow_rules(fr: opfc->rule[i]);
1358	put_flow_table(dev, prio, ft_added: true);
1359	}
1360
1361	put_per_qp_prio(dev, type);
1362	}
1363
1364	void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa)
1365	{
1366	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
1367	struct mlx5_per_qp_opfc *per_qp_opfc;
1368	struct mlx5_ib_op_fc *in_use_opfc;
1369	struct mlx5_ib_flow_prio *prio;
1370	int i, j;
1371
1372	per_qp_opfc = xa_load(qpn_opfc_xa, index: qp->qp_num);
1373	if (!per_qp_opfc)
1374	return;
1375
1376	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1377	i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1378	if (!per_qp_opfc->opfcs[i].fc)
1379	continue;
1380
1381	if (mlx5r_is_opfc_shared_and_in_use(opfcs: per_qp_opfc->opfcs, type: i,
1382	opfc: &in_use_opfc)) {
1383	per_qp_opfc->opfcs[i].fc = NULL;
1384	continue;
1385	}
1386
1387	for (j = 0; j < MAX_OPFC_RULES; j++) {
1388	if (!per_qp_opfc->opfcs[i].rule[j])
1389	continue;
1390	mlx5_del_flow_rules(fr: per_qp_opfc->opfcs[i].rule[j]);
1391	prio = get_opfc_prio(dev, type: i);
1392	put_flow_table(dev, prio, ft_added: true);
1393	}
1394	per_qp_opfc->opfcs[i].fc = NULL;
1395	}
1396
1397	kfree(objp: per_qp_opfc);
1398	xa_erase(qpn_opfc_xa, index: qp->qp_num);
1399	}
1400
1401	int mlx5r_fs_bind_op_fc(struct ib_qp *qp,
1402	struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1403	struct xarray *qpn_opfc_xa, u32 port)
1404	{
1405	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
1406	struct mlx5_per_qp_opfc *per_qp_opfc;
1407	struct mlx5_ib_flow_prio *prio;
1408	struct mlx5_ib_counters *cnts;
1409	struct mlx5_ib_op_fc *opfc;
1410	struct mlx5_fc *in_use_fc;
1411	int i, err, per_qp_type;
1412	bool new;
1413
1414	cnts = &dev->port[port - 1].cnts;
1415
1416	for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {
1417	opfc = &cnts->opfcs[i];
1418	if (!opfc->fc)
1419	continue;
1420
1421	per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1422	prio = get_opfc_prio(dev, type: per_qp_type);
1423	WARN_ON(!prio->flow_table);
1424
1425	if (is_fc_shared_and_in_use(fc_arr, type: per_qp_type, fc: &in_use_fc))
1426	fc_arr[per_qp_type] = in_use_fc;
1427
1428	if (!fc_arr[per_qp_type]) {
1429	fc_arr[per_qp_type] = mlx5_fc_create(dev: dev->mdev, aging: false);
1430	if (IS_ERR(ptr: fc_arr[per_qp_type]))
1431	return PTR_ERR(ptr: fc_arr[per_qp_type]);
1432	}
1433
1434	per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp_num: qp->qp_num, new: &new);
1435	if (!per_qp_opfc) {
1436	err = -ENOMEM;
1437	goto free_fc;
1438	}
1439	err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc,
1440	prio, type: per_qp_type, qp_num: qp->qp_num, port_num: port);
1441	if (err)
1442	goto del_rules;
1443	}
1444
1445	return 0;
1446
1447	del_rules:
1448	mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa);
1449	if (new)
1450	kfree(objp: per_qp_opfc);
1451	free_fc:
1452	if (xa_empty(xa: qpn_opfc_xa))
1453	mlx5r_fs_destroy_fcs(dev, fc_arr);
1454	return err;
1455	}
1456
1457	static void set_underlay_qp(struct mlx5_ib_dev *dev,
1458	struct mlx5_flow_spec *spec,
1459	u32 underlay_qpn)
1460	{
1461	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
1462	spec->match_criteria,
1463	misc_parameters);
1464	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1465	misc_parameters);
1466
1467	if (underlay_qpn &&
1468	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1469	ft_field_support.bth_dst_qp)) {
1470	MLX5_SET(fte_match_set_misc,
1471	misc_params_v, bth_dst_qp, underlay_qpn);
1472	MLX5_SET(fte_match_set_misc,
1473	misc_params_c, bth_dst_qp, 0xffffff);
1474	}
1475	}
1476
1477	static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
1478	struct mlx5_flow_spec *spec,
1479	struct mlx5_eswitch_rep *rep)
1480	{
1481	struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
1482	void *misc;
1483
1484	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1485	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1486	misc_parameters_2);
1487
1488	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1489	mlx5_eswitch_get_vport_metadata_for_match(rep->esw,
1490	rep->vport));
1491	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1492	misc_parameters_2);
1493
1494	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1495	mlx5_eswitch_get_vport_metadata_mask());
1496	} else {
1497	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1498	misc_parameters);
1499
1500	MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
1501
1502	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1503	misc_parameters);
1504
1505	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1506	}
1507	}
1508
1509	static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
1510	struct mlx5_ib_flow_prio *ft_prio,
1511	const struct ib_flow_attr *flow_attr,
1512	struct mlx5_flow_destination *dst,
1513	u32 underlay_qpn,
1514	struct mlx5_ib_create_flow *ucmd)
1515	{
1516	struct mlx5_flow_table *ft = ft_prio->flow_table;
1517	struct mlx5_ib_flow_handler *handler;
1518	struct mlx5_flow_act flow_act = {};
1519	struct mlx5_flow_spec *spec;
1520	struct mlx5_flow_destination dest_arr[2] = {};
1521	struct mlx5_flow_destination *rule_dst = dest_arr;
1522	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
1523	unsigned int spec_index;
1524	u32 prev_type = 0;
1525	int err = 0;
1526	int dest_num = 0;
1527	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1528
1529	if (!is_valid_attr(mdev: dev->mdev, flow_attr))
1530	return ERR_PTR(error: -EINVAL);
1531
1532	if (dev->is_rep && is_egress)
1533	return ERR_PTR(error: -EINVAL);
1534
1535	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1536	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
1537	if (!handler || !spec) {
1538	err = -ENOMEM;
1539	goto free;
1540	}
1541
1542	INIT_LIST_HEAD(list: &handler->list);
1543
1544	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
1545	err = parse_flow_attr(mdev: dev->mdev, spec,
1546	ib_spec: ib_flow, flow_attr, action: &flow_act,
1547	prev_type);
1548	if (err < 0)
1549	goto free;
1550
1551	prev_type = ((union ib_flow_spec *)ib_flow)->type;
1552	ib_flow += ((union ib_flow_spec *)ib_flow)->size;
1553	}
1554
1555	if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
1556	memcpy(&dest_arr[0], dst, sizeof(*dst));
1557	dest_num++;
1558	}
1559
1560	if (!flow_is_multicast_only(ib_attr: flow_attr))
1561	set_underlay_qp(dev, spec, underlay_qpn);
1562
1563	if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) {
1564	struct mlx5_eswitch_rep *rep;
1565
1566	rep = dev->port[flow_attr->port - 1].rep;
1567	if (!rep) {
1568	err = -EINVAL;
1569	goto free;
1570	}
1571
1572	mlx5_ib_set_rule_source_port(dev, spec, rep);
1573	}
1574
1575	spec->match_criteria_enable = get_match_criteria_enable(match_criteria: spec->match_criteria);
1576
1577	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1578	struct mlx5_ib_mcounters *mcounters;
1579
1580	err = mlx5_ib_flow_counters_set_data(ibcounters: flow_act.counters, ucmd);
1581	if (err)
1582	goto free;
1583
1584	mcounters = to_mcounters(ibcntrs: flow_act.counters);
1585	handler->ibcounters = flow_act.counters;
1586	dest_arr[dest_num].type =
1587	MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1588	dest_arr[dest_num].counter =
1589	mcounters->hw_cntrs_hndl;
1590	dest_num++;
1591	}
1592
1593	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
1594	if (!dest_num)
1595	rule_dst = NULL;
1596	} else {
1597	if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
1598	flow_act.action |=
1599	MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1600	if (is_egress)
1601	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1602	else if (dest_num)
1603	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1604	}
1605
1606	if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
1607	(flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1608	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1609	mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1610	spec->flow_context.flow_tag, flow_attr->type);
1611	err = -EINVAL;
1612	goto free;
1613	}
1614	handler->rule = mlx5_add_flow_rules(ft, spec,
1615	flow_act: &flow_act,
1616	dest: rule_dst, num_dest: dest_num);
1617
1618	if (IS_ERR(ptr: handler->rule)) {
1619	err = PTR_ERR(ptr: handler->rule);
1620	goto free;
1621	}
1622
1623	ft_prio->refcount++;
1624	handler->prio = ft_prio;
1625	handler->dev = dev;
1626
1627	ft_prio->flow_table = ft;
1628	free:
1629	if (err && handler) {
1630	mlx5_ib_counters_clear_description(counters: handler->ibcounters);
1631	kfree(objp: handler);
1632	}
1633	kvfree(addr: spec);
1634	return err ? ERR_PTR(error: err) : handler;
1635	}
1636
1637	static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
1638	struct mlx5_ib_flow_prio *ft_prio,
1639	const struct ib_flow_attr *flow_attr,
1640	struct mlx5_flow_destination *dst)
1641	{
1642	return _create_flow_rule(dev, ft_prio, flow_attr, dst, underlay_qpn: 0, NULL);
1643	}
1644
1645	static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
1646	struct mlx5_ib_flow_prio *ft_prio,
1647	struct ib_flow_attr *flow_attr,
1648	struct mlx5_flow_destination *dst)
1649	{
1650	struct mlx5_ib_flow_handler *handler_ucast = NULL;
1651	struct mlx5_ib_flow_handler *handler = NULL;
1652
1653	static struct {
1654	struct ib_flow_spec_eth eth_flow;
1655	struct ib_flow_attr flow_attr;
1656	} leftovers_wc = { .flow_attr = { .num_of_specs = 1,
1657	.size = sizeof(leftovers_wc) },
1658	.eth_flow = {
1659	.type = IB_FLOW_SPEC_ETH,
1660	.size = sizeof(struct ib_flow_spec_eth),
1661	.mask = { .dst_mac = { 0x1 } },
1662	.val = { .dst_mac = { 0x1 } } } };
1663
1664	static struct {
1665	struct ib_flow_spec_eth eth_flow;
1666	struct ib_flow_attr flow_attr;
1667	} leftovers_uc = { .flow_attr = { .num_of_specs = 1,
1668	.size = sizeof(leftovers_uc) },
1669	.eth_flow = {
1670	.type = IB_FLOW_SPEC_ETH,
1671	.size = sizeof(struct ib_flow_spec_eth),
1672	.mask = { .dst_mac = { 0x1 } },
1673	.val = { .dst_mac = {} } } };
1674
1675	handler = create_flow_rule(dev, ft_prio, flow_attr: &leftovers_wc.flow_attr, dst);
1676	if (!IS_ERR(ptr: handler) &&
1677	flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
1678	handler_ucast = create_flow_rule(dev, ft_prio,
1679	flow_attr: &leftovers_uc.flow_attr, dst);
1680	if (IS_ERR(ptr: handler_ucast)) {
1681	mlx5_del_flow_rules(fr: handler->rule);
1682	ft_prio->refcount--;
1683	kfree(objp: handler);
1684	handler = handler_ucast;
1685	} else {
1686	list_add(new: &handler_ucast->list, head: &handler->list);
1687	}
1688	}
1689
1690	return handler;
1691	}
1692
1693	static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
1694	struct mlx5_ib_flow_prio *ft_rx,
1695	struct mlx5_ib_flow_prio *ft_tx,
1696	struct mlx5_flow_destination *dst)
1697	{
1698	struct mlx5_ib_flow_handler *handler_rx;
1699	struct mlx5_ib_flow_handler *handler_tx;
1700	int err;
1701	static const struct ib_flow_attr flow_attr = {
1702	.num_of_specs = 0,
1703	.type = IB_FLOW_ATTR_SNIFFER,
1704	.size = sizeof(flow_attr)
1705	};
1706
1707	handler_rx = create_flow_rule(dev, ft_prio: ft_rx, flow_attr: &flow_attr, dst);
1708	if (IS_ERR(ptr: handler_rx)) {
1709	err = PTR_ERR(ptr: handler_rx);
1710	goto err;
1711	}
1712
1713	handler_tx = create_flow_rule(dev, ft_prio: ft_tx, flow_attr: &flow_attr, dst);
1714	if (IS_ERR(ptr: handler_tx)) {
1715	err = PTR_ERR(ptr: handler_tx);
1716	goto err_tx;
1717	}
1718
1719	list_add(new: &handler_tx->list, head: &handler_rx->list);
1720
1721	return handler_rx;
1722
1723	err_tx:
1724	mlx5_del_flow_rules(fr: handler_rx->rule);
1725	ft_rx->refcount--;
1726	kfree(objp: handler_rx);
1727	err:
1728	return ERR_PTR(error: err);
1729	}
1730
1731	static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
1732	struct ib_flow_attr *flow_attr,
1733	struct ib_udata *udata)
1734	{
1735	struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
1736	struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
1737	struct mlx5_ib_flow_handler *handler = NULL;
1738	struct mlx5_flow_destination *dst = NULL;
1739	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
1740	struct mlx5_ib_flow_prio *ft_prio;
1741	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1742	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
1743	size_t min_ucmd_sz, required_ucmd_sz;
1744	int err;
1745	int underlay_qpn;
1746
1747	if (udata && udata->inlen) {
1748	min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
1749	if (udata->inlen < min_ucmd_sz)
1750	return ERR_PTR(error: -EOPNOTSUPP);
1751
1752	err = ib_copy_from_udata(dest: &ucmd_hdr, udata, len: min_ucmd_sz);
1753	if (err)
1754	return ERR_PTR(error: err);
1755
1756	/* currently supports only one counters data */
1757	if (ucmd_hdr.ncounters_data > 1)
1758	return ERR_PTR(error: -EINVAL);
1759
1760	required_ucmd_sz = min_ucmd_sz +
1761	sizeof(struct mlx5_ib_flow_counters_data) *
1762	ucmd_hdr.ncounters_data;
1763	if (udata->inlen > required_ucmd_sz &&
1764	!ib_is_udata_cleared(udata, offset: required_ucmd_sz,
1765	len: udata->inlen - required_ucmd_sz))
1766	return ERR_PTR(error: -EOPNOTSUPP);
1767
1768	ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
1769	if (!ucmd)
1770	return ERR_PTR(error: -ENOMEM);
1771
1772	err = ib_copy_from_udata(dest: ucmd, udata, len: required_ucmd_sz);
1773	if (err)
1774	goto free_ucmd;
1775	}
1776
1777	if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
1778	err = -ENOMEM;
1779	goto free_ucmd;
1780	}
1781
1782	if (flow_attr->flags &
1783	~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) {
1784	err = -EINVAL;
1785	goto free_ucmd;
1786	}
1787
1788	if (is_egress &&
1789	(flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1790	flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1791	err = -EINVAL;
1792	goto free_ucmd;
1793	}
1794
1795	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1796	if (!dst) {
1797	err = -ENOMEM;
1798	goto free_ucmd;
1799	}
1800
1801	mutex_lock(&dev->flow_db->lock);
1802
1803	ft_prio = get_flow_table(dev, flow_attr,
1804	ft_type: is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
1805	if (IS_ERR(ptr: ft_prio)) {
1806	err = PTR_ERR(ptr: ft_prio);
1807	goto unlock;
1808	}
1809	if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
1810	ft_prio_tx = get_flow_table(dev, flow_attr, ft_type: MLX5_IB_FT_TX);
1811	if (IS_ERR(ptr: ft_prio_tx)) {
1812	err = PTR_ERR(ptr: ft_prio_tx);
1813	ft_prio_tx = NULL;
1814	goto destroy_ft;
1815	}
1816	}
1817
1818	if (is_egress) {
1819	dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1820	} else {
1821	dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1822	if (mqp->is_rss)
1823	dst->tir_num = mqp->rss_qp.tirn;
1824	else
1825	dst->tir_num = mqp->raw_packet_qp.rq.tirn;
1826	}
1827
1828	switch (flow_attr->type) {
1829	case IB_FLOW_ATTR_NORMAL:
1830	underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1831	mqp->underlay_qpn :
1832	0;
1833	handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
1834	underlay_qpn, ucmd);
1835	break;
1836	case IB_FLOW_ATTR_ALL_DEFAULT:
1837	case IB_FLOW_ATTR_MC_DEFAULT:
1838	handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
1839	break;
1840	case IB_FLOW_ATTR_SNIFFER:
1841	handler = create_sniffer_rule(dev, ft_rx: ft_prio, ft_tx: ft_prio_tx, dst);
1842	break;
1843	default:
1844	err = -EINVAL;
1845	goto destroy_ft;
1846	}
1847
1848	if (IS_ERR(ptr: handler)) {
1849	err = PTR_ERR(ptr: handler);
1850	handler = NULL;
1851	goto destroy_ft;
1852	}
1853
1854	mutex_unlock(lock: &dev->flow_db->lock);
1855	kfree(objp: dst);
1856	kfree(objp: ucmd);
1857
1858	return &handler->ibflow;
1859
1860	destroy_ft:
1861	put_flow_table(dev, prio: ft_prio, ft_added: false);
1862	if (ft_prio_tx)
1863	put_flow_table(dev, prio: ft_prio_tx, ft_added: false);
1864	unlock:
1865	mutex_unlock(lock: &dev->flow_db->lock);
1866	kfree(objp: dst);
1867	free_ucmd:
1868	kfree(objp: ucmd);
1869	return ERR_PTR(error: err);
1870	}
1871
1872	static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev,
1873	enum mlx5_flow_namespace_type type,
1874	u32 *flags, u16 *vport_idx,
1875	u16 *vport,
1876	struct mlx5_core_dev **ft_mdev,
1877	u32 ib_port, u16 *esw_owner_vhca_id)
1878	{
1879	struct mlx5_core_dev *esw_mdev;
1880
1881	if (!is_mdev_switchdev_mode(dev: dev->mdev))
1882	return 0;
1883
1884	if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager))
1885	return -EOPNOTSUPP;
1886
1887	if (!dev->port[ib_port - 1].rep)
1888	return -EINVAL;
1889
1890	esw_mdev = mlx5_eswitch_get_core_dev(esw: dev->port[ib_port - 1].rep->esw);
1891	if (esw_mdev != dev->mdev) {
1892	if (!MLX5_CAP_ADV_RDMA(dev->mdev,
1893	rdma_transport_manager_other_eswitch))
1894	return -EOPNOTSUPP;
1895	*flags |= MLX5_FLOW_TABLE_OTHER_ESWITCH;
1896	*esw_owner_vhca_id = MLX5_CAP_GEN(esw_mdev, vhca_id);
1897	}
1898
1899	*flags |= MLX5_FLOW_TABLE_OTHER_VPORT;
1900	*ft_mdev = esw_mdev;
1901	*vport = dev->port[ib_port - 1].rep->vport;
1902	*vport_idx = dev->port[ib_port - 1].rep->vport_index;
1903
1904	return 0;
1905	}
1906
1907	static struct mlx5_ib_flow_prio *
1908	_get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
1909	enum mlx5_flow_namespace_type ns_type,
1910	bool mcast, u32 ib_port)
1911	{
1912	struct mlx5_core_dev *ft_mdev = dev->mdev;
1913	struct mlx5_flow_table_attr ft_attr = {};
1914	struct mlx5_flow_namespace *ns = NULL;
1915	struct mlx5_ib_flow_prio *prio = NULL;
1916	u16 esw_owner_vhca_id = 0;
1917	int max_table_size = 0;
1918	u16 vport_idx = 0;
1919	bool esw_encap;
1920	u32 flags = 0;
1921	u16 vport = 0;
1922	int priority;
1923	int ret;
1924
1925	if (mcast)
1926	priority = MLX5_IB_FLOW_MCAST_PRIO;
1927	else
1928	priority = ib_prio_to_core_prio(priority: user_priority, dont_trap: false);
1929
1930	esw_encap = mlx5_eswitch_get_encap_mode(dev: dev->mdev) !=
1931	DEVLINK_ESWITCH_ENCAP_MODE_NONE;
1932	switch (ns_type) {
1933	case MLX5_FLOW_NAMESPACE_BYPASS:
1934	max_table_size = BIT(
1935	MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size));
1936	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
1937	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1938	if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1939	reformat_l3_tunnel_to_l2) &&
1940	!esw_encap)
1941	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1942	break;
1943	case MLX5_FLOW_NAMESPACE_EGRESS:
1944	max_table_size = BIT(
1945	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
1946	if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) &&
1947	!esw_encap)
1948	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1949	break;
1950	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
1951	max_table_size = BIT(
1952	MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
1953	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
1954	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1955	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev,
1956	reformat_l3_tunnel_to_l2) &&
1957	esw_encap)
1958	flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1959	priority = user_priority;
1960	break;
1961	case MLX5_FLOW_NAMESPACE_RDMA_RX:
1962	max_table_size = BIT(
1963	MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size));
1964	priority = user_priority;
1965	break;
1966	case MLX5_FLOW_NAMESPACE_RDMA_TX:
1967	max_table_size = BIT(
1968	MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size));
1969	priority = user_priority;
1970	break;
1971	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
1972	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
1973	if (ib_port == 0 ||
1974	user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
1975	return ERR_PTR(error: -EINVAL);
1976	ret = mlx5_ib_fill_transport_ns_info(dev, type: ns_type, flags: &flags,
1977	vport_idx: &vport_idx, vport: &vport,
1978	ft_mdev: &ft_mdev, ib_port,
1979	esw_owner_vhca_id: &esw_owner_vhca_id);
1980	if (ret)
1981	return ERR_PTR(error: ret);
1982
1983	if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX)
1984	max_table_size =
1985	BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(
1986	ft_mdev, log_max_ft_size));
1987	else
1988	max_table_size =
1989	BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(
1990	ft_mdev, log_max_ft_size));
1991	priority = user_priority;
1992	break;
1993	default:
1994	break;
1995	}
1996
1997	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
1998
1999	if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX ||
2000	ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)
2001	ns = mlx5_get_flow_vport_namespace(dev: ft_mdev, type: ns_type, vport_idx);
2002	else
2003	ns = mlx5_get_flow_namespace(dev: ft_mdev, type: ns_type);
2004
2005	if (!ns)
2006	return ERR_PTR(error: -EOPNOTSUPP);
2007
2008	switch (ns_type) {
2009	case MLX5_FLOW_NAMESPACE_BYPASS:
2010	prio = &dev->flow_db->prios[priority];
2011	break;
2012	case MLX5_FLOW_NAMESPACE_EGRESS:
2013	prio = &dev->flow_db->egress_prios[priority];
2014	break;
2015	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
2016	prio = &dev->flow_db->fdb[priority];
2017	break;
2018	case MLX5_FLOW_NAMESPACE_RDMA_RX:
2019	prio = &dev->flow_db->rdma_rx[priority];
2020	break;
2021	case MLX5_FLOW_NAMESPACE_RDMA_TX:
2022	prio = &dev->flow_db->rdma_tx[priority];
2023	break;
2024	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
2025	prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1];
2026	break;
2027	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
2028	prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1];
2029	break;
2030	default: return ERR_PTR(error: -EINVAL);
2031	}
2032
2033	if (!prio)
2034	return ERR_PTR(error: -EINVAL);
2035
2036	if (prio->flow_table)
2037	return prio;
2038
2039	ft_attr.prio = priority;
2040	ft_attr.max_fte = max_table_size;
2041	ft_attr.flags = flags;
2042	ft_attr.vport = vport;
2043	ft_attr.esw_owner_vhca_id = esw_owner_vhca_id;
2044	ft_attr.autogroup.max_num_groups = MLX5_FS_MAX_TYPES;
2045	return _get_prio(ns, prio, ft_attr: &ft_attr);
2046	}
2047
2048	static struct mlx5_ib_flow_handler *
2049	_create_raw_flow_rule(struct mlx5_ib_dev *dev,
2050	struct mlx5_ib_flow_prio *ft_prio,
2051	struct mlx5_flow_destination *dst,
2052	struct mlx5_ib_flow_matcher *fs_matcher,
2053	struct mlx5_flow_context *flow_context,
2054	struct mlx5_flow_act *flow_act,
2055	void *cmd_in, int inlen,
2056	int dst_num)
2057	{
2058	struct mlx5_ib_flow_handler *handler;
2059	struct mlx5_flow_spec *spec;
2060	struct mlx5_flow_table *ft = ft_prio->flow_table;
2061	int err = 0;
2062
2063	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2064	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2065	if (!handler || !spec) {
2066	err = -ENOMEM;
2067	goto free;
2068	}
2069
2070	INIT_LIST_HEAD(list: &handler->list);
2071
2072	memcpy(spec->match_value, cmd_in, inlen);
2073	memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
2074	fs_matcher->mask_len);
2075	spec->match_criteria_enable = fs_matcher->match_criteria_enable;
2076	spec->flow_context = *flow_context;
2077
2078	handler->rule = mlx5_add_flow_rules(ft, spec,
2079	flow_act, dest: dst, num_dest: dst_num);
2080
2081	if (IS_ERR(ptr: handler->rule)) {
2082	err = PTR_ERR(ptr: handler->rule);
2083	goto free;
2084	}
2085
2086	ft_prio->refcount++;
2087	handler->prio = ft_prio;
2088	handler->dev = dev;
2089	ft_prio->flow_table = ft;
2090
2091	free:
2092	if (err)
2093	kfree(objp: handler);
2094	kvfree(addr: spec);
2095	return err ? ERR_PTR(error: err) : handler;
2096	}
2097
2098	static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
2099	void *match_v)
2100	{
2101	void *match_c;
2102	void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
2103	void *dmac, *dmac_mask;
2104	void *ipv4, *ipv4_mask;
2105
2106	if (!(fs_matcher->match_criteria_enable &
2107	(1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
2108	return false;
2109
2110	match_c = fs_matcher->matcher_mask.match_params;
2111	match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
2112	outer_headers);
2113	match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
2114	outer_headers);
2115
2116	dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2117	dmac_47_16);
2118	dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2119	dmac_47_16);
2120
2121	if (is_multicast_ether_addr(addr: dmac) &&
2122	is_multicast_ether_addr(addr: dmac_mask))
2123	return true;
2124
2125	ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2126	dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2127
2128	ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2129	dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2130
2131	if (ipv4_is_multicast(addr: *(__be32 *)(ipv4)) &&
2132	ipv4_is_multicast(addr: *(__be32 *)(ipv4_mask)))
2133	return true;
2134
2135	return false;
2136	}
2137
2138	static struct mlx5_ib_flow_handler *raw_fs_rule_add(
2139	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
2140	struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
2141	struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type)
2142	{
2143	struct mlx5_flow_destination *dst;
2144	struct mlx5_ib_flow_prio *ft_prio;
2145	struct mlx5_ib_flow_handler *handler;
2146	int dst_num = 0;
2147	bool mcast;
2148	int err;
2149
2150	if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
2151	return ERR_PTR(error: -EOPNOTSUPP);
2152
2153	if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
2154	return ERR_PTR(error: -ENOMEM);
2155
2156	dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
2157	if (!dst)
2158	return ERR_PTR(error: -ENOMEM);
2159
2160	mcast = raw_fs_is_multicast(fs_matcher, match_v: cmd_in);
2161	mutex_lock(&dev->flow_db->lock);
2162
2163	ft_prio = _get_flow_table(dev, user_priority: fs_matcher->priority,
2164	ns_type: fs_matcher->ns_type, mcast,
2165	ib_port: fs_matcher->ib_port);
2166	if (IS_ERR(ptr: ft_prio)) {
2167	err = PTR_ERR(ptr: ft_prio);
2168	goto unlock;
2169	}
2170
2171	switch (dest_type) {
2172	case MLX5_FLOW_DESTINATION_TYPE_TIR:
2173	dst[dst_num].type = dest_type;
2174	dst[dst_num++].tir_num = dest_id;
2175	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2176	break;
2177	case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
2178	dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
2179	dst[dst_num++].ft_num = dest_id;
2180	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2181	break;
2182	case MLX5_FLOW_DESTINATION_TYPE_PORT:
2183	dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2184	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
2185	break;
2186	default:
2187	break;
2188	}
2189
2190	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
2191	if (WARN_ON(!counter)) {
2192	err = -EINVAL;
2193	goto unlock;
2194	}
2195	dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2196	dst[dst_num].counter = counter;
2197	dst_num++;
2198	}
2199
2200	handler = _create_raw_flow_rule(dev, ft_prio, dst: dst_num ? dst : NULL,
2201	fs_matcher, flow_context, flow_act,
2202	cmd_in, inlen, dst_num);
2203
2204	if (IS_ERR(ptr: handler)) {
2205	err = PTR_ERR(ptr: handler);
2206	goto destroy_ft;
2207	}
2208
2209	mutex_unlock(lock: &dev->flow_db->lock);
2210	atomic_inc(v: &fs_matcher->usecnt);
2211	handler->flow_matcher = fs_matcher;
2212
2213	kfree(objp: dst);
2214
2215	return handler;
2216
2217	destroy_ft:
2218	put_flow_table(dev, prio: ft_prio, ft_added: false);
2219	unlock:
2220	mutex_unlock(lock: &dev->flow_db->lock);
2221	kfree(objp: dst);
2222
2223	return ERR_PTR(error: err);
2224	}
2225
2226	static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
2227	{
2228	switch (maction->flow_action_raw.sub_type) {
2229	case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
2230	mlx5_modify_header_dealloc(dev: maction->flow_action_raw.dev->mdev,
2231	modify_hdr: maction->flow_action_raw.modify_hdr);
2232	break;
2233	case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
2234	mlx5_packet_reformat_dealloc(dev: maction->flow_action_raw.dev->mdev,
2235	reformat: maction->flow_action_raw.pkt_reformat);
2236	break;
2237	case MLX5_IB_FLOW_ACTION_DECAP:
2238	break;
2239	default:
2240	break;
2241	}
2242	}
2243
2244	static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
2245	{
2246	struct mlx5_ib_flow_action *maction = to_mflow_act(ibact: action);
2247
2248	switch (action->type) {
2249	case IB_FLOW_ACTION_UNSPECIFIED:
2250	destroy_flow_action_raw(maction);
2251	break;
2252	default:
2253	WARN_ON(true);
2254	break;
2255	}
2256
2257	kfree(objp: maction);
2258	return 0;
2259	}
2260
2261	static int
2262	mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
2263	enum mlx5_flow_namespace_type *namespace)
2264	{
2265	switch (table_type) {
2266	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
2267	*namespace = MLX5_FLOW_NAMESPACE_BYPASS;
2268	break;
2269	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
2270	*namespace = MLX5_FLOW_NAMESPACE_EGRESS;
2271	break;
2272	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
2273	*namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS;
2274	break;
2275	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
2276	*namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
2277	break;
2278	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
2279	*namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
2280	break;
2281	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX:
2282	*namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX;
2283	break;
2284	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX:
2285	*namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX;
2286	break;
2287	default:
2288	return -EINVAL;
2289	}
2290
2291	return 0;
2292	}
2293
2294	static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
2295	[MLX5_IB_FLOW_TYPE_NORMAL] = {
2296	.type = UVERBS_ATTR_TYPE_PTR_IN,
2297	.u.ptr = {
2298	.len = sizeof(u16), /* data is priority */
2299	.min_len = sizeof(u16),
2300	}
2301	},
2302	[MLX5_IB_FLOW_TYPE_SNIFFER] = {
2303	.type = UVERBS_ATTR_TYPE_PTR_IN,
2304	UVERBS_ATTR_NO_DATA(),
2305	},
2306	[MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
2307	.type = UVERBS_ATTR_TYPE_PTR_IN,
2308	UVERBS_ATTR_NO_DATA(),
2309	},
2310	[MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
2311	.type = UVERBS_ATTR_TYPE_PTR_IN,
2312	UVERBS_ATTR_NO_DATA(),
2313	},
2314	};
2315
2316	static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
2317	{
2318	struct devx_obj *devx_obj = obj;
2319	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2320
2321	switch (opcode) {
2322	case MLX5_CMD_OP_DESTROY_TIR:
2323	*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2324	*dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
2325	obj_id);
2326	return true;
2327
2328	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
2329	*dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2330	*dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
2331	table_id);
2332	return true;
2333	default:
2334	return false;
2335	}
2336	}
2337
2338	static int get_dests(struct uverbs_attr_bundle *attrs,
2339	struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
2340	int *dest_type, struct ib_qp **qp, u32 *flags)
2341	{
2342	bool dest_devx, dest_qp;
2343	void *devx_obj;
2344	int err;
2345
2346	dest_devx = uverbs_attr_is_valid(attrs_bundle: attrs,
2347	idx: MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2348	dest_qp = uverbs_attr_is_valid(attrs_bundle: attrs,
2349	idx: MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2350
2351	*flags = 0;
2352	err = uverbs_get_flags32(to: flags, attrs_bundle: attrs, idx: MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
2353	allowed_bits: MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
2354	MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
2355	if (err)
2356	return err;
2357
2358	/* Both flags are not allowed */
2359	if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
2360	*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2361	return -EINVAL;
2362
2363	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
2364	if (dest_devx && (dest_qp || *flags))
2365	return -EINVAL;
2366	else if (dest_qp && *flags)
2367	return -EINVAL;
2368	}
2369
2370	/* Allow only DEVX object, drop as dest for FDB */
2371	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2372	!(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
2373	return -EINVAL;
2374
2375	/* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
2376	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2377	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2378	((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
2379	return -EINVAL;
2380
2381	*qp = NULL;
2382	if (dest_devx) {
2383	devx_obj =
2384	uverbs_attr_get_obj(attrs_bundle: attrs,
2385	idx: MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2386
2387	/* Verify that the given DEVX object is a flow
2388	* steering destination.
2389	*/
2390	if (!is_flow_dest(obj: devx_obj, dest_id, dest_type))
2391	return -EINVAL;
2392	/* Allow only flow table as dest when inserting to FDB or RDMA_RX */
2393	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS ||
2394	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2395	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2396	*dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
2397	return -EINVAL;
2398	} else if (dest_qp) {
2399	struct mlx5_ib_qp *mqp;
2400
2401	*qp = uverbs_attr_get_obj(attrs_bundle: attrs,
2402	idx: MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2403	if (IS_ERR(ptr: *qp))
2404	return PTR_ERR(ptr: *qp);
2405
2406	if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
2407	return -EINVAL;
2408
2409	mqp = to_mqp(ibqp: *qp);
2410	if (mqp->is_rss)
2411	*dest_id = mqp->rss_qp.tirn;
2412	else
2413	*dest_id = mqp->raw_packet_qp.rq.tirn;
2414	*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2415	} else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2416	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2417	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) &&
2418	!(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) {
2419	*dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2420	}
2421
2422	if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
2423	(fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2424	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2425	fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX))
2426	return -EINVAL;
2427
2428	return 0;
2429	}
2430
2431	static bool
2432	is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size)
2433	{
2434	struct devx_obj *devx_obj = obj;
2435	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2436
2437	if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
2438
2439	if (offset && offset >= devx_obj->flow_counter_bulk_size)
2440	return false;
2441
2442	*fc_bulk_size = devx_obj->flow_counter_bulk_size;
2443	*counter_id = MLX5_GET(dealloc_flow_counter_in,
2444	devx_obj->dinbox,
2445	flow_counter_id);
2446	*counter_id += offset;
2447	return true;
2448	}
2449
2450	return false;
2451	}
2452
2453	#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
2454	static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
2455	struct uverbs_attr_bundle *attrs)
2456	{
2457	struct mlx5_flow_context flow_context = {.flow_tag =
2458	MLX5_FS_DEFAULT_FLOW_TAG};
2459	int dest_id, dest_type = -1, inlen, len, ret, i;
2460	struct mlx5_ib_flow_handler *flow_handler;
2461	struct mlx5_ib_flow_matcher *fs_matcher;
2462	struct ib_uobject **arr_flow_actions;
2463	struct ib_uflow_resources *uflow_res;
2464	struct mlx5_flow_act flow_act = {};
2465	struct mlx5_fc *counter = NULL;
2466	struct ib_qp *qp = NULL;
2467	void *devx_obj, *cmd_in;
2468	struct ib_uobject *uobj;
2469	struct mlx5_ib_dev *dev;
2470	u32 flags;
2471
2472	if (!rdma_uattrs_has_raw_cap(attrs))
2473	return -EPERM;
2474
2475	fs_matcher = uverbs_attr_get_obj(attrs_bundle: attrs,
2476	idx: MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
2477	uobj = uverbs_attr_get_uobject(attrs_bundle: attrs, idx: MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
2478	dev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
2479
2480	if (get_dests(attrs, fs_matcher, dest_id: &dest_id, dest_type: &dest_type, qp: &qp, flags: &flags))
2481	return -EINVAL;
2482
2483	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
2484	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
2485
2486	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2487	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
2488
2489	len = uverbs_attr_get_uobjs_arr(attrs_bundle: attrs,
2490	attr_idx: MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, arr: &arr_flow_actions);
2491	if (len) {
2492	u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0;
2493	devx_obj = arr_flow_actions[0]->object;
2494
2495	if (uverbs_attr_is_valid(attrs_bundle: attrs,
2496	idx: MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
2497
2498	int num_offsets = uverbs_attr_ptr_get_array_size(
2499	attrs,
2500	idx: MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
2501	elem_size: sizeof(u32));
2502
2503	if (num_offsets != 1)
2504	return -EINVAL;
2505
2506	offset_attr = uverbs_attr_get_alloced_ptr(
2507	attrs_bundle: attrs,
2508	idx: MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
2509	offset = *offset_attr;
2510	}
2511
2512	if (!is_flow_counter(obj: devx_obj, offset, counter_id: &counter_id, fc_bulk_size: &fc_bulk_size))
2513	return -EINVAL;
2514	counter = mlx5_fc_local_create(counter_id, offset, bulk_size: fc_bulk_size);
2515	if (IS_ERR(ptr: counter))
2516	return PTR_ERR(ptr: counter);
2517
2518	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2519	}
2520
2521	cmd_in = uverbs_attr_get_alloced_ptr(
2522	attrs_bundle: attrs, idx: MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2523	inlen = uverbs_attr_get_len(attrs_bundle: attrs,
2524	idx: MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2525
2526	uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
2527	if (!uflow_res) {
2528	ret = -ENOMEM;
2529	goto destroy_counter;
2530	}
2531
2532	len = uverbs_attr_get_uobjs_arr(attrs_bundle: attrs,
2533	attr_idx: MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, arr: &arr_flow_actions);
2534	for (i = 0; i < len; i++) {
2535	struct mlx5_ib_flow_action *maction =
2536	to_mflow_act(ibact: arr_flow_actions[i]->object);
2537
2538	ret = parse_flow_flow_action(maction, is_egress: false, action: &flow_act);
2539	if (ret)
2540	goto err_out;
2541	flow_resources_add(uflow_res, type: IB_FLOW_SPEC_ACTION_HANDLE,
2542	ibobj: arr_flow_actions[i]->object);
2543	}
2544
2545	ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
2546	MLX5_IB_ATTR_CREATE_FLOW_TAG);
2547	if (!ret) {
2548	if (flow_context.flow_tag >= BIT(24)) {
2549	ret = -EINVAL;
2550	goto err_out;
2551	}
2552	flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
2553	}
2554
2555	flow_handler =
2556	raw_fs_rule_add(dev, fs_matcher, flow_context: &flow_context, flow_act: &flow_act,
2557	counter, cmd_in, inlen, dest_id, dest_type);
2558	if (IS_ERR(ptr: flow_handler)) {
2559	ret = PTR_ERR(ptr: flow_handler);
2560	goto err_out;
2561	}
2562
2563	ib_set_flow(uobj, ibflow: &flow_handler->ibflow, qp, device: &dev->ib_dev, uflow_res);
2564
2565	return 0;
2566	err_out:
2567	ib_uverbs_flow_resources_free(uflow_res);
2568	destroy_counter:
2569	if (counter)
2570	mlx5_fc_local_destroy(counter);
2571	return ret;
2572	}
2573
2574	static int flow_matcher_cleanup(struct ib_uobject *uobject,
2575	enum rdma_remove_reason why,
2576	struct uverbs_attr_bundle *attrs)
2577	{
2578	struct mlx5_ib_flow_matcher *obj = uobject->object;
2579
2580	if (atomic_read(v: &obj->usecnt))
2581	return -EBUSY;
2582
2583	kfree(objp: obj);
2584	return 0;
2585	}
2586
2587	static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,
2588	struct mlx5_ib_flow_prio *ft_prio,
2589	enum mlx5_flow_namespace_type ns_type)
2590	{
2591	struct mlx5_flow_table_attr ft_attr = {};
2592	struct mlx5_flow_namespace *ns;
2593	struct mlx5_flow_table *ft;
2594
2595	if (ft_prio->anchor.ft)
2596	return 0;
2597
2598	ns = mlx5_get_flow_namespace(dev: dev->mdev, type: ns_type);
2599	if (!ns)
2600	return -EOPNOTSUPP;
2601
2602	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
2603	ft_attr.uid = MLX5_SHARED_RESOURCE_UID;
2604	ft_attr.prio = 0;
2605	ft_attr.max_fte = 2;
2606	ft_attr.level = 1;
2607
2608	ft = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
2609	if (IS_ERR(ptr: ft))
2610	return PTR_ERR(ptr: ft);
2611
2612	ft_prio->anchor.ft = ft;
2613
2614	return 0;
2615	}
2616
2617	static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)
2618	{
2619	if (ft_prio->anchor.ft) {
2620	mlx5_destroy_flow_table(ft: ft_prio->anchor.ft);
2621	ft_prio->anchor.ft = NULL;
2622	}
2623	}
2624
2625	static int
2626	steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2627	{
2628	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2629	struct mlx5_flow_group *fg;
2630	void *flow_group_in;
2631	int err = 0;
2632
2633	if (ft_prio->anchor.fg_drop)
2634	return 0;
2635
2636	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2637	if (!flow_group_in)
2638	return -ENOMEM;
2639
2640	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
2641	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
2642
2643	fg = mlx5_create_flow_group(ft: ft_prio->anchor.ft, in: flow_group_in);
2644	if (IS_ERR(ptr: fg)) {
2645	err = PTR_ERR(ptr: fg);
2646	goto out;
2647	}
2648
2649	ft_prio->anchor.fg_drop = fg;
2650
2651	out:
2652	kvfree(addr: flow_group_in);
2653
2654	return err;
2655	}
2656
2657	static void
2658	steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2659	{
2660	if (ft_prio->anchor.fg_drop) {
2661	mlx5_destroy_flow_group(fg: ft_prio->anchor.fg_drop);
2662	ft_prio->anchor.fg_drop = NULL;
2663	}
2664	}
2665
2666	static int
2667	steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2668	{
2669	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2670	struct mlx5_flow_group *fg;
2671	void *flow_group_in;
2672	int err = 0;
2673
2674	if (ft_prio->anchor.fg_goto_table)
2675	return 0;
2676
2677	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2678	if (!flow_group_in)
2679	return -ENOMEM;
2680
2681	fg = mlx5_create_flow_group(ft: ft_prio->anchor.ft, in: flow_group_in);
2682	if (IS_ERR(ptr: fg)) {
2683	err = PTR_ERR(ptr: fg);
2684	goto out;
2685	}
2686	ft_prio->anchor.fg_goto_table = fg;
2687
2688	out:
2689	kvfree(addr: flow_group_in);
2690
2691	return err;
2692	}
2693
2694	static void
2695	steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2696	{
2697	if (ft_prio->anchor.fg_goto_table) {
2698	mlx5_destroy_flow_group(fg: ft_prio->anchor.fg_goto_table);
2699	ft_prio->anchor.fg_goto_table = NULL;
2700	}
2701	}
2702
2703	static int
2704	steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2705	{
2706	struct mlx5_flow_act flow_act = {};
2707	struct mlx5_flow_handle *handle;
2708
2709	if (ft_prio->anchor.rule_drop)
2710	return 0;
2711
2712	flow_act.fg = ft_prio->anchor.fg_drop;
2713	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2714
2715	handle = mlx5_add_flow_rules(ft: ft_prio->anchor.ft, NULL, flow_act: &flow_act,
2716	NULL, num_dest: 0);
2717	if (IS_ERR(ptr: handle))
2718	return PTR_ERR(ptr: handle);
2719
2720	ft_prio->anchor.rule_drop = handle;
2721
2722	return 0;
2723	}
2724
2725	static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2726	{
2727	if (ft_prio->anchor.rule_drop) {
2728	mlx5_del_flow_rules(fr: ft_prio->anchor.rule_drop);
2729	ft_prio->anchor.rule_drop = NULL;
2730	}
2731	}
2732
2733	static int
2734	steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2735	{
2736	struct mlx5_flow_destination dest = {};
2737	struct mlx5_flow_act flow_act = {};
2738	struct mlx5_flow_handle *handle;
2739
2740	if (ft_prio->anchor.rule_goto_table)
2741	return 0;
2742
2743	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2744	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
2745	flow_act.fg = ft_prio->anchor.fg_goto_table;
2746
2747	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2748	dest.ft = ft_prio->flow_table;
2749
2750	handle = mlx5_add_flow_rules(ft: ft_prio->anchor.ft, NULL, flow_act: &flow_act,
2751	dest: &dest, num_dest: 1);
2752	if (IS_ERR(ptr: handle))
2753	return PTR_ERR(ptr: handle);
2754
2755	ft_prio->anchor.rule_goto_table = handle;
2756
2757	return 0;
2758	}
2759
2760	static void
2761	steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2762	{
2763	if (ft_prio->anchor.rule_goto_table) {
2764	mlx5_del_flow_rules(fr: ft_prio->anchor.rule_goto_table);
2765	ft_prio->anchor.rule_goto_table = NULL;
2766	}
2767	}
2768
2769	static int steering_anchor_create_res(struct mlx5_ib_dev *dev,
2770	struct mlx5_ib_flow_prio *ft_prio,
2771	enum mlx5_flow_namespace_type ns_type)
2772	{
2773	int err;
2774
2775	err = steering_anchor_create_ft(dev, ft_prio, ns_type);
2776	if (err)
2777	return err;
2778
2779	err = steering_anchor_create_fg_drop(ft_prio);
2780	if (err)
2781	goto destroy_ft;
2782
2783	err = steering_anchor_create_fg_goto_table(ft_prio);
2784	if (err)
2785	goto destroy_fg_drop;
2786
2787	err = steering_anchor_create_rule_drop(ft_prio);
2788	if (err)
2789	goto destroy_fg_goto_table;
2790
2791	err = steering_anchor_create_rule_goto_table(ft_prio);
2792	if (err)
2793	goto destroy_rule_drop;
2794
2795	return 0;
2796
2797	destroy_rule_drop:
2798	steering_anchor_destroy_rule_drop(ft_prio);
2799	destroy_fg_goto_table:
2800	steering_anchor_destroy_fg_goto_table(ft_prio);
2801	destroy_fg_drop:
2802	steering_anchor_destroy_fg_drop(ft_prio);
2803	destroy_ft:
2804	steering_anchor_destroy_ft(ft_prio);
2805
2806	return err;
2807	}
2808
2809	static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)
2810	{
2811	steering_anchor_destroy_rule_goto_table(ft_prio);
2812	steering_anchor_destroy_rule_drop(ft_prio);
2813	steering_anchor_destroy_fg_goto_table(ft_prio);
2814	steering_anchor_destroy_fg_drop(ft_prio);
2815	steering_anchor_destroy_ft(ft_prio);
2816	}
2817
2818	static int steering_anchor_cleanup(struct ib_uobject *uobject,
2819	enum rdma_remove_reason why,
2820	struct uverbs_attr_bundle *attrs)
2821	{
2822	struct mlx5_ib_steering_anchor *obj = uobject->object;
2823
2824	if (atomic_read(v: &obj->usecnt))
2825	return -EBUSY;
2826
2827	mutex_lock(&obj->dev->flow_db->lock);
2828	if (!--obj->ft_prio->anchor.rule_goto_table_ref)
2829	steering_anchor_destroy_rule_goto_table(ft_prio: obj->ft_prio);
2830
2831	put_flow_table(dev: obj->dev, prio: obj->ft_prio, ft_added: true);
2832	mutex_unlock(lock: &obj->dev->flow_db->lock);
2833
2834	kfree(objp: obj);
2835	return 0;
2836	}
2837
2838	static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,
2839	int count)
2840	{
2841	while (count--)
2842	mlx5_steering_anchor_destroy_res(ft_prio: &prio[count]);
2843	}
2844
2845	void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)
2846	{
2847	fs_cleanup_anchor(prio: dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT);
2848	fs_cleanup_anchor(prio: dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT);
2849	fs_cleanup_anchor(prio: dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS);
2850	fs_cleanup_anchor(prio: dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS);
2851	fs_cleanup_anchor(prio: dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS);
2852	fs_cleanup_anchor(prio: dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT);
2853	fs_cleanup_anchor(prio: dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT);
2854	}
2855
2856	static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
2857	struct mlx5_ib_flow_matcher *obj)
2858	{
2859	enum mlx5_ib_uapi_flow_table_type ft_type =
2860	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
2861	u32 flags;
2862	int err;
2863
2864	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
2865	* users should switch to it. We leave this to not break userspace
2866	*/
2867	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
2868	uverbs_attr_is_valid(attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
2869	return -EINVAL;
2870
2871	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
2872	err = uverbs_get_const(&ft_type, attrs,
2873	MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
2874	if (err)
2875	return err;
2876
2877	err = mlx5_ib_ft_type_to_namespace(table_type: ft_type, namespace: &obj->ns_type);
2878	if (err)
2879	return err;
2880
2881	return 0;
2882	}
2883
2884	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
2885	err = uverbs_get_flags32(to: &flags, attrs_bundle: attrs,
2886	idx: MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2887	allowed_bits: IB_FLOW_ATTR_FLAGS_EGRESS);
2888	if (err)
2889	return err;
2890
2891	if (flags)
2892	return mlx5_ib_ft_type_to_namespace(
2893	table_type: MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
2894	namespace: &obj->ns_type);
2895	}
2896
2897	obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
2898
2899	return 0;
2900	}
2901
2902	static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps)
2903	{
2904	if (is_mdev_switchdev_mode(dev: dev->mdev))
2905	return UCAP_ENABLED(enabled_caps,
2906	RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
2907
2908	return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL);
2909	}
2910
2911	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
2912	struct uverbs_attr_bundle *attrs)
2913	{
2914	struct ib_uobject *uobj = uverbs_attr_get_uobject(
2915	attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
2916	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
2917	struct mlx5_ib_flow_matcher *obj;
2918	int err;
2919
2920	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
2921	if (!obj)
2922	return -ENOMEM;
2923
2924	obj->mask_len = uverbs_attr_get_len(
2925	attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2926	err = uverbs_copy_from(&obj->matcher_mask,
2927	attrs,
2928	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2929	if (err)
2930	goto end;
2931
2932	obj->flow_type = uverbs_attr_get_enum_id(
2933	attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2934
2935	if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
2936	err = uverbs_copy_from(&obj->priority,
2937	attrs,
2938	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2939	if (err)
2940	goto end;
2941	}
2942
2943	err = uverbs_copy_from(&obj->match_criteria_enable,
2944	attrs,
2945	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
2946	if (err)
2947	goto end;
2948
2949	err = mlx5_ib_matcher_ns(attrs, obj);
2950	if (err)
2951	goto end;
2952
2953	if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2954	mlx5_eswitch_mode(dev: dev->mdev) != MLX5_ESWITCH_OFFLOADS) {
2955	err = -EINVAL;
2956	goto end;
2957	}
2958
2959	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) {
2960	err = uverbs_copy_from(&obj->ib_port, attrs,
2961	MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT);
2962	if (err)
2963	goto end;
2964	if (!rdma_is_port_valid(device: &dev->ib_dev, port: obj->ib_port)) {
2965	err = -EINVAL;
2966	goto end;
2967	}
2968	if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX &&
2969	obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) {
2970	err = -EINVAL;
2971	goto end;
2972	}
2973	if (!verify_context_caps(dev, enabled_caps: uobj->context->enabled_caps)) {
2974	err = -EOPNOTSUPP;
2975	goto end;
2976	}
2977	}
2978
2979	uobj->object = obj;
2980	obj->mdev = dev->mdev;
2981	atomic_set(v: &obj->usecnt, i: 0);
2982	return 0;
2983
2984	end:
2985	kfree(objp: obj);
2986	return err;
2987	}
2988
2989	static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
2990	struct uverbs_attr_bundle *attrs)
2991	{
2992	struct ib_uobject *uobj = uverbs_attr_get_uobject(
2993	attrs_bundle: attrs, idx: MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE);
2994	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
2995	enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type;
2996	enum mlx5_flow_namespace_type ns_type;
2997	struct mlx5_ib_steering_anchor *obj;
2998	struct mlx5_ib_flow_prio *ft_prio;
2999	u16 priority;
3000	u32 ft_id;
3001	int err;
3002
3003	if (!rdma_dev_has_raw_cap(dev: &dev->ib_dev))
3004	return -EPERM;
3005
3006	err = uverbs_get_const(&ib_uapi_ft_type, attrs,
3007	MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE);
3008	if (err)
3009	return err;
3010
3011	err = mlx5_ib_ft_type_to_namespace(table_type: ib_uapi_ft_type, namespace: &ns_type);
3012	if (err)
3013	return err;
3014
3015	err = uverbs_copy_from(&priority, attrs,
3016	MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY);
3017	if (err)
3018	return err;
3019
3020	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3021	if (!obj)
3022	return -ENOMEM;
3023
3024	mutex_lock(&dev->flow_db->lock);
3025
3026	ft_prio = _get_flow_table(dev, user_priority: priority, ns_type, mcast: 0, ib_port: 0);
3027	if (IS_ERR(ptr: ft_prio)) {
3028	err = PTR_ERR(ptr: ft_prio);
3029	goto free_obj;
3030	}
3031
3032	ft_prio->refcount++;
3033
3034	if (!ft_prio->anchor.rule_goto_table_ref) {
3035	err = steering_anchor_create_res(dev, ft_prio, ns_type);
3036	if (err)
3037	goto put_flow_table;
3038	}
3039
3040	ft_prio->anchor.rule_goto_table_ref++;
3041
3042	ft_id = mlx5_flow_table_id(ft: ft_prio->anchor.ft);
3043
3044	err = uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3045	from: &ft_id, size: sizeof(ft_id));
3046	if (err)
3047	goto destroy_res;
3048
3049	mutex_unlock(lock: &dev->flow_db->lock);
3050
3051	uobj->object = obj;
3052	obj->dev = dev;
3053	obj->ft_prio = ft_prio;
3054	atomic_set(v: &obj->usecnt, i: 0);
3055
3056	return 0;
3057
3058	destroy_res:
3059	--ft_prio->anchor.rule_goto_table_ref;
3060	mlx5_steering_anchor_destroy_res(ft_prio);
3061	put_flow_table:
3062	put_flow_table(dev, prio: ft_prio, ft_added: true);
3063	free_obj:
3064	mutex_unlock(lock: &dev->flow_db->lock);
3065	kfree(objp: obj);
3066
3067	return err;
3068	}
3069
3070	static struct ib_flow_action *
3071	mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
3072	enum mlx5_ib_uapi_flow_table_type ft_type,
3073	u8 num_actions, void *in)
3074	{
3075	enum mlx5_flow_namespace_type namespace;
3076	struct mlx5_ib_flow_action *maction;
3077	int ret;
3078
3079	ret = mlx5_ib_ft_type_to_namespace(table_type: ft_type, namespace: &namespace);
3080	if (ret)
3081	return ERR_PTR(error: -EINVAL);
3082
3083	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3084	if (!maction)
3085	return ERR_PTR(error: -ENOMEM);
3086
3087	maction->flow_action_raw.modify_hdr =
3088	mlx5_modify_header_alloc(dev: dev->mdev, ns_type: namespace, num_actions, modify_actions: in);
3089
3090	if (IS_ERR(ptr: maction->flow_action_raw.modify_hdr)) {
3091	ret = PTR_ERR(ptr: maction->flow_action_raw.modify_hdr);
3092	kfree(objp: maction);
3093	return ERR_PTR(error: ret);
3094	}
3095	maction->flow_action_raw.sub_type =
3096	MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
3097	maction->flow_action_raw.dev = dev;
3098
3099	return &maction->ib_action;
3100	}
3101
3102	static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
3103	{
3104	return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3105	max_modify_header_actions) ||
3106	MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3107	max_modify_header_actions) ||
3108	MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
3109	max_modify_header_actions);
3110	}
3111
3112	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
3113	struct uverbs_attr_bundle *attrs)
3114	{
3115	struct ib_uobject *uobj = uverbs_attr_get_uobject(
3116	attrs_bundle: attrs, idx: MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
3117	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
3118	enum mlx5_ib_uapi_flow_table_type ft_type;
3119	struct ib_flow_action *action;
3120	int num_actions;
3121	void *in;
3122	int ret;
3123
3124	if (!mlx5_ib_modify_header_supported(dev: mdev))
3125	return -EOPNOTSUPP;
3126
3127	in = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs,
3128	idx: MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
3129
3130	num_actions = uverbs_attr_ptr_get_array_size(
3131	attrs, idx: MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3132	MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
3133	if (num_actions < 0)
3134	return num_actions;
3135
3136	ret = uverbs_get_const(&ft_type, attrs,
3137	MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
3138	if (ret)
3139	return ret;
3140	action = mlx5_ib_create_modify_header(dev: mdev, ft_type, num_actions, in);
3141	if (IS_ERR(ptr: action))
3142	return PTR_ERR(ptr: action);
3143
3144	uverbs_flow_action_fill_action(action, uobj, ib_dev: &mdev->ib_dev,
3145	type: IB_FLOW_ACTION_UNSPECIFIED);
3146
3147	return 0;
3148	}
3149
3150	static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
3151	u8 packet_reformat_type,
3152	u8 ft_type)
3153	{
3154	switch (packet_reformat_type) {
3155	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3156	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3157	return MLX5_CAP_FLOWTABLE(ibdev->mdev,
3158	encap_general_header);
3159	break;
3160	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3161	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3162	return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
3163	reformat_l2_to_l3_tunnel);
3164	break;
3165	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3166	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3167	return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
3168	reformat_l3_tunnel_to_l2);
3169	break;
3170	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
3171	if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3172	return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
3173	break;
3174	default:
3175	break;
3176	}
3177
3178	return false;
3179	}
3180
3181	static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
3182	{
3183	switch (dv_prt) {
3184	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3185	*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
3186	break;
3187	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3188	*prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
3189	break;
3190	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3191	*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3192	break;
3193	default:
3194	return -EINVAL;
3195	}
3196
3197	return 0;
3198	}
3199
3200	static int mlx5_ib_flow_action_create_packet_reformat_ctx(
3201	struct mlx5_ib_dev *dev,
3202	struct mlx5_ib_flow_action *maction,
3203	u8 ft_type, u8 dv_prt,
3204	void *in, size_t len)
3205	{
3206	struct mlx5_pkt_reformat_params reformat_params;
3207	enum mlx5_flow_namespace_type namespace;
3208	u8 prm_prt;
3209	int ret;
3210
3211	ret = mlx5_ib_ft_type_to_namespace(table_type: ft_type, namespace: &namespace);
3212	if (ret)
3213	return ret;
3214
3215	ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, prm_prt: &prm_prt);
3216	if (ret)
3217	return ret;
3218
3219	memset(&reformat_params, 0, sizeof(reformat_params));
3220	reformat_params.type = prm_prt;
3221	reformat_params.size = len;
3222	reformat_params.data = in;
3223	maction->flow_action_raw.pkt_reformat =
3224	mlx5_packet_reformat_alloc(dev: dev->mdev, params: &reformat_params,
3225	ns_type: namespace);
3226	if (IS_ERR(ptr: maction->flow_action_raw.pkt_reformat)) {
3227	ret = PTR_ERR(ptr: maction->flow_action_raw.pkt_reformat);
3228	return ret;
3229	}
3230
3231	maction->flow_action_raw.sub_type =
3232	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
3233	maction->flow_action_raw.dev = dev;
3234
3235	return 0;
3236	}
3237
3238	static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
3239	struct uverbs_attr_bundle *attrs)
3240	{
3241	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs_bundle: attrs,
3242	idx: MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
3243	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
3244	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
3245	enum mlx5_ib_uapi_flow_table_type ft_type;
3246	struct mlx5_ib_flow_action *maction;
3247	int ret;
3248
3249	ret = uverbs_get_const(&ft_type, attrs,
3250	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
3251	if (ret)
3252	return ret;
3253
3254	ret = uverbs_get_const(&dv_prt, attrs,
3255	MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
3256	if (ret)
3257	return ret;
3258
3259	if (!mlx5_ib_flow_action_packet_reformat_valid(ibdev: mdev, packet_reformat_type: dv_prt, ft_type))
3260	return -EOPNOTSUPP;
3261
3262	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3263	if (!maction)
3264	return -ENOMEM;
3265
3266	if (dv_prt ==
3267	MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
3268	maction->flow_action_raw.sub_type =
3269	MLX5_IB_FLOW_ACTION_DECAP;
3270	maction->flow_action_raw.dev = mdev;
3271	} else {
3272	void *in;
3273	int len;
3274
3275	in = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs,
3276	idx: MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3277	if (IS_ERR(ptr: in)) {
3278	ret = PTR_ERR(ptr: in);
3279	goto free_maction;
3280	}
3281
3282	len = uverbs_attr_get_len(attrs_bundle: attrs,
3283	idx: MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3284
3285	ret = mlx5_ib_flow_action_create_packet_reformat_ctx(dev: mdev,
3286	maction, ft_type, dv_prt, in, len);
3287	if (ret)
3288	goto free_maction;
3289	}
3290
3291	uverbs_flow_action_fill_action(action: &maction->ib_action, uobj, ib_dev: &mdev->ib_dev,
3292	type: IB_FLOW_ACTION_UNSPECIFIED);
3293	return 0;
3294
3295	free_maction:
3296	kfree(objp: maction);
3297	return ret;
3298	}
3299
3300	DECLARE_UVERBS_NAMED_METHOD(
3301	MLX5_IB_METHOD_CREATE_FLOW,
3302	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3303	UVERBS_OBJECT_FLOW,
3304	UVERBS_ACCESS_NEW,
3305	UA_MANDATORY),
3306	UVERBS_ATTR_PTR_IN(
3307	MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
3308	UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3309	UA_MANDATORY,
3310	UA_ALLOC_AND_COPY),
3311	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
3312	MLX5_IB_OBJECT_FLOW_MATCHER,
3313	UVERBS_ACCESS_READ,
3314	UA_MANDATORY),
3315	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
3316	UVERBS_OBJECT_QP,
3317	UVERBS_ACCESS_READ),
3318	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
3319	MLX5_IB_OBJECT_DEVX_OBJ,
3320	UVERBS_ACCESS_READ),
3321	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
3322	UVERBS_OBJECT_FLOW_ACTION,
3323	UVERBS_ACCESS_READ, 1,
3324	MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
3325	UA_OPTIONAL),
3326	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
3327	UVERBS_ATTR_TYPE(u32),
3328	UA_OPTIONAL),
3329	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
3330	MLX5_IB_OBJECT_DEVX_OBJ,
3331	UVERBS_ACCESS_READ, 1, 1,
3332	UA_OPTIONAL),
3333	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
3334	UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
3335	UA_OPTIONAL,
3336	UA_ALLOC_AND_COPY),
3337	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
3338	enum mlx5_ib_create_flow_flags,
3339	UA_OPTIONAL));
3340
3341	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3342	MLX5_IB_METHOD_DESTROY_FLOW,
3343	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3344	UVERBS_OBJECT_FLOW,
3345	UVERBS_ACCESS_DESTROY,
3346	UA_MANDATORY));
3347
3348	ADD_UVERBS_METHODS(mlx5_ib_fs,
3349	UVERBS_OBJECT_FLOW,
3350	&UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
3351	&UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
3352
3353	DECLARE_UVERBS_NAMED_METHOD(
3354	MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
3355	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
3356	UVERBS_OBJECT_FLOW_ACTION,
3357	UVERBS_ACCESS_NEW,
3358	UA_MANDATORY),
3359	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3360	UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
3361	set_add_copy_action_in_auto)),
3362	UA_MANDATORY,
3363	UA_ALLOC_AND_COPY),
3364	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
3365	enum mlx5_ib_uapi_flow_table_type,
3366	UA_MANDATORY));
3367
3368	DECLARE_UVERBS_NAMED_METHOD(
3369	MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
3370	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
3371	UVERBS_OBJECT_FLOW_ACTION,
3372	UVERBS_ACCESS_NEW,
3373	UA_MANDATORY),
3374	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
3375	UVERBS_ATTR_MIN_SIZE(1),
3376	UA_ALLOC_AND_COPY,
3377	UA_OPTIONAL),
3378	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
3379	enum mlx5_ib_uapi_flow_action_packet_reformat_type,
3380	UA_MANDATORY),
3381	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
3382	enum mlx5_ib_uapi_flow_table_type,
3383	UA_MANDATORY));
3384
3385	ADD_UVERBS_METHODS(
3386	mlx5_ib_flow_actions,
3387	UVERBS_OBJECT_FLOW_ACTION,
3388	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
3389	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
3390
3391	DECLARE_UVERBS_NAMED_METHOD(
3392	MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
3393	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
3394	MLX5_IB_OBJECT_FLOW_MATCHER,
3395	UVERBS_ACCESS_NEW,
3396	UA_MANDATORY),
3397	UVERBS_ATTR_PTR_IN(
3398	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
3399	UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3400	UA_MANDATORY),
3401	UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
3402	mlx5_ib_flow_type,
3403	UA_MANDATORY),
3404	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
3405	UVERBS_ATTR_TYPE(u8),
3406	UA_MANDATORY),
3407	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
3408	enum ib_flow_flags,
3409	UA_OPTIONAL),
3410	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
3411	enum mlx5_ib_uapi_flow_table_type,
3412	UA_OPTIONAL),
3413	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT,
3414	UVERBS_ATTR_TYPE(u32),
3415	UA_OPTIONAL));
3416
3417	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3418	MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
3419	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
3420	MLX5_IB_OBJECT_FLOW_MATCHER,
3421	UVERBS_ACCESS_DESTROY,
3422	UA_MANDATORY));
3423
3424	DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
3425	UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
3426	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
3427	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
3428
3429	DECLARE_UVERBS_NAMED_METHOD(
3430	MLX5_IB_METHOD_STEERING_ANCHOR_CREATE,
3431	UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE,
3432	MLX5_IB_OBJECT_STEERING_ANCHOR,
3433	UVERBS_ACCESS_NEW,
3434	UA_MANDATORY),
3435	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE,
3436	enum mlx5_ib_uapi_flow_table_type,
3437	UA_MANDATORY),
3438	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY,
3439	UVERBS_ATTR_TYPE(u16),
3440	UA_MANDATORY),
3441	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3442	UVERBS_ATTR_TYPE(u32),
3443	UA_MANDATORY));
3444
3445	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3446	MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY,
3447	UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE,
3448	MLX5_IB_OBJECT_STEERING_ANCHOR,
3449	UVERBS_ACCESS_DESTROY,
3450	UA_MANDATORY));
3451
3452	DECLARE_UVERBS_NAMED_OBJECT(
3453	MLX5_IB_OBJECT_STEERING_ANCHOR,
3454	UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup),
3455	&UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE),
3456	&UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY));
3457
3458	const struct uapi_definition mlx5_ib_flow_defs[] = {
3459	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3460	MLX5_IB_OBJECT_FLOW_MATCHER),
3461	UAPI_DEF_CHAIN_OBJ_TREE(
3462	UVERBS_OBJECT_FLOW,
3463	&mlx5_ib_fs),
3464	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3465	&mlx5_ib_flow_actions),
3466	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3467	MLX5_IB_OBJECT_STEERING_ANCHOR,
3468	UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)),
3469	{},
3470	};
3471
3472	static const struct ib_device_ops flow_ops = {
3473	.create_flow = mlx5_ib_create_flow,
3474	.destroy_flow = mlx5_ib_destroy_flow,
3475	.destroy_flow_action = mlx5_ib_destroy_flow_action,
3476	};
3477
3478	int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
3479	{
3480	int i, j;
3481
3482	dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
3483
3484	if (!dev->flow_db)
3485	return -ENOMEM;
3486
3487	for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) {
3488	dev->flow_db->rdma_transport_rx[i] =
3489	kcalloc(dev->num_ports,
3490	sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3491	if (!dev->flow_db->rdma_transport_rx[i])
3492	goto free_rdma_transport_rx;
3493	}
3494
3495	for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) {
3496	dev->flow_db->rdma_transport_tx[j] =
3497	kcalloc(dev->num_ports,
3498	sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3499	if (!dev->flow_db->rdma_transport_tx[j])
3500	goto free_rdma_transport_tx;
3501	}
3502
3503	mutex_init(&dev->flow_db->lock);
3504
3505	ib_set_device_ops(device: &dev->ib_dev, ops: &flow_ops);
3506	return 0;
3507
3508	free_rdma_transport_tx:
3509	while (j--)
3510	kfree(objp: dev->flow_db->rdma_transport_tx[j]);
3511	free_rdma_transport_rx:
3512	while (i--)
3513	kfree(objp: dev->flow_db->rdma_transport_rx[i]);
3514	kfree(objp: dev->flow_db);
3515	return -ENOMEM;
3516	}
3517