1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2019 Mellanox Technologies. */
3
4#include "en/params.h"
5#include "en/txrx.h"
6#include "en/port.h"
7#include "en_accel/en_accel.h"
8#include "en_accel/ipsec.h"
9#include "en_accel/psp.h"
10#include <linux/dim.h>
11#include <net/page_pool/types.h>
12#include <net/xdp_sock_drv.h>
13
14#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18
15#define MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ 17
16
17static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev)
18{
19 u8 min_page_shift = MLX5_CAP_GEN_2(mdev, log_min_mkey_entity_size);
20
21 return min_page_shift ? : 12;
22}
23
24u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
25{
26 u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT;
27 u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev);
28
29 /* Regular RQ uses order-0 pages, the NIC must be able to map them. */
30 if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift))
31 min_page_shift = req_page_shift;
32
33 return max(req_page_shift, min_page_shift);
34}
35
36enum mlx5e_mpwrq_umr_mode
37mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
38{
39 /* Different memory management schemes use different mechanisms to map
40 * user-mode memory. The stricter guarantees we have, the faster
41 * mechanisms we use:
42 * 1. MTT - direct mapping in page granularity.
43 * 2. KSM - indirect mapping to another MKey to arbitrary addresses, but
44 * all mappings have the same size.
45 * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and
46 * mappings can have different sizes.
47 */
48 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
49 bool unaligned = xsk ? xsk->unaligned : false;
50 bool oversized = false;
51
52 if (xsk) {
53 oversized = xsk->chunk_size < (1 << page_shift);
54 WARN_ON_ONCE(xsk->chunk_size > (1 << page_shift));
55 }
56
57 /* XSK frame size doesn't match the UMR page size, either because the
58 * frame size is not a power of two, or it's smaller than the minimal
59 * page size supported by the firmware.
60 * It's possible to receive packets bigger than MTU in certain setups.
61 * To avoid writing over the XSK frame boundary, the top region of each
62 * stride is mapped to a garbage page, resulting in two mappings of
63 * different sizes per frame.
64 */
65 if (oversized) {
66 /* An optimization for frame sizes equal to 3 * power_of_two.
67 * 3 KSMs point to the frame, and one KSM points to the garbage
68 * page, which works faster than KLM.
69 */
70 if (xsk->chunk_size % 3 == 0 && is_power_of_2(n: xsk->chunk_size / 3))
71 return MLX5E_MPWRQ_UMR_MODE_TRIPLE;
72
73 return MLX5E_MPWRQ_UMR_MODE_OVERSIZED;
74 }
75
76 /* XSK frames can start at arbitrary unaligned locations, but they all
77 * have the same size which is a power of two. It allows to optimize to
78 * one KSM per frame.
79 */
80 if (unaligned)
81 return MLX5E_MPWRQ_UMR_MODE_UNALIGNED;
82
83 /* XSK: frames are naturally aligned, MTT can be used.
84 * Non-XSK: Allocations happen in units of CPU pages, therefore, the
85 * mappings are naturally aligned.
86 */
87 return MLX5E_MPWRQ_UMR_MODE_ALIGNED;
88}
89
90u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode)
91{
92 switch (mode) {
93 case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
94 return sizeof(struct mlx5_mtt);
95 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
96 return sizeof(struct mlx5_ksm);
97 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
98 return sizeof(struct mlx5_klm) * 2;
99 case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
100 return sizeof(struct mlx5_ksm) * 4;
101 }
102 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", mode);
103 return 1;
104}
105
106u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
107 enum mlx5e_mpwrq_umr_mode umr_mode)
108{
109 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(mode: umr_mode);
110 u8 max_pages_per_wqe, max_log_wqe_size_calc;
111 u8 max_log_wqe_size_cap;
112 u16 max_wqe_size;
113
114 /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */
115 max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB;
116 max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe),
117 MLX5_UMR_FLEX_ALIGNMENT) / umr_entry_size;
118 max_log_wqe_size_calc = ilog2(max_pages_per_wqe) + page_shift;
119
120 WARN_ON_ONCE(max_log_wqe_size_calc < MLX5E_ORDER2_MAX_PACKET_MTU);
121
122 max_log_wqe_size_cap = mlx5_core_is_ecpf(dev: mdev) ?
123 MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ : MLX5_MPWRQ_MAX_LOG_WQE_SZ;
124
125 return min_t(u8, max_log_wqe_size_calc, max_log_wqe_size_cap);
126}
127
128u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
129 enum mlx5e_mpwrq_umr_mode umr_mode)
130{
131 u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
132 u8 pages_per_wqe;
133
134 pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1;
135
136 /* Two MTTs are needed to form an octword. The number of MTTs is encoded
137 * in octwords in a UMR WQE, so we need at least two to avoid mapping
138 * garbage addresses.
139 */
140 if (WARN_ON_ONCE(pages_per_wqe < 2 && umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))
141 pages_per_wqe = 2;
142
143 /* Sanity check for further calculations to succeed. */
144 BUILD_BUG_ON(MLX5_MPWRQ_MAX_PAGES_PER_WQE > 64);
145 if (WARN_ON_ONCE(pages_per_wqe > MLX5_MPWRQ_MAX_PAGES_PER_WQE))
146 return MLX5_MPWRQ_MAX_PAGES_PER_WQE;
147
148 return pages_per_wqe;
149}
150
151u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
152 enum mlx5e_mpwrq_umr_mode umr_mode)
153{
154 u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);
155 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(mode: umr_mode);
156 u16 umr_wqe_sz;
157
158 umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) +
159 ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT);
160
161 WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK);
162
163 return umr_wqe_sz;
164}
165
166u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift,
167 enum mlx5e_mpwrq_umr_mode umr_mode)
168{
169 return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, umr_mode),
170 MLX5_SEND_WQE_BB);
171}
172
173u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
174 enum mlx5e_mpwrq_umr_mode umr_mode)
175{
176 u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);
177
178 /* Add another page as a buffer between WQEs. This page will absorb
179 * write overflow by the hardware, when receiving packets larger than
180 * MTU. These oversize packets are dropped by the driver at a later
181 * stage.
182 */
183 return ALIGN(pages_per_wqe + 1,
184 MLX5_SEND_WQE_BB / mlx5e_mpwrq_umr_entry_size(umr_mode));
185}
186
187u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev,
188 enum mlx5e_mpwrq_umr_mode umr_mode)
189{
190 /* Same limits apply to KSMs and KLMs. */
191 u32 klm_limit = min(MLX5E_MAX_RQ_NUM_KSMS,
192 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size));
193
194 switch (umr_mode) {
195 case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
196 return MLX5E_MAX_RQ_NUM_MTTS;
197 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
198 return klm_limit;
199 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
200 /* Each entry is two KLMs. */
201 return klm_limit / 2;
202 case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
203 /* Each entry is four KSMs. */
204 return klm_limit / 4;
205 }
206 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
207 return 0;
208}
209
210static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift,
211 enum mlx5e_mpwrq_umr_mode umr_mode)
212{
213 u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, umr_mode);
214 u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, umr_mode);
215
216 return ilog2(max_entries / mtts_per_wqe);
217}
218
219u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
220 enum mlx5e_mpwrq_umr_mode umr_mode)
221{
222 return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode) +
223 mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
224 MLX5E_ORDER2_MAX_PACKET_MTU;
225}
226
227u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
228 struct mlx5e_xsk_param *xsk)
229{
230 u16 headroom;
231
232 if (xsk)
233 return xsk->headroom;
234
235 headroom = NET_IP_ALIGN;
236 if (params->xdp_prog)
237 headroom += XDP_PACKET_HEADROOM;
238 else
239 headroom += MLX5_RX_HEADROOM;
240
241 return headroom;
242}
243
244static u32 mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params *params,
245 struct mlx5e_xsk_param *xsk)
246{
247 u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
248
249 return xsk->headroom + hw_mtu;
250}
251
252static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool no_head_tail_room)
253{
254 u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
255 u16 headroom;
256
257 if (no_head_tail_room)
258 return SKB_DATA_ALIGN(hw_mtu);
259 headroom = mlx5e_get_linear_rq_headroom(params, NULL);
260
261 return MLX5_SKB_FRAG_SZ(headroom + hw_mtu);
262}
263
264static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
265 struct mlx5e_params *params,
266 struct mlx5e_xsk_param *xsk,
267 bool mpwqe)
268{
269 bool no_head_tail_room;
270 u32 sz;
271
272 /* XSK frames are mapped as individual pages, because frames may come in
273 * an arbitrary order from random locations in the UMEM.
274 */
275 if (xsk)
276 return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE;
277
278 no_head_tail_room = params->xdp_prog && mpwqe && !mlx5e_rx_is_linear_skb(mdev, params, xsk);
279
280 /* When no_head_tail_room is set, headroom and tailroom are excluded from skb calculations.
281 * no_head_tail_room should be set in the case of XDP with Striding RQ
282 * when SKB is not linear. This is because another page is allocated for the linear part.
283 */
284 sz = roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, no_head_tail_room));
285
286 /* XDP in mlx5e doesn't support multiple packets per page.
287 * Do not assume sz <= PAGE_SIZE if params->xdp_prog is set.
288 */
289 return params->xdp_prog && sz < PAGE_SIZE ? PAGE_SIZE : sz;
290}
291
292static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
293 struct mlx5e_params *params,
294 struct mlx5e_xsk_param *xsk)
295{
296 u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, mpwqe: true);
297 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
298 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
299
300 return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
301 order_base_2(linear_stride_sz);
302}
303
304bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
305 struct mlx5e_params *params,
306 struct mlx5e_xsk_param *xsk)
307{
308 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE)
309 return false;
310
311 /* Call mlx5e_rx_get_linear_sz_skb with the no_head_tail_room parameter set
312 * to exclude headroom and tailroom from calculations.
313 * no_head_tail_room is true when SKB is built on XDP_PASS on XSK RQs
314 * since packet data buffers don't have headroom and tailroom resreved for the SKB.
315 * Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data
316 * must fit into a CPU page.
317 */
318 if (mlx5e_rx_get_linear_sz_skb(params, no_head_tail_room: xsk) > PAGE_SIZE)
319 return false;
320
321 /* XSK frames must be big enough to hold the packet data. */
322 if (xsk && mlx5e_rx_get_linear_sz_xsk(params, xsk) > xsk->chunk_size)
323 return false;
324
325 return true;
326}
327
328static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
329 u8 log_stride_sz, u8 log_num_strides,
330 u8 page_shift,
331 enum mlx5e_mpwrq_umr_mode umr_mode)
332{
333 if (log_stride_sz + log_num_strides !=
334 mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode))
335 return false;
336
337 if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE ||
338 log_stride_sz > MLX5_MPWQE_LOG_STRIDE_SZ_MAX)
339 return false;
340
341 if (log_num_strides > MLX5_MPWQE_LOG_NUM_STRIDES_MAX)
342 return false;
343
344 if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
345 return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_EXT_BASE;
346
347 return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
348}
349
350bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
351 struct mlx5e_params *params,
352 struct mlx5e_xsk_param *xsk)
353{
354 u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
355 u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
356 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
357 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
358
359 return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz: log_wqe_stride_size,
360 log_num_strides: log_wqe_num_of_strides,
361 page_shift, umr_mode);
362}
363
364bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
365 struct mlx5e_params *params,
366 struct mlx5e_xsk_param *xsk)
367{
368 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
369 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
370 u8 log_num_strides;
371 u8 log_stride_sz;
372 u8 log_wqe_sz;
373
374 if (!mlx5e_rx_is_linear_skb(mdev, params, xsk))
375 return false;
376
377 log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
378 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
379
380 if (log_wqe_sz < log_stride_sz)
381 return false;
382
383 log_num_strides = log_wqe_sz - log_stride_sz;
384
385 return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz,
386 log_num_strides, page_shift,
387 umr_mode);
388}
389
390u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
391 struct mlx5e_params *params,
392 struct mlx5e_xsk_param *xsk)
393{
394 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
395 u8 log_pkts_per_wqe, page_shift, max_log_rq_size;
396
397 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);
398 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
399 max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode);
400
401 /* Numbers are unsigned, don't subtract to avoid underflow. */
402 if (params->log_rq_mtu_frames <
403 log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
404 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
405
406 /* Ethtool's rx_max_pending is calculated for regular RQ, that uses
407 * pages of PAGE_SIZE. Max length of an XSK RQ might differ if it uses a
408 * frame size not equal to PAGE_SIZE.
409 * A stricter condition is checked in mlx5e_mpwrq_validate_xsk, WARN on
410 * unexpected failure.
411 */
412 if (WARN_ON_ONCE(params->log_rq_mtu_frames > log_pkts_per_wqe + max_log_rq_size))
413 return max_log_rq_size;
414
415 return params->log_rq_mtu_frames - log_pkts_per_wqe;
416}
417
418static u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5e_params *params)
419{
420 return order_base_2(DIV_ROUND_UP(MLX5E_SHAMPO_WQ_RESRV_SIZE,
421 params->sw_mtu));
422}
423
424u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
425 struct mlx5e_params *params,
426 struct mlx5e_xsk_param *xsk)
427{
428 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
429 return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
430
431 /* XDP in mlx5e doesn't support multiple packets per page. */
432 if (params->xdp_prog)
433 return PAGE_SHIFT;
434
435 return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
436}
437
438u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
439 struct mlx5e_params *params,
440 struct mlx5e_xsk_param *xsk)
441{
442 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
443 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
444 u8 log_wqe_size, log_stride_size;
445
446 log_wqe_size = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
447 log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
448 WARN(log_wqe_size < log_stride_size,
449 "Log WQE size %u < log stride size %u (page shift %u, umr mode %d, xsk on? %d)\n",
450 log_wqe_size, log_stride_size, page_shift, umr_mode, !!xsk);
451 return log_wqe_size - log_stride_size;
452}
453
454u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz)
455{
456#define UMR_WQE_BULK (2)
457 return min_t(unsigned int, UMR_WQE_BULK, wq_sz / 2 - 1);
458}
459
460u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
461 struct mlx5e_params *params,
462 struct mlx5e_xsk_param *xsk)
463{
464 u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
465
466 if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC)
467 return linear_headroom;
468
469 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
470 return linear_headroom;
471
472 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
473 return linear_headroom;
474
475 return 0;
476}
477
478u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
479{
480 bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
481 u16 stop_room;
482
483 stop_room = mlx5e_ktls_get_stop_room(mdev, params);
484 stop_room += mlx5e_stop_room_for_max_wqe(mdev);
485 if (is_mpwqe)
486 /* A MPWQE can take up to the maximum cacheline-aligned WQE +
487 * all the normal stop room can be taken if a new packet breaks
488 * the active MPWQE session and allocates its WQEs right away.
489 */
490 stop_room += mlx5e_stop_room_for_mpwqe(mdev);
491
492 return stop_room;
493}
494
495int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
496{
497 size_t sq_size = 1 << params->log_sq_size;
498 u16 stop_room;
499
500 stop_room = mlx5e_calc_sq_stop_room(mdev, params);
501 if (stop_room >= sq_size) {
502 mlx5_core_err(mdev, "Stop room %u is bigger than the SQ size %zu\n",
503 stop_room, sq_size);
504 return -EINVAL;
505 }
506
507 return 0;
508}
509
510bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
511{
512 u32 link_speed = 0;
513 u32 pci_bw = 0;
514
515 mlx5_port_max_linkspeed(mdev, speed: &link_speed);
516 pci_bw = pcie_bandwidth_available(dev: mdev->pdev, NULL, NULL, NULL);
517 mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
518 link_speed, pci_bw);
519
520#define MLX5E_SLOW_PCI_RATIO (2)
521
522 return link_speed && pci_bw &&
523 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
524}
525
526int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
527{
528 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, NULL);
529 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL);
530
531 if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))
532 return -EOPNOTSUPP;
533
534 return 0;
535}
536
537int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
538 struct mlx5e_xsk_param *xsk)
539{
540 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
541 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
542 u16 max_mtu_pkts;
543
544 if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) {
545 mlx5_core_err(mdev, "Striding RQ for XSK can't be activated with page_shift %u and umr_mode %d\n",
546 page_shift, umr_mode);
547 return -EOPNOTSUPP;
548 }
549
550 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) {
551 mlx5_core_err(mdev, "Striding RQ linear mode for XSK can't be activated with current params\n");
552 return -EINVAL;
553 }
554
555 /* Current RQ length is too big for the given frame size, the
556 * needed number of WQEs exceeds the maximum.
557 */
558 max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE,
559 mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, xsk->unaligned));
560 if (params->log_rq_mtu_frames > max_mtu_pkts) {
561 mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n",
562 1 << params->log_rq_mtu_frames, xsk->chunk_size);
563 return -EINVAL;
564 }
565
566 return 0;
567}
568
569void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
570 struct mlx5e_params *params)
571{
572 params->log_rq_mtu_frames = is_kdump_kernel() ?
573 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
574 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
575}
576
577void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
578{
579 params->rq_wq_type = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
580 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
581 MLX5_WQ_TYPE_CYCLIC;
582}
583
584void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
585 struct mlx5e_params *params)
586{
587 /* Prefer Striding RQ, unless any of the following holds:
588 * - Striding RQ configuration is not possible/supported.
589 * - CQE compression is ON, and stride_index mini_cqe layout is not supported.
590 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
591 *
592 * No XSK params: checking the availability of striding RQ in general.
593 */
594 if ((!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) ||
595 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) &&
596 !mlx5e_mpwrq_validate_regular(mdev, params) &&
597 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ||
598 !mlx5e_rx_is_linear_skb(mdev, params, NULL)))
599 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
600 mlx5e_set_rq_type(mdev, params);
601 mlx5e_init_rq_type_params(mdev, params);
602}
603
604/* Build queue parameters */
605
606void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c)
607{
608 *ccp = (struct mlx5e_create_cq_param) {
609 .netdev = c->netdev,
610 .wq = c->priv->wq,
611 .napi = &c->napi,
612 .ch_stats = c->stats,
613 .node = cpu_to_node(cpu: c->cpu),
614 .ix = c->vec_ix,
615 .uar = c->bfreg->up,
616 };
617}
618
619static int mlx5e_max_nonlinear_mtu(int first_frag_size, int frag_size, bool xdp)
620{
621 if (xdp)
622 /* XDP requires all fragments to be of the same size. */
623 return first_frag_size + (MLX5E_MAX_RX_FRAGS - 1) * frag_size;
624
625 /* Optimization for small packets: the last fragment is bigger than the others. */
626 return first_frag_size + (MLX5E_MAX_RX_FRAGS - 2) * frag_size + PAGE_SIZE;
627}
628
629static void mlx5e_rx_compute_wqe_bulk_params(struct mlx5e_params *params,
630 struct mlx5e_rq_frags_info *info)
631{
632 u16 bulk_bound_rq_size = (1 << params->log_rq_mtu_frames) / 4;
633 u32 bulk_bound_rq_size_in_bytes;
634 u32 sum_frag_strides = 0;
635 u32 wqe_bulk_in_bytes;
636 u16 split_factor;
637 u32 wqe_bulk;
638 int i;
639
640 for (i = 0; i < info->num_frags; i++)
641 sum_frag_strides += info->arr[i].frag_stride;
642
643 /* For MTUs larger than PAGE_SIZE, align to PAGE_SIZE to reflect
644 * amount of consumed pages per wqe in bytes.
645 */
646 if (sum_frag_strides > PAGE_SIZE)
647 sum_frag_strides = ALIGN(sum_frag_strides, PAGE_SIZE);
648
649 bulk_bound_rq_size_in_bytes = bulk_bound_rq_size * sum_frag_strides;
650
651#define MAX_WQE_BULK_BYTES(xdp) ((xdp ? 256 : 512) * 1024)
652
653 /* A WQE bulk should not exceed min(512KB, 1/4 of rq size). For XDP
654 * keep bulk size smaller to avoid filling the page_pool cache on
655 * every bulk refill.
656 */
657 wqe_bulk_in_bytes = min_t(u32, MAX_WQE_BULK_BYTES(params->xdp_prog),
658 bulk_bound_rq_size_in_bytes);
659 wqe_bulk = DIV_ROUND_UP(wqe_bulk_in_bytes, sum_frag_strides);
660
661 /* Make sure that allocations don't start when the page is still used
662 * by older WQEs.
663 */
664 info->wqe_bulk = max_t(u16, info->wqe_index_mask + 1, wqe_bulk);
665
666 split_factor = DIV_ROUND_UP(MAX_WQE_BULK_BYTES(params->xdp_prog),
667 PP_ALLOC_CACHE_REFILL * PAGE_SIZE);
668 info->refill_unit = DIV_ROUND_UP(info->wqe_bulk, split_factor);
669}
670
671#define DEFAULT_FRAG_SIZE (2048)
672
673static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
674 struct mlx5e_params *params,
675 struct mlx5e_xsk_param *xsk,
676 struct mlx5e_rq_frags_info *info,
677 u32 *xdp_frag_size)
678{
679 u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
680 int frag_size_max = DEFAULT_FRAG_SIZE;
681 int first_frag_size_max;
682 u32 buf_size = 0;
683 u16 headroom;
684 int max_mtu;
685 int i;
686
687 if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
688 int frag_stride;
689
690 frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, mpwqe: false);
691
692 info->arr[0].frag_size = byte_count;
693 info->arr[0].frag_stride = frag_stride;
694 info->num_frags = 1;
695
696 /* N WQEs share the same page, N = PAGE_SIZE / frag_stride. The
697 * first WQE in the page is responsible for allocation of this
698 * page, this WQE's index is k*N. If WQEs [k*N+1; k*N+N-1] are
699 * still not completed, the allocation must stop before k*N.
700 */
701 info->wqe_index_mask = (PAGE_SIZE / frag_stride) - 1;
702
703 goto out;
704 }
705
706 headroom = mlx5e_get_linear_rq_headroom(params, xsk);
707 first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom);
708
709 max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size: first_frag_size_max, frag_size: frag_size_max,
710 xdp: params->xdp_prog);
711 if (byte_count > max_mtu || params->xdp_prog) {
712 frag_size_max = PAGE_SIZE;
713 first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom);
714
715 max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size: first_frag_size_max, frag_size: frag_size_max,
716 xdp: params->xdp_prog);
717 if (byte_count > max_mtu) {
718 mlx5_core_err(mdev, "MTU %u is too big for non-linear legacy RQ (max %d)\n",
719 params->sw_mtu, max_mtu);
720 return -EINVAL;
721 }
722 }
723
724 i = 0;
725 while (buf_size < byte_count) {
726 int frag_size = byte_count - buf_size;
727
728 if (i == 0)
729 frag_size = min(frag_size, first_frag_size_max);
730 else if (i < MLX5E_MAX_RX_FRAGS - 1)
731 frag_size = min(frag_size, frag_size_max);
732
733 info->arr[i].frag_size = frag_size;
734 buf_size += frag_size;
735
736 if (params->xdp_prog) {
737 /* XDP multi buffer expects fragments of the same size. */
738 info->arr[i].frag_stride = frag_size_max;
739 } else {
740 if (i == 0) {
741 /* Ensure that headroom and tailroom are included. */
742 frag_size += headroom;
743 frag_size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
744 }
745 info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
746 }
747
748 i++;
749 }
750 info->num_frags = i;
751
752 /* The last fragment of WQE with index 2*N may share the page with the
753 * first fragment of WQE with index 2*N+1 in certain cases. If WQE 2*N+1
754 * is not completed yet, WQE 2*N must not be allocated, as it's
755 * responsible for allocating a new page.
756 */
757 if (frag_size_max == PAGE_SIZE) {
758 /* No WQE can start in the middle of a page. */
759 info->wqe_index_mask = 0;
760 } else {
761 /* PAGE_SIZEs starting from 8192 don't use 2K-sized fragments,
762 * because there would be more than MLX5E_MAX_RX_FRAGS of them.
763 */
764 WARN_ON(PAGE_SIZE != 2 * DEFAULT_FRAG_SIZE);
765
766 /* Odd number of fragments allows to pack the last fragment of
767 * the previous WQE and the first fragment of the next WQE into
768 * the same page.
769 * As long as DEFAULT_FRAG_SIZE is 2048, and MLX5E_MAX_RX_FRAGS
770 * is 4, the last fragment can be bigger than the rest only if
771 * it's the fourth one, so WQEs consisting of 3 fragments will
772 * always share a page.
773 * When a page is shared, WQE bulk size is 2, otherwise just 1.
774 */
775 info->wqe_index_mask = info->num_frags % 2;
776 }
777
778out:
779 /* Bulking optimization to skip allocation until a large enough number
780 * of WQEs can be allocated in a row. Bulking also influences how well
781 * deferred page release works.
782 */
783 mlx5e_rx_compute_wqe_bulk_params(params, info);
784
785 mlx5_core_dbg(mdev, "%s: wqe_bulk = %u, wqe_bulk_refill_unit = %u\n",
786 __func__, info->wqe_bulk, info->refill_unit);
787
788 info->log_num_frags = order_base_2(info->num_frags);
789
790 *xdp_frag_size = info->num_frags > 1 && params->xdp_prog ? PAGE_SIZE : 0;
791
792 return 0;
793}
794
795static u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
796{
797 int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
798
799 switch (wq_type) {
800 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
801 sz += sizeof(struct mlx5e_rx_wqe_ll);
802 break;
803 default: /* MLX5_WQ_TYPE_CYCLIC */
804 sz += sizeof(struct mlx5e_rx_wqe_cyc);
805 }
806
807 return order_base_2(sz);
808}
809
810static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
811 struct mlx5e_cq_param *param)
812{
813 void *cqc = param->cqc;
814
815 MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index);
816 if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
817 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
818}
819
820static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
821 struct mlx5e_params *params,
822 struct mlx5e_xsk_param *xsk)
823{
824 u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
825 u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
826 int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(params));
827 int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
828 int wqe_size = BIT(log_stride_sz) * num_strides;
829 int rsrv_size = MLX5E_SHAMPO_WQ_RESRV_SIZE;
830
831 /* +1 is for the case that the pkt_per_rsrv dont consume the reservation
832 * so we get a filler cqe for the rest of the reservation.
833 */
834 return order_base_2((wqe_size / rsrv_size) * wq_size * (pkt_per_rsrv + 1));
835}
836
837static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
838 struct mlx5e_params *params,
839 struct mlx5e_xsk_param *xsk,
840 struct mlx5e_cq_param *param)
841{
842 bool hw_stridx = false;
843 void *cqc = param->cqc;
844 u8 log_cq_size;
845
846 switch (params->rq_wq_type) {
847 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
848 hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
849 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
850 log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk);
851 else
852 log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) +
853 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
854 break;
855 default: /* MLX5_WQ_TYPE_CYCLIC */
856 log_cq_size = params->log_rq_mtu_frames;
857 }
858
859 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
860 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
861 MLX5_SET(cqc, cqc, mini_cqe_res_format, hw_stridx ?
862 MLX5_CQE_FORMAT_CSUM_STRIDX : MLX5_CQE_FORMAT_CSUM);
863 MLX5_SET(cqc, cqc, cqe_compression_layout,
864 MLX5_CAP_GEN(mdev, enhanced_cqe_compression) ?
865 MLX5_CQE_COMPRESS_LAYOUT_ENHANCED :
866 MLX5_CQE_COMPRESS_LAYOUT_BASIC);
867 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
868 }
869
870 mlx5e_build_common_cq_param(mdev, param);
871 param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
872}
873
874static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
875{
876 bool lro_en = params->packet_merge.type == MLX5E_PACKET_MERGE_LRO;
877 bool ro = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
878
879 return ro && lro_en ?
880 MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
881}
882
883int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
884 struct mlx5e_params *params,
885 struct mlx5e_xsk_param *xsk,
886 struct mlx5e_rq_param *param)
887{
888 void *rqc = param->rqc;
889 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
890 u32 lro_timeout;
891 int ndsegs = 1;
892 int err;
893
894 switch (params->rq_wq_type) {
895 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {
896 u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
897 u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
898 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
899 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
900
901 if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz: log_wqe_stride_size,
902 log_num_strides: log_wqe_num_of_strides,
903 page_shift, umr_mode)) {
904 mlx5_core_err(mdev,
905 "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
906 log_wqe_stride_size, log_wqe_num_of_strides,
907 umr_mode);
908 return -EINVAL;
909 }
910
911 MLX5_SET(wq, wq, log_wqe_num_of_strides,
912 log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
913 MLX5_SET(wq, wq, log_wqe_stride_size,
914 log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
915 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
916 if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO)
917 break;
918
919 MLX5_SET(wq, wq, shampo_enable, true);
920 MLX5_SET(wq, wq, log_reservation_size,
921 MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE -
922 MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT);
923 MLX5_SET(wq, wq,
924 log_max_num_of_packets_per_reservation,
925 mlx5e_shampo_get_log_pkt_per_rsrv(params));
926 MLX5_SET(wq, wq, log_headers_entry_size,
927 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE -
928 MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT);
929 lro_timeout =
930 mlx5e_choose_lro_timeout(mdev,
931 MLX5E_DEFAULT_SHAMPO_TIMEOUT);
932 MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout);
933 MLX5_SET(rqc, rqc, shampo_match_criteria_type,
934 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED);
935 MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity,
936 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE);
937 break;
938 }
939 default: /* MLX5_WQ_TYPE_CYCLIC */
940 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
941 err = mlx5e_build_rq_frags_info(mdev, params, xsk, info: &param->frags_info,
942 xdp_frag_size: &param->xdp_frag_size);
943 if (err)
944 return err;
945 ndsegs = param->frags_info.num_frags;
946 }
947
948 MLX5_SET(wq, wq, wq_type, params->rq_wq_type);
949 MLX5_SET(wq, wq, end_padding_mode, rq_end_pad_mode(mdev, params));
950 MLX5_SET(wq, wq, log_wq_stride,
951 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs));
952 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.hw_objs.pdn);
953 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable);
954 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en);
955
956 param->wq.buf_numa_node = dev_to_node(dev: mlx5_core_dma_dev(dev: mdev));
957 mlx5e_build_rx_cq_param(mdev, params, xsk, param: &param->cqp);
958
959 return 0;
960}
961
962void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
963 struct mlx5e_rq_param *param)
964{
965 void *rqc = param->rqc;
966 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
967
968 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
969 MLX5_SET(wq, wq, log_wq_stride,
970 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
971
972 param->wq.buf_numa_node = dev_to_node(dev: mlx5_core_dma_dev(dev: mdev));
973}
974
975void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev,
976 struct mlx5e_params *params,
977 struct mlx5e_cq_param *param)
978{
979 void *cqc = param->cqc;
980
981 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
982
983 mlx5e_build_common_cq_param(mdev, param);
984 param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
985}
986
987void mlx5e_build_sq_param_common(struct mlx5_core_dev *mdev,
988 struct mlx5e_sq_param *param)
989{
990 void *sqc = param->sqc;
991 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
992
993 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
994 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.hw_objs.pdn);
995
996 param->wq.buf_numa_node = dev_to_node(dev: mlx5_core_dma_dev(dev: mdev));
997}
998
999void mlx5e_build_sq_param(struct mlx5_core_dev *mdev,
1000 struct mlx5e_params *params,
1001 struct mlx5e_sq_param *param)
1002{
1003 void *sqc = param->sqc;
1004 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1005 bool allow_swp;
1006
1007 allow_swp = mlx5_geneve_tx_allowed(mdev) ||
1008 (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO) ||
1009 mlx5_is_psp_device(mdev);
1010 mlx5e_build_sq_param_common(mdev, param);
1011 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
1012 MLX5_SET(sqc, sqc, allow_swp, allow_swp);
1013 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
1014 param->stop_room = mlx5e_calc_sq_stop_room(mdev, params);
1015 mlx5e_build_tx_cq_param(mdev, params, param: &param->cqp);
1016}
1017
1018static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev,
1019 u8 log_wq_size,
1020 struct mlx5e_cq_param *param)
1021{
1022 void *cqc = param->cqc;
1023
1024 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1025
1026 mlx5e_build_common_cq_param(mdev, param);
1027
1028 param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1029}
1030
1031/* This function calculates the maximum number of headers entries that are needed
1032 * per WQE, the formula is based on the size of the reservations and the
1033 * restriction we have about max packets for reservation that is equal to max
1034 * headers per reservation.
1035 */
1036u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev,
1037 struct mlx5e_params *params,
1038 struct mlx5e_rq_param *rq_param)
1039{
1040 u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, NULL));
1041 u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL);
1042 int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(params));
1043 int wqe_size = BIT(log_stride_sz) * num_strides;
1044 int rsrv_size = MLX5E_SHAMPO_WQ_RESRV_SIZE;
1045 u32 hd_per_wqe;
1046
1047 /* Assumption: hd_per_wqe % 8 == 0. */
1048 hd_per_wqe = (wqe_size / rsrv_size) * pkt_per_rsrv;
1049 mlx5_core_dbg(mdev, "%s hd_per_wqe = %d rsrv_size = %d wqe_size = %d pkt_per_rsrv = %d\n",
1050 __func__, hd_per_wqe, rsrv_size, wqe_size, pkt_per_rsrv);
1051 return hd_per_wqe;
1052}
1053
1054/* This function calculates the maximum number of headers entries that are needed
1055 * for the WQ, this value is uesed to allocate the header buffer in HW, thus
1056 * must be a pow of 2.
1057 */
1058u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev,
1059 struct mlx5e_params *params,
1060 struct mlx5e_rq_param *rq_param)
1061{
1062 void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
1063 int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
1064 u32 hd_per_wqe, hd_per_wq;
1065
1066 hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param);
1067 hd_per_wq = roundup_pow_of_two(hd_per_wqe * wq_size);
1068 return hd_per_wq;
1069}
1070
1071static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev,
1072 struct mlx5e_params *params,
1073 struct mlx5e_rq_param *rq_param)
1074{
1075 int max_num_of_umr_per_wqe, max_hd_per_wqe, max_ksm_per_umr, rest;
1076 void *wqc = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
1077 int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
1078 u32 wqebbs;
1079
1080 max_ksm_per_umr = MLX5E_MAX_KSM_PER_WQE(mdev);
1081 max_hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param);
1082 max_num_of_umr_per_wqe = max_hd_per_wqe / max_ksm_per_umr;
1083 rest = max_hd_per_wqe % max_ksm_per_umr;
1084 wqebbs = MLX5E_KSM_UMR_WQEBBS(max_ksm_per_umr) * max_num_of_umr_per_wqe;
1085 if (rest)
1086 wqebbs += MLX5E_KSM_UMR_WQEBBS(rest);
1087 wqebbs *= wq_size;
1088 return wqebbs;
1089}
1090
1091#define MLX5E_LRO_TIMEOUT_ARR_SIZE 4
1092
1093u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
1094{
1095 int i;
1096
1097 /* The supported periods are organized in ascending order */
1098 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
1099 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
1100 break;
1101
1102 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
1103}
1104
1105static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev,
1106 struct mlx5e_params *params,
1107 struct mlx5e_xsk_param *xsk)
1108{
1109 enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
1110 u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
1111 u8 umr_wqebbs;
1112
1113 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
1114
1115 return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
1116}
1117
1118static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
1119 struct mlx5e_params *params,
1120 struct mlx5e_rq_param *rqp)
1121{
1122 u32 wqebbs, total_pages, useful_space;
1123
1124 /* MLX5_WQ_TYPE_CYCLIC */
1125 if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
1126 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1127
1128 /* UMR WQEs for the regular RQ. */
1129 wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
1130
1131 /* If XDP program is attached, XSK may be turned on at any time without
1132 * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of
1133 * both regular RQ and XSK RQ.
1134 *
1135 * XSK uses different values of page_shift, and the total number of UMR
1136 * WQEBBs depends on it. This dependency is complex and not monotonic,
1137 * especially taking into consideration that some of the parameters come
1138 * from capabilities. Hence, we have to try all valid values of XSK
1139 * frame size (and page_shift) to find the maximum.
1140 */
1141 if (params->xdp_prog) {
1142 u32 max_xsk_wqebbs = 0;
1143 u8 frame_shift;
1144
1145 for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
1146 frame_shift <= PAGE_SHIFT; frame_shift++) {
1147 /* The headroom doesn't affect the calculation. */
1148 struct mlx5e_xsk_param xsk = {
1149 .chunk_size = 1 << frame_shift,
1150 .unaligned = false,
1151 };
1152
1153 /* XSK aligned mode. */
1154 max_xsk_wqebbs = max(max_xsk_wqebbs,
1155 mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1156
1157 /* XSK unaligned mode, frame size is a power of two. */
1158 xsk.unaligned = true;
1159 max_xsk_wqebbs = max(max_xsk_wqebbs,
1160 mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1161
1162 /* XSK unaligned mode, frame size is not equal to stride size. */
1163 xsk.chunk_size -= 1;
1164 max_xsk_wqebbs = max(max_xsk_wqebbs,
1165 mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1166
1167 /* XSK unaligned mode, frame size is a triple power of two. */
1168 xsk.chunk_size = (1 << frame_shift) / 4 * 3;
1169 max_xsk_wqebbs = max(max_xsk_wqebbs,
1170 mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
1171 }
1172
1173 wqebbs += max_xsk_wqebbs;
1174 }
1175
1176 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
1177 wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rq_param: rqp);
1178
1179 /* UMR WQEs don't cross the page boundary, they are padded with NOPs.
1180 * This padding is always smaller than the max WQE size. That gives us
1181 * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes
1182 * per page. The number of pages is estimated as the total size of WQEs
1183 * divided by the useful space in page, rounding up. If some WQEs don't
1184 * fully fit into the useful space, they can occupy part of the padding,
1185 * which proves this estimation to be correct (reserve enough space).
1186 */
1187 useful_space = PAGE_SIZE - mlx5e_get_max_sq_wqebbs(mdev) + MLX5_SEND_WQE_BB;
1188 total_pages = DIV_ROUND_UP(wqebbs * MLX5_SEND_WQE_BB, useful_space);
1189 wqebbs = total_pages * (PAGE_SIZE / MLX5_SEND_WQE_BB);
1190
1191 return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE, order_base_2(wqebbs));
1192}
1193
1194static u8 mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev *mdev)
1195{
1196 if (mlx5e_is_ktls_rx(mdev))
1197 return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
1198
1199 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1200}
1201
1202static void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev,
1203 u8 log_wq_size,
1204 struct mlx5e_sq_param *param)
1205{
1206 void *sqc = param->sqc;
1207 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1208
1209 mlx5e_build_sq_param_common(mdev, param);
1210
1211 MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1212 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq));
1213 mlx5e_build_ico_cq_param(mdev, log_wq_size, param: &param->cqp);
1214}
1215
1216static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev,
1217 u8 log_wq_size,
1218 struct mlx5e_sq_param *param)
1219{
1220 void *sqc = param->sqc;
1221 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1222
1223 mlx5e_build_sq_param_common(mdev, param);
1224 param->stop_room = mlx5e_stop_room_for_wqe(mdev, wqe_size: 1); /* for XSK NOP */
1225 param->is_tls = mlx5e_is_ktls_rx(mdev);
1226 if (param->is_tls)
1227 param->stop_room += mlx5e_stop_room_for_wqe(mdev, wqe_size: 1); /* for TLS RX resync NOP */
1228 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq));
1229 MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1230 mlx5e_build_ico_cq_param(mdev, log_wq_size, param: &param->cqp);
1231}
1232
1233void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
1234 struct mlx5e_params *params,
1235 struct mlx5e_sq_param *param)
1236{
1237 void *sqc = param->sqc;
1238 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1239
1240 mlx5e_build_sq_param_common(mdev, param);
1241 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
1242 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
1243 mlx5e_build_tx_cq_param(mdev, params, param: &param->cqp);
1244}
1245
1246int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
1247 struct mlx5e_params *params,
1248 struct mlx5e_channel_param *cparam)
1249{
1250 u8 icosq_log_wq_sz, async_icosq_log_wq_sz;
1251 int err;
1252
1253 err = mlx5e_build_rq_param(mdev, params, NULL, param: &cparam->rq);
1254 if (err)
1255 return err;
1256
1257 icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(mdev, params, rqp: &cparam->rq);
1258 async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(mdev);
1259
1260 mlx5e_build_sq_param(mdev, params, param: &cparam->txq_sq);
1261 mlx5e_build_xdpsq_param(mdev, params, param: &cparam->xdp_sq);
1262 mlx5e_build_icosq_param(mdev, log_wq_size: icosq_log_wq_sz, param: &cparam->icosq);
1263 mlx5e_build_async_icosq_param(mdev, log_wq_size: async_icosq_log_wq_sz, param: &cparam->async_icosq);
1264
1265 return 0;
1266}
1267

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en/params.c