1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3
4#ifndef _IONIC_IBDEV_H_
5#define _IONIC_IBDEV_H_
6
7#include <rdma/ib_umem.h>
8#include <rdma/ib_verbs.h>
9#include <rdma/ib_pack.h>
10#include <rdma/uverbs_ioctl.h>
11
12#include <rdma/ionic-abi.h>
13#include <ionic_api.h>
14#include <ionic_regs.h>
15
16#include "ionic_fw.h"
17#include "ionic_queue.h"
18#include "ionic_res.h"
19
20#include "ionic_lif_cfg.h"
21
22/* Config knobs */
23#define IONIC_EQ_DEPTH 511
24#define IONIC_EQ_COUNT 32
25#define IONIC_AQ_DEPTH 63
26#define IONIC_AQ_COUNT 4
27#define IONIC_EQ_ISR_BUDGET 10
28#define IONIC_EQ_WORK_BUDGET 1000
29#define IONIC_MAX_RD_ATOM 16
30#define IONIC_PKEY_TBL_LEN 1
31#define IONIC_GID_TBL_LEN 256
32
33#define IONIC_MAX_QPID 0xffffff
34#define IONIC_SPEC_HIGH 8
35#define IONIC_MAX_PD 1024
36#define IONIC_SPEC_HIGH 8
37#define IONIC_SQCMB_ORDER 5
38#define IONIC_RQCMB_ORDER 0
39
40#define IONIC_META_LAST ((void *)1ul)
41#define IONIC_META_POSTED ((void *)2ul)
42
43#define IONIC_CQ_GRACE 100
44
45#define IONIC_ROCE_UDP_SPORT 28272
46#define IONIC_DMA_LKEY 0
47#define IONIC_DMA_RKEY IONIC_DMA_LKEY
48
49#define IONIC_CMB_SUPPORTED \
50 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
51 IONIC_CMB_WC | IONIC_CMB_UC)
52
53/* resource is not reserved on the device, indicated in tbl_order */
54#define IONIC_RES_INVALID -1
55
56struct ionic_aq;
57struct ionic_cq;
58struct ionic_eq;
59struct ionic_vcq;
60
61enum ionic_admin_state {
62 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
63 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
64 IONIC_ADMIN_KILLED, /* not submitting, locally completed */
65};
66
67enum ionic_admin_flags {
68 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */
69 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */
70 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */
71};
72
73enum ionic_mmap_flag {
74 IONIC_MMAP_WC = BIT(0),
75};
76
77struct ionic_mmap_entry {
78 struct rdma_user_mmap_entry rdma_entry;
79 unsigned long size;
80 unsigned long pfn;
81 u8 mmap_flags;
82};
83
84struct ionic_ibdev {
85 struct ib_device ibdev;
86
87 struct ionic_lif_cfg lif_cfg;
88
89 struct xarray qp_tbl;
90 struct xarray cq_tbl;
91
92 struct ionic_resid_bits inuse_dbid;
93 struct ionic_resid_bits inuse_pdid;
94 struct ionic_resid_bits inuse_ahid;
95 struct ionic_resid_bits inuse_mrid;
96 struct ionic_resid_bits inuse_qpid;
97 struct ionic_resid_bits inuse_cqid;
98
99 u8 half_cqid_udma_shift;
100 u8 half_qpid_udma_shift;
101 u8 next_qpid_udma_idx;
102 u8 next_mrkey;
103
104 struct work_struct reset_work;
105 bool reset_posted;
106 u32 reset_cnt;
107
108 struct delayed_work admin_dwork;
109 struct ionic_aq **aq_vec;
110 atomic_t admin_state;
111
112 struct ionic_eq **eq_vec;
113
114 struct ionic_v1_stat *hw_stats;
115 void *hw_stats_buf;
116 struct rdma_stat_desc *hw_stats_hdrs;
117 struct ionic_counter_stats *counter_stats;
118 int hw_stats_count;
119};
120
121struct ionic_eq {
122 struct ionic_ibdev *dev;
123
124 u32 eqid;
125 u32 intr;
126
127 struct ionic_queue q;
128
129 int armed;
130 bool enable;
131
132 struct work_struct work;
133
134 int irq;
135 char name[32];
136};
137
138struct ionic_admin_wr {
139 struct completion work;
140 struct list_head aq_ent;
141 struct ionic_v1_admin_wqe wqe;
142 struct ionic_v1_cqe cqe;
143 struct ionic_aq *aq;
144 int status;
145};
146
147struct ionic_admin_wr_q {
148 struct ionic_admin_wr *wr;
149 int wqe_strides;
150};
151
152struct ionic_aq {
153 struct ionic_ibdev *dev;
154 struct ionic_vcq *vcq;
155
156 struct work_struct work;
157
158 atomic_t admin_state;
159 unsigned long stamp;
160 bool armed;
161
162 u32 aqid;
163 u32 cqid;
164
165 spinlock_t lock; /* for posting */
166 struct ionic_queue q;
167 struct ionic_admin_wr_q *q_wr;
168 struct list_head wr_prod;
169 struct list_head wr_post;
170};
171
172struct ionic_ctx {
173 struct ib_ucontext ibctx;
174 u32 dbid;
175 struct rdma_user_mmap_entry *mmap_dbell;
176};
177
178struct ionic_tbl_buf {
179 u32 tbl_limit;
180 u32 tbl_pages;
181 size_t tbl_size;
182 __le64 *tbl_buf;
183 dma_addr_t tbl_dma;
184 u8 page_size_log2;
185};
186
187struct ionic_pd {
188 struct ib_pd ibpd;
189
190 u32 pdid;
191 u32 flags;
192};
193
194struct ionic_cq {
195 struct ionic_vcq *vcq;
196
197 u32 cqid;
198 u32 eqid;
199
200 spinlock_t lock; /* for polling */
201 struct list_head poll_sq;
202 bool flush;
203 struct list_head flush_sq;
204 struct list_head flush_rq;
205 struct list_head ibkill_flush_ent;
206
207 struct ionic_queue q;
208 bool color;
209 int credit;
210 u16 arm_any_prod;
211 u16 arm_sol_prod;
212
213 struct kref cq_kref;
214 struct completion cq_rel_comp;
215
216 /* infrequently accessed, keep at end */
217 struct ib_umem *umem;
218};
219
220struct ionic_vcq {
221 struct ib_cq ibcq;
222 struct ionic_cq cq[2];
223 u8 udma_mask;
224 u8 poll_idx;
225};
226
227struct ionic_sq_meta {
228 u64 wrid;
229 u32 len;
230 u16 seq;
231 u8 ibop;
232 u8 ibsts;
233 u8 remote:1;
234 u8 signal:1;
235 u8 local_comp:1;
236};
237
238struct ionic_rq_meta {
239 struct ionic_rq_meta *next;
240 u64 wrid;
241};
242
243struct ionic_qp {
244 struct ib_qp ibqp;
245 enum ib_qp_state state;
246
247 u32 qpid;
248 u32 ahid;
249 u32 sq_cqid;
250 u32 rq_cqid;
251 u8 udma_idx;
252 u8 has_ah:1;
253 u8 has_sq:1;
254 u8 has_rq:1;
255 u8 sig_all:1;
256
257 struct list_head qp_list_counter;
258
259 struct list_head cq_poll_sq;
260 struct list_head cq_flush_sq;
261 struct list_head cq_flush_rq;
262 struct list_head ibkill_flush_ent;
263
264 spinlock_t sq_lock; /* for posting and polling */
265 struct ionic_queue sq;
266 struct ionic_sq_meta *sq_meta;
267 u16 *sq_msn_idx;
268 int sq_spec;
269 u16 sq_old_prod;
270 u16 sq_msn_prod;
271 u16 sq_msn_cons;
272 u8 sq_cmb;
273 bool sq_flush;
274 bool sq_flush_rcvd;
275
276 spinlock_t rq_lock; /* for posting and polling */
277 struct ionic_queue rq;
278 struct ionic_rq_meta *rq_meta;
279 struct ionic_rq_meta *rq_meta_head;
280 int rq_spec;
281 u16 rq_old_prod;
282 u8 rq_cmb;
283 bool rq_flush;
284
285 struct kref qp_kref;
286 struct completion qp_rel_comp;
287
288 /* infrequently accessed, keep at end */
289 int sgid_index;
290 int sq_cmb_order;
291 u32 sq_cmb_pgid;
292 phys_addr_t sq_cmb_addr;
293 struct rdma_user_mmap_entry *mmap_sq_cmb;
294
295 struct ib_umem *sq_umem;
296
297 int rq_cmb_order;
298 u32 rq_cmb_pgid;
299 phys_addr_t rq_cmb_addr;
300 struct rdma_user_mmap_entry *mmap_rq_cmb;
301
302 struct ib_umem *rq_umem;
303
304 int dcqcn_profile;
305
306 struct ib_ud_header *hdr;
307};
308
309struct ionic_ah {
310 struct ib_ah ibah;
311 u32 ahid;
312 int sgid_index;
313 struct ib_ud_header hdr;
314};
315
316struct ionic_mr {
317 union {
318 struct ib_mr ibmr;
319 struct ib_mw ibmw;
320 };
321
322 u32 mrid;
323 int flags;
324
325 struct ib_umem *umem;
326 struct ionic_tbl_buf buf;
327 bool created;
328};
329
330struct ionic_counter_stats {
331 int queue_stats_count;
332 struct ionic_v1_stat *hdr;
333 struct rdma_stat_desc *stats_hdrs;
334 struct xarray xa_counters;
335};
336
337struct ionic_counter {
338 void *vals;
339 struct list_head qp_list;
340};
341
342static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
343{
344 return container_of(ibdev, struct ionic_ibdev, ibdev);
345}
346
347static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)
348{
349 return container_of(ibctx, struct ionic_ctx, ibctx);
350}
351
352static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)
353{
354 if (!uobj)
355 return NULL;
356
357 if (!uobj->context)
358 return NULL;
359
360 return to_ionic_ctx(ibctx: uobj->context);
361}
362
363static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)
364{
365 return container_of(ibpd, struct ionic_pd, ibpd);
366}
367
368static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)
369{
370 return container_of(ibmr, struct ionic_mr, ibmr);
371}
372
373static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)
374{
375 return container_of(ibmw, struct ionic_mr, ibmw);
376}
377
378static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)
379{
380 return container_of(ibcq, struct ionic_vcq, ibcq);
381}
382
383static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,
384 uint8_t udma_idx)
385{
386 return &to_ionic_vcq(ibcq)->cq[udma_idx];
387}
388
389static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
390{
391 return container_of(ibqp, struct ionic_qp, ibqp);
392}
393
394static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
395{
396 return container_of(ibah, struct ionic_ah, ibah);
397}
398
399static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,
400 struct ionic_ctx *ctx)
401{
402 if (!ctx)
403 return dev->lif_cfg.dbid;
404
405 return ctx->dbid;
406}
407
408static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,
409 struct ib_uobject *uobj)
410{
411 return ionic_ctx_dbid(dev, ctx: to_ionic_ctx_uobj(uobj));
412}
413
414static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)
415{
416 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;
417}
418
419static inline void ionic_qp_complete(struct kref *kref)
420{
421 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);
422
423 complete(&qp->qp_rel_comp);
424}
425
426static inline void ionic_cq_complete(struct kref *kref)
427{
428 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);
429
430 complete(&cq->cq_rel_comp);
431}
432
433/* ionic_admin.c */
434extern struct workqueue_struct *ionic_evt_workq;
435void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
436int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
437 enum ionic_admin_flags);
438
439int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);
440
441int ionic_create_rdma_admin(struct ionic_ibdev *dev);
442void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);
443void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);
444
445/* ionic_controlpath.c */
446int ionic_create_cq_common(struct ionic_vcq *vcq,
447 struct ionic_tbl_buf *buf,
448 const struct ib_cq_init_attr *attr,
449 struct ionic_ctx *ctx,
450 struct ib_udata *udata,
451 struct ionic_qdesc *req_cq,
452 __u32 *resp_cqid,
453 int udma_idx);
454void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
455void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
456void ionic_notify_flush_cq(struct ionic_cq *cq);
457
458int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);
459void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);
460int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);
461void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
462int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
463int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
464int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
465 struct ib_udata *udata);
466int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
467int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);
468struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);
469struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
470 u64 addr, int access, struct ib_dmah *dmah,
471 struct ib_udata *udata);
472struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
473 u64 length, u64 addr, int fd, int access,
474 struct ib_dmah *dmah,
475 struct uverbs_attr_bundle *attrs);
476int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
477struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
478 u32 max_sg);
479int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
480 unsigned int *sg_offset);
481int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
482int ionic_dealloc_mw(struct ib_mw *ibmw);
483int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
484 struct uverbs_attr_bundle *attrs);
485int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
486int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
487 struct ib_udata *udata);
488int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
489 struct ib_udata *udata);
490int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
491 struct ib_qp_init_attr *init_attr);
492int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
493
494/* ionic_datapath.c */
495int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
496 const struct ib_send_wr **bad);
497int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
498 const struct ib_recv_wr **bad);
499int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);
500int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
501
502/* ionic_hw_stats.c */
503void ionic_stats_init(struct ionic_ibdev *dev);
504void ionic_stats_cleanup(struct ionic_ibdev *dev);
505
506/* ionic_pgtbl.c */
507__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);
508__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);
509int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);
510int ionic_pgtbl_init(struct ionic_ibdev *dev,
511 struct ionic_tbl_buf *buf,
512 struct ib_umem *umem,
513 dma_addr_t dma,
514 int limit,
515 u64 page_size);
516void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);
517#endif /* _IONIC_IBDEV_H_ */
518

source code of linux/drivers/infiniband/hw/ionic/ionic_ibdev.h