1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2/* Copyright (c) 2021, Microsoft Corporation. */
3
4#ifndef _GDMA_H
5#define _GDMA_H
6
7#include <linux/dma-mapping.h>
8#include <linux/netdevice.h>
9
10#include "shm_channel.h"
11
12#define GDMA_STATUS_MORE_ENTRIES 0x00000105
13#define GDMA_STATUS_CMD_UNSUPPORTED 0xffffffff
14
15/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
16 * them are naturally aligned and hence don't need __packed.
17 */
18
19enum gdma_request_type {
20 GDMA_VERIFY_VF_DRIVER_VERSION = 1,
21 GDMA_QUERY_MAX_RESOURCES = 2,
22 GDMA_LIST_DEVICES = 3,
23 GDMA_REGISTER_DEVICE = 4,
24 GDMA_DEREGISTER_DEVICE = 5,
25 GDMA_GENERATE_TEST_EQE = 10,
26 GDMA_CREATE_QUEUE = 12,
27 GDMA_DISABLE_QUEUE = 13,
28 GDMA_ALLOCATE_RESOURCE_RANGE = 22,
29 GDMA_DESTROY_RESOURCE_RANGE = 24,
30 GDMA_CREATE_DMA_REGION = 25,
31 GDMA_DMA_REGION_ADD_PAGES = 26,
32 GDMA_DESTROY_DMA_REGION = 27,
33 GDMA_CREATE_PD = 29,
34 GDMA_DESTROY_PD = 30,
35 GDMA_CREATE_MR = 31,
36 GDMA_DESTROY_MR = 32,
37 GDMA_QUERY_HWC_TIMEOUT = 84, /* 0x54 */
38};
39
40#define GDMA_RESOURCE_DOORBELL_PAGE 27
41
42enum gdma_queue_type {
43 GDMA_INVALID_QUEUE,
44 GDMA_SQ,
45 GDMA_RQ,
46 GDMA_CQ,
47 GDMA_EQ,
48};
49
50enum gdma_work_request_flags {
51 GDMA_WR_NONE = 0,
52 GDMA_WR_OOB_IN_SGL = BIT(0),
53 GDMA_WR_PAD_BY_SGE0 = BIT(1),
54};
55
56enum gdma_eqe_type {
57 GDMA_EQE_COMPLETION = 3,
58 GDMA_EQE_TEST_EVENT = 64,
59 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
60 GDMA_EQE_HWC_INIT_DATA = 130,
61 GDMA_EQE_HWC_INIT_DONE = 131,
62 GDMA_EQE_HWC_FPGA_RECONFIG = 132,
63 GDMA_EQE_HWC_SOC_RECONFIG_DATA = 133,
64 GDMA_EQE_HWC_SOC_SERVICE = 134,
65 GDMA_EQE_HWC_RESET_REQUEST = 135,
66 GDMA_EQE_RNIC_QP_FATAL = 176,
67};
68
69enum {
70 GDMA_DEVICE_NONE = 0,
71 GDMA_DEVICE_HWC = 1,
72 GDMA_DEVICE_MANA = 2,
73 GDMA_DEVICE_MANA_IB = 3,
74};
75
76enum gdma_service_type {
77 GDMA_SERVICE_TYPE_NONE = 0,
78 GDMA_SERVICE_TYPE_RDMA_SUSPEND = 1,
79 GDMA_SERVICE_TYPE_RDMA_RESUME = 2,
80};
81
82struct mana_service_work {
83 struct work_struct work;
84 struct gdma_dev *gdma_dev;
85 enum gdma_service_type event;
86};
87
88struct gdma_resource {
89 /* Protect the bitmap */
90 spinlock_t lock;
91
92 /* The bitmap size in bits. */
93 u32 size;
94
95 /* The bitmap tracks the resources. */
96 unsigned long *map;
97};
98
99union gdma_doorbell_entry {
100 u64 as_uint64;
101
102 struct {
103 u64 id : 24;
104 u64 reserved : 8;
105 u64 tail_ptr : 31;
106 u64 arm : 1;
107 } cq;
108
109 struct {
110 u64 id : 24;
111 u64 wqe_cnt : 8;
112 u64 tail_ptr : 32;
113 } rq;
114
115 struct {
116 u64 id : 24;
117 u64 reserved : 8;
118 u64 tail_ptr : 32;
119 } sq;
120
121 struct {
122 u64 id : 16;
123 u64 reserved : 16;
124 u64 tail_ptr : 31;
125 u64 arm : 1;
126 } eq;
127}; /* HW DATA */
128
129struct gdma_msg_hdr {
130 u32 hdr_type;
131 u32 msg_type;
132 u16 msg_version;
133 u16 hwc_msg_id;
134 u32 msg_size;
135}; /* HW DATA */
136
137struct gdma_dev_id {
138 union {
139 struct {
140 u16 type;
141 u16 instance;
142 };
143
144 u32 as_uint32;
145 };
146}; /* HW DATA */
147
148struct gdma_req_hdr {
149 struct gdma_msg_hdr req;
150 struct gdma_msg_hdr resp; /* The expected response */
151 struct gdma_dev_id dev_id;
152 u32 activity_id;
153}; /* HW DATA */
154
155struct gdma_resp_hdr {
156 struct gdma_msg_hdr response;
157 struct gdma_dev_id dev_id;
158 u32 activity_id;
159 u32 status;
160 u32 reserved;
161}; /* HW DATA */
162
163struct gdma_general_req {
164 struct gdma_req_hdr hdr;
165}; /* HW DATA */
166
167#define GDMA_MESSAGE_V1 1
168#define GDMA_MESSAGE_V2 2
169#define GDMA_MESSAGE_V3 3
170#define GDMA_MESSAGE_V4 4
171
172struct gdma_general_resp {
173 struct gdma_resp_hdr hdr;
174}; /* HW DATA */
175
176#define GDMA_STANDARD_HEADER_TYPE 0
177
178static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
179 u32 req_size, u32 resp_size)
180{
181 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
182 hdr->req.msg_type = code;
183 hdr->req.msg_version = GDMA_MESSAGE_V1;
184 hdr->req.msg_size = req_size;
185
186 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
187 hdr->resp.msg_type = code;
188 hdr->resp.msg_version = GDMA_MESSAGE_V1;
189 hdr->resp.msg_size = resp_size;
190}
191
192/* The 16-byte struct is part of the GDMA work queue entry (WQE). */
193struct gdma_sge {
194 u64 address;
195 u32 mem_key;
196 u32 size;
197}; /* HW DATA */
198
199struct gdma_wqe_request {
200 struct gdma_sge *sgl;
201 u32 num_sge;
202
203 u32 inline_oob_size;
204 const void *inline_oob_data;
205
206 u32 flags;
207 u32 client_data_unit;
208};
209
210enum gdma_page_type {
211 GDMA_PAGE_TYPE_4K,
212};
213
214#define GDMA_INVALID_DMA_REGION 0
215
216struct gdma_mem_info {
217 struct device *dev;
218
219 dma_addr_t dma_handle;
220 void *virt_addr;
221 u64 length;
222
223 /* Allocated by the PF driver */
224 u64 dma_region_handle;
225};
226
227#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
228
229struct gdma_dev {
230 struct gdma_context *gdma_context;
231
232 struct gdma_dev_id dev_id;
233
234 u32 pdid;
235 u32 doorbell;
236 u32 gpa_mkey;
237
238 /* GDMA driver specific pointer */
239 void *driver_data;
240
241 struct auxiliary_device *adev;
242 bool is_suspended;
243 bool rdma_teardown;
244};
245
246/* MANA_PAGE_SIZE is the DMA unit */
247#define MANA_PAGE_SHIFT 12
248#define MANA_PAGE_SIZE BIT(MANA_PAGE_SHIFT)
249#define MANA_PAGE_ALIGN(x) ALIGN((x), MANA_PAGE_SIZE)
250#define MANA_PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), MANA_PAGE_SIZE)
251#define MANA_PFN(a) ((a) >> MANA_PAGE_SHIFT)
252
253/* Required by HW */
254#define MANA_MIN_QSIZE MANA_PAGE_SIZE
255
256#define GDMA_CQE_SIZE 64
257#define GDMA_EQE_SIZE 16
258#define GDMA_MAX_SQE_SIZE 512
259#define GDMA_MAX_RQE_SIZE 256
260
261#define GDMA_COMP_DATA_SIZE 0x3C
262
263#define GDMA_EVENT_DATA_SIZE 0xC
264
265/* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
266#define GDMA_WQE_BU_SIZE 32
267
268#define INVALID_PDID UINT_MAX
269#define INVALID_DOORBELL UINT_MAX
270#define INVALID_MEM_KEY UINT_MAX
271#define INVALID_QUEUE_ID UINT_MAX
272#define INVALID_PCI_MSIX_INDEX UINT_MAX
273
274struct gdma_comp {
275 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
276 u32 wq_num;
277 bool is_sq;
278};
279
280struct gdma_event {
281 u32 details[GDMA_EVENT_DATA_SIZE / 4];
282 u8 type;
283};
284
285struct gdma_queue;
286
287struct mana_eq {
288 struct gdma_queue *eq;
289 struct dentry *mana_eq_debugfs;
290};
291
292typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
293 struct gdma_event *e);
294
295typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
296
297/* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
298 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
299 * driver increases the 'head' in BUs rather than in bytes, and notifies
300 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
301 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
302 *
303 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
304 * processed, the driver increases the 'tail' to indicate that WQEs have
305 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
306 *
307 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
308 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
309 * the owner bits mechanism to detect if the queue has become empty.
310 */
311struct gdma_queue {
312 struct gdma_dev *gdma_dev;
313
314 enum gdma_queue_type type;
315 u32 id;
316
317 struct gdma_mem_info mem_info;
318
319 void *queue_mem_ptr;
320 u32 queue_size;
321
322 bool monitor_avl_buf;
323
324 u32 head;
325 u32 tail;
326 struct list_head entry;
327
328 /* Extra fields specific to EQ/CQ. */
329 union {
330 struct {
331 bool disable_needed;
332
333 gdma_eq_callback *callback;
334 void *context;
335
336 unsigned int msix_index;
337
338 u32 log2_throttle_limit;
339 } eq;
340
341 struct {
342 gdma_cq_callback *callback;
343 void *context;
344
345 struct gdma_queue *parent; /* For CQ/EQ relationship */
346 } cq;
347 };
348};
349
350struct gdma_queue_spec {
351 enum gdma_queue_type type;
352 bool monitor_avl_buf;
353 unsigned int queue_size;
354
355 /* Extra fields specific to EQ/CQ. */
356 union {
357 struct {
358 gdma_eq_callback *callback;
359 void *context;
360
361 unsigned long log2_throttle_limit;
362 unsigned int msix_index;
363 } eq;
364
365 struct {
366 gdma_cq_callback *callback;
367 void *context;
368
369 struct gdma_queue *parent_eq;
370
371 } cq;
372 };
373};
374
375#define MANA_IRQ_NAME_SZ 32
376
377struct gdma_irq_context {
378 void (*handler)(void *arg);
379 /* Protect the eq_list */
380 spinlock_t lock;
381 struct list_head eq_list;
382 char name[MANA_IRQ_NAME_SZ];
383};
384
385enum gdma_context_flags {
386 GC_PROBE_SUCCEEDED = 0,
387};
388
389struct gdma_context {
390 struct device *dev;
391 struct dentry *mana_pci_debugfs;
392
393 /* Per-vPort max number of queues */
394 unsigned int max_num_queues;
395 unsigned int max_num_msix;
396 unsigned int num_msix_usable;
397 struct xarray irq_contexts;
398
399 /* L2 MTU */
400 u16 adapter_mtu;
401
402 /* This maps a CQ index to the queue structure. */
403 unsigned int max_num_cqs;
404 struct gdma_queue **cq_table;
405
406 /* Protect eq_test_event and test_event_eq_id */
407 struct mutex eq_test_event_mutex;
408 struct completion eq_test_event;
409 u32 test_event_eq_id;
410
411 bool is_pf;
412 bool in_service;
413
414 phys_addr_t bar0_pa;
415 void __iomem *bar0_va;
416 void __iomem *shm_base;
417 void __iomem *db_page_base;
418 phys_addr_t phys_db_page_base;
419 u32 db_page_size;
420 int numa_node;
421
422 /* Shared memory chanenl (used to bootstrap HWC) */
423 struct shm_channel shm_channel;
424
425 /* Hardware communication channel (HWC) */
426 struct gdma_dev hwc;
427
428 /* Azure network adapter */
429 struct gdma_dev mana;
430
431 /* Azure RDMA adapter */
432 struct gdma_dev mana_ib;
433
434 u64 pf_cap_flags1;
435
436 struct workqueue_struct *service_wq;
437
438 unsigned long flags;
439};
440
441static inline bool mana_gd_is_mana(struct gdma_dev *gd)
442{
443 return gd->dev_id.type == GDMA_DEVICE_MANA;
444}
445
446static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
447{
448 return gd->dev_id.type == GDMA_DEVICE_HWC;
449}
450
451u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
452u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
453
454int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
455
456int mana_gd_create_hwc_queue(struct gdma_dev *gd,
457 const struct gdma_queue_spec *spec,
458 struct gdma_queue **queue_ptr);
459
460int mana_gd_create_mana_eq(struct gdma_dev *gd,
461 const struct gdma_queue_spec *spec,
462 struct gdma_queue **queue_ptr);
463
464int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
465 const struct gdma_queue_spec *spec,
466 struct gdma_queue **queue_ptr);
467
468void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
469
470int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
471
472void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
473
474struct gdma_wqe {
475 u32 reserved :24;
476 u32 last_vbytes :8;
477
478 union {
479 u32 flags;
480
481 struct {
482 u32 num_sge :8;
483 u32 inline_oob_size_div4:3;
484 u32 client_oob_in_sgl :1;
485 u32 reserved1 :4;
486 u32 client_data_unit :14;
487 u32 reserved2 :2;
488 };
489 };
490}; /* HW DATA */
491
492#define INLINE_OOB_SMALL_SIZE 8
493#define INLINE_OOB_LARGE_SIZE 24
494
495#define MANA_MAX_TX_WQE_SGL_ENTRIES 30
496
497#define MAX_TX_WQE_SIZE 512
498#define MAX_RX_WQE_SIZE 256
499
500#define MAX_TX_WQE_SGL_ENTRIES ((GDMA_MAX_SQE_SIZE - \
501 sizeof(struct gdma_sge) - INLINE_OOB_SMALL_SIZE) / \
502 sizeof(struct gdma_sge))
503
504#define MAX_RX_WQE_SGL_ENTRIES ((GDMA_MAX_RQE_SIZE - \
505 sizeof(struct gdma_sge)) / sizeof(struct gdma_sge))
506
507struct gdma_cqe {
508 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
509
510 union {
511 u32 as_uint32;
512
513 struct {
514 u32 wq_num : 24;
515 u32 is_sq : 1;
516 u32 reserved : 4;
517 u32 owner_bits : 3;
518 };
519 } cqe_info;
520}; /* HW DATA */
521
522#define GDMA_CQE_OWNER_BITS 3
523
524#define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
525
526#define SET_ARM_BIT 1
527
528#define GDMA_EQE_OWNER_BITS 3
529
530union gdma_eqe_info {
531 u32 as_uint32;
532
533 struct {
534 u32 type : 8;
535 u32 reserved1 : 8;
536 u32 client_id : 2;
537 u32 reserved2 : 11;
538 u32 owner_bits : 3;
539 };
540}; /* HW DATA */
541
542#define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
543#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
544
545struct gdma_eqe {
546 u32 details[GDMA_EVENT_DATA_SIZE / 4];
547 u32 eqe_info;
548}; /* HW DATA */
549
550#define GDMA_REG_DB_PAGE_OFFSET 8
551#define GDMA_REG_DB_PAGE_SIZE 0x10
552#define GDMA_REG_SHM_OFFSET 0x18
553
554#define GDMA_PF_REG_DB_PAGE_SIZE 0xD0
555#define GDMA_PF_REG_DB_PAGE_OFF 0xC8
556#define GDMA_PF_REG_SHM_OFF 0x70
557
558#define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108
559
560#define MANA_PF_DEVICE_ID 0x00B9
561#define MANA_VF_DEVICE_ID 0x00BA
562
563struct gdma_posted_wqe_info {
564 u32 wqe_size_in_bu;
565};
566
567/* GDMA_GENERATE_TEST_EQE */
568struct gdma_generate_test_event_req {
569 struct gdma_req_hdr hdr;
570 u32 queue_index;
571}; /* HW DATA */
572
573/* GDMA_VERIFY_VF_DRIVER_VERSION */
574enum {
575 GDMA_PROTOCOL_V1 = 1,
576 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
577 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
578};
579
580#define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
581
582/* Advertise to the NIC firmware: the NAPI work_done variable race is fixed,
583 * so the driver is able to reliably support features like busy_poll.
584 */
585#define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2)
586#define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3)
587#define GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB BIT(4)
588#define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5)
589
590/* Driver can handle holes (zeros) in the device list */
591#define GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP BIT(11)
592
593/* Driver supports dynamic MSI-X vector allocation */
594#define GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT BIT(13)
595
596/* Driver can self reset on EQE notification */
597#define GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE BIT(14)
598
599/* Driver can self reset on FPGA Reconfig EQE notification */
600#define GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE BIT(17)
601#define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6)
602
603/* Driver supports linearizing the skb when num_sge exceeds hardware limit */
604#define GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE BIT(20)
605
606/* Driver can send HWC periodically to query stats */
607#define GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY BIT(21)
608
609/* Driver can handle hardware recovery events during probe */
610#define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22)
611
612#define GDMA_DRV_CAP_FLAGS1 \
613 (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
614 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
615 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | \
616 GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT | \
617 GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP | \
618 GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT | \
619 GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE | \
620 GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE | \
621 GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE | \
622 GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \
623 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \
624 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY)
625
626#define GDMA_DRV_CAP_FLAGS2 0
627
628#define GDMA_DRV_CAP_FLAGS3 0
629
630#define GDMA_DRV_CAP_FLAGS4 0
631
632struct gdma_verify_ver_req {
633 struct gdma_req_hdr hdr;
634
635 /* Mandatory fields required for protocol establishment */
636 u64 protocol_ver_min;
637 u64 protocol_ver_max;
638
639 /* Gdma Driver Capability Flags */
640 u64 gd_drv_cap_flags1;
641 u64 gd_drv_cap_flags2;
642 u64 gd_drv_cap_flags3;
643 u64 gd_drv_cap_flags4;
644
645 /* Advisory fields */
646 u64 drv_ver;
647 u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
648 u32 reserved;
649 u32 os_ver_major;
650 u32 os_ver_minor;
651 u32 os_ver_build;
652 u32 os_ver_platform;
653 u64 reserved_2;
654 u8 os_ver_str1[128];
655 u8 os_ver_str2[128];
656 u8 os_ver_str3[128];
657 u8 os_ver_str4[128];
658}; /* HW DATA */
659
660struct gdma_verify_ver_resp {
661 struct gdma_resp_hdr hdr;
662 u64 gdma_protocol_ver;
663 u64 pf_cap_flags1;
664 u64 pf_cap_flags2;
665 u64 pf_cap_flags3;
666 u64 pf_cap_flags4;
667}; /* HW DATA */
668
669/* GDMA_QUERY_MAX_RESOURCES */
670struct gdma_query_max_resources_resp {
671 struct gdma_resp_hdr hdr;
672 u32 status;
673 u32 max_sq;
674 u32 max_rq;
675 u32 max_cq;
676 u32 max_eq;
677 u32 max_db;
678 u32 max_mst;
679 u32 max_cq_mod_ctx;
680 u32 max_mod_cq;
681 u32 max_msix;
682}; /* HW DATA */
683
684/* GDMA_LIST_DEVICES */
685#define GDMA_DEV_LIST_SIZE 64
686struct gdma_list_devices_resp {
687 struct gdma_resp_hdr hdr;
688 u32 num_of_devs;
689 u32 reserved;
690 struct gdma_dev_id devs[GDMA_DEV_LIST_SIZE];
691}; /* HW DATA */
692
693/* GDMA_REGISTER_DEVICE */
694struct gdma_register_device_resp {
695 struct gdma_resp_hdr hdr;
696 u32 pdid;
697 u32 gpa_mkey;
698 u32 db_id;
699}; /* HW DATA */
700
701struct gdma_allocate_resource_range_req {
702 struct gdma_req_hdr hdr;
703 u32 resource_type;
704 u32 num_resources;
705 u32 alignment;
706 u32 allocated_resources;
707};
708
709struct gdma_allocate_resource_range_resp {
710 struct gdma_resp_hdr hdr;
711 u32 allocated_resources;
712};
713
714struct gdma_destroy_resource_range_req {
715 struct gdma_req_hdr hdr;
716 u32 resource_type;
717 u32 num_resources;
718 u32 allocated_resources;
719};
720
721/* GDMA_CREATE_QUEUE */
722struct gdma_create_queue_req {
723 struct gdma_req_hdr hdr;
724 u32 type;
725 u32 reserved1;
726 u32 pdid;
727 u32 doolbell_id;
728 u64 gdma_region;
729 u32 reserved2;
730 u32 queue_size;
731 u32 log2_throttle_limit;
732 u32 eq_pci_msix_index;
733 u32 cq_mod_ctx_id;
734 u32 cq_parent_eq_id;
735 u8 rq_drop_on_overrun;
736 u8 rq_err_on_wqe_overflow;
737 u8 rq_chain_rec_wqes;
738 u8 sq_hw_db;
739 u32 reserved3;
740}; /* HW DATA */
741
742struct gdma_create_queue_resp {
743 struct gdma_resp_hdr hdr;
744 u32 queue_index;
745}; /* HW DATA */
746
747/* GDMA_DISABLE_QUEUE */
748struct gdma_disable_queue_req {
749 struct gdma_req_hdr hdr;
750 u32 type;
751 u32 queue_index;
752 u32 alloc_res_id_on_creation;
753}; /* HW DATA */
754
755/* GDMA_QUERY_HWC_TIMEOUT */
756struct gdma_query_hwc_timeout_req {
757 struct gdma_req_hdr hdr;
758 u32 timeout_ms;
759 u32 reserved;
760};
761
762struct gdma_query_hwc_timeout_resp {
763 struct gdma_resp_hdr hdr;
764 u32 timeout_ms;
765 u32 reserved;
766};
767
768enum gdma_mr_access_flags {
769 GDMA_ACCESS_FLAG_LOCAL_READ = BIT_ULL(0),
770 GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT_ULL(1),
771 GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
772 GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
773 GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
774};
775
776/* GDMA_CREATE_DMA_REGION */
777struct gdma_create_dma_region_req {
778 struct gdma_req_hdr hdr;
779
780 /* The total size of the DMA region */
781 u64 length;
782
783 /* The offset in the first page */
784 u32 offset_in_page;
785
786 /* enum gdma_page_type */
787 u32 gdma_page_type;
788
789 /* The total number of pages */
790 u32 page_count;
791
792 /* If page_addr_list_len is smaller than page_count,
793 * the remaining page addresses will be added via the
794 * message GDMA_DMA_REGION_ADD_PAGES.
795 */
796 u32 page_addr_list_len;
797 u64 page_addr_list[];
798}; /* HW DATA */
799
800struct gdma_create_dma_region_resp {
801 struct gdma_resp_hdr hdr;
802 u64 dma_region_handle;
803}; /* HW DATA */
804
805/* GDMA_DMA_REGION_ADD_PAGES */
806struct gdma_dma_region_add_pages_req {
807 struct gdma_req_hdr hdr;
808
809 u64 dma_region_handle;
810
811 u32 page_addr_list_len;
812 u32 reserved3;
813
814 u64 page_addr_list[];
815}; /* HW DATA */
816
817/* GDMA_DESTROY_DMA_REGION */
818struct gdma_destroy_dma_region_req {
819 struct gdma_req_hdr hdr;
820
821 u64 dma_region_handle;
822}; /* HW DATA */
823
824enum gdma_pd_flags {
825 GDMA_PD_FLAG_INVALID = 0,
826 GDMA_PD_FLAG_ALLOW_GPA_MR = 1,
827};
828
829struct gdma_create_pd_req {
830 struct gdma_req_hdr hdr;
831 enum gdma_pd_flags flags;
832 u32 reserved;
833};/* HW DATA */
834
835struct gdma_create_pd_resp {
836 struct gdma_resp_hdr hdr;
837 u64 pd_handle;
838 u32 pd_id;
839 u32 reserved;
840};/* HW DATA */
841
842struct gdma_destroy_pd_req {
843 struct gdma_req_hdr hdr;
844 u64 pd_handle;
845};/* HW DATA */
846
847struct gdma_destory_pd_resp {
848 struct gdma_resp_hdr hdr;
849};/* HW DATA */
850
851enum gdma_mr_type {
852 /*
853 * Guest Physical Address - MRs of this type allow access
854 * to any DMA-mapped memory using bus-logical address
855 */
856 GDMA_MR_TYPE_GPA = 1,
857 /* Guest Virtual Address - MRs of this type allow access
858 * to memory mapped by PTEs associated with this MR using a virtual
859 * address that is set up in the MST
860 */
861 GDMA_MR_TYPE_GVA = 2,
862 /* Guest zero-based address MRs */
863 GDMA_MR_TYPE_ZBVA = 4,
864};
865
866struct gdma_create_mr_params {
867 u64 pd_handle;
868 enum gdma_mr_type mr_type;
869 union {
870 struct {
871 u64 dma_region_handle;
872 u64 virtual_address;
873 enum gdma_mr_access_flags access_flags;
874 } gva;
875 struct {
876 u64 dma_region_handle;
877 enum gdma_mr_access_flags access_flags;
878 } zbva;
879 };
880};
881
882struct gdma_create_mr_request {
883 struct gdma_req_hdr hdr;
884 u64 pd_handle;
885 enum gdma_mr_type mr_type;
886 u32 reserved_1;
887
888 union {
889 struct {
890 u64 dma_region_handle;
891 u64 virtual_address;
892 enum gdma_mr_access_flags access_flags;
893 } gva;
894 struct {
895 u64 dma_region_handle;
896 enum gdma_mr_access_flags access_flags;
897 } zbva;
898 };
899 u32 reserved_2;
900};/* HW DATA */
901
902struct gdma_create_mr_response {
903 struct gdma_resp_hdr hdr;
904 u64 mr_handle;
905 u32 lkey;
906 u32 rkey;
907};/* HW DATA */
908
909struct gdma_destroy_mr_request {
910 struct gdma_req_hdr hdr;
911 u64 mr_handle;
912};/* HW DATA */
913
914struct gdma_destroy_mr_response {
915 struct gdma_resp_hdr hdr;
916};/* HW DATA */
917
918int mana_gd_verify_vf_version(struct pci_dev *pdev);
919
920int mana_gd_register_device(struct gdma_dev *gd);
921int mana_gd_deregister_device(struct gdma_dev *gd);
922
923int mana_gd_post_work_request(struct gdma_queue *wq,
924 const struct gdma_wqe_request *wqe_req,
925 struct gdma_posted_wqe_info *wqe_info);
926
927int mana_gd_post_and_ring(struct gdma_queue *queue,
928 const struct gdma_wqe_request *wqe,
929 struct gdma_posted_wqe_info *wqe_info);
930
931int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
932void mana_gd_free_res_map(struct gdma_resource *r);
933
934void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
935 struct gdma_queue *queue);
936
937int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
938 struct gdma_mem_info *gmi);
939
940void mana_gd_free_memory(struct gdma_mem_info *gmi);
941
942int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
943 u32 resp_len, void *resp);
944
945int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle);
946void mana_register_debugfs(void);
947void mana_unregister_debugfs(void);
948
949int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event);
950
951int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state);
952int mana_gd_resume(struct pci_dev *pdev);
953
954bool mana_need_log(struct gdma_context *gc, int err);
955
956#endif /* _GDMA_H */
957

source code of linux/include/net/mana/gdma.h