1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved
4 */
5
6#include "mlx5_ib.h"
7#include "data_direct.h"
8
9static LIST_HEAD(mlx5_data_direct_dev_list);
10static LIST_HEAD(mlx5_data_direct_reg_list);
11
12/*
13 * This mutex should be held when accessing either of the above lists
14 */
15static DEFINE_MUTEX(mlx5_data_direct_mutex);
16
17struct mlx5_data_direct_registration {
18 struct mlx5_ib_dev *ibdev;
19 char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];
20 struct list_head list;
21};
22
23static const struct pci_device_id mlx5_data_direct_pci_table[] = {
24 { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */
25 { 0, }
26};
27
28static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev)
29{
30 struct pci_dev *pdev = dev->pdev;
31 unsigned int vpd_size, kw_len;
32 u8 *vpd_data;
33 int start;
34 int ret;
35
36 vpd_data = pci_vpd_alloc(dev: pdev, size: &vpd_size);
37 if (IS_ERR(ptr: vpd_data)) {
38 pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data);
39 return PTR_ERR(ptr: vpd_data);
40 }
41
42 start = pci_vpd_find_ro_info_keyword(buf: vpd_data, len: vpd_size, kw: "VU", size: &kw_len);
43 if (start < 0) {
44 ret = start;
45 pci_err(pdev, "VU keyword not found, err=%d\n", ret);
46 goto end;
47 }
48
49 dev->vuid = kmemdup_nul(s: vpd_data + start, len: kw_len, GFP_KERNEL);
50 ret = dev->vuid ? 0 : -ENOMEM;
51
52end:
53 kfree(objp: vpd_data);
54 return ret;
55}
56
57static void mlx5_data_direct_shutdown(struct pci_dev *pdev)
58{
59 pci_disable_device(dev: pdev);
60}
61
62static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
63{
64 int err;
65
66 err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
67 if (err) {
68 dev_warn(&pdev->dev,
69 "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err);
70 err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32));
71 if (err) {
72 dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err);
73 return err;
74 }
75 }
76
77 dma_set_max_seg_size(dev: &pdev->dev, SZ_2G);
78 return 0;
79}
80
81int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)
82{
83 struct mlx5_data_direct_registration *reg;
84 struct mlx5_data_direct_dev *dev;
85
86 reg = kzalloc(sizeof(*reg), GFP_KERNEL);
87 if (!reg)
88 return -ENOMEM;
89
90 reg->ibdev = ibdev;
91 strcpy(p: reg->vuid, q: vuid);
92
93 mutex_lock(&mlx5_data_direct_mutex);
94 list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
95 if (strcmp(dev->vuid, vuid) == 0) {
96 mlx5_ib_data_direct_bind(ibdev, dev);
97 break;
98 }
99 }
100
101 /* Add the registration to its global list, to be used upon bind/unbind
102 * of its affiliated data direct device
103 */
104 list_add_tail(new: &reg->list, head: &mlx5_data_direct_reg_list);
105 mutex_unlock(lock: &mlx5_data_direct_mutex);
106 return 0;
107}
108
109void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev)
110{
111 struct mlx5_data_direct_registration *reg;
112
113 mutex_lock(&mlx5_data_direct_mutex);
114 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
115 if (reg->ibdev == ibdev) {
116 list_del(entry: &reg->list);
117 kfree(objp: reg);
118 goto end;
119 }
120 }
121
122 WARN_ON(true);
123end:
124 mutex_unlock(lock: &mlx5_data_direct_mutex);
125}
126
127static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
128{
129 struct mlx5_data_direct_registration *reg;
130
131 mutex_lock(&mlx5_data_direct_mutex);
132 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
133 if (strcmp(dev->vuid, reg->vuid) == 0)
134 mlx5_ib_data_direct_bind(ibdev: reg->ibdev, dev);
135 }
136
137 /* Add the data direct device to the global list, further IB devices may
138 * use it later as well
139 */
140 list_add_tail(new: &dev->list, head: &mlx5_data_direct_dev_list);
141 mutex_unlock(lock: &mlx5_data_direct_mutex);
142}
143
144static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)
145{
146 struct mlx5_data_direct_registration *reg;
147
148 mutex_lock(&mlx5_data_direct_mutex);
149 /* Prevent any further affiliations */
150 list_del(entry: &dev->list);
151 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
152 if (strcmp(dev->vuid, reg->vuid) == 0)
153 mlx5_ib_data_direct_unbind(ibdev: reg->ibdev);
154 }
155 mutex_unlock(lock: &mlx5_data_direct_mutex);
156}
157
158static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id)
159{
160 struct mlx5_data_direct_dev *dev;
161 int err;
162
163 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
164 if (!dev)
165 return -ENOMEM;
166
167 dev->device = &pdev->dev;
168 dev->pdev = pdev;
169
170 pci_set_drvdata(pdev: dev->pdev, data: dev);
171 err = pci_enable_device(dev: pdev);
172 if (err) {
173 dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err);
174 goto err;
175 }
176
177 pci_set_master(dev: pdev);
178 err = mlx5_data_direct_set_dma_caps(pdev);
179 if (err)
180 goto err_disable;
181
182 if (pci_enable_atomic_ops_to_root(dev: pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
183 pci_enable_atomic_ops_to_root(dev: pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
184 pci_enable_atomic_ops_to_root(dev: pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))
185 dev_dbg(dev->device, "Enabling pci atomics failed\n");
186
187 err = mlx5_data_direct_vpd_get_vuid(dev);
188 if (err)
189 goto err_disable;
190
191 mlx5_data_direct_dev_reg(dev);
192 return 0;
193
194err_disable:
195 pci_disable_device(dev: pdev);
196err:
197 kfree(objp: dev);
198 return err;
199}
200
201static void mlx5_data_direct_remove(struct pci_dev *pdev)
202{
203 struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev);
204
205 mlx5_data_direct_dev_unreg(dev);
206 pci_disable_device(dev: pdev);
207 kfree(objp: dev->vuid);
208 kfree(objp: dev);
209}
210
211static struct pci_driver mlx5_data_direct_driver = {
212 .name = KBUILD_MODNAME,
213 .id_table = mlx5_data_direct_pci_table,
214 .probe = mlx5_data_direct_probe,
215 .remove = mlx5_data_direct_remove,
216 .shutdown = mlx5_data_direct_shutdown,
217};
218
219int mlx5_data_direct_driver_register(void)
220{
221 return pci_register_driver(&mlx5_data_direct_driver);
222}
223
224void mlx5_data_direct_driver_unregister(void)
225{
226 pci_unregister_driver(dev: &mlx5_data_direct_driver);
227}
228

source code of linux/drivers/infiniband/hw/mlx5/data_direct.c