1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2015 - 2021 Intel Corporation */
3#include <linux/workqueue.h>
4#include <linux/pci.h>
5#include <linux/device.h>
6#include "adf_common_drv.h"
7#include "adf_cfg.h"
8#include "adf_pfvf_pf_msg.h"
9
10#define ADF_VF2PF_RATELIMIT_INTERVAL 8
11#define ADF_VF2PF_RATELIMIT_BURST 130
12
13static struct workqueue_struct *pf2vf_resp_wq;
14
15struct adf_pf2vf_resp {
16 struct work_struct pf2vf_resp_work;
17 struct adf_accel_vf_info *vf_info;
18};
19
20static void adf_iov_send_resp(struct work_struct *work)
21{
22 struct adf_pf2vf_resp *pf2vf_resp =
23 container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work);
24 struct adf_accel_vf_info *vf_info = pf2vf_resp->vf_info;
25 struct adf_accel_dev *accel_dev = vf_info->accel_dev;
26 u32 vf_nr = vf_info->vf_nr;
27 bool ret;
28
29 mutex_lock(&vf_info->pfvf_mig_lock);
30 ret = adf_recv_and_handle_vf2pf_msg(accel_dev, vf_nr);
31 if (ret)
32 /* re-enable interrupt on PF from this VF */
33 adf_enable_vf2pf_interrupts(accel_dev, vf_mask: 1 << vf_nr);
34 mutex_unlock(lock: &vf_info->pfvf_mig_lock);
35
36 kfree(objp: pf2vf_resp);
37}
38
39void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info)
40{
41 struct adf_pf2vf_resp *pf2vf_resp;
42
43 pf2vf_resp = kzalloc(sizeof(*pf2vf_resp), GFP_ATOMIC);
44 if (!pf2vf_resp)
45 return;
46
47 pf2vf_resp->vf_info = vf_info;
48 INIT_WORK(&pf2vf_resp->pf2vf_resp_work, adf_iov_send_resp);
49 queue_work(wq: pf2vf_resp_wq, work: &pf2vf_resp->pf2vf_resp_work);
50}
51
52static int adf_enable_sriov(struct adf_accel_dev *accel_dev)
53{
54 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
55 int totalvfs = pci_sriov_get_totalvfs(dev: pdev);
56 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
57 struct adf_accel_vf_info *vf_info;
58 int i;
59
60 for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs;
61 i++, vf_info++) {
62 /* This ptr will be populated when VFs will be created */
63 vf_info->accel_dev = accel_dev;
64 vf_info->vf_nr = i;
65
66 mutex_init(&vf_info->pf2vf_lock);
67 mutex_init(&vf_info->pfvf_mig_lock);
68 ratelimit_state_init(rs: &vf_info->vf2pf_ratelimit,
69 ADF_VF2PF_RATELIMIT_INTERVAL,
70 ADF_VF2PF_RATELIMIT_BURST);
71 }
72
73 /* Set Valid bits in AE Thread to PCIe Function Mapping */
74 if (hw_data->configure_iov_threads)
75 hw_data->configure_iov_threads(accel_dev, true);
76
77 /* Enable VF to PF interrupts for all VFs */
78 adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1);
79
80 /*
81 * Due to the hardware design, when SR-IOV and the ring arbiter
82 * are enabled all the VFs supported in hardware must be enabled in
83 * order for all the hardware resources (i.e. bundles) to be usable.
84 * When SR-IOV is enabled, each of the VFs will own one bundle.
85 */
86 return pci_enable_sriov(dev: pdev, nr_virtfn: totalvfs);
87}
88
89static int adf_add_sriov_configuration(struct adf_accel_dev *accel_dev)
90{
91 unsigned long val = 0;
92 int ret;
93
94 ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC);
95 if (ret)
96 return ret;
97
98 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY,
99 val: &val, type: ADF_DEC);
100 if (ret)
101 return ret;
102
103 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC,
104 val: &val, type: ADF_DEC);
105 if (ret)
106 return ret;
107
108 set_bit(ADF_STATUS_CONFIGURED, addr: &accel_dev->status);
109
110 return ret;
111}
112
113static int adf_do_disable_sriov(struct adf_accel_dev *accel_dev)
114{
115 int ret;
116
117 if (adf_dev_in_use(accel_dev)) {
118 dev_err(&GET_DEV(accel_dev),
119 "Cannot disable SR-IOV, device in use\n");
120 return -EBUSY;
121 }
122
123 if (adf_dev_started(accel_dev)) {
124 if (adf_devmgr_in_reset(accel_dev)) {
125 dev_err(&GET_DEV(accel_dev),
126 "Cannot disable SR-IOV, device in reset\n");
127 return -EBUSY;
128 }
129
130 ret = adf_dev_down(accel_dev);
131 if (ret)
132 goto err_del_cfg;
133 }
134
135 adf_disable_sriov(accel_dev);
136
137 ret = adf_dev_up(accel_dev, init_config: true);
138 if (ret)
139 goto err_del_cfg;
140
141 return 0;
142
143err_del_cfg:
144 adf_cfg_del_all_except(accel_dev, ADF_GENERAL_SEC);
145 return ret;
146}
147
148static int adf_do_enable_sriov(struct adf_accel_dev *accel_dev)
149{
150 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
151 int totalvfs = pci_sriov_get_totalvfs(dev: pdev);
152 unsigned long val;
153 int ret;
154
155 if (!device_iommu_mapped(dev: &GET_DEV(accel_dev))) {
156 dev_warn(&GET_DEV(accel_dev),
157 "IOMMU should be enabled for SR-IOV to work correctly\n");
158 }
159
160 if (adf_dev_started(accel_dev)) {
161 if (adf_devmgr_in_reset(accel_dev) || adf_dev_in_use(accel_dev)) {
162 dev_err(&GET_DEV(accel_dev), "Device busy\n");
163 return -EBUSY;
164 }
165
166 ret = adf_dev_down(accel_dev);
167 if (ret)
168 return ret;
169 }
170
171 ret = adf_add_sriov_configuration(accel_dev);
172 if (ret)
173 goto err_del_cfg;
174
175 /* Allocate memory for VF info structs */
176 accel_dev->pf.vf_info = kcalloc(totalvfs, sizeof(struct adf_accel_vf_info),
177 GFP_KERNEL);
178 ret = -ENOMEM;
179 if (!accel_dev->pf.vf_info)
180 goto err_del_cfg;
181
182 ret = adf_dev_up(accel_dev, init_config: false);
183 if (ret) {
184 dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
185 accel_dev->accel_id);
186 goto err_free_vf_info;
187 }
188
189 ret = adf_enable_sriov(accel_dev);
190 if (ret)
191 goto err_free_vf_info;
192
193 val = 1;
194 ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, ADF_SRIOV_ENABLED,
195 val: &val, type: ADF_DEC);
196 if (ret)
197 goto err_free_vf_info;
198
199 return totalvfs;
200
201err_free_vf_info:
202 adf_dev_down(accel_dev);
203 kfree(objp: accel_dev->pf.vf_info);
204 accel_dev->pf.vf_info = NULL;
205 return ret;
206err_del_cfg:
207 adf_cfg_del_all_except(accel_dev, ADF_GENERAL_SEC);
208 return ret;
209}
210
211void adf_reenable_sriov(struct adf_accel_dev *accel_dev)
212{
213 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
214 char cfg[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
215
216 if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
217 ADF_SRIOV_ENABLED, value: cfg))
218 return;
219
220 if (!accel_dev->pf.vf_info)
221 return;
222
223 if (adf_add_sriov_configuration(accel_dev))
224 return;
225
226 dev_dbg(&pdev->dev, "Re-enabling SRIOV\n");
227 adf_enable_sriov(accel_dev);
228}
229
230/**
231 * adf_disable_sriov() - Disable SRIOV for the device
232 * @accel_dev: Pointer to accel device.
233 *
234 * Function disables SRIOV for the accel device.
235 *
236 * Return: 0 on success, error code otherwise.
237 */
238void adf_disable_sriov(struct adf_accel_dev *accel_dev)
239{
240 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
241 int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev));
242 struct adf_accel_vf_info *vf;
243 int i;
244
245 if (!accel_dev->pf.vf_info)
246 return;
247
248 adf_pf2vf_notify_restarting(accel_dev);
249 adf_pf2vf_wait_for_restarting_complete(accel_dev);
250 pci_disable_sriov(accel_to_pci_dev(accel_dev));
251
252 /* Disable VF to PF interrupts */
253 adf_disable_all_vf2pf_interrupts(accel_dev);
254
255 /* Clear Valid bits in AE Thread to PCIe Function Mapping */
256 if (hw_data->configure_iov_threads)
257 hw_data->configure_iov_threads(accel_dev, false);
258
259 for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) {
260 mutex_destroy(lock: &vf->pf2vf_lock);
261 mutex_destroy(lock: &vf->pfvf_mig_lock);
262 }
263
264 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
265 kfree(objp: accel_dev->pf.vf_info);
266 accel_dev->pf.vf_info = NULL;
267 }
268}
269EXPORT_SYMBOL_GPL(adf_disable_sriov);
270
271/**
272 * adf_sriov_configure() - Enable SRIOV for the device
273 * @pdev: Pointer to PCI device.
274 * @numvfs: Number of virtual functions (VFs) to enable.
275 *
276 * Note that the @numvfs parameter is ignored and all VFs supported by the
277 * device are enabled due to the design of the hardware.
278 *
279 * Function enables SRIOV for the PCI device.
280 *
281 * Return: number of VFs enabled on success, error code otherwise.
282 */
283int adf_sriov_configure(struct pci_dev *pdev, int numvfs)
284{
285 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pci_dev: pdev);
286
287 if (!accel_dev) {
288 dev_err(&pdev->dev, "Failed to find accel_dev\n");
289 return -EFAULT;
290 }
291
292 if (numvfs)
293 return adf_do_enable_sriov(accel_dev);
294 else
295 return adf_do_disable_sriov(accel_dev);
296}
297EXPORT_SYMBOL_GPL(adf_sriov_configure);
298
299int __init adf_init_pf_wq(void)
300{
301 /* Workqueue for PF2VF responses */
302 pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq",
303 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
304
305 return !pf2vf_resp_wq ? -ENOMEM : 0;
306}
307
308void adf_exit_pf_wq(void)
309{
310 if (pf2vf_resp_wq) {
311 destroy_workqueue(wq: pf2vf_resp_wq);
312 pf2vf_resp_wq = NULL;
313 }
314}
315

source code of linux/drivers/crypto/intel/qat/qat_common/adf_sriov.c