1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2014 - 2020 Intel Corporation */
3#include <linux/delay.h>
4#include <linux/nospec.h>
5#include "adf_accel_devices.h"
6#include "adf_transport_internal.h"
7#include "adf_transport_access_macros.h"
8#include "adf_cfg.h"
9#include "adf_common_drv.h"
10
11#define ADF_MAX_RING_THRESHOLD 80
12#define ADF_PERCENT(tot, percent) (((tot) * (percent)) / 100)
13
14static inline u32 adf_modulo(u32 data, u32 shift)
15{
16 u32 div = data >> shift;
17 u32 mult = div << shift;
18
19 return data - mult;
20}
21
22static inline int adf_check_ring_alignment(u64 addr, u64 size)
23{
24 if (((size - 1) & addr) != 0)
25 return -EFAULT;
26 return 0;
27}
28
29static int adf_verify_ring_size(u32 msg_size, u32 msg_num)
30{
31 int i = ADF_MIN_RING_SIZE;
32
33 for (; i <= ADF_MAX_RING_SIZE; i++)
34 if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i))
35 return i;
36
37 return ADF_DEFAULT_RING_SIZE;
38}
39
40static int adf_reserve_ring(struct adf_etr_bank_data *bank, u32 ring)
41{
42 spin_lock(lock: &bank->lock);
43 if (bank->ring_mask & (1 << ring)) {
44 spin_unlock(lock: &bank->lock);
45 return -EFAULT;
46 }
47 bank->ring_mask |= (1 << ring);
48 spin_unlock(lock: &bank->lock);
49 return 0;
50}
51
52static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring)
53{
54 spin_lock(lock: &bank->lock);
55 bank->ring_mask &= ~(1 << ring);
56 spin_unlock(lock: &bank->lock);
57}
58
59static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
60{
61 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
62
63 spin_lock_bh(lock: &bank->lock);
64 bank->irq_mask |= (1 << ring);
65 spin_unlock_bh(lock: &bank->lock);
66 csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number,
67 bank->irq_mask);
68 csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number,
69 bank->irq_coalesc_timer);
70}
71
72static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
73{
74 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
75
76 spin_lock_bh(lock: &bank->lock);
77 bank->irq_mask &= ~(1 << ring);
78 spin_unlock_bh(lock: &bank->lock);
79 csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number,
80 bank->irq_mask);
81}
82
83bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)
84{
85 return atomic_read(v: ring->inflights) > ring->threshold;
86}
87
88int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)
89{
90 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
91
92 if (atomic_add_return(i: 1, v: ring->inflights) >
93 ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {
94 atomic_dec(v: ring->inflights);
95 return -EAGAIN;
96 }
97 spin_lock_bh(lock: &ring->lock);
98 memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg,
99 ADF_MSG_SIZE_TO_BYTES(ring->msg_size));
100
101 ring->tail = adf_modulo(data: ring->tail +
102 ADF_MSG_SIZE_TO_BYTES(ring->msg_size),
103 ADF_RING_SIZE_MODULO(ring->ring_size));
104 csr_ops->write_csr_ring_tail(ring->bank->csr_addr,
105 ring->bank->bank_number, ring->ring_number,
106 ring->tail);
107 spin_unlock_bh(lock: &ring->lock);
108
109 return 0;
110}
111
112static int adf_handle_response(struct adf_etr_ring_data *ring)
113{
114 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
115 u32 msg_counter = 0;
116 u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
117
118 while (*msg != ADF_RING_EMPTY_SIG) {
119 ring->callback((u32 *)msg);
120 atomic_dec(v: ring->inflights);
121 *msg = ADF_RING_EMPTY_SIG;
122 ring->head = adf_modulo(data: ring->head +
123 ADF_MSG_SIZE_TO_BYTES(ring->msg_size),
124 ADF_RING_SIZE_MODULO(ring->ring_size));
125 msg_counter++;
126 msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
127 }
128 if (msg_counter > 0) {
129 csr_ops->write_csr_ring_head(ring->bank->csr_addr,
130 ring->bank->bank_number,
131 ring->ring_number, ring->head);
132 }
133 return 0;
134}
135
136static void adf_configure_tx_ring(struct adf_etr_ring_data *ring)
137{
138 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
139 u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);
140
141 csr_ops->write_csr_ring_config(ring->bank->csr_addr,
142 ring->bank->bank_number,
143 ring->ring_number, ring_config);
144
145}
146
147static void adf_configure_rx_ring(struct adf_etr_ring_data *ring)
148{
149 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
150 u32 ring_config =
151 BUILD_RESP_RING_CONFIG(ring->ring_size,
152 ADF_RING_NEAR_WATERMARK_512,
153 ADF_RING_NEAR_WATERMARK_0);
154
155 csr_ops->write_csr_ring_config(ring->bank->csr_addr,
156 ring->bank->bank_number,
157 ring->ring_number, ring_config);
158}
159
160static int adf_init_ring(struct adf_etr_ring_data *ring)
161{
162 struct adf_etr_bank_data *bank = ring->bank;
163 struct adf_accel_dev *accel_dev = bank->accel_dev;
164 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
165 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
166 u64 ring_base;
167 u32 ring_size_bytes =
168 ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);
169
170 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);
171 ring->base_addr = dma_alloc_coherent(dev: &GET_DEV(accel_dev),
172 size: ring_size_bytes, dma_handle: &ring->dma_addr,
173 GFP_KERNEL);
174 if (!ring->base_addr)
175 return -ENOMEM;
176
177 memset(ring->base_addr, 0x7F, ring_size_bytes);
178 /* The base_addr has to be aligned to the size of the buffer */
179 if (adf_check_ring_alignment(addr: ring->dma_addr, size: ring_size_bytes)) {
180 dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n");
181 dma_free_coherent(dev: &GET_DEV(accel_dev), size: ring_size_bytes,
182 cpu_addr: ring->base_addr, dma_handle: ring->dma_addr);
183 ring->base_addr = NULL;
184 return -EFAULT;
185 }
186
187 if (hw_data->tx_rings_mask & (1 << ring->ring_number))
188 adf_configure_tx_ring(ring);
189
190 else
191 adf_configure_rx_ring(ring);
192
193 ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr,
194 ring->ring_size);
195
196 csr_ops->write_csr_ring_base(ring->bank->csr_addr,
197 ring->bank->bank_number, ring->ring_number,
198 ring_base);
199 spin_lock_init(&ring->lock);
200 return 0;
201}
202
203static void adf_cleanup_ring(struct adf_etr_ring_data *ring)
204{
205 u32 ring_size_bytes =
206 ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);
207 ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);
208
209 if (ring->base_addr) {
210 memset(ring->base_addr, 0x7F, ring_size_bytes);
211 dma_free_coherent(dev: &GET_DEV(ring->bank->accel_dev),
212 size: ring_size_bytes, cpu_addr: ring->base_addr,
213 dma_handle: ring->dma_addr);
214 }
215}
216
217int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
218 u32 bank_num, u32 num_msgs,
219 u32 msg_size, const char *ring_name,
220 adf_callback_fn callback, int poll_mode,
221 struct adf_etr_ring_data **ring_ptr)
222{
223 struct adf_etr_data *transport_data = accel_dev->transport;
224 u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev);
225 struct adf_etr_bank_data *bank;
226 struct adf_etr_ring_data *ring;
227 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
228 int max_inflights;
229 u32 ring_num;
230 int ret;
231
232 if (bank_num >= GET_MAX_BANKS(accel_dev)) {
233 dev_err(&GET_DEV(accel_dev), "Invalid bank number\n");
234 return -EFAULT;
235 }
236 if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
237 dev_err(&GET_DEV(accel_dev), "Invalid msg size\n");
238 return -EFAULT;
239 }
240 if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs),
241 ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) {
242 dev_err(&GET_DEV(accel_dev),
243 "Invalid ring size for given msg size\n");
244 return -EFAULT;
245 }
246 if (adf_cfg_get_param_value(accel_dev, section, name: ring_name, value: val)) {
247 dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n",
248 section, ring_name);
249 return -EFAULT;
250 }
251 if (kstrtouint(s: val, base: 10, res: &ring_num)) {
252 dev_err(&GET_DEV(accel_dev), "Can't get ring number\n");
253 return -EFAULT;
254 }
255 if (ring_num >= num_rings_per_bank) {
256 dev_err(&GET_DEV(accel_dev), "Invalid ring number\n");
257 return -EFAULT;
258 }
259
260 ring_num = array_index_nospec(ring_num, num_rings_per_bank);
261 bank = &transport_data->banks[bank_num];
262 if (adf_reserve_ring(bank, ring: ring_num)) {
263 dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n",
264 ring_num, ring_name);
265 return -EFAULT;
266 }
267 ring = &bank->rings[ring_num];
268 ring->ring_number = ring_num;
269 ring->bank = bank;
270 ring->callback = callback;
271 ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);
272 ring->ring_size = adf_verify_ring_size(msg_size, msg_num: num_msgs);
273 ring->head = 0;
274 ring->tail = 0;
275 max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);
276 ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD);
277 atomic_set(v: ring->inflights, i: 0);
278 ret = adf_init_ring(ring);
279 if (ret)
280 goto err;
281
282 /* Enable HW arbitration for the given ring */
283 adf_update_ring_arb(ring);
284
285 if (adf_ring_debugfs_add(ring, name: ring_name)) {
286 dev_err(&GET_DEV(accel_dev),
287 "Couldn't add ring debugfs entry\n");
288 ret = -EFAULT;
289 goto err;
290 }
291
292 /* Enable interrupts if needed */
293 if (callback && (!poll_mode))
294 adf_enable_ring_irq(bank, ring: ring->ring_number);
295 *ring_ptr = ring;
296 return 0;
297err:
298 adf_cleanup_ring(ring);
299 adf_unreserve_ring(bank, ring: ring_num);
300 adf_update_ring_arb(ring);
301 return ret;
302}
303
304void adf_remove_ring(struct adf_etr_ring_data *ring)
305{
306 struct adf_etr_bank_data *bank = ring->bank;
307 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
308
309 /* Disable interrupts for the given ring */
310 adf_disable_ring_irq(bank, ring: ring->ring_number);
311
312 /* Clear PCI config space */
313
314 csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number,
315 ring->ring_number, 0);
316 csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number,
317 ring->ring_number, 0);
318 adf_ring_debugfs_rm(ring);
319 adf_unreserve_ring(bank, ring: ring->ring_number);
320 /* Disable HW arbitration for the given ring */
321 adf_update_ring_arb(ring);
322 adf_cleanup_ring(ring);
323}
324
325static void adf_ring_response_handler(struct adf_etr_bank_data *bank)
326{
327 struct adf_accel_dev *accel_dev = bank->accel_dev;
328 u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev);
329 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
330 unsigned long empty_rings;
331 int i;
332
333 empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr,
334 bank->bank_number);
335 empty_rings = ~empty_rings & bank->irq_mask;
336
337 for_each_set_bit(i, &empty_rings, num_rings_per_bank)
338 adf_handle_response(ring: &bank->rings[i]);
339}
340
341void adf_response_handler(uintptr_t bank_addr)
342{
343 struct adf_etr_bank_data *bank = (void *)bank_addr;
344 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
345
346 /* Handle all the responses and reenable IRQs */
347 adf_ring_response_handler(bank);
348
349 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,
350 bank->irq_mask);
351}
352
353static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev,
354 const char *section, const char *format,
355 u32 key, u32 *value)
356{
357 char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
358 char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
359
360 snprintf(buf: key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, fmt: format, key);
361
362 if (adf_cfg_get_param_value(accel_dev, section, name: key_buf, value: val_buf))
363 return -EFAULT;
364
365 if (kstrtouint(s: val_buf, base: 10, res: value))
366 return -EFAULT;
367 return 0;
368}
369
370static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank,
371 const char *section,
372 u32 bank_num_in_accel)
373{
374 if (adf_get_cfg_int(accel_dev: bank->accel_dev, section,
375 ADF_ETRMGR_COALESCE_TIMER_FORMAT,
376 key: bank_num_in_accel, value: &bank->irq_coalesc_timer))
377 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME;
378
379 if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer ||
380 ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer)
381 bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME;
382}
383
384static int adf_init_bank(struct adf_accel_dev *accel_dev,
385 struct adf_etr_bank_data *bank,
386 u32 bank_num, void __iomem *csr_addr)
387{
388 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
389 u8 num_rings_per_bank = hw_data->num_rings_per_bank;
390 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops;
391 u32 irq_mask = BIT(num_rings_per_bank) - 1;
392 struct adf_etr_ring_data *ring;
393 struct adf_etr_ring_data *tx_ring;
394 u32 i, coalesc_enabled = 0;
395 unsigned long ring_mask;
396 int size;
397
398 memset(bank, 0, sizeof(*bank));
399 bank->bank_number = bank_num;
400 bank->csr_addr = csr_addr;
401 bank->accel_dev = accel_dev;
402 spin_lock_init(&bank->lock);
403
404 /* Allocate the rings in the bank */
405 size = num_rings_per_bank * sizeof(struct adf_etr_ring_data);
406 bank->rings = kzalloc_node(size, GFP_KERNEL,
407 dev_to_node(&GET_DEV(accel_dev)));
408 if (!bank->rings)
409 return -ENOMEM;
410
411 /* Enable IRQ coalescing always. This will allow to use
412 * the optimised flag and coalesc register.
413 * If it is disabled in the config file just use min time value */
414 if ((adf_get_cfg_int(accel_dev, section: "Accelerator0",
415 ADF_ETRMGR_COALESCING_ENABLED_FORMAT, key: bank_num,
416 value: &coalesc_enabled) == 0) && coalesc_enabled)
417 adf_get_coalesc_timer(bank, section: "Accelerator0", bank_num_in_accel: bank_num);
418 else
419 bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;
420
421 for (i = 0; i < num_rings_per_bank; i++) {
422 csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0);
423 csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0);
424
425 ring = &bank->rings[i];
426 if (hw_data->tx_rings_mask & (1 << i)) {
427 ring->inflights =
428 kzalloc_node(sizeof(atomic_t),
429 GFP_KERNEL,
430 dev_to_node(&GET_DEV(accel_dev)));
431 if (!ring->inflights)
432 goto err;
433 } else {
434 if (i < hw_data->tx_rx_gap) {
435 dev_err(&GET_DEV(accel_dev),
436 "Invalid tx rings mask config\n");
437 goto err;
438 }
439 tx_ring = &bank->rings[i - hw_data->tx_rx_gap];
440 ring->inflights = tx_ring->inflights;
441 }
442 }
443 if (adf_bank_debugfs_add(bank)) {
444 dev_err(&GET_DEV(accel_dev),
445 "Failed to add bank debugfs entry\n");
446 goto err;
447 }
448
449 csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask);
450 csr_ops->write_csr_int_srcsel(csr_addr, bank_num);
451
452 return 0;
453err:
454 ring_mask = hw_data->tx_rings_mask;
455 for_each_set_bit(i, &ring_mask, num_rings_per_bank) {
456 ring = &bank->rings[i];
457 kfree(objp: ring->inflights);
458 ring->inflights = NULL;
459 }
460 kfree(objp: bank->rings);
461 return -ENOMEM;
462}
463
464/**
465 * adf_init_etr_data() - Initialize transport rings for acceleration device
466 * @accel_dev: Pointer to acceleration device.
467 *
468 * Function is the initializes the communications channels (rings) to the
469 * acceleration device accel_dev.
470 * To be used by QAT device specific drivers.
471 *
472 * Return: 0 on success, error code otherwise.
473 */
474int adf_init_etr_data(struct adf_accel_dev *accel_dev)
475{
476 struct adf_etr_data *etr_data;
477 void __iomem *csr_addr;
478 u32 size;
479 u32 num_banks = 0;
480 int i, ret;
481
482 etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL,
483 dev_to_node(&GET_DEV(accel_dev)));
484 if (!etr_data)
485 return -ENOMEM;
486
487 num_banks = GET_MAX_BANKS(accel_dev);
488 size = num_banks * sizeof(struct adf_etr_bank_data);
489 etr_data->banks = kzalloc_node(size, GFP_KERNEL,
490 dev_to_node(&GET_DEV(accel_dev)));
491 if (!etr_data->banks) {
492 ret = -ENOMEM;
493 goto err_bank;
494 }
495
496 accel_dev->transport = etr_data;
497 csr_addr = adf_get_etr_base(accel_dev);
498
499 /* accel_dev->debugfs_dir should always be non-NULL here */
500 etr_data->debug = debugfs_create_dir(name: "transport",
501 parent: accel_dev->debugfs_dir);
502
503 for (i = 0; i < num_banks; i++) {
504 ret = adf_init_bank(accel_dev, bank: &etr_data->banks[i], bank_num: i,
505 csr_addr);
506 if (ret)
507 goto err_bank_all;
508 }
509
510 return 0;
511
512err_bank_all:
513 debugfs_remove(dentry: etr_data->debug);
514 kfree(objp: etr_data->banks);
515err_bank:
516 kfree(objp: etr_data);
517 accel_dev->transport = NULL;
518 return ret;
519}
520EXPORT_SYMBOL_GPL(adf_init_etr_data);
521
522static void cleanup_bank(struct adf_etr_bank_data *bank)
523{
524 struct adf_accel_dev *accel_dev = bank->accel_dev;
525 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
526 u8 num_rings_per_bank = hw_data->num_rings_per_bank;
527 u32 i;
528
529 for (i = 0; i < num_rings_per_bank; i++) {
530 struct adf_etr_ring_data *ring = &bank->rings[i];
531
532 if (bank->ring_mask & (1 << i))
533 adf_cleanup_ring(ring);
534
535 if (hw_data->tx_rings_mask & (1 << i))
536 kfree(objp: ring->inflights);
537 }
538 kfree(objp: bank->rings);
539 adf_bank_debugfs_rm(bank);
540 memset(bank, 0, sizeof(*bank));
541}
542
543static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev)
544{
545 struct adf_etr_data *etr_data = accel_dev->transport;
546 u32 i, num_banks = GET_MAX_BANKS(accel_dev);
547
548 for (i = 0; i < num_banks; i++)
549 cleanup_bank(bank: &etr_data->banks[i]);
550}
551
552/**
553 * adf_cleanup_etr_data() - Clear transport rings for acceleration device
554 * @accel_dev: Pointer to acceleration device.
555 *
556 * Function is the clears the communications channels (rings) of the
557 * acceleration device accel_dev.
558 * To be used by QAT device specific drivers.
559 *
560 * Return: void
561 */
562void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev)
563{
564 struct adf_etr_data *etr_data = accel_dev->transport;
565
566 if (etr_data) {
567 adf_cleanup_etr_handles(accel_dev);
568 debugfs_remove(dentry: etr_data->debug);
569 kfree(objp: etr_data->banks->rings);
570 kfree(objp: etr_data->banks);
571 kfree(objp: etr_data);
572 accel_dev->transport = NULL;
573 }
574}
575EXPORT_SYMBOL_GPL(adf_cleanup_etr_data);
576

source code of linux/drivers/crypto/intel/qat/qat_common/adf_transport.c