1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
3
4#include <linux/init.h>
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/pci.h>
8#include <linux/device.h>
9#include <linux/iommu.h>
10#include <uapi/linux/idxd.h>
11#include <linux/highmem.h>
12#include <linux/sched/smt.h>
13#include <crypto/internal/acompress.h>
14
15#include "idxd.h"
16#include "iaa_crypto.h"
17#include "iaa_crypto_stats.h"
18
19#ifdef pr_fmt
20#undef pr_fmt
21#endif
22
23#define pr_fmt(fmt) "idxd: " IDXD_SUBDRIVER_NAME ": " fmt
24
25#define IAA_ALG_PRIORITY 300
26
27/* number of iaa instances probed */
28static unsigned int nr_iaa;
29static unsigned int nr_cpus;
30static unsigned int nr_nodes;
31static unsigned int nr_cpus_per_node;
32
33/* Number of physical cpus sharing each iaa instance */
34static unsigned int cpus_per_iaa;
35
36/* Per-cpu lookup table for balanced wqs */
37static struct wq_table_entry __percpu *wq_table;
38
39static struct idxd_wq *wq_table_next_wq(int cpu)
40{
41 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
42
43 if (++entry->cur_wq >= entry->n_wqs)
44 entry->cur_wq = 0;
45
46 if (!entry->wqs[entry->cur_wq])
47 return NULL;
48
49 pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,
50 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,
51 entry->wqs[entry->cur_wq]->id, cpu);
52
53 return entry->wqs[entry->cur_wq];
54}
55
56static void wq_table_add(int cpu, struct idxd_wq *wq)
57{
58 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
59
60 if (WARN_ON(entry->n_wqs == entry->max_wqs))
61 return;
62
63 entry->wqs[entry->n_wqs++] = wq;
64
65 pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__,
66 entry->wqs[entry->n_wqs - 1]->idxd->id,
67 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu);
68}
69
70static void wq_table_free_entry(int cpu)
71{
72 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
73
74 kfree(objp: entry->wqs);
75 memset(entry, 0, sizeof(*entry));
76}
77
78static void wq_table_clear_entry(int cpu)
79{
80 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
81
82 entry->n_wqs = 0;
83 entry->cur_wq = 0;
84 memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));
85}
86
87LIST_HEAD(iaa_devices);
88DEFINE_MUTEX(iaa_devices_lock);
89
90/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */
91static bool iaa_crypto_enabled;
92static bool iaa_crypto_registered;
93
94/* Verify results of IAA compress or not */
95static bool iaa_verify_compress = true;
96
97static ssize_t verify_compress_show(struct device_driver *driver, char *buf)
98{
99 return sprintf(buf, fmt: "%d\n", iaa_verify_compress);
100}
101
102static ssize_t verify_compress_store(struct device_driver *driver,
103 const char *buf, size_t count)
104{
105 int ret = -EBUSY;
106
107 mutex_lock(&iaa_devices_lock);
108
109 if (iaa_crypto_enabled)
110 goto out;
111
112 ret = kstrtobool(s: buf, res: &iaa_verify_compress);
113 if (ret)
114 goto out;
115
116 ret = count;
117out:
118 mutex_unlock(lock: &iaa_devices_lock);
119
120 return ret;
121}
122static DRIVER_ATTR_RW(verify_compress);
123
124/*
125 * The iaa crypto driver supports three 'sync' methods determining how
126 * compressions and decompressions are performed:
127 *
128 * - sync: the compression or decompression completes before
129 * returning. This is the mode used by the async crypto
130 * interface when the sync mode is set to 'sync' and by
131 * the sync crypto interface regardless of setting.
132 *
133 * - async: the compression or decompression is submitted and returns
134 * immediately. Completion interrupts are not used so
135 * the caller is responsible for polling the descriptor
136 * for completion. This mode is applicable to only the
137 * async crypto interface and is ignored for anything
138 * else.
139 *
140 * - async_irq: the compression or decompression is submitted and
141 * returns immediately. Completion interrupts are
142 * enabled so the caller can wait for the completion and
143 * yield to other threads. When the compression or
144 * decompression completes, the completion is signaled
145 * and the caller awakened. This mode is applicable to
146 * only the async crypto interface and is ignored for
147 * anything else.
148 *
149 * These modes can be set using the iaa_crypto sync_mode driver
150 * attribute.
151 */
152
153/* Use async mode */
154static bool async_mode;
155/* Use interrupts */
156static bool use_irq;
157
158/**
159 * set_iaa_sync_mode - Set IAA sync mode
160 * @name: The name of the sync mode
161 *
162 * Make the IAA sync mode named @name the current sync mode used by
163 * compression/decompression.
164 */
165
166static int set_iaa_sync_mode(const char *name)
167{
168 int ret = 0;
169
170 if (sysfs_streq(s1: name, s2: "sync")) {
171 async_mode = false;
172 use_irq = false;
173 } else if (sysfs_streq(s1: name, s2: "async")) {
174 async_mode = false;
175 use_irq = false;
176 } else if (sysfs_streq(s1: name, s2: "async_irq")) {
177 async_mode = true;
178 use_irq = true;
179 } else {
180 ret = -EINVAL;
181 }
182
183 return ret;
184}
185
186static ssize_t sync_mode_show(struct device_driver *driver, char *buf)
187{
188 int ret = 0;
189
190 if (!async_mode && !use_irq)
191 ret = sprintf(buf, fmt: "%s\n", "sync");
192 else if (async_mode && !use_irq)
193 ret = sprintf(buf, fmt: "%s\n", "async");
194 else if (async_mode && use_irq)
195 ret = sprintf(buf, fmt: "%s\n", "async_irq");
196
197 return ret;
198}
199
200static ssize_t sync_mode_store(struct device_driver *driver,
201 const char *buf, size_t count)
202{
203 int ret = -EBUSY;
204
205 mutex_lock(&iaa_devices_lock);
206
207 if (iaa_crypto_enabled)
208 goto out;
209
210 ret = set_iaa_sync_mode(buf);
211 if (ret == 0)
212 ret = count;
213out:
214 mutex_unlock(lock: &iaa_devices_lock);
215
216 return ret;
217}
218static DRIVER_ATTR_RW(sync_mode);
219
220static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];
221
222static int find_empty_iaa_compression_mode(void)
223{
224 int i = -EINVAL;
225
226 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
227 if (iaa_compression_modes[i])
228 continue;
229 break;
230 }
231
232 return i;
233}
234
235static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx)
236{
237 struct iaa_compression_mode *mode;
238 int i;
239
240 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
241 mode = iaa_compression_modes[i];
242 if (!mode)
243 continue;
244
245 if (!strcmp(mode->name, name)) {
246 *idx = i;
247 return iaa_compression_modes[i];
248 }
249 }
250
251 return NULL;
252}
253
254static void free_iaa_compression_mode(struct iaa_compression_mode *mode)
255{
256 kfree(objp: mode->name);
257 kfree(objp: mode->ll_table);
258 kfree(objp: mode->d_table);
259
260 kfree(objp: mode);
261}
262
263/*
264 * IAA Compression modes are defined by an ll_table and a d_table.
265 * These tables are typically generated and captured using statistics
266 * collected from running actual compress/decompress workloads.
267 *
268 * A module or other kernel code can add and remove compression modes
269 * with a given name using the exported @add_iaa_compression_mode()
270 * and @remove_iaa_compression_mode functions.
271 *
272 * When a new compression mode is added, the tables are saved in a
273 * global compression mode list. When IAA devices are added, a
274 * per-IAA device dma mapping is created for each IAA device, for each
275 * compression mode. These are the tables used to do the actual
276 * compression/deccompression and are unmapped if/when the devices are
277 * removed. Currently, compression modes must be added before any
278 * device is added, and removed after all devices have been removed.
279 */
280
281/**
282 * remove_iaa_compression_mode - Remove an IAA compression mode
283 * @name: The name the compression mode will be known as
284 *
285 * Remove the IAA compression mode named @name.
286 */
287void remove_iaa_compression_mode(const char *name)
288{
289 struct iaa_compression_mode *mode;
290 int idx;
291
292 mutex_lock(&iaa_devices_lock);
293
294 if (!list_empty(head: &iaa_devices))
295 goto out;
296
297 mode = find_iaa_compression_mode(name, idx: &idx);
298 if (mode) {
299 free_iaa_compression_mode(mode);
300 iaa_compression_modes[idx] = NULL;
301 }
302out:
303 mutex_unlock(lock: &iaa_devices_lock);
304}
305EXPORT_SYMBOL_GPL(remove_iaa_compression_mode);
306
307/**
308 * add_iaa_compression_mode - Add an IAA compression mode
309 * @name: The name the compression mode will be known as
310 * @ll_table: The ll table
311 * @ll_table_size: The ll table size in bytes
312 * @d_table: The d table
313 * @d_table_size: The d table size in bytes
314 * @init: Optional callback function to init the compression mode data
315 * @free: Optional callback function to free the compression mode data
316 *
317 * Add a new IAA compression mode named @name.
318 *
319 * Returns 0 if successful, errcode otherwise.
320 */
321int add_iaa_compression_mode(const char *name,
322 const u32 *ll_table,
323 int ll_table_size,
324 const u32 *d_table,
325 int d_table_size,
326 iaa_dev_comp_init_fn_t init,
327 iaa_dev_comp_free_fn_t free)
328{
329 struct iaa_compression_mode *mode;
330 int idx, ret = -ENOMEM;
331
332 mutex_lock(&iaa_devices_lock);
333
334 if (!list_empty(head: &iaa_devices)) {
335 ret = -EBUSY;
336 goto out;
337 }
338
339 mode = kzalloc(sizeof(*mode), GFP_KERNEL);
340 if (!mode)
341 goto out;
342
343 mode->name = kstrdup(s: name, GFP_KERNEL);
344 if (!mode->name)
345 goto free;
346
347 if (ll_table) {
348 mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL);
349 if (!mode->ll_table)
350 goto free;
351 mode->ll_table_size = ll_table_size;
352 }
353
354 if (d_table) {
355 mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL);
356 if (!mode->d_table)
357 goto free;
358 mode->d_table_size = d_table_size;
359 }
360
361 mode->init = init;
362 mode->free = free;
363
364 idx = find_empty_iaa_compression_mode();
365 if (idx < 0)
366 goto free;
367
368 pr_debug("IAA compression mode %s added at idx %d\n",
369 mode->name, idx);
370
371 iaa_compression_modes[idx] = mode;
372
373 ret = 0;
374out:
375 mutex_unlock(lock: &iaa_devices_lock);
376
377 return ret;
378free:
379 free_iaa_compression_mode(mode);
380 goto out;
381}
382EXPORT_SYMBOL_GPL(add_iaa_compression_mode);
383
384static struct iaa_device_compression_mode *
385get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx)
386{
387 return iaa_device->compression_modes[idx];
388}
389
390static void free_device_compression_mode(struct iaa_device *iaa_device,
391 struct iaa_device_compression_mode *device_mode)
392{
393 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
394 struct device *dev = &iaa_device->idxd->pdev->dev;
395
396 kfree(objp: device_mode->name);
397
398 if (device_mode->aecs_comp_table)
399 dma_free_coherent(dev, size, cpu_addr: device_mode->aecs_comp_table,
400 dma_handle: device_mode->aecs_comp_table_dma_addr);
401 kfree(objp: device_mode);
402}
403
404#define IDXD_OP_FLAG_AECS_RW_TGLS 0x400000
405#define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC)
406#define IAX_AECS_COMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
407#define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
408#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \
409 IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \
410 IDXD_OP_FLAG_AECS_RW_TGLS)
411
412static int check_completion(struct device *dev,
413 struct iax_completion_record *comp,
414 bool compress,
415 bool only_once);
416
417static int init_device_compression_mode(struct iaa_device *iaa_device,
418 struct iaa_compression_mode *mode,
419 int idx, struct idxd_wq *wq)
420{
421 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
422 struct device *dev = &iaa_device->idxd->pdev->dev;
423 struct iaa_device_compression_mode *device_mode;
424 int ret = -ENOMEM;
425
426 device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL);
427 if (!device_mode)
428 return -ENOMEM;
429
430 device_mode->name = kstrdup(s: mode->name, GFP_KERNEL);
431 if (!device_mode->name)
432 goto free;
433
434 device_mode->aecs_comp_table = dma_alloc_coherent(dev, size,
435 dma_handle: &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL);
436 if (!device_mode->aecs_comp_table)
437 goto free;
438
439 /* Add Huffman table to aecs */
440 memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));
441 memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size);
442 memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size);
443
444 if (mode->init) {
445 ret = mode->init(device_mode);
446 if (ret)
447 goto free;
448 }
449
450 /* mode index should match iaa_compression_modes idx */
451 iaa_device->compression_modes[idx] = device_mode;
452
453 pr_debug("IAA %s compression mode initialized for iaa device %d\n",
454 mode->name, iaa_device->idxd->id);
455
456 ret = 0;
457out:
458 return ret;
459free:
460 pr_debug("IAA %s compression mode initialization failed for iaa device %d\n",
461 mode->name, iaa_device->idxd->id);
462
463 free_device_compression_mode(iaa_device, device_mode);
464 goto out;
465}
466
467static int init_device_compression_modes(struct iaa_device *iaa_device,
468 struct idxd_wq *wq)
469{
470 struct iaa_compression_mode *mode;
471 int i, ret = 0;
472
473 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
474 mode = iaa_compression_modes[i];
475 if (!mode)
476 continue;
477
478 ret = init_device_compression_mode(iaa_device, mode, idx: i, wq);
479 if (ret)
480 break;
481 }
482
483 return ret;
484}
485
486static void remove_device_compression_modes(struct iaa_device *iaa_device)
487{
488 struct iaa_device_compression_mode *device_mode;
489 int i;
490
491 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
492 device_mode = iaa_device->compression_modes[i];
493 if (!device_mode)
494 continue;
495
496 if (iaa_compression_modes[i]->free)
497 iaa_compression_modes[i]->free(device_mode);
498 free_device_compression_mode(iaa_device, device_mode);
499 iaa_device->compression_modes[i] = NULL;
500 }
501}
502
503static struct iaa_device *iaa_device_alloc(void)
504{
505 struct iaa_device *iaa_device;
506
507 iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL);
508 if (!iaa_device)
509 return NULL;
510
511 INIT_LIST_HEAD(list: &iaa_device->wqs);
512
513 return iaa_device;
514}
515
516static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
517{
518 struct iaa_wq *iaa_wq;
519
520 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
521 if (iaa_wq->wq == wq)
522 return true;
523 }
524
525 return false;
526}
527
528static struct iaa_device *add_iaa_device(struct idxd_device *idxd)
529{
530 struct iaa_device *iaa_device;
531
532 iaa_device = iaa_device_alloc();
533 if (!iaa_device)
534 return NULL;
535
536 iaa_device->idxd = idxd;
537
538 list_add_tail(new: &iaa_device->list, head: &iaa_devices);
539
540 nr_iaa++;
541
542 return iaa_device;
543}
544
545static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq)
546{
547 int ret = 0;
548
549 ret = init_device_compression_modes(iaa_device, wq: iaa_wq->wq);
550 if (ret)
551 return ret;
552
553 return ret;
554}
555
556static void del_iaa_device(struct iaa_device *iaa_device)
557{
558 list_del(entry: &iaa_device->list);
559
560 nr_iaa--;
561}
562
563static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq,
564 struct iaa_wq **new_wq)
565{
566 struct idxd_device *idxd = iaa_device->idxd;
567 struct pci_dev *pdev = idxd->pdev;
568 struct device *dev = &pdev->dev;
569 struct iaa_wq *iaa_wq;
570
571 iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL);
572 if (!iaa_wq)
573 return -ENOMEM;
574
575 iaa_wq->wq = wq;
576 iaa_wq->iaa_device = iaa_device;
577 idxd_wq_set_private(wq, iaa_wq);
578
579 list_add_tail(new: &iaa_wq->list, head: &iaa_device->wqs);
580
581 iaa_device->n_wq++;
582
583 if (new_wq)
584 *new_wq = iaa_wq;
585
586 dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n",
587 wq->id, iaa_device->idxd->id, iaa_device->n_wq);
588
589 return 0;
590}
591
592static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
593{
594 struct idxd_device *idxd = iaa_device->idxd;
595 struct pci_dev *pdev = idxd->pdev;
596 struct device *dev = &pdev->dev;
597 struct iaa_wq *iaa_wq;
598
599 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
600 if (iaa_wq->wq == wq) {
601 list_del(entry: &iaa_wq->list);
602 iaa_device->n_wq--;
603
604 dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n",
605 wq->id, iaa_device->idxd->id,
606 iaa_device->n_wq, nr_iaa);
607
608 if (iaa_device->n_wq == 0)
609 del_iaa_device(iaa_device);
610 break;
611 }
612 }
613}
614
615static void clear_wq_table(void)
616{
617 int cpu;
618
619 for (cpu = 0; cpu < nr_cpus; cpu++)
620 wq_table_clear_entry(cpu);
621
622 pr_debug("cleared wq table\n");
623}
624
625static void free_iaa_device(struct iaa_device *iaa_device)
626{
627 if (!iaa_device)
628 return;
629
630 remove_device_compression_modes(iaa_device);
631 kfree(objp: iaa_device);
632}
633
634static void __free_iaa_wq(struct iaa_wq *iaa_wq)
635{
636 struct iaa_device *iaa_device;
637
638 if (!iaa_wq)
639 return;
640
641 iaa_device = iaa_wq->iaa_device;
642 if (iaa_device->n_wq == 0)
643 free_iaa_device(iaa_device: iaa_wq->iaa_device);
644}
645
646static void free_iaa_wq(struct iaa_wq *iaa_wq)
647{
648 struct idxd_wq *wq;
649
650 __free_iaa_wq(iaa_wq);
651
652 wq = iaa_wq->wq;
653
654 kfree(objp: iaa_wq);
655 idxd_wq_set_private(wq, NULL);
656}
657
658static int iaa_wq_get(struct idxd_wq *wq)
659{
660 struct idxd_device *idxd = wq->idxd;
661 struct iaa_wq *iaa_wq;
662 int ret = 0;
663
664 spin_lock(lock: &idxd->dev_lock);
665 iaa_wq = idxd_wq_get_private(wq);
666 if (iaa_wq && !iaa_wq->remove) {
667 iaa_wq->ref++;
668 idxd_wq_get(wq);
669 } else {
670 ret = -ENODEV;
671 }
672 spin_unlock(lock: &idxd->dev_lock);
673
674 return ret;
675}
676
677static int iaa_wq_put(struct idxd_wq *wq)
678{
679 struct idxd_device *idxd = wq->idxd;
680 struct iaa_wq *iaa_wq;
681 bool free = false;
682 int ret = 0;
683
684 spin_lock(lock: &idxd->dev_lock);
685 iaa_wq = idxd_wq_get_private(wq);
686 if (iaa_wq) {
687 iaa_wq->ref--;
688 if (iaa_wq->ref == 0 && iaa_wq->remove) {
689 idxd_wq_set_private(wq, NULL);
690 free = true;
691 }
692 idxd_wq_put(wq);
693 } else {
694 ret = -ENODEV;
695 }
696 spin_unlock(lock: &idxd->dev_lock);
697 if (free) {
698 __free_iaa_wq(iaa_wq);
699 kfree(objp: iaa_wq);
700 }
701
702 return ret;
703}
704
705static void free_wq_table(void)
706{
707 int cpu;
708
709 for (cpu = 0; cpu < nr_cpus; cpu++)
710 wq_table_free_entry(cpu);
711
712 free_percpu(pdata: wq_table);
713
714 pr_debug("freed wq table\n");
715}
716
717static int alloc_wq_table(int max_wqs)
718{
719 struct wq_table_entry *entry;
720 int cpu;
721
722 wq_table = alloc_percpu(struct wq_table_entry);
723 if (!wq_table)
724 return -ENOMEM;
725
726 for (cpu = 0; cpu < nr_cpus; cpu++) {
727 entry = per_cpu_ptr(wq_table, cpu);
728 entry->wqs = kcalloc(max_wqs, sizeof(*entry->wqs), GFP_KERNEL);
729 if (!entry->wqs) {
730 free_wq_table();
731 return -ENOMEM;
732 }
733
734 entry->max_wqs = max_wqs;
735 }
736
737 pr_debug("initialized wq table\n");
738
739 return 0;
740}
741
742static int save_iaa_wq(struct idxd_wq *wq)
743{
744 struct iaa_device *iaa_device, *found = NULL;
745 struct idxd_device *idxd;
746 struct pci_dev *pdev;
747 struct device *dev;
748 int ret = 0;
749
750 list_for_each_entry(iaa_device, &iaa_devices, list) {
751 if (iaa_device->idxd == wq->idxd) {
752 idxd = iaa_device->idxd;
753 pdev = idxd->pdev;
754 dev = &pdev->dev;
755 /*
756 * Check to see that we don't already have this wq.
757 * Shouldn't happen but we don't control probing.
758 */
759 if (iaa_has_wq(iaa_device, wq)) {
760 dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n",
761 iaa_device);
762 goto out;
763 }
764
765 found = iaa_device;
766
767 ret = add_iaa_wq(iaa_device, wq, NULL);
768 if (ret)
769 goto out;
770
771 break;
772 }
773 }
774
775 if (!found) {
776 struct iaa_device *new_device;
777 struct iaa_wq *new_wq;
778
779 new_device = add_iaa_device(idxd: wq->idxd);
780 if (!new_device) {
781 ret = -ENOMEM;
782 goto out;
783 }
784
785 ret = add_iaa_wq(iaa_device: new_device, wq, new_wq: &new_wq);
786 if (ret) {
787 del_iaa_device(iaa_device: new_device);
788 free_iaa_device(iaa_device: new_device);
789 goto out;
790 }
791
792 ret = init_iaa_device(iaa_device: new_device, iaa_wq: new_wq);
793 if (ret) {
794 del_iaa_wq(iaa_device: new_device, wq: new_wq->wq);
795 del_iaa_device(iaa_device: new_device);
796 free_iaa_wq(iaa_wq: new_wq);
797 goto out;
798 }
799 }
800
801 if (WARN_ON(nr_iaa == 0))
802 return -EINVAL;
803
804 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
805 if (!cpus_per_iaa)
806 cpus_per_iaa = 1;
807out:
808 return ret;
809}
810
811static void remove_iaa_wq(struct idxd_wq *wq)
812{
813 struct iaa_device *iaa_device;
814
815 list_for_each_entry(iaa_device, &iaa_devices, list) {
816 if (iaa_has_wq(iaa_device, wq)) {
817 del_iaa_wq(iaa_device, wq);
818 break;
819 }
820 }
821
822 if (nr_iaa) {
823 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
824 if (!cpus_per_iaa)
825 cpus_per_iaa = 1;
826 } else
827 cpus_per_iaa = 1;
828}
829
830static int wq_table_add_wqs(int iaa, int cpu)
831{
832 struct iaa_device *iaa_device, *found_device = NULL;
833 int ret = 0, cur_iaa = 0, n_wqs_added = 0;
834 struct idxd_device *idxd;
835 struct iaa_wq *iaa_wq;
836 struct pci_dev *pdev;
837 struct device *dev;
838
839 list_for_each_entry(iaa_device, &iaa_devices, list) {
840 idxd = iaa_device->idxd;
841 pdev = idxd->pdev;
842 dev = &pdev->dev;
843
844 if (cur_iaa != iaa) {
845 cur_iaa++;
846 continue;
847 }
848
849 found_device = iaa_device;
850 dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n",
851 found_device->idxd->id, cur_iaa);
852 break;
853 }
854
855 if (!found_device) {
856 found_device = list_first_entry_or_null(&iaa_devices,
857 struct iaa_device, list);
858 if (!found_device) {
859 pr_debug("couldn't find any iaa devices with wqs!\n");
860 ret = -EINVAL;
861 goto out;
862 }
863 cur_iaa = 0;
864
865 idxd = found_device->idxd;
866 pdev = idxd->pdev;
867 dev = &pdev->dev;
868 dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n",
869 found_device->idxd->id, cur_iaa);
870 }
871
872 list_for_each_entry(iaa_wq, &found_device->wqs, list) {
873 wq_table_add(cpu, wq: iaa_wq->wq);
874 pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",
875 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id);
876 n_wqs_added++;
877 }
878
879 if (!n_wqs_added) {
880 pr_debug("couldn't find any iaa wqs!\n");
881 ret = -EINVAL;
882 goto out;
883 }
884out:
885 return ret;
886}
887
888/*
889 * Rebalance the wq table so that given a cpu, it's easy to find the
890 * closest IAA instance. The idea is to try to choose the most
891 * appropriate IAA instance for a caller and spread available
892 * workqueues around to clients.
893 */
894static void rebalance_wq_table(void)
895{
896 const struct cpumask *node_cpus;
897 int node_cpu, node, cpu, iaa = 0;
898
899 if (nr_iaa == 0)
900 return;
901
902 pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n",
903 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa);
904
905 clear_wq_table();
906
907 if (nr_iaa == 1) {
908 for_each_possible_cpu(cpu) {
909 if (WARN_ON(wq_table_add_wqs(0, cpu)))
910 goto err;
911 }
912
913 return;
914 }
915
916 for_each_node_with_cpus(node) {
917 cpu = 0;
918 node_cpus = cpumask_of_node(node);
919
920 for_each_cpu(node_cpu, node_cpus) {
921 iaa = cpu / cpus_per_iaa;
922 if (WARN_ON(wq_table_add_wqs(iaa, node_cpu)))
923 goto err;
924 cpu++;
925 }
926 }
927
928 return;
929err:
930 pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu);
931}
932
933static inline int check_completion(struct device *dev,
934 struct iax_completion_record *comp,
935 bool compress,
936 bool only_once)
937{
938 char *op_str = compress ? "compress" : "decompress";
939 int status_checks = 0;
940 int ret = 0;
941
942 while (!comp->status) {
943 if (only_once)
944 return -EAGAIN;
945 cpu_relax();
946 if (status_checks++ >= IAA_COMPLETION_TIMEOUT) {
947 /* Something is wrong with the hw, disable it. */
948 dev_err(dev, "%s completion timed out - "
949 "assuming broken hw, iaa_crypto now DISABLED\n",
950 op_str);
951 iaa_crypto_enabled = false;
952 ret = -ETIMEDOUT;
953 goto out;
954 }
955 }
956
957 if (comp->status != IAX_COMP_SUCCESS) {
958 if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) {
959 ret = -ETIMEDOUT;
960 dev_dbg(dev, "%s timed out, size=0x%x\n",
961 op_str, comp->output_size);
962 update_completion_timeout_errs();
963 goto out;
964 }
965
966 if (comp->status == IAA_ANALYTICS_ERROR &&
967 comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) {
968 ret = -E2BIG;
969 dev_dbg(dev, "compressed > uncompressed size,"
970 " not compressing, size=0x%x\n",
971 comp->output_size);
972 update_completion_comp_buf_overflow_errs();
973 goto out;
974 }
975
976 if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) {
977 ret = -EOVERFLOW;
978 goto out;
979 }
980
981 ret = -EINVAL;
982 dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",
983 op_str, comp->status, comp->error_code, comp->output_size);
984 print_hex_dump(KERN_INFO, prefix_str: "cmp-rec: ", prefix_type: DUMP_PREFIX_OFFSET, rowsize: 8, groupsize: 1, buf: comp, len: 64, ascii: 0);
985 update_completion_einval_errs();
986
987 goto out;
988 }
989out:
990 return ret;
991}
992
993static int deflate_generic_decompress(struct acomp_req *req)
994{
995 ACOMP_FBREQ_ON_STACK(fbreq, req);
996 int ret;
997
998 ret = crypto_acomp_decompress(req: fbreq);
999 req->dlen = fbreq->dlen;
1000
1001 update_total_sw_decomp_calls();
1002
1003 return ret;
1004}
1005
1006static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1007 struct acomp_req *req,
1008 dma_addr_t *src_addr, dma_addr_t *dst_addr);
1009
1010static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1011 struct idxd_wq *wq,
1012 dma_addr_t src_addr, unsigned int slen,
1013 dma_addr_t dst_addr, unsigned int *dlen);
1014
1015static void iaa_desc_complete(struct idxd_desc *idxd_desc,
1016 enum idxd_complete_type comp_type,
1017 bool free_desc, void *__ctx,
1018 u32 *status)
1019{
1020 struct iaa_device_compression_mode *active_compression_mode;
1021 struct iaa_compression_ctx *compression_ctx;
1022 struct crypto_ctx *ctx = __ctx;
1023 struct iaa_device *iaa_device;
1024 struct idxd_device *idxd;
1025 struct iaa_wq *iaa_wq;
1026 struct pci_dev *pdev;
1027 struct device *dev;
1028 int ret, err = 0;
1029
1030 compression_ctx = crypto_tfm_ctx(tfm: ctx->tfm);
1031
1032 iaa_wq = idxd_wq_get_private(idxd_desc->wq);
1033 iaa_device = iaa_wq->iaa_device;
1034 idxd = iaa_device->idxd;
1035 pdev = idxd->pdev;
1036 dev = &pdev->dev;
1037
1038 active_compression_mode = get_iaa_device_compression_mode(iaa_device,
1039 idx: compression_ctx->mode);
1040 dev_dbg(dev, "%s: compression mode %s,"
1041 " ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,
1042 active_compression_mode->name,
1043 ctx->src_addr, ctx->dst_addr);
1044
1045 ret = check_completion(dev, comp: idxd_desc->iax_completion,
1046 compress: ctx->compress, only_once: false);
1047 if (ret) {
1048 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1049 if (!ctx->compress &&
1050 idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1051 pr_warn("%s: falling back to deflate-generic decompress, "
1052 "analytics error code %x\n", __func__,
1053 idxd_desc->iax_completion->error_code);
1054 ret = deflate_generic_decompress(req: ctx->req);
1055 if (ret) {
1056 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1057 __func__, ret);
1058 err = -EIO;
1059 goto err;
1060 }
1061 } else {
1062 err = -EIO;
1063 goto err;
1064 }
1065 } else {
1066 ctx->req->dlen = idxd_desc->iax_completion->output_size;
1067 }
1068
1069 /* Update stats */
1070 if (ctx->compress) {
1071 update_total_comp_bytes_out(n: ctx->req->dlen);
1072 update_wq_comp_bytes(idxd_wq: iaa_wq->wq, n: ctx->req->dlen);
1073 } else {
1074 update_total_decomp_bytes_in(n: ctx->req->slen);
1075 update_wq_decomp_bytes(idxd_wq: iaa_wq->wq, n: ctx->req->slen);
1076 }
1077
1078 if (ctx->compress && compression_ctx->verify_compress) {
1079 u32 *compression_crc = acomp_request_ctx(req: ctx->req);
1080 dma_addr_t src_addr, dst_addr;
1081
1082 *compression_crc = idxd_desc->iax_completion->crc;
1083
1084 ret = iaa_remap_for_verify(dev, iaa_wq, req: ctx->req, src_addr: &src_addr, dst_addr: &dst_addr);
1085 if (ret) {
1086 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1087 err = -EIO;
1088 goto out;
1089 }
1090
1091 ret = iaa_compress_verify(tfm: ctx->tfm, req: ctx->req, wq: iaa_wq->wq, src_addr,
1092 slen: ctx->req->slen, dst_addr, dlen: &ctx->req->dlen);
1093 if (ret) {
1094 dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
1095 err = -EIO;
1096 }
1097
1098 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
1099 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);
1100
1101 goto out;
1102 }
1103err:
1104 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
1105 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
1106out:
1107 if (ret != 0)
1108 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1109
1110 if (ctx->req->base.complete)
1111 acomp_request_complete(req: ctx->req, err);
1112
1113 if (free_desc)
1114 idxd_free_desc(idxd_desc->wq, idxd_desc);
1115 iaa_wq_put(wq: idxd_desc->wq);
1116}
1117
1118static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
1119 struct idxd_wq *wq,
1120 dma_addr_t src_addr, unsigned int slen,
1121 dma_addr_t dst_addr, unsigned int *dlen)
1122{
1123 struct iaa_device_compression_mode *active_compression_mode;
1124 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1125 u32 *compression_crc = acomp_request_ctx(req);
1126 struct iaa_device *iaa_device;
1127 struct idxd_desc *idxd_desc;
1128 struct iax_hw_desc *desc;
1129 struct idxd_device *idxd;
1130 struct iaa_wq *iaa_wq;
1131 struct pci_dev *pdev;
1132 struct device *dev;
1133 int ret = 0;
1134
1135 iaa_wq = idxd_wq_get_private(wq);
1136 iaa_device = iaa_wq->iaa_device;
1137 idxd = iaa_device->idxd;
1138 pdev = idxd->pdev;
1139 dev = &pdev->dev;
1140
1141 active_compression_mode = get_iaa_device_compression_mode(iaa_device, idx: ctx->mode);
1142
1143 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1144 if (IS_ERR(ptr: idxd_desc)) {
1145 dev_dbg(dev, "idxd descriptor allocation failed\n");
1146 dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));
1147 return PTR_ERR(ptr: idxd_desc);
1148 }
1149 desc = idxd_desc->iax_hw;
1150
1151 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |
1152 IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;
1153 desc->opcode = IAX_OPCODE_COMPRESS;
1154 desc->compr_flags = IAA_COMP_FLAGS;
1155 desc->priv = 0;
1156
1157 desc->src1_addr = (u64)src_addr;
1158 desc->src1_size = slen;
1159 desc->dst_addr = (u64)dst_addr;
1160 desc->max_dst_size = *dlen;
1161 desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
1162 desc->src2_size = sizeof(struct aecs_comp_table_record);
1163 desc->completion_addr = idxd_desc->compl_dma;
1164
1165 if (ctx->use_irq) {
1166 desc->flags |= IDXD_OP_FLAG_RCI;
1167
1168 idxd_desc->crypto.req = req;
1169 idxd_desc->crypto.tfm = tfm;
1170 idxd_desc->crypto.src_addr = src_addr;
1171 idxd_desc->crypto.dst_addr = dst_addr;
1172 idxd_desc->crypto.compress = true;
1173
1174 dev_dbg(dev, "%s use_async_irq: compression mode %s,"
1175 " src_addr %llx, dst_addr %llx\n", __func__,
1176 active_compression_mode->name,
1177 src_addr, dst_addr);
1178 }
1179
1180 dev_dbg(dev, "%s: compression mode %s,"
1181 " desc->src1_addr %llx, desc->src1_size %d,"
1182 " desc->dst_addr %llx, desc->max_dst_size %d,"
1183 " desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1184 active_compression_mode->name,
1185 desc->src1_addr, desc->src1_size, desc->dst_addr,
1186 desc->max_dst_size, desc->src2_addr, desc->src2_size);
1187
1188 ret = idxd_submit_desc(wq, idxd_desc);
1189 if (ret) {
1190 dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1191 goto err;
1192 }
1193
1194 /* Update stats */
1195 update_total_comp_calls();
1196 update_wq_comp_calls(idxd_wq: wq);
1197
1198 if (ctx->async_mode) {
1199 ret = -EINPROGRESS;
1200 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1201 goto out;
1202 }
1203
1204 ret = check_completion(dev, comp: idxd_desc->iax_completion, compress: true, only_once: false);
1205 if (ret) {
1206 dev_dbg(dev, "check_completion failed ret=%d\n", ret);
1207 goto err;
1208 }
1209
1210 *dlen = idxd_desc->iax_completion->output_size;
1211
1212 /* Update stats */
1213 update_total_comp_bytes_out(n: *dlen);
1214 update_wq_comp_bytes(idxd_wq: wq, n: *dlen);
1215
1216 *compression_crc = idxd_desc->iax_completion->crc;
1217
1218 if (!ctx->async_mode)
1219 idxd_free_desc(wq, idxd_desc);
1220out:
1221 return ret;
1222err:
1223 idxd_free_desc(wq, idxd_desc);
1224 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1225
1226 goto out;
1227}
1228
1229static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1230 struct acomp_req *req,
1231 dma_addr_t *src_addr, dma_addr_t *dst_addr)
1232{
1233 int ret = 0;
1234 int nr_sgs;
1235
1236 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1237 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1238
1239 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1240 if (nr_sgs <= 0 || nr_sgs > 1) {
1241 dev_dbg(dev, "verify: couldn't map src sg for iaa device %d,"
1242 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1243 iaa_wq->wq->id, ret);
1244 ret = -EIO;
1245 goto out;
1246 }
1247 *src_addr = sg_dma_address(req->src);
1248 dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1249 " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,
1250 req->src, req->slen, sg_dma_len(req->src));
1251
1252 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1253 if (nr_sgs <= 0 || nr_sgs > 1) {
1254 dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d,"
1255 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1256 iaa_wq->wq->id, ret);
1257 ret = -EIO;
1258 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1259 goto out;
1260 }
1261 *dst_addr = sg_dma_address(req->dst);
1262 dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1263 " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,
1264 req->dst, req->dlen, sg_dma_len(req->dst));
1265out:
1266 return ret;
1267}
1268
1269static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1270 struct idxd_wq *wq,
1271 dma_addr_t src_addr, unsigned int slen,
1272 dma_addr_t dst_addr, unsigned int *dlen)
1273{
1274 struct iaa_device_compression_mode *active_compression_mode;
1275 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1276 u32 *compression_crc = acomp_request_ctx(req);
1277 struct iaa_device *iaa_device;
1278 struct idxd_desc *idxd_desc;
1279 struct iax_hw_desc *desc;
1280 struct idxd_device *idxd;
1281 struct iaa_wq *iaa_wq;
1282 struct pci_dev *pdev;
1283 struct device *dev;
1284 int ret = 0;
1285
1286 iaa_wq = idxd_wq_get_private(wq);
1287 iaa_device = iaa_wq->iaa_device;
1288 idxd = iaa_device->idxd;
1289 pdev = idxd->pdev;
1290 dev = &pdev->dev;
1291
1292 active_compression_mode = get_iaa_device_compression_mode(iaa_device, idx: ctx->mode);
1293
1294 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1295 if (IS_ERR(ptr: idxd_desc)) {
1296 dev_dbg(dev, "idxd descriptor allocation failed\n");
1297 dev_dbg(dev, "iaa compress failed: ret=%ld\n",
1298 PTR_ERR(idxd_desc));
1299 return PTR_ERR(ptr: idxd_desc);
1300 }
1301 desc = idxd_desc->iax_hw;
1302
1303 /* Verify (optional) - decompress and check crc, suppress dest write */
1304
1305 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1306 desc->opcode = IAX_OPCODE_DECOMPRESS;
1307 desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT;
1308 desc->priv = 0;
1309
1310 desc->src1_addr = (u64)dst_addr;
1311 desc->src1_size = *dlen;
1312 desc->dst_addr = (u64)src_addr;
1313 desc->max_dst_size = slen;
1314 desc->completion_addr = idxd_desc->compl_dma;
1315
1316 dev_dbg(dev, "(verify) compression mode %s,"
1317 " desc->src1_addr %llx, desc->src1_size %d,"
1318 " desc->dst_addr %llx, desc->max_dst_size %d,"
1319 " desc->src2_addr %llx, desc->src2_size %d\n",
1320 active_compression_mode->name,
1321 desc->src1_addr, desc->src1_size, desc->dst_addr,
1322 desc->max_dst_size, desc->src2_addr, desc->src2_size);
1323
1324 ret = idxd_submit_desc(wq, idxd_desc);
1325 if (ret) {
1326 dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret);
1327 goto err;
1328 }
1329
1330 ret = check_completion(dev, comp: idxd_desc->iax_completion, compress: false, only_once: false);
1331 if (ret) {
1332 dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret);
1333 goto err;
1334 }
1335
1336 if (*compression_crc != idxd_desc->iax_completion->crc) {
1337 ret = -EINVAL;
1338 dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
1339 " comp=0x%x, decomp=0x%x\n", *compression_crc,
1340 idxd_desc->iax_completion->crc);
1341 print_hex_dump(KERN_INFO, prefix_str: "cmp-rec: ", prefix_type: DUMP_PREFIX_OFFSET,
1342 rowsize: 8, groupsize: 1, buf: idxd_desc->iax_completion, len: 64, ascii: 0);
1343 goto err;
1344 }
1345
1346 idxd_free_desc(wq, idxd_desc);
1347out:
1348 return ret;
1349err:
1350 idxd_free_desc(wq, idxd_desc);
1351 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1352
1353 goto out;
1354}
1355
1356static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
1357 struct idxd_wq *wq,
1358 dma_addr_t src_addr, unsigned int slen,
1359 dma_addr_t dst_addr, unsigned int *dlen)
1360{
1361 struct iaa_device_compression_mode *active_compression_mode;
1362 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1363 struct iaa_device *iaa_device;
1364 struct idxd_desc *idxd_desc;
1365 struct iax_hw_desc *desc;
1366 struct idxd_device *idxd;
1367 struct iaa_wq *iaa_wq;
1368 struct pci_dev *pdev;
1369 struct device *dev;
1370 int ret = 0;
1371
1372 iaa_wq = idxd_wq_get_private(wq);
1373 iaa_device = iaa_wq->iaa_device;
1374 idxd = iaa_device->idxd;
1375 pdev = idxd->pdev;
1376 dev = &pdev->dev;
1377
1378 active_compression_mode = get_iaa_device_compression_mode(iaa_device, idx: ctx->mode);
1379
1380 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1381 if (IS_ERR(ptr: idxd_desc)) {
1382 dev_dbg(dev, "idxd descriptor allocation failed\n");
1383 dev_dbg(dev, "iaa decompress failed: ret=%ld\n",
1384 PTR_ERR(idxd_desc));
1385 return PTR_ERR(ptr: idxd_desc);
1386 }
1387 desc = idxd_desc->iax_hw;
1388
1389 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1390 desc->opcode = IAX_OPCODE_DECOMPRESS;
1391 desc->max_dst_size = PAGE_SIZE;
1392 desc->decompr_flags = IAA_DECOMP_FLAGS;
1393 desc->priv = 0;
1394
1395 desc->src1_addr = (u64)src_addr;
1396 desc->dst_addr = (u64)dst_addr;
1397 desc->max_dst_size = *dlen;
1398 desc->src1_size = slen;
1399 desc->completion_addr = idxd_desc->compl_dma;
1400
1401 if (ctx->use_irq) {
1402 desc->flags |= IDXD_OP_FLAG_RCI;
1403
1404 idxd_desc->crypto.req = req;
1405 idxd_desc->crypto.tfm = tfm;
1406 idxd_desc->crypto.src_addr = src_addr;
1407 idxd_desc->crypto.dst_addr = dst_addr;
1408 idxd_desc->crypto.compress = false;
1409
1410 dev_dbg(dev, "%s: use_async_irq compression mode %s,"
1411 " src_addr %llx, dst_addr %llx\n", __func__,
1412 active_compression_mode->name,
1413 src_addr, dst_addr);
1414 }
1415
1416 dev_dbg(dev, "%s: decompression mode %s,"
1417 " desc->src1_addr %llx, desc->src1_size %d,"
1418 " desc->dst_addr %llx, desc->max_dst_size %d,"
1419 " desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1420 active_compression_mode->name,
1421 desc->src1_addr, desc->src1_size, desc->dst_addr,
1422 desc->max_dst_size, desc->src2_addr, desc->src2_size);
1423
1424 ret = idxd_submit_desc(wq, idxd_desc);
1425 if (ret) {
1426 dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1427 goto err;
1428 }
1429
1430 /* Update stats */
1431 update_total_decomp_calls();
1432 update_wq_decomp_calls(idxd_wq: wq);
1433
1434 if (ctx->async_mode) {
1435 ret = -EINPROGRESS;
1436 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1437 goto out;
1438 }
1439
1440 ret = check_completion(dev, comp: idxd_desc->iax_completion, compress: false, only_once: false);
1441 if (ret) {
1442 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1443 if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1444 pr_warn("%s: falling back to deflate-generic decompress, "
1445 "analytics error code %x\n", __func__,
1446 idxd_desc->iax_completion->error_code);
1447 ret = deflate_generic_decompress(req);
1448 if (ret) {
1449 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1450 __func__, ret);
1451 goto err;
1452 }
1453 } else {
1454 goto err;
1455 }
1456 } else {
1457 req->dlen = idxd_desc->iax_completion->output_size;
1458 }
1459
1460 *dlen = req->dlen;
1461
1462 if (!ctx->async_mode)
1463 idxd_free_desc(wq, idxd_desc);
1464
1465 /* Update stats */
1466 update_total_decomp_bytes_in(n: slen);
1467 update_wq_decomp_bytes(idxd_wq: wq, n: slen);
1468out:
1469 return ret;
1470err:
1471 idxd_free_desc(wq, idxd_desc);
1472 dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret);
1473
1474 goto out;
1475}
1476
1477static int iaa_comp_acompress(struct acomp_req *req)
1478{
1479 struct iaa_compression_ctx *compression_ctx;
1480 struct crypto_tfm *tfm = req->base.tfm;
1481 dma_addr_t src_addr, dst_addr;
1482 int nr_sgs, cpu, ret = 0;
1483 struct iaa_wq *iaa_wq;
1484 struct idxd_wq *wq;
1485 struct device *dev;
1486
1487 compression_ctx = crypto_tfm_ctx(tfm);
1488
1489 if (!iaa_crypto_enabled) {
1490 pr_debug("iaa_crypto disabled, not compressing\n");
1491 return -ENODEV;
1492 }
1493
1494 if (!req->src || !req->slen) {
1495 pr_debug("invalid src, not compressing\n");
1496 return -EINVAL;
1497 }
1498
1499 cpu = get_cpu();
1500 wq = wq_table_next_wq(cpu);
1501 put_cpu();
1502 if (!wq) {
1503 pr_debug("no wq configured for cpu=%d\n", cpu);
1504 return -ENODEV;
1505 }
1506
1507 ret = iaa_wq_get(wq);
1508 if (ret) {
1509 pr_debug("no wq available for cpu=%d\n", cpu);
1510 return -ENODEV;
1511 }
1512
1513 iaa_wq = idxd_wq_get_private(wq);
1514
1515 dev = &wq->idxd->pdev->dev;
1516
1517 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1518 if (nr_sgs <= 0 || nr_sgs > 1) {
1519 dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1520 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1521 iaa_wq->wq->id, ret);
1522 ret = -EIO;
1523 goto out;
1524 }
1525 src_addr = sg_dma_address(req->src);
1526 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1527 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1528 req->src, req->slen, sg_dma_len(req->src));
1529
1530 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1531 if (nr_sgs <= 0 || nr_sgs > 1) {
1532 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1533 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1534 iaa_wq->wq->id, ret);
1535 ret = -EIO;
1536 goto err_map_dst;
1537 }
1538 dst_addr = sg_dma_address(req->dst);
1539 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1540 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1541 req->dst, req->dlen, sg_dma_len(req->dst));
1542
1543 ret = iaa_compress(tfm, req, wq, src_addr, slen: req->slen, dst_addr,
1544 dlen: &req->dlen);
1545 if (ret == -EINPROGRESS)
1546 return ret;
1547
1548 if (!ret && compression_ctx->verify_compress) {
1549 ret = iaa_remap_for_verify(dev, iaa_wq, req, src_addr: &src_addr, dst_addr: &dst_addr);
1550 if (ret) {
1551 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1552 goto out;
1553 }
1554
1555 ret = iaa_compress_verify(tfm, req, wq, src_addr, slen: req->slen,
1556 dst_addr, dlen: &req->dlen);
1557 if (ret)
1558 dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret);
1559
1560 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1561 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1562
1563 goto out;
1564 }
1565
1566 if (ret)
1567 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1568
1569 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1570err_map_dst:
1571 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1572out:
1573 iaa_wq_put(wq);
1574
1575 return ret;
1576}
1577
1578static int iaa_comp_adecompress(struct acomp_req *req)
1579{
1580 struct crypto_tfm *tfm = req->base.tfm;
1581 dma_addr_t src_addr, dst_addr;
1582 int nr_sgs, cpu, ret = 0;
1583 struct iaa_wq *iaa_wq;
1584 struct device *dev;
1585 struct idxd_wq *wq;
1586
1587 if (!iaa_crypto_enabled) {
1588 pr_debug("iaa_crypto disabled, not decompressing\n");
1589 return -ENODEV;
1590 }
1591
1592 if (!req->src || !req->slen) {
1593 pr_debug("invalid src, not decompressing\n");
1594 return -EINVAL;
1595 }
1596
1597 cpu = get_cpu();
1598 wq = wq_table_next_wq(cpu);
1599 put_cpu();
1600 if (!wq) {
1601 pr_debug("no wq configured for cpu=%d\n", cpu);
1602 return -ENODEV;
1603 }
1604
1605 ret = iaa_wq_get(wq);
1606 if (ret) {
1607 pr_debug("no wq available for cpu=%d\n", cpu);
1608 return -ENODEV;
1609 }
1610
1611 iaa_wq = idxd_wq_get_private(wq);
1612
1613 dev = &wq->idxd->pdev->dev;
1614
1615 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1616 if (nr_sgs <= 0 || nr_sgs > 1) {
1617 dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1618 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1619 iaa_wq->wq->id, ret);
1620 ret = -EIO;
1621 goto out;
1622 }
1623 src_addr = sg_dma_address(req->src);
1624 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1625 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1626 req->src, req->slen, sg_dma_len(req->src));
1627
1628 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1629 if (nr_sgs <= 0 || nr_sgs > 1) {
1630 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1631 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1632 iaa_wq->wq->id, ret);
1633 ret = -EIO;
1634 goto err_map_dst;
1635 }
1636 dst_addr = sg_dma_address(req->dst);
1637 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1638 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1639 req->dst, req->dlen, sg_dma_len(req->dst));
1640
1641 ret = iaa_decompress(tfm, req, wq, src_addr, slen: req->slen,
1642 dst_addr, dlen: &req->dlen);
1643 if (ret == -EINPROGRESS)
1644 return ret;
1645
1646 if (ret != 0)
1647 dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1648
1649 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1650err_map_dst:
1651 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1652out:
1653 iaa_wq_put(wq);
1654
1655 return ret;
1656}
1657
1658static void compression_ctx_init(struct iaa_compression_ctx *ctx)
1659{
1660 ctx->verify_compress = iaa_verify_compress;
1661 ctx->async_mode = async_mode;
1662 ctx->use_irq = use_irq;
1663}
1664
1665static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)
1666{
1667 struct crypto_tfm *tfm = crypto_acomp_tfm(tfm: acomp_tfm);
1668 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1669
1670 compression_ctx_init(ctx);
1671
1672 ctx->mode = IAA_MODE_FIXED;
1673
1674 return 0;
1675}
1676
1677static struct acomp_alg iaa_acomp_fixed_deflate = {
1678 .init = iaa_comp_init_fixed,
1679 .compress = iaa_comp_acompress,
1680 .decompress = iaa_comp_adecompress,
1681 .base = {
1682 .cra_name = "deflate",
1683 .cra_driver_name = "deflate-iaa",
1684 .cra_flags = CRYPTO_ALG_ASYNC,
1685 .cra_ctxsize = sizeof(struct iaa_compression_ctx),
1686 .cra_reqsize = sizeof(u32),
1687 .cra_module = THIS_MODULE,
1688 .cra_priority = IAA_ALG_PRIORITY,
1689 }
1690};
1691
1692static int iaa_register_compression_device(void)
1693{
1694 int ret;
1695
1696 ret = crypto_register_acomp(alg: &iaa_acomp_fixed_deflate);
1697 if (ret) {
1698 pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);
1699 goto out;
1700 }
1701
1702 iaa_crypto_registered = true;
1703out:
1704 return ret;
1705}
1706
1707static int iaa_unregister_compression_device(void)
1708{
1709 if (iaa_crypto_registered)
1710 crypto_unregister_acomp(alg: &iaa_acomp_fixed_deflate);
1711
1712 return 0;
1713}
1714
1715static int iaa_crypto_probe(struct idxd_dev *idxd_dev)
1716{
1717 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1718 struct idxd_device *idxd = wq->idxd;
1719 struct idxd_driver_data *data = idxd->data;
1720 struct device *dev = &idxd_dev->conf_dev;
1721 bool first_wq = false;
1722 int ret = 0;
1723
1724 if (idxd->state != IDXD_DEV_ENABLED)
1725 return -ENXIO;
1726
1727 if (data->type != IDXD_TYPE_IAX)
1728 return -ENODEV;
1729
1730 mutex_lock(&wq->wq_lock);
1731
1732 if (idxd_wq_get_private(wq)) {
1733 mutex_unlock(lock: &wq->wq_lock);
1734 return -EBUSY;
1735 }
1736
1737 if (!idxd_wq_driver_name_match(wq, dev)) {
1738 dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n",
1739 idxd->id, wq->id, wq->driver_name, dev->driver->name);
1740 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
1741 ret = -ENODEV;
1742 goto err;
1743 }
1744
1745 wq->type = IDXD_WQT_KERNEL;
1746
1747 ret = idxd_drv_enable_wq(wq);
1748 if (ret < 0) {
1749 dev_dbg(dev, "enable wq %d.%d failed: %d\n",
1750 idxd->id, wq->id, ret);
1751 ret = -ENXIO;
1752 goto err;
1753 }
1754
1755 mutex_lock(&iaa_devices_lock);
1756
1757 if (list_empty(head: &iaa_devices)) {
1758 ret = alloc_wq_table(max_wqs: wq->idxd->max_wqs);
1759 if (ret)
1760 goto err_alloc;
1761 first_wq = true;
1762 }
1763
1764 ret = save_iaa_wq(wq);
1765 if (ret)
1766 goto err_save;
1767
1768 rebalance_wq_table();
1769
1770 if (first_wq) {
1771 iaa_crypto_enabled = true;
1772 ret = iaa_register_compression_device();
1773 if (ret != 0) {
1774 iaa_crypto_enabled = false;
1775 dev_dbg(dev, "IAA compression device registration failed\n");
1776 goto err_register;
1777 }
1778 try_module_get(THIS_MODULE);
1779
1780 pr_info("iaa_crypto now ENABLED\n");
1781 }
1782
1783 mutex_unlock(lock: &iaa_devices_lock);
1784out:
1785 mutex_unlock(lock: &wq->wq_lock);
1786
1787 return ret;
1788
1789err_register:
1790 remove_iaa_wq(wq);
1791 free_iaa_wq(iaa_wq: idxd_wq_get_private(wq));
1792err_save:
1793 if (first_wq)
1794 free_wq_table();
1795err_alloc:
1796 mutex_unlock(lock: &iaa_devices_lock);
1797 idxd_drv_disable_wq(wq);
1798err:
1799 wq->type = IDXD_WQT_NONE;
1800
1801 goto out;
1802}
1803
1804static void iaa_crypto_remove(struct idxd_dev *idxd_dev)
1805{
1806 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1807 struct idxd_device *idxd = wq->idxd;
1808 struct iaa_wq *iaa_wq;
1809 bool free = false;
1810
1811 idxd_wq_quiesce(wq);
1812
1813 mutex_lock(&wq->wq_lock);
1814 mutex_lock(&iaa_devices_lock);
1815
1816 remove_iaa_wq(wq);
1817
1818 spin_lock(lock: &idxd->dev_lock);
1819 iaa_wq = idxd_wq_get_private(wq);
1820 if (!iaa_wq) {
1821 spin_unlock(lock: &idxd->dev_lock);
1822 pr_err("%s: no iaa_wq available to remove\n", __func__);
1823 goto out;
1824 }
1825
1826 if (iaa_wq->ref) {
1827 iaa_wq->remove = true;
1828 } else {
1829 wq = iaa_wq->wq;
1830 idxd_wq_set_private(wq, NULL);
1831 free = true;
1832 }
1833 spin_unlock(lock: &idxd->dev_lock);
1834 if (free) {
1835 __free_iaa_wq(iaa_wq);
1836 kfree(objp: iaa_wq);
1837 }
1838
1839 idxd_drv_disable_wq(wq);
1840 rebalance_wq_table();
1841
1842 if (nr_iaa == 0) {
1843 iaa_crypto_enabled = false;
1844 free_wq_table();
1845 module_put(THIS_MODULE);
1846
1847 pr_info("iaa_crypto now DISABLED\n");
1848 }
1849out:
1850 mutex_unlock(lock: &iaa_devices_lock);
1851 mutex_unlock(lock: &wq->wq_lock);
1852}
1853
1854static enum idxd_dev_type dev_types[] = {
1855 IDXD_DEV_WQ,
1856 IDXD_DEV_NONE,
1857};
1858
1859static struct idxd_device_driver iaa_crypto_driver = {
1860 .probe = iaa_crypto_probe,
1861 .remove = iaa_crypto_remove,
1862 .name = IDXD_SUBDRIVER_NAME,
1863 .type = dev_types,
1864 .desc_complete = iaa_desc_complete,
1865};
1866
1867static int __init iaa_crypto_init_module(void)
1868{
1869 int ret = 0;
1870 int node;
1871
1872 nr_cpus = num_possible_cpus();
1873 for_each_node_with_cpus(node)
1874 nr_nodes++;
1875 if (!nr_nodes) {
1876 pr_err("IAA couldn't find any nodes with cpus\n");
1877 return -ENODEV;
1878 }
1879 nr_cpus_per_node = nr_cpus / nr_nodes;
1880
1881 ret = iaa_aecs_init_fixed();
1882 if (ret < 0) {
1883 pr_debug("IAA fixed compression mode init failed\n");
1884 goto err_aecs_init;
1885 }
1886
1887 ret = idxd_driver_register(&iaa_crypto_driver);
1888 if (ret) {
1889 pr_debug("IAA wq sub-driver registration failed\n");
1890 goto err_driver_reg;
1891 }
1892
1893 ret = driver_create_file(driver: &iaa_crypto_driver.drv,
1894 attr: &driver_attr_verify_compress);
1895 if (ret) {
1896 pr_debug("IAA verify_compress attr creation failed\n");
1897 goto err_verify_attr_create;
1898 }
1899
1900 ret = driver_create_file(driver: &iaa_crypto_driver.drv,
1901 attr: &driver_attr_sync_mode);
1902 if (ret) {
1903 pr_debug("IAA sync mode attr creation failed\n");
1904 goto err_sync_attr_create;
1905 }
1906
1907 if (iaa_crypto_debugfs_init())
1908 pr_warn("debugfs init failed, stats not available\n");
1909
1910 pr_debug("initialized\n");
1911out:
1912 return ret;
1913
1914err_sync_attr_create:
1915 driver_remove_file(driver: &iaa_crypto_driver.drv,
1916 attr: &driver_attr_verify_compress);
1917err_verify_attr_create:
1918 idxd_driver_unregister(&iaa_crypto_driver);
1919err_driver_reg:
1920 iaa_aecs_cleanup_fixed();
1921err_aecs_init:
1922
1923 goto out;
1924}
1925
1926static void __exit iaa_crypto_cleanup_module(void)
1927{
1928 if (iaa_unregister_compression_device())
1929 pr_debug("IAA compression device unregister failed\n");
1930
1931 iaa_crypto_debugfs_cleanup();
1932 driver_remove_file(driver: &iaa_crypto_driver.drv,
1933 attr: &driver_attr_sync_mode);
1934 driver_remove_file(driver: &iaa_crypto_driver.drv,
1935 attr: &driver_attr_verify_compress);
1936 idxd_driver_unregister(&iaa_crypto_driver);
1937 iaa_aecs_cleanup_fixed();
1938
1939 pr_debug("cleaned up\n");
1940}
1941
1942MODULE_IMPORT_NS("IDXD");
1943MODULE_LICENSE("GPL");
1944MODULE_ALIAS_IDXD_DEVICE(0);
1945MODULE_AUTHOR("Intel Corporation");
1946MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");
1947
1948module_init(iaa_crypto_init_module);
1949module_exit(iaa_crypto_cleanup_module);
1950

source code of linux/drivers/crypto/intel/iaa/iaa_crypto_main.c