1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation */
3#include <linux/crypto.h>
4#include <crypto/acompress.h>
5#include <crypto/internal/acompress.h>
6#include <crypto/scatterwalk.h>
7#include <linux/dma-mapping.h>
8#include <linux/workqueue.h>
9#include "adf_accel_devices.h"
10#include "adf_common_drv.h"
11#include "adf_dc.h"
12#include "qat_bl.h"
13#include "qat_comp_req.h"
14#include "qat_compression.h"
15#include "qat_algs_send.h"
16
17static DEFINE_MUTEX(algs_lock);
18static unsigned int active_devs;
19
20enum direction {
21 DECOMPRESSION = 0,
22 COMPRESSION = 1,
23};
24
25struct qat_compression_req;
26
27struct qat_compression_ctx {
28 u8 comp_ctx[QAT_COMP_CTX_SIZE];
29 struct qat_compression_instance *inst;
30 int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp);
31};
32
33struct qat_compression_req {
34 u8 req[QAT_COMP_REQ_SIZE];
35 struct qat_compression_ctx *qat_compression_ctx;
36 struct acomp_req *acompress_req;
37 struct qat_request_buffs buf;
38 enum direction dir;
39 int actual_dlen;
40 struct qat_alg_req alg_req;
41};
42
43static int qat_alg_send_dc_message(struct qat_compression_req *qat_req,
44 struct qat_compression_instance *inst,
45 struct crypto_async_request *base)
46{
47 struct qat_alg_req *alg_req = &qat_req->alg_req;
48
49 alg_req->fw_req = (u32 *)&qat_req->req;
50 alg_req->tx_ring = inst->dc_tx;
51 alg_req->base = base;
52 alg_req->backlog = &inst->backlog;
53
54 return qat_alg_send_message(req: alg_req);
55}
56
57static void qat_comp_generic_callback(struct qat_compression_req *qat_req,
58 void *resp)
59{
60 struct acomp_req *areq = qat_req->acompress_req;
61 struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
62 struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
63 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req: areq);
64 struct qat_compression_instance *inst = ctx->inst;
65 int consumed, produced;
66 s8 cmp_err, xlt_err;
67 int res = -EBADMSG;
68 int status;
69 u8 cnv;
70
71 status = qat_comp_get_cmp_status(resp);
72 status |= qat_comp_get_xlt_status(resp);
73 cmp_err = qat_comp_get_cmp_err(resp);
74 xlt_err = qat_comp_get_xlt_err(resp);
75
76 consumed = qat_comp_get_consumed_ctr(resp);
77 produced = qat_comp_get_produced_ctr(resp);
78
79 dev_dbg(&GET_DEV(accel_dev),
80 "[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d",
81 crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
82 qat_req->dir == COMPRESSION ? "comp " : "decomp",
83 status ? "ERR" : "OK ",
84 areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err);
85
86 areq->dlen = 0;
87
88 if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
89 goto end;
90
91 if (qat_req->dir == COMPRESSION) {
92 cnv = qat_comp_get_cmp_cnv_flag(resp);
93 if (unlikely(!cnv)) {
94 dev_err(&GET_DEV(accel_dev),
95 "Verified compression not supported\n");
96 goto end;
97 }
98
99 if (unlikely(produced > qat_req->actual_dlen)) {
100 memset(inst->dc_data->ovf_buff, 0,
101 inst->dc_data->ovf_buff_sz);
102 dev_dbg(&GET_DEV(accel_dev),
103 "Actual buffer overflow: produced=%d, dlen=%d\n",
104 produced, qat_req->actual_dlen);
105 goto end;
106 }
107 }
108
109 res = 0;
110 areq->dlen = produced;
111
112 if (ctx->qat_comp_callback)
113 res = ctx->qat_comp_callback(qat_req, resp);
114
115end:
116 qat_bl_free_bufl(accel_dev, buf: &qat_req->buf);
117 acomp_request_complete(req: areq, err: res);
118}
119
120void qat_comp_alg_callback(void *resp)
121{
122 struct qat_compression_req *qat_req =
123 (void *)(__force long)qat_comp_get_opaque(resp);
124 struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
125
126 qat_comp_generic_callback(qat_req, resp);
127
128 qat_alg_send_backlog(backlog);
129}
130
131static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm)
132{
133 struct crypto_tfm *tfm = crypto_acomp_tfm(tfm: acomp_tfm);
134 struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
135 struct qat_compression_instance *inst;
136 int node;
137
138 if (tfm->node == NUMA_NO_NODE)
139 node = numa_node_id();
140 else
141 node = tfm->node;
142
143 memset(ctx, 0, sizeof(*ctx));
144 inst = qat_compression_get_instance_node(node);
145 if (!inst)
146 return -EINVAL;
147 ctx->inst = inst;
148
149 return qat_comp_build_ctx(accel_dev: inst->accel_dev, ctx: ctx->comp_ctx, algo: QAT_DEFLATE);
150}
151
152static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm)
153{
154 struct crypto_tfm *tfm = crypto_acomp_tfm(tfm: acomp_tfm);
155 struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
156
157 qat_compression_put_instance(inst: ctx->inst);
158 memset(ctx, 0, sizeof(*ctx));
159}
160
161static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir,
162 unsigned int shdr, unsigned int sftr,
163 unsigned int dhdr, unsigned int dftr)
164{
165 struct qat_compression_req *qat_req = acomp_request_ctx(req: areq);
166 struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req: areq);
167 struct crypto_tfm *tfm = crypto_acomp_tfm(tfm: acomp_tfm);
168 struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
169 struct qat_compression_instance *inst = ctx->inst;
170 gfp_t f = qat_algs_alloc_flags(req: &areq->base);
171 struct qat_sgl_to_bufl_params params = {0};
172 int slen = areq->slen - shdr - sftr;
173 int dlen = areq->dlen - dhdr - dftr;
174 dma_addr_t sfbuf, dfbuf;
175 u8 *req = qat_req->req;
176 size_t ovf_buff_sz;
177 int ret;
178
179 params.sskip = shdr;
180 params.dskip = dhdr;
181
182 if (!areq->src || !slen)
183 return -EINVAL;
184
185 if (!areq->dst || !dlen)
186 return -EINVAL;
187
188 if (dir == COMPRESSION) {
189 params.extra_dst_buff = inst->dc_data->ovf_buff_p;
190 ovf_buff_sz = inst->dc_data->ovf_buff_sz;
191 params.sz_extra_dst_buff = ovf_buff_sz;
192 }
193
194 ret = qat_bl_sgl_to_bufl(accel_dev: ctx->inst->accel_dev, sgl: areq->src, sglout: areq->dst,
195 buf: &qat_req->buf, params: &params, flags: f);
196 if (unlikely(ret))
197 return ret;
198
199 sfbuf = qat_req->buf.blp;
200 dfbuf = qat_req->buf.bloutp;
201 qat_req->qat_compression_ctx = ctx;
202 qat_req->acompress_req = areq;
203 qat_req->dir = dir;
204
205 if (dir == COMPRESSION) {
206 qat_req->actual_dlen = dlen;
207 dlen += ovf_buff_sz;
208 qat_comp_create_compression_req(ctx: ctx->comp_ctx, req,
209 src: (u64)(__force long)sfbuf, slen,
210 dst: (u64)(__force long)dfbuf, dlen,
211 opaque: (u64)(__force long)qat_req);
212 } else {
213 qat_comp_create_decompression_req(ctx: ctx->comp_ctx, req,
214 src: (u64)(__force long)sfbuf, slen,
215 dst: (u64)(__force long)dfbuf, dlen,
216 opaque: (u64)(__force long)qat_req);
217 }
218
219 ret = qat_alg_send_dc_message(qat_req, inst, base: &areq->base);
220 if (ret == -ENOSPC)
221 qat_bl_free_bufl(accel_dev: inst->accel_dev, buf: &qat_req->buf);
222
223 return ret;
224}
225
226static int qat_comp_alg_compress(struct acomp_req *req)
227{
228 return qat_comp_alg_compress_decompress(areq: req, dir: COMPRESSION, shdr: 0, sftr: 0, dhdr: 0, dftr: 0);
229}
230
231static int qat_comp_alg_decompress(struct acomp_req *req)
232{
233 return qat_comp_alg_compress_decompress(areq: req, dir: DECOMPRESSION, shdr: 0, sftr: 0, dhdr: 0, dftr: 0);
234}
235
236static struct acomp_alg qat_acomp[] = { {
237 .base = {
238 .cra_name = "deflate",
239 .cra_driver_name = "qat_deflate",
240 .cra_priority = 4001,
241 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
242 .cra_ctxsize = sizeof(struct qat_compression_ctx),
243 .cra_reqsize = sizeof(struct qat_compression_req),
244 .cra_module = THIS_MODULE,
245 },
246 .init = qat_comp_alg_init_tfm,
247 .exit = qat_comp_alg_exit_tfm,
248 .compress = qat_comp_alg_compress,
249 .decompress = qat_comp_alg_decompress,
250}};
251
252int qat_comp_algs_register(void)
253{
254 int ret = 0;
255
256 mutex_lock(&algs_lock);
257 if (++active_devs == 1)
258 ret = crypto_register_acomps(algs: qat_acomp, ARRAY_SIZE(qat_acomp));
259 mutex_unlock(lock: &algs_lock);
260 return ret;
261}
262
263void qat_comp_algs_unregister(void)
264{
265 mutex_lock(&algs_lock);
266 if (--active_devs == 0)
267 crypto_unregister_acomps(algs: qat_acomp, ARRAY_SIZE(qat_acomp));
268 mutex_unlock(lock: &algs_lock);
269}
270

source code of linux/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c