1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2014 - 2020 Intel Corporation */
3#include <linux/module.h>
4#include <linux/slab.h>
5#include <linux/crypto.h>
6#include <crypto/internal/aead.h>
7#include <crypto/internal/cipher.h>
8#include <crypto/internal/skcipher.h>
9#include <crypto/aes.h>
10#include <crypto/sha1.h>
11#include <crypto/sha2.h>
12#include <crypto/algapi.h>
13#include <crypto/authenc.h>
14#include <crypto/scatterwalk.h>
15#include <crypto/xts.h>
16#include <linux/dma-mapping.h>
17#include "adf_accel_devices.h"
18#include "qat_algs_send.h"
19#include "adf_common_drv.h"
20#include "qat_crypto.h"
21#include "icp_qat_hw.h"
22#include "icp_qat_fw.h"
23#include "icp_qat_fw_la.h"
24#include "qat_bl.h"
25
26#define QAT_AES_HW_CONFIG_ENC(alg, mode) \
27 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
28 ICP_QAT_HW_CIPHER_NO_CONVERT, \
29 ICP_QAT_HW_CIPHER_ENCRYPT)
30
31#define QAT_AES_HW_CONFIG_DEC(alg, mode) \
32 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
33 ICP_QAT_HW_CIPHER_KEY_CONVERT, \
34 ICP_QAT_HW_CIPHER_DECRYPT)
35
36#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \
37 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
38 ICP_QAT_HW_CIPHER_NO_CONVERT, \
39 ICP_QAT_HW_CIPHER_DECRYPT)
40
41#define HW_CAP_AES_V2(accel_dev) \
42 (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \
43 ICP_ACCEL_CAPABILITIES_AES_V2)
44
45static DEFINE_MUTEX(algs_lock);
46static unsigned int active_devs;
47
48/* Common content descriptor */
49struct qat_alg_cd {
50 union {
51 struct qat_enc { /* Encrypt content desc */
52 struct icp_qat_hw_cipher_algo_blk cipher;
53 struct icp_qat_hw_auth_algo_blk hash;
54 } qat_enc_cd;
55 struct qat_dec { /* Decrypt content desc */
56 struct icp_qat_hw_auth_algo_blk hash;
57 struct icp_qat_hw_cipher_algo_blk cipher;
58 } qat_dec_cd;
59 };
60} __aligned(64);
61
62struct qat_alg_aead_ctx {
63 struct qat_alg_cd *enc_cd;
64 struct qat_alg_cd *dec_cd;
65 dma_addr_t enc_cd_paddr;
66 dma_addr_t dec_cd_paddr;
67 struct icp_qat_fw_la_bulk_req enc_fw_req;
68 struct icp_qat_fw_la_bulk_req dec_fw_req;
69 enum icp_qat_hw_auth_algo qat_hash_alg;
70 unsigned int hash_digestsize;
71 unsigned int hash_blocksize;
72 struct qat_crypto_instance *inst;
73};
74
75struct qat_alg_skcipher_ctx {
76 struct icp_qat_hw_cipher_algo_blk *enc_cd;
77 struct icp_qat_hw_cipher_algo_blk *dec_cd;
78 dma_addr_t enc_cd_paddr;
79 dma_addr_t dec_cd_paddr;
80 struct icp_qat_fw_la_bulk_req enc_fw_req;
81 struct icp_qat_fw_la_bulk_req dec_fw_req;
82 struct qat_crypto_instance *inst;
83 struct crypto_skcipher *ftfm;
84 struct crypto_cipher *tweak;
85 bool fallback;
86 int mode;
87};
88
89static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
90 struct qat_alg_aead_ctx *ctx,
91 const u8 *auth_key,
92 unsigned int auth_keylen)
93{
94 switch (ctx->qat_hash_alg) {
95 case ICP_QAT_HW_AUTH_ALGO_SHA1: {
96 struct hmac_sha1_key key;
97 __be32 *istate = (__be32 *)hash->sha.state1;
98 __be32 *ostate = (__be32 *)(hash->sha.state1 +
99 round_up(sizeof(key.istate.h), 8));
100
101 hmac_sha1_preparekey(key: &key, raw_key: auth_key, raw_key_len: auth_keylen);
102 for (int i = 0; i < ARRAY_SIZE(key.istate.h); i++) {
103 istate[i] = cpu_to_be32(key.istate.h[i]);
104 ostate[i] = cpu_to_be32(key.ostate.h[i]);
105 }
106 memzero_explicit(s: &key, count: sizeof(key));
107 return 0;
108 }
109 case ICP_QAT_HW_AUTH_ALGO_SHA256: {
110 struct hmac_sha256_key key;
111 __be32 *istate = (__be32 *)hash->sha.state1;
112 __be32 *ostate = (__be32 *)(hash->sha.state1 +
113 sizeof(key.key.istate.h));
114
115 hmac_sha256_preparekey(key: &key, raw_key: auth_key, raw_key_len: auth_keylen);
116 for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) {
117 istate[i] = cpu_to_be32(key.key.istate.h[i]);
118 ostate[i] = cpu_to_be32(key.key.ostate.h[i]);
119 }
120 memzero_explicit(s: &key, count: sizeof(key));
121 return 0;
122 }
123 case ICP_QAT_HW_AUTH_ALGO_SHA512: {
124 struct hmac_sha512_key key;
125 __be64 *istate = (__be64 *)hash->sha.state1;
126 __be64 *ostate = (__be64 *)(hash->sha.state1 +
127 sizeof(key.key.istate.h));
128
129 hmac_sha512_preparekey(key: &key, raw_key: auth_key, raw_key_len: auth_keylen);
130 for (int i = 0; i < ARRAY_SIZE(key.key.istate.h); i++) {
131 istate[i] = cpu_to_be64(key.key.istate.h[i]);
132 ostate[i] = cpu_to_be64(key.key.ostate.h[i]);
133 }
134 memzero_explicit(s: &key, count: sizeof(key));
135 return 0;
136 }
137 default:
138 return -EFAULT;
139 }
140}
141
142static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
143{
144 header->hdr_flags =
145 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
146 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
147 header->comn_req_flags =
148 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
149 QAT_COMN_PTR_TYPE_SGL);
150 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
151 ICP_QAT_FW_LA_PARTIAL_NONE);
152 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
153 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
154 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
155 ICP_QAT_FW_LA_NO_PROTO);
156 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
157 ICP_QAT_FW_LA_NO_UPDATE_STATE);
158}
159
160static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
161 int alg,
162 struct crypto_authenc_keys *keys,
163 int mode)
164{
165 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm: aead_tfm);
166 unsigned int digestsize = crypto_aead_authsize(tfm: aead_tfm);
167 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
168 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
169 struct icp_qat_hw_auth_algo_blk *hash =
170 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
171 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
172 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
173 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
174 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
175 void *ptr = &req_tmpl->cd_ctrl;
176 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
177 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
178
179 /* CD setup */
180 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
181 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
182 hash->sha.inner_setup.auth_config.config =
183 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
184 ctx->qat_hash_alg, digestsize);
185 hash->sha.inner_setup.auth_counter.counter =
186 cpu_to_be32(ctx->hash_blocksize);
187
188 if (qat_alg_do_precomputes(hash, ctx, auth_key: keys->authkey, auth_keylen: keys->authkeylen))
189 return -EFAULT;
190
191 /* Request setup */
192 qat_alg_init_common_hdr(header);
193 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
194 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
195 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
196 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
197 ICP_QAT_FW_LA_RET_AUTH_RES);
198 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
199 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
200 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
201 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
202
203 /* Cipher CD config setup */
204 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
205 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
206 cipher_cd_ctrl->cipher_cfg_offset = 0;
207 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
208 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
209 /* Auth CD config setup */
210 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
211 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
212 hash_cd_ctrl->inner_res_sz = digestsize;
213 hash_cd_ctrl->final_sz = digestsize;
214
215 switch (ctx->qat_hash_alg) {
216 case ICP_QAT_HW_AUTH_ALGO_SHA1:
217 hash_cd_ctrl->inner_state1_sz =
218 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
219 hash_cd_ctrl->inner_state2_sz =
220 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
221 break;
222 case ICP_QAT_HW_AUTH_ALGO_SHA256:
223 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
224 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
225 break;
226 case ICP_QAT_HW_AUTH_ALGO_SHA512:
227 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
228 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
229 break;
230 default:
231 break;
232 }
233 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
234 ((sizeof(struct icp_qat_hw_auth_setup) +
235 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
236 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
237 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
238 return 0;
239}
240
241static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
242 int alg,
243 struct crypto_authenc_keys *keys,
244 int mode)
245{
246 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm: aead_tfm);
247 unsigned int digestsize = crypto_aead_authsize(tfm: aead_tfm);
248 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
249 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
250 struct icp_qat_hw_cipher_algo_blk *cipher =
251 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
252 sizeof(struct icp_qat_hw_auth_setup) +
253 roundup(ctx->hash_digestsize, 8) * 2);
254 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
255 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
256 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
257 void *ptr = &req_tmpl->cd_ctrl;
258 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
259 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
260 struct icp_qat_fw_la_auth_req_params *auth_param =
261 (struct icp_qat_fw_la_auth_req_params *)
262 ((char *)&req_tmpl->serv_specif_rqpars +
263 sizeof(struct icp_qat_fw_la_cipher_req_params));
264
265 /* CD setup */
266 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
267 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
268 hash->sha.inner_setup.auth_config.config =
269 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
270 ctx->qat_hash_alg,
271 digestsize);
272 hash->sha.inner_setup.auth_counter.counter =
273 cpu_to_be32(ctx->hash_blocksize);
274
275 if (qat_alg_do_precomputes(hash, ctx, auth_key: keys->authkey, auth_keylen: keys->authkeylen))
276 return -EFAULT;
277
278 /* Request setup */
279 qat_alg_init_common_hdr(header);
280 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
281 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
282 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
283 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
284 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
285 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
286 ICP_QAT_FW_LA_CMP_AUTH_RES);
287 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
288 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
289
290 /* Cipher CD config setup */
291 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
292 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
293 cipher_cd_ctrl->cipher_cfg_offset =
294 (sizeof(struct icp_qat_hw_auth_setup) +
295 roundup(ctx->hash_digestsize, 8) * 2) >> 3;
296 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
297 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
298
299 /* Auth CD config setup */
300 hash_cd_ctrl->hash_cfg_offset = 0;
301 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
302 hash_cd_ctrl->inner_res_sz = digestsize;
303 hash_cd_ctrl->final_sz = digestsize;
304
305 switch (ctx->qat_hash_alg) {
306 case ICP_QAT_HW_AUTH_ALGO_SHA1:
307 hash_cd_ctrl->inner_state1_sz =
308 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
309 hash_cd_ctrl->inner_state2_sz =
310 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
311 break;
312 case ICP_QAT_HW_AUTH_ALGO_SHA256:
313 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
314 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
315 break;
316 case ICP_QAT_HW_AUTH_ALGO_SHA512:
317 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
318 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
319 break;
320 default:
321 break;
322 }
323
324 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
325 ((sizeof(struct icp_qat_hw_auth_setup) +
326 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
327 auth_param->auth_res_sz = digestsize;
328 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
329 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
330 return 0;
331}
332
333static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,
334 struct icp_qat_fw_la_bulk_req *req,
335 struct icp_qat_hw_cipher_algo_blk *cd,
336 const u8 *key, unsigned int keylen)
337{
338 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
339 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
340 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
341 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
342 int mode = ctx->mode;
343
344 qat_alg_init_common_hdr(header);
345 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
346 cd_pars->u.s.content_desc_params_sz =
347 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
348
349 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
350 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
351 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
352
353 /* Store both XTS keys in CD, only the first key is sent
354 * to the HW, the second key is used for tweak calculation
355 */
356 memcpy(cd->ucs_aes.key, key, keylen);
357 keylen = keylen / 2;
358 } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
359 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
360 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
361 memcpy(cd->ucs_aes.key, key, keylen);
362 keylen = round_up(keylen, 16);
363 } else {
364 memcpy(cd->aes.key, key, keylen);
365 }
366
367 /* Cipher CD config setup */
368 cd_ctrl->cipher_key_sz = keylen >> 3;
369 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
370 cd_ctrl->cipher_cfg_offset = 0;
371 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
372 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
373}
374
375static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
376 int alg, const u8 *key,
377 unsigned int keylen, int mode)
378{
379 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
380 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
381 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
382
383 qat_alg_skcipher_init_com(ctx, req, cd: enc_cd, key, keylen);
384 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
385 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
386}
387
388static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
389 u8 *key_reverse)
390{
391 struct crypto_aes_ctx aes_expanded;
392 int nrounds;
393 u8 *key;
394
395 aes_expandkey(ctx: &aes_expanded, in_key: key_forward, key_len: keylen);
396 if (keylen == AES_KEYSIZE_128) {
397 nrounds = 10;
398 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
399 memcpy(key_reverse, key, AES_BLOCK_SIZE);
400 } else {
401 /* AES_KEYSIZE_256 */
402 nrounds = 14;
403 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
404 memcpy(key_reverse, key, AES_BLOCK_SIZE);
405 memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
406 AES_BLOCK_SIZE);
407 }
408}
409
410static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
411 int alg, const u8 *key,
412 unsigned int keylen, int mode)
413{
414 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
415 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
416 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
417 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
418
419 qat_alg_skcipher_init_com(ctx, req, cd: dec_cd, key, keylen);
420 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
421
422 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
423 /* Key reversing not supported, set no convert */
424 dec_cd->aes.cipher_config.val =
425 QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode);
426
427 /* In-place key reversal */
428 qat_alg_xts_reverse_key(key_forward: dec_cd->ucs_aes.key, keylen: keylen / 2,
429 key_reverse: dec_cd->ucs_aes.key);
430 } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) {
431 dec_cd->aes.cipher_config.val =
432 QAT_AES_HW_CONFIG_DEC(alg, mode);
433 } else {
434 dec_cd->aes.cipher_config.val =
435 QAT_AES_HW_CONFIG_ENC(alg, mode);
436 }
437}
438
439static int qat_alg_validate_key(int key_len, int *alg, int mode)
440{
441 if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
442 switch (key_len) {
443 case AES_KEYSIZE_128:
444 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
445 break;
446 case AES_KEYSIZE_192:
447 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
448 break;
449 case AES_KEYSIZE_256:
450 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
451 break;
452 default:
453 return -EINVAL;
454 }
455 } else {
456 switch (key_len) {
457 case AES_KEYSIZE_128 << 1:
458 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
459 break;
460 case AES_KEYSIZE_256 << 1:
461 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
462 break;
463 default:
464 return -EINVAL;
465 }
466 }
467 return 0;
468}
469
470static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
471 unsigned int keylen, int mode)
472{
473 struct crypto_authenc_keys keys;
474 int alg;
475
476 if (crypto_authenc_extractkeys(keys: &keys, key, keylen))
477 goto bad_key;
478
479 if (qat_alg_validate_key(key_len: keys.enckeylen, alg: &alg, mode))
480 goto bad_key;
481
482 if (qat_alg_aead_init_enc_session(aead_tfm: tfm, alg, keys: &keys, mode))
483 goto error;
484
485 if (qat_alg_aead_init_dec_session(aead_tfm: tfm, alg, keys: &keys, mode))
486 goto error;
487
488 memzero_explicit(s: &keys, count: sizeof(keys));
489 return 0;
490bad_key:
491 memzero_explicit(s: &keys, count: sizeof(keys));
492 return -EINVAL;
493error:
494 memzero_explicit(s: &keys, count: sizeof(keys));
495 return -EFAULT;
496}
497
498static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,
499 const u8 *key,
500 unsigned int keylen,
501 int mode)
502{
503 int alg;
504
505 if (qat_alg_validate_key(key_len: keylen, alg: &alg, mode))
506 return -EINVAL;
507
508 qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);
509 qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);
510 return 0;
511}
512
513static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key,
514 unsigned int keylen)
515{
516 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
517
518 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
519 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
520 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
521 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
522
523 return qat_alg_aead_init_sessions(tfm, key, keylen,
524 mode: ICP_QAT_HW_CIPHER_CBC_MODE);
525}
526
527static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key,
528 unsigned int keylen)
529{
530 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
531 struct qat_crypto_instance *inst = NULL;
532 int node = numa_node_id();
533 struct device *dev;
534 int ret;
535
536 inst = qat_crypto_get_instance_node(node);
537 if (!inst)
538 return -EINVAL;
539 dev = &GET_DEV(inst->accel_dev);
540 ctx->inst = inst;
541 ctx->enc_cd = dma_alloc_coherent(dev, size: sizeof(*ctx->enc_cd),
542 dma_handle: &ctx->enc_cd_paddr,
543 GFP_ATOMIC);
544 if (!ctx->enc_cd) {
545 ret = -ENOMEM;
546 goto out_free_inst;
547 }
548 ctx->dec_cd = dma_alloc_coherent(dev, size: sizeof(*ctx->dec_cd),
549 dma_handle: &ctx->dec_cd_paddr,
550 GFP_ATOMIC);
551 if (!ctx->dec_cd) {
552 ret = -ENOMEM;
553 goto out_free_enc;
554 }
555
556 ret = qat_alg_aead_init_sessions(tfm, key, keylen,
557 mode: ICP_QAT_HW_CIPHER_CBC_MODE);
558 if (ret)
559 goto out_free_all;
560
561 return 0;
562
563out_free_all:
564 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
565 dma_free_coherent(dev, size: sizeof(struct qat_alg_cd),
566 cpu_addr: ctx->dec_cd, dma_handle: ctx->dec_cd_paddr);
567 ctx->dec_cd = NULL;
568out_free_enc:
569 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
570 dma_free_coherent(dev, size: sizeof(struct qat_alg_cd),
571 cpu_addr: ctx->enc_cd, dma_handle: ctx->enc_cd_paddr);
572 ctx->enc_cd = NULL;
573out_free_inst:
574 ctx->inst = NULL;
575 qat_crypto_put_instance(inst);
576 return ret;
577}
578
579static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key,
580 unsigned int keylen)
581{
582 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
583
584 if (ctx->enc_cd)
585 return qat_alg_aead_rekey(tfm, key, keylen);
586 else
587 return qat_alg_aead_newkey(tfm, key, keylen);
588}
589
590static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
591 struct qat_crypto_request *qat_req)
592{
593 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
594 struct qat_crypto_instance *inst = ctx->inst;
595 struct aead_request *areq = qat_req->aead_req;
596 u8 stat_filed = qat_resp->comn_resp.comn_status;
597 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
598
599 qat_bl_free_bufl(accel_dev: inst->accel_dev, buf: &qat_req->buf);
600 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
601 res = -EBADMSG;
602 aead_request_complete(req: areq, err: res);
603}
604
605static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req)
606{
607 struct skcipher_request *sreq = qat_req->skcipher_req;
608 u64 iv_lo_prev;
609 u64 iv_lo;
610 u64 iv_hi;
611
612 memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE);
613
614 iv_lo = be64_to_cpu(qat_req->iv_lo);
615 iv_hi = be64_to_cpu(qat_req->iv_hi);
616
617 iv_lo_prev = iv_lo;
618 iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE);
619 if (iv_lo < iv_lo_prev)
620 iv_hi++;
621
622 qat_req->iv_lo = cpu_to_be64(iv_lo);
623 qat_req->iv_hi = cpu_to_be64(iv_hi);
624}
625
626static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req)
627{
628 struct skcipher_request *sreq = qat_req->skcipher_req;
629 int offset = sreq->cryptlen - AES_BLOCK_SIZE;
630 struct scatterlist *sgl;
631
632 if (qat_req->encryption)
633 sgl = sreq->dst;
634 else
635 sgl = sreq->src;
636
637 scatterwalk_map_and_copy(buf: qat_req->iv, sg: sgl, start: offset, AES_BLOCK_SIZE, out: 0);
638}
639
640static void qat_alg_update_iv(struct qat_crypto_request *qat_req)
641{
642 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
643 struct device *dev = &GET_DEV(ctx->inst->accel_dev);
644
645 switch (ctx->mode) {
646 case ICP_QAT_HW_CIPHER_CTR_MODE:
647 qat_alg_update_iv_ctr_mode(qat_req);
648 break;
649 case ICP_QAT_HW_CIPHER_CBC_MODE:
650 qat_alg_update_iv_cbc_mode(qat_req);
651 break;
652 case ICP_QAT_HW_CIPHER_XTS_MODE:
653 break;
654 default:
655 dev_warn(dev, "Unsupported IV update for cipher mode %d\n",
656 ctx->mode);
657 }
658}
659
660static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
661 struct qat_crypto_request *qat_req)
662{
663 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
664 struct qat_crypto_instance *inst = ctx->inst;
665 struct skcipher_request *sreq = qat_req->skcipher_req;
666 u8 stat_filed = qat_resp->comn_resp.comn_status;
667 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
668
669 qat_bl_free_bufl(accel_dev: inst->accel_dev, buf: &qat_req->buf);
670 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
671 res = -EINVAL;
672
673 if (qat_req->encryption)
674 qat_alg_update_iv(qat_req);
675
676 memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
677
678 skcipher_request_complete(req: sreq, err: res);
679}
680
681void qat_alg_callback(void *resp)
682{
683 struct icp_qat_fw_la_resp *qat_resp = resp;
684 struct qat_crypto_request *qat_req =
685 (void *)(__force long)qat_resp->opaque_data;
686 struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
687
688 qat_req->cb(qat_resp, qat_req);
689
690 qat_alg_send_backlog(backlog);
691}
692
693static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,
694 struct qat_crypto_instance *inst,
695 struct crypto_async_request *base)
696{
697 struct qat_alg_req *alg_req = &qat_req->alg_req;
698
699 alg_req->fw_req = (u32 *)&qat_req->req;
700 alg_req->tx_ring = inst->sym_tx;
701 alg_req->base = base;
702 alg_req->backlog = &inst->backlog;
703
704 return qat_alg_send_message(req: alg_req);
705}
706
707static int qat_alg_aead_dec(struct aead_request *areq)
708{
709 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(req: areq);
710 struct crypto_tfm *tfm = crypto_aead_tfm(tfm: aead_tfm);
711 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
712 struct qat_crypto_request *qat_req = aead_request_ctx(req: areq);
713 struct icp_qat_fw_la_cipher_req_params *cipher_param;
714 struct icp_qat_fw_la_auth_req_params *auth_param;
715 struct icp_qat_fw_la_bulk_req *msg;
716 int digst_size = crypto_aead_authsize(tfm: aead_tfm);
717 gfp_t f = qat_algs_alloc_flags(req: &areq->base);
718 int ret;
719 u32 cipher_len;
720
721 cipher_len = areq->cryptlen - digst_size;
722 if (cipher_len % AES_BLOCK_SIZE != 0)
723 return -EINVAL;
724
725 ret = qat_bl_sgl_to_bufl(accel_dev: ctx->inst->accel_dev, sgl: areq->src, sglout: areq->dst,
726 buf: &qat_req->buf, NULL, flags: f);
727 if (unlikely(ret))
728 return ret;
729
730 msg = &qat_req->req;
731 *msg = ctx->dec_fw_req;
732 qat_req->aead_ctx = ctx;
733 qat_req->aead_req = areq;
734 qat_req->cb = qat_aead_alg_callback;
735 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
736 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
737 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
738 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
739 cipher_param->cipher_length = cipher_len;
740 cipher_param->cipher_offset = areq->assoclen;
741 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
742 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
743 auth_param->auth_off = 0;
744 auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
745
746 ret = qat_alg_send_sym_message(qat_req, inst: ctx->inst, base: &areq->base);
747 if (ret == -ENOSPC)
748 qat_bl_free_bufl(accel_dev: ctx->inst->accel_dev, buf: &qat_req->buf);
749
750 return ret;
751}
752
753static int qat_alg_aead_enc(struct aead_request *areq)
754{
755 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(req: areq);
756 struct crypto_tfm *tfm = crypto_aead_tfm(tfm: aead_tfm);
757 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
758 struct qat_crypto_request *qat_req = aead_request_ctx(req: areq);
759 struct icp_qat_fw_la_cipher_req_params *cipher_param;
760 struct icp_qat_fw_la_auth_req_params *auth_param;
761 gfp_t f = qat_algs_alloc_flags(req: &areq->base);
762 struct icp_qat_fw_la_bulk_req *msg;
763 u8 *iv = areq->iv;
764 int ret;
765
766 if (areq->cryptlen % AES_BLOCK_SIZE != 0)
767 return -EINVAL;
768
769 ret = qat_bl_sgl_to_bufl(accel_dev: ctx->inst->accel_dev, sgl: areq->src, sglout: areq->dst,
770 buf: &qat_req->buf, NULL, flags: f);
771 if (unlikely(ret))
772 return ret;
773
774 msg = &qat_req->req;
775 *msg = ctx->enc_fw_req;
776 qat_req->aead_ctx = ctx;
777 qat_req->aead_req = areq;
778 qat_req->cb = qat_aead_alg_callback;
779 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
780 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
781 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
782 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
783 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
784
785 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
786 cipher_param->cipher_length = areq->cryptlen;
787 cipher_param->cipher_offset = areq->assoclen;
788
789 auth_param->auth_off = 0;
790 auth_param->auth_len = areq->assoclen + areq->cryptlen;
791
792 ret = qat_alg_send_sym_message(qat_req, inst: ctx->inst, base: &areq->base);
793 if (ret == -ENOSPC)
794 qat_bl_free_bufl(accel_dev: ctx->inst->accel_dev, buf: &qat_req->buf);
795
796 return ret;
797}
798
799static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
800 const u8 *key, unsigned int keylen,
801 int mode)
802{
803 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
804 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
805 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
806 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
807
808 return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
809}
810
811static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,
812 const u8 *key, unsigned int keylen,
813 int mode)
814{
815 struct qat_crypto_instance *inst = NULL;
816 struct device *dev;
817 int node = numa_node_id();
818 int ret;
819
820 inst = qat_crypto_get_instance_node(node);
821 if (!inst)
822 return -EINVAL;
823 dev = &GET_DEV(inst->accel_dev);
824 ctx->inst = inst;
825 ctx->enc_cd = dma_alloc_coherent(dev, size: sizeof(*ctx->enc_cd),
826 dma_handle: &ctx->enc_cd_paddr,
827 GFP_ATOMIC);
828 if (!ctx->enc_cd) {
829 ret = -ENOMEM;
830 goto out_free_instance;
831 }
832 ctx->dec_cd = dma_alloc_coherent(dev, size: sizeof(*ctx->dec_cd),
833 dma_handle: &ctx->dec_cd_paddr,
834 GFP_ATOMIC);
835 if (!ctx->dec_cd) {
836 ret = -ENOMEM;
837 goto out_free_enc;
838 }
839
840 ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
841 if (ret)
842 goto out_free_all;
843
844 return 0;
845
846out_free_all:
847 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
848 dma_free_coherent(dev, size: sizeof(*ctx->dec_cd),
849 cpu_addr: ctx->dec_cd, dma_handle: ctx->dec_cd_paddr);
850 ctx->dec_cd = NULL;
851out_free_enc:
852 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
853 dma_free_coherent(dev, size: sizeof(*ctx->enc_cd),
854 cpu_addr: ctx->enc_cd, dma_handle: ctx->enc_cd_paddr);
855 ctx->enc_cd = NULL;
856out_free_instance:
857 ctx->inst = NULL;
858 qat_crypto_put_instance(inst);
859 return ret;
860}
861
862static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,
863 const u8 *key, unsigned int keylen,
864 int mode)
865{
866 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
867
868 ctx->mode = mode;
869
870 if (ctx->enc_cd)
871 return qat_alg_skcipher_rekey(ctx, key, keylen, mode);
872 else
873 return qat_alg_skcipher_newkey(ctx, key, keylen, mode);
874}
875
876static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,
877 const u8 *key, unsigned int keylen)
878{
879 return qat_alg_skcipher_setkey(tfm, key, keylen,
880 mode: ICP_QAT_HW_CIPHER_CBC_MODE);
881}
882
883static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,
884 const u8 *key, unsigned int keylen)
885{
886 return qat_alg_skcipher_setkey(tfm, key, keylen,
887 mode: ICP_QAT_HW_CIPHER_CTR_MODE);
888}
889
890static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,
891 const u8 *key, unsigned int keylen)
892{
893 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
894 int ret;
895
896 ret = xts_verify_key(tfm, key, keylen);
897 if (ret)
898 return ret;
899
900 if (keylen >> 1 == AES_KEYSIZE_192) {
901 ret = crypto_skcipher_setkey(tfm: ctx->ftfm, key, keylen);
902 if (ret)
903 return ret;
904
905 ctx->fallback = true;
906
907 return 0;
908 }
909
910 ctx->fallback = false;
911
912 ret = qat_alg_skcipher_setkey(tfm, key, keylen,
913 mode: ICP_QAT_HW_CIPHER_XTS_MODE);
914 if (ret)
915 return ret;
916
917 if (HW_CAP_AES_V2(ctx->inst->accel_dev))
918 ret = crypto_cipher_setkey(tfm: ctx->tweak, key: key + (keylen / 2),
919 keylen: keylen / 2);
920
921 return ret;
922}
923
924static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req)
925{
926 struct icp_qat_fw_la_cipher_req_params *cipher_param;
927 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
928 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
929 u8 *iv = qat_req->skcipher_req->iv;
930
931 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
932
933 if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE)
934 crypto_cipher_encrypt_one(tfm: ctx->tweak,
935 dst: (u8 *)cipher_param->u.cipher_IV_array,
936 src: iv);
937 else
938 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
939}
940
941static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
942{
943 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
944 struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: stfm);
945 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
946 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
947 struct icp_qat_fw_la_cipher_req_params *cipher_param;
948 gfp_t f = qat_algs_alloc_flags(req: &req->base);
949 struct icp_qat_fw_la_bulk_req *msg;
950 int ret;
951
952 if (req->cryptlen == 0)
953 return 0;
954
955 ret = qat_bl_sgl_to_bufl(accel_dev: ctx->inst->accel_dev, sgl: req->src, sglout: req->dst,
956 buf: &qat_req->buf, NULL, flags: f);
957 if (unlikely(ret))
958 return ret;
959
960 msg = &qat_req->req;
961 *msg = ctx->enc_fw_req;
962 qat_req->skcipher_ctx = ctx;
963 qat_req->skcipher_req = req;
964 qat_req->cb = qat_skcipher_alg_callback;
965 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
966 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
967 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
968 qat_req->encryption = true;
969 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
970 cipher_param->cipher_length = req->cryptlen;
971 cipher_param->cipher_offset = 0;
972
973 qat_alg_set_req_iv(qat_req);
974
975 ret = qat_alg_send_sym_message(qat_req, inst: ctx->inst, base: &req->base);
976 if (ret == -ENOSPC)
977 qat_bl_free_bufl(accel_dev: ctx->inst->accel_dev, buf: &qat_req->buf);
978
979 return ret;
980}
981
982static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
983{
984 if (req->cryptlen % AES_BLOCK_SIZE != 0)
985 return -EINVAL;
986
987 return qat_alg_skcipher_encrypt(req);
988}
989
990static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req)
991{
992 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
993 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm: stfm);
994 struct skcipher_request *nreq = skcipher_request_ctx(req);
995
996 if (req->cryptlen < XTS_BLOCK_SIZE)
997 return -EINVAL;
998
999 if (ctx->fallback) {
1000 memcpy(nreq, req, sizeof(*req));
1001 skcipher_request_set_tfm(req: nreq, tfm: ctx->ftfm);
1002 return crypto_skcipher_encrypt(req: nreq);
1003 }
1004
1005 return qat_alg_skcipher_encrypt(req);
1006}
1007
1008static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
1009{
1010 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1011 struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: stfm);
1012 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1013 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1014 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1015 gfp_t f = qat_algs_alloc_flags(req: &req->base);
1016 struct icp_qat_fw_la_bulk_req *msg;
1017 int ret;
1018
1019 if (req->cryptlen == 0)
1020 return 0;
1021
1022 ret = qat_bl_sgl_to_bufl(accel_dev: ctx->inst->accel_dev, sgl: req->src, sglout: req->dst,
1023 buf: &qat_req->buf, NULL, flags: f);
1024 if (unlikely(ret))
1025 return ret;
1026
1027 msg = &qat_req->req;
1028 *msg = ctx->dec_fw_req;
1029 qat_req->skcipher_ctx = ctx;
1030 qat_req->skcipher_req = req;
1031 qat_req->cb = qat_skcipher_alg_callback;
1032 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1033 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1034 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1035 qat_req->encryption = false;
1036 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1037 cipher_param->cipher_length = req->cryptlen;
1038 cipher_param->cipher_offset = 0;
1039
1040 qat_alg_set_req_iv(qat_req);
1041 qat_alg_update_iv(qat_req);
1042
1043 ret = qat_alg_send_sym_message(qat_req, inst: ctx->inst, base: &req->base);
1044 if (ret == -ENOSPC)
1045 qat_bl_free_bufl(accel_dev: ctx->inst->accel_dev, buf: &qat_req->buf);
1046
1047 return ret;
1048}
1049
1050static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
1051{
1052 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1053 return -EINVAL;
1054
1055 return qat_alg_skcipher_decrypt(req);
1056}
1057
1058static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)
1059{
1060 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1061 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm: stfm);
1062 struct skcipher_request *nreq = skcipher_request_ctx(req);
1063
1064 if (req->cryptlen < XTS_BLOCK_SIZE)
1065 return -EINVAL;
1066
1067 if (ctx->fallback) {
1068 memcpy(nreq, req, sizeof(*req));
1069 skcipher_request_set_tfm(req: nreq, tfm: ctx->ftfm);
1070 return crypto_skcipher_decrypt(req: nreq);
1071 }
1072
1073 return qat_alg_skcipher_decrypt(req);
1074}
1075
1076static int qat_alg_aead_init(struct crypto_aead *tfm,
1077 enum icp_qat_hw_auth_algo hash_alg,
1078 unsigned int hash_digestsize,
1079 unsigned int hash_blocksize)
1080{
1081 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1082
1083 ctx->qat_hash_alg = hash_alg;
1084 ctx->hash_digestsize = hash_digestsize;
1085 ctx->hash_blocksize = hash_blocksize;
1086 crypto_aead_set_reqsize(aead: tfm, reqsize: sizeof(struct qat_crypto_request));
1087 return 0;
1088}
1089
1090static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1091{
1092 return qat_alg_aead_init(tfm, hash_alg: ICP_QAT_HW_AUTH_ALGO_SHA1,
1093 SHA1_DIGEST_SIZE, SHA1_BLOCK_SIZE);
1094}
1095
1096static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1097{
1098 return qat_alg_aead_init(tfm, hash_alg: ICP_QAT_HW_AUTH_ALGO_SHA256,
1099 SHA256_DIGEST_SIZE, SHA256_BLOCK_SIZE);
1100}
1101
1102static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1103{
1104 return qat_alg_aead_init(tfm, hash_alg: ICP_QAT_HW_AUTH_ALGO_SHA512,
1105 SHA512_DIGEST_SIZE, SHA512_BLOCK_SIZE);
1106}
1107
1108static void qat_alg_aead_exit(struct crypto_aead *tfm)
1109{
1110 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1111 struct qat_crypto_instance *inst = ctx->inst;
1112 struct device *dev;
1113
1114 if (!inst)
1115 return;
1116
1117 dev = &GET_DEV(inst->accel_dev);
1118 if (ctx->enc_cd) {
1119 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1120 dma_free_coherent(dev, size: sizeof(struct qat_alg_cd),
1121 cpu_addr: ctx->enc_cd, dma_handle: ctx->enc_cd_paddr);
1122 }
1123 if (ctx->dec_cd) {
1124 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1125 dma_free_coherent(dev, size: sizeof(struct qat_alg_cd),
1126 cpu_addr: ctx->dec_cd, dma_handle: ctx->dec_cd_paddr);
1127 }
1128 qat_crypto_put_instance(inst);
1129}
1130
1131static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
1132{
1133 crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct qat_crypto_request));
1134 return 0;
1135}
1136
1137static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
1138{
1139 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1140 int reqsize;
1141
1142 ctx->ftfm = crypto_alloc_skcipher(alg_name: "xts(aes)", type: 0,
1143 CRYPTO_ALG_NEED_FALLBACK);
1144 if (IS_ERR(ptr: ctx->ftfm))
1145 return PTR_ERR(ptr: ctx->ftfm);
1146
1147 ctx->tweak = crypto_alloc_cipher(alg_name: "aes", type: 0, mask: 0);
1148 if (IS_ERR(ptr: ctx->tweak)) {
1149 crypto_free_skcipher(tfm: ctx->ftfm);
1150 return PTR_ERR(ptr: ctx->tweak);
1151 }
1152
1153 reqsize = max(sizeof(struct qat_crypto_request),
1154 sizeof(struct skcipher_request) +
1155 crypto_skcipher_reqsize(ctx->ftfm));
1156 crypto_skcipher_set_reqsize(skcipher: tfm, reqsize);
1157
1158 return 0;
1159}
1160
1161static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
1162{
1163 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1164 struct qat_crypto_instance *inst = ctx->inst;
1165 struct device *dev;
1166
1167 if (!inst)
1168 return;
1169
1170 dev = &GET_DEV(inst->accel_dev);
1171 if (ctx->enc_cd) {
1172 memset(ctx->enc_cd, 0,
1173 sizeof(struct icp_qat_hw_cipher_algo_blk));
1174 dma_free_coherent(dev,
1175 size: sizeof(struct icp_qat_hw_cipher_algo_blk),
1176 cpu_addr: ctx->enc_cd, dma_handle: ctx->enc_cd_paddr);
1177 }
1178 if (ctx->dec_cd) {
1179 memset(ctx->dec_cd, 0,
1180 sizeof(struct icp_qat_hw_cipher_algo_blk));
1181 dma_free_coherent(dev,
1182 size: sizeof(struct icp_qat_hw_cipher_algo_blk),
1183 cpu_addr: ctx->dec_cd, dma_handle: ctx->dec_cd_paddr);
1184 }
1185 qat_crypto_put_instance(inst);
1186}
1187
1188static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm)
1189{
1190 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1191
1192 if (ctx->ftfm)
1193 crypto_free_skcipher(tfm: ctx->ftfm);
1194
1195 if (ctx->tweak)
1196 crypto_free_cipher(tfm: ctx->tweak);
1197
1198 qat_alg_skcipher_exit_tfm(tfm);
1199}
1200
1201static struct aead_alg qat_aeads[] = { {
1202 .base = {
1203 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1204 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1205 .cra_priority = 100,
1206 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1207 .cra_blocksize = AES_BLOCK_SIZE,
1208 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1209 .cra_module = THIS_MODULE,
1210 },
1211 .init = qat_alg_aead_sha1_init,
1212 .exit = qat_alg_aead_exit,
1213 .setkey = qat_alg_aead_setkey,
1214 .decrypt = qat_alg_aead_dec,
1215 .encrypt = qat_alg_aead_enc,
1216 .ivsize = AES_BLOCK_SIZE,
1217 .maxauthsize = SHA1_DIGEST_SIZE,
1218}, {
1219 .base = {
1220 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1221 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1222 .cra_priority = 100,
1223 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1224 .cra_blocksize = AES_BLOCK_SIZE,
1225 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1226 .cra_module = THIS_MODULE,
1227 },
1228 .init = qat_alg_aead_sha256_init,
1229 .exit = qat_alg_aead_exit,
1230 .setkey = qat_alg_aead_setkey,
1231 .decrypt = qat_alg_aead_dec,
1232 .encrypt = qat_alg_aead_enc,
1233 .ivsize = AES_BLOCK_SIZE,
1234 .maxauthsize = SHA256_DIGEST_SIZE,
1235}, {
1236 .base = {
1237 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1238 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1239 .cra_priority = 100,
1240 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1241 .cra_blocksize = AES_BLOCK_SIZE,
1242 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1243 .cra_module = THIS_MODULE,
1244 },
1245 .init = qat_alg_aead_sha512_init,
1246 .exit = qat_alg_aead_exit,
1247 .setkey = qat_alg_aead_setkey,
1248 .decrypt = qat_alg_aead_dec,
1249 .encrypt = qat_alg_aead_enc,
1250 .ivsize = AES_BLOCK_SIZE,
1251 .maxauthsize = SHA512_DIGEST_SIZE,
1252} };
1253
1254static struct skcipher_alg qat_skciphers[] = { {
1255 .base.cra_name = "cbc(aes)",
1256 .base.cra_driver_name = "qat_aes_cbc",
1257 .base.cra_priority = 100,
1258 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1259 .base.cra_blocksize = AES_BLOCK_SIZE,
1260 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1261 .base.cra_alignmask = 0,
1262 .base.cra_module = THIS_MODULE,
1263
1264 .init = qat_alg_skcipher_init_tfm,
1265 .exit = qat_alg_skcipher_exit_tfm,
1266 .setkey = qat_alg_skcipher_cbc_setkey,
1267 .decrypt = qat_alg_skcipher_blk_decrypt,
1268 .encrypt = qat_alg_skcipher_blk_encrypt,
1269 .min_keysize = AES_MIN_KEY_SIZE,
1270 .max_keysize = AES_MAX_KEY_SIZE,
1271 .ivsize = AES_BLOCK_SIZE,
1272}, {
1273 .base.cra_name = "ctr(aes)",
1274 .base.cra_driver_name = "qat_aes_ctr",
1275 .base.cra_priority = 100,
1276 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1277 .base.cra_blocksize = 1,
1278 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1279 .base.cra_alignmask = 0,
1280 .base.cra_module = THIS_MODULE,
1281
1282 .init = qat_alg_skcipher_init_tfm,
1283 .exit = qat_alg_skcipher_exit_tfm,
1284 .setkey = qat_alg_skcipher_ctr_setkey,
1285 .decrypt = qat_alg_skcipher_decrypt,
1286 .encrypt = qat_alg_skcipher_encrypt,
1287 .min_keysize = AES_MIN_KEY_SIZE,
1288 .max_keysize = AES_MAX_KEY_SIZE,
1289 .ivsize = AES_BLOCK_SIZE,
1290}, {
1291 .base.cra_name = "xts(aes)",
1292 .base.cra_driver_name = "qat_aes_xts",
1293 .base.cra_priority = 100,
1294 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
1295 CRYPTO_ALG_ALLOCATES_MEMORY,
1296 .base.cra_blocksize = AES_BLOCK_SIZE,
1297 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1298 .base.cra_alignmask = 0,
1299 .base.cra_module = THIS_MODULE,
1300
1301 .init = qat_alg_skcipher_init_xts_tfm,
1302 .exit = qat_alg_skcipher_exit_xts_tfm,
1303 .setkey = qat_alg_skcipher_xts_setkey,
1304 .decrypt = qat_alg_skcipher_xts_decrypt,
1305 .encrypt = qat_alg_skcipher_xts_encrypt,
1306 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1307 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1308 .ivsize = AES_BLOCK_SIZE,
1309} };
1310
1311int qat_algs_register(void)
1312{
1313 int ret = 0;
1314
1315 mutex_lock(&algs_lock);
1316 if (++active_devs != 1)
1317 goto unlock;
1318
1319 ret = crypto_register_skciphers(algs: qat_skciphers,
1320 ARRAY_SIZE(qat_skciphers));
1321 if (ret)
1322 goto unlock;
1323
1324 ret = crypto_register_aeads(algs: qat_aeads, ARRAY_SIZE(qat_aeads));
1325 if (ret)
1326 goto unreg_algs;
1327
1328unlock:
1329 mutex_unlock(lock: &algs_lock);
1330 return ret;
1331
1332unreg_algs:
1333 crypto_unregister_skciphers(algs: qat_skciphers, ARRAY_SIZE(qat_skciphers));
1334 goto unlock;
1335}
1336
1337void qat_algs_unregister(void)
1338{
1339 mutex_lock(&algs_lock);
1340 if (--active_devs != 0)
1341 goto unlock;
1342
1343 crypto_unregister_aeads(algs: qat_aeads, ARRAY_SIZE(qat_aeads));
1344 crypto_unregister_skciphers(algs: qat_skciphers, ARRAY_SIZE(qat_skciphers));
1345
1346unlock:
1347 mutex_unlock(lock: &algs_lock);
1348}
1349

source code of linux/drivers/crypto/intel/qat/qat_common/qat_algs.c